diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,11956 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 100,
+  "global_step": 7642,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.535947712418301e-09,
+      "logits/chosen": -2.29970645904541,
+      "logits/rejected": -2.22867488861084,
+      "logps/chosen": -220.96466064453125,
+      "logps/rejected": -239.84609985351562,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.535947712418302e-08,
+      "logits/chosen": -2.2488739490509033,
+      "logits/rejected": -2.1726131439208984,
+      "logps/chosen": -271.4999694824219,
+      "logps/rejected": -247.97677612304688,
+      "loss": 0.693,
+      "rewards/accuracies": 0.1944444477558136,
+      "rewards/chosen": -4.116101627005264e-05,
+      "rewards/margins": 0.000303717446513474,
+      "rewards/rejected": -0.00034487841185182333,
+      "step": 10
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.3071895424836603e-07,
+      "logits/chosen": -2.2442140579223633,
+      "logits/rejected": -2.2172605991363525,
+      "logps/chosen": -264.39996337890625,
+      "logps/rejected": -269.8542785644531,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.00035499135265126824,
+      "rewards/margins": -0.000299239152809605,
+      "rewards/rejected": -5.575223985943012e-05,
+      "step": 20
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.9607843137254904e-07,
+      "logits/chosen": -2.224724054336548,
+      "logits/rejected": -2.128314256668091,
+      "logps/chosen": -235.7845001220703,
+      "logps/rejected": -211.7075958251953,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -5.135792162036523e-05,
+      "rewards/margins": 0.0006848908378742635,
+      "rewards/rejected": -0.0007362487376667559,
+      "step": 30
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 2.6143790849673207e-07,
+      "logits/chosen": -2.3858389854431152,
+      "logits/rejected": -2.249871015548706,
+      "logps/chosen": -264.62255859375,
+      "logps/rejected": -244.1466827392578,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.00010533479508012533,
+      "rewards/margins": 0.000317855563480407,
+      "rewards/rejected": -0.0002125207829521969,
+      "step": 40
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 3.267973856209151e-07,
+      "logits/chosen": -2.3077752590179443,
+      "logits/rejected": -2.2615857124328613,
+      "logps/chosen": -226.8096923828125,
+      "logps/rejected": -226.6692657470703,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.00021501348237507045,
+      "rewards/margins": 0.00017367670079693198,
+      "rewards/rejected": 4.1336799768032506e-05,
+      "step": 50
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 3.921568627450981e-07,
+      "logits/chosen": -2.243070125579834,
+      "logits/rejected": -2.1937053203582764,
+      "logps/chosen": -267.62103271484375,
+      "logps/rejected": -242.4007110595703,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 7.660783739993349e-05,
+      "rewards/margins": 0.00017966241284739226,
+      "rewards/rejected": -0.00010305454634362832,
+      "step": 60
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 4.5751633986928105e-07,
+      "logits/chosen": -2.274949789047241,
+      "logits/rejected": -2.2325363159179688,
+      "logps/chosen": -246.0950469970703,
+      "logps/rejected": -205.44601440429688,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 9.732646867632866e-05,
+      "rewards/margins": 8.030790922930464e-05,
+      "rewards/rejected": 1.701857036096044e-05,
+      "step": 70
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.228758169934641e-07,
+      "logits/chosen": -2.2184979915618896,
+      "logits/rejected": -2.1940760612487793,
+      "logps/chosen": -289.24444580078125,
+      "logps/rejected": -281.460693359375,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.00038058884092606604,
+      "rewards/margins": -0.0002509167534299195,
+      "rewards/rejected": 0.0006315055652521551,
+      "step": 80
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.882352941176471e-07,
+      "logits/chosen": -2.3450047969818115,
+      "logits/rejected": -2.2712016105651855,
+      "logps/chosen": -254.7499542236328,
+      "logps/rejected": -225.3871612548828,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.00030283976229839027,
+      "rewards/margins": -0.0004797237052116543,
+      "rewards/rejected": 0.00017688400112092495,
+      "step": 90
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.535947712418302e-07,
+      "logits/chosen": -2.239689588546753,
+      "logits/rejected": -2.2465834617614746,
+      "logps/chosen": -227.69287109375,
+      "logps/rejected": -235.5568389892578,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0005908356397412717,
+      "rewards/margins": 0.00014562085561919957,
+      "rewards/rejected": 0.0004452148568816483,
+      "step": 100
+    },
+    {
+      "epoch": 0.01,
+      "eval_logits/chosen": -2.216303825378418,
+      "eval_logits/rejected": -2.1334962844848633,
+      "eval_logps/chosen": -268.5094909667969,
+      "eval_logps/rejected": -248.78553771972656,
+      "eval_loss": 0.6930069327354431,
+      "eval_rewards/accuracies": 0.5134999752044678,
+      "eval_rewards/chosen": 0.0005006304709240794,
+      "eval_rewards/margins": 0.0002866178110707551,
+      "eval_rewards/rejected": 0.00021401270350906998,
+      "eval_runtime": 1345.5315,
+      "eval_samples_per_second": 1.486,
+      "eval_steps_per_second": 0.186,
+      "step": 100
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.189542483660131e-07,
+      "logits/chosen": -2.301734209060669,
+      "logits/rejected": -2.1840476989746094,
+      "logps/chosen": -269.32525634765625,
+      "logps/rejected": -251.1839141845703,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0006390978232957423,
+      "rewards/margins": 0.0005474867066368461,
+      "rewards/rejected": 9.161119669442996e-05,
+      "step": 110
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 7.843137254901962e-07,
+      "logits/chosen": -2.2919907569885254,
+      "logits/rejected": -2.193392276763916,
+      "logps/chosen": -286.66033935546875,
+      "logps/rejected": -229.22476196289062,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0010297037661075592,
+      "rewards/margins": 0.0013585933484137058,
+      "rewards/rejected": -0.0003288895823061466,
+      "step": 120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 8.496732026143792e-07,
+      "logits/chosen": -2.3159279823303223,
+      "logits/rejected": -2.314882755279541,
+      "logps/chosen": -251.7840576171875,
+      "logps/rejected": -224.52389526367188,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.00013725504686590284,
+      "rewards/margins": 0.0005960448761470616,
+      "rewards/rejected": -0.0004587897565215826,
+      "step": 130
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.150326797385621e-07,
+      "logits/chosen": -2.27937650680542,
+      "logits/rejected": -2.232600688934326,
+      "logps/chosen": -260.2637939453125,
+      "logps/rejected": -231.6568603515625,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.001226735534146428,
+      "rewards/margins": 0.001199987600557506,
+      "rewards/rejected": 2.674796814972069e-05,
+      "step": 140
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.80392156862745e-07,
+      "logits/chosen": -2.3943567276000977,
+      "logits/rejected": -2.257779359817505,
+      "logps/chosen": -317.0682067871094,
+      "logps/rejected": -283.7619323730469,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.000854751851875335,
+      "rewards/margins": 0.0006376241217367351,
+      "rewards/rejected": 0.0002171276864828542,
+      "step": 150
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0457516339869283e-06,
+      "logits/chosen": -2.3294196128845215,
+      "logits/rejected": -2.193216562271118,
+      "logps/chosen": -246.99520874023438,
+      "logps/rejected": -224.45498657226562,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.00034151150612160563,
+      "rewards/margins": 0.0006650895811617374,
+      "rewards/rejected": -0.00032357795862481,
+      "step": 160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.111111111111111e-06,
+      "logits/chosen": -2.206002950668335,
+      "logits/rejected": -2.149352550506592,
+      "logps/chosen": -228.5664520263672,
+      "logps/rejected": -214.8862762451172,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0014570723287761211,
+      "rewards/margins": 0.0008309429395012558,
+      "rewards/rejected": 0.0006261293892748654,
+      "step": 170
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.1764705882352942e-06,
+      "logits/chosen": -2.3910346031188965,
+      "logits/rejected": -2.180985927581787,
+      "logps/chosen": -329.5492858886719,
+      "logps/rejected": -258.0649108886719,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.001763816922903061,
+      "rewards/margins": 0.0007384417112916708,
+      "rewards/rejected": 0.0010253752116113901,
+      "step": 180
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.2418300653594772e-06,
+      "logits/chosen": -2.195915699005127,
+      "logits/rejected": -2.14906644821167,
+      "logps/chosen": -249.59085083007812,
+      "logps/rejected": -210.2095489501953,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.0017713718116283417,
+      "rewards/margins": 0.0012134118005633354,
+      "rewards/rejected": 0.0005579598946496844,
+      "step": 190
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.3071895424836604e-06,
+      "logits/chosen": -2.3219058513641357,
+      "logits/rejected": -2.181734085083008,
+      "logps/chosen": -245.27432250976562,
+      "logps/rejected": -250.915283203125,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.001385509385727346,
+      "rewards/margins": 0.0011166484327986836,
+      "rewards/rejected": 0.00026886104024015367,
+      "step": 200
+    },
+    {
+      "epoch": 0.03,
+      "eval_logits/chosen": -2.2157464027404785,
+      "eval_logits/rejected": -2.1330456733703613,
+      "eval_logps/chosen": -268.33306884765625,
+      "eval_logps/rejected": -248.72244262695312,
+      "eval_loss": 0.6924427151679993,
+      "eval_rewards/accuracies": 0.5885000228881836,
+      "eval_rewards/chosen": 0.0022651778999716043,
+      "eval_rewards/margins": 0.0014198434073477983,
+      "eval_rewards/rejected": 0.0008453346672467887,
+      "eval_runtime": 1344.8236,
+      "eval_samples_per_second": 1.487,
+      "eval_steps_per_second": 0.186,
+      "step": 200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.3725490196078434e-06,
+      "logits/chosen": -2.4165990352630615,
+      "logits/rejected": -2.3491058349609375,
+      "logps/chosen": -267.70062255859375,
+      "logps/rejected": -244.712646484375,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0022462736815214157,
+      "rewards/margins": 0.0013150768354535103,
+      "rewards/rejected": 0.0009311968460679054,
+      "step": 210
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.4379084967320261e-06,
+      "logits/chosen": -2.25294828414917,
+      "logits/rejected": -2.147373914718628,
+      "logps/chosen": -267.0906982421875,
+      "logps/rejected": -245.4513397216797,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0026935755740851164,
+      "rewards/margins": 0.0018555322894826531,
+      "rewards/rejected": 0.0008380432846024632,
+      "step": 220
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5032679738562091e-06,
+      "logits/chosen": -2.172698497772217,
+      "logits/rejected": -2.1678102016448975,
+      "logps/chosen": -218.66287231445312,
+      "logps/rejected": -262.49371337890625,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0030288477428257465,
+      "rewards/margins": 0.000991313369013369,
+      "rewards/rejected": 0.0020375349558889866,
+      "step": 230
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5686274509803923e-06,
+      "logits/chosen": -2.2649569511413574,
+      "logits/rejected": -2.2200064659118652,
+      "logps/chosen": -234.3190460205078,
+      "logps/rejected": -253.21572875976562,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.003117733635008335,
+      "rewards/margins": 0.0017040821257978678,
+      "rewards/rejected": 0.0014136515092104673,
+      "step": 240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.6339869281045753e-06,
+      "logits/chosen": -2.2958521842956543,
+      "logits/rejected": -2.1578164100646973,
+      "logps/chosen": -300.43707275390625,
+      "logps/rejected": -258.9712829589844,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.004412081092596054,
+      "rewards/margins": 0.003268744796514511,
+      "rewards/rejected": 0.0011433359468355775,
+      "step": 250
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.6993464052287585e-06,
+      "logits/chosen": -2.3434669971466064,
+      "logits/rejected": -2.246519088745117,
+      "logps/chosen": -279.12591552734375,
+      "logps/rejected": -246.7017364501953,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.007248834706842899,
+      "rewards/margins": 0.005665811710059643,
+      "rewards/rejected": 0.0015830222982913256,
+      "step": 260
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.7647058823529414e-06,
+      "logits/chosen": -2.2550342082977295,
+      "logits/rejected": -2.178781270980835,
+      "logps/chosen": -235.0968780517578,
+      "logps/rejected": -214.44772338867188,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.007135806139558554,
+      "rewards/margins": 0.006511199288070202,
+      "rewards/rejected": 0.0006246067932806909,
+      "step": 270
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.8300653594771242e-06,
+      "logits/chosen": -2.391324996948242,
+      "logits/rejected": -2.265413761138916,
+      "logps/chosen": -303.04833984375,
+      "logps/rejected": -271.86480712890625,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.009703542105853558,
+      "rewards/margins": 0.00541451433673501,
+      "rewards/rejected": 0.004289029166102409,
+      "step": 280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.8954248366013072e-06,
+      "logits/chosen": -2.2713468074798584,
+      "logits/rejected": -2.232816219329834,
+      "logps/chosen": -274.16015625,
+      "logps/rejected": -256.81488037109375,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.01044213306158781,
+      "rewards/margins": 0.004127039574086666,
+      "rewards/rejected": 0.006315095815807581,
+      "step": 290
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.96078431372549e-06,
+      "logits/chosen": -2.432267904281616,
+      "logits/rejected": -2.303013324737549,
+      "logps/chosen": -286.23333740234375,
+      "logps/rejected": -236.09896850585938,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.011049175634980202,
+      "rewards/margins": 0.005651378072798252,
+      "rewards/rejected": 0.00539779756218195,
+      "step": 300
+    },
+    {
+      "epoch": 0.04,
+      "eval_logits/chosen": -2.2193877696990967,
+      "eval_logits/rejected": -2.137324333190918,
+      "eval_logps/chosen": -267.3080139160156,
+      "eval_logps/rejected": -248.17079162597656,
+      "eval_loss": 0.6901101469993591,
+      "eval_rewards/accuracies": 0.6474999785423279,
+      "eval_rewards/chosen": 0.012515785172581673,
+      "eval_rewards/margins": 0.006154170259833336,
+      "eval_rewards/rejected": 0.006361616309732199,
+      "eval_runtime": 1348.9305,
+      "eval_samples_per_second": 1.483,
+      "eval_steps_per_second": 0.185,
+      "step": 300
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0261437908496734e-06,
+      "logits/chosen": -2.281259059906006,
+      "logits/rejected": -2.2281479835510254,
+      "logps/chosen": -275.324462890625,
+      "logps/rejected": -271.2378845214844,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.010945344343781471,
+      "rewards/margins": 0.005101542919874191,
+      "rewards/rejected": 0.005843800958245993,
+      "step": 310
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0915032679738565e-06,
+      "logits/chosen": -2.3443284034729004,
+      "logits/rejected": -2.2085940837860107,
+      "logps/chosen": -250.2330780029297,
+      "logps/rejected": -241.8087921142578,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.012956800870597363,
+      "rewards/margins": 0.006753178779035807,
+      "rewards/rejected": 0.0062036230228841305,
+      "step": 320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.1568627450980393e-06,
+      "logits/chosen": -2.355668544769287,
+      "logits/rejected": -2.222667694091797,
+      "logps/chosen": -284.8980407714844,
+      "logps/rejected": -239.30996704101562,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.01640392653644085,
+      "rewards/margins": 0.00757676362991333,
+      "rewards/rejected": 0.00882716290652752,
+      "step": 330
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.222222222222222e-06,
+      "logits/chosen": -2.295743942260742,
+      "logits/rejected": -2.1351170539855957,
+      "logps/chosen": -257.31011962890625,
+      "logps/rejected": -215.67660522460938,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.01751278154551983,
+      "rewards/margins": 0.010068411007523537,
+      "rewards/rejected": 0.007444371934980154,
+      "step": 340
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.2875816993464053e-06,
+      "logits/chosen": -2.404604196548462,
+      "logits/rejected": -2.2168691158294678,
+      "logps/chosen": -306.09136962890625,
+      "logps/rejected": -253.8984832763672,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.022151336073875427,
+      "rewards/margins": 0.012735260650515556,
+      "rewards/rejected": 0.009416075423359871,
+      "step": 350
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.3529411764705885e-06,
+      "logits/chosen": -2.3716368675231934,
+      "logits/rejected": -2.2997515201568604,
+      "logps/chosen": -248.0289306640625,
+      "logps/rejected": -245.43032836914062,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.023434137925505638,
+      "rewards/margins": 0.011598402634263039,
+      "rewards/rejected": 0.011835736222565174,
+      "step": 360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.4183006535947716e-06,
+      "logits/chosen": -2.2913079261779785,
+      "logits/rejected": -2.262873649597168,
+      "logps/chosen": -251.10775756835938,
+      "logps/rejected": -229.71969604492188,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.023998061195015907,
+      "rewards/margins": 0.01162667479366064,
+      "rewards/rejected": 0.012371386401355267,
+      "step": 370
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.4836601307189544e-06,
+      "logits/chosen": -2.2983736991882324,
+      "logits/rejected": -2.253401756286621,
+      "logps/chosen": -272.18658447265625,
+      "logps/rejected": -239.8662567138672,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.028478026390075684,
+      "rewards/margins": 0.01902223378419876,
+      "rewards/rejected": 0.009455792605876923,
+      "step": 380
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.549019607843137e-06,
+      "logits/chosen": -2.278621196746826,
+      "logits/rejected": -2.1427106857299805,
+      "logps/chosen": -287.32666015625,
+      "logps/rejected": -232.88766479492188,
+      "loss": 0.6806,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.0392298623919487,
+      "rewards/margins": 0.025771820917725563,
+      "rewards/rejected": 0.01345804799348116,
+      "step": 390
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.6143790849673208e-06,
+      "logits/chosen": -2.3590660095214844,
+      "logits/rejected": -2.3591723442077637,
+      "logps/chosen": -272.09381103515625,
+      "logps/rejected": -284.07489013671875,
+      "loss": 0.6868,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.0330638512969017,
+      "rewards/margins": 0.013295474462211132,
+      "rewards/rejected": 0.019768375903367996,
+      "step": 400
+    },
+    {
+      "epoch": 0.05,
+      "eval_logits/chosen": -2.217874526977539,
+      "eval_logits/rejected": -2.135589599609375,
+      "eval_logps/chosen": -264.7627258300781,
+      "eval_logps/rejected": -247.1243438720703,
+      "eval_loss": 0.6829787492752075,
+      "eval_rewards/accuracies": 0.6610000133514404,
+      "eval_rewards/chosen": 0.037968553602695465,
+      "eval_rewards/margins": 0.02114250510931015,
+      "eval_rewards/rejected": 0.016826050356030464,
+      "eval_runtime": 1346.6707,
+      "eval_samples_per_second": 1.485,
+      "eval_steps_per_second": 0.186,
+      "step": 400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.6797385620915036e-06,
+      "logits/chosen": -2.197803020477295,
+      "logits/rejected": -2.1066477298736572,
+      "logps/chosen": -245.6123046875,
+      "logps/rejected": -204.1496124267578,
+      "loss": 0.6841,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.035216279327869415,
+      "rewards/margins": 0.019117429852485657,
+      "rewards/rejected": 0.016098853200674057,
+      "step": 410
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.7450980392156867e-06,
+      "logits/chosen": -2.260650157928467,
+      "logits/rejected": -2.19161319732666,
+      "logps/chosen": -262.06365966796875,
+      "logps/rejected": -264.2047424316406,
+      "loss": 0.6784,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.044983021914958954,
+      "rewards/margins": 0.030561596155166626,
+      "rewards/rejected": 0.014421427622437477,
+      "step": 420
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.8104575163398695e-06,
+      "logits/chosen": -2.317704200744629,
+      "logits/rejected": -2.2272989749908447,
+      "logps/chosen": -266.327392578125,
+      "logps/rejected": -254.08602905273438,
+      "loss": 0.6819,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0401587150990963,
+      "rewards/margins": 0.024031201377511024,
+      "rewards/rejected": 0.016127515584230423,
+      "step": 430
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.8758169934640523e-06,
+      "logits/chosen": -2.378629207611084,
+      "logits/rejected": -2.3443870544433594,
+      "logps/chosen": -256.9866638183594,
+      "logps/rejected": -233.60433959960938,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.03785388544201851,
+      "rewards/margins": 0.02240253984928131,
+      "rewards/rejected": 0.0154513418674469,
+      "step": 440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.9411764705882355e-06,
+      "logits/chosen": -2.3383307456970215,
+      "logits/rejected": -2.33845853805542,
+      "logps/chosen": -273.9598693847656,
+      "logps/rejected": -291.20794677734375,
+      "loss": 0.6828,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.03283879533410072,
+      "rewards/margins": 0.0222325436770916,
+      "rewards/rejected": 0.010606248863041401,
+      "step": 450
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.0065359477124182e-06,
+      "logits/chosen": -2.2138919830322266,
+      "logits/rejected": -2.1119112968444824,
+      "logps/chosen": -239.9493408203125,
+      "logps/rejected": -234.13693237304688,
+      "loss": 0.6775,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.04935748130083084,
+      "rewards/margins": 0.03295300528407097,
+      "rewards/rejected": 0.016404472291469574,
+      "step": 460
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.071895424836602e-06,
+      "logits/chosen": -2.282121181488037,
+      "logits/rejected": -2.2568697929382324,
+      "logps/chosen": -260.3847351074219,
+      "logps/rejected": -243.64340209960938,
+      "loss": 0.6715,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0625983253121376,
+      "rewards/margins": 0.04548066481947899,
+      "rewards/rejected": 0.017117660492658615,
+      "step": 470
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.1372549019607846e-06,
+      "logits/chosen": -2.2041165828704834,
+      "logits/rejected": -2.186706781387329,
+      "logps/chosen": -267.3668212890625,
+      "logps/rejected": -227.5152587890625,
+      "loss": 0.6662,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.06482601910829544,
+      "rewards/margins": 0.056661974638700485,
+      "rewards/rejected": 0.008164034225046635,
+      "step": 480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.2026143790849674e-06,
+      "logits/chosen": -2.3111612796783447,
+      "logits/rejected": -2.1605923175811768,
+      "logps/chosen": -258.001708984375,
+      "logps/rejected": -215.0772247314453,
+      "loss": 0.6634,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.04909733682870865,
+      "rewards/margins": 0.06452682614326477,
+      "rewards/rejected": -0.015429483726620674,
+      "step": 490
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.2679738562091506e-06,
+      "logits/chosen": -2.2743332386016846,
+      "logits/rejected": -2.1727280616760254,
+      "logps/chosen": -238.04263305664062,
+      "logps/rejected": -224.16592407226562,
+      "loss": 0.6781,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.009544854052364826,
+      "rewards/margins": 0.03458785265684128,
+      "rewards/rejected": -0.025042995810508728,
+      "step": 500
+    },
+    {
+      "epoch": 0.07,
+      "eval_logits/chosen": -2.159041404724121,
+      "eval_logits/rejected": -2.0747883319854736,
+      "eval_logps/chosen": -266.5388488769531,
+      "eval_logps/rejected": -252.3708038330078,
+      "eval_loss": 0.6678662896156311,
+      "eval_rewards/accuracies": 0.6784999966621399,
+      "eval_rewards/chosen": 0.020207293331623077,
+      "eval_rewards/margins": 0.05584573373198509,
+      "eval_rewards/rejected": -0.03563844412565231,
+      "eval_runtime": 1345.2254,
+      "eval_samples_per_second": 1.487,
+      "eval_steps_per_second": 0.186,
+      "step": 500
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.3333333333333333e-06,
+      "logits/chosen": -2.1650888919830322,
+      "logits/rejected": -2.1543965339660645,
+      "logps/chosen": -249.5853271484375,
+      "logps/rejected": -251.1168212890625,
+      "loss": 0.6726,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.007822860032320023,
+      "rewards/margins": 0.04704464226961136,
+      "rewards/rejected": -0.039221785962581635,
+      "step": 510
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.398692810457517e-06,
+      "logits/chosen": -2.1978859901428223,
+      "logits/rejected": -2.0505318641662598,
+      "logps/chosen": -245.8839874267578,
+      "logps/rejected": -250.81405639648438,
+      "loss": 0.6669,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.02485199086368084,
+      "rewards/margins": 0.059726566076278687,
+      "rewards/rejected": -0.08457855880260468,
+      "step": 520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.4640522875816997e-06,
+      "logits/chosen": -2.1094107627868652,
+      "logits/rejected": -2.04038405418396,
+      "logps/chosen": -249.32870483398438,
+      "logps/rejected": -235.13290405273438,
+      "loss": 0.6608,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.01553401630371809,
+      "rewards/margins": 0.07144078612327576,
+      "rewards/rejected": -0.08697480708360672,
+      "step": 530
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.529411764705883e-06,
+      "logits/chosen": -2.225689649581909,
+      "logits/rejected": -2.079272985458374,
+      "logps/chosen": -274.75543212890625,
+      "logps/rejected": -260.36004638671875,
+      "loss": 0.6558,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.019599393010139465,
+      "rewards/margins": 0.08329910784959793,
+      "rewards/rejected": -0.1028985008597374,
+      "step": 540
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.5947712418300657e-06,
+      "logits/chosen": -2.1707935333251953,
+      "logits/rejected": -2.113981008529663,
+      "logps/chosen": -246.376220703125,
+      "logps/rejected": -238.87417602539062,
+      "loss": 0.643,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.01092623546719551,
+      "rewards/margins": 0.11469154059886932,
+      "rewards/rejected": -0.12561777234077454,
+      "step": 550
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.6601307189542484e-06,
+      "logits/chosen": -2.2364041805267334,
+      "logits/rejected": -2.1267893314361572,
+      "logps/chosen": -279.9202575683594,
+      "logps/rejected": -300.40679931640625,
+      "loss": 0.6572,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.05697600916028023,
+      "rewards/margins": 0.08570733666419983,
+      "rewards/rejected": -0.14268335700035095,
+      "step": 560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.7254901960784316e-06,
+      "logits/chosen": -2.2800512313842773,
+      "logits/rejected": -2.203979015350342,
+      "logps/chosen": -259.62908935546875,
+      "logps/rejected": -256.3851318359375,
+      "loss": 0.6429,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.039192236959934235,
+      "rewards/margins": 0.11163588613271713,
+      "rewards/rejected": -0.15082812309265137,
+      "step": 570
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.7908496732026144e-06,
+      "logits/chosen": -2.213808536529541,
+      "logits/rejected": -2.078162431716919,
+      "logps/chosen": -301.599609375,
+      "logps/rejected": -284.9830322265625,
+      "loss": 0.632,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.01635696552693844,
+      "rewards/margins": 0.1448306441307068,
+      "rewards/rejected": -0.16118761897087097,
+      "step": 580
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.856209150326798e-06,
+      "logits/chosen": -2.244199752807617,
+      "logits/rejected": -2.0686116218566895,
+      "logps/chosen": -286.9233093261719,
+      "logps/rejected": -259.3969421386719,
+      "loss": 0.6437,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.06086136773228645,
+      "rewards/margins": 0.12167215347290039,
+      "rewards/rejected": -0.18253351747989655,
+      "step": 590
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.92156862745098e-06,
+      "logits/chosen": -2.0786757469177246,
+      "logits/rejected": -1.9320405721664429,
+      "logps/chosen": -242.836181640625,
+      "logps/rejected": -253.8811798095703,
+      "loss": 0.6565,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.09613479673862457,
+      "rewards/margins": 0.10148496925830841,
+      "rewards/rejected": -0.1976197510957718,
+      "step": 600
+    },
+    {
+      "epoch": 0.08,
+      "eval_logits/chosen": -2.068483591079712,
+      "eval_logits/rejected": -1.9762624502182007,
+      "eval_logps/chosen": -278.922607421875,
+      "eval_logps/rejected": -272.44207763671875,
+      "eval_loss": 0.6402838230133057,
+      "eval_rewards/accuracies": 0.6804999709129333,
+      "eval_rewards/chosen": -0.10363046079874039,
+      "eval_rewards/margins": 0.1327204555273056,
+      "eval_rewards/rejected": -0.23635093867778778,
+      "eval_runtime": 1345.261,
+      "eval_samples_per_second": 1.487,
+      "eval_steps_per_second": 0.186,
+      "step": 600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.986928104575164e-06,
+      "logits/chosen": -2.1265506744384766,
+      "logits/rejected": -1.959896445274353,
+      "logps/chosen": -233.31161499023438,
+      "logps/rejected": -217.30517578125,
+      "loss": 0.628,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.07696186006069183,
+      "rewards/margins": 0.15258391201496124,
+      "rewards/rejected": -0.22954575717449188,
+      "step": 610
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.052287581699347e-06,
+      "logits/chosen": -2.1296346187591553,
+      "logits/rejected": -1.984848976135254,
+      "logps/chosen": -288.03912353515625,
+      "logps/rejected": -286.7398986816406,
+      "loss": 0.6076,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.11124956607818604,
+      "rewards/margins": 0.20728126168251038,
+      "rewards/rejected": -0.318530797958374,
+      "step": 620
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.11764705882353e-06,
+      "logits/chosen": -2.099287748336792,
+      "logits/rejected": -2.044370651245117,
+      "logps/chosen": -284.10955810546875,
+      "logps/rejected": -279.2649841308594,
+      "loss": 0.644,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.20420393347740173,
+      "rewards/margins": 0.1350855976343155,
+      "rewards/rejected": -0.3392895758152008,
+      "step": 630
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.183006535947713e-06,
+      "logits/chosen": -2.179161310195923,
+      "logits/rejected": -2.028665065765381,
+      "logps/chosen": -286.99542236328125,
+      "logps/rejected": -287.5129089355469,
+      "loss": 0.6607,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.3067086338996887,
+      "rewards/margins": 0.0990782156586647,
+      "rewards/rejected": -0.40578681230545044,
+      "step": 640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.2483660130718954e-06,
+      "logits/chosen": -2.1218552589416504,
+      "logits/rejected": -2.070801258087158,
+      "logps/chosen": -286.2041015625,
+      "logps/rejected": -284.69085693359375,
+      "loss": 0.657,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.3266238272190094,
+      "rewards/margins": 0.11262993514537811,
+      "rewards/rejected": -0.4392537474632263,
+      "step": 650
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.313725490196079e-06,
+      "logits/chosen": -2.1642332077026367,
+      "logits/rejected": -2.0245330333709717,
+      "logps/chosen": -320.84771728515625,
+      "logps/rejected": -280.52398681640625,
+      "loss": 0.6261,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.24117839336395264,
+      "rewards/margins": 0.16396468877792358,
+      "rewards/rejected": -0.4051431119441986,
+      "step": 660
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.379084967320262e-06,
+      "logits/chosen": -2.2846810817718506,
+      "logits/rejected": -2.1570193767547607,
+      "logps/chosen": -292.38970947265625,
+      "logps/rejected": -324.989013671875,
+      "loss": 0.6209,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1101335883140564,
+      "rewards/margins": 0.183219313621521,
+      "rewards/rejected": -0.2933529019355774,
+      "step": 670
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.444444444444444e-06,
+      "logits/chosen": -2.272421360015869,
+      "logits/rejected": -2.1363396644592285,
+      "logps/chosen": -281.40045166015625,
+      "logps/rejected": -274.00457763671875,
+      "loss": 0.6574,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.1361692249774933,
+      "rewards/margins": 0.10350640118122101,
+      "rewards/rejected": -0.2396756112575531,
+      "step": 680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.509803921568628e-06,
+      "logits/chosen": -2.1876792907714844,
+      "logits/rejected": -2.097910165786743,
+      "logps/chosen": -308.5197448730469,
+      "logps/rejected": -279.525390625,
+      "loss": 0.6436,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.14121045172214508,
+      "rewards/margins": 0.14382004737854004,
+      "rewards/rejected": -0.2850305140018463,
+      "step": 690
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.5751633986928105e-06,
+      "logits/chosen": -2.168365001678467,
+      "logits/rejected": -2.1110668182373047,
+      "logps/chosen": -287.0357666015625,
+      "logps/rejected": -290.0934143066406,
+      "loss": 0.6411,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.19541046023368835,
+      "rewards/margins": 0.15668657422065735,
+      "rewards/rejected": -0.3520970642566681,
+      "step": 700
+    },
+    {
+      "epoch": 0.09,
+      "eval_logits/chosen": -2.0180885791778564,
+      "eval_logits/rejected": -1.919668436050415,
+      "eval_logps/chosen": -283.8720397949219,
+      "eval_logps/rejected": -282.3092041015625,
+      "eval_loss": 0.6254200339317322,
+      "eval_rewards/accuracies": 0.6819999814033508,
+      "eval_rewards/chosen": -0.15312448143959045,
+      "eval_rewards/margins": 0.18189799785614014,
+      "eval_rewards/rejected": -0.3350224792957306,
+      "eval_runtime": 1346.673,
+      "eval_samples_per_second": 1.485,
+      "eval_steps_per_second": 0.186,
+      "step": 700
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.640522875816994e-06,
+      "logits/chosen": -2.1289966106414795,
+      "logits/rejected": -2.0453264713287354,
+      "logps/chosen": -285.4678649902344,
+      "logps/rejected": -289.77227783203125,
+      "loss": 0.6191,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.10420187562704086,
+      "rewards/margins": 0.20611795783042908,
+      "rewards/rejected": -0.31031984090805054,
+      "step": 710
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.705882352941177e-06,
+      "logits/chosen": -2.1111762523651123,
+      "logits/rejected": -2.0428059101104736,
+      "logps/chosen": -325.65167236328125,
+      "logps/rejected": -322.3482360839844,
+      "loss": 0.6086,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1119033545255661,
+      "rewards/margins": 0.23122373223304749,
+      "rewards/rejected": -0.3431270718574524,
+      "step": 720
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.77124183006536e-06,
+      "logits/chosen": -2.121738910675049,
+      "logits/rejected": -2.060598850250244,
+      "logps/chosen": -288.5126647949219,
+      "logps/rejected": -295.1073913574219,
+      "loss": 0.6138,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.10401918739080429,
+      "rewards/margins": 0.19626674056053162,
+      "rewards/rejected": -0.3002859055995941,
+      "step": 730
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.836601307189543e-06,
+      "logits/chosen": -2.07651948928833,
+      "logits/rejected": -1.910945177078247,
+      "logps/chosen": -305.45599365234375,
+      "logps/rejected": -284.6485900878906,
+      "loss": 0.6079,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1200060099363327,
+      "rewards/margins": 0.2301141321659088,
+      "rewards/rejected": -0.3501201570034027,
+      "step": 740
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.901960784313726e-06,
+      "logits/chosen": -2.095877170562744,
+      "logits/rejected": -2.005570650100708,
+      "logps/chosen": -256.7840881347656,
+      "logps/rejected": -267.2419738769531,
+      "loss": 0.5989,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.1889648139476776,
+      "rewards/margins": 0.2477390021085739,
+      "rewards/rejected": -0.4367038607597351,
+      "step": 750
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.967320261437909e-06,
+      "logits/chosen": -2.111067295074463,
+      "logits/rejected": -1.9038223028182983,
+      "logps/chosen": -290.6449279785156,
+      "logps/rejected": -266.15972900390625,
+      "loss": 0.6199,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.19410932064056396,
+      "rewards/margins": 0.19198177754878998,
+      "rewards/rejected": -0.38609108328819275,
+      "step": 760
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.99999347843947e-06,
+      "logits/chosen": -2.033259391784668,
+      "logits/rejected": -2.011737823486328,
+      "logps/chosen": -298.57135009765625,
+      "logps/rejected": -296.5043640136719,
+      "loss": 0.6363,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.17297473549842834,
+      "rewards/margins": 0.18455691635608673,
+      "rewards/rejected": -0.3575316369533539,
+      "step": 770
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999941306159375e-06,
+      "logits/chosen": -2.1474356651306152,
+      "logits/rejected": -1.9568185806274414,
+      "logps/chosen": -310.86865234375,
+      "logps/rejected": -304.8359069824219,
+      "loss": 0.599,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.20156534016132355,
+      "rewards/margins": 0.2747187912464142,
+      "rewards/rejected": -0.4762841761112213,
+      "step": 780
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999836962687967e-06,
+      "logits/chosen": -2.033233165740967,
+      "logits/rejected": -2.039140462875366,
+      "logps/chosen": -307.1895446777344,
+      "logps/rejected": -336.08331298828125,
+      "loss": 0.5866,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.31010061502456665,
+      "rewards/margins": 0.3080299496650696,
+      "rewards/rejected": -0.618130624294281,
+      "step": 790
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999680450202786e-06,
+      "logits/chosen": -1.819702386856079,
+      "logits/rejected": -1.810394525527954,
+      "logps/chosen": -290.6375427246094,
+      "logps/rejected": -325.89654541015625,
+      "loss": 0.6177,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.3965233862400055,
+      "rewards/margins": 0.25661128759384155,
+      "rewards/rejected": -0.6531347036361694,
+      "step": 800
+    },
+    {
+      "epoch": 0.1,
+      "eval_logits/chosen": -1.930443525314331,
+      "eval_logits/rejected": -1.8202427625656128,
+      "eval_logps/chosen": -307.0185546875,
+      "eval_logps/rejected": -313.1127624511719,
+      "eval_loss": 0.6134334206581116,
+      "eval_rewards/accuracies": 0.6765000224113464,
+      "eval_rewards/chosen": -0.3845898509025574,
+      "eval_rewards/margins": 0.25846797227859497,
+      "eval_rewards/rejected": -0.6430578827857971,
+      "eval_runtime": 1347.2508,
+      "eval_samples_per_second": 1.485,
+      "eval_steps_per_second": 0.186,
+      "step": 800
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.999471771970087e-06,
+      "logits/chosen": -2.04258394241333,
+      "logits/rejected": -1.9091304540634155,
+      "logps/chosen": -315.5386047363281,
+      "logps/rejected": -313.8574523925781,
+      "loss": 0.5916,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3548921048641205,
+      "rewards/margins": 0.289425790309906,
+      "rewards/rejected": -0.6443178653717041,
+      "step": 810
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.999210932344767e-06,
+      "logits/chosen": -2.1729469299316406,
+      "logits/rejected": -2.030179023742676,
+      "logps/chosen": -335.8345031738281,
+      "logps/rejected": -319.205078125,
+      "loss": 0.5943,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.3209165036678314,
+      "rewards/margins": 0.28550076484680176,
+      "rewards/rejected": -0.6064172983169556,
+      "step": 820
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.998897936770281e-06,
+      "logits/chosen": -1.9827296733856201,
+      "logits/rejected": -2.0415992736816406,
+      "logps/chosen": -257.7474670410156,
+      "logps/rejected": -314.6881408691406,
+      "loss": 0.5921,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.29162853956222534,
+      "rewards/margins": 0.29887694120407104,
+      "rewards/rejected": -0.5905055403709412,
+      "step": 830
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.998532791778521e-06,
+      "logits/chosen": -1.9418144226074219,
+      "logits/rejected": -1.7791082859039307,
+      "logps/chosen": -318.93170166015625,
+      "logps/rejected": -291.5006103515625,
+      "loss": 0.5887,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.35302484035491943,
+      "rewards/margins": 0.32170987129211426,
+      "rewards/rejected": -0.6747347116470337,
+      "step": 840
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.9981155049896885e-06,
+      "logits/chosen": -1.990808129310608,
+      "logits/rejected": -1.9234561920166016,
+      "logps/chosen": -315.5331726074219,
+      "logps/rejected": -359.76123046875,
+      "loss": 0.6135,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.43257227540016174,
+      "rewards/margins": 0.276610791683197,
+      "rewards/rejected": -0.7091829776763916,
+      "step": 850
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.997646085112126e-06,
+      "logits/chosen": -2.009455680847168,
+      "logits/rejected": -1.9086802005767822,
+      "logps/chosen": -292.7685546875,
+      "logps/rejected": -319.24676513671875,
+      "loss": 0.6211,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.5454513430595398,
+      "rewards/margins": 0.2806360125541687,
+      "rewards/rejected": -0.8260873556137085,
+      "step": 860
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.997124541942141e-06,
+      "logits/chosen": -2.0661356449127197,
+      "logits/rejected": -1.9607994556427002,
+      "logps/chosen": -319.46124267578125,
+      "logps/rejected": -332.58465576171875,
+      "loss": 0.6352,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.43448805809020996,
+      "rewards/margins": 0.231379434466362,
+      "rewards/rejected": -0.6658674478530884,
+      "step": 870
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.996550886363801e-06,
+      "logits/chosen": -1.9991384744644165,
+      "logits/rejected": -1.9224755764007568,
+      "logps/chosen": -297.3063659667969,
+      "logps/rejected": -330.55364990234375,
+      "loss": 0.6501,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.47025617957115173,
+      "rewards/margins": 0.20600691437721252,
+      "rewards/rejected": -0.6762630343437195,
+      "step": 880
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.995925130348706e-06,
+      "logits/chosen": -2.0374197959899902,
+      "logits/rejected": -1.9763472080230713,
+      "logps/chosen": -291.50054931640625,
+      "logps/rejected": -322.2240295410156,
+      "loss": 0.626,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.387765109539032,
+      "rewards/margins": 0.23342053592205048,
+      "rewards/rejected": -0.6211856603622437,
+      "step": 890
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.995247286955734e-06,
+      "logits/chosen": -2.044053554534912,
+      "logits/rejected": -1.8695251941680908,
+      "logps/chosen": -295.85137939453125,
+      "logps/rejected": -287.2956848144531,
+      "loss": 0.6333,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5495660305023193,
+      "rewards/margins": 0.2384297400712967,
+      "rewards/rejected": -0.7879956960678101,
+      "step": 900
+    },
+    {
+      "epoch": 0.12,
+      "eval_logits/chosen": -1.9660098552703857,
+      "eval_logits/rejected": -1.8565987348556519,
+      "eval_logps/chosen": -308.6199035644531,
+      "eval_logps/rejected": -317.15264892578125,
+      "eval_loss": 0.6081622838973999,
+      "eval_rewards/accuracies": 0.6740000247955322,
+      "eval_rewards/chosen": -0.40060317516326904,
+      "eval_rewards/margins": 0.28285351395606995,
+      "eval_rewards/rejected": -0.6834567189216614,
+      "eval_runtime": 1348.8352,
+      "eval_samples_per_second": 1.483,
+      "eval_steps_per_second": 0.185,
+      "step": 900
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.994517370330779e-06,
+      "logits/chosen": -1.987975835800171,
+      "logits/rejected": -2.027632474899292,
+      "logps/chosen": -306.1188049316406,
+      "logps/rejected": -366.084228515625,
+      "loss": 0.5879,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.3855474889278412,
+      "rewards/margins": 0.3462125062942505,
+      "rewards/rejected": -0.7317599058151245,
+      "step": 910
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.993735395706446e-06,
+      "logits/chosen": -2.085257053375244,
+      "logits/rejected": -1.9146783351898193,
+      "logps/chosen": -269.17791748046875,
+      "logps/rejected": -305.6871032714844,
+      "loss": 0.5722,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.41887688636779785,
+      "rewards/margins": 0.37890392541885376,
+      "rewards/rejected": -0.7977808117866516,
+      "step": 920
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.992901379401737e-06,
+      "logits/chosen": -2.1707818508148193,
+      "logits/rejected": -2.048088312149048,
+      "logps/chosen": -349.13177490234375,
+      "logps/rejected": -356.95135498046875,
+      "loss": 0.602,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.4841079115867615,
+      "rewards/margins": 0.3337276577949524,
+      "rewards/rejected": -0.8178355097770691,
+      "step": 930
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.992015338821711e-06,
+      "logits/chosen": -2.1080679893493652,
+      "logits/rejected": -1.9318233728408813,
+      "logps/chosen": -342.4098815917969,
+      "logps/rejected": -362.0527648925781,
+      "loss": 0.5658,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.6447209119796753,
+      "rewards/margins": 0.36333078145980835,
+      "rewards/rejected": -1.0080516338348389,
+      "step": 940
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.991077292457117e-06,
+      "logits/chosen": -2.117976188659668,
+      "logits/rejected": -1.9404761791229248,
+      "logps/chosen": -330.13812255859375,
+      "logps/rejected": -331.7317810058594,
+      "loss": 0.5881,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.5888674259185791,
+      "rewards/margins": 0.35803017020225525,
+      "rewards/rejected": -0.9468975067138672,
+      "step": 950
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.990087259884016e-06,
+      "logits/chosen": -2.113210916519165,
+      "logits/rejected": -2.0068416595458984,
+      "logps/chosen": -284.6747131347656,
+      "logps/rejected": -291.47100830078125,
+      "loss": 0.6023,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.4492453634738922,
+      "rewards/margins": 0.33002012968063354,
+      "rewards/rejected": -0.7792654037475586,
+      "step": 960
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.989045261763362e-06,
+      "logits/chosen": -2.160799503326416,
+      "logits/rejected": -2.1110472679138184,
+      "logps/chosen": -311.8405456542969,
+      "logps/rejected": -344.5122375488281,
+      "loss": 0.5186,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.3906407952308655,
+      "rewards/margins": 0.472191721200943,
+      "rewards/rejected": -0.8628325462341309,
+      "step": 970
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.98795131984058e-06,
+      "logits/chosen": -1.9881279468536377,
+      "logits/rejected": -1.943418264389038,
+      "logps/chosen": -303.29058837890625,
+      "logps/rejected": -309.2772521972656,
+      "loss": 0.6097,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.5643569231033325,
+      "rewards/margins": 0.35135558247566223,
+      "rewards/rejected": -0.9157125353813171,
+      "step": 980
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.986805456945107e-06,
+      "logits/chosen": -2.0823357105255127,
+      "logits/rejected": -1.9641021490097046,
+      "logps/chosen": -281.0516357421875,
+      "logps/rejected": -313.0377502441406,
+      "loss": 0.546,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.5891399383544922,
+      "rewards/margins": 0.5124160647392273,
+      "rewards/rejected": -1.1015560626983643,
+      "step": 990
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.985607696989919e-06,
+      "logits/chosen": -2.1286559104919434,
+      "logits/rejected": -2.1203885078430176,
+      "logps/chosen": -330.2342529296875,
+      "logps/rejected": -364.49468994140625,
+      "loss": 0.5776,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.6434313654899597,
+      "rewards/margins": 0.44300857186317444,
+      "rewards/rejected": -1.0864399671554565,
+      "step": 1000
+    },
+    {
+      "epoch": 0.13,
+      "eval_logits/chosen": -2.003793954849243,
+      "eval_logits/rejected": -1.8956444263458252,
+      "eval_logps/chosen": -335.0627136230469,
+      "eval_logps/rejected": -351.8793640136719,
+      "eval_loss": 0.6065914034843445,
+      "eval_rewards/accuracies": 0.6735000014305115,
+      "eval_rewards/chosen": -0.665031373500824,
+      "eval_rewards/margins": 0.365693062543869,
+      "eval_rewards/rejected": -1.0307244062423706,
+      "eval_runtime": 1346.3958,
+      "eval_samples_per_second": 1.485,
+      "eval_steps_per_second": 0.186,
+      "step": 1000
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.984358064971026e-06,
+      "logits/chosen": -2.0841567516326904,
+      "logits/rejected": -1.9348751306533813,
+      "logps/chosen": -332.2041015625,
+      "logps/rejected": -326.98809814453125,
+      "loss": 0.6494,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.7189096212387085,
+      "rewards/margins": 0.22981159389019012,
+      "rewards/rejected": -0.9487211108207703,
+      "step": 1010
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.983056586966958e-06,
+      "logits/chosen": -2.0408473014831543,
+      "logits/rejected": -1.890376329421997,
+      "logps/chosen": -361.5563049316406,
+      "logps/rejected": -338.8031311035156,
+      "loss": 0.5289,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.5372278094291687,
+      "rewards/margins": 0.49330466985702515,
+      "rewards/rejected": -1.0305324792861938,
+      "step": 1020
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.981703290138215e-06,
+      "logits/chosen": -2.160186290740967,
+      "logits/rejected": -2.051741600036621,
+      "logps/chosen": -373.94537353515625,
+      "logps/rejected": -394.06414794921875,
+      "loss": 0.5962,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.7640237808227539,
+      "rewards/margins": 0.4476410746574402,
+      "rewards/rejected": -1.2116649150848389,
+      "step": 1030
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.980298202726706e-06,
+      "logits/chosen": -2.264606237411499,
+      "logits/rejected": -2.1024699211120605,
+      "logps/chosen": -379.8427429199219,
+      "logps/rejected": -394.89111328125,
+      "loss": 0.528,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.7532575726509094,
+      "rewards/margins": 0.5998953580856323,
+      "rewards/rejected": -1.3531529903411865,
+      "step": 1040
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.978841354055148e-06,
+      "logits/chosen": -2.0314176082611084,
+      "logits/rejected": -1.9465423822402954,
+      "logps/chosen": -345.968017578125,
+      "logps/rejected": -362.8763122558594,
+      "loss": 0.619,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.904411792755127,
+      "rewards/margins": 0.3138309121131897,
+      "rewards/rejected": -1.2182427644729614,
+      "step": 1050
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.977332774526471e-06,
+      "logits/chosen": -1.9704465866088867,
+      "logits/rejected": -1.6791388988494873,
+      "logps/chosen": -336.9738464355469,
+      "logps/rejected": -326.60406494140625,
+      "loss": 0.5847,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.6676698327064514,
+      "rewards/margins": 0.4430173933506012,
+      "rewards/rejected": -1.1106873750686646,
+      "step": 1060
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.97577249562317e-06,
+      "logits/chosen": -1.9557006359100342,
+      "logits/rejected": -1.8785998821258545,
+      "logps/chosen": -301.1720886230469,
+      "logps/rejected": -341.7430725097656,
+      "loss": 0.5322,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.45251208543777466,
+      "rewards/margins": 0.5301971435546875,
+      "rewards/rejected": -0.9827091097831726,
+      "step": 1070
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.974160549906652e-06,
+      "logits/chosen": -1.9266964197158813,
+      "logits/rejected": -1.827126145362854,
+      "logps/chosen": -292.7878723144531,
+      "logps/rejected": -310.02056884765625,
+      "loss": 0.5964,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.4741947054862976,
+      "rewards/margins": 0.35469281673431396,
+      "rewards/rejected": -0.8288875818252563,
+      "step": 1080
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.972496971016559e-06,
+      "logits/chosen": -1.9643986225128174,
+      "logits/rejected": -1.9294040203094482,
+      "logps/chosen": -287.1049499511719,
+      "logps/rejected": -301.7425537109375,
+      "loss": 0.5462,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.38289979100227356,
+      "rewards/margins": 0.49448949098587036,
+      "rewards/rejected": -0.8773892521858215,
+      "step": 1090
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.9707817936700635e-06,
+      "logits/chosen": -2.0732617378234863,
+      "logits/rejected": -1.930274248123169,
+      "logps/chosen": -317.2794189453125,
+      "logps/rejected": -328.32550048828125,
+      "loss": 0.6093,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.512849748134613,
+      "rewards/margins": 0.32138776779174805,
+      "rewards/rejected": -0.8342375755310059,
+      "step": 1100
+    },
+    {
+      "epoch": 0.14,
+      "eval_logits/chosen": -2.0021698474884033,
+      "eval_logits/rejected": -1.9019155502319336,
+      "eval_logps/chosen": -324.48455810546875,
+      "eval_logps/rejected": -341.52301025390625,
+      "eval_loss": 0.6074594259262085,
+      "eval_rewards/accuracies": 0.6740000247955322,
+      "eval_rewards/chosen": -0.5592499375343323,
+      "eval_rewards/margins": 0.36791032552719116,
+      "eval_rewards/rejected": -0.9271602630615234,
+      "eval_runtime": 1346.3334,
+      "eval_samples_per_second": 1.486,
+      "eval_steps_per_second": 0.186,
+      "step": 1100
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.969015053661142e-06,
+      "logits/chosen": -1.9875437021255493,
+      "logits/rejected": -1.8942997455596924,
+      "logps/chosen": -327.0362243652344,
+      "logps/rejected": -337.4400939941406,
+      "loss": 0.6393,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.5959460139274597,
+      "rewards/margins": 0.33921343088150024,
+      "rewards/rejected": -0.9351595044136047,
+      "step": 1110
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.967196787859835e-06,
+      "logits/chosen": -2.131622314453125,
+      "logits/rejected": -2.010146379470825,
+      "logps/chosen": -344.14471435546875,
+      "logps/rejected": -369.2281188964844,
+      "loss": 0.5931,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.6821675896644592,
+      "rewards/margins": 0.3684613108634949,
+      "rewards/rejected": -1.0506290197372437,
+      "step": 1120
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.965327034211469e-06,
+      "logits/chosen": -2.0446646213531494,
+      "logits/rejected": -1.9094947576522827,
+      "logps/chosen": -346.3064270019531,
+      "logps/rejected": -359.83135986328125,
+      "loss": 0.523,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5417262315750122,
+      "rewards/margins": 0.5890461802482605,
+      "rewards/rejected": -1.130772352218628,
+      "step": 1130
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.96340583173587e-06,
+      "logits/chosen": -2.0372977256774902,
+      "logits/rejected": -1.8208487033843994,
+      "logps/chosen": -294.8141174316406,
+      "logps/rejected": -309.7556457519531,
+      "loss": 0.5627,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.604282021522522,
+      "rewards/margins": 0.43121084570884705,
+      "rewards/rejected": -1.0354927778244019,
+      "step": 1140
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.96143322052655e-06,
+      "logits/chosen": -2.02699875831604,
+      "logits/rejected": -2.104145050048828,
+      "logps/chosen": -323.2965087890625,
+      "logps/rejected": -351.07012939453125,
+      "loss": 0.5735,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.5178147554397583,
+      "rewards/margins": 0.3925105035305023,
+      "rewards/rejected": -0.9103251695632935,
+      "step": 1150
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.959409241749864e-06,
+      "logits/chosen": -2.0998756885528564,
+      "logits/rejected": -1.9320350885391235,
+      "logps/chosen": -323.5308532714844,
+      "logps/rejected": -368.880859375,
+      "loss": 0.5674,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.5751530528068542,
+      "rewards/margins": 0.45413222908973694,
+      "rewards/rejected": -1.029285192489624,
+      "step": 1160
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.957333937644159e-06,
+      "logits/chosen": -2.1263599395751953,
+      "logits/rejected": -2.02579665184021,
+      "logps/chosen": -348.6449279785156,
+      "logps/rejected": -370.02642822265625,
+      "loss": 0.5391,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.6443693041801453,
+      "rewards/margins": 0.5169423818588257,
+      "rewards/rejected": -1.1613117456436157,
+      "step": 1170
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.955207351518885e-06,
+      "logits/chosen": -2.1276612281799316,
+      "logits/rejected": -2.097865581512451,
+      "logps/chosen": -324.40899658203125,
+      "logps/rejected": -328.3305969238281,
+      "loss": 0.6008,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.6907325983047485,
+      "rewards/margins": 0.3582058548927307,
+      "rewards/rejected": -1.048938512802124,
+      "step": 1180
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.953029527753699e-06,
+      "logits/chosen": -2.0528275966644287,
+      "logits/rejected": -2.047828435897827,
+      "logps/chosen": -309.662109375,
+      "logps/rejected": -412.89788818359375,
+      "loss": 0.6198,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.806117057800293,
+      "rewards/margins": 0.34065908193588257,
+      "rewards/rejected": -1.1467760801315308,
+      "step": 1190
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.95080051179753e-06,
+      "logits/chosen": -1.955240249633789,
+      "logits/rejected": -1.9605426788330078,
+      "logps/chosen": -323.08514404296875,
+      "logps/rejected": -367.53924560546875,
+      "loss": 0.5607,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.825681209564209,
+      "rewards/margins": 0.46968841552734375,
+      "rewards/rejected": -1.2953693866729736,
+      "step": 1200
+    },
+    {
+      "epoch": 0.16,
+      "eval_logits/chosen": -1.9182138442993164,
+      "eval_logits/rejected": -1.8081015348434448,
+      "eval_logps/chosen": -352.83721923828125,
+      "eval_logps/rejected": -375.3465576171875,
+      "eval_loss": 0.5970308780670166,
+      "eval_rewards/accuracies": 0.6800000071525574,
+      "eval_rewards/chosen": -0.8427762389183044,
+      "eval_rewards/margins": 0.4226198196411133,
+      "eval_rewards/rejected": -1.265395998954773,
+      "eval_runtime": 1348.871,
+      "eval_samples_per_second": 1.483,
+      "eval_steps_per_second": 0.185,
+      "step": 1200
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.948520350167637e-06,
+      "logits/chosen": -1.9895503520965576,
+      "logits/rejected": -1.897316575050354,
+      "logps/chosen": -351.7968444824219,
+      "logps/rejected": -382.85748291015625,
+      "loss": 0.5691,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.8973641395568848,
+      "rewards/margins": 0.46902114152908325,
+      "rewards/rejected": -1.3663853406906128,
+      "step": 1210
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.946189090448639e-06,
+      "logits/chosen": -1.922502875328064,
+      "logits/rejected": -1.8165347576141357,
+      "logps/chosen": -360.2001647949219,
+      "logps/rejected": -336.5096435546875,
+      "loss": 0.714,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.9478484988212585,
+      "rewards/margins": 0.13732792437076569,
+      "rewards/rejected": -1.0851763486862183,
+      "step": 1220
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.943806781291515e-06,
+      "logits/chosen": -2.002378225326538,
+      "logits/rejected": -1.84847092628479,
+      "logps/chosen": -312.14483642578125,
+      "logps/rejected": -326.6715087890625,
+      "loss": 0.5578,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5726705193519592,
+      "rewards/margins": 0.4877811372280121,
+      "rewards/rejected": -1.0604517459869385,
+      "step": 1230
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.941373472412595e-06,
+      "logits/chosen": -1.9809534549713135,
+      "logits/rejected": -1.9042818546295166,
+      "logps/chosen": -373.47161865234375,
+      "logps/rejected": -391.1158752441406,
+      "loss": 0.5844,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.6417821049690247,
+      "rewards/margins": 0.39200183749198914,
+      "rewards/rejected": -1.0337839126586914,
+      "step": 1240
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.938889214592521e-06,
+      "logits/chosen": -1.9450843334197998,
+      "logits/rejected": -1.8153011798858643,
+      "logps/chosen": -381.24560546875,
+      "logps/rejected": -378.72698974609375,
+      "loss": 0.6158,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.6465792655944824,
+      "rewards/margins": 0.3059485852718353,
+      "rewards/rejected": -0.9525278210639954,
+      "step": 1250
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.936354059675186e-06,
+      "logits/chosen": -1.892221212387085,
+      "logits/rejected": -1.7443745136260986,
+      "logps/chosen": -341.537353515625,
+      "logps/rejected": -342.2924499511719,
+      "loss": 0.5684,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.7926312685012817,
+      "rewards/margins": 0.38524889945983887,
+      "rewards/rejected": -1.177880048751831,
+      "step": 1260
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.933768060566654e-06,
+      "logits/chosen": -1.7476335763931274,
+      "logits/rejected": -1.5973765850067139,
+      "logps/chosen": -369.65081787109375,
+      "logps/rejected": -383.3055725097656,
+      "loss": 0.5981,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.0846364498138428,
+      "rewards/margins": 0.35797643661499023,
+      "rewards/rejected": -1.4426130056381226,
+      "step": 1270
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.931131271234052e-06,
+      "logits/chosen": -1.6848058700561523,
+      "logits/rejected": -1.3915191888809204,
+      "logps/chosen": -374.5550537109375,
+      "logps/rejected": -370.0396423339844,
+      "loss": 0.5877,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.2266910076141357,
+      "rewards/margins": 0.34090346097946167,
+      "rewards/rejected": -1.567594289779663,
+      "step": 1280
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.928443746704448e-06,
+      "logits/chosen": -1.50856614112854,
+      "logits/rejected": -1.3586972951889038,
+      "logps/chosen": -434.8419494628906,
+      "logps/rejected": -429.79254150390625,
+      "loss": 0.5896,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.321608304977417,
+      "rewards/margins": 0.42720308899879456,
+      "rewards/rejected": -1.7488113641738892,
+      "step": 1290
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.925705543063703e-06,
+      "logits/chosen": -1.3820029497146606,
+      "logits/rejected": -1.4208353757858276,
+      "logps/chosen": -399.9224548339844,
+      "logps/rejected": -480.36077880859375,
+      "loss": 0.5627,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.2707209587097168,
+      "rewards/margins": 0.4977220594882965,
+      "rewards/rejected": -1.7684433460235596,
+      "step": 1300
+    },
+    {
+      "epoch": 0.17,
+      "eval_logits/chosen": -1.3202824592590332,
+      "eval_logits/rejected": -1.1519203186035156,
+      "eval_logps/chosen": -411.944580078125,
+      "eval_logps/rejected": -433.7877197265625,
+      "eval_loss": 0.5935286283493042,
+      "eval_rewards/accuracies": 0.6850000023841858,
+      "eval_rewards/chosen": -1.4338501691818237,
+      "eval_rewards/margins": 0.415957510471344,
+      "eval_rewards/rejected": -1.8498077392578125,
+      "eval_runtime": 1348.1226,
+      "eval_samples_per_second": 1.484,
+      "eval_steps_per_second": 0.185,
+      "step": 1300
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.922916717455297e-06,
+      "logits/chosen": -1.5533722639083862,
+      "logits/rejected": -1.2233012914657593,
+      "logps/chosen": -429.00457763671875,
+      "logps/rejected": -415.05999755859375,
+      "loss": 0.5778,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.415178656578064,
+      "rewards/margins": 0.4627149999141693,
+      "rewards/rejected": -1.8778936862945557,
+      "step": 1310
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.920077328079136e-06,
+      "logits/chosen": -1.5570157766342163,
+      "logits/rejected": -1.2419131994247437,
+      "logps/chosen": -416.5760192871094,
+      "logps/rejected": -406.1861572265625,
+      "loss": 0.6179,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.358478307723999,
+      "rewards/margins": 0.34959474205970764,
+      "rewards/rejected": -1.7080730199813843,
+      "step": 1320
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.9171874341903445e-06,
+      "logits/chosen": -1.418280839920044,
+      "logits/rejected": -1.2672127485275269,
+      "logps/chosen": -394.59918212890625,
+      "logps/rejected": -458.80059814453125,
+      "loss": 0.5081,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.306119680404663,
+      "rewards/margins": 0.6360645294189453,
+      "rewards/rejected": -1.9421840906143188,
+      "step": 1330
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.914247096098019e-06,
+      "logits/chosen": -1.3951623439788818,
+      "logits/rejected": -1.3654650449752808,
+      "logps/chosen": -399.6478271484375,
+      "logps/rejected": -422.133056640625,
+      "loss": 0.6578,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.3707911968231201,
+      "rewards/margins": 0.21062076091766357,
+      "rewards/rejected": -1.5814119577407837,
+      "step": 1340
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.911256375163977e-06,
+      "logits/chosen": -1.1869124174118042,
+      "logits/rejected": -1.3260669708251953,
+      "logps/chosen": -336.62066650390625,
+      "logps/rejected": -408.3683776855469,
+      "loss": 0.5649,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.1873462200164795,
+      "rewards/margins": 0.5111492872238159,
+      "rewards/rejected": -1.698495626449585,
+      "step": 1350
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.908215333801474e-06,
+      "logits/chosen": -1.4656002521514893,
+      "logits/rejected": -1.303011417388916,
+      "logps/chosen": -355.81195068359375,
+      "logps/rejected": -376.6522521972656,
+      "loss": 0.5316,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.1016346216201782,
+      "rewards/margins": 0.5347610712051392,
+      "rewards/rejected": -1.6363956928253174,
+      "step": 1360
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.9051240354739004e-06,
+      "logits/chosen": -1.4606434106826782,
+      "logits/rejected": -1.3179041147232056,
+      "logps/chosen": -413.5399475097656,
+      "logps/rejected": -417.6502380371094,
+      "loss": 0.6447,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.1709959506988525,
+      "rewards/margins": 0.2992558479309082,
+      "rewards/rejected": -1.4702519178390503,
+      "step": 1370
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.901982544693457e-06,
+      "logits/chosen": -1.5680989027023315,
+      "logits/rejected": -1.3608156442642212,
+      "logps/chosen": -375.6697082519531,
+      "logps/rejected": -419.6275329589844,
+      "loss": 0.4882,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8651687502861023,
+      "rewards/margins": 0.707001268863678,
+      "rewards/rejected": -1.5721700191497803,
+      "step": 1380
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.898790927019809e-06,
+      "logits/chosen": -1.2418692111968994,
+      "logits/rejected": -1.0983781814575195,
+      "logps/chosen": -382.36395263671875,
+      "logps/rejected": -423.8787536621094,
+      "loss": 0.5961,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.2077820301055908,
+      "rewards/margins": 0.5080002546310425,
+      "rewards/rejected": -1.7157824039459229,
+      "step": 1390
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.895549249058718e-06,
+      "logits/chosen": -1.132901906967163,
+      "logits/rejected": -0.7660635709762573,
+      "logps/chosen": -363.669189453125,
+      "logps/rejected": -394.21063232421875,
+      "loss": 0.5853,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.0653022527694702,
+      "rewards/margins": 0.5547600984573364,
+      "rewards/rejected": -1.620062232017517,
+      "step": 1400
+    },
+    {
+      "epoch": 0.18,
+      "eval_logits/chosen": -1.0520106554031372,
+      "eval_logits/rejected": -0.8708193302154541,
+      "eval_logps/chosen": -389.5525207519531,
+      "eval_logps/rejected": -417.2325439453125,
+      "eval_loss": 0.58421790599823,
+      "eval_rewards/accuracies": 0.6949999928474426,
+      "eval_rewards/chosen": -1.2099299430847168,
+      "eval_rewards/margins": 0.4743254780769348,
+      "eval_rewards/rejected": -1.6842552423477173,
+      "eval_runtime": 1346.416,
+      "eval_samples_per_second": 1.485,
+      "eval_steps_per_second": 0.186,
+      "step": 1400
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.892257578460656e-06,
+      "logits/chosen": -1.1538102626800537,
+      "logits/rejected": -1.0136797428131104,
+      "logps/chosen": -389.5459899902344,
+      "logps/rejected": -429.15576171875,
+      "loss": 0.6171,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.3148356676101685,
+      "rewards/margins": 0.4683291018009186,
+      "rewards/rejected": -1.783164620399475,
+      "step": 1410
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.888915983919383e-06,
+      "logits/chosen": -1.3282802104949951,
+      "logits/rejected": -1.1823840141296387,
+      "logps/chosen": -442.50225830078125,
+      "logps/rejected": -484.5198669433594,
+      "loss": 0.5577,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.3251651525497437,
+      "rewards/margins": 0.5749329924583435,
+      "rewards/rejected": -1.900098204612732,
+      "step": 1420
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.885524535170525e-06,
+      "logits/chosen": -1.12726628780365,
+      "logits/rejected": -1.0665578842163086,
+      "logps/chosen": -387.0057373046875,
+      "logps/rejected": -398.9828186035156,
+      "loss": 0.5768,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.1461999416351318,
+      "rewards/margins": 0.43426451086997986,
+      "rewards/rejected": -1.580464482307434,
+      "step": 1430
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.882083302990113e-06,
+      "logits/chosen": -1.2161692380905151,
+      "logits/rejected": -1.1059772968292236,
+      "logps/chosen": -436.05035400390625,
+      "logps/rejected": -461.33172607421875,
+      "loss": 0.5454,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.256879210472107,
+      "rewards/margins": 0.5998114347457886,
+      "rewards/rejected": -1.8566906452178955,
+      "step": 1440
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.878592359193104e-06,
+      "logits/chosen": -1.1037123203277588,
+      "logits/rejected": -1.0302155017852783,
+      "logps/chosen": -392.379638671875,
+      "logps/rejected": -418.23553466796875,
+      "loss": 0.5535,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.3421741724014282,
+      "rewards/margins": 0.5467718839645386,
+      "rewards/rejected": -1.8889458179473877,
+      "step": 1450
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.875051776631888e-06,
+      "logits/chosen": -1.097325325012207,
+      "logits/rejected": -0.8905621767044067,
+      "logps/chosen": -465.3509826660156,
+      "logps/rejected": -470.8297424316406,
+      "loss": 0.5854,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.3240224123001099,
+      "rewards/margins": 0.5567721128463745,
+      "rewards/rejected": -1.8807942867279053,
+      "step": 1460
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.871461629194764e-06,
+      "logits/chosen": -0.8902055025100708,
+      "logits/rejected": -0.5290982723236084,
+      "logps/chosen": -412.06707763671875,
+      "logps/rejected": -420.2411193847656,
+      "loss": 0.6299,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6500478982925415,
+      "rewards/margins": 0.4187239706516266,
+      "rewards/rejected": -2.0687718391418457,
+      "step": 1470
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.8678219918043984e-06,
+      "logits/chosen": -0.7797793745994568,
+      "logits/rejected": -0.3388027846813202,
+      "logps/chosen": -439.6683654785156,
+      "logps/rejected": -446.1436462402344,
+      "loss": 0.6022,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7856781482696533,
+      "rewards/margins": 0.5206558108329773,
+      "rewards/rejected": -2.3063340187072754,
+      "step": 1480
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.864132940416262e-06,
+      "logits/chosen": -0.917339026927948,
+      "logits/rejected": -0.6046267747879028,
+      "logps/chosen": -431.1070251464844,
+      "logps/rejected": -445.59832763671875,
+      "loss": 0.4937,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.3777341842651367,
+      "rewards/margins": 0.7288486361503601,
+      "rewards/rejected": -2.1065826416015625,
+      "step": 1490
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.860394552017044e-06,
+      "logits/chosen": -0.7166872024536133,
+      "logits/rejected": -0.7468100190162659,
+      "logps/chosen": -425.39752197265625,
+      "logps/rejected": -487.08795166015625,
+      "loss": 0.5622,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.5589123964309692,
+      "rewards/margins": 0.5889005661010742,
+      "rewards/rejected": -2.147813081741333,
+      "step": 1500
+    },
+    {
+      "epoch": 0.2,
+      "eval_logits/chosen": -0.6560909748077393,
+      "eval_logits/rejected": -0.4323239028453827,
+      "eval_logps/chosen": -419.2692565917969,
+      "eval_logps/rejected": -453.90203857421875,
+      "eval_loss": 0.5712173581123352,
+      "eval_rewards/accuracies": 0.6990000009536743,
+      "eval_rewards/chosen": -1.5070966482162476,
+      "eval_rewards/margins": 0.5438539385795593,
+      "eval_rewards/rejected": -2.050950765609741,
+      "eval_runtime": 1348.4057,
+      "eval_samples_per_second": 1.483,
+      "eval_steps_per_second": 0.185,
+      "step": 1500
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.856606904623047e-06,
+      "logits/chosen": -0.556477963924408,
+      "logits/rejected": -0.4889760911464691,
+      "logps/chosen": -363.55963134765625,
+      "logps/rejected": -426.9551696777344,
+      "loss": 0.591,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.474310040473938,
+      "rewards/margins": 0.4846143126487732,
+      "rewards/rejected": -1.9589242935180664,
+      "step": 1510
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.852770077278557e-06,
+      "logits/chosen": -0.6768211126327515,
+      "logits/rejected": -0.3175960183143616,
+      "logps/chosen": -421.33154296875,
+      "logps/rejected": -458.45074462890625,
+      "loss": 0.4786,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.6068187952041626,
+      "rewards/margins": 0.7079833149909973,
+      "rewards/rejected": -2.314802408218384,
+      "step": 1520
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.848884150054196e-06,
+      "logits/chosen": -0.5115082263946533,
+      "logits/rejected": -0.3035307824611664,
+      "logps/chosen": -445.80255126953125,
+      "logps/rejected": -447.6587829589844,
+      "loss": 0.6743,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.983443021774292,
+      "rewards/margins": 0.3264579176902771,
+      "rewards/rejected": -2.309900999069214,
+      "step": 1530
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.8449492040452495e-06,
+      "logits/chosen": -0.7161798477172852,
+      "logits/rejected": -0.48103800415992737,
+      "logps/chosen": -450.58612060546875,
+      "logps/rejected": -464.6512145996094,
+      "loss": 0.5429,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.7402369976043701,
+      "rewards/margins": 0.6388343572616577,
+      "rewards/rejected": -2.3790714740753174,
+      "step": 1540
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.840965321369973e-06,
+      "logits/chosen": -0.5199601054191589,
+      "logits/rejected": -0.07315714657306671,
+      "logps/chosen": -428.27569580078125,
+      "logps/rejected": -446.408447265625,
+      "loss": 0.5816,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.614469289779663,
+      "rewards/margins": 0.5205715298652649,
+      "rewards/rejected": -2.135040760040283,
+      "step": 1550
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.8369325851678795e-06,
+      "logits/chosen": -0.5739278793334961,
+      "logits/rejected": -0.2974932789802551,
+      "logps/chosen": -418.03533935546875,
+      "logps/rejected": -462.3472595214844,
+      "loss": 0.5015,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.4371753931045532,
+      "rewards/margins": 0.7006227970123291,
+      "rewards/rejected": -2.1377980709075928,
+      "step": 1560
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.832851079598007e-06,
+      "logits/chosen": -0.6279390454292297,
+      "logits/rejected": -0.5385897755622864,
+      "logps/chosen": -373.42572021484375,
+      "logps/rejected": -448.22052001953125,
+      "loss": 0.5524,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.3597685098648071,
+      "rewards/margins": 0.6313223838806152,
+      "rewards/rejected": -1.991091012954712,
+      "step": 1570
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.828720889837158e-06,
+      "logits/chosen": -0.6760931611061096,
+      "logits/rejected": -0.5581658482551575,
+      "logps/chosen": -401.9905700683594,
+      "logps/rejected": -400.3769226074219,
+      "loss": 0.5935,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.3365936279296875,
+      "rewards/margins": 0.45313510298728943,
+      "rewards/rejected": -1.7897287607192993,
+      "step": 1580
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.824542102078125e-06,
+      "logits/chosen": -0.754819393157959,
+      "logits/rejected": -0.659611165523529,
+      "logps/chosen": -378.61077880859375,
+      "logps/rejected": -403.50921630859375,
+      "loss": 0.5855,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.2485653162002563,
+      "rewards/margins": 0.48787546157836914,
+      "rewards/rejected": -1.736440896987915,
+      "step": 1590
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.820314803527888e-06,
+      "logits/chosen": -0.7386206984519958,
+      "logits/rejected": -0.44416993856430054,
+      "logps/chosen": -372.7093200683594,
+      "logps/rejected": -436.66546630859375,
+      "loss": 0.4815,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.3660434484481812,
+      "rewards/margins": 0.7934683561325073,
+      "rewards/rejected": -2.1595120429992676,
+      "step": 1600
+    },
+    {
+      "epoch": 0.21,
+      "eval_logits/chosen": -0.581002414226532,
+      "eval_logits/rejected": -0.341525673866272,
+      "eval_logps/chosen": -421.0228271484375,
+      "eval_logps/rejected": -464.60430908203125,
+      "eval_loss": 0.5662662386894226,
+      "eval_rewards/accuracies": 0.703499972820282,
+      "eval_rewards/chosen": -1.5246328115463257,
+      "eval_rewards/margins": 0.6333407759666443,
+      "eval_rewards/rejected": -2.157973289489746,
+      "eval_runtime": 1347.3247,
+      "eval_samples_per_second": 1.484,
+      "eval_steps_per_second": 0.186,
+      "step": 1600
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.816039082405799e-06,
+      "logits/chosen": -0.6247519254684448,
+      "logits/rejected": -0.4443454146385193,
+      "logps/chosen": -402.0728759765625,
+      "logps/rejected": -463.2029724121094,
+      "loss": 0.5448,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.4785516262054443,
+      "rewards/margins": 0.6134171485900879,
+      "rewards/rejected": -2.0919687747955322,
+      "step": 1610
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.81171502794174e-06,
+      "logits/chosen": -0.6708969473838806,
+      "logits/rejected": -0.575433611869812,
+      "logps/chosen": -444.38653564453125,
+      "logps/rejected": -467.5558166503906,
+      "loss": 0.6019,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.4077513217926025,
+      "rewards/margins": 0.5450264811515808,
+      "rewards/rejected": -1.9527778625488281,
+      "step": 1620
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.8073427303742584e-06,
+      "logits/chosen": -0.818834125995636,
+      "logits/rejected": -0.5491704344749451,
+      "logps/chosen": -366.55096435546875,
+      "logps/rejected": -431.32696533203125,
+      "loss": 0.5472,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.370957374572754,
+      "rewards/margins": 0.7625428438186646,
+      "rewards/rejected": -2.133500337600708,
+      "step": 1630
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.802922280948685e-06,
+      "logits/chosen": -0.49336114525794983,
+      "logits/rejected": -0.47838038206100464,
+      "logps/chosen": -367.27301025390625,
+      "logps/rejected": -459.3961486816406,
+      "loss": 0.5981,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.4365627765655518,
+      "rewards/margins": 0.5842777490615845,
+      "rewards/rejected": -2.020840644836426,
+      "step": 1640
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.798453771915231e-06,
+      "logits/chosen": -0.6987757682800293,
+      "logits/rejected": -0.3112920820713043,
+      "logps/chosen": -380.4547119140625,
+      "logps/rejected": -433.07574462890625,
+      "loss": 0.5011,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.2082726955413818,
+      "rewards/margins": 0.7442606687545776,
+      "rewards/rejected": -1.9525333642959595,
+      "step": 1650
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.793937296527062e-06,
+      "logits/chosen": -0.7503664493560791,
+      "logits/rejected": -0.36433500051498413,
+      "logps/chosen": -444.39801025390625,
+      "logps/rejected": -505.1778869628906,
+      "loss": 0.5485,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.3104846477508545,
+      "rewards/margins": 0.7326589822769165,
+      "rewards/rejected": -2.0431437492370605,
+      "step": 1660
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.78937294903835e-06,
+      "logits/chosen": -0.45205217599868774,
+      "logits/rejected": -0.10327209532260895,
+      "logps/chosen": -363.23846435546875,
+      "logps/rejected": -417.77716064453125,
+      "loss": 0.5073,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.1292226314544678,
+      "rewards/margins": 0.7744971513748169,
+      "rewards/rejected": -1.9037199020385742,
+      "step": 1670
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.78476082470231e-06,
+      "logits/chosen": -0.3769649863243103,
+      "logits/rejected": 0.06695427745580673,
+      "logps/chosen": -383.61846923828125,
+      "logps/rejected": -437.2826232910156,
+      "loss": 0.5432,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.3478237390518188,
+      "rewards/margins": 0.75844407081604,
+      "rewards/rejected": -2.1062679290771484,
+      "step": 1680
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.780101019769212e-06,
+      "logits/chosen": -0.19569449126720428,
+      "logits/rejected": 0.16784332692623138,
+      "logps/chosen": -473.6358947753906,
+      "logps/rejected": -497.500732421875,
+      "loss": 0.5732,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.8416274785995483,
+      "rewards/margins": 0.6301079988479614,
+      "rewards/rejected": -2.4717354774475098,
+      "step": 1690
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.775393631484368e-06,
+      "logits/chosen": 0.04904654622077942,
+      "logits/rejected": 0.33445030450820923,
+      "logps/chosen": -456.16741943359375,
+      "logps/rejected": -534.6590576171875,
+      "loss": 0.4698,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.711970567703247,
+      "rewards/margins": 0.9142668843269348,
+      "rewards/rejected": -2.626237392425537,
+      "step": 1700
+    },
+    {
+      "epoch": 0.22,
+      "eval_logits/chosen": 0.2594228982925415,
+      "eval_logits/rejected": 0.5641188621520996,
+      "eval_logps/chosen": -450.21026611328125,
+      "eval_logps/rejected": -498.6651916503906,
+      "eval_loss": 0.5697379112243652,
+      "eval_rewards/accuracies": 0.6990000009536743,
+      "eval_rewards/chosen": -1.81650710105896,
+      "eval_rewards/margins": 0.6820747256278992,
+      "eval_rewards/rejected": -2.498581886291504,
+      "eval_runtime": 1348.5773,
+      "eval_samples_per_second": 1.483,
+      "eval_steps_per_second": 0.185,
+      "step": 1700
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.770638758086105e-06,
+      "logits/chosen": 0.18382230401039124,
+      "logits/rejected": 0.507646381855011,
+      "logps/chosen": -442.58319091796875,
+      "logps/rejected": -470.44580078125,
+      "loss": 0.6142,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.754159688949585,
+      "rewards/margins": 0.5679591298103333,
+      "rewards/rejected": -2.3221187591552734,
+      "step": 1710
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.7658364988037184e-06,
+      "logits/chosen": -0.07166196405887604,
+      "logits/rejected": 0.15328675508499146,
+      "logps/chosen": -445.88494873046875,
+      "logps/rejected": -486.57025146484375,
+      "loss": 0.5461,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.54258131980896,
+      "rewards/margins": 0.6698669791221619,
+      "rewards/rejected": -2.2124481201171875,
+      "step": 1720
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.760986953855395e-06,
+      "logits/chosen": -0.2639234960079193,
+      "logits/rejected": -0.1371985226869583,
+      "logps/chosen": -405.93115234375,
+      "logps/rejected": -479.60321044921875,
+      "loss": 0.5395,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.5491055250167847,
+      "rewards/margins": 0.6250948905944824,
+      "rewards/rejected": -2.1742005348205566,
+      "step": 1730
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.756090224446127e-06,
+      "logits/chosen": -0.3448755741119385,
+      "logits/rejected": 0.07972880452871323,
+      "logps/chosen": -449.4403381347656,
+      "logps/rejected": -480.0484313964844,
+      "loss": 0.5777,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.4526723623275757,
+      "rewards/margins": 0.592454731464386,
+      "rewards/rejected": -2.0451271533966064,
+      "step": 1740
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.7511464127655945e-06,
+      "logits/chosen": -0.18566124141216278,
+      "logits/rejected": 0.47423624992370605,
+      "logps/chosen": -420.04730224609375,
+      "logps/rejected": -476.8564453125,
+      "loss": 0.4909,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.5504100322723389,
+      "rewards/margins": 0.8979840278625488,
+      "rewards/rejected": -2.4483940601348877,
+      "step": 1750
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.74615562198604e-06,
+      "logits/chosen": -0.2191568911075592,
+      "logits/rejected": 0.2801164984703064,
+      "logps/chosen": -391.5447998046875,
+      "logps/rejected": -462.55450439453125,
+      "loss": 0.4937,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4210954904556274,
+      "rewards/margins": 1.0278735160827637,
+      "rewards/rejected": -2.4489688873291016,
+      "step": 1760
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.741117956260107e-06,
+      "logits/chosen": -0.09145449846982956,
+      "logits/rejected": 0.14795629680156708,
+      "logps/chosen": -376.05340576171875,
+      "logps/rejected": -454.7176208496094,
+      "loss": 0.5262,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.1599403619766235,
+      "rewards/margins": 0.8291015625,
+      "rewards/rejected": -1.9890419244766235,
+      "step": 1770
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.736033520718672e-06,
+      "logits/chosen": -0.5329246520996094,
+      "logits/rejected": 0.04998129978775978,
+      "logps/chosen": -402.61187744140625,
+      "logps/rejected": -450.05133056640625,
+      "loss": 0.5402,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.0866920948028564,
+      "rewards/margins": 0.7409444451332092,
+      "rewards/rejected": -1.827636480331421,
+      "step": 1780
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.730902421468652e-06,
+      "logits/chosen": -0.28307825326919556,
+      "logits/rejected": 0.11154267936944962,
+      "logps/chosen": -366.70684814453125,
+      "logps/rejected": -447.93182373046875,
+      "loss": 0.5422,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.2073009014129639,
+      "rewards/margins": 0.752407431602478,
+      "rewards/rejected": -1.9597084522247314,
+      "step": 1790
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.7257247655907854e-06,
+      "logits/chosen": 0.30156201124191284,
+      "logits/rejected": 0.42981958389282227,
+      "logps/chosen": -351.79229736328125,
+      "logps/rejected": -450.8818359375,
+      "loss": 0.5213,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.284209966659546,
+      "rewards/margins": 0.841025710105896,
+      "rewards/rejected": -2.1252357959747314,
+      "step": 1800
+    },
+    {
+      "epoch": 0.24,
+      "eval_logits/chosen": 0.29522255063056946,
+      "eval_logits/rejected": 0.6172813773155212,
+      "eval_logps/chosen": -410.91522216796875,
+      "eval_logps/rejected": -458.92144775390625,
+      "eval_loss": 0.5669854879379272,
+      "eval_rewards/accuracies": 0.7055000066757202,
+      "eval_rewards/chosen": -1.4235563278198242,
+      "eval_rewards/margins": 0.6775884628295898,
+      "eval_rewards/rejected": -2.101144790649414,
+      "eval_runtime": 1348.013,
+      "eval_samples_per_second": 1.484,
+      "eval_steps_per_second": 0.185,
+      "step": 1800
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.720500661137397e-06,
+      "logits/chosen": -0.024377573281526566,
+      "logits/rejected": 0.19398494064807892,
+      "logps/chosen": -397.6201171875,
+      "logps/rejected": -452.4983825683594,
+      "loss": 0.6507,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.4880634546279907,
+      "rewards/margins": 0.5089324116706848,
+      "rewards/rejected": -1.9969959259033203,
+      "step": 1810
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.71523021713015e-06,
+      "logits/chosen": -0.5791071653366089,
+      "logits/rejected": -0.443958044052124,
+      "logps/chosen": -470.78863525390625,
+      "logps/rejected": -513.630126953125,
+      "loss": 0.6143,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.3055927753448486,
+      "rewards/margins": 0.47979336977005005,
+      "rewards/rejected": -1.785386085510254,
+      "step": 1820
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.709913543557761e-06,
+      "logits/chosen": -0.3587265610694885,
+      "logits/rejected": -0.10842130333185196,
+      "logps/chosen": -440.23345947265625,
+      "logps/rejected": -488.025146484375,
+      "loss": 0.5245,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.411224126815796,
+      "rewards/margins": 0.6483907699584961,
+      "rewards/rejected": -2.059614896774292,
+      "step": 1830
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.704550751373715e-06,
+      "logits/chosen": -0.027194689959287643,
+      "logits/rejected": 0.4479443430900574,
+      "logps/chosen": -501.24212646484375,
+      "logps/rejected": -536.3165283203125,
+      "loss": 0.4708,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.5418251752853394,
+      "rewards/margins": 0.9211807250976562,
+      "rewards/rejected": -2.463005781173706,
+      "step": 1840
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.699141952493941e-06,
+      "logits/chosen": 0.4470292031764984,
+      "logits/rejected": 0.7221449613571167,
+      "logps/chosen": -469.3924865722656,
+      "logps/rejected": -532.7666625976562,
+      "loss": 0.6152,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.1157889366149902,
+      "rewards/margins": 0.6170716285705566,
+      "rewards/rejected": -2.732860565185547,
+      "step": 1850
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.6936872597944814e-06,
+      "logits/chosen": 0.10170433670282364,
+      "logits/rejected": 0.870053768157959,
+      "logps/chosen": -486.12408447265625,
+      "logps/rejected": -509.8555603027344,
+      "loss": 0.4243,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.8883590698242188,
+      "rewards/margins": 1.0139433145523071,
+      "rewards/rejected": -2.9023022651672363,
+      "step": 1860
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.688186787109136e-06,
+      "logits/chosen": 0.09663492441177368,
+      "logits/rejected": 0.32683372497558594,
+      "logps/chosen": -479.57879638671875,
+      "logps/rejected": -554.6630859375,
+      "loss": 0.5406,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.8735917806625366,
+      "rewards/margins": 0.8642719984054565,
+      "rewards/rejected": -2.737863779067993,
+      "step": 1870
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.682640649227085e-06,
+      "logits/chosen": 0.12810960412025452,
+      "logits/rejected": 0.4416731297969818,
+      "logps/chosen": -439.32135009765625,
+      "logps/rejected": -492.44219970703125,
+      "loss": 0.4771,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.7408326864242554,
+      "rewards/margins": 0.8746045231819153,
+      "rewards/rejected": -2.6154370307922363,
+      "step": 1880
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.677048961890492e-06,
+      "logits/chosen": 0.49400216341018677,
+      "logits/rejected": 0.8036004900932312,
+      "logps/chosen": -444.3980407714844,
+      "logps/rejected": -459.29498291015625,
+      "loss": 0.643,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7791763544082642,
+      "rewards/margins": 0.5157355666160583,
+      "rewards/rejected": -2.294912099838257,
+      "step": 1890
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.671411841792096e-06,
+      "logits/chosen": 0.44327154755592346,
+      "logits/rejected": 0.5383572578430176,
+      "logps/chosen": -481.62762451171875,
+      "logps/rejected": -582.154541015625,
+      "loss": 0.5295,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.9470043182373047,
+      "rewards/margins": 0.9709717631340027,
+      "rewards/rejected": -2.9179763793945312,
+      "step": 1900
+    },
+    {
+      "epoch": 0.25,
+      "eval_logits/chosen": 0.5819193720817566,
+      "eval_logits/rejected": 0.8940628170967102,
+      "eval_logps/chosen": -466.5294494628906,
+      "eval_logps/rejected": -518.3280029296875,
+      "eval_loss": 0.5606327652931213,
+      "eval_rewards/accuracies": 0.6945000290870667,
+      "eval_rewards/chosen": -1.9796984195709229,
+      "eval_rewards/margins": 0.7155116200447083,
+      "eval_rewards/rejected": -2.6952099800109863,
+      "eval_runtime": 1350.3842,
+      "eval_samples_per_second": 1.481,
+      "eval_steps_per_second": 0.185,
+      "step": 1900
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.665729406572764e-06,
+      "logits/chosen": 0.622109591960907,
+      "logits/rejected": 0.7973288893699646,
+      "logps/chosen": -456.37548828125,
+      "logps/rejected": -506.8421936035156,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.2507548332214355,
+      "rewards/margins": 0.5436421036720276,
+      "rewards/rejected": -2.7943971157073975,
+      "step": 1910
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.660001774819048e-06,
+      "logits/chosen": -0.023168373852968216,
+      "logits/rejected": 0.45402470231056213,
+      "logps/chosen": -525.577880859375,
+      "logps/rejected": -518.7696533203125,
+      "loss": 0.6013,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.235229015350342,
+      "rewards/margins": 0.5837393403053284,
+      "rewards/rejected": -2.8189685344696045,
+      "step": 1920
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.654229066060702e-06,
+      "logits/chosen": 0.17349359393119812,
+      "logits/rejected": 0.6238114237785339,
+      "logps/chosen": -468.94122314453125,
+      "logps/rejected": -476.7464294433594,
+      "loss": 0.5894,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.0981879234313965,
+      "rewards/margins": 0.459352970123291,
+      "rewards/rejected": -2.5575408935546875,
+      "step": 1930
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.648411400768193e-06,
+      "logits/chosen": -0.03141383081674576,
+      "logits/rejected": 0.6579645872116089,
+      "logps/chosen": -451.70928955078125,
+      "logps/rejected": -445.5111389160156,
+      "loss": 0.518,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.7957528829574585,
+      "rewards/margins": 0.6112611293792725,
+      "rewards/rejected": -2.4070143699645996,
+      "step": 1940
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.642548900350182e-06,
+      "logits/chosen": 0.3349805772304535,
+      "logits/rejected": 0.40201663970947266,
+      "logps/chosen": -392.16876220703125,
+      "logps/rejected": -450.25689697265625,
+      "loss": 0.5105,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.6065629720687866,
+      "rewards/margins": 0.6597822904586792,
+      "rewards/rejected": -2.266345262527466,
+      "step": 1950
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.636641687150994e-06,
+      "logits/chosen": 0.020562540739774704,
+      "logits/rejected": 0.1890704184770584,
+      "logps/chosen": -424.3070373535156,
+      "logps/rejected": -480.12548828125,
+      "loss": 0.5026,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.4197609424591064,
+      "rewards/margins": 0.7624276280403137,
+      "rewards/rejected": -2.1821885108947754,
+      "step": 1960
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.6306898844480615e-06,
+      "logits/chosen": 0.21109429001808167,
+      "logits/rejected": 0.4792759418487549,
+      "logps/chosen": -425.04412841796875,
+      "logps/rejected": -442.88836669921875,
+      "loss": 0.5706,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.3820161819458008,
+      "rewards/margins": 0.4862526059150696,
+      "rewards/rejected": -1.8682689666748047,
+      "step": 1970
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.624693616449358e-06,
+      "logits/chosen": -0.05729494243860245,
+      "logits/rejected": 0.07928317785263062,
+      "logps/chosen": -350.982666015625,
+      "logps/rejected": -388.08892822265625,
+      "loss": 0.6067,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.2988847494125366,
+      "rewards/margins": 0.475615918636322,
+      "rewards/rejected": -1.774500846862793,
+      "step": 1980
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.6186530082908e-06,
+      "logits/chosen": -0.20739535987377167,
+      "logits/rejected": 0.06369415670633316,
+      "logps/chosen": -432.6578063964844,
+      "logps/rejected": -471.60638427734375,
+      "loss": 0.5956,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.3366018533706665,
+      "rewards/margins": 0.5623012781143188,
+      "rewards/rejected": -1.8989031314849854,
+      "step": 1990
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.612568186033633e-06,
+      "logits/chosen": -0.36816614866256714,
+      "logits/rejected": -0.16022469103336334,
+      "logps/chosen": -387.3262634277344,
+      "logps/rejected": -443.4298400878906,
+      "loss": 0.6074,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.4646406173706055,
+      "rewards/margins": 0.47723302245140076,
+      "rewards/rejected": -1.94187331199646,
+      "step": 2000
+    },
+    {
+      "epoch": 0.26,
+      "eval_logits/chosen": 0.027064476162195206,
+      "eval_logits/rejected": 0.34493085741996765,
+      "eval_logps/chosen": -387.03961181640625,
+      "eval_logps/rejected": -427.6169738769531,
+      "eval_loss": 0.5524502992630005,
+      "eval_rewards/accuracies": 0.7164999842643738,
+      "eval_rewards/chosen": -1.1848002672195435,
+      "eval_rewards/margins": 0.603299617767334,
+      "eval_rewards/rejected": -1.7881001234054565,
+      "eval_runtime": 1351.0677,
+      "eval_samples_per_second": 1.48,
+      "eval_steps_per_second": 0.185,
+      "step": 2000
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.6064392766618125e-06,
+      "logits/chosen": 0.10670644044876099,
+      "logits/rejected": 0.4181477129459381,
+      "logps/chosen": -405.8658447265625,
+      "logps/rejected": -438.45452880859375,
+      "loss": 0.6023,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.2449086904525757,
+      "rewards/margins": 0.5699526071548462,
+      "rewards/rejected": -1.814861536026001,
+      "step": 2010
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.60026640807934e-06,
+      "logits/chosen": 0.02254806086421013,
+      "logits/rejected": 0.49392470717430115,
+      "logps/chosen": -386.99285888671875,
+      "logps/rejected": -417.5672302246094,
+      "loss": 0.4989,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.0919376611709595,
+      "rewards/margins": 0.6963882446289062,
+      "rewards/rejected": -1.7883260250091553,
+      "step": 2020
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.594049709107604e-06,
+      "logits/chosen": 0.3108864724636078,
+      "logits/rejected": 1.0099918842315674,
+      "logps/chosen": -408.521240234375,
+      "logps/rejected": -457.60809326171875,
+      "loss": 0.5026,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.3481906652450562,
+      "rewards/margins": 0.8482155799865723,
+      "rewards/rejected": -2.196406126022339,
+      "step": 2030
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.587789309482687e-06,
+      "logits/chosen": 1.0027506351470947,
+      "logits/rejected": 1.1039046049118042,
+      "logps/chosen": -374.0387878417969,
+      "logps/rejected": -481.02593994140625,
+      "loss": 0.5294,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.366990089416504,
+      "rewards/margins": 0.7354560494422913,
+      "rewards/rejected": -2.1024460792541504,
+      "step": 2040
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.581485339852659e-06,
+      "logits/chosen": 0.7044609189033508,
+      "logits/rejected": 1.1121299266815186,
+      "logps/chosen": -443.6524963378906,
+      "logps/rejected": -456.810302734375,
+      "loss": 0.6068,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.4939275979995728,
+      "rewards/margins": 0.5384919047355652,
+      "rewards/rejected": -2.0324196815490723,
+      "step": 2050
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.5751379317748514e-06,
+      "logits/chosen": 0.6495908498764038,
+      "logits/rejected": 0.8748601675033569,
+      "logps/chosen": -391.97509765625,
+      "logps/rejected": -433.9190979003906,
+      "loss": 0.5737,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.3713769912719727,
+      "rewards/margins": 0.48151645064353943,
+      "rewards/rejected": -1.8528934717178345,
+      "step": 2060
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.56874721771311e-06,
+      "logits/chosen": 0.2890474200248718,
+      "logits/rejected": 0.6486330032348633,
+      "logps/chosen": -409.802490234375,
+      "logps/rejected": -427.52252197265625,
+      "loss": 0.5732,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.4637774229049683,
+      "rewards/margins": 0.5432168245315552,
+      "rewards/rejected": -2.0069940090179443,
+      "step": 2070
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.562313331035032e-06,
+      "logits/chosen": 0.9071200489997864,
+      "logits/rejected": 1.1255214214324951,
+      "logps/chosen": -390.31298828125,
+      "logps/rejected": -489.7353515625,
+      "loss": 0.5158,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5372436046600342,
+      "rewards/margins": 0.7855271100997925,
+      "rewards/rejected": -2.322770833969116,
+      "step": 2080
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.555836406009183e-06,
+      "logits/chosen": 1.0377204418182373,
+      "logits/rejected": 1.2941482067108154,
+      "logps/chosen": -418.8136291503906,
+      "logps/rejected": -467.39349365234375,
+      "loss": 0.5568,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.8789348602294922,
+      "rewards/margins": 0.6682177186012268,
+      "rewards/rejected": -2.547152519226074,
+      "step": 2090
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.5493165778022945e-06,
+      "logits/chosen": 0.8948332071304321,
+      "logits/rejected": 1.2317169904708862,
+      "logps/chosen": -404.481689453125,
+      "logps/rejected": -428.8797302246094,
+      "loss": 0.568,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.6655458211898804,
+      "rewards/margins": 0.5730478167533875,
+      "rewards/rejected": -2.238593578338623,
+      "step": 2100
+    },
+    {
+      "epoch": 0.27,
+      "eval_logits/chosen": 0.9785990118980408,
+      "eval_logits/rejected": 1.349733829498291,
+      "eval_logps/chosen": -425.226318359375,
+      "eval_logps/rejected": -473.691162109375,
+      "eval_loss": 0.538813591003418,
+      "eval_rewards/accuracies": 0.722000002861023,
+      "eval_rewards/chosen": -1.566667079925537,
+      "eval_rewards/margins": 0.682174801826477,
+      "eval_rewards/rejected": -2.2488420009613037,
+      "eval_runtime": 1350.4269,
+      "eval_samples_per_second": 1.481,
+      "eval_steps_per_second": 0.185,
+      "step": 2100
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.542753982476443e-06,
+      "logits/chosen": 0.5831831097602844,
+      "logits/rejected": 1.2383759021759033,
+      "logps/chosen": -441.15594482421875,
+      "logps/rejected": -467.72979736328125,
+      "loss": 0.5461,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4610209465026855,
+      "rewards/margins": 0.6836363673210144,
+      "rewards/rejected": -2.1446573734283447,
+      "step": 2110
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.53614875698621e-06,
+      "logits/chosen": 0.45439672470092773,
+      "logits/rejected": 0.8214502334594727,
+      "logps/chosen": -459.13238525390625,
+      "logps/rejected": -515.7960205078125,
+      "loss": 0.5203,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.5529654026031494,
+      "rewards/margins": 0.6917394995689392,
+      "rewards/rejected": -2.2447047233581543,
+      "step": 2120
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.529501039175824e-06,
+      "logits/chosen": 1.047241449356079,
+      "logits/rejected": 1.3419172763824463,
+      "logps/chosen": -407.1217346191406,
+      "logps/rejected": -463.77606201171875,
+      "loss": 0.6148,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.838382363319397,
+      "rewards/margins": 0.6213015913963318,
+      "rewards/rejected": -2.459683656692505,
+      "step": 2130
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.522810967776287e-06,
+      "logits/chosen": 1.1739928722381592,
+      "logits/rejected": 1.8392829895019531,
+      "logps/chosen": -464.14056396484375,
+      "logps/rejected": -472.36053466796875,
+      "loss": 0.4707,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.8977082967758179,
+      "rewards/margins": 0.7907202839851379,
+      "rewards/rejected": -2.6884286403656006,
+      "step": 2140
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.516078682402473e-06,
+      "logits/chosen": 0.9727063179016113,
+      "logits/rejected": 1.5400081872940063,
+      "logps/chosen": -471.14227294921875,
+      "logps/rejected": -526.6529541015625,
+      "loss": 0.5428,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.9313971996307373,
+      "rewards/margins": 0.7983649373054504,
+      "rewards/rejected": -2.729762315750122,
+      "step": 2150
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.509304323550221e-06,
+      "logits/chosen": 0.9346233606338501,
+      "logits/rejected": 1.2992526292800903,
+      "logps/chosen": -470.99407958984375,
+      "logps/rejected": -533.6473999023438,
+      "loss": 0.5223,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6646778583526611,
+      "rewards/margins": 0.8623111844062805,
+      "rewards/rejected": -2.526988983154297,
+      "step": 2160
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.502488032593398e-06,
+      "logits/chosen": 0.6454982757568359,
+      "logits/rejected": 1.2728111743927002,
+      "logps/chosen": -443.3365173339844,
+      "logps/rejected": -487.98699951171875,
+      "loss": 0.5189,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.6750648021697998,
+      "rewards/margins": 0.7639718055725098,
+      "rewards/rejected": -2.4390366077423096,
+      "step": 2170
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.495629951780951e-06,
+      "logits/chosen": 0.5267282724380493,
+      "logits/rejected": 1.3138864040374756,
+      "logps/chosen": -476.30877685546875,
+      "logps/rejected": -476.84173583984375,
+      "loss": 0.4679,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.3605176210403442,
+      "rewards/margins": 0.9084545969963074,
+      "rewards/rejected": -2.268972158432007,
+      "step": 2180
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.488730224233941e-06,
+      "logits/chosen": 1.021937608718872,
+      "logits/rejected": 1.2827173471450806,
+      "logps/chosen": -377.60394287109375,
+      "logps/rejected": -490.66986083984375,
+      "loss": 0.4319,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.5510553121566772,
+      "rewards/margins": 0.978097140789032,
+      "rewards/rejected": -2.5291523933410645,
+      "step": 2190
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.481788993942547e-06,
+      "logits/chosen": 1.2917654514312744,
+      "logits/rejected": 1.8904330730438232,
+      "logps/chosen": -464.09832763671875,
+      "logps/rejected": -501.93731689453125,
+      "loss": 0.5643,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.9287099838256836,
+      "rewards/margins": 0.7453629970550537,
+      "rewards/rejected": -2.6740729808807373,
+      "step": 2200
+    },
+    {
+      "epoch": 0.29,
+      "eval_logits/chosen": 1.556535243988037,
+      "eval_logits/rejected": 1.9603238105773926,
+      "eval_logps/chosen": -449.6773681640625,
+      "eval_logps/rejected": -510.6461181640625,
+      "eval_loss": 0.5538718104362488,
+      "eval_rewards/accuracies": 0.7145000100135803,
+      "eval_rewards/chosen": -1.811178207397461,
+      "eval_rewards/margins": 0.8072137236595154,
+      "eval_rewards/rejected": -2.618391752243042,
+      "eval_runtime": 1347.2905,
+      "eval_samples_per_second": 1.484,
+      "eval_steps_per_second": 0.186,
+      "step": 2200
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.474806405763076e-06,
+      "logits/chosen": 0.9370642900466919,
+      "logits/rejected": 1.1973974704742432,
+      "logps/chosen": -436.38726806640625,
+      "logps/rejected": -491.88079833984375,
+      "loss": 0.5655,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.6625728607177734,
+      "rewards/margins": 0.7996843457221985,
+      "rewards/rejected": -2.462257146835327,
+      "step": 2210
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.4677826054149235e-06,
+      "logits/chosen": 0.7612293362617493,
+      "logits/rejected": 1.112367868423462,
+      "logps/chosen": -439.87481689453125,
+      "logps/rejected": -478.6990661621094,
+      "loss": 0.5757,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.3228178024291992,
+      "rewards/margins": 0.6328498721122742,
+      "rewards/rejected": -1.9556677341461182,
+      "step": 2220
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.460717739477543e-06,
+      "logits/chosen": 0.5900286436080933,
+      "logits/rejected": 0.6329357624053955,
+      "logps/chosen": -376.19091796875,
+      "logps/rejected": -467.05645751953125,
+      "loss": 0.59,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.2272824048995972,
+      "rewards/margins": 0.5313378572463989,
+      "rewards/rejected": -1.758620262145996,
+      "step": 2230
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.4536119553873866e-06,
+      "logits/chosen": 0.8423527479171753,
+      "logits/rejected": 0.9933163523674011,
+      "logps/chosen": -370.025634765625,
+      "logps/rejected": -456.2650451660156,
+      "loss": 0.4239,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.151697039604187,
+      "rewards/margins": 0.92719966173172,
+      "rewards/rejected": -2.0788967609405518,
+      "step": 2240
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.446465401434824e-06,
+      "logits/chosen": 0.8776890635490417,
+      "logits/rejected": 1.1721899509429932,
+      "logps/chosen": -415.385986328125,
+      "logps/rejected": -533.56640625,
+      "loss": 0.5433,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.8485475778579712,
+      "rewards/margins": 0.8066433668136597,
+      "rewards/rejected": -2.6551907062530518,
+      "step": 2250
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.43927822676105e-06,
+      "logits/chosen": 1.0181093215942383,
+      "logits/rejected": 1.7029005289077759,
+      "logps/chosen": -496.4100036621094,
+      "logps/rejected": -538.6514282226562,
+      "loss": 0.5184,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.101989269256592,
+      "rewards/margins": 0.7876111268997192,
+      "rewards/rejected": -2.8896005153656006,
+      "step": 2260
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.432050581354972e-06,
+      "logits/chosen": 0.9238463640213013,
+      "logits/rejected": 1.4000639915466309,
+      "logps/chosen": -440.0384826660156,
+      "logps/rejected": -490.71044921875,
+      "loss": 0.6116,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.7824652194976807,
+      "rewards/margins": 0.6415243744850159,
+      "rewards/rejected": -2.423989772796631,
+      "step": 2270
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.424782616050078e-06,
+      "logits/chosen": 0.9443693161010742,
+      "logits/rejected": 1.2565793991088867,
+      "logps/chosen": -415.452880859375,
+      "logps/rejected": -460.87261962890625,
+      "loss": 0.5797,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.5016992092132568,
+      "rewards/margins": 0.6675459146499634,
+      "rewards/rejected": -2.1692450046539307,
+      "step": 2280
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.4174744825212954e-06,
+      "logits/chosen": 0.8250998258590698,
+      "logits/rejected": 0.8220119476318359,
+      "logps/chosen": -389.514892578125,
+      "logps/rejected": -449.9290466308594,
+      "loss": 0.706,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.4745213985443115,
+      "rewards/margins": 0.4119679927825928,
+      "rewards/rejected": -1.8864895105361938,
+      "step": 2290
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.410126333281815e-06,
+      "logits/chosen": 0.8298419713973999,
+      "logits/rejected": 1.080414056777954,
+      "logps/chosen": -467.6815490722656,
+      "logps/rejected": -554.405029296875,
+      "loss": 0.5226,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.5661046504974365,
+      "rewards/margins": 0.8343645334243774,
+      "rewards/rejected": -2.4004693031311035,
+      "step": 2300
+    },
+    {
+      "epoch": 0.3,
+      "eval_logits/chosen": 0.9660720229148865,
+      "eval_logits/rejected": 1.3672971725463867,
+      "eval_logps/chosen": -428.75531005859375,
+      "eval_logps/rejected": -484.68389892578125,
+      "eval_loss": 0.5353882312774658,
+      "eval_rewards/accuracies": 0.7245000004768372,
+      "eval_rewards/chosen": -1.6019575595855713,
+      "eval_rewards/margins": 0.7568122744560242,
+      "eval_rewards/rejected": -2.3587698936462402,
+      "eval_runtime": 1347.7361,
+      "eval_samples_per_second": 1.484,
+      "eval_steps_per_second": 0.185,
+      "step": 2300
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.402738321679918e-06,
+      "logits/chosen": 0.7783528566360474,
+      "logits/rejected": 1.4660738706588745,
+      "logps/chosen": -442.6292419433594,
+      "logps/rejected": -442.45733642578125,
+      "loss": 0.5536,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5938727855682373,
+      "rewards/margins": 0.6696248054504395,
+      "rewards/rejected": -2.263497829437256,
+      "step": 2310
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.395310601895772e-06,
+      "logits/chosen": 0.5439037084579468,
+      "logits/rejected": 0.8595826029777527,
+      "logps/chosen": -430.41851806640625,
+      "logps/rejected": -504.6600646972656,
+      "loss": 0.5026,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.5620139837265015,
+      "rewards/margins": 0.8716133832931519,
+      "rewards/rejected": -2.4336273670196533,
+      "step": 2320
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.38784332893821e-06,
+      "logits/chosen": 0.623258113861084,
+      "logits/rejected": 0.8479729890823364,
+      "logps/chosen": -332.442138671875,
+      "logps/rejected": -431.9659729003906,
+      "loss": 0.4765,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.3472410440444946,
+      "rewards/margins": 0.9654043316841125,
+      "rewards/rejected": -2.312645435333252,
+      "step": 2330
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.380336658641503e-06,
+      "logits/chosen": 0.3584001660346985,
+      "logits/rejected": 1.3645209074020386,
+      "logps/chosen": -431.05682373046875,
+      "logps/rejected": -475.0547790527344,
+      "loss": 0.4756,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.5529234409332275,
+      "rewards/margins": 0.8567450642585754,
+      "rewards/rejected": -2.409668445587158,
+      "step": 2340
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.372790747662101e-06,
+      "logits/chosen": 0.9099920392036438,
+      "logits/rejected": 1.1361488103866577,
+      "logps/chosen": -442.63818359375,
+      "logps/rejected": -501.41015625,
+      "loss": 0.527,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.8081750869750977,
+      "rewards/margins": 0.7763790488243103,
+      "rewards/rejected": -2.5845541954040527,
+      "step": 2350
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.365205753475367e-06,
+      "logits/chosen": 0.7123975157737732,
+      "logits/rejected": 1.5183513164520264,
+      "logps/chosen": -496.0164489746094,
+      "logps/rejected": -513.8137817382812,
+      "loss": 0.4981,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.6035009622573853,
+      "rewards/margins": 0.93915194272995,
+      "rewards/rejected": -2.5426530838012695,
+      "step": 2360
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.35758183437229e-06,
+      "logits/chosen": 0.4071558117866516,
+      "logits/rejected": 0.8936365246772766,
+      "logps/chosen": -481.523193359375,
+      "logps/rejected": -545.0813598632812,
+      "loss": 0.5409,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.562885046005249,
+      "rewards/margins": 0.7595075368881226,
+      "rewards/rejected": -2.322392702102661,
+      "step": 2370
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.3499191494561835e-06,
+      "logits/chosen": 1.1250611543655396,
+      "logits/rejected": 1.3284550905227661,
+      "logps/chosen": -446.35028076171875,
+      "logps/rejected": -483.1497497558594,
+      "loss": 0.5545,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7999433279037476,
+      "rewards/margins": 0.7792242169380188,
+      "rewards/rejected": -2.5791678428649902,
+      "step": 2380
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.3422178586393615e-06,
+      "logits/chosen": 0.4511619210243225,
+      "logits/rejected": 1.0610133409500122,
+      "logps/chosen": -515.4139404296875,
+      "logps/rejected": -549.369140625,
+      "loss": 0.48,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.011679172515869,
+      "rewards/margins": 0.9340222477912903,
+      "rewards/rejected": -2.9457013607025146,
+      "step": 2390
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.334478122639804e-06,
+      "logits/chosen": 0.6562846302986145,
+      "logits/rejected": 1.704473853111267,
+      "logps/chosen": -452.92486572265625,
+      "logps/rejected": -500.90948486328125,
+      "loss": 0.4144,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.026827573776245,
+      "rewards/margins": 1.2147269248962402,
+      "rewards/rejected": -3.2415542602539062,
+      "step": 2400
+    },
+    {
+      "epoch": 0.31,
+      "eval_logits/chosen": 1.254248023033142,
+      "eval_logits/rejected": 1.6609251499176025,
+      "eval_logps/chosen": -469.65570068359375,
+      "eval_logps/rejected": -531.568115234375,
+      "eval_loss": 0.5338253378868103,
+      "eval_rewards/accuracies": 0.7245000004768372,
+      "eval_rewards/chosen": -2.010960578918457,
+      "eval_rewards/margins": 0.816650390625,
+      "eval_rewards/rejected": -2.827610969543457,
+      "eval_runtime": 1346.1313,
+      "eval_samples_per_second": 1.486,
+      "eval_steps_per_second": 0.186,
+      "step": 2400
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.3267001029778015e-06,
+      "logits/chosen": 0.7201460599899292,
+      "logits/rejected": 1.3868685960769653,
+      "logps/chosen": -483.8265075683594,
+      "logps/rejected": -470.66387939453125,
+      "loss": 0.6302,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.9128888845443726,
+      "rewards/margins": 0.5651463866233826,
+      "rewards/rejected": -2.4780352115631104,
+      "step": 2410
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.318883961972585e-06,
+      "logits/chosen": 0.8093687295913696,
+      "logits/rejected": 0.856730580329895,
+      "logps/chosen": -432.450927734375,
+      "logps/rejected": -478.2227478027344,
+      "loss": 0.5893,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7508265972137451,
+      "rewards/margins": 0.6114845275878906,
+      "rewards/rejected": -2.3623111248016357,
+      "step": 2420
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.311029862738942e-06,
+      "logits/chosen": 0.47912222146987915,
+      "logits/rejected": 1.0415494441986084,
+      "logps/chosen": -436.6087951660156,
+      "logps/rejected": -461.66229248046875,
+      "loss": 0.5562,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6141655445098877,
+      "rewards/margins": 0.545906126499176,
+      "rewards/rejected": -2.160071611404419,
+      "step": 2430
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.303137969183804e-06,
+      "logits/chosen": 0.5455479621887207,
+      "logits/rejected": 0.9843274354934692,
+      "logps/chosen": -407.2878723144531,
+      "logps/rejected": -468.61798095703125,
+      "loss": 0.4345,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.501356840133667,
+      "rewards/margins": 0.8992450833320618,
+      "rewards/rejected": -2.400601863861084,
+      "step": 2440
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.295208446002832e-06,
+      "logits/chosen": 0.8205218315124512,
+      "logits/rejected": 1.3320242166519165,
+      "logps/chosen": -388.4240417480469,
+      "logps/rejected": -440.4716796875,
+      "loss": 0.5557,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6460374593734741,
+      "rewards/margins": 0.7431990504264832,
+      "rewards/rejected": -2.3892364501953125,
+      "step": 2450
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.287241458676981e-06,
+      "logits/chosen": 1.020278811454773,
+      "logits/rejected": 1.2911311388015747,
+      "logps/chosen": -480.62335205078125,
+      "logps/rejected": -542.1524047851562,
+      "loss": 0.4606,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.669176697731018,
+      "rewards/margins": 1.0525093078613281,
+      "rewards/rejected": -2.7216861248016357,
+      "step": 2460
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.279237173469043e-06,
+      "logits/chosen": 0.7975326776504517,
+      "logits/rejected": 1.5225236415863037,
+      "logps/chosen": -440.745849609375,
+      "logps/rejected": -494.22802734375,
+      "loss": 0.5043,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7947410345077515,
+      "rewards/margins": 0.8784025311470032,
+      "rewards/rejected": -2.6731436252593994,
+      "step": 2470
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.271195757420177e-06,
+      "logits/chosen": 1.3535038232803345,
+      "logits/rejected": 1.4873548746109009,
+      "logps/chosen": -467.2286071777344,
+      "logps/rejected": -575.8056640625,
+      "loss": 0.5234,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.02435302734375,
+      "rewards/margins": 0.9957019686698914,
+      "rewards/rejected": -3.020055055618286,
+      "step": 2480
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.263117378346425e-06,
+      "logits/chosen": 0.9210482835769653,
+      "logits/rejected": 1.7769590616226196,
+      "logps/chosen": -476.7568359375,
+      "logps/rejected": -542.2296752929688,
+      "loss": 0.5014,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.864184021949768,
+      "rewards/margins": 1.0771721601486206,
+      "rewards/rejected": -2.9413561820983887,
+      "step": 2490
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.255002204835208e-06,
+      "logits/chosen": 0.8788352012634277,
+      "logits/rejected": 1.2017310857772827,
+      "logps/chosen": -465.7417907714844,
+      "logps/rejected": -518.2665405273438,
+      "loss": 0.5233,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7486317157745361,
+      "rewards/margins": 0.7791939377784729,
+      "rewards/rejected": -2.5278258323669434,
+      "step": 2500
+    },
+    {
+      "epoch": 0.33,
+      "eval_logits/chosen": 1.3092632293701172,
+      "eval_logits/rejected": 1.7390477657318115,
+      "eval_logps/chosen": -458.5733947753906,
+      "eval_logps/rejected": -521.7109375,
+      "eval_loss": 0.5387481451034546,
+      "eval_rewards/accuracies": 0.7245000004768372,
+      "eval_rewards/chosen": -1.9001383781433105,
+      "eval_rewards/margins": 0.8289017081260681,
+      "eval_rewards/rejected": -2.7290403842926025,
+      "eval_runtime": 1348.6701,
+      "eval_samples_per_second": 1.483,
+      "eval_steps_per_second": 0.185,
+      "step": 2500
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.246850406241812e-06,
+      "logits/chosen": 1.0379670858383179,
+      "logits/rejected": 1.4408442974090576,
+      "logps/chosen": -381.1977233886719,
+      "logps/rejected": -520.677734375,
+      "loss": 0.4871,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7298479080200195,
+      "rewards/margins": 1.1202685832977295,
+      "rewards/rejected": -2.850116729736328,
+      "step": 2510
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2386621526858465e-06,
+      "logits/chosen": 1.0765659809112549,
+      "logits/rejected": 1.6101245880126953,
+      "logps/chosen": -409.4563293457031,
+      "logps/rejected": -450.95489501953125,
+      "loss": 0.5423,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.8959506750106812,
+      "rewards/margins": 0.7661338448524475,
+      "rewards/rejected": -2.6620841026306152,
+      "step": 2520
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2304376150477015e-06,
+      "logits/chosen": 0.6722235679626465,
+      "logits/rejected": 0.9519909620285034,
+      "logps/chosen": -457.364501953125,
+      "logps/rejected": -544.7530517578125,
+      "loss": 0.5058,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.8167307376861572,
+      "rewards/margins": 0.9830341339111328,
+      "rewards/rejected": -2.799765110015869,
+      "step": 2530
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.222176964964977e-06,
+      "logits/chosen": 0.9938064813613892,
+      "logits/rejected": 1.4047633409500122,
+      "logps/chosen": -469.493896484375,
+      "logps/rejected": -517.1723022460938,
+      "loss": 0.5862,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.0123093128204346,
+      "rewards/margins": 0.5795112252235413,
+      "rewards/rejected": -2.591820478439331,
+      "step": 2540
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.213880374828903e-06,
+      "logits/chosen": 1.090275526046753,
+      "logits/rejected": 1.423802137374878,
+      "logps/chosen": -423.1133728027344,
+      "logps/rejected": -515.8557739257812,
+      "loss": 0.4663,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.9112498760223389,
+      "rewards/margins": 0.8618186116218567,
+      "rewards/rejected": -2.77306866645813,
+      "step": 2550
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2055480177807406e-06,
+      "logits/chosen": 1.3426724672317505,
+      "logits/rejected": 1.5009058713912964,
+      "logps/chosen": -411.602783203125,
+      "logps/rejected": -459.12152099609375,
+      "loss": 0.5932,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.9755351543426514,
+      "rewards/margins": 0.5162807106971741,
+      "rewards/rejected": -2.4918155670166016,
+      "step": 2560
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.1971800677081696e-06,
+      "logits/chosen": 1.6340980529785156,
+      "logits/rejected": 1.6496193408966064,
+      "logps/chosen": -477.47369384765625,
+      "logps/rejected": -558.4954833984375,
+      "loss": 0.579,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.12569260597229,
+      "rewards/margins": 0.596170961856842,
+      "rewards/rejected": -2.7218637466430664,
+      "step": 2570
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.188776699241661e-06,
+      "logits/chosen": 0.9561182856559753,
+      "logits/rejected": 1.7183074951171875,
+      "logps/chosen": -483.1239318847656,
+      "logps/rejected": -528.122802734375,
+      "loss": 0.5394,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.050997734069824,
+      "rewards/margins": 0.6786635518074036,
+      "rewards/rejected": -2.729661226272583,
+      "step": 2580
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.180338087750827e-06,
+      "logits/chosen": 1.683789610862732,
+      "logits/rejected": 1.7157539129257202,
+      "logps/chosen": -463.20306396484375,
+      "logps/rejected": -557.5487670898438,
+      "loss": 0.5256,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.076094627380371,
+      "rewards/margins": 0.8124367594718933,
+      "rewards/rejected": -2.88853120803833,
+      "step": 2590
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.1718644093407704e-06,
+      "logits/chosen": 1.5733778476715088,
+      "logits/rejected": 2.047729969024658,
+      "logps/chosen": -495.30535888671875,
+      "logps/rejected": -561.2083129882812,
+      "loss": 0.5425,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.249206304550171,
+      "rewards/margins": 0.9049404263496399,
+      "rewards/rejected": -3.154146909713745,
+      "step": 2600
+    },
+    {
+      "epoch": 0.34,
+      "eval_logits/chosen": 1.92619788646698,
+      "eval_logits/rejected": 2.4431087970733643,
+      "eval_logps/chosen": -514.7514038085938,
+      "eval_logps/rejected": -582.4703979492188,
+      "eval_loss": 0.5430271029472351,
+      "eval_rewards/accuracies": 0.7225000262260437,
+      "eval_rewards/chosen": -2.4619174003601074,
+      "eval_rewards/margins": 0.8747171759605408,
+      "eval_rewards/rejected": -3.336634635925293,
+      "eval_runtime": 1349.5313,
+      "eval_samples_per_second": 1.482,
+      "eval_steps_per_second": 0.185,
+      "step": 2600
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.163355840848401e-06,
+      "logits/chosen": 1.4102129936218262,
+      "logits/rejected": 2.106313705444336,
+      "logps/chosen": -536.1241455078125,
+      "logps/rejected": -613.0098876953125,
+      "loss": 0.6003,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.5259006023406982,
+      "rewards/margins": 0.9065718650817871,
+      "rewards/rejected": -3.4324727058410645,
+      "step": 2610
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.154812559838748e-06,
+      "logits/chosen": 1.683789849281311,
+      "logits/rejected": 2.341182231903076,
+      "logps/chosen": -476.7560119628906,
+      "logps/rejected": -542.1966552734375,
+      "loss": 0.4689,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.2628796100616455,
+      "rewards/margins": 0.9704042673110962,
+      "rewards/rejected": -3.233283519744873,
+      "step": 2620
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.146234744601259e-06,
+      "logits/chosen": 2.2674190998077393,
+      "logits/rejected": 2.664658784866333,
+      "logps/chosen": -489.93426513671875,
+      "logps/rejected": -619.0794677734375,
+      "loss": 0.4576,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.517807960510254,
+      "rewards/margins": 1.2175263166427612,
+      "rewards/rejected": -3.7353336811065674,
+      "step": 2630
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.137622574146071e-06,
+      "logits/chosen": 1.8759933710098267,
+      "logits/rejected": 2.658112049102783,
+      "logps/chosen": -503.9085998535156,
+      "logps/rejected": -544.957275390625,
+      "loss": 0.5691,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.5300498008728027,
+      "rewards/margins": 0.8022888898849487,
+      "rewards/rejected": -3.332338809967041,
+      "step": 2640
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.12897622820028e-06,
+      "logits/chosen": 1.4453976154327393,
+      "logits/rejected": 2.1845076084136963,
+      "logps/chosen": -540.5733642578125,
+      "logps/rejected": -593.5223999023438,
+      "loss": 0.5428,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.4543707370758057,
+      "rewards/margins": 0.8197662234306335,
+      "rewards/rejected": -3.274137020111084,
+      "step": 2650
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.120295887204191e-06,
+      "logits/chosen": 1.4862298965454102,
+      "logits/rejected": 2.1127517223358154,
+      "logps/chosen": -497.09417724609375,
+      "logps/rejected": -582.1761474609375,
+      "loss": 0.6127,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.546721935272217,
+      "rewards/margins": 0.7177943587303162,
+      "rewards/rejected": -3.2645161151885986,
+      "step": 2660
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.111581732307548e-06,
+      "logits/chosen": 1.4486029148101807,
+      "logits/rejected": 1.781240463256836,
+      "logps/chosen": -470.76312255859375,
+      "logps/rejected": -555.5183715820312,
+      "loss": 0.5928,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.4050707817077637,
+      "rewards/margins": 0.7954873442649841,
+      "rewards/rejected": -3.2005581855773926,
+      "step": 2670
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.1028339453657595e-06,
+      "logits/chosen": 1.0702714920043945,
+      "logits/rejected": 1.9701976776123047,
+      "logps/chosen": -493.88702392578125,
+      "logps/rejected": -521.0225219726562,
+      "loss": 0.5478,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.814436912536621,
+      "rewards/margins": 0.7960997819900513,
+      "rewards/rejected": -2.610536813735962,
+      "step": 2680
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.094052708936096e-06,
+      "logits/chosen": 0.9257051348686218,
+      "logits/rejected": 1.1736860275268555,
+      "logps/chosen": -445.78094482421875,
+      "logps/rejected": -565.4486694335938,
+      "loss": 0.4675,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.6142679452896118,
+      "rewards/margins": 0.9910353422164917,
+      "rewards/rejected": -2.6053032875061035,
+      "step": 2690
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.0852382062738874e-06,
+      "logits/chosen": 0.9413816332817078,
+      "logits/rejected": 1.2453538179397583,
+      "logps/chosen": -470.5220642089844,
+      "logps/rejected": -553.0546264648438,
+      "loss": 0.4719,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.896549940109253,
+      "rewards/margins": 0.8988428115844727,
+      "rewards/rejected": -2.7953925132751465,
+      "step": 2700
+    },
+    {
+      "epoch": 0.35,
+      "eval_logits/chosen": 1.6066195964813232,
+      "eval_logits/rejected": 2.0585551261901855,
+      "eval_logps/chosen": -459.7815856933594,
+      "eval_logps/rejected": -519.9871826171875,
+      "eval_loss": 0.5308552384376526,
+      "eval_rewards/accuracies": 0.7285000085830688,
+      "eval_rewards/chosen": -1.9122200012207031,
+      "eval_rewards/margins": 0.7995823621749878,
+      "eval_rewards/rejected": -2.7118022441864014,
+      "eval_runtime": 1345.8752,
+      "eval_samples_per_second": 1.486,
+      "eval_steps_per_second": 0.186,
+      "step": 2700
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.076390621328693e-06,
+      "logits/chosen": 1.2057842016220093,
+      "logits/rejected": 2.069288730621338,
+      "logps/chosen": -517.2399291992188,
+      "logps/rejected": -559.2868041992188,
+      "loss": 0.4674,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.1037609577178955,
+      "rewards/margins": 0.8918477296829224,
+      "rewards/rejected": -2.9956085681915283,
+      "step": 2710
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.067510138740467e-06,
+      "logits/chosen": 1.6564505100250244,
+      "logits/rejected": 1.9685461521148682,
+      "logps/chosen": -502.56658935546875,
+      "logps/rejected": -579.8209228515625,
+      "loss": 0.5314,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.049485683441162,
+      "rewards/margins": 0.943230926990509,
+      "rewards/rejected": -2.9927167892456055,
+      "step": 2720
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.058596943835703e-06,
+      "logits/chosen": 1.1909606456756592,
+      "logits/rejected": 1.5680088996887207,
+      "logps/chosen": -467.0550231933594,
+      "logps/rejected": -482.2935485839844,
+      "loss": 0.6493,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.9316520690917969,
+      "rewards/margins": 0.5345866084098816,
+      "rewards/rejected": -2.4662387371063232,
+      "step": 2730
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.049651222623568e-06,
+      "logits/chosen": 0.9478217363357544,
+      "logits/rejected": 1.3925116062164307,
+      "logps/chosen": -431.4820251464844,
+      "logps/rejected": -455.96295166015625,
+      "loss": 0.5248,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.6732879877090454,
+      "rewards/margins": 0.6313273906707764,
+      "rewards/rejected": -2.3046154975891113,
+      "step": 2740
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.040673161792014e-06,
+      "logits/chosen": 0.557174026966095,
+      "logits/rejected": 1.1926147937774658,
+      "logps/chosen": -475.2584533691406,
+      "logps/rejected": -545.0785522460938,
+      "loss": 0.517,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.8062732219696045,
+      "rewards/margins": 0.8304523229598999,
+      "rewards/rejected": -2.636725664138794,
+      "step": 2750
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.031662948703896e-06,
+      "logits/chosen": 0.7812498211860657,
+      "logits/rejected": 1.2353713512420654,
+      "logps/chosen": -435.7364196777344,
+      "logps/rejected": -506.4576721191406,
+      "loss": 0.5255,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6220506429672241,
+      "rewards/margins": 0.9081017374992371,
+      "rewards/rejected": -2.5301523208618164,
+      "step": 2760
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.022620771393047e-06,
+      "logits/chosen": 0.5790234804153442,
+      "logits/rejected": 1.1668779850006104,
+      "logps/chosen": -456.9969787597656,
+      "logps/rejected": -488.02978515625,
+      "loss": 0.5588,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.797832727432251,
+      "rewards/margins": 0.6820603609085083,
+      "rewards/rejected": -2.4798929691314697,
+      "step": 2770
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.013546818560362e-06,
+      "logits/chosen": 0.7289198637008667,
+      "logits/rejected": 1.2590051889419556,
+      "logps/chosen": -405.9442138671875,
+      "logps/rejected": -438.5257873535156,
+      "loss": 0.6239,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6182396411895752,
+      "rewards/margins": 0.5406599044799805,
+      "rewards/rejected": -2.1588995456695557,
+      "step": 2780
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 4.00444127956986e-06,
+      "logits/chosen": 0.42976975440979004,
+      "logits/rejected": 1.1965467929840088,
+      "logps/chosen": -402.8417663574219,
+      "logps/rejected": -461.88336181640625,
+      "loss": 0.4922,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.5902665853500366,
+      "rewards/margins": 0.8525587320327759,
+      "rewards/rejected": -2.4428253173828125,
+      "step": 2790
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.9953043444447255e-06,
+      "logits/chosen": 0.5072175860404968,
+      "logits/rejected": 0.8735979199409485,
+      "logps/chosen": -463.3907165527344,
+      "logps/rejected": -490.46453857421875,
+      "loss": 0.5543,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.650398850440979,
+      "rewards/margins": 0.574273943901062,
+      "rewards/rejected": -2.224672794342041,
+      "step": 2800
+    },
+    {
+      "epoch": 0.37,
+      "eval_logits/chosen": 1.1720820665359497,
+      "eval_logits/rejected": 1.5924084186553955,
+      "eval_logps/chosen": -439.21624755859375,
+      "eval_logps/rejected": -495.0331726074219,
+      "eval_loss": 0.5267750024795532,
+      "eval_rewards/accuracies": 0.7225000262260437,
+      "eval_rewards/chosen": -1.7065664529800415,
+      "eval_rewards/margins": 0.7556957602500916,
+      "eval_rewards/rejected": -2.462261915206909,
+      "eval_runtime": 1346.002,
+      "eval_samples_per_second": 1.486,
+      "eval_steps_per_second": 0.186,
+      "step": 2800
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.986136203863355e-06,
+      "logits/chosen": 0.7749293446540833,
+      "logits/rejected": 1.0836472511291504,
+      "logps/chosen": -476.359130859375,
+      "logps/rejected": -528.639892578125,
+      "loss": 0.5521,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.778302788734436,
+      "rewards/margins": 0.7972840070724487,
+      "rewards/rejected": -2.5755867958068848,
+      "step": 2810
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.976937049155365e-06,
+      "logits/chosen": 1.1629598140716553,
+      "logits/rejected": 1.5726840496063232,
+      "logps/chosen": -446.14990234375,
+      "logps/rejected": -550.91259765625,
+      "loss": 0.4494,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.8462921380996704,
+      "rewards/margins": 0.9811326265335083,
+      "rewards/rejected": -2.8274245262145996,
+      "step": 2820
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.967707072297608e-06,
+      "logits/chosen": 1.3382699489593506,
+      "logits/rejected": 1.666267991065979,
+      "logps/chosen": -524.681640625,
+      "logps/rejected": -602.2101440429688,
+      "loss": 0.55,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.4141881465911865,
+      "rewards/margins": 0.8269587755203247,
+      "rewards/rejected": -3.2411468029022217,
+      "step": 2830
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.958446465910159e-06,
+      "logits/chosen": 1.0163097381591797,
+      "logits/rejected": 1.2590783834457397,
+      "logps/chosen": -509.1556091308594,
+      "logps/rejected": -610.1139526367188,
+      "loss": 0.4743,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.4810359477996826,
+      "rewards/margins": 1.0028384923934937,
+      "rewards/rejected": -3.483874559402466,
+      "step": 2840
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.9491554232523066e-06,
+      "logits/chosen": 1.702996015548706,
+      "logits/rejected": 1.5542683601379395,
+      "logps/chosen": -450.76483154296875,
+      "logps/rejected": -534.9152221679688,
+      "loss": 0.6099,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.3840889930725098,
+      "rewards/margins": 0.565158486366272,
+      "rewards/rejected": -2.9492478370666504,
+      "step": 2850
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.939834138218505e-06,
+      "logits/chosen": 0.9692273139953613,
+      "logits/rejected": 1.3489946126937866,
+      "logps/chosen": -508.81951904296875,
+      "logps/rejected": -591.5501708984375,
+      "loss": 0.5307,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.379051685333252,
+      "rewards/margins": 1.0336570739746094,
+      "rewards/rejected": -3.4127089977264404,
+      "step": 2860
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.930482805334339e-06,
+      "logits/chosen": 1.2421374320983887,
+      "logits/rejected": 1.6894123554229736,
+      "logps/chosen": -462.6669006347656,
+      "logps/rejected": -542.070556640625,
+      "loss": 0.5082,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.187364101409912,
+      "rewards/margins": 0.945517361164093,
+      "rewards/rejected": -3.1328816413879395,
+      "step": 2870
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.921101619752464e-06,
+      "logits/chosen": 0.9331795573234558,
+      "logits/rejected": 1.6218369007110596,
+      "logps/chosen": -455.52325439453125,
+      "logps/rejected": -511.2289123535156,
+      "loss": 0.4595,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.9294874668121338,
+      "rewards/margins": 0.9207557439804077,
+      "rewards/rejected": -2.850243330001831,
+      "step": 2880
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.911690777248525e-06,
+      "logits/chosen": 1.006848692893982,
+      "logits/rejected": 1.6681101322174072,
+      "logps/chosen": -473.966796875,
+      "logps/rejected": -560.3245849609375,
+      "loss": 0.4874,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.041447162628174,
+      "rewards/margins": 0.8777435421943665,
+      "rewards/rejected": -2.9191908836364746,
+      "step": 2890
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.902250474217079e-06,
+      "logits/chosen": 1.176171064376831,
+      "logits/rejected": 1.6354738473892212,
+      "logps/chosen": -468.18572998046875,
+      "logps/rejected": -540.3934326171875,
+      "loss": 0.5409,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.180406093597412,
+      "rewards/margins": 1.0076518058776855,
+      "rewards/rejected": -3.1880578994750977,
+      "step": 2900
+    },
+    {
+      "epoch": 0.38,
+      "eval_logits/chosen": 1.6062495708465576,
+      "eval_logits/rejected": 2.088993787765503,
+      "eval_logps/chosen": -487.3476867675781,
+      "eval_logps/rejected": -564.322021484375,
+      "eval_loss": 0.5400366187095642,
+      "eval_rewards/accuracies": 0.7174999713897705,
+      "eval_rewards/chosen": -2.1878812313079834,
+      "eval_rewards/margins": 0.967268705368042,
+      "eval_rewards/rejected": -3.1551499366760254,
+      "eval_runtime": 1348.7765,
+      "eval_samples_per_second": 1.483,
+      "eval_steps_per_second": 0.185,
+      "step": 2900
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.892780907667495e-06,
+      "logits/chosen": 1.2155516147613525,
+      "logits/rejected": 1.6637327671051025,
+      "logps/chosen": -457.01031494140625,
+      "logps/rejected": -541.6937255859375,
+      "loss": 0.563,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.9876590967178345,
+      "rewards/margins": 0.9567297101020813,
+      "rewards/rejected": -2.9443888664245605,
+      "step": 2910
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.883282275219837e-06,
+      "logits/chosen": 0.7854558825492859,
+      "logits/rejected": 1.3646886348724365,
+      "logps/chosen": -479.4649963378906,
+      "logps/rejected": -504.91827392578125,
+      "loss": 0.577,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.0149054527282715,
+      "rewards/margins": 0.6518548727035522,
+      "rewards/rejected": -2.6667604446411133,
+      "step": 2920
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.873754775100751e-06,
+      "logits/chosen": 0.5739338994026184,
+      "logits/rejected": 1.2258188724517822,
+      "logps/chosen": -426.21136474609375,
+      "logps/rejected": -493.7691345214844,
+      "loss": 0.5316,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.5058035850524902,
+      "rewards/margins": 0.9343582391738892,
+      "rewards/rejected": -2.44016170501709,
+      "step": 2930
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.8641986061393145e-06,
+      "logits/chosen": 0.7734447121620178,
+      "logits/rejected": 1.2065842151641846,
+      "logps/chosen": -402.43804931640625,
+      "logps/rejected": -538.2296752929688,
+      "loss": 0.4186,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.4549567699432373,
+      "rewards/margins": 1.145514726638794,
+      "rewards/rejected": -2.6004717350006104,
+      "step": 2940
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.854613967762898e-06,
+      "logits/chosen": 0.7447450757026672,
+      "logits/rejected": 1.4527260065078735,
+      "logps/chosen": -470.2606506347656,
+      "logps/rejected": -511.93609619140625,
+      "loss": 0.5396,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.599679946899414,
+      "rewards/margins": 0.878267765045166,
+      "rewards/rejected": -2.47794771194458,
+      "step": 2950
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.845001059992999e-06,
+      "logits/chosen": 0.5066796541213989,
+      "logits/rejected": 0.976254940032959,
+      "logps/chosen": -441.29632568359375,
+      "logps/rejected": -488.44873046875,
+      "loss": 0.5053,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.605376958847046,
+      "rewards/margins": 0.9187203645706177,
+      "rewards/rejected": -2.524097442626953,
+      "step": 2960
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.835360083441067e-06,
+      "logits/chosen": 0.9353615641593933,
+      "logits/rejected": 1.2030768394470215,
+      "logps/chosen": -387.2866516113281,
+      "logps/rejected": -479.06201171875,
+      "loss": 0.4577,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.4047024250030518,
+      "rewards/margins": 1.10038161277771,
+      "rewards/rejected": -2.5050840377807617,
+      "step": 2970
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.825691239304318e-06,
+      "logits/chosen": 0.7916946411132812,
+      "logits/rejected": 1.1446305513381958,
+      "logps/chosen": -410.592529296875,
+      "logps/rejected": -501.0753479003906,
+      "loss": 0.4923,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.5499457120895386,
+      "rewards/margins": 0.9437626600265503,
+      "rewards/rejected": -2.4937081336975098,
+      "step": 2980
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.8159947293615385e-06,
+      "logits/chosen": 0.4882342219352722,
+      "logits/rejected": 1.0783809423446655,
+      "logps/chosen": -455.2822265625,
+      "logps/rejected": -500.0282287597656,
+      "loss": 0.4993,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.5454859733581543,
+      "rewards/margins": 0.8720490336418152,
+      "rewards/rejected": -2.417534828186035,
+      "step": 2990
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.806270755968866e-06,
+      "logits/chosen": 0.4040301740169525,
+      "logits/rejected": 0.990578830242157,
+      "logps/chosen": -488.35430908203125,
+      "logps/rejected": -512.38232421875,
+      "loss": 0.4956,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.6512552499771118,
+      "rewards/margins": 0.9614316821098328,
+      "rewards/rejected": -2.6126868724823,
+      "step": 3000
+    },
+    {
+      "epoch": 0.39,
+      "eval_logits/chosen": 1.1678742170333862,
+      "eval_logits/rejected": 1.646390438079834,
+      "eval_logps/chosen": -452.4431457519531,
+      "eval_logps/rejected": -520.4592895507812,
+      "eval_loss": 0.5285268425941467,
+      "eval_rewards/accuracies": 0.7285000085830688,
+      "eval_rewards/chosen": -1.8388358354568481,
+      "eval_rewards/margins": 0.8776875138282776,
+      "eval_rewards/rejected": -2.7165234088897705,
+      "eval_runtime": 1348.2958,
+      "eval_samples_per_second": 1.483,
+      "eval_steps_per_second": 0.185,
+      "step": 3000
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.7965195220555784e-06,
+      "logits/chosen": 1.372248649597168,
+      "logits/rejected": 1.4761378765106201,
+      "logps/chosen": -441.6908264160156,
+      "logps/rejected": -499.46435546875,
+      "loss": 0.6113,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.1146788597106934,
+      "rewards/margins": 0.6008113026618958,
+      "rewards/rejected": -2.7154901027679443,
+      "step": 3010
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.786741231119847e-06,
+      "logits/chosen": 0.40710383653640747,
+      "logits/rejected": 1.1151280403137207,
+      "logps/chosen": -430.97454833984375,
+      "logps/rejected": -473.3141174316406,
+      "loss": 0.5606,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6159788370132446,
+      "rewards/margins": 0.6816727519035339,
+      "rewards/rejected": -2.297651529312134,
+      "step": 3020
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7769360872244992e-06,
+      "logits/chosen": 0.8422456979751587,
+      "logits/rejected": 1.1742428541183472,
+      "logps/chosen": -391.97491455078125,
+      "logps/rejected": -444.60003662109375,
+      "loss": 0.5438,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.4630100727081299,
+      "rewards/margins": 0.7062714695930481,
+      "rewards/rejected": -2.169281482696533,
+      "step": 3030
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.767104294992754e-06,
+      "logits/chosen": 0.9450136423110962,
+      "logits/rejected": 1.7667958736419678,
+      "logps/chosen": -477.14862060546875,
+      "logps/rejected": -449.6900329589844,
+      "loss": 0.6412,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7589792013168335,
+      "rewards/margins": 0.6161889433860779,
+      "rewards/rejected": -2.3751680850982666,
+      "step": 3040
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7572460596039524e-06,
+      "logits/chosen": 1.0094486474990845,
+      "logits/rejected": 1.339747667312622,
+      "logps/chosen": -448.03082275390625,
+      "logps/rejected": -514.642578125,
+      "loss": 0.6051,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.8631420135498047,
+      "rewards/margins": 0.7241281270980835,
+      "rewards/rejected": -2.5872702598571777,
+      "step": 3050
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.74736158678928e-06,
+      "logits/chosen": 0.9307212829589844,
+      "logits/rejected": 1.5582621097564697,
+      "logps/chosen": -465.6565856933594,
+      "logps/rejected": -500.3355407714844,
+      "loss": 0.4838,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.5386617183685303,
+      "rewards/margins": 0.9806914329528809,
+      "rewards/rejected": -2.519353151321411,
+      "step": 3060
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7374510828274673e-06,
+      "logits/chosen": 0.3841513693332672,
+      "logits/rejected": 1.1715351343154907,
+      "logps/chosen": -488.7295837402344,
+      "logps/rejected": -492.0480041503906,
+      "loss": 0.5624,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.5639508962631226,
+      "rewards/margins": 0.7329056262969971,
+      "rewards/rejected": -2.29685640335083,
+      "step": 3070
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.72751475454049e-06,
+      "logits/chosen": 0.8612874150276184,
+      "logits/rejected": 1.154017448425293,
+      "logps/chosen": -443.53851318359375,
+      "logps/rejected": -490.7236328125,
+      "loss": 0.5749,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.7006498575210571,
+      "rewards/margins": 0.6338729858398438,
+      "rewards/rejected": -2.3345227241516113,
+      "step": 3080
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7175528092892503e-06,
+      "logits/chosen": 0.8836231231689453,
+      "logits/rejected": 1.5137784481048584,
+      "logps/chosen": -433.352783203125,
+      "logps/rejected": -490.69732666015625,
+      "loss": 0.5754,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.8206474781036377,
+      "rewards/margins": 0.6728289723396301,
+      "rewards/rejected": -2.493476152420044,
+      "step": 3090
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.7075654549692498e-06,
+      "logits/chosen": 0.6033874750137329,
+      "logits/rejected": 1.4928462505340576,
+      "logps/chosen": -513.2379150390625,
+      "logps/rejected": -497.65240478515625,
+      "loss": 0.4572,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -1.669196367263794,
+      "rewards/margins": 0.8243755102157593,
+      "rewards/rejected": -2.4935717582702637,
+      "step": 3100
+    },
+    {
+      "epoch": 0.41,
+      "eval_logits/chosen": 1.2994097471237183,
+      "eval_logits/rejected": 1.7627153396606445,
+      "eval_logps/chosen": -434.95050048828125,
+      "eval_logps/rejected": -491.4958190917969,
+      "eval_loss": 0.5197616219520569,
+      "eval_rewards/accuracies": 0.7264999747276306,
+      "eval_rewards/chosen": -1.6639093160629272,
+      "eval_rewards/margins": 0.7629793882369995,
+      "eval_rewards/rejected": -2.4268887042999268,
+      "eval_runtime": 1347.1237,
+      "eval_samples_per_second": 1.485,
+      "eval_steps_per_second": 0.186,
+      "step": 3100
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.697552900006249e-06,
+      "logits/chosen": 1.2331713438034058,
+      "logits/rejected": 1.4610141515731812,
+      "logps/chosen": -446.64898681640625,
+      "logps/rejected": -499.29052734375,
+      "loss": 0.526,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.8795382976531982,
+      "rewards/margins": 0.795755922794342,
+      "rewards/rejected": -2.6752943992614746,
+      "step": 3110
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6875153533519244e-06,
+      "logits/chosen": 1.1177051067352295,
+      "logits/rejected": 1.7195285558700562,
+      "logps/chosen": -458.1422424316406,
+      "logps/rejected": -503.0191345214844,
+      "loss": 0.5332,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.8283450603485107,
+      "rewards/margins": 0.7121249437332153,
+      "rewards/rejected": -2.5404701232910156,
+      "step": 3120
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6774530244794992e-06,
+      "logits/chosen": 1.114457368850708,
+      "logits/rejected": 1.7580559253692627,
+      "logps/chosen": -453.89434814453125,
+      "logps/rejected": -466.11505126953125,
+      "loss": 0.5635,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7303870916366577,
+      "rewards/margins": 0.7310529351234436,
+      "rewards/rejected": -2.461440086364746,
+      "step": 3130
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.667366123379378e-06,
+      "logits/chosen": 1.3230986595153809,
+      "logits/rejected": 2.0303707122802734,
+      "logps/chosen": -395.96844482421875,
+      "logps/rejected": -447.9563903808594,
+      "loss": 0.5089,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.8301582336425781,
+      "rewards/margins": 0.6995818018913269,
+      "rewards/rejected": -2.5297398567199707,
+      "step": 3140
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6572548605547607e-06,
+      "logits/chosen": 0.8152780532836914,
+      "logits/rejected": 1.2985804080963135,
+      "logps/chosen": -418.112060546875,
+      "logps/rejected": -519.9182739257812,
+      "loss": 0.5073,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7351443767547607,
+      "rewards/margins": 0.910904586315155,
+      "rewards/rejected": -2.6460492610931396,
+      "step": 3150
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6471194470172538e-06,
+      "logits/chosen": 1.3877384662628174,
+      "logits/rejected": 1.593894124031067,
+      "logps/chosen": -447.94085693359375,
+      "logps/rejected": -542.073486328125,
+      "loss": 0.5002,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.8057979345321655,
+      "rewards/margins": 0.9341724514961243,
+      "rewards/rejected": -2.7399704456329346,
+      "step": 3160
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.636960094282461e-06,
+      "logits/chosen": 1.31181001663208,
+      "logits/rejected": 1.7865238189697266,
+      "logps/chosen": -478.12310791015625,
+      "logps/rejected": -522.6786499023438,
+      "loss": 0.5053,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.0036494731903076,
+      "rewards/margins": 0.9192601442337036,
+      "rewards/rejected": -2.92290997505188,
+      "step": 3170
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.6267770143655743e-06,
+      "logits/chosen": 0.8400815725326538,
+      "logits/rejected": 1.4613516330718994,
+      "logps/chosen": -471.61651611328125,
+      "logps/rejected": -510.92218017578125,
+      "loss": 0.5264,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.932928442955017,
+      "rewards/margins": 0.7337788343429565,
+      "rewards/rejected": -2.6667070388793945,
+      "step": 3180
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.6165704197769484e-06,
+      "logits/chosen": 1.0123398303985596,
+      "logits/rejected": 1.2186167240142822,
+      "logps/chosen": -440.90478515625,
+      "logps/rejected": -487.68157958984375,
+      "loss": 0.6579,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.7840601205825806,
+      "rewards/margins": 0.5804886817932129,
+      "rewards/rejected": -2.364548444747925,
+      "step": 3190
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.606340523517663e-06,
+      "logits/chosen": 0.9449944496154785,
+      "logits/rejected": 1.3913816213607788,
+      "logps/chosen": -429.3963928222656,
+      "logps/rejected": -505.7931213378906,
+      "loss": 0.4962,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.6913959980010986,
+      "rewards/margins": 0.9350394010543823,
+      "rewards/rejected": -2.6264355182647705,
+      "step": 3200
+    },
+    {
+      "epoch": 0.42,
+      "eval_logits/chosen": 1.1779999732971191,
+      "eval_logits/rejected": 1.6452065706253052,
+      "eval_logps/chosen": -437.69940185546875,
+      "eval_logps/rejected": -500.95111083984375,
+      "eval_loss": 0.518135666847229,
+      "eval_rewards/accuracies": 0.7264999747276306,
+      "eval_rewards/chosen": -1.6913982629776,
+      "eval_rewards/margins": 0.8300431370735168,
+      "eval_rewards/rejected": -2.5214414596557617,
+      "eval_runtime": 1343.9012,
+      "eval_samples_per_second": 1.488,
+      "eval_steps_per_second": 0.186,
+      "step": 3200
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.5960875390750793e-06,
+      "logits/chosen": 0.6452552080154419,
+      "logits/rejected": 1.625881552696228,
+      "logps/chosen": -500.6880798339844,
+      "logps/rejected": -524.5322265625,
+      "loss": 0.4618,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.7134166955947876,
+      "rewards/margins": 0.9759113192558289,
+      "rewards/rejected": -2.689328193664551,
+      "step": 3210
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.585811680418386e-06,
+      "logits/chosen": 1.1903812885284424,
+      "logits/rejected": 1.7203998565673828,
+      "logps/chosen": -461.4546813964844,
+      "logps/rejected": -535.4126586914062,
+      "loss": 0.5158,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.821473479270935,
+      "rewards/margins": 0.993263840675354,
+      "rewards/rejected": -2.814737558364868,
+      "step": 3220
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.5755131619941347e-06,
+      "logits/chosen": 1.4167505502700806,
+      "logits/rejected": 1.889325499534607,
+      "logps/chosen": -432.9002990722656,
+      "logps/rejected": -492.9075622558594,
+      "loss": 0.5078,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.7830183506011963,
+      "rewards/margins": 0.9155006408691406,
+      "rewards/rejected": -2.698519229888916,
+      "step": 3230
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.565192198721759e-06,
+      "logits/chosen": 1.428459882736206,
+      "logits/rejected": 1.745610237121582,
+      "logps/chosen": -434.2123107910156,
+      "logps/rejected": -575.7933959960938,
+      "loss": 0.4542,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.9869998693466187,
+      "rewards/margins": 1.1341489553451538,
+      "rewards/rejected": -3.1211490631103516,
+      "step": 3240
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5548490059890965e-06,
+      "logits/chosen": 1.730893850326538,
+      "logits/rejected": 2.2235589027404785,
+      "logps/chosen": -520.96728515625,
+      "logps/rejected": -630.299560546875,
+      "loss": 0.4728,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.232191801071167,
+      "rewards/margins": 1.2577087879180908,
+      "rewards/rejected": -3.4899001121520996,
+      "step": 3250
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5444837996478903e-06,
+      "logits/chosen": 1.2619092464447021,
+      "logits/rejected": 1.7568824291229248,
+      "logps/chosen": -539.0397338867188,
+      "logps/rejected": -645.3343505859375,
+      "loss": 0.5949,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.369567394256592,
+      "rewards/margins": 0.9820758700370789,
+      "rewards/rejected": -3.3516433238983154,
+      "step": 3260
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.534096796009282e-06,
+      "logits/chosen": 1.4912410974502563,
+      "logits/rejected": 1.9062105417251587,
+      "logps/chosen": -492.0826721191406,
+      "logps/rejected": -606.6270751953125,
+      "loss": 0.5747,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.358466386795044,
+      "rewards/margins": 1.1837238073349,
+      "rewards/rejected": -3.5421900749206543,
+      "step": 3270
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5236882118393046e-06,
+      "logits/chosen": 1.0840097665786743,
+      "logits/rejected": 1.5367692708969116,
+      "logps/chosen": -505.703125,
+      "logps/rejected": -539.6936645507812,
+      "loss": 0.5717,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.0457935333251953,
+      "rewards/margins": 0.7116455435752869,
+      "rewards/rejected": -2.757439136505127,
+      "step": 3280
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5132582643543513e-06,
+      "logits/chosen": 0.8981589078903198,
+      "logits/rejected": 1.4330666065216064,
+      "logps/chosen": -455.79241943359375,
+      "logps/rejected": -531.706787109375,
+      "loss": 0.496,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.832550287246704,
+      "rewards/margins": 1.0026943683624268,
+      "rewards/rejected": -2.835244655609131,
+      "step": 3290
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5028071712166456e-06,
+      "logits/chosen": 0.6667258143424988,
+      "logits/rejected": 1.4452887773513794,
+      "logps/chosen": -459.93096923828125,
+      "logps/rejected": -475.8651428222656,
+      "loss": 0.6098,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7569339275360107,
+      "rewards/margins": 0.7255094051361084,
+      "rewards/rejected": -2.4824435710906982,
+      "step": 3300
+    },
+    {
+      "epoch": 0.43,
+      "eval_logits/chosen": 1.0616835355758667,
+      "eval_logits/rejected": 1.5141295194625854,
+      "eval_logps/chosen": -428.9995422363281,
+      "eval_logps/rejected": -492.60223388671875,
+      "eval_loss": 0.5188182592391968,
+      "eval_rewards/accuracies": 0.7310000061988831,
+      "eval_rewards/chosen": -1.6043996810913086,
+      "eval_rewards/margins": 0.8335530161857605,
+      "eval_rewards/rejected": -2.4379525184631348,
+      "eval_runtime": 1346.7524,
+      "eval_samples_per_second": 1.485,
+      "eval_steps_per_second": 0.186,
+      "step": 3300
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.4923351505297008e-06,
+      "logits/chosen": 0.5315229296684265,
+      "logits/rejected": 1.0150467157363892,
+      "logps/chosen": -461.950439453125,
+      "logps/rejected": -524.130615234375,
+      "loss": 0.4958,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.6741793155670166,
+      "rewards/margins": 0.8332613706588745,
+      "rewards/rejected": -2.5074405670166016,
+      "step": 3310
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.481842420833766e-06,
+      "logits/chosen": 0.5714815855026245,
+      "logits/rejected": 1.2654632329940796,
+      "logps/chosen": -496.99725341796875,
+      "logps/rejected": -485.7489318847656,
+      "loss": 0.543,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.656964898109436,
+      "rewards/margins": 0.7108852863311768,
+      "rewards/rejected": -2.3678503036499023,
+      "step": 3320
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4713292011012645e-06,
+      "logits/chosen": 0.8555871248245239,
+      "logits/rejected": 1.5854310989379883,
+      "logps/chosen": -445.5460510253906,
+      "logps/rejected": -491.4014587402344,
+      "loss": 0.5503,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5714675188064575,
+      "rewards/margins": 0.8530597686767578,
+      "rewards/rejected": -2.424527406692505,
+      "step": 3330
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4607957107322277e-06,
+      "logits/chosen": 0.8446288108825684,
+      "logits/rejected": 1.6464910507202148,
+      "logps/chosen": -412.71258544921875,
+      "logps/rejected": -478.8643493652344,
+      "loss": 0.5002,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.8122217655181885,
+      "rewards/margins": 0.8502435684204102,
+      "rewards/rejected": -2.6624653339385986,
+      "step": 3340
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4502421695497112e-06,
+      "logits/chosen": 1.0822343826293945,
+      "logits/rejected": 1.4068114757537842,
+      "logps/chosen": -440.36383056640625,
+      "logps/rejected": -484.6875915527344,
+      "loss": 0.5778,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.7022686004638672,
+      "rewards/margins": 0.6010777354240417,
+      "rewards/rejected": -2.3033463954925537,
+      "step": 3350
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4396687977952137e-06,
+      "logits/chosen": 0.7362917065620422,
+      "logits/rejected": 1.3217395544052124,
+      "logps/chosen": -425.94061279296875,
+      "logps/rejected": -493.17913818359375,
+      "loss": 0.5245,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.687774896621704,
+      "rewards/margins": 0.9045063257217407,
+      "rewards/rejected": -2.5922811031341553,
+      "step": 3360
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.429075816124075e-06,
+      "logits/chosen": 0.7948580980300903,
+      "logits/rejected": 1.158984899520874,
+      "logps/chosen": -454.0587463378906,
+      "logps/rejected": -518.7434692382812,
+      "loss": 0.5,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.6684925556182861,
+      "rewards/margins": 0.8132556676864624,
+      "rewards/rejected": -2.481748342514038,
+      "step": 3370
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.418463445600874e-06,
+      "logits/chosen": 0.6248632669448853,
+      "logits/rejected": 1.210164189338684,
+      "logps/chosen": -379.62652587890625,
+      "logps/rejected": -469.27587890625,
+      "loss": 0.5095,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.5749435424804688,
+      "rewards/margins": 0.8910201787948608,
+      "rewards/rejected": -2.46596360206604,
+      "step": 3380
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4078319076948173e-06,
+      "logits/chosen": 0.6288284063339233,
+      "logits/rejected": 1.0596554279327393,
+      "logps/chosen": -378.13232421875,
+      "logps/rejected": -449.1874084472656,
+      "loss": 0.5162,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.3207027912139893,
+      "rewards/margins": 0.9178522229194641,
+      "rewards/rejected": -2.2385549545288086,
+      "step": 3390
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.3971814242751123e-06,
+      "logits/chosen": 0.43135276436805725,
+      "logits/rejected": 1.0462135076522827,
+      "logps/chosen": -417.98431396484375,
+      "logps/rejected": -448.4012756347656,
+      "loss": 0.5349,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.2526473999023438,
+      "rewards/margins": 0.7096043229103088,
+      "rewards/rejected": -1.9622516632080078,
+      "step": 3400
+    },
+    {
+      "epoch": 0.44,
+      "eval_logits/chosen": 1.0370838642120361,
+      "eval_logits/rejected": 1.4965388774871826,
+      "eval_logps/chosen": -415.7578125,
+      "eval_logps/rejected": -479.7060546875,
+      "eval_loss": 0.5210185050964355,
+      "eval_rewards/accuracies": 0.7285000085830688,
+      "eval_rewards/chosen": -1.4719822406768799,
+      "eval_rewards/margins": 0.8370087742805481,
+      "eval_rewards/rejected": -2.308990955352783,
+      "eval_runtime": 1349.8289,
+      "eval_samples_per_second": 1.482,
+      "eval_steps_per_second": 0.185,
+      "step": 3400
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.386512217606339e-06,
+      "logits/chosen": 1.0526317358016968,
+      "logits/rejected": 1.447126865386963,
+      "logps/chosen": -424.024169921875,
+      "logps/rejected": -462.27789306640625,
+      "loss": 0.5427,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.565047264099121,
+      "rewards/margins": 0.7917253375053406,
+      "rewards/rejected": -2.3567726612091064,
+      "step": 3410
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.375824510343816e-06,
+      "logits/chosen": 1.2742136716842651,
+      "logits/rejected": 1.851244568824768,
+      "logps/chosen": -480.64202880859375,
+      "logps/rejected": -567.9425659179688,
+      "loss": 0.5112,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.9135100841522217,
+      "rewards/margins": 0.8371618986129761,
+      "rewards/rejected": -2.7506721019744873,
+      "step": 3420
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3651185255289466e-06,
+      "logits/chosen": 1.2147417068481445,
+      "logits/rejected": 1.8255727291107178,
+      "logps/chosen": -501.2945251464844,
+      "logps/rejected": -578.0968017578125,
+      "loss": 0.4648,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.9875972270965576,
+      "rewards/margins": 0.9468340873718262,
+      "rewards/rejected": -2.934431552886963,
+      "step": 3430
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.354394486584568e-06,
+      "logits/chosen": 1.0248808860778809,
+      "logits/rejected": 1.6474864482879639,
+      "logps/chosen": -467.7376403808594,
+      "logps/rejected": -552.8011474609375,
+      "loss": 0.4732,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.6010745763778687,
+      "rewards/margins": 0.948132336139679,
+      "rewards/rejected": -2.5492069721221924,
+      "step": 3440
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3436526173102913e-06,
+      "logits/chosen": 1.4321781396865845,
+      "logits/rejected": 2.235511302947998,
+      "logps/chosen": -418.10321044921875,
+      "logps/rejected": -451.8190002441406,
+      "loss": 0.5161,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.8352901935577393,
+      "rewards/margins": 0.8485320806503296,
+      "rewards/rejected": -2.6838221549987793,
+      "step": 3450
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3328931418778254e-06,
+      "logits/chosen": 1.3957363367080688,
+      "logits/rejected": 1.565422773361206,
+      "logps/chosen": -429.4269104003906,
+      "logps/rejected": -548.0386962890625,
+      "loss": 0.4128,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.6326179504394531,
+      "rewards/margins": 1.2800357341766357,
+      "rewards/rejected": -2.912653684616089,
+      "step": 3460
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3221162848263028e-06,
+      "logits/chosen": 1.4535284042358398,
+      "logits/rejected": 2.0320611000061035,
+      "logps/chosen": -474.25018310546875,
+      "logps/rejected": -544.2798461914062,
+      "loss": 0.4054,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.8090862035751343,
+      "rewards/margins": 1.2316696643829346,
+      "rewards/rejected": -3.0407557487487793,
+      "step": 3470
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.3113222710575914e-06,
+      "logits/chosen": 1.2613544464111328,
+      "logits/rejected": 1.9117896556854248,
+      "logps/chosen": -485.325439453125,
+      "logps/rejected": -530.9984130859375,
+      "loss": 0.651,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.9451086521148682,
+      "rewards/margins": 0.6674405336380005,
+      "rewards/rejected": -2.612549304962158,
+      "step": 3480
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.300511325831603e-06,
+      "logits/chosen": 1.5713465213775635,
+      "logits/rejected": 1.713397741317749,
+      "logps/chosen": -419.45269775390625,
+      "logps/rejected": -516.1373901367188,
+      "loss": 0.5098,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.5190571546554565,
+      "rewards/margins": 0.9378665089607239,
+      "rewards/rejected": -2.456923246383667,
+      "step": 3490
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.289683674761592e-06,
+      "logits/chosen": 1.1915028095245361,
+      "logits/rejected": 1.8072803020477295,
+      "logps/chosen": -417.5728454589844,
+      "logps/rejected": -467.2867126464844,
+      "loss": 0.4773,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.4632017612457275,
+      "rewards/margins": 0.8837030529975891,
+      "rewards/rejected": -2.346904754638672,
+      "step": 3500
+    },
+    {
+      "epoch": 0.46,
+      "eval_logits/chosen": 1.3263517618179321,
+      "eval_logits/rejected": 1.8090280294418335,
+      "eval_logps/chosen": -412.80621337890625,
+      "eval_logps/rejected": -471.6596984863281,
+      "eval_loss": 0.520591676235199,
+      "eval_rewards/accuracies": 0.7279999852180481,
+      "eval_rewards/chosen": -1.4424660205841064,
+      "eval_rewards/margins": 0.7860610485076904,
+      "eval_rewards/rejected": -2.228527069091797,
+      "eval_runtime": 1345.6267,
+      "eval_samples_per_second": 1.486,
+      "eval_steps_per_second": 0.186,
+      "step": 3500
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2788395438094444e-06,
+      "logits/chosen": 1.2892673015594482,
+      "logits/rejected": 1.7906087636947632,
+      "logps/chosen": -439.6426696777344,
+      "logps/rejected": -489.35302734375,
+      "loss": 0.5324,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.62063467502594,
+      "rewards/margins": 0.7643077969551086,
+      "rewards/rejected": -2.3849425315856934,
+      "step": 3510
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2679791592809653e-06,
+      "logits/chosen": 1.4143444299697876,
+      "logits/rejected": 1.933223009109497,
+      "logps/chosen": -387.8199768066406,
+      "logps/rejected": -473.97216796875,
+      "loss": 0.4968,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.4349174499511719,
+      "rewards/margins": 0.7744017839431763,
+      "rewards/rejected": -2.2093191146850586,
+      "step": 3520
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.257102747821157e-06,
+      "logits/chosen": 1.6398617029190063,
+      "logits/rejected": 2.4579176902770996,
+      "logps/chosen": -457.726806640625,
+      "logps/rejected": -544.8309326171875,
+      "loss": 0.4879,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.6612077951431274,
+      "rewards/margins": 1.1296989917755127,
+      "rewards/rejected": -2.7909066677093506,
+      "step": 3530
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.246210536409484e-06,
+      "logits/chosen": 1.8480857610702515,
+      "logits/rejected": 2.22153377532959,
+      "logps/chosen": -449.11016845703125,
+      "logps/rejected": -468.74908447265625,
+      "loss": 0.603,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.7867637872695923,
+      "rewards/margins": 0.6125444173812866,
+      "rewards/rejected": -2.399308204650879,
+      "step": 3540
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.235302752355142e-06,
+      "logits/chosen": 1.2989999055862427,
+      "logits/rejected": 2.3280489444732666,
+      "logps/chosen": -468.44110107421875,
+      "logps/rejected": -477.34808349609375,
+      "loss": 0.4542,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.599212408065796,
+      "rewards/margins": 0.9258416891098022,
+      "rewards/rejected": -2.5250542163848877,
+      "step": 3550
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.2243796232923097e-06,
+      "logits/chosen": 1.28660249710083,
+      "logits/rejected": 1.9692814350128174,
+      "logps/chosen": -500.02569580078125,
+      "logps/rejected": -559.4823608398438,
+      "loss": 0.5085,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.932739019393921,
+      "rewards/margins": 0.7877759337425232,
+      "rewards/rejected": -2.720515012741089,
+      "step": 3560
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.2134413771754037e-06,
+      "logits/chosen": 1.5761184692382812,
+      "logits/rejected": 1.9535433053970337,
+      "logps/chosen": -485.031005859375,
+      "logps/rejected": -532.835693359375,
+      "loss": 0.5654,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.2038707733154297,
+      "rewards/margins": 0.6359282732009888,
+      "rewards/rejected": -2.839799165725708,
+      "step": 3570
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.2024882422743118e-06,
+      "logits/chosen": 1.8086954355239868,
+      "logits/rejected": 2.427356719970703,
+      "logps/chosen": -433.95343017578125,
+      "logps/rejected": -540.0693969726562,
+      "loss": 0.471,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.0142340660095215,
+      "rewards/margins": 1.0213309526443481,
+      "rewards/rejected": -3.03556489944458,
+      "step": 3580
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.1915204471696425e-06,
+      "logits/chosen": 1.6601234674453735,
+      "logits/rejected": 2.600374221801758,
+      "logps/chosen": -477.4906311035156,
+      "logps/rejected": -502.08123779296875,
+      "loss": 0.5613,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.0841822624206543,
+      "rewards/margins": 0.8141940832138062,
+      "rewards/rejected": -2.898376226425171,
+      "step": 3590
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.180538220747943e-06,
+      "logits/chosen": 1.7485212087631226,
+      "logits/rejected": 2.3923451900482178,
+      "logps/chosen": -470.86474609375,
+      "logps/rejected": -538.1746215820312,
+      "loss": 0.5312,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.0638813972473145,
+      "rewards/margins": 0.9722470045089722,
+      "rewards/rejected": -3.036128520965576,
+      "step": 3600
+    },
+    {
+      "epoch": 0.47,
+      "eval_logits/chosen": 2.019141912460327,
+      "eval_logits/rejected": 2.533942222595215,
+      "eval_logps/chosen": -449.8386535644531,
+      "eval_logps/rejected": -515.9942626953125,
+      "eval_loss": 0.5195835828781128,
+      "eval_rewards/accuracies": 0.7319999933242798,
+      "eval_rewards/chosen": -1.8127903938293457,
+      "eval_rewards/margins": 0.8590822815895081,
+      "eval_rewards/rejected": -2.671872854232788,
+      "eval_runtime": 1347.8562,
+      "eval_samples_per_second": 1.484,
+      "eval_steps_per_second": 0.185,
+      "step": 3600
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.1695417921969287e-06,
+      "logits/chosen": 1.3130995035171509,
+      "logits/rejected": 1.591094732284546,
+      "logps/chosen": -444.1183166503906,
+      "logps/rejected": -521.2981567382812,
+      "loss": 0.614,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8694114685058594,
+      "rewards/margins": 0.596592366695404,
+      "rewards/rejected": -2.466003894805908,
+      "step": 3610
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.158531391000697e-06,
+      "logits/chosen": 1.6498037576675415,
+      "logits/rejected": 2.4687371253967285,
+      "logps/chosen": -465.9458923339844,
+      "logps/rejected": -519.3355712890625,
+      "loss": 0.4851,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.8775802850723267,
+      "rewards/margins": 0.9237555265426636,
+      "rewards/rejected": -2.8013358116149902,
+      "step": 3620
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.147507246934943e-06,
+      "logits/chosen": 1.3147783279418945,
+      "logits/rejected": 1.57236647605896,
+      "logps/chosen": -487.71612548828125,
+      "logps/rejected": -559.159423828125,
+      "loss": 0.4994,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.8661190271377563,
+      "rewards/margins": 0.8776202201843262,
+      "rewards/rejected": -2.743738889694214,
+      "step": 3630
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.136469590062158e-06,
+      "logits/chosen": 1.4456905126571655,
+      "logits/rejected": 2.133394241333008,
+      "logps/chosen": -428.09478759765625,
+      "logps/rejected": -484.08734130859375,
+      "loss": 0.4681,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.8653593063354492,
+      "rewards/margins": 0.8778258562088013,
+      "rewards/rejected": -2.743185043334961,
+      "step": 3640
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1254186507268354e-06,
+      "logits/chosen": 1.3756492137908936,
+      "logits/rejected": 1.8904682397842407,
+      "logps/chosen": -423.1832580566406,
+      "logps/rejected": -501.38934326171875,
+      "loss": 0.5105,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8500111103057861,
+      "rewards/margins": 0.8337074518203735,
+      "rewards/rejected": -2.68371844291687,
+      "step": 3650
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.114354659550656e-06,
+      "logits/chosen": 1.8524181842803955,
+      "logits/rejected": 2.4592716693878174,
+      "logps/chosen": -481.6455078125,
+      "logps/rejected": -550.0005493164062,
+      "loss": 0.4985,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.05267333984375,
+      "rewards/margins": 1.1118876934051514,
+      "rewards/rejected": -3.1645612716674805,
+      "step": 3660
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1032778474276816e-06,
+      "logits/chosen": 1.546240210533142,
+      "logits/rejected": 2.442688465118408,
+      "logps/chosen": -474.2201232910156,
+      "logps/rejected": -511.53570556640625,
+      "loss": 0.4755,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.969080924987793,
+      "rewards/margins": 0.90904700756073,
+      "rewards/rejected": -2.8781278133392334,
+      "step": 3670
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.092188445519532e-06,
+      "logits/chosen": 1.6645681858062744,
+      "logits/rejected": 2.637698173522949,
+      "logps/chosen": -467.86993408203125,
+      "logps/rejected": -527.5178833007812,
+      "loss": 0.4865,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.948889136314392,
+      "rewards/margins": 1.0606005191802979,
+      "rewards/rejected": -3.0094897747039795,
+      "step": 3680
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.081086685250565e-06,
+      "logits/chosen": 1.934732437133789,
+      "logits/rejected": 2.7185251712799072,
+      "logps/chosen": -444.38507080078125,
+      "logps/rejected": -482.70928955078125,
+      "loss": 0.5187,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.9130178689956665,
+      "rewards/margins": 0.875904381275177,
+      "rewards/rejected": -2.7889223098754883,
+      "step": 3690
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.0699727983030434e-06,
+      "logits/chosen": 1.7284419536590576,
+      "logits/rejected": 2.68674898147583,
+      "logps/chosen": -502.0184631347656,
+      "logps/rejected": -508.72998046875,
+      "loss": 0.5879,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.1163885593414307,
+      "rewards/margins": 0.6249352693557739,
+      "rewards/rejected": -2.741323947906494,
+      "step": 3700
+    },
+    {
+      "epoch": 0.48,
+      "eval_logits/chosen": 2.3934340476989746,
+      "eval_logits/rejected": 2.9390172958374023,
+      "eval_logps/chosen": -460.812255859375,
+      "eval_logps/rejected": -528.5556030273438,
+      "eval_loss": 0.5127580761909485,
+      "eval_rewards/accuracies": 0.7354999780654907,
+      "eval_rewards/chosen": -1.9225265979766846,
+      "eval_rewards/margins": 0.8749601244926453,
+      "eval_rewards/rejected": -2.7974865436553955,
+      "eval_runtime": 1349.3004,
+      "eval_samples_per_second": 1.482,
+      "eval_steps_per_second": 0.185,
+      "step": 3700
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.058847016612301e-06,
+      "logits/chosen": 1.6397192478179932,
+      "logits/rejected": 1.8890368938446045,
+      "logps/chosen": -504.09521484375,
+      "logps/rejected": -567.0184326171875,
+      "loss": 0.6041,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.942374587059021,
+      "rewards/margins": 0.6444495916366577,
+      "rewards/rejected": -2.5868241786956787,
+      "step": 3710
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0477095723619034e-06,
+      "logits/chosen": 1.0804823637008667,
+      "logits/rejected": 2.315983533859253,
+      "logps/chosen": -409.70440673828125,
+      "logps/rejected": -500.25555419921875,
+      "loss": 0.4622,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.626031517982483,
+      "rewards/margins": 1.1232527494430542,
+      "rewards/rejected": -2.749284267425537,
+      "step": 3720
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0365606979788003e-06,
+      "logits/chosen": 1.2249141931533813,
+      "logits/rejected": 2.131930112838745,
+      "logps/chosen": -440.2842712402344,
+      "logps/rejected": -466.86309814453125,
+      "loss": 0.5609,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.684000015258789,
+      "rewards/margins": 0.6666284799575806,
+      "rewards/rejected": -2.35062837600708,
+      "step": 3730
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0254006261284786e-06,
+      "logits/chosen": 1.4237465858459473,
+      "logits/rejected": 1.9983221292495728,
+      "logps/chosen": -417.606201171875,
+      "logps/rejected": -480.5640563964844,
+      "loss": 0.507,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.571561336517334,
+      "rewards/margins": 0.8193261027336121,
+      "rewards/rejected": -2.39088773727417,
+      "step": 3740
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0142295897101032e-06,
+      "logits/chosen": 1.4703900814056396,
+      "logits/rejected": 2.453923225402832,
+      "logps/chosen": -434.49053955078125,
+      "logps/rejected": -498.53680419921875,
+      "loss": 0.3917,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.516781210899353,
+      "rewards/margins": 1.1974669694900513,
+      "rewards/rejected": -2.714247941970825,
+      "step": 3750
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0030478218516578e-06,
+      "logits/chosen": 1.7173267602920532,
+      "logits/rejected": 2.0685720443725586,
+      "logps/chosen": -421.38201904296875,
+      "logps/rejected": -496.91632080078125,
+      "loss": 0.4581,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.5563424825668335,
+      "rewards/margins": 1.0363523960113525,
+      "rewards/rejected": -2.5926952362060547,
+      "step": 3760
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.9918555559050826e-06,
+      "logits/chosen": 1.3105838298797607,
+      "logits/rejected": 1.7742143869400024,
+      "logps/chosen": -432.66461181640625,
+      "logps/rejected": -480.446533203125,
+      "loss": 0.5189,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.6999588012695312,
+      "rewards/margins": 0.8144930601119995,
+      "rewards/rejected": -2.5144519805908203,
+      "step": 3770
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.980653025441399e-06,
+      "logits/chosen": 1.4036800861358643,
+      "logits/rejected": 2.019293785095215,
+      "logps/chosen": -474.83746337890625,
+      "logps/rejected": -519.8162231445312,
+      "loss": 0.4627,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.664132833480835,
+      "rewards/margins": 0.94652259349823,
+      "rewards/rejected": -2.6106553077697754,
+      "step": 3780
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.969440464245841e-06,
+      "logits/chosen": 1.2756503820419312,
+      "logits/rejected": 1.8688790798187256,
+      "logps/chosen": -479.37823486328125,
+      "logps/rejected": -503.56903076171875,
+      "loss": 0.5606,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.8139938116073608,
+      "rewards/margins": 0.7126478552818298,
+      "rewards/rejected": -2.526641368865967,
+      "step": 3790
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.95821810631297e-06,
+      "logits/chosen": 1.6649503707885742,
+      "logits/rejected": 1.834276795387268,
+      "logps/chosen": -410.50286865234375,
+      "logps/rejected": -491.81109619140625,
+      "loss": 0.5202,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.8058888912200928,
+      "rewards/margins": 0.9338420629501343,
+      "rewards/rejected": -2.7397310733795166,
+      "step": 3800
+    },
+    {
+      "epoch": 0.5,
+      "eval_logits/chosen": 1.7444523572921753,
+      "eval_logits/rejected": 2.27280592918396,
+      "eval_logps/chosen": -451.4666748046875,
+      "eval_logps/rejected": -520.3418579101562,
+      "eval_loss": 0.5155265927314758,
+      "eval_rewards/accuracies": 0.7329999804496765,
+      "eval_rewards/chosen": -1.829070806503296,
+      "eval_rewards/margins": 0.8862784504890442,
+      "eval_rewards/rejected": -2.7153491973876953,
+      "eval_runtime": 1349.2282,
+      "eval_samples_per_second": 1.482,
+      "eval_steps_per_second": 0.185,
+      "step": 3800
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.946986185841801e-06,
+      "logits/chosen": 1.7209484577178955,
+      "logits/rejected": 2.219358444213867,
+      "logps/chosen": -450.63092041015625,
+      "logps/rejected": -509.802490234375,
+      "loss": 0.522,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.8785607814788818,
+      "rewards/margins": 0.7590249180793762,
+      "rewards/rejected": -2.6375856399536133,
+      "step": 3810
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.935744937230903e-06,
+      "logits/chosen": 1.6572405099868774,
+      "logits/rejected": 2.077040672302246,
+      "logps/chosen": -480.9547424316406,
+      "logps/rejected": -522.3167724609375,
+      "loss": 0.5795,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.8544946908950806,
+      "rewards/margins": 0.7630246877670288,
+      "rewards/rejected": -2.6175193786621094,
+      "step": 3820
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.924494595073517e-06,
+      "logits/chosen": 1.4180971384048462,
+      "logits/rejected": 1.8616039752960205,
+      "logps/chosen": -471.89190673828125,
+      "logps/rejected": -554.1817016601562,
+      "loss": 0.4458,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.9011123180389404,
+      "rewards/margins": 1.1739518642425537,
+      "rewards/rejected": -3.075064182281494,
+      "step": 3830
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9132353941526575e-06,
+      "logits/chosen": 1.3230631351470947,
+      "logits/rejected": 2.034977912902832,
+      "logps/chosen": -467.30029296875,
+      "logps/rejected": -552.8757934570312,
+      "loss": 0.5541,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.104778289794922,
+      "rewards/margins": 0.8113514184951782,
+      "rewards/rejected": -2.9161300659179688,
+      "step": 3840
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.901967569436209e-06,
+      "logits/chosen": 1.4525129795074463,
+      "logits/rejected": 2.0421319007873535,
+      "logps/chosen": -413.902587890625,
+      "logps/rejected": -490.19415283203125,
+      "loss": 0.4785,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.8785244226455688,
+      "rewards/margins": 0.9411863088607788,
+      "rewards/rejected": -2.8197109699249268,
+      "step": 3850
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.89069135607203e-06,
+      "logits/chosen": 1.417065978050232,
+      "logits/rejected": 1.6673557758331299,
+      "logps/chosen": -489.1497497558594,
+      "logps/rejected": -525.7388916015625,
+      "loss": 0.5459,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.0849897861480713,
+      "rewards/margins": 0.7166804075241089,
+      "rewards/rejected": -2.8016700744628906,
+      "step": 3860
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8794069893830386e-06,
+      "logits/chosen": 1.6072677373886108,
+      "logits/rejected": 2.1457881927490234,
+      "logps/chosen": -457.97259521484375,
+      "logps/rejected": -518.5877075195312,
+      "loss": 0.5641,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.092526435852051,
+      "rewards/margins": 0.6483314037322998,
+      "rewards/rejected": -2.7408576011657715,
+      "step": 3870
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8681147048623038e-06,
+      "logits/chosen": 1.3772077560424805,
+      "logits/rejected": 1.7642300128936768,
+      "logps/chosen": -481.44879150390625,
+      "logps/rejected": -572.1488037109375,
+      "loss": 0.484,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.8889554738998413,
+      "rewards/margins": 1.0506285429000854,
+      "rewards/rejected": -2.9395840167999268,
+      "step": 3880
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8568147381681333e-06,
+      "logits/chosen": 1.3481850624084473,
+      "logits/rejected": 1.8976227045059204,
+      "logps/chosen": -434.4345703125,
+      "logps/rejected": -537.3731689453125,
+      "loss": 0.4765,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.8758662939071655,
+      "rewards/margins": 1.0942527055740356,
+      "rewards/rejected": -2.970118999481201,
+      "step": 3890
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8455073251191533e-06,
+      "logits/chosen": 1.3988851308822632,
+      "logits/rejected": 1.8135325908660889,
+      "logps/chosen": -467.7666931152344,
+      "logps/rejected": -553.7589111328125,
+      "loss": 0.5116,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.0215046405792236,
+      "rewards/margins": 0.9531213641166687,
+      "rewards/rejected": -2.974626064300537,
+      "step": 3900
+    },
+    {
+      "epoch": 0.51,
+      "eval_logits/chosen": 1.7291210889816284,
+      "eval_logits/rejected": 2.2721054553985596,
+      "eval_logps/chosen": -475.875244140625,
+      "eval_logps/rejected": -553.0798950195312,
+      "eval_loss": 0.5187917351722717,
+      "eval_rewards/accuracies": 0.7285000085830688,
+      "eval_rewards/chosen": -2.0731570720672607,
+      "eval_rewards/margins": 0.9695719480514526,
+      "eval_rewards/rejected": -3.042728900909424,
+      "eval_runtime": 1348.9898,
+      "eval_samples_per_second": 1.483,
+      "eval_steps_per_second": 0.185,
+      "step": 3900
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8341927016893887e-06,
+      "logits/chosen": 1.312760591506958,
+      "logits/rejected": 1.7260150909423828,
+      "logps/chosen": -447.3658142089844,
+      "logps/rejected": -545.6050415039062,
+      "loss": 0.5319,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.1263701915740967,
+      "rewards/margins": 0.9401434659957886,
+      "rewards/rejected": -3.0665135383605957,
+      "step": 3910
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.822871104003335e-06,
+      "logits/chosen": 1.9234113693237305,
+      "logits/rejected": 1.956146001815796,
+      "logps/chosen": -423.244384765625,
+      "logps/rejected": -550.2393798828125,
+      "loss": 0.504,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.94733464717865,
+      "rewards/margins": 0.977364718914032,
+      "rewards/rejected": -2.924699306488037,
+      "step": 3920
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8115427683310355e-06,
+      "logits/chosen": 0.9371951818466187,
+      "logits/rejected": 1.8139762878417969,
+      "logps/chosen": -508.7724609375,
+      "logps/rejected": -579.633056640625,
+      "loss": 0.4478,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.050142765045166,
+      "rewards/margins": 1.0734608173370361,
+      "rewards/rejected": -3.123603343963623,
+      "step": 3930
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.8002079310831477e-06,
+      "logits/chosen": 1.5121482610702515,
+      "logits/rejected": 2.12965726852417,
+      "logps/chosen": -473.70233154296875,
+      "logps/rejected": -540.3966064453125,
+      "loss": 0.609,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.293128490447998,
+      "rewards/margins": 0.6268256306648254,
+      "rewards/rejected": -2.9199540615081787,
+      "step": 3940
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7888668288060095e-06,
+      "logits/chosen": 1.356512427330017,
+      "logits/rejected": 2.4057295322418213,
+      "logps/chosen": -509.6851501464844,
+      "logps/rejected": -612.4816284179688,
+      "loss": 0.3855,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.204204559326172,
+      "rewards/margins": 1.3057889938354492,
+      "rewards/rejected": -3.5099937915802,
+      "step": 3950
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7775196981767044e-06,
+      "logits/chosen": 1.84615957736969,
+      "logits/rejected": 2.5423619747161865,
+      "logps/chosen": -486.559326171875,
+      "logps/rejected": -608.5115966796875,
+      "loss": 0.5297,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.4462332725524902,
+      "rewards/margins": 1.1890199184417725,
+      "rewards/rejected": -3.6352531909942627,
+      "step": 3960
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7661667759981213e-06,
+      "logits/chosen": 1.7176733016967773,
+      "logits/rejected": 2.0130772590637207,
+      "logps/chosen": -488.48150634765625,
+      "logps/rejected": -600.0150146484375,
+      "loss": 0.491,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.572052001953125,
+      "rewards/margins": 1.085648775100708,
+      "rewards/rejected": -3.657701015472412,
+      "step": 3970
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7548082991940137e-06,
+      "logits/chosen": 1.509436845779419,
+      "logits/rejected": 2.070053815841675,
+      "logps/chosen": -528.0103149414062,
+      "logps/rejected": -597.5302734375,
+      "loss": 0.671,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.6207382678985596,
+      "rewards/margins": 0.7578615546226501,
+      "rewards/rejected": -3.3785996437072754,
+      "step": 3980
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.743444504804051e-06,
+      "logits/chosen": 1.9147052764892578,
+      "logits/rejected": 2.2859673500061035,
+      "logps/chosen": -453.40533447265625,
+      "logps/rejected": -522.7704467773438,
+      "loss": 0.5936,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.4376797676086426,
+      "rewards/margins": 0.7642915844917297,
+      "rewards/rejected": -3.2019710540771484,
+      "step": 3990
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7320756299788788e-06,
+      "logits/chosen": 1.5613954067230225,
+      "logits/rejected": 2.2871968746185303,
+      "logps/chosen": -482.3287658691406,
+      "logps/rejected": -524.7835083007812,
+      "loss": 0.5521,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.094331979751587,
+      "rewards/margins": 0.6762000322341919,
+      "rewards/rejected": -2.7705318927764893,
+      "step": 4000
+    },
+    {
+      "epoch": 0.52,
+      "eval_logits/chosen": 1.7449135780334473,
+      "eval_logits/rejected": 2.2576889991760254,
+      "eval_logps/chosen": -501.3871765136719,
+      "eval_logps/rejected": -579.346923828125,
+      "eval_loss": 0.5161105990409851,
+      "eval_rewards/accuracies": 0.7254999876022339,
+      "eval_rewards/chosen": -2.3282761573791504,
+      "eval_rewards/margins": 0.9771235585212708,
+      "eval_rewards/rejected": -3.3053994178771973,
+      "eval_runtime": 1349.5569,
+      "eval_samples_per_second": 1.482,
+      "eval_steps_per_second": 0.185,
+      "step": 4000
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7207019119751644e-06,
+      "logits/chosen": 1.232839584350586,
+      "logits/rejected": 2.0548617839813232,
+      "logps/chosen": -481.10400390625,
+      "logps/rejected": -539.7461547851562,
+      "loss": 0.4746,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.39994215965271,
+      "rewards/margins": 1.0269447565078735,
+      "rewards/rejected": -3.426886796951294,
+      "step": 4010
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.7093235881506474e-06,
+      "logits/chosen": 1.112256407737732,
+      "logits/rejected": 1.9274839162826538,
+      "logps/chosen": -474.70172119140625,
+      "logps/rejected": -589.70556640625,
+      "loss": 0.4242,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.9605392217636108,
+      "rewards/margins": 1.3502711057662964,
+      "rewards/rejected": -3.310810089111328,
+      "step": 4020
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6979408959591863e-06,
+      "logits/chosen": 1.0921329259872437,
+      "logits/rejected": 1.2823255062103271,
+      "logps/chosen": -421.07403564453125,
+      "logps/rejected": -510.64404296875,
+      "loss": 0.5436,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.795819640159607,
+      "rewards/margins": 0.8993387222290039,
+      "rewards/rejected": -2.6951584815979004,
+      "step": 4030
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6865540729458034e-06,
+      "logits/chosen": 1.1424638032913208,
+      "logits/rejected": 1.946634292602539,
+      "logps/chosen": -468.6031799316406,
+      "logps/rejected": -576.2097778320312,
+      "loss": 0.4296,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.8103950023651123,
+      "rewards/margins": 1.2279468774795532,
+      "rewards/rejected": -3.038341999053955,
+      "step": 4040
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.675163356741726e-06,
+      "logits/chosen": 1.128174901008606,
+      "logits/rejected": 1.841668725013733,
+      "logps/chosen": -421.0733947753906,
+      "logps/rejected": -531.6262817382812,
+      "loss": 0.5032,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.7898073196411133,
+      "rewards/margins": 1.0664771795272827,
+      "rewards/rejected": -2.8562843799591064,
+      "step": 4050
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6637689850594285e-06,
+      "logits/chosen": 1.1155699491500854,
+      "logits/rejected": 1.4345989227294922,
+      "logps/chosen": -450.415283203125,
+      "logps/rejected": -518.7340698242188,
+      "loss": 0.6349,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.0943007469177246,
+      "rewards/margins": 0.5381532907485962,
+      "rewards/rejected": -2.6324543952941895,
+      "step": 4060
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.652371195687671e-06,
+      "logits/chosen": 0.9411834478378296,
+      "logits/rejected": 1.5208265781402588,
+      "logps/chosen": -425.16650390625,
+      "logps/rejected": -544.6481323242188,
+      "loss": 0.4471,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.621567964553833,
+      "rewards/margins": 1.246022343635559,
+      "rewards/rejected": -2.8675904273986816,
+      "step": 4070
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.64097022648654e-06,
+      "logits/chosen": 0.5625711679458618,
+      "logits/rejected": 1.5168582201004028,
+      "logps/chosen": -456.169677734375,
+      "logps/rejected": -508.9212341308594,
+      "loss": 0.4456,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.599456548690796,
+      "rewards/margins": 1.0814018249511719,
+      "rewards/rejected": -2.6808583736419678,
+      "step": 4080
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.6295663153824774e-06,
+      "logits/chosen": 0.4188370108604431,
+      "logits/rejected": 1.2125314474105835,
+      "logps/chosen": -451.51495361328125,
+      "logps/rejected": -488.12615966796875,
+      "loss": 0.468,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.586099624633789,
+      "rewards/margins": 1.0054364204406738,
+      "rewards/rejected": -2.591536045074463,
+      "step": 4090
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.6181597003633218e-06,
+      "logits/chosen": 0.9930766820907593,
+      "logits/rejected": 1.5937138795852661,
+      "logps/chosen": -424.79248046875,
+      "logps/rejected": -501.5884704589844,
+      "loss": 0.5107,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.615247130393982,
+      "rewards/margins": 0.9743593335151672,
+      "rewards/rejected": -2.589606523513794,
+      "step": 4100
+    },
+    {
+      "epoch": 0.54,
+      "eval_logits/chosen": 1.222239375114441,
+      "eval_logits/rejected": 1.7677592039108276,
+      "eval_logps/chosen": -450.4802551269531,
+      "eval_logps/rejected": -522.2897338867188,
+      "eval_loss": 0.5196803212165833,
+      "eval_rewards/accuracies": 0.7214999794960022,
+      "eval_rewards/chosen": -1.8192068338394165,
+      "eval_rewards/margins": 0.9156206250190735,
+      "eval_rewards/rejected": -2.7348272800445557,
+      "eval_runtime": 1350.6097,
+      "eval_samples_per_second": 1.481,
+      "eval_steps_per_second": 0.185,
+      "step": 4100
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.606750619473342e-06,
+      "logits/chosen": 1.3754876852035522,
+      "logits/rejected": 1.7925678491592407,
+      "logps/chosen": -423.62542724609375,
+      "logps/rejected": -550.4591674804688,
+      "loss": 0.4468,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.809665322303772,
+      "rewards/margins": 1.1548588275909424,
+      "rewards/rejected": -2.964524269104004,
+      "step": 4110
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.595339310808262e-06,
+      "logits/chosen": 0.9116421937942505,
+      "logits/rejected": 1.1701269149780273,
+      "logps/chosen": -409.7557067871094,
+      "logps/rejected": -518.2631225585938,
+      "loss": 0.4888,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.672642469406128,
+      "rewards/margins": 1.0149104595184326,
+      "rewards/rejected": -2.6875529289245605,
+      "step": 4120
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5839260125103004e-06,
+      "logits/chosen": 1.3330891132354736,
+      "logits/rejected": 2.106353521347046,
+      "logps/chosen": -467.6688537597656,
+      "logps/rejected": -549.4391479492188,
+      "loss": 0.5469,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.174787759780884,
+      "rewards/margins": 0.9655358195304871,
+      "rewards/rejected": -3.1403234004974365,
+      "step": 4130
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5725109627631984e-06,
+      "logits/chosen": 1.2201000452041626,
+      "logits/rejected": 1.6247581243515015,
+      "logps/chosen": -481.80352783203125,
+      "logps/rejected": -510.554443359375,
+      "loss": 0.5964,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.0002753734588623,
+      "rewards/margins": 0.8849322199821472,
+      "rewards/rejected": -2.8852076530456543,
+      "step": 4140
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5610943997872443e-06,
+      "logits/chosen": 1.1533983945846558,
+      "logits/rejected": 1.6274654865264893,
+      "logps/chosen": -457.3433532714844,
+      "logps/rejected": -532.3827514648438,
+      "loss": 0.516,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.0298244953155518,
+      "rewards/margins": 0.8265906572341919,
+      "rewards/rejected": -2.856415271759033,
+      "step": 4150
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5496765618343096e-06,
+      "logits/chosen": 1.0701457262039185,
+      "logits/rejected": 1.001036524772644,
+      "logps/chosen": -371.4546203613281,
+      "logps/rejected": -496.13470458984375,
+      "loss": 0.5111,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.5883476734161377,
+      "rewards/margins": 0.8016815185546875,
+      "rewards/rejected": -2.3900294303894043,
+      "step": 4160
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.538257687182871e-06,
+      "logits/chosen": 0.9576324224472046,
+      "logits/rejected": 1.3845566511154175,
+      "logps/chosen": -473.50579833984375,
+      "logps/rejected": -560.6783447265625,
+      "loss": 0.5811,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.0806198120117188,
+      "rewards/margins": 0.9803606867790222,
+      "rewards/rejected": -3.0609803199768066,
+      "step": 4170
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.526838014133041e-06,
+      "logits/chosen": 0.6895111799240112,
+      "logits/rejected": 1.5417369604110718,
+      "logps/chosen": -458.15240478515625,
+      "logps/rejected": -512.6672973632812,
+      "loss": 0.5718,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.0201404094696045,
+      "rewards/margins": 0.7214975357055664,
+      "rewards/rejected": -2.741637706756592,
+      "step": 4180
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.515417781001594e-06,
+      "logits/chosen": 1.0739127397537231,
+      "logits/rejected": 1.7107112407684326,
+      "logps/chosen": -459.8623962402344,
+      "logps/rejected": -530.8646240234375,
+      "loss": 0.473,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.962693452835083,
+      "rewards/margins": 1.135125756263733,
+      "rewards/rejected": -3.0978188514709473,
+      "step": 4190
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.503997226116992e-06,
+      "logits/chosen": 1.1395686864852905,
+      "logits/rejected": 1.460407018661499,
+      "logps/chosen": -451.47607421875,
+      "logps/rejected": -561.359130859375,
+      "loss": 0.4773,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.008143186569214,
+      "rewards/margins": 1.006184458732605,
+      "rewards/rejected": -3.0143282413482666,
+      "step": 4200
+    },
+    {
+      "epoch": 0.55,
+      "eval_logits/chosen": 1.3121181726455688,
+      "eval_logits/rejected": 1.8497090339660645,
+      "eval_logps/chosen": -487.4992370605469,
+      "eval_logps/rejected": -564.3450927734375,
+      "eval_loss": 0.5163218975067139,
+      "eval_rewards/accuracies": 0.7264999747276306,
+      "eval_rewards/chosen": -2.189396619796753,
+      "eval_rewards/margins": 0.9659839272499084,
+      "eval_rewards/rejected": -3.1553807258605957,
+      "eval_runtime": 1350.2389,
+      "eval_samples_per_second": 1.481,
+      "eval_steps_per_second": 0.185,
+      "step": 4200
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4925765878144115e-06,
+      "logits/chosen": 1.1307320594787598,
+      "logits/rejected": 1.7652254104614258,
+      "logps/chosen": -457.52520751953125,
+      "logps/rejected": -570.0340576171875,
+      "loss": 0.4339,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.1189956665039062,
+      "rewards/margins": 1.274686574935913,
+      "rewards/rejected": -3.3936820030212402,
+      "step": 4210
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4811561044307727e-06,
+      "logits/chosen": 1.369770884513855,
+      "logits/rejected": 1.8304027318954468,
+      "logps/chosen": -449.43701171875,
+      "logps/rejected": -611.7244873046875,
+      "loss": 0.3777,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.1294121742248535,
+      "rewards/margins": 1.5083189010620117,
+      "rewards/rejected": -3.637730836868286,
+      "step": 4220
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.469736014299758e-06,
+      "logits/chosen": 1.0193202495574951,
+      "logits/rejected": 1.568139910697937,
+      "logps/chosen": -519.9937133789062,
+      "logps/rejected": -601.6876831054688,
+      "loss": 0.5344,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.376950740814209,
+      "rewards/margins": 0.8890069127082825,
+      "rewards/rejected": -3.2659575939178467,
+      "step": 4230
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.458316555746846e-06,
+      "logits/chosen": 1.0642564296722412,
+      "logits/rejected": 1.5705610513687134,
+      "logps/chosen": -572.1160888671875,
+      "logps/rejected": -644.0897216796875,
+      "loss": 0.4339,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.4768519401550293,
+      "rewards/margins": 1.1803462505340576,
+      "rewards/rejected": -3.657198667526245,
+      "step": 4240
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.446897967084334e-06,
+      "logits/chosen": 1.552398920059204,
+      "logits/rejected": 2.16156268119812,
+      "logps/chosen": -575.1290283203125,
+      "logps/rejected": -632.1162109375,
+      "loss": 0.4945,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.7811026573181152,
+      "rewards/margins": 1.2048923969268799,
+      "rewards/rejected": -3.985995054244995,
+      "step": 4250
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4354804866063684e-06,
+      "logits/chosen": 1.3526991605758667,
+      "logits/rejected": 2.040238380432129,
+      "logps/chosen": -528.0109252929688,
+      "logps/rejected": -637.324462890625,
+      "loss": 0.4597,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.5519847869873047,
+      "rewards/margins": 1.2211592197418213,
+      "rewards/rejected": -3.7731432914733887,
+      "step": 4260
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.424064352583964e-06,
+      "logits/chosen": 1.4652588367462158,
+      "logits/rejected": 2.1928470134735107,
+      "logps/chosen": -517.3566284179688,
+      "logps/rejected": -598.0098876953125,
+      "loss": 0.4721,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.553633451461792,
+      "rewards/margins": 0.9991529583930969,
+      "rewards/rejected": -3.552786350250244,
+      "step": 4270
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4126498032600403e-06,
+      "logits/chosen": 1.160603404045105,
+      "logits/rejected": 1.4777276515960693,
+      "logps/chosen": -516.5755615234375,
+      "logps/rejected": -612.9207763671875,
+      "loss": 0.5601,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.413853168487549,
+      "rewards/margins": 0.8914909362792969,
+      "rewards/rejected": -3.3053441047668457,
+      "step": 4280
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.401237076844445e-06,
+      "logits/chosen": 1.891185998916626,
+      "logits/rejected": 2.358264446258545,
+      "logps/chosen": -486.514892578125,
+      "logps/rejected": -516.6591796875,
+      "loss": 0.6073,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.458522319793701,
+      "rewards/margins": 0.6096151471138,
+      "rewards/rejected": -3.0681376457214355,
+      "step": 4290
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.38982641150898e-06,
+      "logits/chosen": 1.6550731658935547,
+      "logits/rejected": 2.238009214401245,
+      "logps/chosen": -456.22088623046875,
+      "logps/rejected": -543.3316650390625,
+      "loss": 0.4315,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.0792737007141113,
+      "rewards/margins": 1.1827647686004639,
+      "rewards/rejected": -3.262038469314575,
+      "step": 4300
+    },
+    {
+      "epoch": 0.56,
+      "eval_logits/chosen": 1.6783313751220703,
+      "eval_logits/rejected": 2.203875780105591,
+      "eval_logps/chosen": -477.2872009277344,
+      "eval_logps/rejected": -552.970458984375,
+      "eval_loss": 0.5096888542175293,
+      "eval_rewards/accuracies": 0.734000027179718,
+      "eval_rewards/chosen": -2.0872764587402344,
+      "eval_rewards/margins": 0.9543583393096924,
+      "eval_rewards/rejected": -3.0416345596313477,
+      "eval_runtime": 1348.0916,
+      "eval_samples_per_second": 1.484,
+      "eval_steps_per_second": 0.185,
+      "step": 4300
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.3784180453824414e-06,
+      "logits/chosen": 1.2046362161636353,
+      "logits/rejected": 1.6807031631469727,
+      "logps/chosen": -461.3589782714844,
+      "logps/rejected": -523.1180419921875,
+      "loss": 0.5534,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.1297290325164795,
+      "rewards/margins": 0.7883184552192688,
+      "rewards/rejected": -2.9180474281311035,
+      "step": 4310
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.367012216545638e-06,
+      "logits/chosen": 1.3626408576965332,
+      "logits/rejected": 1.9960730075836182,
+      "logps/chosen": -444.55078125,
+      "logps/rejected": -508.3905334472656,
+      "loss": 0.5068,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.9940459728240967,
+      "rewards/margins": 0.8987429738044739,
+      "rewards/rejected": -2.892788887023926,
+      "step": 4320
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3556091630264294e-06,
+      "logits/chosen": 1.07587730884552,
+      "logits/rejected": 1.8802881240844727,
+      "logps/chosen": -454.011962890625,
+      "logps/rejected": -489.26605224609375,
+      "loss": 0.5926,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.9290491342544556,
+      "rewards/margins": 0.7649563550949097,
+      "rewards/rejected": -2.6940054893493652,
+      "step": 4330
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.344209122794757e-06,
+      "logits/chosen": 1.2137640714645386,
+      "logits/rejected": 1.4355902671813965,
+      "logps/chosen": -473.57659912109375,
+      "logps/rejected": -570.0810546875,
+      "loss": 0.4937,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.094072103500366,
+      "rewards/margins": 0.9700179100036621,
+      "rewards/rejected": -3.0640902519226074,
+      "step": 4340
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3328123337576787e-06,
+      "logits/chosen": 1.3968803882598877,
+      "logits/rejected": 1.3642232418060303,
+      "logps/chosen": -428.2279357910156,
+      "logps/rejected": -581.060791015625,
+      "loss": 0.4259,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.9399982690811157,
+      "rewards/margins": 1.2616885900497437,
+      "rewards/rejected": -3.2016868591308594,
+      "step": 4350
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3214190337544017e-06,
+      "logits/chosen": 0.9632726907730103,
+      "logits/rejected": 1.7064374685287476,
+      "logps/chosen": -462.50347900390625,
+      "logps/rejected": -555.4581298828125,
+      "loss": 0.3957,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.066453695297241,
+      "rewards/margins": 1.118513822555542,
+      "rewards/rejected": -3.1849677562713623,
+      "step": 4360
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.310029460551323e-06,
+      "logits/chosen": 1.1379252672195435,
+      "logits/rejected": 2.3016974925994873,
+      "logps/chosen": -516.5641479492188,
+      "logps/rejected": -568.5375366210938,
+      "loss": 0.4628,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.202714681625366,
+      "rewards/margins": 1.1053831577301025,
+      "rewards/rejected": -3.3080978393554688,
+      "step": 4370
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.2986438518370645e-06,
+      "logits/chosen": 1.0676512718200684,
+      "logits/rejected": 2.015590190887451,
+      "logps/chosen": -499.21173095703125,
+      "logps/rejected": -600.5050048828125,
+      "loss": 0.494,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.4465959072113037,
+      "rewards/margins": 1.1969095468521118,
+      "rewards/rejected": -3.643505573272705,
+      "step": 4380
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.2872624452175123e-06,
+      "logits/chosen": 1.6105989217758179,
+      "logits/rejected": 2.049776792526245,
+      "logps/chosen": -531.1690063476562,
+      "logps/rejected": -580.7404174804688,
+      "loss": 0.5167,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.5410590171813965,
+      "rewards/margins": 0.8904156684875488,
+      "rewards/rejected": -3.4314746856689453,
+      "step": 4390
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2758854782108584e-06,
+      "logits/chosen": 1.2695518732070923,
+      "logits/rejected": 1.7854732275009155,
+      "logps/chosen": -547.46240234375,
+      "logps/rejected": -621.9617309570312,
+      "loss": 0.5176,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.5525646209716797,
+      "rewards/margins": 0.9455914497375488,
+      "rewards/rejected": -3.4981560707092285,
+      "step": 4400
+    },
+    {
+      "epoch": 0.58,
+      "eval_logits/chosen": 1.5387457609176636,
+      "eval_logits/rejected": 2.078198194503784,
+      "eval_logps/chosen": -493.41455078125,
+      "eval_logps/rejected": -572.8978881835938,
+      "eval_loss": 0.5096824765205383,
+      "eval_rewards/accuracies": 0.7289999723434448,
+      "eval_rewards/chosen": -2.2485501766204834,
+      "eval_rewards/margins": 0.9923587441444397,
+      "eval_rewards/rejected": -3.2409090995788574,
+      "eval_runtime": 1347.0607,
+      "eval_samples_per_second": 1.485,
+      "eval_steps_per_second": 0.186,
+      "step": 4400
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2645131882426458e-06,
+      "logits/chosen": 1.236730933189392,
+      "logits/rejected": 1.5685746669769287,
+      "logps/chosen": -470.15594482421875,
+      "logps/rejected": -549.2040405273438,
+      "loss": 0.5585,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.0850582122802734,
+      "rewards/margins": 0.8133324384689331,
+      "rewards/rejected": -2.898390293121338,
+      "step": 4410
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2531458126408154e-06,
+      "logits/chosen": 1.2222044467926025,
+      "logits/rejected": 1.8391752243041992,
+      "logps/chosen": -466.7596130371094,
+      "logps/rejected": -521.5641479492188,
+      "loss": 0.5527,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.0936551094055176,
+      "rewards/margins": 0.8788488507270813,
+      "rewards/rejected": -2.972504138946533,
+      "step": 4420
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2417835886307452e-06,
+      "logits/chosen": 0.949698805809021,
+      "logits/rejected": 1.535921335220337,
+      "logps/chosen": -448.20867919921875,
+      "logps/rejected": -526.8872680664062,
+      "loss": 0.4481,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.9574615955352783,
+      "rewards/margins": 1.0204461812973022,
+      "rewards/rejected": -2.977907657623291,
+      "step": 4430
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2304267533303075e-06,
+      "logits/chosen": 1.0622180700302124,
+      "logits/rejected": 1.7385826110839844,
+      "logps/chosen": -492.62493896484375,
+      "logps/rejected": -558.9925537109375,
+      "loss": 0.4546,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.048485279083252,
+      "rewards/margins": 1.0082876682281494,
+      "rewards/rejected": -3.0567729473114014,
+      "step": 4440
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.219075543744918e-06,
+      "logits/chosen": 0.8114604949951172,
+      "logits/rejected": 1.5278441905975342,
+      "logps/chosen": -495.5421447753906,
+      "logps/rejected": -542.13818359375,
+      "loss": 0.5969,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.2880332469940186,
+      "rewards/margins": 0.7602779269218445,
+      "rewards/rejected": -3.048311710357666,
+      "step": 4450
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.207730196762589e-06,
+      "logits/chosen": 0.9587013125419617,
+      "logits/rejected": 1.969460129737854,
+      "logps/chosen": -498.8384704589844,
+      "logps/rejected": -572.7350463867188,
+      "loss": 0.4433,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.1771950721740723,
+      "rewards/margins": 1.1226615905761719,
+      "rewards/rejected": -3.299856662750244,
+      "step": 4460
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.1963909491489846e-06,
+      "logits/chosen": 1.2846577167510986,
+      "logits/rejected": 1.7563025951385498,
+      "logps/chosen": -474.389404296875,
+      "logps/rejected": -573.5692138671875,
+      "loss": 0.5201,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.9463615417480469,
+      "rewards/margins": 0.9242898225784302,
+      "rewards/rejected": -2.8706517219543457,
+      "step": 4470
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.185058037542486e-06,
+      "logits/chosen": 0.9459007382392883,
+      "logits/rejected": 1.6969045400619507,
+      "logps/chosen": -488.2107849121094,
+      "logps/rejected": -543.5244140625,
+      "loss": 0.4907,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.1091866493225098,
+      "rewards/margins": 0.8848414421081543,
+      "rewards/rejected": -2.994028091430664,
+      "step": 4480
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.173731698449244e-06,
+      "logits/chosen": 0.8182988166809082,
+      "logits/rejected": 1.281505823135376,
+      "logps/chosen": -446.75701904296875,
+      "logps/rejected": -551.4588623046875,
+      "loss": 0.4752,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.910454511642456,
+      "rewards/margins": 1.078656792640686,
+      "rewards/rejected": -2.9891109466552734,
+      "step": 4490
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1624121682382495e-06,
+      "logits/chosen": 1.0567976236343384,
+      "logits/rejected": 2.1221261024475098,
+      "logps/chosen": -445.9683532714844,
+      "logps/rejected": -516.5798950195312,
+      "loss": 0.4487,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.9384996891021729,
+      "rewards/margins": 1.0074058771133423,
+      "rewards/rejected": -2.9459054470062256,
+      "step": 4500
+    },
+    {
+      "epoch": 0.59,
+      "eval_logits/chosen": 1.4967700242996216,
+      "eval_logits/rejected": 2.0676395893096924,
+      "eval_logps/chosen": -471.12823486328125,
+      "eval_logps/rejected": -550.2474975585938,
+      "eval_loss": 0.5131733417510986,
+      "eval_rewards/accuracies": 0.7245000004768372,
+      "eval_rewards/chosen": -2.025686502456665,
+      "eval_rewards/margins": 0.9887187480926514,
+      "eval_rewards/rejected": -3.0144052505493164,
+      "eval_runtime": 1350.621,
+      "eval_samples_per_second": 1.481,
+      "eval_steps_per_second": 0.185,
+      "step": 4500
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1510996831363993e-06,
+      "logits/chosen": 1.1638802289962769,
+      "logits/rejected": 2.1318185329437256,
+      "logps/chosen": -460.8349609375,
+      "logps/rejected": -506.89111328125,
+      "loss": 0.5306,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.097872495651245,
+      "rewards/margins": 0.9646193385124207,
+      "rewards/rejected": -3.0624918937683105,
+      "step": 4510
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.139794479223565e-06,
+      "logits/chosen": 1.5130078792572021,
+      "logits/rejected": 1.864434003829956,
+      "logps/chosen": -514.8096313476562,
+      "logps/rejected": -561.0484619140625,
+      "loss": 0.5475,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.156911849975586,
+      "rewards/margins": 0.9767072796821594,
+      "rewards/rejected": -3.1336190700531006,
+      "step": 4520
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.128496792427669e-06,
+      "logits/chosen": 1.3294789791107178,
+      "logits/rejected": 1.437835693359375,
+      "logps/chosen": -489.97064208984375,
+      "logps/rejected": -579.2623291015625,
+      "loss": 0.5147,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.0266828536987305,
+      "rewards/margins": 0.8329746127128601,
+      "rewards/rejected": -2.8596572875976562,
+      "step": 4530
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.117206858519758e-06,
+      "logits/chosen": 1.2132080793380737,
+      "logits/rejected": 1.2598590850830078,
+      "logps/chosen": -435.6639099121094,
+      "logps/rejected": -531.2625732421875,
+      "loss": 0.5979,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.9387620687484741,
+      "rewards/margins": 0.6938289403915405,
+      "rewards/rejected": -2.6325910091400146,
+      "step": 4540
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.1059249131090844e-06,
+      "logits/chosen": 1.2094806432724,
+      "logits/rejected": 1.8091529607772827,
+      "logps/chosen": -415.89697265625,
+      "logps/rejected": -470.177001953125,
+      "loss": 0.6029,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.9523341655731201,
+      "rewards/margins": 0.8241564035415649,
+      "rewards/rejected": -2.7764906883239746,
+      "step": 4550
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.094651191638189e-06,
+      "logits/chosen": 1.0723166465759277,
+      "logits/rejected": 1.5394939184188843,
+      "logps/chosen": -440.8968811035156,
+      "logps/rejected": -482.54254150390625,
+      "loss": 0.5652,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.8494970798492432,
+      "rewards/margins": 0.7115729451179504,
+      "rewards/rejected": -2.561069965362549,
+      "step": 4560
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0833859293779867e-06,
+      "logits/chosen": 0.9671816825866699,
+      "logits/rejected": 1.646967887878418,
+      "logps/chosen": -443.7911071777344,
+      "logps/rejected": -526.4573974609375,
+      "loss": 0.5237,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.78009033203125,
+      "rewards/margins": 0.9070240259170532,
+      "rewards/rejected": -2.6871142387390137,
+      "step": 4570
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0721293614228568e-06,
+      "logits/chosen": 1.1051685810089111,
+      "logits/rejected": 1.5212745666503906,
+      "logps/chosen": -419.5252990722656,
+      "logps/rejected": -555.3690795898438,
+      "loss": 0.503,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.8462114334106445,
+      "rewards/margins": 0.9561260938644409,
+      "rewards/rejected": -2.802337646484375,
+      "step": 4580
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.060881722685742e-06,
+      "logits/chosen": 1.3264110088348389,
+      "logits/rejected": 1.3881210088729858,
+      "logps/chosen": -455.6913146972656,
+      "logps/rejected": -559.4048461914062,
+      "loss": 0.4983,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.056220293045044,
+      "rewards/margins": 0.9094651341438293,
+      "rewards/rejected": -2.9656851291656494,
+      "step": 4590
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.049643247893235e-06,
+      "logits/chosen": 1.3746750354766846,
+      "logits/rejected": 1.7783405780792236,
+      "logps/chosen": -478.1459045410156,
+      "logps/rejected": -558.3126831054688,
+      "loss": 0.478,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.16377592086792,
+      "rewards/margins": 1.0386617183685303,
+      "rewards/rejected": -3.20243763923645,
+      "step": 4600
+    },
+    {
+      "epoch": 0.6,
+      "eval_logits/chosen": 1.5401781797409058,
+      "eval_logits/rejected": 2.1064529418945312,
+      "eval_logps/chosen": -474.2083740234375,
+      "eval_logps/rejected": -552.2376098632812,
+      "eval_loss": 0.5082429051399231,
+      "eval_rewards/accuracies": 0.7269999980926514,
+      "eval_rewards/chosen": -2.056487798690796,
+      "eval_rewards/margins": 0.9778187870979309,
+      "eval_rewards/rejected": -3.034306526184082,
+      "eval_runtime": 1350.6659,
+      "eval_samples_per_second": 1.481,
+      "eval_steps_per_second": 0.185,
+      "step": 4600
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0384141715806903e-06,
+      "logits/chosen": 1.2565066814422607,
+      "logits/rejected": 1.5821117162704468,
+      "logps/chosen": -511.9517517089844,
+      "logps/rejected": -568.5081176757812,
+      "loss": 0.5461,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.137535572052002,
+      "rewards/margins": 0.8168964385986328,
+      "rewards/rejected": -2.954432249069214,
+      "step": 4610
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0271947280873255e-06,
+      "logits/chosen": 0.8720534443855286,
+      "logits/rejected": 1.5173699855804443,
+      "logps/chosen": -443.9607849121094,
+      "logps/rejected": -526.7023315429688,
+      "loss": 0.4874,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.851604700088501,
+      "rewards/margins": 1.0082629919052124,
+      "rewards/rejected": -2.859867572784424,
+      "step": 4620
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.0159851515513302e-06,
+      "logits/chosen": 1.6047923564910889,
+      "logits/rejected": 1.7259162664413452,
+      "logps/chosen": -393.366943359375,
+      "logps/rejected": -511.61383056640625,
+      "loss": 0.5197,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7584097385406494,
+      "rewards/margins": 0.9344598650932312,
+      "rewards/rejected": -2.6928694248199463,
+      "step": 4630
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.004785675904982e-06,
+      "logits/chosen": 1.185978651046753,
+      "logits/rejected": 1.4965678453445435,
+      "logps/chosen": -467.0824279785156,
+      "logps/rejected": -558.9572143554688,
+      "loss": 0.4999,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.8907064199447632,
+      "rewards/margins": 1.0817322731018066,
+      "rewards/rejected": -2.9724388122558594,
+      "step": 4640
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9935965348697624e-06,
+      "logits/chosen": 0.9723242521286011,
+      "logits/rejected": 1.8149017095565796,
+      "logps/chosen": -454.5943298339844,
+      "logps/rejected": -485.91461181640625,
+      "loss": 0.4961,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.7425225973129272,
+      "rewards/margins": 0.9134918451309204,
+      "rewards/rejected": -2.656014919281006,
+      "step": 4650
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9824179619514807e-06,
+      "logits/chosen": 1.2578637599945068,
+      "logits/rejected": 1.6150343418121338,
+      "logps/chosen": -485.50341796875,
+      "logps/rejected": -574.8892822265625,
+      "loss": 0.5788,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.063918352127075,
+      "rewards/margins": 0.8207401037216187,
+      "rewards/rejected": -2.8846583366394043,
+      "step": 4660
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9712501904354004e-06,
+      "logits/chosen": 1.263235330581665,
+      "logits/rejected": 1.5865004062652588,
+      "logps/chosen": -438.1275329589844,
+      "logps/rejected": -540.5255737304688,
+      "loss": 0.4526,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8934905529022217,
+      "rewards/margins": 1.071181058883667,
+      "rewards/rejected": -2.9646716117858887,
+      "step": 4670
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.960093453381369e-06,
+      "logits/chosen": 1.4069606065750122,
+      "logits/rejected": 1.65280282497406,
+      "logps/chosen": -462.9320373535156,
+      "logps/rejected": -538.8988037109375,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.1185081005096436,
+      "rewards/margins": 0.6407272815704346,
+      "rewards/rejected": -2.759235382080078,
+      "step": 4680
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.948947983618962e-06,
+      "logits/chosen": 0.9151169061660767,
+      "logits/rejected": 1.5992482900619507,
+      "logps/chosen": -484.33721923828125,
+      "logps/rejected": -541.8434448242188,
+      "loss": 0.4845,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.016294002532959,
+      "rewards/margins": 0.9646770358085632,
+      "rewards/rejected": -2.980971336364746,
+      "step": 4690
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.937814013742611e-06,
+      "logits/chosen": 1.5325231552124023,
+      "logits/rejected": 1.9264907836914062,
+      "logps/chosen": -430.5271911621094,
+      "logps/rejected": -515.6851806640625,
+      "loss": 0.5351,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.9069957733154297,
+      "rewards/margins": 0.817613422870636,
+      "rewards/rejected": -2.724608898162842,
+      "step": 4700
+    },
+    {
+      "epoch": 0.62,
+      "eval_logits/chosen": 1.5017454624176025,
+      "eval_logits/rejected": 2.0488224029541016,
+      "eval_logps/chosen": -464.8120422363281,
+      "eval_logps/rejected": -538.7389526367188,
+      "eval_loss": 0.5038378238677979,
+      "eval_rewards/accuracies": 0.7285000085830688,
+      "eval_rewards/chosen": -1.9625245332717896,
+      "eval_rewards/margins": 0.9367952942848206,
+      "eval_rewards/rejected": -2.899319887161255,
+      "eval_runtime": 1347.9299,
+      "eval_samples_per_second": 1.484,
+      "eval_steps_per_second": 0.185,
+      "step": 4700
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9266917761067617e-06,
+      "logits/chosen": 1.3719656467437744,
+      "logits/rejected": 2.0552589893341064,
+      "logps/chosen": -413.7361755371094,
+      "logps/rejected": -543.5679321289062,
+      "loss": 0.383,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.7675737142562866,
+      "rewards/margins": 1.3390953540802002,
+      "rewards/rejected": -3.1066689491271973,
+      "step": 4710
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.915581502821017e-06,
+      "logits/chosen": 1.5292431116104126,
+      "logits/rejected": 1.7938716411590576,
+      "logps/chosen": -481.78009033203125,
+      "logps/rejected": -541.44091796875,
+      "loss": 0.6434,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.2409629821777344,
+      "rewards/margins": 0.600003719329834,
+      "rewards/rejected": -2.8409667015075684,
+      "step": 4720
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9044834257452997e-06,
+      "logits/chosen": 1.3596161603927612,
+      "logits/rejected": 1.6001783609390259,
+      "logps/chosen": -473.0391540527344,
+      "logps/rejected": -541.8897705078125,
+      "loss": 0.4544,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.8972028493881226,
+      "rewards/margins": 0.9681664705276489,
+      "rewards/rejected": -2.8653693199157715,
+      "step": 4730
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.893397776485006e-06,
+      "logits/chosen": 1.7671854496002197,
+      "logits/rejected": 2.342480182647705,
+      "logps/chosen": -421.7418518066406,
+      "logps/rejected": -510.4185485839844,
+      "loss": 0.4851,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.9983059167861938,
+      "rewards/margins": 0.9934319257736206,
+      "rewards/rejected": -2.9917380809783936,
+      "step": 4740
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8823247863861804e-06,
+      "logits/chosen": 1.740416169166565,
+      "logits/rejected": 2.443150520324707,
+      "logps/chosen": -464.9256286621094,
+      "logps/rejected": -539.0505981445312,
+      "loss": 0.5186,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.1636714935302734,
+      "rewards/margins": 1.1274700164794922,
+      "rewards/rejected": -3.2911415100097656,
+      "step": 4750
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8712646865306822e-06,
+      "logits/chosen": 1.8207957744598389,
+      "logits/rejected": 1.9273515939712524,
+      "logps/chosen": -496.93011474609375,
+      "logps/rejected": -586.597412109375,
+      "loss": 0.5573,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.3262219429016113,
+      "rewards/margins": 0.828659176826477,
+      "rewards/rejected": -3.154881477355957,
+      "step": 4760
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8602177077313631e-06,
+      "logits/chosen": 1.835579514503479,
+      "logits/rejected": 1.931449294090271,
+      "logps/chosen": -457.050537109375,
+      "logps/rejected": -575.4301147460938,
+      "loss": 0.516,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.0982136726379395,
+      "rewards/margins": 0.9348455667495728,
+      "rewards/rejected": -3.0330593585968018,
+      "step": 4770
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8491840805272546e-06,
+      "logits/chosen": 1.555099368095398,
+      "logits/rejected": 2.0824811458587646,
+      "logps/chosen": -457.87939453125,
+      "logps/rejected": -526.5816650390625,
+      "loss": 0.4671,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.1487972736358643,
+      "rewards/margins": 0.932784378528595,
+      "rewards/rejected": -3.0815815925598145,
+      "step": 4780
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8381640351787516e-06,
+      "logits/chosen": 1.8381952047348022,
+      "logits/rejected": 2.573172092437744,
+      "logps/chosen": -460.7508850097656,
+      "logps/rejected": -537.609375,
+      "loss": 0.4833,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.1327645778656006,
+      "rewards/margins": 0.953527569770813,
+      "rewards/rejected": -3.086291790008545,
+      "step": 4790
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8271578016628122e-06,
+      "logits/chosen": 1.3897019624710083,
+      "logits/rejected": 2.0009140968322754,
+      "logps/chosen": -502.94189453125,
+      "logps/rejected": -572.9285888671875,
+      "loss": 0.4942,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.2561757564544678,
+      "rewards/margins": 1.0010895729064941,
+      "rewards/rejected": -3.257265567779541,
+      "step": 4800
+    },
+    {
+      "epoch": 0.63,
+      "eval_logits/chosen": 1.9470590353012085,
+      "eval_logits/rejected": 2.5209600925445557,
+      "eval_logps/chosen": -494.25750732421875,
+      "eval_logps/rejected": -573.5953979492188,
+      "eval_loss": 0.5058150291442871,
+      "eval_rewards/accuracies": 0.7304999828338623,
+      "eval_rewards/chosen": -2.25697922706604,
+      "eval_rewards/margins": 0.9909048676490784,
+      "eval_rewards/rejected": -3.2478837966918945,
+      "eval_runtime": 1350.69,
+      "eval_samples_per_second": 1.481,
+      "eval_steps_per_second": 0.185,
+      "step": 4800
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8161656096681546e-06,
+      "logits/chosen": 1.3728210926055908,
+      "logits/rejected": 2.371026039123535,
+      "logps/chosen": -471.84765625,
+      "logps/rejected": -558.6546020507812,
+      "loss": 0.5012,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.087430477142334,
+      "rewards/margins": 1.2432823181152344,
+      "rewards/rejected": -3.3307125568389893,
+      "step": 4810
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8051876885904645e-06,
+      "logits/chosen": 1.9149010181427002,
+      "logits/rejected": 2.388388156890869,
+      "logps/chosen": -446.7649841308594,
+      "logps/rejected": -528.5233154296875,
+      "loss": 0.5534,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.252214193344116,
+      "rewards/margins": 0.9581782221794128,
+      "rewards/rejected": -3.210392475128174,
+      "step": 4820
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7942242675276098e-06,
+      "logits/chosen": 1.9972965717315674,
+      "logits/rejected": 2.561474561691284,
+      "logps/chosen": -503.34307861328125,
+      "logps/rejected": -630.5084228515625,
+      "loss": 0.4656,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.345182418823242,
+      "rewards/margins": 1.2717478275299072,
+      "rewards/rejected": -3.6169307231903076,
+      "step": 4830
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.783275575274856e-06,
+      "logits/chosen": 2.1411635875701904,
+      "logits/rejected": 2.9087636470794678,
+      "logps/chosen": -484.42620849609375,
+      "logps/rejected": -581.2403564453125,
+      "loss": 0.4446,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.4242682456970215,
+      "rewards/margins": 1.1327811479568481,
+      "rewards/rejected": -3.55704927444458,
+      "step": 4840
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7723418403200943e-06,
+      "logits/chosen": 1.742540717124939,
+      "logits/rejected": 2.5372486114501953,
+      "logps/chosen": -470.41650390625,
+      "logps/rejected": -591.961669921875,
+      "loss": 0.4601,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.1441421508789062,
+      "rewards/margins": 1.1709234714508057,
+      "rewards/rejected": -3.315065860748291,
+      "step": 4850
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7614232908390748e-06,
+      "logits/chosen": 2.332051992416382,
+      "logits/rejected": 2.7874271869659424,
+      "logps/chosen": -497.0062561035156,
+      "logps/rejected": -571.5010986328125,
+      "loss": 0.5125,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.3403408527374268,
+      "rewards/margins": 1.1052398681640625,
+      "rewards/rejected": -3.445580244064331,
+      "step": 4860
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7505201546906398e-06,
+      "logits/chosen": 1.4827157258987427,
+      "logits/rejected": 2.3149659633636475,
+      "logps/chosen": -475.12841796875,
+      "logps/rejected": -559.3888549804688,
+      "loss": 0.4428,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.9785665273666382,
+      "rewards/margins": 1.1883295774459839,
+      "rewards/rejected": -3.166896343231201,
+      "step": 4870
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7396326594119717e-06,
+      "logits/chosen": 1.8245060443878174,
+      "logits/rejected": 1.7831999063491821,
+      "logps/chosen": -501.25616455078125,
+      "logps/rejected": -634.8753662109375,
+      "loss": 0.4176,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.2555172443389893,
+      "rewards/margins": 1.1453604698181152,
+      "rewards/rejected": -3.4008774757385254,
+      "step": 4880
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7287610322138449e-06,
+      "logits/chosen": 1.7876285314559937,
+      "logits/rejected": 2.3779330253601074,
+      "logps/chosen": -465.4222717285156,
+      "logps/rejected": -541.3952026367188,
+      "loss": 0.511,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.205435276031494,
+      "rewards/margins": 0.9296838045120239,
+      "rewards/rejected": -3.1351189613342285,
+      "step": 4890
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7179054999758817e-06,
+      "logits/chosen": 1.7829234600067139,
+      "logits/rejected": 2.5234198570251465,
+      "logps/chosen": -522.3741455078125,
+      "logps/rejected": -597.4845581054688,
+      "loss": 0.4918,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.4341843128204346,
+      "rewards/margins": 1.086076021194458,
+      "rewards/rejected": -3.5202603340148926,
+      "step": 4900
+    },
+    {
+      "epoch": 0.64,
+      "eval_logits/chosen": 2.2467591762542725,
+      "eval_logits/rejected": 2.829533576965332,
+      "eval_logps/chosen": -516.3652954101562,
+      "eval_logps/rejected": -602.0275268554688,
+      "eval_loss": 0.5128796100616455,
+      "eval_rewards/accuracies": 0.7350000143051147,
+      "eval_rewards/chosen": -2.4780566692352295,
+      "eval_rewards/margins": 1.0541480779647827,
+      "eval_rewards/rejected": -3.532205104827881,
+      "eval_runtime": 1350.4247,
+      "eval_samples_per_second": 1.481,
+      "eval_steps_per_second": 0.185,
+      "step": 4900
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7070662892418225e-06,
+      "logits/chosen": 1.9229376316070557,
+      "logits/rejected": 2.3540873527526855,
+      "logps/chosen": -482.9024353027344,
+      "logps/rejected": -619.5520629882812,
+      "loss": 0.4092,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.291491746902466,
+      "rewards/margins": 1.363003134727478,
+      "rewards/rejected": -3.6544952392578125,
+      "step": 4910
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.6962436262147913e-06,
+      "logits/chosen": 1.7321851253509521,
+      "logits/rejected": 2.324427604675293,
+      "logps/chosen": -577.123779296875,
+      "logps/rejected": -686.9881591796875,
+      "loss": 0.543,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.8536272048950195,
+      "rewards/margins": 1.103878378868103,
+      "rewards/rejected": -3.957504987716675,
+      "step": 4920
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6854377367525814e-06,
+      "logits/chosen": 2.1816630363464355,
+      "logits/rejected": 2.71958327293396,
+      "logps/chosen": -491.7640075683594,
+      "logps/rejected": -628.5194091796875,
+      "loss": 0.4128,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.3798747062683105,
+      "rewards/margins": 1.4266116619110107,
+      "rewards/rejected": -3.8064866065979004,
+      "step": 4930
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6746488463629362e-06,
+      "logits/chosen": 1.6970103979110718,
+      "logits/rejected": 2.421438694000244,
+      "logps/chosen": -556.524169921875,
+      "logps/rejected": -607.4085083007812,
+      "loss": 0.5932,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.6201248168945312,
+      "rewards/margins": 1.0148919820785522,
+      "rewards/rejected": -3.635016918182373,
+      "step": 4940
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6638771801988483e-06,
+      "logits/chosen": 1.7660362720489502,
+      "logits/rejected": 2.399137496948242,
+      "logps/chosen": -536.6796875,
+      "logps/rejected": -607.7869262695312,
+      "loss": 0.5056,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.398643732070923,
+      "rewards/margins": 0.9375537633895874,
+      "rewards/rejected": -3.3361973762512207,
+      "step": 4950
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.653122963053857e-06,
+      "logits/chosen": 1.7342414855957031,
+      "logits/rejected": 2.0754549503326416,
+      "logps/chosen": -482.79608154296875,
+      "logps/rejected": -571.0006103515625,
+      "loss": 0.4824,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.3455636501312256,
+      "rewards/margins": 1.0675238370895386,
+      "rewards/rejected": -3.4130873680114746,
+      "step": 4960
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6423864193573606e-06,
+      "logits/chosen": 1.7145580053329468,
+      "logits/rejected": 2.474611759185791,
+      "logps/chosen": -508.37823486328125,
+      "logps/rejected": -608.32958984375,
+      "loss": 0.5012,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.4307332038879395,
+      "rewards/margins": 1.0467889308929443,
+      "rewards/rejected": -3.477522373199463,
+      "step": 4970
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6316677731699286e-06,
+      "logits/chosen": 1.5840505361557007,
+      "logits/rejected": 2.531991481781006,
+      "logps/chosen": -532.1738891601562,
+      "logps/rejected": -607.3749389648438,
+      "loss": 0.5389,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.5308117866516113,
+      "rewards/margins": 1.1180236339569092,
+      "rewards/rejected": -3.6488349437713623,
+      "step": 4980
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6209672481786302e-06,
+      "logits/chosen": 1.7518901824951172,
+      "logits/rejected": 2.2096705436706543,
+      "logps/chosen": -493.8329162597656,
+      "logps/rejected": -591.2696533203125,
+      "loss": 0.4979,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.412578582763672,
+      "rewards/margins": 0.9104771614074707,
+      "rewards/rejected": -3.3230559825897217,
+      "step": 4990
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6102850676923616e-06,
+      "logits/chosen": 2.2818374633789062,
+      "logits/rejected": 2.2372803688049316,
+      "logps/chosen": -422.70574951171875,
+      "logps/rejected": -555.5460205078125,
+      "loss": 0.4693,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.179816484451294,
+      "rewards/margins": 1.1581186056137085,
+      "rewards/rejected": -3.337935209274292,
+      "step": 5000
+    },
+    {
+      "epoch": 0.65,
+      "eval_logits/chosen": 2.1136746406555176,
+      "eval_logits/rejected": 2.6930837631225586,
+      "eval_logps/chosen": -498.2968444824219,
+      "eval_logps/rejected": -584.69873046875,
+      "eval_loss": 0.5130857825279236,
+      "eval_rewards/accuracies": 0.7315000295639038,
+      "eval_rewards/chosen": -2.297372579574585,
+      "eval_rewards/margins": 1.0615448951721191,
+      "eval_rewards/rejected": -3.358917236328125,
+      "eval_runtime": 1347.3021,
+      "eval_samples_per_second": 1.484,
+      "eval_steps_per_second": 0.186,
+      "step": 5000
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5996214546371888e-06,
+      "logits/chosen": 1.8514554500579834,
+      "logits/rejected": 2.3251304626464844,
+      "logps/chosen": -479.25848388671875,
+      "logps/rejected": -617.2220458984375,
+      "loss": 0.3962,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -2.0973076820373535,
+      "rewards/margins": 1.345058560371399,
+      "rewards/rejected": -3.442366123199463,
+      "step": 5010
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.588976631551697e-06,
+      "logits/chosen": 1.7371189594268799,
+      "logits/rejected": 3.0962324142456055,
+      "logps/chosen": -534.461181640625,
+      "logps/rejected": -609.4308471679688,
+      "loss": 0.4711,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.526381015777588,
+      "rewards/margins": 1.168291449546814,
+      "rewards/rejected": -3.6946728229522705,
+      "step": 5020
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5783508205823412e-06,
+      "logits/chosen": 1.8225364685058594,
+      "logits/rejected": 2.568248748779297,
+      "logps/chosen": -517.3311767578125,
+      "logps/rejected": -634.4415283203125,
+      "loss": 0.5319,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.5617117881774902,
+      "rewards/margins": 1.385089635848999,
+      "rewards/rejected": -3.9468014240264893,
+      "step": 5030
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5677442434788143e-06,
+      "logits/chosen": 1.8760391473770142,
+      "logits/rejected": 2.354264497756958,
+      "logps/chosen": -539.271484375,
+      "logps/rejected": -572.6505737304688,
+      "loss": 0.5246,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.3990871906280518,
+      "rewards/margins": 0.8671795129776001,
+      "rewards/rejected": -3.2662670612335205,
+      "step": 5040
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5571571215894181e-06,
+      "logits/chosen": 1.8587557077407837,
+      "logits/rejected": 2.3229153156280518,
+      "logps/chosen": -440.3995056152344,
+      "logps/rejected": -576.4422607421875,
+      "loss": 0.3813,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.0747156143188477,
+      "rewards/margins": 1.5198169946670532,
+      "rewards/rejected": -3.5945327281951904,
+      "step": 5050
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5465896758564452e-06,
+      "logits/chosen": 1.581708550453186,
+      "logits/rejected": 1.9403880834579468,
+      "logps/chosen": -473.6387634277344,
+      "logps/rejected": -540.385009765625,
+      "loss": 0.5306,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.0461907386779785,
+      "rewards/margins": 0.9337705373764038,
+      "rewards/rejected": -2.979961395263672,
+      "step": 5060
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5360421268115653e-06,
+      "logits/chosen": 1.2373251914978027,
+      "logits/rejected": 1.9060701131820679,
+      "logps/chosen": -465.56103515625,
+      "logps/rejected": -546.8648681640625,
+      "loss": 0.512,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.0983774662017822,
+      "rewards/margins": 0.9021574854850769,
+      "rewards/rejected": -3.000534772872925,
+      "step": 5070
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5255146945712267e-06,
+      "logits/chosen": 1.3490861654281616,
+      "logits/rejected": 1.50093674659729,
+      "logps/chosen": -440.2433166503906,
+      "logps/rejected": -526.07763671875,
+      "loss": 0.5318,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.9903907775878906,
+      "rewards/margins": 0.9573873281478882,
+      "rewards/rejected": -2.9477779865264893,
+      "step": 5080
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.5150075988320594e-06,
+      "logits/chosen": 1.3113791942596436,
+      "logits/rejected": 2.4158082008361816,
+      "logps/chosen": -445.6539611816406,
+      "logps/rejected": -514.7459716796875,
+      "loss": 0.5265,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.052556276321411,
+      "rewards/margins": 1.1268374919891357,
+      "rewards/rejected": -3.179393768310547,
+      "step": 5090
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.5045210588662929e-06,
+      "logits/chosen": 1.6021820306777954,
+      "logits/rejected": 2.033743381500244,
+      "logps/chosen": -529.7374267578125,
+      "logps/rejected": -580.7431030273438,
+      "loss": 0.5796,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.492642641067505,
+      "rewards/margins": 0.6846088767051697,
+      "rewards/rejected": -3.1772518157958984,
+      "step": 5100
+    },
+    {
+      "epoch": 0.67,
+      "eval_logits/chosen": 1.9365460872650146,
+      "eval_logits/rejected": 2.492507219314575,
+      "eval_logps/chosen": -483.41131591796875,
+      "eval_logps/rejected": -565.8975219726562,
+      "eval_loss": 0.5083523988723755,
+      "eval_rewards/accuracies": 0.7300000190734863,
+      "eval_rewards/chosen": -2.148517608642578,
+      "eval_rewards/margins": 1.0223873853683472,
+      "eval_rewards/rejected": -3.1709048748016357,
+      "eval_runtime": 1351.4934,
+      "eval_samples_per_second": 1.48,
+      "eval_steps_per_second": 0.185,
+      "step": 5100
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4940552935171781e-06,
+      "logits/chosen": 1.7719790935516357,
+      "logits/rejected": 2.5644233226776123,
+      "logps/chosen": -474.07928466796875,
+      "logps/rejected": -568.5455932617188,
+      "loss": 0.4725,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.128190279006958,
+      "rewards/margins": 1.2509822845458984,
+      "rewards/rejected": -3.3791728019714355,
+      "step": 5110
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.483610521194419e-06,
+      "logits/chosen": 1.831482172012329,
+      "logits/rejected": 2.2708048820495605,
+      "logps/chosen": -448.7881774902344,
+      "logps/rejected": -524.2953491210938,
+      "loss": 0.52,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.3062729835510254,
+      "rewards/margins": 0.8814659118652344,
+      "rewards/rejected": -3.1877386569976807,
+      "step": 5120
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4731869598696226e-06,
+      "logits/chosen": 1.3386174440383911,
+      "logits/rejected": 1.8742462396621704,
+      "logps/chosen": -556.6773071289062,
+      "logps/rejected": -625.0130615234375,
+      "loss": 0.5261,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.445650339126587,
+      "rewards/margins": 1.0798041820526123,
+      "rewards/rejected": -3.5254547595977783,
+      "step": 5130
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4627848270717387e-06,
+      "logits/chosen": 1.30894935131073,
+      "logits/rejected": 2.0028789043426514,
+      "logps/chosen": -486.5819396972656,
+      "logps/rejected": -585.2606811523438,
+      "loss": 0.4634,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.2083120346069336,
+      "rewards/margins": 1.2071669101715088,
+      "rewards/rejected": -3.4154789447784424,
+      "step": 5140
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4524043398825277e-06,
+      "logits/chosen": 1.2474445104599,
+      "logits/rejected": 2.7433581352233887,
+      "logps/chosen": -509.2044372558594,
+      "logps/rejected": -515.4666748046875,
+      "loss": 0.4914,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.217095375061035,
+      "rewards/margins": 1.03823983669281,
+      "rewards/rejected": -3.255335569381714,
+      "step": 5150
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4420457149320299e-06,
+      "logits/chosen": 1.0699405670166016,
+      "logits/rejected": 2.0535435676574707,
+      "logps/chosen": -523.6400146484375,
+      "logps/rejected": -590.9120483398438,
+      "loss": 0.5117,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.3313043117523193,
+      "rewards/margins": 0.992363452911377,
+      "rewards/rejected": -3.3236680030822754,
+      "step": 5160
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.431709168394042e-06,
+      "logits/chosen": 1.559014081954956,
+      "logits/rejected": 2.0179953575134277,
+      "logps/chosen": -473.36090087890625,
+      "logps/rejected": -569.082275390625,
+      "loss": 0.6213,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.3286890983581543,
+      "rewards/margins": 0.9337849617004395,
+      "rewards/rejected": -3.2624740600585938,
+      "step": 5170
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4213949159816059e-06,
+      "logits/chosen": 1.7814000844955444,
+      "logits/rejected": 2.1790690422058105,
+      "logps/chosen": -472.4271545410156,
+      "logps/rejected": -564.2132568359375,
+      "loss": 0.5802,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.2520670890808105,
+      "rewards/margins": 0.8396795988082886,
+      "rewards/rejected": -3.0917465686798096,
+      "step": 5180
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4111031729425103e-06,
+      "logits/chosen": 1.3565289974212646,
+      "logits/rejected": 1.7993520498275757,
+      "logps/chosen": -480.3685607910156,
+      "logps/rejected": -567.4840698242188,
+      "loss": 0.4673,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.013777494430542,
+      "rewards/margins": 1.0025947093963623,
+      "rewards/rejected": -3.0163724422454834,
+      "step": 5190
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4008341540547965e-06,
+      "logits/chosen": 1.205325961112976,
+      "logits/rejected": 1.6385034322738647,
+      "logps/chosen": -446.7149353027344,
+      "logps/rejected": -477.7730407714844,
+      "loss": 0.5137,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.1694726943969727,
+      "rewards/margins": 0.74607253074646,
+      "rewards/rejected": -2.9155452251434326,
+      "step": 5200
+    },
+    {
+      "epoch": 0.68,
+      "eval_logits/chosen": 1.5737698078155518,
+      "eval_logits/rejected": 2.096890449523926,
+      "eval_logps/chosen": -469.3902893066406,
+      "eval_logps/rejected": -542.50732421875,
+      "eval_loss": 0.501226544380188,
+      "eval_rewards/accuracies": 0.7365000247955322,
+      "eval_rewards/chosen": -2.0083069801330566,
+      "eval_rewards/margins": 0.9286957383155823,
+      "eval_rewards/rejected": -2.9370028972625732,
+      "eval_runtime": 1351.5573,
+      "eval_samples_per_second": 1.48,
+      "eval_steps_per_second": 0.185,
+      "step": 5200
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3905880736222737e-06,
+      "logits/chosen": 1.5835494995117188,
+      "logits/rejected": 2.0422065258026123,
+      "logps/chosen": -426.63311767578125,
+      "logps/rejected": -494.61553955078125,
+      "loss": 0.4842,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.9298732280731201,
+      "rewards/margins": 0.923658549785614,
+      "rewards/rejected": -2.8535315990448,
+      "step": 5210
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3803651454700531e-06,
+      "logits/chosen": 1.2102668285369873,
+      "logits/rejected": 1.7876434326171875,
+      "logps/chosen": -486.9481506347656,
+      "logps/rejected": -548.3073120117188,
+      "loss": 0.4636,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.9990739822387695,
+      "rewards/margins": 0.9765180349349976,
+      "rewards/rejected": -2.9755921363830566,
+      "step": 5220
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3701655829400773e-06,
+      "logits/chosen": 1.554851770401001,
+      "logits/rejected": 2.1653993129730225,
+      "logps/chosen": -454.7589416503906,
+      "logps/rejected": -535.565185546875,
+      "loss": 0.5577,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.1235334873199463,
+      "rewards/margins": 0.8729076385498047,
+      "rewards/rejected": -2.996441125869751,
+      "step": 5230
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3599895988866756e-06,
+      "logits/chosen": 1.4060057401657104,
+      "logits/rejected": 1.7991788387298584,
+      "logps/chosen": -433.3150939941406,
+      "logps/rejected": -551.0670166015625,
+      "loss": 0.4454,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.0311315059661865,
+      "rewards/margins": 1.0842902660369873,
+      "rewards/rejected": -3.115422010421753,
+      "step": 5240
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3498374056721198e-06,
+      "logits/chosen": 1.3356964588165283,
+      "logits/rejected": 2.13816237449646,
+      "logps/chosen": -508.8194274902344,
+      "logps/rejected": -522.1041259765625,
+      "loss": 0.6085,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.1835639476776123,
+      "rewards/margins": 0.6619709730148315,
+      "rewards/rejected": -2.8455350399017334,
+      "step": 5250
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3397092151621883e-06,
+      "logits/chosen": 1.5083988904953003,
+      "logits/rejected": 2.101290225982666,
+      "logps/chosen": -500.5271911621094,
+      "logps/rejected": -604.2184448242188,
+      "loss": 0.5239,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.408400774002075,
+      "rewards/margins": 0.9796700477600098,
+      "rewards/rejected": -3.388070583343506,
+      "step": 5260
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3296052387217484e-06,
+      "logits/chosen": 1.2947309017181396,
+      "logits/rejected": 1.7775700092315674,
+      "logps/chosen": -504.0699157714844,
+      "logps/rejected": -572.1639404296875,
+      "loss": 0.4642,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.0995423793792725,
+      "rewards/margins": 0.9391013383865356,
+      "rewards/rejected": -3.0386435985565186,
+      "step": 5270
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3195256872103476e-06,
+      "logits/chosen": 0.8099312782287598,
+      "logits/rejected": 2.195142984390259,
+      "logps/chosen": -540.7571411132812,
+      "logps/rejected": -546.215576171875,
+      "loss": 0.4742,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.1803600788116455,
+      "rewards/margins": 1.006661295890808,
+      "rewards/rejected": -3.187021017074585,
+      "step": 5280
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3094707709778068e-06,
+      "logits/chosen": 1.4804357290267944,
+      "logits/rejected": 1.7239437103271484,
+      "logps/chosen": -534.307861328125,
+      "logps/rejected": -587.14697265625,
+      "loss": 0.5808,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.470363140106201,
+      "rewards/margins": 0.7377761602401733,
+      "rewards/rejected": -3.208139419555664,
+      "step": 5290
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.2994406998598364e-06,
+      "logits/chosen": 1.144063115119934,
+      "logits/rejected": 1.781019926071167,
+      "logps/chosen": -467.21795654296875,
+      "logps/rejected": -543.3238525390625,
+      "loss": 0.4484,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.9340791702270508,
+      "rewards/margins": 1.080896019935608,
+      "rewards/rejected": -3.0149753093719482,
+      "step": 5300
+    },
+    {
+      "epoch": 0.69,
+      "eval_logits/chosen": 1.7154160737991333,
+      "eval_logits/rejected": 2.2539069652557373,
+      "eval_logps/chosen": -480.0531311035156,
+      "eval_logps/rejected": -556.4617919921875,
+      "eval_loss": 0.5021650195121765,
+      "eval_rewards/accuracies": 0.734499990940094,
+      "eval_rewards/chosen": -2.1149356365203857,
+      "eval_rewards/margins": 0.9616131782531738,
+      "eval_rewards/rejected": -3.0765483379364014,
+      "eval_runtime": 1351.1957,
+      "eval_samples_per_second": 1.48,
+      "eval_steps_per_second": 0.185,
+      "step": 5300
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.2894356831736558e-06,
+      "logits/chosen": 1.211223840713501,
+      "logits/rejected": 1.5505168437957764,
+      "logps/chosen": -527.10888671875,
+      "logps/rejected": -570.4957275390625,
+      "loss": 0.5633,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.17678165435791,
+      "rewards/margins": 0.8642389178276062,
+      "rewards/rejected": -3.041020393371582,
+      "step": 5310
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2794559297136203e-06,
+      "logits/chosen": 1.4967700242996216,
+      "logits/rejected": 1.9913291931152344,
+      "logps/chosen": -494.332763671875,
+      "logps/rejected": -571.6278686523438,
+      "loss": 0.609,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.312108278274536,
+      "rewards/margins": 0.6342388987541199,
+      "rewards/rejected": -2.9463469982147217,
+      "step": 5320
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2695016477468724e-06,
+      "logits/chosen": 1.1873748302459717,
+      "logits/rejected": 1.5668585300445557,
+      "logps/chosen": -430.52447509765625,
+      "logps/rejected": -576.4597778320312,
+      "loss": 0.4466,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.9823791980743408,
+      "rewards/margins": 1.2965465784072876,
+      "rewards/rejected": -3.278925657272339,
+      "step": 5330
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2595730450089874e-06,
+      "logits/chosen": 1.0061835050582886,
+      "logits/rejected": 1.6807470321655273,
+      "logps/chosen": -501.7490234375,
+      "logps/rejected": -563.7476806640625,
+      "loss": 0.4222,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.8900600671768188,
+      "rewards/margins": 1.1204086542129517,
+      "rewards/rejected": -3.0104689598083496,
+      "step": 5340
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2496703286996433e-06,
+      "logits/chosen": 0.9863855242729187,
+      "logits/rejected": 1.6997339725494385,
+      "logps/chosen": -504.4463806152344,
+      "logps/rejected": -584.2822265625,
+      "loss": 0.55,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.1821701526641846,
+      "rewards/margins": 0.9166151881217957,
+      "rewards/rejected": -3.098785161972046,
+      "step": 5350
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2397937054782961e-06,
+      "logits/chosen": 1.1537480354309082,
+      "logits/rejected": 1.8737424612045288,
+      "logps/chosen": -486.51068115234375,
+      "logps/rejected": -538.9456787109375,
+      "loss": 0.6584,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.2745912075042725,
+      "rewards/margins": 0.6943547129631042,
+      "rewards/rejected": -2.9689457416534424,
+      "step": 5360
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2299433814598635e-06,
+      "logits/chosen": 1.1881624460220337,
+      "logits/rejected": 1.7410074472427368,
+      "logps/chosen": -517.01513671875,
+      "logps/rejected": -568.9479370117188,
+      "loss": 0.5428,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.205918073654175,
+      "rewards/margins": 0.6970504522323608,
+      "rewards/rejected": -2.902968645095825,
+      "step": 5370
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2201195622104265e-06,
+      "logits/chosen": 1.4046329259872437,
+      "logits/rejected": 2.1186203956604004,
+      "logps/chosen": -488.14569091796875,
+      "logps/rejected": -572.571533203125,
+      "loss": 0.4804,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.2033238410949707,
+      "rewards/margins": 1.0035836696624756,
+      "rewards/rejected": -3.206907272338867,
+      "step": 5380
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.2103224527429417e-06,
+      "logits/chosen": 1.1378192901611328,
+      "logits/rejected": 1.4404375553131104,
+      "logps/chosen": -486.9642639160156,
+      "logps/rejected": -562.9290771484375,
+      "loss": 0.4969,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.0345826148986816,
+      "rewards/margins": 0.9460740089416504,
+      "rewards/rejected": -2.980656862258911,
+      "step": 5390
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.2005522575129559e-06,
+      "logits/chosen": 0.9280862808227539,
+      "logits/rejected": 1.6083498001098633,
+      "logps/chosen": -494.250244140625,
+      "logps/rejected": -594.7198486328125,
+      "loss": 0.4608,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.9917570352554321,
+      "rewards/margins": 1.1567471027374268,
+      "rewards/rejected": -3.1485037803649902,
+      "step": 5400
+    },
+    {
+      "epoch": 0.71,
+      "eval_logits/chosen": 1.6704055070877075,
+      "eval_logits/rejected": 2.222350597381592,
+      "eval_logps/chosen": -484.9484558105469,
+      "eval_logps/rejected": -564.666259765625,
+      "eval_loss": 0.5034918785095215,
+      "eval_rewards/accuracies": 0.7379999756813049,
+      "eval_rewards/chosen": -2.163888692855835,
+      "eval_rewards/margins": 0.9947044849395752,
+      "eval_rewards/rejected": -3.158592939376831,
+      "eval_runtime": 1351.1157,
+      "eval_samples_per_second": 1.48,
+      "eval_steps_per_second": 0.185,
+      "step": 5400
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1908091804143469e-06,
+      "logits/chosen": 1.754124641418457,
+      "logits/rejected": 2.076094150543213,
+      "logps/chosen": -468.14837646484375,
+      "logps/rejected": -543.490234375,
+      "loss": 0.4768,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.1779284477233887,
+      "rewards/margins": 0.964095950126648,
+      "rewards/rejected": -3.142024517059326,
+      "step": 5410
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1810934247750649e-06,
+      "logits/chosen": 1.11089026927948,
+      "logits/rejected": 1.9022104740142822,
+      "logps/chosen": -521.2377319335938,
+      "logps/rejected": -560.9497680664062,
+      "loss": 0.5685,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.26832914352417,
+      "rewards/margins": 0.93254554271698,
+      "rewards/rejected": -3.2008748054504395,
+      "step": 5420
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1714051933528881e-06,
+      "logits/chosen": 1.4646847248077393,
+      "logits/rejected": 1.9254957437515259,
+      "logps/chosen": -474.38592529296875,
+      "logps/rejected": -572.9240112304688,
+      "loss": 0.4644,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.13691782951355,
+      "rewards/margins": 1.1846344470977783,
+      "rewards/rejected": -3.321552276611328,
+      "step": 5430
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.161744688331192e-06,
+      "logits/chosen": 1.5409965515136719,
+      "logits/rejected": 2.231088638305664,
+      "logps/chosen": -479.615966796875,
+      "logps/rejected": -575.2286376953125,
+      "loss": 0.4095,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.1384658813476562,
+      "rewards/margins": 1.1418513059616089,
+      "rewards/rejected": -3.2803173065185547,
+      "step": 5440
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.152112111314733e-06,
+      "logits/chosen": 1.5389087200164795,
+      "logits/rejected": 1.5896430015563965,
+      "logps/chosen": -499.2042541503906,
+      "logps/rejected": -575.2064208984375,
+      "loss": 0.5154,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.297316789627075,
+      "rewards/margins": 0.9257675409317017,
+      "rewards/rejected": -3.223083972930908,
+      "step": 5450
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.142507663325439e-06,
+      "logits/chosen": 1.833674430847168,
+      "logits/rejected": 2.3198883533477783,
+      "logps/chosen": -493.2935485839844,
+      "logps/rejected": -577.0165405273438,
+      "loss": 0.4467,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.1435275077819824,
+      "rewards/margins": 1.3296173810958862,
+      "rewards/rejected": -3.47314453125,
+      "step": 5460
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.132931544798211e-06,
+      "logits/chosen": 1.1300413608551025,
+      "logits/rejected": 2.187528133392334,
+      "logps/chosen": -518.1541748046875,
+      "logps/rejected": -555.7781982421875,
+      "loss": 0.522,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.3925600051879883,
+      "rewards/margins": 1.0563757419586182,
+      "rewards/rejected": -3.4489357471466064,
+      "step": 5470
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1233839555767482e-06,
+      "logits/chosen": 1.57352614402771,
+      "logits/rejected": 1.8808791637420654,
+      "logps/chosen": -456.79119873046875,
+      "logps/rejected": -597.9332275390625,
+      "loss": 0.476,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.3361334800720215,
+      "rewards/margins": 1.2098755836486816,
+      "rewards/rejected": -3.546009063720703,
+      "step": 5480
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1138650949093668e-06,
+      "logits/chosen": 1.2874473333358765,
+      "logits/rejected": 1.7039320468902588,
+      "logps/chosen": -466.4269104003906,
+      "logps/rejected": -554.2048950195312,
+      "loss": 0.5749,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.2714924812316895,
+      "rewards/margins": 0.7845773696899414,
+      "rewards/rejected": -3.056069850921631,
+      "step": 5490
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1043751614448543e-06,
+      "logits/chosen": 1.6732509136199951,
+      "logits/rejected": 1.7993825674057007,
+      "logps/chosen": -513.9376220703125,
+      "logps/rejected": -618.1824340820312,
+      "loss": 0.5746,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.526108503341675,
+      "rewards/margins": 1.0179903507232666,
+      "rewards/rejected": -3.5440986156463623,
+      "step": 5500
+    },
+    {
+      "epoch": 0.72,
+      "eval_logits/chosen": 1.6561747789382935,
+      "eval_logits/rejected": 2.21342134475708,
+      "eval_logps/chosen": -504.5519714355469,
+      "eval_logps/rejected": -589.0369873046875,
+      "eval_loss": 0.5045461654663086,
+      "eval_rewards/accuracies": 0.7319999933242798,
+      "eval_rewards/chosen": -2.359923839569092,
+      "eval_rewards/margins": 1.0423763990402222,
+      "eval_rewards/rejected": -3.4023003578186035,
+      "eval_runtime": 1350.8821,
+      "eval_samples_per_second": 1.481,
+      "eval_steps_per_second": 0.185,
+      "step": 5500
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0949143532283107e-06,
+      "logits/chosen": 1.0113441944122314,
+      "logits/rejected": 2.051943302154541,
+      "logps/chosen": -494.387939453125,
+      "logps/rejected": -583.12646484375,
+      "loss": 0.4723,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.393023729324341,
+      "rewards/margins": 1.269960641860962,
+      "rewards/rejected": -3.6629843711853027,
+      "step": 5510
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0854828676970275e-06,
+      "logits/chosen": 1.4438446760177612,
+      "logits/rejected": 2.046376943588257,
+      "logps/chosen": -531.700927734375,
+      "logps/rejected": -543.64453125,
+      "loss": 0.5264,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.439065933227539,
+      "rewards/margins": 0.7978044748306274,
+      "rewards/rejected": -3.236870527267456,
+      "step": 5520
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.076080901676361e-06,
+      "logits/chosen": 1.4071542024612427,
+      "logits/rejected": 1.1845344305038452,
+      "logps/chosen": -490.24810791015625,
+      "logps/rejected": -589.6234130859375,
+      "loss": 0.6032,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.3688700199127197,
+      "rewards/margins": 0.6264444589614868,
+      "rewards/rejected": -2.995314121246338,
+      "step": 5530
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0667086513756234e-06,
+      "logits/chosen": 1.4568307399749756,
+      "logits/rejected": 1.9023765325546265,
+      "logps/chosen": -431.7606506347656,
+      "logps/rejected": -534.075439453125,
+      "loss": 0.4395,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.1697888374328613,
+      "rewards/margins": 1.1363937854766846,
+      "rewards/rejected": -3.306182861328125,
+      "step": 5540
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0573663123839912e-06,
+      "logits/chosen": 1.3489223718643188,
+      "logits/rejected": 1.9295543432235718,
+      "logps/chosen": -430.6036071777344,
+      "logps/rejected": -520.8906860351562,
+      "loss": 0.4463,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.194150924682617,
+      "rewards/margins": 0.9387199282646179,
+      "rewards/rejected": -3.13287091255188,
+      "step": 5550
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0480540796664251e-06,
+      "logits/chosen": 1.2379951477050781,
+      "logits/rejected": 1.6586802005767822,
+      "logps/chosen": -542.0699462890625,
+      "logps/rejected": -612.1685791015625,
+      "loss": 0.5408,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.2761611938476562,
+      "rewards/margins": 0.918903648853302,
+      "rewards/rejected": -3.1950647830963135,
+      "step": 5560
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0387721475595978e-06,
+      "logits/chosen": 1.1465680599212646,
+      "logits/rejected": 1.605237364768982,
+      "logps/chosen": -532.4811401367188,
+      "logps/rejected": -576.0885620117188,
+      "loss": 0.5655,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.334770679473877,
+      "rewards/margins": 0.8122250437736511,
+      "rewards/rejected": -3.1469955444335938,
+      "step": 5570
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0295207097678378e-06,
+      "logits/chosen": 0.9449742436408997,
+      "logits/rejected": 1.907265067100525,
+      "logps/chosen": -490.6373596191406,
+      "logps/rejected": -536.0383911132812,
+      "loss": 0.5806,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.2731540203094482,
+      "rewards/margins": 0.8993282318115234,
+      "rewards/rejected": -3.1724822521209717,
+      "step": 5580
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0202999593590924e-06,
+      "logits/chosen": 1.2909094095230103,
+      "logits/rejected": 1.8930532932281494,
+      "logps/chosen": -453.7796936035156,
+      "logps/rejected": -501.70184326171875,
+      "loss": 0.5327,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.203911066055298,
+      "rewards/margins": 0.8938743472099304,
+      "rewards/rejected": -3.097785472869873,
+      "step": 5590
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.011110088760891e-06,
+      "logits/chosen": 0.6917427778244019,
+      "logits/rejected": 1.4272220134735107,
+      "logps/chosen": -481.73516845703125,
+      "logps/rejected": -526.750244140625,
+      "loss": 0.5768,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.0975871086120605,
+      "rewards/margins": 0.9376052618026733,
+      "rewards/rejected": -3.0351924896240234,
+      "step": 5600
+    },
+    {
+      "epoch": 0.73,
+      "eval_logits/chosen": 1.2667248249053955,
+      "eval_logits/rejected": 1.8199137449264526,
+      "eval_logps/chosen": -475.1830139160156,
+      "eval_logps/rejected": -553.1031494140625,
+      "eval_loss": 0.5010562539100647,
+      "eval_rewards/accuracies": 0.737500011920929,
+      "eval_rewards/chosen": -2.0662341117858887,
+      "eval_rewards/margins": 0.9767271876335144,
+      "eval_rewards/rejected": -3.042961359024048,
+      "eval_runtime": 1351.7561,
+      "eval_samples_per_second": 1.48,
+      "eval_steps_per_second": 0.185,
+      "step": 5600
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0019512897563347e-06,
+      "logits/chosen": 0.9792037010192871,
+      "logits/rejected": 1.4095228910446167,
+      "logps/chosen": -427.49371337890625,
+      "logps/rejected": -571.4511108398438,
+      "loss": 0.3932,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.9337078332901,
+      "rewards/margins": 1.3885209560394287,
+      "rewards/rejected": -3.3222289085388184,
+      "step": 5610
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.928237534800935e-07,
+      "logits/chosen": 0.669500470161438,
+      "logits/rejected": 1.7472620010375977,
+      "logps/chosen": -501.66143798828125,
+      "logps/rejected": -571.5609130859375,
+      "loss": 0.4731,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.3167357444763184,
+      "rewards/margins": 1.0845458507537842,
+      "rewards/rejected": -3.401282548904419,
+      "step": 5620
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.837276704144174e-07,
+      "logits/chosen": 1.515411615371704,
+      "logits/rejected": 1.5292515754699707,
+      "logps/chosen": -434.12860107421875,
+      "logps/rejected": -549.5562133789062,
+      "loss": 0.5639,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.1806459426879883,
+      "rewards/margins": 0.749146580696106,
+      "rewards/rejected": -2.9297924041748047,
+      "step": 5630
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.746632303851569e-07,
+      "logits/chosen": 0.9732497334480286,
+      "logits/rejected": 1.5330772399902344,
+      "logps/chosen": -481.242431640625,
+      "logps/rejected": -539.4158935546875,
+      "loss": 0.4749,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.0362632274627686,
+      "rewards/margins": 0.9089013934135437,
+      "rewards/rejected": -2.9451651573181152,
+      "step": 5640
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.65630622557809e-07,
+      "logits/chosen": 1.665758728981018,
+      "logits/rejected": 1.9480602741241455,
+      "logps/chosen": -467.7523498535156,
+      "logps/rejected": -554.9008178710938,
+      "loss": 0.5405,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.2450926303863525,
+      "rewards/margins": 0.9270321130752563,
+      "rewards/rejected": -3.1721243858337402,
+      "step": 5650
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.56630035433561e-07,
+      "logits/chosen": 0.7561038732528687,
+      "logits/rejected": 1.725600004196167,
+      "logps/chosen": -491.6058044433594,
+      "logps/rejected": -590.5841064453125,
+      "loss": 0.3691,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.189763069152832,
+      "rewards/margins": 1.3421342372894287,
+      "rewards/rejected": -3.5318970680236816,
+      "step": 5660
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.476616568453659e-07,
+      "logits/chosen": 1.2824413776397705,
+      "logits/rejected": 2.044991970062256,
+      "logps/chosen": -481.43316650390625,
+      "logps/rejected": -539.0975341796875,
+      "loss": 0.5242,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.2171339988708496,
+      "rewards/margins": 0.9363832473754883,
+      "rewards/rejected": -3.153517246246338,
+      "step": 5670
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.387256739540162e-07,
+      "logits/chosen": 1.2762155532836914,
+      "logits/rejected": 2.1030337810516357,
+      "logps/chosen": -427.1504821777344,
+      "logps/rejected": -497.1144104003906,
+      "loss": 0.5199,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.0391926765441895,
+      "rewards/margins": 1.041942834854126,
+      "rewards/rejected": -3.0811352729797363,
+      "step": 5680
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.298222732442377e-07,
+      "logits/chosen": 1.0183273553848267,
+      "logits/rejected": 1.3000823259353638,
+      "logps/chosen": -455.291748046875,
+      "logps/rejected": -606.7677612304688,
+      "loss": 0.4416,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.0766406059265137,
+      "rewards/margins": 1.3346563577651978,
+      "rewards/rejected": -3.411296844482422,
+      "step": 5690
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.20951640520803e-07,
+      "logits/chosen": 0.6322634220123291,
+      "logits/rejected": 1.0998876094818115,
+      "logps/chosen": -495.69427490234375,
+      "logps/rejected": -609.1307373046875,
+      "loss": 0.4359,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.9762344360351562,
+      "rewards/margins": 1.2228240966796875,
+      "rewards/rejected": -3.1990585327148438,
+      "step": 5700
+    },
+    {
+      "epoch": 0.75,
+      "eval_logits/chosen": 1.350328803062439,
+      "eval_logits/rejected": 1.9073015451431274,
+      "eval_logps/chosen": -477.8932189941406,
+      "eval_logps/rejected": -559.804931640625,
+      "eval_loss": 0.5031585693359375,
+      "eval_rewards/accuracies": 0.7350000143051147,
+      "eval_rewards/chosen": -2.0933358669281006,
+      "eval_rewards/margins": 1.01664400100708,
+      "eval_rewards/rejected": -3.1099796295166016,
+      "eval_runtime": 1350.1474,
+      "eval_samples_per_second": 1.481,
+      "eval_steps_per_second": 0.185,
+      "step": 5700
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.121139609046484e-07,
+      "logits/chosen": 0.7715672850608826,
+      "logits/rejected": 1.6318953037261963,
+      "logps/chosen": -479.60308837890625,
+      "logps/rejected": -542.7793579101562,
+      "loss": 0.5555,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.2636046409606934,
+      "rewards/margins": 0.9329112768173218,
+      "rewards/rejected": -3.1965160369873047,
+      "step": 5710
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.033094188290121e-07,
+      "logits/chosen": 0.7050351500511169,
+      "logits/rejected": 1.2993714809417725,
+      "logps/chosen": -456.8487243652344,
+      "logps/rejected": -598.6525268554688,
+      "loss": 0.4222,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.9550931453704834,
+      "rewards/margins": 1.3325374126434326,
+      "rewards/rejected": -3.287630558013916,
+      "step": 5720
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.945381980355889e-07,
+      "logits/chosen": 0.6951960325241089,
+      "logits/rejected": 1.2937911748886108,
+      "logps/chosen": -473.1756286621094,
+      "logps/rejected": -552.302734375,
+      "loss": 0.5595,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.092557907104492,
+      "rewards/margins": 0.7794235944747925,
+      "rewards/rejected": -2.8719818592071533,
+      "step": 5730
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.858004815706919e-07,
+      "logits/chosen": 1.1467959880828857,
+      "logits/rejected": 1.3040180206298828,
+      "logps/chosen": -422.026611328125,
+      "logps/rejected": -551.6525268554688,
+      "loss": 0.4887,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.9531104564666748,
+      "rewards/margins": 1.2445043325424194,
+      "rewards/rejected": -3.1976146697998047,
+      "step": 5740
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.77096451781432e-07,
+      "logits/chosen": 1.3914406299591064,
+      "logits/rejected": 1.6102991104125977,
+      "logps/chosen": -452.4950256347656,
+      "logps/rejected": -532.6602783203125,
+      "loss": 0.5251,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.0720486640930176,
+      "rewards/margins": 1.0228497982025146,
+      "rewards/rejected": -3.0948984622955322,
+      "step": 5750
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.684262903119165e-07,
+      "logits/chosen": 1.1006872653961182,
+      "logits/rejected": 2.0346176624298096,
+      "logps/chosen": -505.7814025878906,
+      "logps/rejected": -549.6341552734375,
+      "loss": 0.5088,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.2264130115509033,
+      "rewards/margins": 1.0577672719955444,
+      "rewards/rejected": -3.284180164337158,
+      "step": 5760
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.597901780994525e-07,
+      "logits/chosen": 1.5478460788726807,
+      "logits/rejected": 1.8454002141952515,
+      "logps/chosen": -479.65557861328125,
+      "logps/rejected": -595.908203125,
+      "loss": 0.487,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.081925630569458,
+      "rewards/margins": 1.1287730932235718,
+      "rewards/rejected": -3.2106986045837402,
+      "step": 5770
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.511882953707773e-07,
+      "logits/chosen": 1.294874906539917,
+      "logits/rejected": 1.2901678085327148,
+      "logps/chosen": -468.7681579589844,
+      "logps/rejected": -594.8966674804688,
+      "loss": 0.4787,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.128744125366211,
+      "rewards/margins": 1.2591350078582764,
+      "rewards/rejected": -3.3878791332244873,
+      "step": 5780
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.426208216382944e-07,
+      "logits/chosen": 1.6592957973480225,
+      "logits/rejected": 1.7846893072128296,
+      "logps/chosen": -415.2164001464844,
+      "logps/rejected": -565.8373413085938,
+      "loss": 0.462,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.060683488845825,
+      "rewards/margins": 1.2238519191741943,
+      "rewards/rejected": -3.2845356464385986,
+      "step": 5790
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.340879356963245e-07,
+      "logits/chosen": 1.4348406791687012,
+      "logits/rejected": 1.7727413177490234,
+      "logps/chosen": -454.76861572265625,
+      "logps/rejected": -583.7518310546875,
+      "loss": 0.4812,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.2436611652374268,
+      "rewards/margins": 1.2402582168579102,
+      "rewards/rejected": -3.483919620513916,
+      "step": 5800
+    },
+    {
+      "epoch": 0.76,
+      "eval_logits/chosen": 1.550770878791809,
+      "eval_logits/rejected": 2.123439073562622,
+      "eval_logps/chosen": -497.8670959472656,
+      "eval_logps/rejected": -585.2068481445312,
+      "eval_loss": 0.5055537223815918,
+      "eval_rewards/accuracies": 0.7319999933242798,
+      "eval_rewards/chosen": -2.2930750846862793,
+      "eval_rewards/margins": 1.070924162864685,
+      "eval_rewards/rejected": -3.363999605178833,
+      "eval_runtime": 1350.8063,
+      "eval_samples_per_second": 1.481,
+      "eval_steps_per_second": 0.185,
+      "step": 5800
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.255898156173777e-07,
+      "logits/chosen": 1.4885269403457642,
+      "logits/rejected": 1.9609041213989258,
+      "logps/chosen": -561.2666015625,
+      "logps/rejected": -604.8101806640625,
+      "loss": 0.579,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.5989909172058105,
+      "rewards/margins": 0.8462007641792297,
+      "rewards/rejected": -3.4451918601989746,
+      "step": 5810
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.171266387484389e-07,
+      "logits/chosen": 1.5601074695587158,
+      "logits/rejected": 1.875687599182129,
+      "logps/chosen": -482.86273193359375,
+      "logps/rejected": -601.8505859375,
+      "loss": 0.4969,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.191948413848877,
+      "rewards/margins": 1.03483247756958,
+      "rewards/rejected": -3.226781129837036,
+      "step": 5820
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.086985817072604e-07,
+      "logits/chosen": 1.0829532146453857,
+      "logits/rejected": 1.7410335540771484,
+      "logps/chosen": -455.19921875,
+      "logps/rejected": -553.7186279296875,
+      "loss": 0.4741,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.145019292831421,
+      "rewards/margins": 1.2013393640518188,
+      "rewards/rejected": -3.3463587760925293,
+      "step": 5830
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.003058203786835e-07,
+      "logits/chosen": 0.8813959360122681,
+      "logits/rejected": 1.5128105878829956,
+      "logps/chosen": -473.0982360839844,
+      "logps/rejected": -498.4776306152344,
+      "loss": 0.5942,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.2035470008850098,
+      "rewards/margins": 0.5761550068855286,
+      "rewards/rejected": -2.7797021865844727,
+      "step": 5840
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.91948529910963e-07,
+      "logits/chosen": 0.9583026170730591,
+      "logits/rejected": 1.7379636764526367,
+      "logps/chosen": -556.1282348632812,
+      "logps/rejected": -600.1179809570312,
+      "loss": 0.5818,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.2369632720947266,
+      "rewards/margins": 1.0085150003433228,
+      "rewards/rejected": -3.245478391647339,
+      "step": 5850
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.836268847121126e-07,
+      "logits/chosen": 1.0308722257614136,
+      "logits/rejected": 2.037480592727661,
+      "logps/chosen": -482.66064453125,
+      "logps/rejected": -570.44580078125,
+      "loss": 0.4393,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.0046544075012207,
+      "rewards/margins": 1.1127052307128906,
+      "rewards/rejected": -3.1173596382141113,
+      "step": 5860
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.753410584462681e-07,
+      "logits/chosen": 1.0271632671356201,
+      "logits/rejected": 1.2409298419952393,
+      "logps/chosen": -502.3153381347656,
+      "logps/rejected": -618.0609130859375,
+      "loss": 0.5062,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.3595948219299316,
+      "rewards/margins": 1.049361228942871,
+      "rewards/rejected": -3.4089558124542236,
+      "step": 5870
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.670912240300596e-07,
+      "logits/chosen": 1.081769585609436,
+      "logits/rejected": 1.6072752475738525,
+      "logps/chosen": -447.9144592285156,
+      "logps/rejected": -576.9791870117188,
+      "loss": 0.4578,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.9362945556640625,
+      "rewards/margins": 1.196453332901001,
+      "rewards/rejected": -3.1327478885650635,
+      "step": 5880
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.588775536290035e-07,
+      "logits/chosen": 0.7779386043548584,
+      "logits/rejected": 1.5611240863800049,
+      "logps/chosen": -477.61944580078125,
+      "logps/rejected": -555.985107421875,
+      "loss": 0.5985,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.9901539087295532,
+      "rewards/margins": 1.002282977104187,
+      "rewards/rejected": -2.9924368858337402,
+      "step": 5890
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.507002186539147e-07,
+      "logits/chosen": 0.748561441898346,
+      "logits/rejected": 1.257095217704773,
+      "logps/chosen": -485.76373291015625,
+      "logps/rejected": -572.6224365234375,
+      "loss": 0.5048,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.019212007522583,
+      "rewards/margins": 1.0997709035873413,
+      "rewards/rejected": -3.118983030319214,
+      "step": 5900
+    },
+    {
+      "epoch": 0.77,
+      "eval_logits/chosen": 1.2367382049560547,
+      "eval_logits/rejected": 1.7970218658447266,
+      "eval_logps/chosen": -462.8023681640625,
+      "eval_logps/rejected": -541.667236328125,
+      "eval_loss": 0.5036435127258301,
+      "eval_rewards/accuracies": 0.7335000038146973,
+      "eval_rewards/chosen": -1.9424282312393188,
+      "eval_rewards/margins": 0.9861749410629272,
+      "eval_rewards/rejected": -2.928602933883667,
+      "eval_runtime": 1349.0566,
+      "eval_samples_per_second": 1.483,
+      "eval_steps_per_second": 0.185,
+      "step": 5900
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.425593897573216e-07,
+      "logits/chosen": 1.2500572204589844,
+      "logits/rejected": 1.7614637613296509,
+      "logps/chosen": -429.0987854003906,
+      "logps/rejected": -520.119140625,
+      "loss": 0.4737,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.7603803873062134,
+      "rewards/margins": 0.9675158262252808,
+      "rewards/rejected": -2.727896213531494,
+      "step": 5910
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.344552368299088e-07,
+      "logits/chosen": 1.1057970523834229,
+      "logits/rejected": 1.6236732006072998,
+      "logps/chosen": -435.447509765625,
+      "logps/rejected": -565.1829223632812,
+      "loss": 0.4331,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.0624637603759766,
+      "rewards/margins": 1.2321687936782837,
+      "rewards/rejected": -3.2946324348449707,
+      "step": 5920
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.26387928996973e-07,
+      "logits/chosen": 1.1262363195419312,
+      "logits/rejected": 1.8808482885360718,
+      "logps/chosen": -455.239501953125,
+      "logps/rejected": -509.94140625,
+      "loss": 0.504,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.9708709716796875,
+      "rewards/margins": 0.8497422337532043,
+      "rewards/rejected": -2.820613145828247,
+      "step": 5930
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.183576346148899e-07,
+      "logits/chosen": 0.9126480221748352,
+      "logits/rejected": 1.415241003036499,
+      "logps/chosen": -462.6504821777344,
+      "logps/rejected": -562.460205078125,
+      "loss": 0.3995,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.7664072513580322,
+      "rewards/margins": 1.1610538959503174,
+      "rewards/rejected": -2.9274611473083496,
+      "step": 5940
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.103645212676044e-07,
+      "logits/chosen": 1.0327112674713135,
+      "logits/rejected": 1.5767911672592163,
+      "logps/chosen": -489.48492431640625,
+      "logps/rejected": -559.7766723632812,
+      "loss": 0.5194,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.9195845127105713,
+      "rewards/margins": 1.0718343257904053,
+      "rewards/rejected": -2.9914188385009766,
+      "step": 5950
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.024087557631318e-07,
+      "logits/chosen": 0.7663835287094116,
+      "logits/rejected": 1.4398205280303955,
+      "logps/chosen": -478.8651428222656,
+      "logps/rejected": -554.8570556640625,
+      "loss": 0.5265,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.0895943641662598,
+      "rewards/margins": 0.9745588302612305,
+      "rewards/rejected": -3.0641531944274902,
+      "step": 5960
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.944905041300739e-07,
+      "logits/chosen": 0.8068726658821106,
+      "logits/rejected": 1.432417869567871,
+      "logps/chosen": -482.126220703125,
+      "logps/rejected": -525.658203125,
+      "loss": 0.5332,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.8830230236053467,
+      "rewards/margins": 0.9656939506530762,
+      "rewards/rejected": -2.848716974258423,
+      "step": 5970
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.866099316141606e-07,
+      "logits/chosen": 1.1562412977218628,
+      "logits/rejected": 1.471755027770996,
+      "logps/chosen": -483.470703125,
+      "logps/rejected": -553.72021484375,
+      "loss": 0.5357,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.001695394515991,
+      "rewards/margins": 1.0708293914794922,
+      "rewards/rejected": -3.0725247859954834,
+      "step": 5980
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.787672026747946e-07,
+      "logits/chosen": 1.2011348009109497,
+      "logits/rejected": 1.8071174621582031,
+      "logps/chosen": -431.57208251953125,
+      "logps/rejected": -564.7637939453125,
+      "loss": 0.3997,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.7622629404067993,
+      "rewards/margins": 1.3659493923187256,
+      "rewards/rejected": -3.1282122135162354,
+      "step": 5990
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.709624809816223e-07,
+      "logits/chosen": 0.984458327293396,
+      "logits/rejected": 1.8598114252090454,
+      "logps/chosen": -456.0328063964844,
+      "logps/rejected": -522.6248779296875,
+      "loss": 0.4505,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.0299830436706543,
+      "rewards/margins": 1.026553988456726,
+      "rewards/rejected": -3.056536912918091,
+      "step": 6000
+    },
+    {
+      "epoch": 0.79,
+      "eval_logits/chosen": 1.3843281269073486,
+      "eval_logits/rejected": 1.9581965208053589,
+      "eval_logps/chosen": -467.3695373535156,
+      "eval_logps/rejected": -547.7703247070312,
+      "eval_loss": 0.5052816867828369,
+      "eval_rewards/accuracies": 0.7329999804496765,
+      "eval_rewards/chosen": -1.9880995750427246,
+      "eval_rewards/margins": 1.0015342235565186,
+      "eval_rewards/rejected": -2.989633560180664,
+      "eval_runtime": 1350.0676,
+      "eval_samples_per_second": 1.481,
+      "eval_steps_per_second": 0.185,
+      "step": 6000
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.6319592941112e-07,
+      "logits/chosen": 1.0520964860916138,
+      "logits/rejected": 1.612426519393921,
+      "logps/chosen": -476.35015869140625,
+      "logps/rejected": -540.3294067382812,
+      "loss": 0.4216,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -2.075939178466797,
+      "rewards/margins": 1.0815153121948242,
+      "rewards/rejected": -3.157454013824463,
+      "step": 6010
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.554677100431927e-07,
+      "logits/chosen": 1.1474238634109497,
+      "logits/rejected": 1.9181575775146484,
+      "logps/chosen": -427.70977783203125,
+      "logps/rejected": -536.0413208007812,
+      "loss": 0.4457,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.9163522720336914,
+      "rewards/margins": 1.3633421659469604,
+      "rewards/rejected": -3.2796943187713623,
+      "step": 6020
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.4777798415779e-07,
+      "logits/chosen": 0.471513032913208,
+      "logits/rejected": 1.3180567026138306,
+      "logps/chosen": -480.142578125,
+      "logps/rejected": -520.0164184570312,
+      "loss": 0.5356,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.9671579599380493,
+      "rewards/margins": 0.7752783894538879,
+      "rewards/rejected": -2.742436408996582,
+      "step": 6030
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.401269122315451e-07,
+      "logits/chosen": 1.3245112895965576,
+      "logits/rejected": 1.473232626914978,
+      "logps/chosen": -477.04949951171875,
+      "logps/rejected": -597.4310302734375,
+      "loss": 0.5312,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.1192777156829834,
+      "rewards/margins": 0.9577383995056152,
+      "rewards/rejected": -3.0770161151885986,
+      "step": 6040
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.325146539344196e-07,
+      "logits/chosen": 1.1838363409042358,
+      "logits/rejected": 2.0153896808624268,
+      "logps/chosen": -593.576416015625,
+      "logps/rejected": -631.4378051757812,
+      "loss": 0.5535,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.4238779544830322,
+      "rewards/margins": 0.9913672208786011,
+      "rewards/rejected": -3.4152450561523438,
+      "step": 6050
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.249413681263782e-07,
+      "logits/chosen": 1.2767165899276733,
+      "logits/rejected": 1.6955763101577759,
+      "logps/chosen": -403.29571533203125,
+      "logps/rejected": -589.2863159179688,
+      "loss": 0.5033,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.9824445247650146,
+      "rewards/margins": 1.3527143001556396,
+      "rewards/rejected": -3.3351588249206543,
+      "step": 6060
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.174072128540686e-07,
+      "logits/chosen": 0.6846606731414795,
+      "logits/rejected": 1.2512080669403076,
+      "logps/chosen": -479.2460021972656,
+      "logps/rejected": -514.6866455078125,
+      "loss": 0.5369,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.8355529308319092,
+      "rewards/margins": 0.852269172668457,
+      "rewards/rejected": -2.687821865081787,
+      "step": 6070
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.099123453475245e-07,
+      "logits/chosen": 0.9443454742431641,
+      "logits/rejected": 1.8545773029327393,
+      "logps/chosen": -497.82220458984375,
+      "logps/rejected": -556.7174682617188,
+      "loss": 0.5242,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.1323070526123047,
+      "rewards/margins": 0.8927971124649048,
+      "rewards/rejected": -3.02510404586792,
+      "step": 6080
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.024569220168836e-07,
+      "logits/chosen": 1.1339746713638306,
+      "logits/rejected": 2.2085318565368652,
+      "logps/chosen": -463.0975646972656,
+      "logps/rejected": -556.5795288085938,
+      "loss": 0.396,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.7871935367584229,
+      "rewards/margins": 1.4249250888824463,
+      "rewards/rejected": -3.212118625640869,
+      "step": 6090
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.950410984491268e-07,
+      "logits/chosen": 1.2697426080703735,
+      "logits/rejected": 1.555204153060913,
+      "logps/chosen": -508.49249267578125,
+      "logps/rejected": -589.2627563476562,
+      "loss": 0.5197,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.2372868061065674,
+      "rewards/margins": 0.8701568841934204,
+      "rewards/rejected": -3.1074435710906982,
+      "step": 6100
+    },
+    {
+      "epoch": 0.8,
+      "eval_logits/chosen": 1.4340654611587524,
+      "eval_logits/rejected": 2.0118377208709717,
+      "eval_logps/chosen": -470.9445495605469,
+      "eval_logps/rejected": -552.71533203125,
+      "eval_loss": 0.5070981979370117,
+      "eval_rewards/accuracies": 0.7315000295639038,
+      "eval_rewards/chosen": -2.0238494873046875,
+      "eval_rewards/margins": 1.0152342319488525,
+      "eval_rewards/rejected": -3.03908371925354,
+      "eval_runtime": 1348.3734,
+      "eval_samples_per_second": 1.483,
+      "eval_steps_per_second": 0.185,
+      "step": 6100
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.876650294048262e-07,
+      "logits/chosen": 1.4878935813903809,
+      "logits/rejected": 1.7394554615020752,
+      "logps/chosen": -428.736572265625,
+      "logps/rejected": -516.4510498046875,
+      "loss": 0.5539,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.9505386352539062,
+      "rewards/margins": 0.8216649889945984,
+      "rewards/rejected": -2.7722036838531494,
+      "step": 6110
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.8032886881492e-07,
+      "logits/chosen": 1.517862319946289,
+      "logits/rejected": 2.2016451358795166,
+      "logps/chosen": -410.40765380859375,
+      "logps/rejected": -473.9432678222656,
+      "loss": 0.586,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.9991931915283203,
+      "rewards/margins": 0.8155339360237122,
+      "rewards/rejected": -2.8147268295288086,
+      "step": 6120
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.730327697774988e-07,
+      "logits/chosen": 1.3214466571807861,
+      "logits/rejected": 1.6739966869354248,
+      "logps/chosen": -439.2711486816406,
+      "logps/rejected": -571.9884033203125,
+      "loss": 0.4368,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.9049676656723022,
+      "rewards/margins": 1.112154245376587,
+      "rewards/rejected": -3.0171217918395996,
+      "step": 6130
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.657768845546068e-07,
+      "logits/chosen": 1.445155382156372,
+      "logits/rejected": 2.029951572418213,
+      "logps/chosen": -401.62255859375,
+      "logps/rejected": -504.14306640625,
+      "loss": 0.4885,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.9933397769927979,
+      "rewards/margins": 1.0306072235107422,
+      "rewards/rejected": -3.02394700050354,
+      "step": 6140
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.585613645690713e-07,
+      "logits/chosen": 1.192810297012329,
+      "logits/rejected": 1.7521553039550781,
+      "logps/chosen": -450.53057861328125,
+      "logps/rejected": -565.3319702148438,
+      "loss": 0.4309,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.0455074310302734,
+      "rewards/margins": 1.2288362979888916,
+      "rewards/rejected": -3.274343490600586,
+      "step": 6150
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.513863604013355e-07,
+      "logits/chosen": 0.9030055999755859,
+      "logits/rejected": 1.8799047470092773,
+      "logps/chosen": -525.406494140625,
+      "logps/rejected": -556.5892333984375,
+      "loss": 0.6402,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.340468406677246,
+      "rewards/margins": 0.8067280650138855,
+      "rewards/rejected": -3.1471967697143555,
+      "step": 6160
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.442520217863215e-07,
+      "logits/chosen": 1.4492144584655762,
+      "logits/rejected": 1.909208059310913,
+      "logps/chosen": -454.86322021484375,
+      "logps/rejected": -540.5611572265625,
+      "loss": 0.5375,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.090507984161377,
+      "rewards/margins": 1.020002841949463,
+      "rewards/rejected": -3.110511064529419,
+      "step": 6170
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.371584976103034e-07,
+      "logits/chosen": 0.9736919403076172,
+      "logits/rejected": 1.8472111225128174,
+      "logps/chosen": -454.79864501953125,
+      "logps/rejected": -586.4794921875,
+      "loss": 0.4084,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.0333003997802734,
+      "rewards/margins": 1.345563530921936,
+      "rewards/rejected": -3.378864288330078,
+      "step": 6180
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.301059359077987e-07,
+      "logits/chosen": 1.7344433069229126,
+      "logits/rejected": 2.225065231323242,
+      "logps/chosen": -429.40972900390625,
+      "logps/rejected": -530.9122314453125,
+      "loss": 0.5617,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.9893436431884766,
+      "rewards/margins": 1.059349775314331,
+      "rewards/rejected": -3.0486934185028076,
+      "step": 6190
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.230944838584806e-07,
+      "logits/chosen": 0.998323917388916,
+      "logits/rejected": 1.474500060081482,
+      "logps/chosen": -508.4969177246094,
+      "logps/rejected": -578.9632568359375,
+      "loss": 0.6046,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.241689682006836,
+      "rewards/margins": 0.8613544702529907,
+      "rewards/rejected": -3.103044033050537,
+      "step": 6200
+    },
+    {
+      "epoch": 0.81,
+      "eval_logits/chosen": 1.5328489542007446,
+      "eval_logits/rejected": 2.1151063442230225,
+      "eval_logps/chosen": -476.5938720703125,
+      "eval_logps/rejected": -559.9707641601562,
+      "eval_loss": 0.5063574314117432,
+      "eval_rewards/accuracies": 0.7310000061988831,
+      "eval_rewards/chosen": -2.080343246459961,
+      "eval_rewards/margins": 1.031294822692871,
+      "eval_rewards/rejected": -3.111637830734253,
+      "eval_runtime": 1349.6517,
+      "eval_samples_per_second": 1.482,
+      "eval_steps_per_second": 0.185,
+      "step": 6200
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.161242877841083e-07,
+      "logits/chosen": 0.9407013654708862,
+      "logits/rejected": 1.7394936084747314,
+      "logps/chosen": -485.3163146972656,
+      "logps/rejected": -582.8829345703125,
+      "loss": 0.4519,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.159008502960205,
+      "rewards/margins": 1.259042501449585,
+      "rewards/rejected": -3.418050765991211,
+      "step": 6210
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.091954931454682e-07,
+      "logits/chosen": 1.2999640703201294,
+      "logits/rejected": 1.8846088647842407,
+      "logps/chosen": -455.0072326660156,
+      "logps/rejected": -548.2179565429688,
+      "loss": 0.4716,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.9322246313095093,
+      "rewards/margins": 1.0663868188858032,
+      "rewards/rejected": -2.9986114501953125,
+      "step": 6220
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 5.023082445393446e-07,
+      "logits/chosen": 1.3162462711334229,
+      "logits/rejected": 2.076570987701416,
+      "logps/chosen": -451.0628967285156,
+      "logps/rejected": -539.8049926757812,
+      "loss": 0.4103,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.8992286920547485,
+      "rewards/margins": 1.3221932649612427,
+      "rewards/rejected": -3.2214221954345703,
+      "step": 6230
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.95462685695498e-07,
+      "logits/chosen": 0.9100130796432495,
+      "logits/rejected": 1.6328957080841064,
+      "logps/chosen": -481.1402282714844,
+      "logps/rejected": -540.2344970703125,
+      "loss": 0.516,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.1461784839630127,
+      "rewards/margins": 1.0078233480453491,
+      "rewards/rejected": -3.1540021896362305,
+      "step": 6240
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.88658959473666e-07,
+      "logits/chosen": 1.2769775390625,
+      "logits/rejected": 2.1803529262542725,
+      "logps/chosen": -480.28839111328125,
+      "logps/rejected": -548.637451171875,
+      "loss": 0.457,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.9999210834503174,
+      "rewards/margins": 1.1817798614501953,
+      "rewards/rejected": -3.1817009449005127,
+      "step": 6250
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.818972078605821e-07,
+      "logits/chosen": 1.1566002368927002,
+      "logits/rejected": 2.3512704372406006,
+      "logps/chosen": -508.625,
+      "logps/rejected": -562.2999877929688,
+      "loss": 0.4531,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.007694721221924,
+      "rewards/margins": 1.3290417194366455,
+      "rewards/rejected": -3.336735963821411,
+      "step": 6260
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.7517757196701514e-07,
+      "logits/chosen": 0.931840717792511,
+      "logits/rejected": 1.811882734298706,
+      "logps/chosen": -425.78277587890625,
+      "logps/rejected": -546.6446533203125,
+      "loss": 0.4263,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.9187036752700806,
+      "rewards/margins": 1.4656442403793335,
+      "rewards/rejected": -3.384347915649414,
+      "step": 6270
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.6850019202482193e-07,
+      "logits/chosen": 1.4084501266479492,
+      "logits/rejected": 2.1319353580474854,
+      "logps/chosen": -423.0577087402344,
+      "logps/rejected": -514.2741088867188,
+      "loss": 0.4873,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.923856496810913,
+      "rewards/margins": 1.0924780368804932,
+      "rewards/rejected": -3.0163347721099854,
+      "step": 6280
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.618652073840188e-07,
+      "logits/chosen": 0.9233554601669312,
+      "logits/rejected": 1.7832212448120117,
+      "logps/chosen": -498.699951171875,
+      "logps/rejected": -563.3565063476562,
+      "loss": 0.5303,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.065371036529541,
+      "rewards/margins": 1.102463722229004,
+      "rewards/rejected": -3.167834758758545,
+      "step": 6290
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.5527275650987965e-07,
+      "logits/chosen": 1.077385663986206,
+      "logits/rejected": 1.7431968450546265,
+      "logps/chosen": -469.2884216308594,
+      "logps/rejected": -549.5968017578125,
+      "loss": 0.4669,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.0613627433776855,
+      "rewards/margins": 1.1245476007461548,
+      "rewards/rejected": -3.18591046333313,
+      "step": 6300
+    },
+    {
+      "epoch": 0.82,
+      "eval_logits/chosen": 1.639417290687561,
+      "eval_logits/rejected": 2.2263967990875244,
+      "eval_logps/chosen": -478.6570129394531,
+      "eval_logps/rejected": -564.2191772460938,
+      "eval_loss": 0.5071954727172852,
+      "eval_rewards/accuracies": 0.7310000061988831,
+      "eval_rewards/chosen": -2.1009740829467773,
+      "eval_rewards/margins": 1.0531479120254517,
+      "eval_rewards/rejected": -3.1541216373443604,
+      "eval_runtime": 1350.0942,
+      "eval_samples_per_second": 1.481,
+      "eval_steps_per_second": 0.185,
+      "step": 6300
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.487229769800394e-07,
+      "logits/chosen": 1.7093207836151123,
+      "logits/rejected": 2.278848886489868,
+      "logps/chosen": -429.5550231933594,
+      "logps/rejected": -490.27716064453125,
+      "loss": 0.5977,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.0858054161071777,
+      "rewards/margins": 0.786652684211731,
+      "rewards/rejected": -2.872457981109619,
+      "step": 6310
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.422160054816285e-07,
+      "logits/chosen": 1.4057409763336182,
+      "logits/rejected": 2.069688320159912,
+      "logps/chosen": -509.1127014160156,
+      "logps/rejected": -585.1344604492188,
+      "loss": 0.5564,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.159127712249756,
+      "rewards/margins": 0.8664734959602356,
+      "rewards/rejected": -3.025601387023926,
+      "step": 6320
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.35751977808416e-07,
+      "logits/chosen": 1.1788089275360107,
+      "logits/rejected": 1.9736534357070923,
+      "logps/chosen": -458.32586669921875,
+      "logps/rejected": -573.1961059570312,
+      "loss": 0.3739,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.9391695261001587,
+      "rewards/margins": 1.4471466541290283,
+      "rewards/rejected": -3.3863162994384766,
+      "step": 6330
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.293310288579794e-07,
+      "logits/chosen": 1.9138660430908203,
+      "logits/rejected": 2.005502700805664,
+      "logps/chosen": -459.32220458984375,
+      "logps/rejected": -547.9752807617188,
+      "loss": 0.5509,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.1588072776794434,
+      "rewards/margins": 0.9634833335876465,
+      "rewards/rejected": -3.122290849685669,
+      "step": 6340
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.2295329262888733e-07,
+      "logits/chosen": 1.659541130065918,
+      "logits/rejected": 2.415644407272339,
+      "logps/chosen": -466.9901428222656,
+      "logps/rejected": -520.4453125,
+      "loss": 0.5662,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.1395952701568604,
+      "rewards/margins": 0.8698142170906067,
+      "rewards/rejected": -3.0094096660614014,
+      "step": 6350
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.1661890221790316e-07,
+      "logits/chosen": 1.3984253406524658,
+      "logits/rejected": 1.5648412704467773,
+      "logps/chosen": -435.675048828125,
+      "logps/rejected": -593.08935546875,
+      "loss": 0.4144,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.882380723953247,
+      "rewards/margins": 1.3236576318740845,
+      "rewards/rejected": -3.206038236618042,
+      "step": 6360
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.103279898172072e-07,
+      "logits/chosen": 1.0725916624069214,
+      "logits/rejected": 2.069240093231201,
+      "logps/chosen": -529.6309814453125,
+      "logps/rejected": -565.3829345703125,
+      "loss": 0.5725,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.2158706188201904,
+      "rewards/margins": 0.926199734210968,
+      "rewards/rejected": -3.1420705318450928,
+      "step": 6370
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.040806867116401e-07,
+      "logits/chosen": 1.1758654117584229,
+      "logits/rejected": 1.4364426136016846,
+      "logps/chosen": -476.8794860839844,
+      "logps/rejected": -546.7584838867188,
+      "loss": 0.5121,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.073026418685913,
+      "rewards/margins": 0.8902866244316101,
+      "rewards/rejected": -2.963313102722168,
+      "step": 6380
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.978771232759615e-07,
+      "logits/chosen": 1.464595913887024,
+      "logits/rejected": 1.8888301849365234,
+      "logps/chosen": -442.5675354003906,
+      "logps/rejected": -540.0778198242188,
+      "loss": 0.5456,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.1246681213378906,
+      "rewards/margins": 0.981758713722229,
+      "rewards/rejected": -3.106426477432251,
+      "step": 6390
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.917174289721276e-07,
+      "logits/chosen": 1.4766385555267334,
+      "logits/rejected": 1.8044464588165283,
+      "logps/chosen": -479.61065673828125,
+      "logps/rejected": -530.1144409179688,
+      "loss": 0.5631,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.0680642127990723,
+      "rewards/margins": 0.7943180799484253,
+      "rewards/rejected": -2.862382411956787,
+      "step": 6400
+    },
+    {
+      "epoch": 0.84,
+      "eval_logits/chosen": 1.723006248474121,
+      "eval_logits/rejected": 2.3071978092193604,
+      "eval_logps/chosen": -477.9385070800781,
+      "eval_logps/rejected": -562.65283203125,
+      "eval_loss": 0.5055004358291626,
+      "eval_rewards/accuracies": 0.7304999828338623,
+      "eval_rewards/chosen": -2.0937891006469727,
+      "eval_rewards/margins": 1.0446698665618896,
+      "eval_rewards/rejected": -3.1384589672088623,
+      "eval_runtime": 1350.1999,
+      "eval_samples_per_second": 1.481,
+      "eval_steps_per_second": 0.185,
+      "step": 6400
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.856017323465938e-07,
+      "logits/chosen": 1.690515160560608,
+      "logits/rejected": 2.077540397644043,
+      "logps/chosen": -448.4349060058594,
+      "logps/rejected": -553.067626953125,
+      "loss": 0.4925,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.210909843444824,
+      "rewards/margins": 1.1146156787872314,
+      "rewards/rejected": -3.3255257606506348,
+      "step": 6410
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.7953016102762695e-07,
+      "logits/chosen": 1.6206096410751343,
+      "logits/rejected": 1.8189092874526978,
+      "logps/chosen": -481.15802001953125,
+      "logps/rejected": -543.9744262695312,
+      "loss": 0.5612,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.1497743129730225,
+      "rewards/margins": 0.8574525117874146,
+      "rewards/rejected": -3.0072269439697266,
+      "step": 6420
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.7350284172264493e-07,
+      "logits/chosen": 1.571506142616272,
+      "logits/rejected": 1.7734636068344116,
+      "logps/chosen": -433.11676025390625,
+      "logps/rejected": -560.2884521484375,
+      "loss": 0.5001,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.110581159591675,
+      "rewards/margins": 1.0816186666488647,
+      "rewards/rejected": -3.19219970703125,
+      "step": 6430
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.67519900215573e-07,
+      "logits/chosen": 1.3697388172149658,
+      "logits/rejected": 2.2321481704711914,
+      "logps/chosen": -455.712158203125,
+      "logps/rejected": -542.8861083984375,
+      "loss": 0.4719,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.9866927862167358,
+      "rewards/margins": 1.1989667415618896,
+      "rewards/rejected": -3.185659885406494,
+      "step": 6440
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.615814613642174e-07,
+      "logits/chosen": 1.1395235061645508,
+      "logits/rejected": 2.379185438156128,
+      "logps/chosen": -511.09857177734375,
+      "logps/rejected": -540.2202758789062,
+      "loss": 0.5616,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.0880825519561768,
+      "rewards/margins": 0.9573913812637329,
+      "rewards/rejected": -3.04547381401062,
+      "step": 6450
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.5568764909765795e-07,
+      "logits/chosen": 1.3384441137313843,
+      "logits/rejected": 2.1021268367767334,
+      "logps/chosen": -470.8004455566406,
+      "logps/rejected": -557.8414916992188,
+      "loss": 0.46,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.9508905410766602,
+      "rewards/margins": 1.076872706413269,
+      "rewards/rejected": -3.0277633666992188,
+      "step": 6460
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.498385864136672e-07,
+      "logits/chosen": 1.3269442319869995,
+      "logits/rejected": 1.879699468612671,
+      "logps/chosen": -483.760498046875,
+      "logps/rejected": -564.81982421875,
+      "loss": 0.5612,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.265338182449341,
+      "rewards/margins": 0.8522366285324097,
+      "rewards/rejected": -3.11757493019104,
+      "step": 6470
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.440343953761363e-07,
+      "logits/chosen": 1.5149481296539307,
+      "logits/rejected": 1.9883750677108765,
+      "logps/chosen": -462.28729248046875,
+      "logps/rejected": -509.1297302246094,
+      "loss": 0.6418,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.129033088684082,
+      "rewards/margins": 0.718868613243103,
+      "rewards/rejected": -2.8479018211364746,
+      "step": 6480
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.382751971125345e-07,
+      "logits/chosen": 1.8467496633529663,
+      "logits/rejected": 2.180973529815674,
+      "logps/chosen": -453.90869140625,
+      "logps/rejected": -518.36865234375,
+      "loss": 0.5721,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.0848388671875,
+      "rewards/margins": 0.8801206350326538,
+      "rewards/rejected": -2.9649596214294434,
+      "step": 6490
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.3256111181137753e-07,
+      "logits/chosen": 1.1316883563995361,
+      "logits/rejected": 1.827903389930725,
+      "logps/chosen": -451.13385009765625,
+      "logps/rejected": -528.8464965820312,
+      "loss": 0.433,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.0000321865081787,
+      "rewards/margins": 1.0588886737823486,
+      "rewards/rejected": -3.0589210987091064,
+      "step": 6500
+    },
+    {
+      "epoch": 0.85,
+      "eval_logits/chosen": 1.6963303089141846,
+      "eval_logits/rejected": 2.276027202606201,
+      "eval_logps/chosen": -474.8586120605469,
+      "eval_logps/rejected": -558.1637573242188,
+      "eval_loss": 0.504417359828949,
+      "eval_rewards/accuracies": 0.7289999723434448,
+      "eval_rewards/chosen": -2.062990427017212,
+      "eval_rewards/margins": 1.0305776596069336,
+      "eval_rewards/rejected": -3.0935683250427246,
+      "eval_runtime": 1350.0169,
+      "eval_samples_per_second": 1.481,
+      "eval_steps_per_second": 0.185,
+      "step": 6500
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.2689225871971905e-07,
+      "logits/chosen": 1.1778087615966797,
+      "logits/rejected": 1.671144723892212,
+      "logps/chosen": -481.48919677734375,
+      "logps/rejected": -598.1636352539062,
+      "loss": 0.5961,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.131434917449951,
+      "rewards/margins": 1.1485437154769897,
+      "rewards/rejected": -3.2799789905548096,
+      "step": 6510
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.2126875614066523e-07,
+      "logits/chosen": 1.400327205657959,
+      "logits/rejected": 1.8389770984649658,
+      "logps/chosen": -442.63067626953125,
+      "logps/rejected": -504.4331970214844,
+      "loss": 0.5526,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.9644542932510376,
+      "rewards/margins": 0.8293323516845703,
+      "rewards/rejected": -2.7937865257263184,
+      "step": 6520
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.156907214309024e-07,
+      "logits/chosen": 1.2499854564666748,
+      "logits/rejected": 1.9847021102905273,
+      "logps/chosen": -441.12860107421875,
+      "logps/rejected": -553.3264770507812,
+      "loss": 0.4618,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.052786350250244,
+      "rewards/margins": 1.1508277654647827,
+      "rewards/rejected": -3.2036144733428955,
+      "step": 6530
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.1015827099824923e-07,
+      "logits/chosen": 0.9569733738899231,
+      "logits/rejected": 1.9669069051742554,
+      "logps/chosen": -489.81878662109375,
+      "logps/rejected": -555.5958251953125,
+      "loss": 0.4936,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.1646628379821777,
+      "rewards/margins": 0.9500352144241333,
+      "rewards/rejected": -3.1146979331970215,
+      "step": 6540
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.0467152029922926e-07,
+      "logits/chosen": 1.7571938037872314,
+      "logits/rejected": 1.9496371746063232,
+      "logps/chosen": -452.1244201660156,
+      "logps/rejected": -569.55078125,
+      "loss": 0.5703,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.2034170627593994,
+      "rewards/margins": 0.8420044779777527,
+      "rewards/rejected": -3.0454213619232178,
+      "step": 6550
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.992305838366591e-07,
+      "logits/chosen": 1.0713802576065063,
+      "logits/rejected": 1.689044713973999,
+      "logps/chosen": -509.22283935546875,
+      "logps/rejected": -588.2781372070312,
+      "loss": 0.4756,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.0089187622070312,
+      "rewards/margins": 1.1974365711212158,
+      "rewards/rejected": -3.206355333328247,
+      "step": 6560
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.938355751572583e-07,
+      "logits/chosen": 1.6999753713607788,
+      "logits/rejected": 2.4180939197540283,
+      "logps/chosen": -482.21392822265625,
+      "logps/rejected": -562.8911743164062,
+      "loss": 0.6033,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.192016839981079,
+      "rewards/margins": 1.1564048528671265,
+      "rewards/rejected": -3.348421573638916,
+      "step": 6570
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.8848660684928307e-07,
+      "logits/chosen": 1.4272618293762207,
+      "logits/rejected": 2.1681792736053467,
+      "logps/chosen": -488.7395935058594,
+      "logps/rejected": -543.8782958984375,
+      "loss": 0.5111,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.107006072998047,
+      "rewards/margins": 0.9169102907180786,
+      "rewards/rejected": -3.023916244506836,
+      "step": 6580
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.8318379054017383e-07,
+      "logits/chosen": 1.4116415977478027,
+      "logits/rejected": 2.233696699142456,
+      "logps/chosen": -479.3365173339844,
+      "logps/rejected": -544.3157348632812,
+      "loss": 0.4514,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.9850895404815674,
+      "rewards/margins": 1.249487280845642,
+      "rewards/rejected": -3.23457670211792,
+      "step": 6590
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.779272368942246e-07,
+      "logits/chosen": 1.1742923259735107,
+      "logits/rejected": 2.0967671871185303,
+      "logps/chosen": -488.69146728515625,
+      "logps/rejected": -583.5924682617188,
+      "loss": 0.4908,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.047158718109131,
+      "rewards/margins": 1.2081985473632812,
+      "rewards/rejected": -3.255357027053833,
+      "step": 6600
+    },
+    {
+      "epoch": 0.86,
+      "eval_logits/chosen": 1.7556606531143188,
+      "eval_logits/rejected": 2.3343164920806885,
+      "eval_logps/chosen": -474.2539978027344,
+      "eval_logps/rejected": -557.4365234375,
+      "eval_loss": 0.5042900443077087,
+      "eval_rewards/accuracies": 0.7294999957084656,
+      "eval_rewards/chosen": -2.0569441318511963,
+      "eval_rewards/margins": 1.0293512344360352,
+      "eval_rewards/rejected": -3.0862953662872314,
+      "eval_runtime": 1349.9153,
+      "eval_samples_per_second": 1.482,
+      "eval_steps_per_second": 0.185,
+      "step": 6600
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.7271705561027986e-07,
+      "logits/chosen": 1.12348210811615,
+      "logits/rejected": 2.082200288772583,
+      "logps/chosen": -509.3050842285156,
+      "logps/rejected": -559.39453125,
+      "loss": 0.4986,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.0815956592559814,
+      "rewards/margins": 1.0864602327346802,
+      "rewards/rejected": -3.168056011199951,
+      "step": 6610
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.6755335541943677e-07,
+      "logits/chosen": 1.6275326013565063,
+      "logits/rejected": 1.7634668350219727,
+      "logps/chosen": -476.7122497558594,
+      "logps/rejected": -548.7210693359375,
+      "loss": 0.5774,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.170102596282959,
+      "rewards/margins": 0.7067625522613525,
+      "rewards/rejected": -2.8768656253814697,
+      "step": 6620
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.62436244082781e-07,
+      "logits/chosen": 1.2020237445831299,
+      "logits/rejected": 1.5702972412109375,
+      "logps/chosen": -497.9236755371094,
+      "logps/rejected": -597.7046508789062,
+      "loss": 0.4128,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.095738172531128,
+      "rewards/margins": 1.2979549169540405,
+      "rewards/rejected": -3.393693447113037,
+      "step": 6630
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.5736582838913836e-07,
+      "logits/chosen": 1.3650001287460327,
+      "logits/rejected": 2.2222132682800293,
+      "logps/chosen": -473.80645751953125,
+      "logps/rejected": -559.5433959960938,
+      "loss": 0.4482,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.018864393234253,
+      "rewards/margins": 1.3088419437408447,
+      "rewards/rejected": -3.3277060985565186,
+      "step": 6640
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.5234221415284363e-07,
+      "logits/chosen": 1.630008339881897,
+      "logits/rejected": 2.1548168659210205,
+      "logps/chosen": -485.123046875,
+      "logps/rejected": -586.856201171875,
+      "loss": 0.5472,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.0703234672546387,
+      "rewards/margins": 0.9899441599845886,
+      "rewards/rejected": -3.060267686843872,
+      "step": 6650
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.4736550621153375e-07,
+      "logits/chosen": 2.019033193588257,
+      "logits/rejected": 2.8152642250061035,
+      "logps/chosen": -448.20367431640625,
+      "logps/rejected": -559.1297607421875,
+      "loss": 0.4668,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.152010679244995,
+      "rewards/margins": 1.2867562770843506,
+      "rewards/rejected": -3.4387669563293457,
+      "step": 6660
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.424358084239609e-07,
+      "logits/chosen": 1.4462095499038696,
+      "logits/rejected": 1.5666110515594482,
+      "logps/chosen": -463.9244689941406,
+      "logps/rejected": -532.5508422851562,
+      "loss": 0.5804,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.0727927684783936,
+      "rewards/margins": 0.66776043176651,
+      "rewards/rejected": -2.740553379058838,
+      "step": 6670
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.3755322366782158e-07,
+      "logits/chosen": 1.5529290437698364,
+      "logits/rejected": 2.2098095417022705,
+      "logps/chosen": -459.36138916015625,
+      "logps/rejected": -547.589111328125,
+      "loss": 0.466,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.897678017616272,
+      "rewards/margins": 1.2030918598175049,
+      "rewards/rejected": -3.100769519805908,
+      "step": 6680
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.3271785383761431e-07,
+      "logits/chosen": 1.3178908824920654,
+      "logits/rejected": 1.6003563404083252,
+      "logps/chosen": -476.467529296875,
+      "logps/rejected": -509.8314514160156,
+      "loss": 0.6207,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.1006667613983154,
+      "rewards/margins": 0.5864391326904297,
+      "rewards/rejected": -2.687105894088745,
+      "step": 6690
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.2792979984250978e-07,
+      "logits/chosen": 1.141798734664917,
+      "logits/rejected": 1.843541145324707,
+      "logps/chosen": -481.78582763671875,
+      "logps/rejected": -526.7293090820312,
+      "loss": 0.522,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.9672702550888062,
+      "rewards/margins": 0.9249141812324524,
+      "rewards/rejected": -2.8921844959259033,
+      "step": 6700
+    },
+    {
+      "epoch": 0.88,
+      "eval_logits/chosen": 1.7706228494644165,
+      "eval_logits/rejected": 2.3468830585479736,
+      "eval_logps/chosen": -476.112548828125,
+      "eval_logps/rejected": -559.4036865234375,
+      "eval_loss": 0.503921627998352,
+      "eval_rewards/accuracies": 0.7300000190734863,
+      "eval_rewards/chosen": -2.0755295753479004,
+      "eval_rewards/margins": 1.0304373502731323,
+      "eval_rewards/rejected": -3.1059670448303223,
+      "eval_runtime": 1348.9247,
+      "eval_samples_per_second": 1.483,
+      "eval_steps_per_second": 0.185,
+      "step": 6700
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.231891616042453e-07,
+      "logits/chosen": 0.9167200326919556,
+      "logits/rejected": 1.595862627029419,
+      "logps/chosen": -490.7945861816406,
+      "logps/rejected": -595.1524658203125,
+      "loss": 0.5071,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.1621007919311523,
+      "rewards/margins": 1.162212610244751,
+      "rewards/rejected": -3.3243136405944824,
+      "step": 6710
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.1849603805504328e-07,
+      "logits/chosen": 0.9640613794326782,
+      "logits/rejected": 1.7189490795135498,
+      "logps/chosen": -480.1543884277344,
+      "logps/rejected": -548.0877685546875,
+      "loss": 0.5229,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.9537487030029297,
+      "rewards/margins": 0.9588750004768372,
+      "rewards/rejected": -2.912623643875122,
+      "step": 6720
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.1385052713554066e-07,
+      "logits/chosen": 2.030742645263672,
+      "logits/rejected": 2.115102767944336,
+      "logps/chosen": -448.92681884765625,
+      "logps/rejected": -575.0702514648438,
+      "loss": 0.4615,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.055258274078369,
+      "rewards/margins": 1.2751580476760864,
+      "rewards/rejected": -3.330416440963745,
+      "step": 6730
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.0925272579274873e-07,
+      "logits/chosen": 1.3698285818099976,
+      "logits/rejected": 1.5558454990386963,
+      "logps/chosen": -459.74664306640625,
+      "logps/rejected": -582.8912353515625,
+      "loss": 0.4538,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.0338211059570312,
+      "rewards/margins": 1.2058879137039185,
+      "rewards/rejected": -3.2397091388702393,
+      "step": 6740
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.047027299780302e-07,
+      "logits/chosen": 1.4781476259231567,
+      "logits/rejected": 1.8085733652114868,
+      "logps/chosen": -505.6385192871094,
+      "logps/rejected": -547.7869262695312,
+      "loss": 0.5994,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.114323139190674,
+      "rewards/margins": 0.7734920978546143,
+      "rewards/rejected": -2.887815475463867,
+      "step": 6750
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.0020063464509492e-07,
+      "logits/chosen": 1.7144548892974854,
+      "logits/rejected": 2.051178455352783,
+      "logps/chosen": -447.4488220214844,
+      "logps/rejected": -549.2349243164062,
+      "loss": 0.3953,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.8397657871246338,
+      "rewards/margins": 1.2791472673416138,
+      "rewards/rejected": -3.1189124584198,
+      "step": 6760
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.957465337480191e-07,
+      "logits/chosen": 1.4801374673843384,
+      "logits/rejected": 2.172884941101074,
+      "logps/chosen": -499.571044921875,
+      "logps/rejected": -584.143310546875,
+      "loss": 0.5172,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.0284764766693115,
+      "rewards/margins": 1.0863863229751587,
+      "rewards/rejected": -3.1148629188537598,
+      "step": 6770
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.9134052023928622e-07,
+      "logits/chosen": 1.6794211864471436,
+      "logits/rejected": 2.0311288833618164,
+      "logps/chosen": -463.77728271484375,
+      "logps/rejected": -554.3758544921875,
+      "loss": 0.5269,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.2532241344451904,
+      "rewards/margins": 0.9340078234672546,
+      "rewards/rejected": -3.1872317790985107,
+      "step": 6780
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.8698268606784392e-07,
+      "logits/chosen": 1.573906660079956,
+      "logits/rejected": 1.8606685400009155,
+      "logps/chosen": -440.43316650390625,
+      "logps/rejected": -563.8428955078125,
+      "loss": 0.4891,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.089843511581421,
+      "rewards/margins": 1.2432950735092163,
+      "rewards/rejected": -3.3331387042999268,
+      "step": 6790
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.826731221771866e-07,
+      "logits/chosen": 1.3942010402679443,
+      "logits/rejected": 2.3982625007629395,
+      "logps/chosen": -499.33160400390625,
+      "logps/rejected": -573.6112060546875,
+      "loss": 0.4953,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.1394927501678467,
+      "rewards/margins": 1.2987289428710938,
+      "rewards/rejected": -3.4382221698760986,
+      "step": 6800
+    },
+    {
+      "epoch": 0.89,
+      "eval_logits/chosen": 1.8129349946975708,
+      "eval_logits/rejected": 2.38814115524292,
+      "eval_logps/chosen": -477.73876953125,
+      "eval_logps/rejected": -561.1604614257812,
+      "eval_loss": 0.5038527250289917,
+      "eval_rewards/accuracies": 0.7289999723434448,
+      "eval_rewards/chosen": -2.0917916297912598,
+      "eval_rewards/margins": 1.0317424535751343,
+      "eval_rewards/rejected": -3.1235344409942627,
+      "eval_runtime": 1350.6063,
+      "eval_samples_per_second": 1.481,
+      "eval_steps_per_second": 0.185,
+      "step": 6800
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.7841191850345967e-07,
+      "logits/chosen": 1.6474618911743164,
+      "logits/rejected": 2.0439670085906982,
+      "logps/chosen": -478.4677734375,
+      "logps/rejected": -617.0116577148438,
+      "loss": 0.4235,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.031907796859741,
+      "rewards/margins": 1.4322130680084229,
+      "rewards/rejected": -3.464120864868164,
+      "step": 6810
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.7419916397357905e-07,
+      "logits/chosen": 1.182619571685791,
+      "logits/rejected": 2.2269234657287598,
+      "logps/chosen": -515.2547607421875,
+      "logps/rejected": -566.9799194335938,
+      "loss": 0.5674,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.317580223083496,
+      "rewards/margins": 0.9180108904838562,
+      "rewards/rejected": -3.235590696334839,
+      "step": 6820
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.700349465033782e-07,
+      "logits/chosen": 1.376081109046936,
+      "logits/rejected": 1.829991340637207,
+      "logps/chosen": -479.0719299316406,
+      "logps/rejected": -554.9691162109375,
+      "loss": 0.5019,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.0843653678894043,
+      "rewards/margins": 0.9625455737113953,
+      "rewards/rejected": -3.0469107627868652,
+      "step": 6830
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.6591935299577227e-07,
+      "logits/chosen": 1.6951709985733032,
+      "logits/rejected": 2.2439935207366943,
+      "logps/chosen": -455.78594970703125,
+      "logps/rejected": -516.5980834960938,
+      "loss": 0.439,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.9713783264160156,
+      "rewards/margins": 1.1705067157745361,
+      "rewards/rejected": -3.1418850421905518,
+      "step": 6840
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.6185246933894338e-07,
+      "logits/chosen": 1.4182054996490479,
+      "logits/rejected": 1.6353418827056885,
+      "logps/chosen": -465.1398010253906,
+      "logps/rejected": -548.318359375,
+      "loss": 0.5583,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.003892660140991,
+      "rewards/margins": 0.9650713801383972,
+      "rewards/rejected": -2.968964099884033,
+      "step": 6850
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.5783438040455097e-07,
+      "logits/chosen": 1.3360683917999268,
+      "logits/rejected": 2.2278339862823486,
+      "logps/chosen": -498.10455322265625,
+      "logps/rejected": -526.9678955078125,
+      "loss": 0.4552,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.1999521255493164,
+      "rewards/margins": 0.9584857225418091,
+      "rewards/rejected": -3.158437728881836,
+      "step": 6860
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.538651700459576e-07,
+      "logits/chosen": 1.5857367515563965,
+      "logits/rejected": 2.085242986679077,
+      "logps/chosen": -500.7560119628906,
+      "logps/rejected": -663.60107421875,
+      "loss": 0.3796,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -2.0150258541107178,
+      "rewards/margins": 1.3807644844055176,
+      "rewards/rejected": -3.3957905769348145,
+      "step": 6870
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.4994492109648151e-07,
+      "logits/chosen": 1.9010179042816162,
+      "logits/rejected": 2.6883623600006104,
+      "logps/chosen": -448.3954162597656,
+      "logps/rejected": -531.6165161132812,
+      "loss": 0.504,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.164825916290283,
+      "rewards/margins": 1.1142840385437012,
+      "rewards/rejected": -3.2791099548339844,
+      "step": 6880
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.4607371536766695e-07,
+      "logits/chosen": 1.3107022047042847,
+      "logits/rejected": 1.8201125860214233,
+      "logps/chosen": -444.1551208496094,
+      "logps/rejected": -537.4769287109375,
+      "loss": 0.5123,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.2259302139282227,
+      "rewards/margins": 1.0542100667953491,
+      "rewards/rejected": -3.2801403999328613,
+      "step": 6890
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.4225163364757655e-07,
+      "logits/chosen": 1.7681090831756592,
+      "logits/rejected": 2.513566493988037,
+      "logps/chosen": -511.04339599609375,
+      "logps/rejected": -567.5250854492188,
+      "loss": 0.5683,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.4051239490509033,
+      "rewards/margins": 0.7946692705154419,
+      "rewards/rejected": -3.1997933387756348,
+      "step": 6900
+    },
+    {
+      "epoch": 0.9,
+      "eval_logits/chosen": 1.7897045612335205,
+      "eval_logits/rejected": 2.364945411682129,
+      "eval_logps/chosen": -477.5472412109375,
+      "eval_logps/rejected": -560.8372802734375,
+      "eval_loss": 0.5036479234695435,
+      "eval_rewards/accuracies": 0.7300000190734863,
+      "eval_rewards/chosen": -2.089876890182495,
+      "eval_rewards/margins": 1.0304259061813354,
+      "eval_rewards/rejected": -3.120303153991699,
+      "eval_runtime": 1350.5526,
+      "eval_samples_per_second": 1.481,
+      "eval_steps_per_second": 0.185,
+      "step": 6900
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.3847875569910462e-07,
+      "logits/chosen": 1.4367663860321045,
+      "logits/rejected": 2.0270867347717285,
+      "logps/chosen": -465.47467041015625,
+      "logps/rejected": -527.0831298828125,
+      "loss": 0.532,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.0552918910980225,
+      "rewards/margins": 0.7905406951904297,
+      "rewards/rejected": -2.845832347869873,
+      "step": 6910
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.3475516025831552e-07,
+      "logits/chosen": 1.3050683736801147,
+      "logits/rejected": 1.6783323287963867,
+      "logps/chosen": -465.7059020996094,
+      "logps/rejected": -570.4100341796875,
+      "loss": 0.4781,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.093655824661255,
+      "rewards/margins": 1.068086862564087,
+      "rewards/rejected": -3.161742925643921,
+      "step": 6920
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.310809250327974e-07,
+      "logits/chosen": 1.763871192932129,
+      "logits/rejected": 2.3861560821533203,
+      "logps/chosen": -448.56689453125,
+      "logps/rejected": -522.2000732421875,
+      "loss": 0.5009,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.0917561054229736,
+      "rewards/margins": 1.0084232091903687,
+      "rewards/rejected": -3.1001787185668945,
+      "step": 6930
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.2745612670004153e-07,
+      "logits/chosen": 1.5332653522491455,
+      "logits/rejected": 1.8925899267196655,
+      "logps/chosen": -466.7227478027344,
+      "logps/rejected": -628.6092529296875,
+      "loss": 0.4747,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.210294485092163,
+      "rewards/margins": 1.1151539087295532,
+      "rewards/rejected": -3.3254482746124268,
+      "step": 6940
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.2388084090584395e-07,
+      "logits/chosen": 1.9033037424087524,
+      "logits/rejected": 2.0931131839752197,
+      "logps/chosen": -471.924560546875,
+      "logps/rejected": -590.5072021484375,
+      "loss": 0.5169,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.378572940826416,
+      "rewards/margins": 1.035857915878296,
+      "rewards/rejected": -3.414430618286133,
+      "step": 6950
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.2035514226272305e-07,
+      "logits/chosen": 1.804849624633789,
+      "logits/rejected": 2.29634428024292,
+      "logps/chosen": -443.5736389160156,
+      "logps/rejected": -556.4137573242188,
+      "loss": 0.5026,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.0660266876220703,
+      "rewards/margins": 1.1615619659423828,
+      "rewards/rejected": -3.2275891304016113,
+      "step": 6960
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1687910434836607e-07,
+      "logits/chosen": 1.8551757335662842,
+      "logits/rejected": 2.122899055480957,
+      "logps/chosen": -472.8878479003906,
+      "logps/rejected": -556.80810546875,
+      "loss": 0.5317,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.202465772628784,
+      "rewards/margins": 0.9380524754524231,
+      "rewards/rejected": -3.1405181884765625,
+      "step": 6970
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1345279970409128e-07,
+      "logits/chosen": 1.9365707635879517,
+      "logits/rejected": 2.0756518840789795,
+      "logps/chosen": -414.45806884765625,
+      "logps/rejected": -534.6054077148438,
+      "loss": 0.5615,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.9993774890899658,
+      "rewards/margins": 0.945439338684082,
+      "rewards/rejected": -2.944816827774048,
+      "step": 6980
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1007629983333629e-07,
+      "logits/chosen": 1.4802647829055786,
+      "logits/rejected": 2.290517807006836,
+      "logps/chosen": -449.20672607421875,
+      "logps/rejected": -518.4752197265625,
+      "loss": 0.4674,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.966091513633728,
+      "rewards/margins": 0.9973756670951843,
+      "rewards/rejected": -2.9634673595428467,
+      "step": 6990
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.067496752001626e-07,
+      "logits/chosen": 1.8506180047988892,
+      "logits/rejected": 2.136117935180664,
+      "logps/chosen": -495.5009765625,
+      "logps/rejected": -561.6314697265625,
+      "loss": 0.5399,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.309641122817993,
+      "rewards/margins": 1.1153204441070557,
+      "rewards/rejected": -3.424961805343628,
+      "step": 7000
+    },
+    {
+      "epoch": 0.92,
+      "eval_logits/chosen": 1.7831629514694214,
+      "eval_logits/rejected": 2.358971357345581,
+      "eval_logps/chosen": -476.87213134765625,
+      "eval_logps/rejected": -560.0004272460938,
+      "eval_loss": 0.5037100315093994,
+      "eval_rewards/accuracies": 0.7294999957084656,
+      "eval_rewards/chosen": -2.0831258296966553,
+      "eval_rewards/margins": 1.0288087129592896,
+      "eval_rewards/rejected": -3.1119344234466553,
+      "eval_runtime": 1352.3698,
+      "eval_samples_per_second": 1.479,
+      "eval_steps_per_second": 0.185,
+      "step": 7000
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.0347299522778909e-07,
+      "logits/chosen": 1.349867582321167,
+      "logits/rejected": 1.9197378158569336,
+      "logps/chosen": -493.3343811035156,
+      "logps/rejected": -586.4623413085938,
+      "loss": 0.3853,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -2.054044008255005,
+      "rewards/margins": 1.3054800033569336,
+      "rewards/rejected": -3.3595237731933594,
+      "step": 7010
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.0024632829713971e-07,
+      "logits/chosen": 1.7153570652008057,
+      "logits/rejected": 1.8305761814117432,
+      "logps/chosen": -445.87591552734375,
+      "logps/rejected": -555.9547729492188,
+      "loss": 0.5149,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.1093852519989014,
+      "rewards/margins": 0.915780246257782,
+      "rewards/rejected": -3.025165557861328,
+      "step": 7020
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.706974174541889e-08,
+      "logits/chosen": 1.5524930953979492,
+      "logits/rejected": 2.390944004058838,
+      "logps/chosen": -476.3097229003906,
+      "logps/rejected": -566.9791259765625,
+      "loss": 0.3976,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.897104024887085,
+      "rewards/margins": 1.4008325338363647,
+      "rewards/rejected": -3.2979366779327393,
+      "step": 7030
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.39433018647043e-08,
+      "logits/chosen": 1.0797309875488281,
+      "logits/rejected": 1.4154889583587646,
+      "logps/chosen": -466.6759338378906,
+      "logps/rejected": -574.3694458007812,
+      "loss": 0.4972,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.9545814990997314,
+      "rewards/margins": 1.0115101337432861,
+      "rewards/rejected": -2.9660916328430176,
+      "step": 7040
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.086707390056543e-08,
+      "logits/chosen": 1.370980143547058,
+      "logits/rejected": 1.9756882190704346,
+      "logps/chosen": -491.34771728515625,
+      "logps/rejected": -563.2071533203125,
+      "loss": 0.4868,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.119713068008423,
+      "rewards/margins": 1.0249578952789307,
+      "rewards/rejected": -3.1446709632873535,
+      "step": 7050
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.784112205070083e-08,
+      "logits/chosen": 1.299039602279663,
+      "logits/rejected": 1.6571731567382812,
+      "logps/chosen": -442.0696716308594,
+      "logps/rejected": -548.4847412109375,
+      "loss": 0.5277,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.0241847038269043,
+      "rewards/margins": 0.8732501864433289,
+      "rewards/rejected": -2.897434949874878,
+      "step": 7060
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 8.486550946359779e-08,
+      "logits/chosen": 1.3464136123657227,
+      "logits/rejected": 1.987272024154663,
+      "logps/chosen": -467.614013671875,
+      "logps/rejected": -516.166015625,
+      "loss": 0.5722,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.0513296127319336,
+      "rewards/margins": 0.7225589752197266,
+      "rewards/rejected": -2.77388858795166,
+      "step": 7070
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 8.194029823721556e-08,
+      "logits/chosen": 1.6114628314971924,
+      "logits/rejected": 2.4454283714294434,
+      "logps/chosen": -463.339599609375,
+      "logps/rejected": -540.651123046875,
+      "loss": 0.5075,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.088026523590088,
+      "rewards/margins": 1.003624677658081,
+      "rewards/rejected": -3.091651439666748,
+      "step": 7080
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.906554941768896e-08,
+      "logits/chosen": 1.6810016632080078,
+      "logits/rejected": 2.0003983974456787,
+      "logps/chosen": -464.04974365234375,
+      "logps/rejected": -545.1192626953125,
+      "loss": 0.5616,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.305964231491089,
+      "rewards/margins": 0.8182452917098999,
+      "rewards/rejected": -3.12420916557312,
+      "step": 7090
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.624132299805575e-08,
+      "logits/chosen": 1.6606857776641846,
+      "logits/rejected": 1.7308368682861328,
+      "logps/chosen": -512.5149536132812,
+      "logps/rejected": -601.4288330078125,
+      "loss": 0.4628,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.098553419113159,
+      "rewards/margins": 1.14769446849823,
+      "rewards/rejected": -3.246248245239258,
+      "step": 7100
+    },
+    {
+      "epoch": 0.93,
+      "eval_logits/chosen": 1.7910367250442505,
+      "eval_logits/rejected": 2.365868330001831,
+      "eval_logps/chosen": -477.3761291503906,
+      "eval_logps/rejected": -560.6895751953125,
+      "eval_loss": 0.5034900307655334,
+      "eval_rewards/accuracies": 0.7300000190734863,
+      "eval_rewards/chosen": -2.088165283203125,
+      "eval_rewards/margins": 1.0306603908538818,
+      "eval_rewards/rejected": -3.118825674057007,
+      "eval_runtime": 1351.4217,
+      "eval_samples_per_second": 1.48,
+      "eval_steps_per_second": 0.185,
+      "step": 7100
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.346767791700127e-08,
+      "logits/chosen": 1.5037165880203247,
+      "logits/rejected": 2.0673274993896484,
+      "logps/chosen": -496.1266174316406,
+      "logps/rejected": -583.6951904296875,
+      "loss": 0.4097,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.9274057149887085,
+      "rewards/margins": 1.3257863521575928,
+      "rewards/rejected": -3.253192186355591,
+      "step": 7110
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.07446720576327e-08,
+      "logits/chosen": 1.8694560527801514,
+      "logits/rejected": 2.5695881843566895,
+      "logps/chosen": -457.87646484375,
+      "logps/rejected": -540.10302734375,
+      "loss": 0.5411,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.2093558311462402,
+      "rewards/margins": 1.0019371509552002,
+      "rewards/rejected": -3.2112929821014404,
+      "step": 7120
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.807236224626701e-08,
+      "logits/chosen": 1.3616082668304443,
+      "logits/rejected": 1.805204153060913,
+      "logps/chosen": -482.5126953125,
+      "logps/rejected": -545.8123779296875,
+      "loss": 0.4922,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.0851895809173584,
+      "rewards/margins": 0.8939981460571289,
+      "rewards/rejected": -2.979187488555908,
+      "step": 7130
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.545080425124888e-08,
+      "logits/chosen": 1.6385771036148071,
+      "logits/rejected": 1.8407480716705322,
+      "logps/chosen": -464.08392333984375,
+      "logps/rejected": -557.6373291015625,
+      "loss": 0.5435,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.053687572479248,
+      "rewards/margins": 0.9564380645751953,
+      "rewards/rejected": -3.0101258754730225,
+      "step": 7140
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.288005278178382e-08,
+      "logits/chosen": 1.2109508514404297,
+      "logits/rejected": 1.8342506885528564,
+      "logps/chosen": -465.4290466308594,
+      "logps/rejected": -549.1767578125,
+      "loss": 0.4736,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.9375137090682983,
+      "rewards/margins": 1.0112565755844116,
+      "rewards/rejected": -2.948770523071289,
+      "step": 7150
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.036016148679825e-08,
+      "logits/chosen": 1.542161226272583,
+      "logits/rejected": 2.386200428009033,
+      "logps/chosen": -485.739501953125,
+      "logps/rejected": -507.75274658203125,
+      "loss": 0.5121,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.139554500579834,
+      "rewards/margins": 0.8059576153755188,
+      "rewards/rejected": -2.945512294769287,
+      "step": 7160
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.7891182953819235e-08,
+      "logits/chosen": 1.4941692352294922,
+      "logits/rejected": 1.8872270584106445,
+      "logps/chosen": -431.12811279296875,
+      "logps/rejected": -562.6895141601562,
+      "loss": 0.4651,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.0269341468811035,
+      "rewards/margins": 1.0946818590164185,
+      "rewards/rejected": -3.1216161251068115,
+      "step": 7170
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.547316870787689e-08,
+      "logits/chosen": 1.330713152885437,
+      "logits/rejected": 2.2656915187835693,
+      "logps/chosen": -493.731689453125,
+      "logps/rejected": -566.7503662109375,
+      "loss": 0.4873,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.068589687347412,
+      "rewards/margins": 1.1464108228683472,
+      "rewards/rejected": -3.215000629425049,
+      "step": 7180
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.310616921042927e-08,
+      "logits/chosen": 1.158499836921692,
+      "logits/rejected": 2.168647289276123,
+      "logps/chosen": -528.9005126953125,
+      "logps/rejected": -583.1807250976562,
+      "loss": 0.5035,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.1934781074523926,
+      "rewards/margins": 1.0433975458145142,
+      "rewards/rejected": -3.236875534057617,
+      "step": 7190
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.079023385830939e-08,
+      "logits/chosen": 1.237884759902954,
+      "logits/rejected": 2.009561538696289,
+      "logps/chosen": -528.780029296875,
+      "logps/rejected": -572.2435302734375,
+      "loss": 0.5273,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.288360118865967,
+      "rewards/margins": 1.0411651134490967,
+      "rewards/rejected": -3.3295249938964844,
+      "step": 7200
+    },
+    {
+      "epoch": 0.94,
+      "eval_logits/chosen": 1.7852716445922852,
+      "eval_logits/rejected": 2.359431266784668,
+      "eval_logps/chosen": -477.53173828125,
+      "eval_logps/rejected": -560.8275146484375,
+      "eval_loss": 0.5035795569419861,
+      "eval_rewards/accuracies": 0.7294999957084656,
+      "eval_rewards/chosen": -2.0897216796875,
+      "eval_rewards/margins": 1.030483365058899,
+      "eval_rewards/rejected": -3.1202054023742676,
+      "eval_runtime": 1352.8929,
+      "eval_samples_per_second": 1.478,
+      "eval_steps_per_second": 0.185,
+      "step": 7200
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 4.8525410982695476e-08,
+      "logits/chosen": 1.179303526878357,
+      "logits/rejected": 2.1191606521606445,
+      "logps/chosen": -501.01068115234375,
+      "logps/rejected": -586.3238525390625,
+      "loss": 0.4467,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.9667949676513672,
+      "rewards/margins": 1.1312482357025146,
+      "rewards/rejected": -3.0980429649353027,
+      "step": 7210
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 4.6311747848099e-08,
+      "logits/chosen": 1.4426491260528564,
+      "logits/rejected": 2.4274449348449707,
+      "logps/chosen": -448.96636962890625,
+      "logps/rejected": -544.4766845703125,
+      "loss": 0.4745,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.1399025917053223,
+      "rewards/margins": 1.2233902215957642,
+      "rewards/rejected": -3.363292694091797,
+      "step": 7220
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.4149290651382405e-08,
+      "logits/chosen": 1.8652222156524658,
+      "logits/rejected": 1.758488655090332,
+      "logps/chosen": -410.63507080078125,
+      "logps/rejected": -527.044677734375,
+      "loss": 0.5549,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.078183650970459,
+      "rewards/margins": 0.9267053604125977,
+      "rewards/rejected": -3.0048892498016357,
+      "step": 7230
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.203808452079211e-08,
+      "logits/chosen": 1.6954448223114014,
+      "logits/rejected": 2.087514877319336,
+      "logps/chosen": -467.0596618652344,
+      "logps/rejected": -557.9234619140625,
+      "loss": 0.433,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.9111039638519287,
+      "rewards/margins": 1.0559155941009521,
+      "rewards/rejected": -2.967019557952881,
+      "step": 7240
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.9978173515018427e-08,
+      "logits/chosen": 1.0878384113311768,
+      "logits/rejected": 2.007995843887329,
+      "logps/chosen": -520.1403198242188,
+      "logps/rejected": -581.5050048828125,
+      "loss": 0.4802,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.0365946292877197,
+      "rewards/margins": 1.238152265548706,
+      "rewards/rejected": -3.2747466564178467,
+      "step": 7250
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.7969600622274614e-08,
+      "logits/chosen": 1.0851434469223022,
+      "logits/rejected": 1.6826393604278564,
+      "logps/chosen": -532.126953125,
+      "logps/rejected": -517.87451171875,
+      "loss": 0.6007,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.1615326404571533,
+      "rewards/margins": 0.6440030336380005,
+      "rewards/rejected": -2.8055357933044434,
+      "step": 7260
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.601240775940151e-08,
+      "logits/chosen": 1.7193288803100586,
+      "logits/rejected": 1.8620322942733765,
+      "logps/chosen": -459.6993103027344,
+      "logps/rejected": -539.3936767578125,
+      "loss": 0.6492,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.1847825050354004,
+      "rewards/margins": 0.8445892333984375,
+      "rewards/rejected": -3.029371500015259,
+      "step": 7270
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.410663577099071e-08,
+      "logits/chosen": 1.172057032585144,
+      "logits/rejected": 2.0547006130218506,
+      "logps/chosen": -547.5747680664062,
+      "logps/rejected": -601.03515625,
+      "loss": 0.6063,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.2760438919067383,
+      "rewards/margins": 0.9378223419189453,
+      "rewards/rejected": -3.2138659954071045,
+      "step": 7280
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.2252324428534986e-08,
+      "logits/chosen": 1.371187686920166,
+      "logits/rejected": 2.305682420730591,
+      "logps/chosen": -488.50189208984375,
+      "logps/rejected": -565.6369018554688,
+      "loss": 0.4587,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.90292489528656,
+      "rewards/margins": 1.1992566585540771,
+      "rewards/rejected": -3.102181911468506,
+      "step": 7290
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.0449512429594486e-08,
+      "logits/chosen": 1.575221300125122,
+      "logits/rejected": 2.3022565841674805,
+      "logps/chosen": -474.8644104003906,
+      "logps/rejected": -565.8113403320312,
+      "loss": 0.4445,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.9912481307983398,
+      "rewards/margins": 1.2342274188995361,
+      "rewards/rejected": -3.225475788116455,
+      "step": 7300
+    },
+    {
+      "epoch": 0.96,
+      "eval_logits/chosen": 1.7871266603469849,
+      "eval_logits/rejected": 2.3613994121551514,
+      "eval_logps/chosen": -477.4447326660156,
+      "eval_logps/rejected": -560.7728881835938,
+      "eval_loss": 0.5034801363945007,
+      "eval_rewards/accuracies": 0.7304999828338623,
+      "eval_rewards/chosen": -2.088850975036621,
+      "eval_rewards/margins": 1.0308077335357666,
+      "eval_rewards/rejected": -3.119658946990967,
+      "eval_runtime": 1351.4971,
+      "eval_samples_per_second": 1.48,
+      "eval_steps_per_second": 0.185,
+      "step": 7300
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.8698237396992956e-08,
+      "logits/chosen": 1.5441524982452393,
+      "logits/rejected": 1.6202573776245117,
+      "logps/chosen": -473.77728271484375,
+      "logps/rejected": -575.0803833007812,
+      "loss": 0.4913,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.0911903381347656,
+      "rewards/margins": 1.0879924297332764,
+      "rewards/rejected": -3.1791832447052,
+      "step": 7310
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.6998535878030584e-08,
+      "logits/chosen": 1.5388976335525513,
+      "logits/rejected": 2.5617949962615967,
+      "logps/chosen": -397.65869140625,
+      "logps/rejected": -457.47894287109375,
+      "loss": 0.4961,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.9129102230072021,
+      "rewards/margins": 1.0359675884246826,
+      "rewards/rejected": -2.9488778114318848,
+      "step": 7320
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.535044334372072e-08,
+      "logits/chosen": 1.6406481266021729,
+      "logits/rejected": 2.3773553371429443,
+      "logps/chosen": -532.7373657226562,
+      "logps/rejected": -569.6896362304688,
+      "loss": 0.5327,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.057133197784424,
+      "rewards/margins": 1.087259292602539,
+      "rewards/rejected": -3.144392251968384,
+      "step": 7330
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.3753994188051853e-08,
+      "logits/chosen": 1.0717352628707886,
+      "logits/rejected": 2.2860207557678223,
+      "logps/chosen": -486.56011962890625,
+      "logps/rejected": -574.9765014648438,
+      "loss": 0.4391,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.091378688812256,
+      "rewards/margins": 1.2134711742401123,
+      "rewards/rejected": -3.3048501014709473,
+      "step": 7340
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.220922172726764e-08,
+      "logits/chosen": 1.3720486164093018,
+      "logits/rejected": 1.9279594421386719,
+      "logps/chosen": -470.49322509765625,
+      "logps/rejected": -532.9813232421875,
+      "loss": 0.5442,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.0626983642578125,
+      "rewards/margins": 0.9828320741653442,
+      "rewards/rejected": -3.045530319213867,
+      "step": 7350
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.071615819917244e-08,
+      "logits/chosen": 1.4359339475631714,
+      "logits/rejected": 1.4438194036483765,
+      "logps/chosen": -473.2745666503906,
+      "logps/rejected": -560.4413452148438,
+      "loss": 0.5146,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.0602781772613525,
+      "rewards/margins": 0.8161458969116211,
+      "rewards/rejected": -2.8764240741729736,
+      "step": 7360
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9274834762459393e-08,
+      "logits/chosen": 1.6191987991333008,
+      "logits/rejected": 2.6226930618286133,
+      "logps/chosen": -494.58978271484375,
+      "logps/rejected": -562.642578125,
+      "loss": 0.4142,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.0491795539855957,
+      "rewards/margins": 1.300944447517395,
+      "rewards/rejected": -3.350123882293701,
+      "step": 7370
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7885281496058947e-08,
+      "logits/chosen": 1.264670729637146,
+      "logits/rejected": 1.6359466314315796,
+      "logps/chosen": -501.2201232910156,
+      "logps/rejected": -625.55859375,
+      "loss": 0.4558,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.1520655155181885,
+      "rewards/margins": 0.9725669026374817,
+      "rewards/rejected": -3.1246321201324463,
+      "step": 7380
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.654752739851134e-08,
+      "logits/chosen": 1.3897452354431152,
+      "logits/rejected": 2.307743787765503,
+      "logps/chosen": -487.04632568359375,
+      "logps/rejected": -530.5057373046875,
+      "loss": 0.5465,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.090754270553589,
+      "rewards/margins": 0.919838547706604,
+      "rewards/rejected": -3.0105929374694824,
+      "step": 7390
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.526160038736235e-08,
+      "logits/chosen": 1.1125606298446655,
+      "logits/rejected": 2.2139885425567627,
+      "logps/chosen": -540.2608642578125,
+      "logps/rejected": -582.2575073242188,
+      "loss": 0.4839,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.130701780319214,
+      "rewards/margins": 1.0939173698425293,
+      "rewards/rejected": -3.2246193885803223,
+      "step": 7400
+    },
+    {
+      "epoch": 0.97,
+      "eval_logits/chosen": 1.7896208763122559,
+      "eval_logits/rejected": 2.364553213119507,
+      "eval_logps/chosen": -477.5042419433594,
+      "eval_logps/rejected": -560.7960815429688,
+      "eval_loss": 0.5035399794578552,
+      "eval_rewards/accuracies": 0.7310000061988831,
+      "eval_rewards/chosen": -2.089447021484375,
+      "eval_rewards/margins": 1.0304434299468994,
+      "eval_rewards/rejected": -3.1198906898498535,
+      "eval_runtime": 1351.3557,
+      "eval_samples_per_second": 1.48,
+      "eval_steps_per_second": 0.185,
+      "step": 7400
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.402752729857959e-08,
+      "logits/chosen": 1.6876471042633057,
+      "logits/rejected": 1.9298269748687744,
+      "logps/chosen": -454.95977783203125,
+      "logps/rejected": -573.255615234375,
+      "loss": 0.4879,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.064669370651245,
+      "rewards/margins": 1.0874347686767578,
+      "rewards/rejected": -3.152103900909424,
+      "step": 7410
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.2845333885992683e-08,
+      "logits/chosen": 1.4165942668914795,
+      "logits/rejected": 2.105940818786621,
+      "logps/chosen": -469.54571533203125,
+      "logps/rejected": -548.0072021484375,
+      "loss": 0.4729,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.9717060327529907,
+      "rewards/margins": 1.168196439743042,
+      "rewards/rejected": -3.1399025917053223,
+      "step": 7420
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.171504482075675e-08,
+      "logits/chosen": 0.8542730212211609,
+      "logits/rejected": 1.5652596950531006,
+      "logps/chosen": -530.7537231445312,
+      "logps/rejected": -616.4605102539062,
+      "loss": 0.5075,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.1917450428009033,
+      "rewards/margins": 1.0583924055099487,
+      "rewards/rejected": -3.2501378059387207,
+      "step": 7430
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.0636683690836147e-08,
+      "logits/chosen": 1.1933740377426147,
+      "logits/rejected": 2.0062317848205566,
+      "logps/chosen": -462.7686462402344,
+      "logps/rejected": -541.359375,
+      "loss": 0.5029,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.0580408573150635,
+      "rewards/margins": 0.9420655369758606,
+      "rewards/rejected": -3.0001060962677,
+      "step": 7440
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 9.610273000513203e-09,
+      "logits/chosen": 1.3310664892196655,
+      "logits/rejected": 2.07230544090271,
+      "logps/chosen": -529.7361450195312,
+      "logps/rejected": -596.6788330078125,
+      "loss": 0.5739,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.3096230030059814,
+      "rewards/margins": 0.7905694246292114,
+      "rewards/rejected": -3.1001923084259033,
+      "step": 7450
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.635834169918312e-09,
+      "logits/chosen": 1.5401674509048462,
+      "logits/rejected": 1.8369882106781006,
+      "logps/chosen": -484.8273010253906,
+      "logps/rejected": -563.3812255859375,
+      "loss": 0.5777,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.011173725128174,
+      "rewards/margins": 0.7867377996444702,
+      "rewards/rejected": -2.7979118824005127,
+      "step": 7460
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 7.713387534582506e-09,
+      "logits/chosen": 1.4142358303070068,
+      "logits/rejected": 2.0463063716888428,
+      "logps/chosen": -452.90130615234375,
+      "logps/rejected": -596.63330078125,
+      "loss": 0.4415,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.053887367248535,
+      "rewards/margins": 1.2912774085998535,
+      "rewards/rejected": -3.3451647758483887,
+      "step": 7470
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 6.84295234501392e-09,
+      "logits/chosen": 1.4514764547348022,
+      "logits/rejected": 2.0671112537384033,
+      "logps/chosen": -482.8954162597656,
+      "logps/rejected": -524.7696533203125,
+      "loss": 0.5347,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.0976595878601074,
+      "rewards/margins": 0.8478931188583374,
+      "rewards/rejected": -2.9455525875091553,
+      "step": 7480
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 6.024546766295325e-09,
+      "logits/chosen": 1.4061548709869385,
+      "logits/rejected": 2.595026731491089,
+      "logps/chosen": -515.5316162109375,
+      "logps/rejected": -561.0677490234375,
+      "loss": 0.4829,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.14540696144104,
+      "rewards/margins": 1.1225945949554443,
+      "rewards/rejected": -3.2680015563964844,
+      "step": 7490
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 5.2581878777049895e-09,
+      "logits/chosen": 1.2910041809082031,
+      "logits/rejected": 2.12312650680542,
+      "logps/chosen": -479.82281494140625,
+      "logps/rejected": -583.8944091796875,
+      "loss": 0.4425,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.003810167312622,
+      "rewards/margins": 1.2062535285949707,
+      "rewards/rejected": -3.2100632190704346,
+      "step": 7500
+    },
+    {
+      "epoch": 0.98,
+      "eval_logits/chosen": 1.789110541343689,
+      "eval_logits/rejected": 2.363781213760376,
+      "eval_logps/chosen": -477.48095703125,
+      "eval_logps/rejected": -560.772216796875,
+      "eval_loss": 0.5036382079124451,
+      "eval_rewards/accuracies": 0.7294999957084656,
+      "eval_rewards/chosen": -2.0892136096954346,
+      "eval_rewards/margins": 1.0304385423660278,
+      "eval_rewards/rejected": -3.119652509689331,
+      "eval_runtime": 1352.5396,
+      "eval_samples_per_second": 1.479,
+      "eval_steps_per_second": 0.185,
+      "step": 7500
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 4.543891672361411e-09,
+      "logits/chosen": 1.2577922344207764,
+      "logits/rejected": 1.6616942882537842,
+      "logps/chosen": -474.1512145996094,
+      "logps/rejected": -563.4071044921875,
+      "loss": 0.5158,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.0339672565460205,
+      "rewards/margins": 0.9651128649711609,
+      "rewards/rejected": -2.9990804195404053,
+      "step": 7510
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 3.881673056887747e-09,
+      "logits/chosen": 1.3830997943878174,
+      "logits/rejected": 1.8997008800506592,
+      "logps/chosen": -506.72906494140625,
+      "logps/rejected": -568.8949584960938,
+      "loss": 0.5782,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.2226126194000244,
+      "rewards/margins": 0.9263314008712769,
+      "rewards/rejected": -3.1489439010620117,
+      "step": 7520
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.2715458511023425e-09,
+      "logits/chosen": 1.5419468879699707,
+      "logits/rejected": 1.9950015544891357,
+      "logps/chosen": -474.4693298339844,
+      "logps/rejected": -576.6512451171875,
+      "loss": 0.5019,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.9757168292999268,
+      "rewards/margins": 1.0253888368606567,
+      "rewards/rejected": -3.001105308532715,
+      "step": 7530
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.7135227877289617e-09,
+      "logits/chosen": 1.3359543085098267,
+      "logits/rejected": 1.294131875038147,
+      "logps/chosen": -452.1876525878906,
+      "logps/rejected": -564.711181640625,
+      "loss": 0.5042,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.9297090768814087,
+      "rewards/margins": 0.9726053476333618,
+      "rewards/rejected": -2.9023139476776123,
+      "step": 7540
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.2076155121328326e-09,
+      "logits/chosen": 1.512879729270935,
+      "logits/rejected": 2.404162883758545,
+      "logps/chosen": -439.26470947265625,
+      "logps/rejected": -551.4031982421875,
+      "loss": 0.405,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -1.962259292602539,
+      "rewards/margins": 1.355453610420227,
+      "rewards/rejected": -3.3177127838134766,
+      "step": 7550
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.7538345820755641e-09,
+      "logits/chosen": 1.199231743812561,
+      "logits/rejected": 2.2076611518859863,
+      "logps/chosen": -515.0853271484375,
+      "logps/rejected": -548.8587646484375,
+      "loss": 0.499,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.1014609336853027,
+      "rewards/margins": 0.9294778108596802,
+      "rewards/rejected": -3.0309383869171143,
+      "step": 7560
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.3521894674961567e-09,
+      "logits/chosen": 1.5572443008422852,
+      "logits/rejected": 1.9144580364227295,
+      "logps/chosen": -489.3985290527344,
+      "logps/rejected": -560.0220947265625,
+      "loss": 0.5296,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.060938596725464,
+      "rewards/margins": 0.9170997738838196,
+      "rewards/rejected": -2.9780383110046387,
+      "step": 7570
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.0026885503131023e-09,
+      "logits/chosen": 1.0403639078140259,
+      "logits/rejected": 2.0525689125061035,
+      "logps/chosen": -492.7122497558594,
+      "logps/rejected": -552.2716064453125,
+      "loss": 0.4739,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.1970009803771973,
+      "rewards/margins": 1.237358570098877,
+      "rewards/rejected": -3.434359073638916,
+      "step": 7580
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.053391242492491e-10,
+      "logits/chosen": 1.895965576171875,
+      "logits/rejected": 2.3480334281921387,
+      "logps/chosen": -491.33856201171875,
+      "logps/rejected": -556.0880737304688,
+      "loss": 0.6229,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.3857264518737793,
+      "rewards/margins": 0.7232415080070496,
+      "rewards/rejected": -3.1089677810668945,
+      "step": 7590
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.6014739467997725e-10,
+      "logits/chosen": 1.737116813659668,
+      "logits/rejected": 2.1178250312805176,
+      "logps/chosen": -469.6908264160156,
+      "logps/rejected": -541.1561279296875,
+      "loss": 0.5195,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.0848071575164795,
+      "rewards/margins": 0.879688560962677,
+      "rewards/rejected": -2.964495897293091,
+      "step": 7600
+    },
+    {
+      "epoch": 0.99,
+      "eval_logits/chosen": 1.789110541343689,
+      "eval_logits/rejected": 2.363781213760376,
+      "eval_logps/chosen": -477.48095703125,
+      "eval_logps/rejected": -560.772216796875,
+      "eval_loss": 0.5036382079124451,
+      "eval_rewards/accuracies": 0.7294999957084656,
+      "eval_rewards/chosen": -2.0892136096954346,
+      "eval_rewards/margins": 1.0304385423660278,
+      "eval_rewards/rejected": -3.119652509689331,
+      "eval_runtime": 1352.72,
+      "eval_samples_per_second": 1.479,
+      "eval_steps_per_second": 0.185,
+      "step": 7600
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.671184785033032e-10,
+      "logits/chosen": 0.9431979060173035,
+      "logits/rejected": 1.651685357093811,
+      "logps/chosen": -487.22119140625,
+      "logps/rejected": -587.4962768554688,
+      "loss": 0.4563,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.9482799768447876,
+      "rewards/margins": 1.1132206916809082,
+      "rewards/rejected": -3.0615007877349854,
+      "step": 7610
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.2625640403302054e-10,
+      "logits/chosen": 1.1383507251739502,
+      "logits/rejected": 1.779808759689331,
+      "logps/chosen": -482.02044677734375,
+      "logps/rejected": -584.72216796875,
+      "loss": 0.48,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.037858486175537,
+      "rewards/margins": 0.975435733795166,
+      "rewards/rejected": -3.013294219970703,
+      "step": 7620
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.756411091515588e-11,
+      "logits/chosen": 1.5609556436538696,
+      "logits/rejected": 1.9194170236587524,
+      "logps/chosen": -524.3682861328125,
+      "logps/rejected": -606.0785522460938,
+      "loss": 0.5484,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.113037109375,
+      "rewards/margins": 1.1586177349090576,
+      "rewards/rejected": -3.2716546058654785,
+      "step": 7630
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.0434500657963143e-12,
+      "logits/chosen": 1.7576706409454346,
+      "logits/rejected": 2.020904064178467,
+      "logps/chosen": -467.19189453125,
+      "logps/rejected": -589.6717529296875,
+      "loss": 0.4419,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.112278938293457,
+      "rewards/margins": 1.146576166152954,
+      "rewards/rejected": -3.258855104446411,
+      "step": 7640
+    },
+    {
+      "epoch": 1.0,
+      "step": 7642,
+      "total_flos": 0.0,
+      "train_loss": 0.4069632120280327,
+      "train_runtime": 138556.5872,
+      "train_samples_per_second": 0.441,
+      "train_steps_per_second": 0.055
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 7642,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}