diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,19313 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 100,
+  "global_step": 11608,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.00017229496898690558,
+      "grad_norm": 2.18241548538208,
+      "learning_rate": 4.306632213608957e-11,
+      "logits/chosen": -2.967046022415161,
+      "logits/rejected": -2.9243061542510986,
+      "logps/chosen": -43.99115753173828,
+      "logps/rejected": -41.627906799316406,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.0017229496898690559,
+      "grad_norm": 2.379453659057617,
+      "learning_rate": 4.306632213608958e-10,
+      "logits/chosen": -3.0551884174346924,
+      "logits/rejected": -3.0257153511047363,
+      "logps/chosen": -50.47557067871094,
+      "logps/rejected": -49.60466766357422,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4513888955116272,
+      "rewards/chosen": -9.277687786379829e-05,
+      "rewards/margins": 0.00010332252713851631,
+      "rewards/rejected": -0.00019609939772635698,
+      "step": 10
+    },
+    {
+      "epoch": 0.0034458993797381117,
+      "grad_norm": 2.242704153060913,
+      "learning_rate": 8.613264427217916e-10,
+      "logits/chosen": -3.1189393997192383,
+      "logits/rejected": -3.1107640266418457,
+      "logps/chosen": -52.659149169921875,
+      "logps/rejected": -52.98278045654297,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": 3.9344689866993576e-05,
+      "rewards/margins": -0.00018578903109300882,
+      "rewards/rejected": 0.00022513372823596,
+      "step": 20
+    },
+    {
+      "epoch": 0.005168849069607168,
+      "grad_norm": 2.5800223350524902,
+      "learning_rate": 1.2919896640826872e-09,
+      "logits/chosen": -3.091651439666748,
+      "logits/rejected": -3.0679259300231934,
+      "logps/chosen": -56.789154052734375,
+      "logps/rejected": -58.42307662963867,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 6.968002708163112e-05,
+      "rewards/margins": -7.167726289480925e-05,
+      "rewards/rejected": 0.00014135728997644037,
+      "step": 30
+    },
+    {
+      "epoch": 0.006891798759476223,
+      "grad_norm": 2.011408567428589,
+      "learning_rate": 1.7226528854435832e-09,
+      "logits/chosen": -3.1051833629608154,
+      "logits/rejected": -3.0736680030822754,
+      "logps/chosen": -55.264076232910156,
+      "logps/rejected": -50.67798614501953,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 8.091173367574811e-05,
+      "rewards/margins": 7.482850833184784e-06,
+      "rewards/rejected": 7.342894241446629e-05,
+      "step": 40
+    },
+    {
+      "epoch": 0.00861474844934528,
+      "grad_norm": 2.3961024284362793,
+      "learning_rate": 2.153316106804479e-09,
+      "logits/chosen": -3.1010234355926514,
+      "logits/rejected": -3.0845837593078613,
+      "logps/chosen": -53.13024139404297,
+      "logps/rejected": -51.508338928222656,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.00013639405369758606,
+      "rewards/margins": -5.1460356189636514e-05,
+      "rewards/rejected": -8.493369386997074e-05,
+      "step": 50
+    },
+    {
+      "epoch": 0.010337698139214336,
+      "grad_norm": 2.789827585220337,
+      "learning_rate": 2.5839793281653743e-09,
+      "logits/chosen": -3.1541037559509277,
+      "logits/rejected": -3.1243040561676025,
+      "logps/chosen": -57.57470703125,
+      "logps/rejected": -54.148826599121094,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": 0.0001171809490188025,
+      "rewards/margins": -0.00021332548931241035,
+      "rewards/rejected": 0.00033050641650334,
+      "step": 60
+    },
+    {
+      "epoch": 0.012060647829083391,
+      "grad_norm": 2.199442148208618,
+      "learning_rate": 3.01464254952627e-09,
+      "logits/chosen": -3.05102276802063,
+      "logits/rejected": -3.031003475189209,
+      "logps/chosen": -53.7640266418457,
+      "logps/rejected": -53.21064376831055,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": 5.2153354772599414e-05,
+      "rewards/margins": -8.695709402672946e-05,
+      "rewards/rejected": 0.00013911043060943484,
+      "step": 70
+    },
+    {
+      "epoch": 0.013783597518952447,
+      "grad_norm": 2.4367082118988037,
+      "learning_rate": 3.4453057708871665e-09,
+      "logits/chosen": -3.159820079803467,
+      "logits/rejected": -3.1265246868133545,
+      "logps/chosen": -59.09244918823242,
+      "logps/rejected": -54.107215881347656,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 3.980521796620451e-05,
+      "rewards/margins": 0.0001560368837090209,
+      "rewards/rejected": -0.00011623169120866805,
+      "step": 80
+    },
+    {
+      "epoch": 0.015506547208821502,
+      "grad_norm": 2.474703311920166,
+      "learning_rate": 3.8759689922480615e-09,
+      "logits/chosen": -2.9933581352233887,
+      "logits/rejected": -2.978722095489502,
+      "logps/chosen": -53.47351837158203,
+      "logps/rejected": -52.8353157043457,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -2.4621247575851157e-05,
+      "rewards/margins": -9.236435289494693e-05,
+      "rewards/rejected": 6.774312350898981e-05,
+      "step": 90
+    },
+    {
+      "epoch": 0.01722949689869056,
+      "grad_norm": 2.4847445487976074,
+      "learning_rate": 4.306632213608958e-09,
+      "logits/chosen": -3.1697986125946045,
+      "logits/rejected": -3.107689380645752,
+      "logps/chosen": -55.950645446777344,
+      "logps/rejected": -49.6440544128418,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -5.733591387979686e-05,
+      "rewards/margins": 8.698813326191157e-05,
+      "rewards/rejected": -0.00014432406169362366,
+      "step": 100
+    },
+    {
+      "epoch": 0.01722949689869056,
+      "eval_logits/chosen": -3.163391590118408,
+      "eval_logits/rejected": -3.1577107906341553,
+      "eval_logps/chosen": -58.705955505371094,
+      "eval_logps/rejected": -63.17074966430664,
+      "eval_loss": 0.6931650638580322,
+      "eval_rewards/accuracies": 0.4830390214920044,
+      "eval_rewards/chosen": 5.938978210906498e-05,
+      "eval_rewards/margins": -3.4367883927188814e-05,
+      "eval_rewards/rejected": 9.375767695019022e-05,
+      "eval_runtime": 384.6232,
+      "eval_samples_per_second": 11.19,
+      "eval_steps_per_second": 1.399,
+      "step": 100
+    },
+    {
+      "epoch": 0.018952446588559616,
+      "grad_norm": 2.545112371444702,
+      "learning_rate": 4.737295434969853e-09,
+      "logits/chosen": -3.121964454650879,
+      "logits/rejected": -3.0981342792510986,
+      "logps/chosen": -55.60063552856445,
+      "logps/rejected": -52.33555221557617,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.0002358659403398633,
+      "rewards/margins": -0.0001902415679069236,
+      "rewards/rejected": -4.56243687949609e-05,
+      "step": 110
+    },
+    {
+      "epoch": 0.02067539627842867,
+      "grad_norm": 2.565199136734009,
+      "learning_rate": 5.167958656330749e-09,
+      "logits/chosen": -3.0656113624572754,
+      "logits/rejected": -3.0501232147216797,
+      "logps/chosen": -53.173988342285156,
+      "logps/rejected": -55.5782356262207,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 8.139597775880247e-05,
+      "rewards/margins": 0.00014234792615752667,
+      "rewards/rejected": -6.095195931266062e-05,
+      "step": 120
+    },
+    {
+      "epoch": 0.022398345968297727,
+      "grad_norm": 2.1395199298858643,
+      "learning_rate": 5.598621877691645e-09,
+      "logits/chosen": -3.1010420322418213,
+      "logits/rejected": -3.0870213508605957,
+      "logps/chosen": -55.18474578857422,
+      "logps/rejected": -53.75429153442383,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 8.881503163138404e-05,
+      "rewards/margins": 2.743530785664916e-05,
+      "rewards/rejected": 6.13797310506925e-05,
+      "step": 130
+    },
+    {
+      "epoch": 0.024121295658166782,
+      "grad_norm": 2.427537679672241,
+      "learning_rate": 6.02928509905254e-09,
+      "logits/chosen": -3.1228744983673096,
+      "logits/rejected": -3.1045634746551514,
+      "logps/chosen": -54.176483154296875,
+      "logps/rejected": -53.76659393310547,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": 9.501051681581885e-05,
+      "rewards/margins": 0.00018827730673365295,
+      "rewards/rejected": -9.326677536591887e-05,
+      "step": 140
+    },
+    {
+      "epoch": 0.025844245348035838,
+      "grad_norm": 2.217176914215088,
+      "learning_rate": 6.459948320413436e-09,
+      "logits/chosen": -3.0275871753692627,
+      "logits/rejected": -3.0097999572753906,
+      "logps/chosen": -52.621315002441406,
+      "logps/rejected": -52.406578063964844,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 5.4861029639141634e-05,
+      "rewards/margins": 9.930648229783401e-05,
+      "rewards/rejected": -4.444545629667118e-05,
+      "step": 150
+    },
+    {
+      "epoch": 0.027567195037904894,
+      "grad_norm": 2.1536264419555664,
+      "learning_rate": 6.890611541774333e-09,
+      "logits/chosen": -3.0886142253875732,
+      "logits/rejected": -3.067762851715088,
+      "logps/chosen": -53.504295349121094,
+      "logps/rejected": -54.707183837890625,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.00017055129865184426,
+      "rewards/margins": -0.00012665479152929038,
+      "rewards/rejected": -4.389649984659627e-05,
+      "step": 160
+    },
+    {
+      "epoch": 0.02929014472777395,
+      "grad_norm": 2.371796131134033,
+      "learning_rate": 7.321274763135228e-09,
+      "logits/chosen": -3.0764527320861816,
+      "logits/rejected": -3.0569329261779785,
+      "logps/chosen": -56.2569580078125,
+      "logps/rejected": -51.33256149291992,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0002764415694400668,
+      "rewards/margins": 0.0003880424192175269,
+      "rewards/rejected": -0.00011160083522554487,
+      "step": 170
+    },
+    {
+      "epoch": 0.031013094417643005,
+      "grad_norm": 2.6078555583953857,
+      "learning_rate": 7.751937984496123e-09,
+      "logits/chosen": -3.062596321105957,
+      "logits/rejected": -3.0437824726104736,
+      "logps/chosen": -56.3991813659668,
+      "logps/rejected": -53.76967239379883,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 6.471871165558696e-05,
+      "rewards/margins": -4.70122613478452e-05,
+      "rewards/rejected": 0.00011173098755534738,
+      "step": 180
+    },
+    {
+      "epoch": 0.03273604410751206,
+      "grad_norm": 2.631075620651245,
+      "learning_rate": 8.18260120585702e-09,
+      "logits/chosen": -3.12394118309021,
+      "logits/rejected": -3.080615997314453,
+      "logps/chosen": -58.177490234375,
+      "logps/rejected": -52.5438232421875,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 6.044554174877703e-05,
+      "rewards/margins": 0.00012063176836818457,
+      "rewards/rejected": -6.018621934344992e-05,
+      "step": 190
+    },
+    {
+      "epoch": 0.03445899379738112,
+      "grad_norm": 2.578320026397705,
+      "learning_rate": 8.613264427217916e-09,
+      "logits/chosen": -3.05964994430542,
+      "logits/rejected": -3.0439488887786865,
+      "logps/chosen": -54.1138916015625,
+      "logps/rejected": -54.71663284301758,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 8.812195301288739e-05,
+      "rewards/margins": 3.2142568670678884e-05,
+      "rewards/rejected": 5.597937706625089e-05,
+      "step": 200
+    },
+    {
+      "epoch": 0.03445899379738112,
+      "eval_logits/chosen": -3.1632590293884277,
+      "eval_logits/rejected": -3.1575653553009033,
+      "eval_logps/chosen": -58.70980453491211,
+      "eval_logps/rejected": -63.166080474853516,
+      "eval_loss": 0.6932074427604675,
+      "eval_rewards/accuracies": 0.4763011038303375,
+      "eval_rewards/chosen": 2.09207119041821e-05,
+      "eval_rewards/margins": -0.00011948727478738874,
+      "eval_rewards/rejected": 0.0001404080103384331,
+      "eval_runtime": 384.6382,
+      "eval_samples_per_second": 11.19,
+      "eval_steps_per_second": 1.399,
+      "step": 200
+    },
+    {
+      "epoch": 0.03618194348725017,
+      "grad_norm": 2.293426513671875,
+      "learning_rate": 9.043927648578812e-09,
+      "logits/chosen": -3.0142459869384766,
+      "logits/rejected": -3.005631446838379,
+      "logps/chosen": -53.2763557434082,
+      "logps/rejected": -57.27679443359375,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": -0.0001231295900652185,
+      "rewards/margins": -0.0002601429296191782,
+      "rewards/rejected": 0.00013701329589821398,
+      "step": 210
+    },
+    {
+      "epoch": 0.03790489317711923,
+      "grad_norm": 2.318727493286133,
+      "learning_rate": 9.474590869939706e-09,
+      "logits/chosen": -3.050718069076538,
+      "logits/rejected": -3.0194239616394043,
+      "logps/chosen": -52.210899353027344,
+      "logps/rejected": -51.304832458496094,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": -0.00011435164924478158,
+      "rewards/margins": -0.000227766897296533,
+      "rewards/rejected": 0.00011341525532770902,
+      "step": 220
+    },
+    {
+      "epoch": 0.03962784286698828,
+      "grad_norm": 2.393052339553833,
+      "learning_rate": 9.905254091300603e-09,
+      "logits/chosen": -3.0512566566467285,
+      "logits/rejected": -3.0329246520996094,
+      "logps/chosen": -48.9101448059082,
+      "logps/rejected": -49.956642150878906,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -5.238472567725694e-06,
+      "rewards/margins": 0.00011344092490617186,
+      "rewards/rejected": -0.00011867940338561311,
+      "step": 230
+    },
+    {
+      "epoch": 0.04135079255685734,
+      "grad_norm": 2.250718832015991,
+      "learning_rate": 1.0335917312661497e-08,
+      "logits/chosen": -3.0246143341064453,
+      "logits/rejected": -2.982182502746582,
+      "logps/chosen": -55.954872131347656,
+      "logps/rejected": -52.138389587402344,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -7.040909167699283e-06,
+      "rewards/margins": -0.0001695117389317602,
+      "rewards/rejected": 0.00016247082385234535,
+      "step": 240
+    },
+    {
+      "epoch": 0.043073742246726394,
+      "grad_norm": 2.314748764038086,
+      "learning_rate": 1.0766580534022395e-08,
+      "logits/chosen": -3.1180419921875,
+      "logits/rejected": -3.0976483821868896,
+      "logps/chosen": -52.29320526123047,
+      "logps/rejected": -51.09782791137695,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.41874998807907104,
+      "rewards/chosen": -0.00013127316196914762,
+      "rewards/margins": -0.00012907147174701095,
+      "rewards/rejected": -2.2016661205270793e-06,
+      "step": 250
+    },
+    {
+      "epoch": 0.044796691936595454,
+      "grad_norm": 2.3100082874298096,
+      "learning_rate": 1.119724375538329e-08,
+      "logits/chosen": -3.094325304031372,
+      "logits/rejected": -3.0820565223693848,
+      "logps/chosen": -54.84743118286133,
+      "logps/rejected": -56.62158203125,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": 0.00021743346587754786,
+      "rewards/margins": 0.00026062963297590613,
+      "rewards/rejected": -4.3196203478146344e-05,
+      "step": 260
+    },
+    {
+      "epoch": 0.046519641626464506,
+      "grad_norm": 2.2106575965881348,
+      "learning_rate": 1.1627906976744186e-08,
+      "logits/chosen": -3.0329341888427734,
+      "logits/rejected": -3.0148322582244873,
+      "logps/chosen": -53.122947692871094,
+      "logps/rejected": -54.30078887939453,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -3.401874710107222e-05,
+      "rewards/margins": -7.825206557754427e-05,
+      "rewards/rejected": 4.423331847647205e-05,
+      "step": 270
+    },
+    {
+      "epoch": 0.048242591316333565,
+      "grad_norm": 2.429263114929199,
+      "learning_rate": 1.205857019810508e-08,
+      "logits/chosen": -3.1244781017303467,
+      "logits/rejected": -3.09040904045105,
+      "logps/chosen": -57.611549377441406,
+      "logps/rejected": -53.40376663208008,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 1.628863719815854e-05,
+      "rewards/margins": -9.053622488863766e-05,
+      "rewards/rejected": 0.00010682487481972203,
+      "step": 280
+    },
+    {
+      "epoch": 0.04996554100620262,
+      "grad_norm": 2.248253583908081,
+      "learning_rate": 1.2489233419465977e-08,
+      "logits/chosen": -3.047151565551758,
+      "logits/rejected": -3.0328431129455566,
+      "logps/chosen": -55.37665939331055,
+      "logps/rejected": -54.281166076660156,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -2.120883436873555e-06,
+      "rewards/margins": -7.98687142378185e-06,
+      "rewards/rejected": 5.865987532160943e-06,
+      "step": 290
+    },
+    {
+      "epoch": 0.051688490696071676,
+      "grad_norm": 2.360860586166382,
+      "learning_rate": 1.2919896640826872e-08,
+      "logits/chosen": -3.0028676986694336,
+      "logits/rejected": -2.99418044090271,
+      "logps/chosen": -52.83058547973633,
+      "logps/rejected": -53.940582275390625,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.00011773269216064364,
+      "rewards/margins": 8.197128045139834e-05,
+      "rewards/rejected": 3.576142626116052e-05,
+      "step": 300
+    },
+    {
+      "epoch": 0.051688490696071676,
+      "eval_logits/chosen": -3.1634702682495117,
+      "eval_logits/rejected": -3.157820224761963,
+      "eval_logps/chosen": -58.71290969848633,
+      "eval_logps/rejected": -63.17586898803711,
+      "eval_loss": 0.6931741237640381,
+      "eval_rewards/accuracies": 0.4893122613430023,
+      "eval_rewards/chosen": -1.0121803825313691e-05,
+      "eval_rewards/margins": -5.267354936222546e-05,
+      "eval_rewards/rejected": 4.255173553247005e-05,
+      "eval_runtime": 383.3081,
+      "eval_samples_per_second": 11.229,
+      "eval_steps_per_second": 1.404,
+      "step": 300
+    },
+    {
+      "epoch": 0.05341144038594073,
+      "grad_norm": 2.4704978466033936,
+      "learning_rate": 1.3350559862187768e-08,
+      "logits/chosen": -3.065483570098877,
+      "logits/rejected": -3.05995512008667,
+      "logps/chosen": -53.50603103637695,
+      "logps/rejected": -53.28745651245117,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0001336863060714677,
+      "rewards/margins": 5.5023447202984244e-05,
+      "rewards/rejected": 7.86628806963563e-05,
+      "step": 310
+    },
+    {
+      "epoch": 0.05513439007580979,
+      "grad_norm": 2.38741397857666,
+      "learning_rate": 1.3781223083548666e-08,
+      "logits/chosen": -3.0231926441192627,
+      "logits/rejected": -2.996873378753662,
+      "logps/chosen": -54.51049041748047,
+      "logps/rejected": -49.271846771240234,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -2.4914557798183523e-05,
+      "rewards/margins": 0.00029169180197641253,
+      "rewards/rejected": -0.0003166063397657126,
+      "step": 320
+    },
+    {
+      "epoch": 0.05685733976567884,
+      "grad_norm": 2.3458266258239746,
+      "learning_rate": 1.4211886304909559e-08,
+      "logits/chosen": -3.083329200744629,
+      "logits/rejected": -3.059753894805908,
+      "logps/chosen": -55.02418899536133,
+      "logps/rejected": -52.21124267578125,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 8.943701686803252e-05,
+      "rewards/margins": 9.8983189673163e-05,
+      "rewards/rejected": -9.546163710183464e-06,
+      "step": 330
+    },
+    {
+      "epoch": 0.0585802894555479,
+      "grad_norm": 2.1573686599731445,
+      "learning_rate": 1.4642549526270457e-08,
+      "logits/chosen": -3.0052781105041504,
+      "logits/rejected": -2.9837374687194824,
+      "logps/chosen": -52.53722381591797,
+      "logps/rejected": -51.92158126831055,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.00010671249037841335,
+      "rewards/margins": -0.0002460604009684175,
+      "rewards/rejected": 0.00013934790331404656,
+      "step": 340
+    },
+    {
+      "epoch": 0.06030323914541695,
+      "grad_norm": 2.317518949508667,
+      "learning_rate": 1.507321274763135e-08,
+      "logits/chosen": -2.9774985313415527,
+      "logits/rejected": -2.9380526542663574,
+      "logps/chosen": -56.21097946166992,
+      "logps/rejected": -53.56806564331055,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.5194231309578754e-05,
+      "rewards/margins": 9.991490514948964e-05,
+      "rewards/rejected": -0.0001151091419160366,
+      "step": 350
+    },
+    {
+      "epoch": 0.06202618883528601,
+      "grad_norm": 2.406733274459839,
+      "learning_rate": 1.5503875968992246e-08,
+      "logits/chosen": -3.1285297870635986,
+      "logits/rejected": -3.105591297149658,
+      "logps/chosen": -54.59253692626953,
+      "logps/rejected": -50.519187927246094,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -5.032217450207099e-05,
+      "rewards/margins": 8.953655310506292e-07,
+      "rewards/rejected": -5.12175502080936e-05,
+      "step": 360
+    },
+    {
+      "epoch": 0.06374913852515507,
+      "grad_norm": 2.325828790664673,
+      "learning_rate": 1.5934539190353144e-08,
+      "logits/chosen": -3.103545665740967,
+      "logits/rejected": -3.074301242828369,
+      "logps/chosen": -52.3903923034668,
+      "logps/rejected": -51.360008239746094,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -5.227896690485068e-05,
+      "rewards/margins": 0.00013777246931567788,
+      "rewards/rejected": -0.00019005144713446498,
+      "step": 370
+    },
+    {
+      "epoch": 0.06547208821502412,
+      "grad_norm": 2.0818049907684326,
+      "learning_rate": 1.636520241171404e-08,
+      "logits/chosen": -3.203943967819214,
+      "logits/rejected": -3.1791841983795166,
+      "logps/chosen": -53.51023483276367,
+      "logps/rejected": -52.2191276550293,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -6.515652785310522e-05,
+      "rewards/margins": 0.00022023550991434604,
+      "rewards/rejected": -0.0002853920159395784,
+      "step": 380
+    },
+    {
+      "epoch": 0.06719503790489317,
+      "grad_norm": 2.4084341526031494,
+      "learning_rate": 1.6795865633074936e-08,
+      "logits/chosen": -3.1006531715393066,
+      "logits/rejected": -3.0751090049743652,
+      "logps/chosen": -56.07448196411133,
+      "logps/rejected": -55.197418212890625,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": -8.944308501668274e-05,
+      "rewards/margins": -0.00018932360399048775,
+      "rewards/rejected": 9.988051897380501e-05,
+      "step": 390
+    },
+    {
+      "epoch": 0.06891798759476224,
+      "grad_norm": 2.091031551361084,
+      "learning_rate": 1.722652885443583e-08,
+      "logits/chosen": -3.0707168579101562,
+      "logits/rejected": -3.054976463317871,
+      "logps/chosen": -52.6497802734375,
+      "logps/rejected": -52.77490234375,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.00010567819845164195,
+      "rewards/margins": -3.149460462736897e-05,
+      "rewards/rejected": -7.418356835842133e-05,
+      "step": 400
+    },
+    {
+      "epoch": 0.06891798759476224,
+      "eval_logits/chosen": -3.1633570194244385,
+      "eval_logits/rejected": -3.1577041149139404,
+      "eval_logps/chosen": -58.698055267333984,
+      "eval_logps/rejected": -63.15386962890625,
+      "eval_loss": 0.6932098269462585,
+      "eval_rewards/accuracies": 0.46305760741233826,
+      "eval_rewards/chosen": 0.00013840175233781338,
+      "eval_rewards/margins": -0.00012417207472026348,
+      "eval_rewards/rejected": 0.00026257382705807686,
+      "eval_runtime": 384.6562,
+      "eval_samples_per_second": 11.189,
+      "eval_steps_per_second": 1.399,
+      "step": 400
+    },
+    {
+      "epoch": 0.07064093728463129,
+      "grad_norm": 2.156712293624878,
+      "learning_rate": 1.7657192075796726e-08,
+      "logits/chosen": -3.0744612216949463,
+      "logits/rejected": -3.0702929496765137,
+      "logps/chosen": -50.75202178955078,
+      "logps/rejected": -55.54851531982422,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -6.53189345030114e-05,
+      "rewards/margins": 0.00013954550377093256,
+      "rewards/rejected": -0.0002048644528258592,
+      "step": 410
+    },
+    {
+      "epoch": 0.07236388697450034,
+      "grad_norm": 2.5379998683929443,
+      "learning_rate": 1.8087855297157624e-08,
+      "logits/chosen": -3.06006121635437,
+      "logits/rejected": -3.0521535873413086,
+      "logps/chosen": -54.168479919433594,
+      "logps/rejected": -53.900001525878906,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -2.4979224690468982e-05,
+      "rewards/margins": -0.00016279231931548566,
+      "rewards/rejected": 0.00013781306915916502,
+      "step": 420
+    },
+    {
+      "epoch": 0.0740868366643694,
+      "grad_norm": 2.2448506355285645,
+      "learning_rate": 1.8518518518518518e-08,
+      "logits/chosen": -3.088217258453369,
+      "logits/rejected": -3.0743932723999023,
+      "logps/chosen": -53.09238815307617,
+      "logps/rejected": -54.16203689575195,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -1.2775502909789793e-05,
+      "rewards/margins": -2.3485516067012213e-05,
+      "rewards/rejected": 1.0709994057833683e-05,
+      "step": 430
+    },
+    {
+      "epoch": 0.07580978635423846,
+      "grad_norm": 2.5049097537994385,
+      "learning_rate": 1.8949181739879413e-08,
+      "logits/chosen": -3.1334779262542725,
+      "logits/rejected": -3.0982742309570312,
+      "logps/chosen": -54.30324172973633,
+      "logps/rejected": -53.26799392700195,
+      "loss": 0.693,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 1.5067227650433779e-05,
+      "rewards/margins": 0.00023381877690553665,
+      "rewards/rejected": -0.0002187515638070181,
+      "step": 440
+    },
+    {
+      "epoch": 0.07753273604410751,
+      "grad_norm": 2.283006191253662,
+      "learning_rate": 1.937984496124031e-08,
+      "logits/chosen": -3.055915117263794,
+      "logits/rejected": -3.0233635902404785,
+      "logps/chosen": -56.093841552734375,
+      "logps/rejected": -54.61299514770508,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 2.0395926185301505e-05,
+      "rewards/margins": 0.00013481700443662703,
+      "rewards/rejected": -0.00011442105460446328,
+      "step": 450
+    },
+    {
+      "epoch": 0.07925568573397657,
+      "grad_norm": 2.3945493698120117,
+      "learning_rate": 1.9810508182601205e-08,
+      "logits/chosen": -3.0244760513305664,
+      "logits/rejected": -3.004972457885742,
+      "logps/chosen": -56.20159149169922,
+      "logps/rejected": -53.006324768066406,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2788770618499257e-05,
+      "rewards/margins": 0.0001769150112522766,
+      "rewards/rejected": -0.0001897038018796593,
+      "step": 460
+    },
+    {
+      "epoch": 0.08097863542384562,
+      "grad_norm": 2.2040019035339355,
+      "learning_rate": 2.02411714039621e-08,
+      "logits/chosen": -3.047471523284912,
+      "logits/rejected": -3.0158817768096924,
+      "logps/chosen": -53.29467010498047,
+      "logps/rejected": -51.39592742919922,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4312500059604645,
+      "rewards/chosen": -0.00029085157439112663,
+      "rewards/margins": -0.0003255879564676434,
+      "rewards/rejected": 3.473638207651675e-05,
+      "step": 470
+    },
+    {
+      "epoch": 0.08270158511371468,
+      "grad_norm": 2.4485998153686523,
+      "learning_rate": 2.0671834625322995e-08,
+      "logits/chosen": -3.0454659461975098,
+      "logits/rejected": -3.040191173553467,
+      "logps/chosen": -54.217689514160156,
+      "logps/rejected": -58.958770751953125,
+      "loss": 0.693,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.0002150393556803465,
+      "rewards/margins": 0.0002061827399302274,
+      "rewards/rejected": 8.856653948896565e-06,
+      "step": 480
+    },
+    {
+      "epoch": 0.08442453480358374,
+      "grad_norm": 2.4696004390716553,
+      "learning_rate": 2.1102497846683892e-08,
+      "logits/chosen": -2.9552059173583984,
+      "logits/rejected": -2.9054675102233887,
+      "logps/chosen": -60.59929656982422,
+      "logps/rejected": -51.36865997314453,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.00016204144048970193,
+      "rewards/margins": 0.00046071287943050265,
+      "rewards/rejected": -0.0002986714825965464,
+      "step": 490
+    },
+    {
+      "epoch": 0.08614748449345279,
+      "grad_norm": 2.2612617015838623,
+      "learning_rate": 2.153316106804479e-08,
+      "logits/chosen": -3.017484188079834,
+      "logits/rejected": -2.9888033866882324,
+      "logps/chosen": -54.98015213012695,
+      "logps/rejected": -51.623558044433594,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.00030283088563010097,
+      "rewards/margins": 4.6169217966962606e-05,
+      "rewards/rejected": -0.00034900003811344504,
+      "step": 500
+    },
+    {
+      "epoch": 0.08614748449345279,
+      "eval_logits/chosen": -3.1633315086364746,
+      "eval_logits/rejected": -3.157658815383911,
+      "eval_logps/chosen": -58.706417083740234,
+      "eval_logps/rejected": -63.16279983520508,
+      "eval_loss": 0.6932070255279541,
+      "eval_rewards/accuracies": 0.48420074582099915,
+      "eval_rewards/chosen": 5.475644866237417e-05,
+      "eval_rewards/margins": -0.00011850229930132627,
+      "eval_rewards/rejected": 0.00017325876979157329,
+      "eval_runtime": 384.9904,
+      "eval_samples_per_second": 11.179,
+      "eval_steps_per_second": 1.397,
+      "step": 500
+    },
+    {
+      "epoch": 0.08787043418332184,
+      "grad_norm": 2.2379634380340576,
+      "learning_rate": 2.1963824289405682e-08,
+      "logits/chosen": -3.009848117828369,
+      "logits/rejected": -2.988564968109131,
+      "logps/chosen": -58.29756546020508,
+      "logps/rejected": -52.009971618652344,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.00022627852740697563,
+      "rewards/margins": -7.393761188723147e-05,
+      "rewards/rejected": -0.00015234094462357461,
+      "step": 510
+    },
+    {
+      "epoch": 0.08959338387319091,
+      "grad_norm": 2.065375804901123,
+      "learning_rate": 2.239448751076658e-08,
+      "logits/chosen": -3.0577759742736816,
+      "logits/rejected": -3.032174587249756,
+      "logps/chosen": -56.432098388671875,
+      "logps/rejected": -51.66090774536133,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.0002516114036552608,
+      "rewards/margins": 5.810005313833244e-05,
+      "rewards/rejected": -0.00030971146770752966,
+      "step": 520
+    },
+    {
+      "epoch": 0.09131633356305996,
+      "grad_norm": 2.0607364177703857,
+      "learning_rate": 2.2825150732127478e-08,
+      "logits/chosen": -3.05430269241333,
+      "logits/rejected": -3.012406349182129,
+      "logps/chosen": -55.683311462402344,
+      "logps/rejected": -51.21955490112305,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.000163088392582722,
+      "rewards/margins": 0.0004993419861420989,
+      "rewards/rejected": -0.0003362536372151226,
+      "step": 530
+    },
+    {
+      "epoch": 0.09303928325292901,
+      "grad_norm": 2.232825756072998,
+      "learning_rate": 2.3255813953488372e-08,
+      "logits/chosen": -3.040478229522705,
+      "logits/rejected": -3.024061679840088,
+      "logps/chosen": -52.8683967590332,
+      "logps/rejected": -52.940589904785156,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.00016143513494171202,
+      "rewards/margins": 6.717621727148071e-05,
+      "rewards/rejected": -0.00022861137404106557,
+      "step": 540
+    },
+    {
+      "epoch": 0.09476223294279806,
+      "grad_norm": 2.261385917663574,
+      "learning_rate": 2.3686477174849267e-08,
+      "logits/chosen": -3.1011884212493896,
+      "logits/rejected": -3.0831618309020996,
+      "logps/chosen": -53.58256912231445,
+      "logps/rejected": -52.0014762878418,
+      "loss": 0.693,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -8.711746340850368e-05,
+      "rewards/margins": 0.0002997437841258943,
+      "rewards/rejected": -0.0003868612984661013,
+      "step": 550
+    },
+    {
+      "epoch": 0.09648518263266713,
+      "grad_norm": 2.576868772506714,
+      "learning_rate": 2.411714039621016e-08,
+      "logits/chosen": -3.072585105895996,
+      "logits/rejected": -3.0646235942840576,
+      "logps/chosen": -52.1091194152832,
+      "logps/rejected": -54.990638732910156,
+      "loss": 0.693,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -5.415815394371748e-06,
+      "rewards/margins": 0.00019889110990334302,
+      "rewards/rejected": -0.00020430695440154523,
+      "step": 560
+    },
+    {
+      "epoch": 0.09820813232253618,
+      "grad_norm": 2.2414772510528564,
+      "learning_rate": 2.454780361757106e-08,
+      "logits/chosen": -3.0451016426086426,
+      "logits/rejected": -3.03639554977417,
+      "logps/chosen": -51.368141174316406,
+      "logps/rejected": -53.84248733520508,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -5.9789126680698246e-05,
+      "rewards/margins": 0.0001447344257030636,
+      "rewards/rejected": -0.00020452355965971947,
+      "step": 570
+    },
+    {
+      "epoch": 0.09993108201240523,
+      "grad_norm": 1.7708972692489624,
+      "learning_rate": 2.4978466838931954e-08,
+      "logits/chosen": -3.0519888401031494,
+      "logits/rejected": -3.046243190765381,
+      "logps/chosen": -51.128501892089844,
+      "logps/rejected": -53.25567626953125,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.00020434039470274001,
+      "rewards/margins": -6.637627666350454e-05,
+      "rewards/rejected": -0.00013796411803923547,
+      "step": 580
+    },
+    {
+      "epoch": 0.1016540317022743,
+      "grad_norm": 2.066856622695923,
+      "learning_rate": 2.540913006029285e-08,
+      "logits/chosen": -3.050938606262207,
+      "logits/rejected": -3.029092788696289,
+      "logps/chosen": -54.890525817871094,
+      "logps/rejected": -54.8263053894043,
+      "loss": 0.693,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.00011597380216699094,
+      "rewards/margins": 0.0002852606412488967,
+      "rewards/rejected": -0.0004012344288639724,
+      "step": 590
+    },
+    {
+      "epoch": 0.10337698139214335,
+      "grad_norm": 2.326631546020508,
+      "learning_rate": 2.5839793281653743e-08,
+      "logits/chosen": -3.0242562294006348,
+      "logits/rejected": -2.999917984008789,
+      "logps/chosen": -53.8814811706543,
+      "logps/rejected": -56.718788146972656,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 2.2674797946820036e-05,
+      "rewards/margins": 0.0005269134417176247,
+      "rewards/rejected": -0.0005042386474087834,
+      "step": 600
+    },
+    {
+      "epoch": 0.10337698139214335,
+      "eval_logits/chosen": -3.1630170345306396,
+      "eval_logits/rejected": -3.1573736667633057,
+      "eval_logps/chosen": -58.69740295410156,
+      "eval_logps/rejected": -63.162841796875,
+      "eval_loss": 0.6931617259979248,
+      "eval_rewards/accuracies": 0.4869888424873352,
+      "eval_rewards/chosen": 0.00014496293442789465,
+      "eval_rewards/margins": -2.785699507512618e-05,
+      "eval_rewards/rejected": 0.00017281992768403143,
+      "eval_runtime": 385.2833,
+      "eval_samples_per_second": 11.171,
+      "eval_steps_per_second": 1.396,
+      "step": 600
+    },
+    {
+      "epoch": 0.1050999310820124,
+      "grad_norm": 2.225328207015991,
+      "learning_rate": 2.6270456503014644e-08,
+      "logits/chosen": -2.9901328086853027,
+      "logits/rejected": -2.987546682357788,
+      "logps/chosen": -52.56848907470703,
+      "logps/rejected": -53.27485275268555,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -3.8782523915870115e-05,
+      "rewards/margins": 9.988024976337329e-05,
+      "rewards/rejected": -0.00013866278459317982,
+      "step": 610
+    },
+    {
+      "epoch": 0.10682288077188146,
+      "grad_norm": 2.3660664558410645,
+      "learning_rate": 2.6701119724375536e-08,
+      "logits/chosen": -3.1494712829589844,
+      "logits/rejected": -3.1227805614471436,
+      "logps/chosen": -55.0382080078125,
+      "logps/rejected": -53.294105529785156,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.00019198787049390376,
+      "rewards/margins": 7.998466026037931e-05,
+      "rewards/rejected": -0.00027197253075428307,
+      "step": 620
+    },
+    {
+      "epoch": 0.10854583046175052,
+      "grad_norm": 2.477909803390503,
+      "learning_rate": 2.713178294573643e-08,
+      "logits/chosen": -3.1327731609344482,
+      "logits/rejected": -3.1058883666992188,
+      "logps/chosen": -53.891265869140625,
+      "logps/rejected": -50.907020568847656,
+      "loss": 0.693,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.0002646218053996563,
+      "rewards/margins": 0.0002667299995664507,
+      "rewards/rejected": -0.0005313518340699375,
+      "step": 630
+    },
+    {
+      "epoch": 0.11026878015161957,
+      "grad_norm": 2.475726366043091,
+      "learning_rate": 2.756244616709733e-08,
+      "logits/chosen": -3.1008944511413574,
+      "logits/rejected": -3.089931011199951,
+      "logps/chosen": -52.93054962158203,
+      "logps/rejected": -54.35607147216797,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.00018455352983437479,
+      "rewards/margins": 0.00011149556667078286,
+      "rewards/rejected": -0.0002960491110570729,
+      "step": 640
+    },
+    {
+      "epoch": 0.11199172984148863,
+      "grad_norm": 2.6768577098846436,
+      "learning_rate": 2.7993109388458226e-08,
+      "logits/chosen": -3.113297939300537,
+      "logits/rejected": -3.114703416824341,
+      "logps/chosen": -51.5147819519043,
+      "logps/rejected": -54.85554885864258,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.0003988946264144033,
+      "rewards/margins": -5.776577745564282e-05,
+      "rewards/rejected": -0.00034112887806259096,
+      "step": 650
+    },
+    {
+      "epoch": 0.11371467953135768,
+      "grad_norm": 2.236011266708374,
+      "learning_rate": 2.8423772609819118e-08,
+      "logits/chosen": -3.0034189224243164,
+      "logits/rejected": -2.997283458709717,
+      "logps/chosen": -54.66306686401367,
+      "logps/rejected": -52.2958869934082,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.0003502237086649984,
+      "rewards/margins": -2.6918458388536237e-05,
+      "rewards/rejected": -0.00032330522662959993,
+      "step": 660
+    },
+    {
+      "epoch": 0.11543762922122675,
+      "grad_norm": 2.2036397457122803,
+      "learning_rate": 2.885443583118002e-08,
+      "logits/chosen": -3.026435613632202,
+      "logits/rejected": -3.0215084552764893,
+      "logps/chosen": -53.05399703979492,
+      "logps/rejected": -57.551170349121094,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.0004840382025577128,
+      "rewards/margins": -0.00017842079978436232,
+      "rewards/rejected": -0.00030561737366952,
+      "step": 670
+    },
+    {
+      "epoch": 0.1171605789110958,
+      "grad_norm": 2.316985845565796,
+      "learning_rate": 2.9285099052540913e-08,
+      "logits/chosen": -2.9783945083618164,
+      "logits/rejected": -2.9534618854522705,
+      "logps/chosen": -53.95965576171875,
+      "logps/rejected": -50.90400314331055,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.00026430556317791343,
+      "rewards/margins": 0.00012199376942589879,
+      "rewards/rejected": -0.0003862993326038122,
+      "step": 680
+    },
+    {
+      "epoch": 0.11888352860096485,
+      "grad_norm": 2.5839884281158447,
+      "learning_rate": 2.9715762273901808e-08,
+      "logits/chosen": -3.126861095428467,
+      "logits/rejected": -3.0969622135162354,
+      "logps/chosen": -59.1032829284668,
+      "logps/rejected": -50.57086944580078,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.00020788065739907324,
+      "rewards/margins": 8.284212526632473e-05,
+      "rewards/rejected": -0.00029072276083752513,
+      "step": 690
+    },
+    {
+      "epoch": 0.1206064782908339,
+      "grad_norm": 2.193490743637085,
+      "learning_rate": 3.01464254952627e-08,
+      "logits/chosen": -3.084601640701294,
+      "logits/rejected": -3.0562877655029297,
+      "logps/chosen": -55.746856689453125,
+      "logps/rejected": -53.032745361328125,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.000186169141670689,
+      "rewards/margins": 0.0003339126124046743,
+      "rewards/rejected": -0.0005200817249715328,
+      "step": 700
+    },
+    {
+      "epoch": 0.1206064782908339,
+      "eval_logits/chosen": -3.1629295349121094,
+      "eval_logits/rejected": -3.1573126316070557,
+      "eval_logps/chosen": -58.69445037841797,
+      "eval_logps/rejected": -63.160179138183594,
+      "eval_loss": 0.693160355091095,
+      "eval_rewards/accuracies": 0.48652416467666626,
+      "eval_rewards/chosen": 0.00017447922436986119,
+      "eval_rewards/margins": -2.497924469935242e-05,
+      "eval_rewards/rejected": 0.00019945848907809705,
+      "eval_runtime": 384.834,
+      "eval_samples_per_second": 11.184,
+      "eval_steps_per_second": 1.398,
+      "step": 700
+    },
+    {
+      "epoch": 0.12232942798070297,
+      "grad_norm": 2.264024257659912,
+      "learning_rate": 3.05770887166236e-08,
+      "logits/chosen": -3.0631766319274902,
+      "logits/rejected": -3.034384250640869,
+      "logps/chosen": -54.652748107910156,
+      "logps/rejected": -54.749244689941406,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.045735840918496e-05,
+      "rewards/margins": 0.0004714590613730252,
+      "rewards/rejected": -0.0005119164707139134,
+      "step": 710
+    },
+    {
+      "epoch": 0.12405237767057202,
+      "grad_norm": 2.503174066543579,
+      "learning_rate": 3.100775193798449e-08,
+      "logits/chosen": -3.024672031402588,
+      "logits/rejected": -3.020382881164551,
+      "logps/chosen": -53.344268798828125,
+      "logps/rejected": -54.44537353515625,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.00027154560666531324,
+      "rewards/margins": 0.00020538867102004588,
+      "rewards/rejected": -0.0004769343067891896,
+      "step": 720
+    },
+    {
+      "epoch": 0.12577532736044109,
+      "grad_norm": 2.3664326667785645,
+      "learning_rate": 3.143841515934539e-08,
+      "logits/chosen": -3.1416659355163574,
+      "logits/rejected": -3.1165900230407715,
+      "logps/chosen": -56.359092712402344,
+      "logps/rejected": -52.56962203979492,
+      "loss": 0.693,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.00026833382435142994,
+      "rewards/margins": 0.00021698088676203042,
+      "rewards/rejected": -0.00048531469656154513,
+      "step": 730
+    },
+    {
+      "epoch": 0.12749827705031014,
+      "grad_norm": 2.2199554443359375,
+      "learning_rate": 3.186907838070629e-08,
+      "logits/chosen": -3.0236122608184814,
+      "logits/rejected": -2.9979326725006104,
+      "logps/chosen": -54.92462158203125,
+      "logps/rejected": -53.774696350097656,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -7.765614282106981e-05,
+      "rewards/margins": 0.0006780828116461635,
+      "rewards/rejected": -0.0007557389326393604,
+      "step": 740
+    },
+    {
+      "epoch": 0.1292212267401792,
+      "grad_norm": 2.419062614440918,
+      "learning_rate": 3.229974160206718e-08,
+      "logits/chosen": -3.197396755218506,
+      "logits/rejected": -3.169250249862671,
+      "logps/chosen": -56.0156364440918,
+      "logps/rejected": -54.06449508666992,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -3.135430233669467e-05,
+      "rewards/margins": 0.0005592834786511958,
+      "rewards/rejected": -0.0005906378501094878,
+      "step": 750
+    },
+    {
+      "epoch": 0.13094417643004824,
+      "grad_norm": 2.5335209369659424,
+      "learning_rate": 3.273040482342808e-08,
+      "logits/chosen": -3.0475993156433105,
+      "logits/rejected": -3.008817672729492,
+      "logps/chosen": -54.46016311645508,
+      "logps/rejected": -49.59763717651367,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.00023703320766799152,
+      "rewards/margins": 0.0005621311138384044,
+      "rewards/rejected": -0.0007991644670255482,
+      "step": 760
+    },
+    {
+      "epoch": 0.1326671261199173,
+      "grad_norm": 2.0989232063293457,
+      "learning_rate": 3.3161068044788975e-08,
+      "logits/chosen": -3.0981945991516113,
+      "logits/rejected": -3.074923038482666,
+      "logps/chosen": -52.93854522705078,
+      "logps/rejected": -52.26714324951172,
+      "loss": 0.693,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.00019248048192821443,
+      "rewards/margins": 0.0003921961470041424,
+      "rewards/rejected": -0.0005846765707246959,
+      "step": 770
+    },
+    {
+      "epoch": 0.13439007580978635,
+      "grad_norm": 2.6708788871765137,
+      "learning_rate": 3.359173126614987e-08,
+      "logits/chosen": -3.0934255123138428,
+      "logits/rejected": -3.0623326301574707,
+      "logps/chosen": -53.17230987548828,
+      "logps/rejected": -51.2599983215332,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0003232863382436335,
+      "rewards/margins": 0.0005654854467138648,
+      "rewards/rejected": -0.0008887718431651592,
+      "step": 780
+    },
+    {
+      "epoch": 0.1361130254996554,
+      "grad_norm": 2.2480077743530273,
+      "learning_rate": 3.4022394487510764e-08,
+      "logits/chosen": -3.1010355949401855,
+      "logits/rejected": -3.0677378177642822,
+      "logps/chosen": -53.946929931640625,
+      "logps/rejected": -53.62763214111328,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.00040287169395014644,
+      "rewards/margins": 0.00018724618712440133,
+      "rewards/rejected": -0.0005901179392822087,
+      "step": 790
+    },
+    {
+      "epoch": 0.13783597518952448,
+      "grad_norm": 2.673172950744629,
+      "learning_rate": 3.445305770887166e-08,
+      "logits/chosen": -2.9871883392333984,
+      "logits/rejected": -2.9610915184020996,
+      "logps/chosen": -55.04201126098633,
+      "logps/rejected": -54.506614685058594,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.00044451188296079636,
+      "rewards/margins": 0.0006691280868835747,
+      "rewards/rejected": -0.001113640028052032,
+      "step": 800
+    },
+    {
+      "epoch": 0.13783597518952448,
+      "eval_logits/chosen": -3.162649393081665,
+      "eval_logits/rejected": -3.156989574432373,
+      "eval_logps/chosen": -58.67864990234375,
+      "eval_logps/rejected": -63.15027618408203,
+      "eval_loss": 0.6931309700012207,
+      "eval_rewards/accuracies": 0.500464677810669,
+      "eval_rewards/chosen": 0.00033243352663703263,
+      "eval_rewards/margins": 3.4030228562187403e-05,
+      "eval_rewards/rejected": 0.00029840326169505715,
+      "eval_runtime": 384.7183,
+      "eval_samples_per_second": 11.187,
+      "eval_steps_per_second": 1.398,
+      "step": 800
+    },
+    {
+      "epoch": 0.13955892487939353,
+      "grad_norm": 2.2532660961151123,
+      "learning_rate": 3.488372093023256e-08,
+      "logits/chosen": -3.061872720718384,
+      "logits/rejected": -3.0337612628936768,
+      "logps/chosen": -56.68253707885742,
+      "logps/rejected": -55.6502571105957,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.00045143062015995383,
+      "rewards/margins": 0.00042007811134681106,
+      "rewards/rejected": -0.0008715087315067649,
+      "step": 810
+    },
+    {
+      "epoch": 0.14128187456926258,
+      "grad_norm": 2.159388780593872,
+      "learning_rate": 3.531438415159345e-08,
+      "logits/chosen": -3.111081123352051,
+      "logits/rejected": -3.0853843688964844,
+      "logps/chosen": -51.60670852661133,
+      "logps/rejected": -50.825965881347656,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.0005902333068661392,
+      "rewards/margins": 0.00040004943730309606,
+      "rewards/rejected": -0.0009902827441692352,
+      "step": 820
+    },
+    {
+      "epoch": 0.14300482425913164,
+      "grad_norm": 2.432778835296631,
+      "learning_rate": 3.574504737295434e-08,
+      "logits/chosen": -3.039113998413086,
+      "logits/rejected": -3.0236411094665527,
+      "logps/chosen": -54.623687744140625,
+      "logps/rejected": -54.103050231933594,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.00043887770152650774,
+      "rewards/margins": 0.0008294513681903481,
+      "rewards/rejected": -0.0012683289824053645,
+      "step": 830
+    },
+    {
+      "epoch": 0.1447277739490007,
+      "grad_norm": 2.550287961959839,
+      "learning_rate": 3.617571059431525e-08,
+      "logits/chosen": -3.1427431106567383,
+      "logits/rejected": -3.116264581680298,
+      "logps/chosen": -54.12895584106445,
+      "logps/rejected": -49.640846252441406,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.00034066507942043245,
+      "rewards/margins": 0.0006451265071518719,
+      "rewards/rejected": -0.0009857916738837957,
+      "step": 840
+    },
+    {
+      "epoch": 0.14645072363886974,
+      "grad_norm": 2.298295021057129,
+      "learning_rate": 3.660637381567614e-08,
+      "logits/chosen": -3.0181753635406494,
+      "logits/rejected": -3.007498264312744,
+      "logps/chosen": -50.89386749267578,
+      "logps/rejected": -55.006080627441406,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.0007277115946635604,
+      "rewards/margins": -2.364696774748154e-05,
+      "rewards/rejected": -0.0007040645577944815,
+      "step": 850
+    },
+    {
+      "epoch": 0.1481736733287388,
+      "grad_norm": 2.32586407661438,
+      "learning_rate": 3.7037037037037036e-08,
+      "logits/chosen": -3.043879985809326,
+      "logits/rejected": -3.023777484893799,
+      "logps/chosen": -52.9006462097168,
+      "logps/rejected": -52.421607971191406,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.0004140673263464123,
+      "rewards/margins": 0.000737765512894839,
+      "rewards/rejected": -0.00115183275192976,
+      "step": 860
+    },
+    {
+      "epoch": 0.14989662301860784,
+      "grad_norm": 1.925281286239624,
+      "learning_rate": 3.7467700258397934e-08,
+      "logits/chosen": -3.11802077293396,
+      "logits/rejected": -3.114187240600586,
+      "logps/chosen": -51.35333251953125,
+      "logps/rejected": -53.79102325439453,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.0005176192498765886,
+      "rewards/margins": 0.0008342757937498391,
+      "rewards/rejected": -0.0013518951600417495,
+      "step": 870
+    },
+    {
+      "epoch": 0.15161957270847692,
+      "grad_norm": 1.907193899154663,
+      "learning_rate": 3.7898363479758826e-08,
+      "logits/chosen": -3.031480312347412,
+      "logits/rejected": -3.008613109588623,
+      "logps/chosen": -51.805419921875,
+      "logps/rejected": -51.525291442871094,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.000755091430619359,
+      "rewards/margins": 0.0006963929045014083,
+      "rewards/rejected": -0.0014514842769131064,
+      "step": 880
+    },
+    {
+      "epoch": 0.15334252239834598,
+      "grad_norm": 2.257608413696289,
+      "learning_rate": 3.8329026701119724e-08,
+      "logits/chosen": -3.059802532196045,
+      "logits/rejected": -3.0223753452301025,
+      "logps/chosen": -58.479454040527344,
+      "logps/rejected": -54.29310989379883,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 7.615011418238282e-05,
+      "rewards/margins": 0.0014042813563719392,
+      "rewards/rejected": -0.0013281311839818954,
+      "step": 890
+    },
+    {
+      "epoch": 0.15506547208821503,
+      "grad_norm": 2.1525301933288574,
+      "learning_rate": 3.875968992248062e-08,
+      "logits/chosen": -3.0800440311431885,
+      "logits/rejected": -3.0696609020233154,
+      "logps/chosen": -54.2166748046875,
+      "logps/rejected": -52.520423889160156,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0007992695900611579,
+      "rewards/margins": 0.0005503349239006639,
+      "rewards/rejected": -0.0013496044557541609,
+      "step": 900
+    },
+    {
+      "epoch": 0.15506547208821503,
+      "eval_logits/chosen": -3.1620073318481445,
+      "eval_logits/rejected": -3.15635085105896,
+      "eval_logps/chosen": -58.651519775390625,
+      "eval_logps/rejected": -63.13766860961914,
+      "eval_loss": 0.6930587291717529,
+      "eval_rewards/accuracies": 0.5113847851753235,
+      "eval_rewards/chosen": 0.0006038006395101547,
+      "eval_rewards/margins": 0.00017923797713592649,
+      "eval_rewards/rejected": 0.0004245626914780587,
+      "eval_runtime": 384.6723,
+      "eval_samples_per_second": 11.189,
+      "eval_steps_per_second": 1.399,
+      "step": 900
+    },
+    {
+      "epoch": 0.15678842177808408,
+      "grad_norm": 2.124608039855957,
+      "learning_rate": 3.919035314384151e-08,
+      "logits/chosen": -3.0508179664611816,
+      "logits/rejected": -3.0427260398864746,
+      "logps/chosen": -51.09298324584961,
+      "logps/rejected": -52.15949630737305,
+      "loss": 0.693,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.0006594097940251231,
+      "rewards/margins": 0.0003589142288547009,
+      "rewards/rejected": -0.0010183239355683327,
+      "step": 910
+    },
+    {
+      "epoch": 0.15851137146795313,
+      "grad_norm": 2.318742036819458,
+      "learning_rate": 3.962101636520241e-08,
+      "logits/chosen": -3.092485189437866,
+      "logits/rejected": -3.048938274383545,
+      "logps/chosen": -54.40534591674805,
+      "logps/rejected": -49.63042068481445,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.0008317944593727589,
+      "rewards/margins": 0.0009431432699784636,
+      "rewards/rejected": -0.0017749378457665443,
+      "step": 920
+    },
+    {
+      "epoch": 0.16023432115782218,
+      "grad_norm": 2.5763866901397705,
+      "learning_rate": 4.005167958656331e-08,
+      "logits/chosen": -3.153640031814575,
+      "logits/rejected": -3.1376469135284424,
+      "logps/chosen": -52.54253005981445,
+      "logps/rejected": -54.658164978027344,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.0005866853171028197,
+      "rewards/margins": 0.0010867745149880648,
+      "rewards/rejected": -0.0016734597738832235,
+      "step": 930
+    },
+    {
+      "epoch": 0.16195727084769124,
+      "grad_norm": 2.196305513381958,
+      "learning_rate": 4.04823428079242e-08,
+      "logits/chosen": -3.133608818054199,
+      "logits/rejected": -3.096038818359375,
+      "logps/chosen": -60.2542724609375,
+      "logps/rejected": -55.04746627807617,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.000489344703964889,
+      "rewards/margins": 0.001133394311182201,
+      "rewards/rejected": -0.0016227388987317681,
+      "step": 940
+    },
+    {
+      "epoch": 0.16368022053756032,
+      "grad_norm": 2.1961252689361572,
+      "learning_rate": 4.09130060292851e-08,
+      "logits/chosen": -2.9263529777526855,
+      "logits/rejected": -2.908989429473877,
+      "logps/chosen": -55.38722610473633,
+      "logps/rejected": -55.83245849609375,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.001158432918600738,
+      "rewards/margins": 0.0008559624548070133,
+      "rewards/rejected": -0.0020143953152000904,
+      "step": 950
+    },
+    {
+      "epoch": 0.16540317022742937,
+      "grad_norm": 2.281458616256714,
+      "learning_rate": 4.134366925064599e-08,
+      "logits/chosen": -2.8953421115875244,
+      "logits/rejected": -2.898488998413086,
+      "logps/chosen": -50.881263732910156,
+      "logps/rejected": -55.753257751464844,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.0013995501212775707,
+      "rewards/margins": 0.00013200611283537,
+      "rewards/rejected": -0.0015315564814954996,
+      "step": 960
+    },
+    {
+      "epoch": 0.16712611991729842,
+      "grad_norm": 2.3103432655334473,
+      "learning_rate": 4.177433247200689e-08,
+      "logits/chosen": -3.0658106803894043,
+      "logits/rejected": -3.030211925506592,
+      "logps/chosen": -60.7714729309082,
+      "logps/rejected": -52.61222457885742,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.0010795213747769594,
+      "rewards/margins": 0.0005947933532297611,
+      "rewards/rejected": -0.0016743147280067205,
+      "step": 970
+    },
+    {
+      "epoch": 0.16884906960716747,
+      "grad_norm": 3.0539042949676514,
+      "learning_rate": 4.2204995693367785e-08,
+      "logits/chosen": -3.1680197715759277,
+      "logits/rejected": -3.1482954025268555,
+      "logps/chosen": -56.08203887939453,
+      "logps/rejected": -55.020782470703125,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.000332908850396052,
+      "rewards/margins": 0.0014755966840311885,
+      "rewards/rejected": -0.00180850550532341,
+      "step": 980
+    },
+    {
+      "epoch": 0.17057201929703653,
+      "grad_norm": 2.456150531768799,
+      "learning_rate": 4.2635658914728676e-08,
+      "logits/chosen": -3.034815549850464,
+      "logits/rejected": -3.0113401412963867,
+      "logps/chosen": -54.68064498901367,
+      "logps/rejected": -52.834197998046875,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.0011139125563204288,
+      "rewards/margins": 0.0010427940869703889,
+      "rewards/rejected": -0.002156706526875496,
+      "step": 990
+    },
+    {
+      "epoch": 0.17229496898690558,
+      "grad_norm": 2.200680732727051,
+      "learning_rate": 4.306632213608958e-08,
+      "logits/chosen": -2.986654758453369,
+      "logits/rejected": -2.9588630199432373,
+      "logps/chosen": -56.979515075683594,
+      "logps/rejected": -51.90175247192383,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.0011130155762657523,
+      "rewards/margins": 0.00047593802446499467,
+      "rewards/rejected": -0.001588953658938408,
+      "step": 1000
+    },
+    {
+      "epoch": 0.17229496898690558,
+      "eval_logits/chosen": -3.161055088043213,
+      "eval_logits/rejected": -3.1554207801818848,
+      "eval_logps/chosen": -58.646148681640625,
+      "eval_logps/rejected": -63.13682174682617,
+      "eval_loss": 0.6930364370346069,
+      "eval_rewards/accuracies": 0.5162639617919922,
+      "eval_rewards/chosen": 0.0006574814324267209,
+      "eval_rewards/margins": 0.00022444974456448108,
+      "eval_rewards/rejected": 0.00043303167331032455,
+      "eval_runtime": 384.8769,
+      "eval_samples_per_second": 11.183,
+      "eval_steps_per_second": 1.398,
+      "step": 1000
+    },
+    {
+      "epoch": 0.17401791867677463,
+      "grad_norm": 2.37037992477417,
+      "learning_rate": 4.349698535745047e-08,
+      "logits/chosen": -2.9222261905670166,
+      "logits/rejected": -2.9327034950256348,
+      "logps/chosen": -53.912513732910156,
+      "logps/rejected": -58.098915100097656,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.0015469321515411139,
+      "rewards/margins": 0.0005805426044389606,
+      "rewards/rejected": -0.0021274748723953962,
+      "step": 1010
+    },
+    {
+      "epoch": 0.17574086836664368,
+      "grad_norm": 2.2179789543151855,
+      "learning_rate": 4.3927648578811363e-08,
+      "logits/chosen": -3.1055715084075928,
+      "logits/rejected": -3.069932222366333,
+      "logps/chosen": -57.186676025390625,
+      "logps/rejected": -54.4721565246582,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.0008405259577557445,
+      "rewards/margins": 0.001570344204083085,
+      "rewards/rejected": -0.002410870511084795,
+      "step": 1020
+    },
+    {
+      "epoch": 0.17746381805651276,
+      "grad_norm": 2.36515736579895,
+      "learning_rate": 4.435831180017227e-08,
+      "logits/chosen": -3.154853105545044,
+      "logits/rejected": -3.1275923252105713,
+      "logps/chosen": -53.790496826171875,
+      "logps/rejected": -52.090797424316406,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.0007170508033595979,
+      "rewards/margins": 0.0010657899547368288,
+      "rewards/rejected": -0.0017828406998887658,
+      "step": 1030
+    },
+    {
+      "epoch": 0.17918676774638181,
+      "grad_norm": 2.4404468536376953,
+      "learning_rate": 4.478897502153316e-08,
+      "logits/chosen": -3.0700807571411133,
+      "logits/rejected": -3.0596654415130615,
+      "logps/chosen": -53.5359001159668,
+      "logps/rejected": -55.77339553833008,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.0013753273524343967,
+      "rewards/margins": 0.0009833540534600616,
+      "rewards/rejected": -0.00235868152230978,
+      "step": 1040
+    },
+    {
+      "epoch": 0.18090971743625087,
+      "grad_norm": 2.453272819519043,
+      "learning_rate": 4.521963824289405e-08,
+      "logits/chosen": -2.9913229942321777,
+      "logits/rejected": -2.9559338092803955,
+      "logps/chosen": -57.256309509277344,
+      "logps/rejected": -50.811302185058594,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.0011325918603688478,
+      "rewards/margins": 0.0012491911184042692,
+      "rewards/rejected": -0.0023817827459424734,
+      "step": 1050
+    },
+    {
+      "epoch": 0.18263266712611992,
+      "grad_norm": 2.4378652572631836,
+      "learning_rate": 4.5650301464254955e-08,
+      "logits/chosen": -2.9999990463256836,
+      "logits/rejected": -2.980663299560547,
+      "logps/chosen": -56.63576126098633,
+      "logps/rejected": -55.161354064941406,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.0013450079131871462,
+      "rewards/margins": 0.0005044653080403805,
+      "rewards/rejected": -0.001849473686888814,
+      "step": 1060
+    },
+    {
+      "epoch": 0.18435561681598897,
+      "grad_norm": 2.2646141052246094,
+      "learning_rate": 4.6080964685615846e-08,
+      "logits/chosen": -3.1439247131347656,
+      "logits/rejected": -3.109926700592041,
+      "logps/chosen": -56.06062698364258,
+      "logps/rejected": -53.506126403808594,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.001293791807256639,
+      "rewards/margins": 0.0012843736913055182,
+      "rewards/rejected": -0.0025781658478081226,
+      "step": 1070
+    },
+    {
+      "epoch": 0.18607856650585802,
+      "grad_norm": 2.265031099319458,
+      "learning_rate": 4.6511627906976744e-08,
+      "logits/chosen": -3.121396541595459,
+      "logits/rejected": -3.0980241298675537,
+      "logps/chosen": -55.934959411621094,
+      "logps/rejected": -51.0109748840332,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.0014917576918378472,
+      "rewards/margins": 0.0013125650584697723,
+      "rewards/rejected": -0.0028043226338922977,
+      "step": 1080
+    },
+    {
+      "epoch": 0.18780151619572708,
+      "grad_norm": 2.3563618659973145,
+      "learning_rate": 4.6942291128337636e-08,
+      "logits/chosen": -2.998483657836914,
+      "logits/rejected": -2.9900708198547363,
+      "logps/chosen": -52.405494689941406,
+      "logps/rejected": -52.89825439453125,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.0017381549114361405,
+      "rewards/margins": 0.0009068640065379441,
+      "rewards/rejected": -0.0026450185105204582,
+      "step": 1090
+    },
+    {
+      "epoch": 0.18952446588559613,
+      "grad_norm": 2.120204210281372,
+      "learning_rate": 4.7372954349698534e-08,
+      "logits/chosen": -3.0643551349639893,
+      "logits/rejected": -3.06142520904541,
+      "logps/chosen": -51.033565521240234,
+      "logps/rejected": -54.82084274291992,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.0012612906284630299,
+      "rewards/margins": 0.0008697055163793266,
+      "rewards/rejected": -0.0021309959702193737,
+      "step": 1100
+    },
+    {
+      "epoch": 0.18952446588559613,
+      "eval_logits/chosen": -3.160189390182495,
+      "eval_logits/rejected": -3.1545591354370117,
+      "eval_logps/chosen": -58.63004684448242,
+      "eval_logps/rejected": -63.12809371948242,
+      "eval_loss": 0.693000078201294,
+      "eval_rewards/accuracies": 0.535315990447998,
+      "eval_rewards/chosen": 0.0008185264887288213,
+      "eval_rewards/margins": 0.0002982157457154244,
+      "eval_rewards/rejected": 0.0005203107139095664,
+      "eval_runtime": 385.0595,
+      "eval_samples_per_second": 11.177,
+      "eval_steps_per_second": 1.397,
+      "step": 1100
+    },
+    {
+      "epoch": 0.1912474155754652,
+      "grad_norm": 2.642375946044922,
+      "learning_rate": 4.780361757105943e-08,
+      "logits/chosen": -3.0617642402648926,
+      "logits/rejected": -3.074127674102783,
+      "logps/chosen": -53.102210998535156,
+      "logps/rejected": -56.76555633544922,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0020637321285903454,
+      "rewards/margins": 0.0010714172385632992,
+      "rewards/rejected": -0.003135149134323001,
+      "step": 1110
+    },
+    {
+      "epoch": 0.19297036526533426,
+      "grad_norm": 2.3946757316589355,
+      "learning_rate": 4.823428079242032e-08,
+      "logits/chosen": -3.0919437408447266,
+      "logits/rejected": -3.070047378540039,
+      "logps/chosen": -56.52800369262695,
+      "logps/rejected": -54.03032302856445,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.001569713233038783,
+      "rewards/margins": 0.0012932121753692627,
+      "rewards/rejected": -0.0028629254084080458,
+      "step": 1120
+    },
+    {
+      "epoch": 0.1946933149552033,
+      "grad_norm": 2.235377311706543,
+      "learning_rate": 4.866494401378122e-08,
+      "logits/chosen": -3.1583805084228516,
+      "logits/rejected": -3.1338725090026855,
+      "logps/chosen": -52.188377380371094,
+      "logps/rejected": -54.559593200683594,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.0015601011691614985,
+      "rewards/margins": 0.002112725516781211,
+      "rewards/rejected": -0.003672827035188675,
+      "step": 1130
+    },
+    {
+      "epoch": 0.19641626464507236,
+      "grad_norm": 2.4177064895629883,
+      "learning_rate": 4.909560723514212e-08,
+      "logits/chosen": -3.0652709007263184,
+      "logits/rejected": -3.0315542221069336,
+      "logps/chosen": -57.006141662597656,
+      "logps/rejected": -53.281494140625,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.0012478515272960067,
+      "rewards/margins": 0.0017186368349939585,
+      "rewards/rejected": -0.0029664882458746433,
+      "step": 1140
+    },
+    {
+      "epoch": 0.19813921433494142,
+      "grad_norm": 2.364077568054199,
+      "learning_rate": 4.952627045650301e-08,
+      "logits/chosen": -3.0051112174987793,
+      "logits/rejected": -2.9858992099761963,
+      "logps/chosen": -53.52112579345703,
+      "logps/rejected": -54.783485412597656,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.0016474571311846375,
+      "rewards/margins": 0.0013315407559275627,
+      "rewards/rejected": -0.0029789977706968784,
+      "step": 1150
+    },
+    {
+      "epoch": 0.19986216402481047,
+      "grad_norm": 2.226754903793335,
+      "learning_rate": 4.995693367786391e-08,
+      "logits/chosen": -3.1013810634613037,
+      "logits/rejected": -3.0926613807678223,
+      "logps/chosen": -54.12010955810547,
+      "logps/rejected": -54.82440948486328,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.0009447716292925179,
+      "rewards/margins": 0.001819248660467565,
+      "rewards/rejected": -0.0027640205807983875,
+      "step": 1160
+    },
+    {
+      "epoch": 0.20158511371467952,
+      "grad_norm": 2.1314942836761475,
+      "learning_rate": 4.9999908438832287e-08,
+      "logits/chosen": -2.9734387397766113,
+      "logits/rejected": -2.960632801055908,
+      "logps/chosen": -52.26427459716797,
+      "logps/rejected": -54.9915657043457,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.0018498034914955497,
+      "rewards/margins": 0.0013319350546225905,
+      "rewards/rejected": -0.0031817383132874966,
+      "step": 1170
+    },
+    {
+      "epoch": 0.2033080634045486,
+      "grad_norm": 2.5605907440185547,
+      "learning_rate": 4.999959193195308e-08,
+      "logits/chosen": -2.945619583129883,
+      "logits/rejected": -2.9143805503845215,
+      "logps/chosen": -53.00127410888672,
+      "logps/rejected": -51.44978713989258,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.001994710648432374,
+      "rewards/margins": 0.0024826552253216505,
+      "rewards/rejected": -0.0044773658737540245,
+      "step": 1180
+    },
+    {
+      "epoch": 0.20503101309441765,
+      "grad_norm": 2.503920078277588,
+      "learning_rate": 4.9999049351839105e-08,
+      "logits/chosen": -3.1429283618927,
+      "logits/rejected": -3.1077914237976074,
+      "logps/chosen": -58.91096878051758,
+      "logps/rejected": -50.78543472290039,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.002114911563694477,
+      "rewards/margins": 0.0017405018443241715,
+      "rewards/rejected": -0.0038554135244339705,
+      "step": 1190
+    },
+    {
+      "epoch": 0.2067539627842867,
+      "grad_norm": 2.1020853519439697,
+      "learning_rate": 4.9998280703396977e-08,
+      "logits/chosen": -2.974677562713623,
+      "logits/rejected": -2.9596338272094727,
+      "logps/chosen": -53.3643798828125,
+      "logps/rejected": -51.5868034362793,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.0019991318695247173,
+      "rewards/margins": 0.0011514651123434305,
+      "rewards/rejected": -0.0031505972146987915,
+      "step": 1200
+    },
+    {
+      "epoch": 0.2067539627842867,
+      "eval_logits/chosen": -3.1589746475219727,
+      "eval_logits/rejected": -3.153330087661743,
+      "eval_logps/chosen": -58.59716796875,
+      "eval_logps/rejected": -63.10630416870117,
+      "eval_loss": 0.692945659160614,
+      "eval_rewards/accuracies": 0.533224880695343,
+      "eval_rewards/chosen": 0.0011472852202132344,
+      "eval_rewards/margins": 0.0004091500595677644,
+      "eval_rewards/rejected": 0.0007381352479569614,
+      "eval_runtime": 385.098,
+      "eval_samples_per_second": 11.176,
+      "eval_steps_per_second": 1.397,
+      "step": 1200
+    },
+    {
+      "epoch": 0.20847691247415576,
+      "grad_norm": 2.282421827316284,
+      "learning_rate": 4.9997285993577624e-08,
+      "logits/chosen": -3.074262857437134,
+      "logits/rejected": -3.0402169227600098,
+      "logps/chosen": -53.918556213378906,
+      "logps/rejected": -53.387733459472656,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.0013029166730120778,
+      "rewards/margins": 0.0024175033904612064,
+      "rewards/rejected": -0.0037204199470579624,
+      "step": 1210
+    },
+    {
+      "epoch": 0.2101998621640248,
+      "grad_norm": 2.127676010131836,
+      "learning_rate": 4.999606523137628e-08,
+      "logits/chosen": -3.0853443145751953,
+      "logits/rejected": -3.0583527088165283,
+      "logps/chosen": -53.62201690673828,
+      "logps/rejected": -52.71541213989258,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.0024071610532701015,
+      "rewards/margins": 0.001912166946567595,
+      "rewards/rejected": -0.004319327883422375,
+      "step": 1220
+    },
+    {
+      "epoch": 0.21192281185389386,
+      "grad_norm": 2.5638203620910645,
+      "learning_rate": 4.99946184278324e-08,
+      "logits/chosen": -3.1554501056671143,
+      "logits/rejected": -3.1132664680480957,
+      "logps/chosen": -55.89921188354492,
+      "logps/rejected": -52.68207550048828,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.001483491505496204,
+      "rewards/margins": 0.003089385572820902,
+      "rewards/rejected": -0.004572877194732428,
+      "step": 1230
+    },
+    {
+      "epoch": 0.2136457615437629,
+      "grad_norm": 2.0821352005004883,
+      "learning_rate": 4.9992945596029545e-08,
+      "logits/chosen": -3.0165340900421143,
+      "logits/rejected": -2.999882221221924,
+      "logps/chosen": -52.704795837402344,
+      "logps/rejected": -53.27036666870117,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.002677032258361578,
+      "rewards/margins": 0.0018253729213029146,
+      "rewards/rejected": -0.004502404946833849,
+      "step": 1240
+    },
+    {
+      "epoch": 0.21536871123363197,
+      "grad_norm": 2.5075433254241943,
+      "learning_rate": 4.999104675109525e-08,
+      "logits/chosen": -3.1366803646087646,
+      "logits/rejected": -3.100405693054199,
+      "logps/chosen": -53.95045852661133,
+      "logps/rejected": -51.615234375,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.0024642706848680973,
+      "rewards/margins": 0.0018142291810363531,
+      "rewards/rejected": -0.004278500098735094,
+      "step": 1250
+    },
+    {
+      "epoch": 0.21709166092350105,
+      "grad_norm": 2.1204674243927,
+      "learning_rate": 4.998892191020092e-08,
+      "logits/chosen": -2.984978199005127,
+      "logits/rejected": -2.9535746574401855,
+      "logps/chosen": -53.21050262451172,
+      "logps/rejected": -50.77629089355469,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.002351625356823206,
+      "rewards/margins": 0.0025678465608507395,
+      "rewards/rejected": -0.004919471684843302,
+      "step": 1260
+    },
+    {
+      "epoch": 0.2188146106133701,
+      "grad_norm": 2.455671787261963,
+      "learning_rate": 4.9986571092561664e-08,
+      "logits/chosen": -3.045783042907715,
+      "logits/rejected": -3.039344549179077,
+      "logps/chosen": -52.09208297729492,
+      "logps/rejected": -55.18883514404297,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.00216267560608685,
+      "rewards/margins": 0.0026770096737891436,
+      "rewards/rejected": -0.004839685279875994,
+      "step": 1270
+    },
+    {
+      "epoch": 0.22053756030323915,
+      "grad_norm": 2.309521436691284,
+      "learning_rate": 4.9983994319436093e-08,
+      "logits/chosen": -3.117283344268799,
+      "logits/rejected": -3.124330520629883,
+      "logps/chosen": -51.40971755981445,
+      "logps/rejected": -60.30560302734375,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.0023816998582333326,
+      "rewards/margins": 0.0015409220941364765,
+      "rewards/rejected": -0.00392262265086174,
+      "step": 1280
+    },
+    {
+      "epoch": 0.2222605099931082,
+      "grad_norm": 2.195352792739868,
+      "learning_rate": 4.998119161412618e-08,
+      "logits/chosen": -3.017302989959717,
+      "logits/rejected": -2.9857184886932373,
+      "logps/chosen": -54.80310821533203,
+      "logps/rejected": -50.89992141723633,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.0020218256395310163,
+      "rewards/margins": 0.00278019392862916,
+      "rewards/rejected": -0.004802019335329533,
+      "step": 1290
+    },
+    {
+      "epoch": 0.22398345968297725,
+      "grad_norm": 2.1362576484680176,
+      "learning_rate": 4.997816300197699e-08,
+      "logits/chosen": -3.0973095893859863,
+      "logits/rejected": -3.0846035480499268,
+      "logps/chosen": -50.580631256103516,
+      "logps/rejected": -54.2244758605957,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.003656886052340269,
+      "rewards/margins": 0.0013638673117384315,
+      "rewards/rejected": -0.005020753480494022,
+      "step": 1300
+    },
+    {
+      "epoch": 0.22398345968297725,
+      "eval_logits/chosen": -3.1577372550964355,
+      "eval_logits/rejected": -3.1521053314208984,
+      "eval_logps/chosen": -58.57061767578125,
+      "eval_logps/rejected": -63.09934616088867,
+      "eval_loss": 0.69284987449646,
+      "eval_rewards/accuracies": 0.5550650358200073,
+      "eval_rewards/chosen": 0.0014127852628007531,
+      "eval_rewards/margins": 0.0006050537922419608,
+      "eval_rewards/rejected": 0.0008077314705587924,
+      "eval_runtime": 384.8259,
+      "eval_samples_per_second": 11.184,
+      "eval_steps_per_second": 1.398,
+      "step": 1300
+    },
+    {
+      "epoch": 0.2257064093728463,
+      "grad_norm": 1.9189567565917969,
+      "learning_rate": 4.99749085103765e-08,
+      "logits/chosen": -3.097705125808716,
+      "logits/rejected": -3.0621659755706787,
+      "logps/chosen": -53.61412811279297,
+      "logps/rejected": -50.056190490722656,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.00273595517501235,
+      "rewards/margins": 0.003625641344115138,
+      "rewards/rejected": -0.0063615962862968445,
+      "step": 1310
+    },
+    {
+      "epoch": 0.22742935906271536,
+      "grad_norm": 2.271080732345581,
+      "learning_rate": 4.9971428168755336e-08,
+      "logits/chosen": -3.070673704147339,
+      "logits/rejected": -3.04512357711792,
+      "logps/chosen": -55.60701370239258,
+      "logps/rejected": -51.29752731323242,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.001872202381491661,
+      "rewards/margins": 0.0037411178927868605,
+      "rewards/rejected": -0.005613320041447878,
+      "step": 1320
+    },
+    {
+      "epoch": 0.22915230875258444,
+      "grad_norm": 2.4264307022094727,
+      "learning_rate": 4.9967722008586484e-08,
+      "logits/chosen": -3.1497802734375,
+      "logits/rejected": -3.119029998779297,
+      "logps/chosen": -55.407188415527344,
+      "logps/rejected": -50.62321090698242,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.0018958871951326728,
+      "rewards/margins": 0.00294573069550097,
+      "rewards/rejected": -0.004841617774218321,
+      "step": 1330
+    },
+    {
+      "epoch": 0.2308752584424535,
+      "grad_norm": 2.294578790664673,
+      "learning_rate": 4.996379006338504e-08,
+      "logits/chosen": -2.9913744926452637,
+      "logits/rejected": -2.969874143600464,
+      "logps/chosen": -55.33652877807617,
+      "logps/rejected": -53.40034103393555,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.001971257384866476,
+      "rewards/margins": 0.0033557876013219357,
+      "rewards/rejected": -0.005327045917510986,
+      "step": 1340
+    },
+    {
+      "epoch": 0.23259820813232254,
+      "grad_norm": 2.2022767066955566,
+      "learning_rate": 4.995963236870789e-08,
+      "logits/chosen": -3.0566794872283936,
+      "logits/rejected": -3.0319995880126953,
+      "logps/chosen": -54.408775329589844,
+      "logps/rejected": -53.8236083984375,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.0030749414581805468,
+      "rewards/margins": 0.0028214040212333202,
+      "rewards/rejected": -0.005896345246583223,
+      "step": 1350
+    },
+    {
+      "epoch": 0.2343211578221916,
+      "grad_norm": 2.3132011890411377,
+      "learning_rate": 4.995524896215339e-08,
+      "logits/chosen": -3.014080762863159,
+      "logits/rejected": -3.001769542694092,
+      "logps/chosen": -55.51990509033203,
+      "logps/rejected": -56.56157302856445,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.003173989709466696,
+      "rewards/margins": 0.002192703541368246,
+      "rewards/rejected": -0.0053666927851736546,
+      "step": 1360
+    },
+    {
+      "epoch": 0.23604410751206065,
+      "grad_norm": 2.3685615062713623,
+      "learning_rate": 4.9950639883361015e-08,
+      "logits/chosen": -3.1277716159820557,
+      "logits/rejected": -3.108008861541748,
+      "logps/chosen": -52.97749710083008,
+      "logps/rejected": -53.440521240234375,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.0030265036039054394,
+      "rewards/margins": 0.003505756612867117,
+      "rewards/rejected": -0.006532260216772556,
+      "step": 1370
+    },
+    {
+      "epoch": 0.2377670572019297,
+      "grad_norm": 2.0827066898345947,
+      "learning_rate": 4.9945805174011024e-08,
+      "logits/chosen": -3.0194902420043945,
+      "logits/rejected": -3.001113176345825,
+      "logps/chosen": -54.50963592529297,
+      "logps/rejected": -52.89838790893555,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.0017685778439044952,
+      "rewards/margins": 0.0040413858368992805,
+      "rewards/rejected": -0.005809963680803776,
+      "step": 1380
+    },
+    {
+      "epoch": 0.23949000689179875,
+      "grad_norm": 2.3470730781555176,
+      "learning_rate": 4.994074487782406e-08,
+      "logits/chosen": -3.169210910797119,
+      "logits/rejected": -3.1417160034179688,
+      "logps/chosen": -57.14221954345703,
+      "logps/rejected": -54.09027099609375,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.0024838510435074568,
+      "rewards/margins": 0.0038540579844266176,
+      "rewards/rejected": -0.006337909493595362,
+      "step": 1390
+    },
+    {
+      "epoch": 0.2412129565816678,
+      "grad_norm": 2.115109443664551,
+      "learning_rate": 4.9935459040560776e-08,
+      "logits/chosen": -2.988971471786499,
+      "logits/rejected": -2.963332176208496,
+      "logps/chosen": -56.10506057739258,
+      "logps/rejected": -53.50803756713867,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.00296258763410151,
+      "rewards/margins": 0.004066202789545059,
+      "rewards/rejected": -0.007028789725154638,
+      "step": 1400
+    },
+    {
+      "epoch": 0.2412129565816678,
+      "eval_logits/chosen": -3.155876398086548,
+      "eval_logits/rejected": -3.150266170501709,
+      "eval_logps/chosen": -58.55189514160156,
+      "eval_logps/rejected": -63.11566162109375,
+      "eval_loss": 0.6926776766777039,
+      "eval_rewards/accuracies": 0.5536710023880005,
+      "eval_rewards/chosen": 0.001600018935278058,
+      "eval_rewards/margins": 0.0009554560529068112,
+      "eval_rewards/rejected": 0.0006445628823712468,
+      "eval_runtime": 384.9141,
+      "eval_samples_per_second": 11.182,
+      "eval_steps_per_second": 1.398,
+      "step": 1400
+    },
+    {
+      "epoch": 0.24293590627153688,
+      "grad_norm": 2.01434588432312,
+      "learning_rate": 4.9929947710021415e-08,
+      "logits/chosen": -3.053598642349243,
+      "logits/rejected": -3.0415971279144287,
+      "logps/chosen": -54.74721145629883,
+      "logps/rejected": -53.88715744018555,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.00373550271615386,
+      "rewards/margins": 0.0031329584307968616,
+      "rewards/rejected": -0.006868461612612009,
+      "step": 1410
+    },
+    {
+      "epoch": 0.24465885596140594,
+      "grad_norm": 2.1197314262390137,
+      "learning_rate": 4.992421093604534e-08,
+      "logits/chosen": -2.9713237285614014,
+      "logits/rejected": -2.977278470993042,
+      "logps/chosen": -51.14799499511719,
+      "logps/rejected": -55.19654083251953,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.003938421607017517,
+      "rewards/margins": 0.0011937494855374098,
+      "rewards/rejected": -0.005132170859724283,
+      "step": 1420
+    },
+    {
+      "epoch": 0.246381805651275,
+      "grad_norm": 2.437990427017212,
+      "learning_rate": 4.9918248770510664e-08,
+      "logits/chosen": -3.0907440185546875,
+      "logits/rejected": -3.079118013381958,
+      "logps/chosen": -53.923736572265625,
+      "logps/rejected": -57.2022590637207,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.003390853526070714,
+      "rewards/margins": 0.003733040764927864,
+      "rewards/rejected": -0.007123894058167934,
+      "step": 1430
+    },
+    {
+      "epoch": 0.24810475534114404,
+      "grad_norm": 2.141449451446533,
+      "learning_rate": 4.9912061267333696e-08,
+      "logits/chosen": -3.018070697784424,
+      "logits/rejected": -2.9866745471954346,
+      "logps/chosen": -50.89799118041992,
+      "logps/rejected": -49.23502731323242,
+      "loss": 0.691,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.004355059005320072,
+      "rewards/margins": 0.004351002164185047,
+      "rewards/rejected": -0.00870606116950512,
+      "step": 1440
+    },
+    {
+      "epoch": 0.2498277050310131,
+      "grad_norm": 2.7689812183380127,
+      "learning_rate": 4.99056484824685e-08,
+      "logits/chosen": -3.031834840774536,
+      "logits/rejected": -2.9943737983703613,
+      "logps/chosen": -55.75330352783203,
+      "logps/rejected": -52.69499969482422,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0027819196693599224,
+      "rewards/margins": 0.004967660643160343,
+      "rewards/rejected": -0.007749579846858978,
+      "step": 1450
+    },
+    {
+      "epoch": 0.25155065472088217,
+      "grad_norm": 2.2638187408447266,
+      "learning_rate": 4.98990104739064e-08,
+      "logits/chosen": -3.034271717071533,
+      "logits/rejected": -3.001857280731201,
+      "logps/chosen": -52.20647048950195,
+      "logps/rejected": -51.38420486450195,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.005068499129265547,
+      "rewards/margins": 0.004121784586459398,
+      "rewards/rejected": -0.009190283715724945,
+      "step": 1460
+    },
+    {
+      "epoch": 0.2532736044107512,
+      "grad_norm": 2.439119577407837,
+      "learning_rate": 4.98921473016754e-08,
+      "logits/chosen": -3.199528217315674,
+      "logits/rejected": -3.1634347438812256,
+      "logps/chosen": -57.339027404785156,
+      "logps/rejected": -55.59828567504883,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.003153699915856123,
+      "rewards/margins": 0.006065295077860355,
+      "rewards/rejected": -0.00921899639070034,
+      "step": 1470
+    },
+    {
+      "epoch": 0.2549965541006203,
+      "grad_norm": 2.1636698246002197,
+      "learning_rate": 4.9885059027839705e-08,
+      "logits/chosen": -3.166796922683716,
+      "logits/rejected": -3.130619525909424,
+      "logps/chosen": -51.73503494262695,
+      "logps/rejected": -52.469932556152344,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.004243043251335621,
+      "rewards/margins": 0.0030524968169629574,
+      "rewards/rejected": -0.007295540068298578,
+      "step": 1480
+    },
+    {
+      "epoch": 0.2567195037904893,
+      "grad_norm": 2.379169225692749,
+      "learning_rate": 4.987774571649912e-08,
+      "logits/chosen": -3.0981640815734863,
+      "logits/rejected": -3.0734543800354004,
+      "logps/chosen": -57.82059860229492,
+      "logps/rejected": -54.51519012451172,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.0043691531755030155,
+      "rewards/margins": 0.004084555897861719,
+      "rewards/rejected": -0.00845370814204216,
+      "step": 1490
+    },
+    {
+      "epoch": 0.2584424534803584,
+      "grad_norm": 2.303616762161255,
+      "learning_rate": 4.987020743378848e-08,
+      "logits/chosen": -3.0071284770965576,
+      "logits/rejected": -3.0044360160827637,
+      "logps/chosen": -50.43038558959961,
+      "logps/rejected": -53.623497009277344,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.004416764713823795,
+      "rewards/margins": 0.005153198726475239,
+      "rewards/rejected": -0.009569964371621609,
+      "step": 1500
+    },
+    {
+      "epoch": 0.2584424534803584,
+      "eval_logits/chosen": -3.1544973850250244,
+      "eval_logits/rejected": -3.148864984512329,
+      "eval_logps/chosen": -58.52912902832031,
+      "eval_logps/rejected": -63.12455749511719,
+      "eval_loss": 0.6925247311592102,
+      "eval_rewards/accuracies": 0.5643587112426758,
+      "eval_rewards/chosen": 0.0018276170594617724,
+      "eval_rewards/margins": 0.0012719028163701296,
+      "eval_rewards/rejected": 0.0005557141848839819,
+      "eval_runtime": 384.7132,
+      "eval_samples_per_second": 11.188,
+      "eval_steps_per_second": 1.398,
+      "step": 1500
+    },
+    {
+      "epoch": 0.2601654031702274,
+      "grad_norm": 2.452697277069092,
+      "learning_rate": 4.9862444247877054e-08,
+      "logits/chosen": -2.9563889503479004,
+      "logits/rejected": -2.9207029342651367,
+      "logps/chosen": -58.9956169128418,
+      "logps/rejected": -53.739662170410156,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.003896885085850954,
+      "rewards/margins": 0.005234030075371265,
+      "rewards/rejected": -0.009130915626883507,
+      "step": 1510
+    },
+    {
+      "epoch": 0.2618883528600965,
+      "grad_norm": 2.5282890796661377,
+      "learning_rate": 4.985445622896794e-08,
+      "logits/chosen": -3.069789171218872,
+      "logits/rejected": -3.061765193939209,
+      "logps/chosen": -52.86076736450195,
+      "logps/rejected": -52.27109909057617,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.006101188249886036,
+      "rewards/margins": 0.0016308920457959175,
+      "rewards/rejected": -0.007732079830020666,
+      "step": 1520
+    },
+    {
+      "epoch": 0.26361130254996556,
+      "grad_norm": 2.563699245452881,
+      "learning_rate": 4.98462434492974e-08,
+      "logits/chosen": -2.955782651901245,
+      "logits/rejected": -2.939286708831787,
+      "logps/chosen": -52.213172912597656,
+      "logps/rejected": -51.718475341796875,
+      "loss": 0.691,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.005245491862297058,
+      "rewards/margins": 0.0043971119448542595,
+      "rewards/rejected": -0.009642602875828743,
+      "step": 1530
+    },
+    {
+      "epoch": 0.2653342522398346,
+      "grad_norm": 2.2898590564727783,
+      "learning_rate": 4.983780598313423e-08,
+      "logits/chosen": -3.0857558250427246,
+      "logits/rejected": -3.053680419921875,
+      "logps/chosen": -56.428077697753906,
+      "logps/rejected": -52.0638313293457,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.005226061679422855,
+      "rewards/margins": 0.005378265865147114,
+      "rewards/rejected": -0.010604326613247395,
+      "step": 1540
+    },
+    {
+      "epoch": 0.26705720192970367,
+      "grad_norm": 2.1219980716705322,
+      "learning_rate": 4.982914390677909e-08,
+      "logits/chosen": -3.003917694091797,
+      "logits/rejected": -2.9839022159576416,
+      "logps/chosen": -53.95481491088867,
+      "logps/rejected": -51.64226150512695,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.005854880437254906,
+      "rewards/margins": 0.005161617416888475,
+      "rewards/rejected": -0.011016499251127243,
+      "step": 1550
+    },
+    {
+      "epoch": 0.2687801516195727,
+      "grad_norm": 2.2792575359344482,
+      "learning_rate": 4.982025729856381e-08,
+      "logits/chosen": -3.018204927444458,
+      "logits/rejected": -3.000903844833374,
+      "logps/chosen": -51.55298614501953,
+      "logps/rejected": -51.710304260253906,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.007052140776067972,
+      "rewards/margins": 0.0034485969226807356,
+      "rewards/rejected": -0.010500738397240639,
+      "step": 1560
+    },
+    {
+      "epoch": 0.2705031013094418,
+      "grad_norm": 2.2385740280151367,
+      "learning_rate": 4.981114623885067e-08,
+      "logits/chosen": -3.0696630477905273,
+      "logits/rejected": -3.071538209915161,
+      "logps/chosen": -52.41461181640625,
+      "logps/rejected": -57.18894577026367,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.004985310602933168,
+      "rewards/margins": 0.004602603614330292,
+      "rewards/rejected": -0.009587914682924747,
+      "step": 1570
+    },
+    {
+      "epoch": 0.2722260509993108,
+      "grad_norm": 2.690396785736084,
+      "learning_rate": 4.980181081003167e-08,
+      "logits/chosen": -3.00474214553833,
+      "logits/rejected": -2.9946131706237793,
+      "logps/chosen": -53.684814453125,
+      "logps/rejected": -54.92816162109375,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.007695530541241169,
+      "rewards/margins": 0.002679653000086546,
+      "rewards/rejected": -0.010375184938311577,
+      "step": 1580
+    },
+    {
+      "epoch": 0.2739490006891799,
+      "grad_norm": 2.4085440635681152,
+      "learning_rate": 4.9792251096527826e-08,
+      "logits/chosen": -3.0508296489715576,
+      "logits/rejected": -3.03615140914917,
+      "logps/chosen": -54.58491134643555,
+      "logps/rejected": -55.44964599609375,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.004544821567833424,
+      "rewards/margins": 0.0035300354938954115,
+      "rewards/rejected": -0.008074857294559479,
+      "step": 1590
+    },
+    {
+      "epoch": 0.27567195037904896,
+      "grad_norm": 2.3438265323638916,
+      "learning_rate": 4.978246718478836e-08,
+      "logits/chosen": -3.0426697731018066,
+      "logits/rejected": -3.0044350624084473,
+      "logps/chosen": -52.823509216308594,
+      "logps/rejected": -51.93804931640625,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.006442557089030743,
+      "rewards/margins": 0.0034379821736365557,
+      "rewards/rejected": -0.00988053996115923,
+      "step": 1600
+    },
+    {
+      "epoch": 0.27567195037904896,
+      "eval_logits/chosen": -3.152864933013916,
+      "eval_logits/rejected": -3.147214889526367,
+      "eval_logps/chosen": -58.51839828491211,
+      "eval_logps/rejected": -63.13454818725586,
+      "eval_loss": 0.6924260258674622,
+      "eval_rewards/accuracies": 0.5659851431846619,
+      "eval_rewards/chosen": 0.0019349841168150306,
+      "eval_rewards/margins": 0.001479199854657054,
+      "eval_rewards/rejected": 0.0004557841457426548,
+      "eval_runtime": 384.7901,
+      "eval_samples_per_second": 11.185,
+      "eval_steps_per_second": 1.398,
+      "step": 1600
+    },
+    {
+      "epoch": 0.277394900068918,
+      "grad_norm": 2.1588258743286133,
+      "learning_rate": 4.9772459163289934e-08,
+      "logits/chosen": -3.0817911624908447,
+      "logits/rejected": -3.056546688079834,
+      "logps/chosen": -56.615623474121094,
+      "logps/rejected": -58.79069900512695,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.007215184159576893,
+      "rewards/margins": 0.0037064203061163425,
+      "rewards/rejected": -0.010921604000031948,
+      "step": 1610
+    },
+    {
+      "epoch": 0.27911784975878706,
+      "grad_norm": 2.1080305576324463,
+      "learning_rate": 4.976222712253587e-08,
+      "logits/chosen": -3.003810167312622,
+      "logits/rejected": -2.981851100921631,
+      "logps/chosen": -52.130332946777344,
+      "logps/rejected": -55.112945556640625,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.006902821362018585,
+      "rewards/margins": 0.0035740062594413757,
+      "rewards/rejected": -0.010476827621459961,
+      "step": 1620
+    },
+    {
+      "epoch": 0.2808407994486561,
+      "grad_norm": 2.3390843868255615,
+      "learning_rate": 4.9751771155055295e-08,
+      "logits/chosen": -3.0741169452667236,
+      "logits/rejected": -3.0476431846618652,
+      "logps/chosen": -56.03596115112305,
+      "logps/rejected": -52.89629364013672,
+      "loss": 0.69,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0052620768547058105,
+      "rewards/margins": 0.0064377509988844395,
+      "rewards/rejected": -0.011699827387928963,
+      "step": 1630
+    },
+    {
+      "epoch": 0.28256374913852517,
+      "grad_norm": 2.375474452972412,
+      "learning_rate": 4.974109135540232e-08,
+      "logits/chosen": -3.1166491508483887,
+      "logits/rejected": -3.0795044898986816,
+      "logps/chosen": -57.4476203918457,
+      "logps/rejected": -52.65089797973633,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.004932580050081015,
+      "rewards/margins": 0.008146191947162151,
+      "rewards/rejected": -0.013078773394227028,
+      "step": 1640
+    },
+    {
+      "epoch": 0.2842866988283942,
+      "grad_norm": 2.5040836334228516,
+      "learning_rate": 4.97301878201552e-08,
+      "logits/chosen": -3.086254596710205,
+      "logits/rejected": -3.0585713386535645,
+      "logps/chosen": -57.601463317871094,
+      "logps/rejected": -55.981971740722656,
+      "loss": 0.69,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.0036502510774880648,
+      "rewards/margins": 0.006314043886959553,
+      "rewards/rejected": -0.0099642938002944,
+      "step": 1650
+    },
+    {
+      "epoch": 0.28600964851826327,
+      "grad_norm": 2.4886910915374756,
+      "learning_rate": 4.971906064791544e-08,
+      "logits/chosen": -3.121581554412842,
+      "logits/rejected": -3.074537754058838,
+      "logps/chosen": -55.42299270629883,
+      "logps/rejected": -51.00796890258789,
+      "loss": 0.689,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.004514411091804504,
+      "rewards/margins": 0.008407974615693092,
+      "rewards/rejected": -0.012922385707497597,
+      "step": 1660
+    },
+    {
+      "epoch": 0.2877325982081323,
+      "grad_norm": 2.512758255004883,
+      "learning_rate": 4.970770993930693e-08,
+      "logits/chosen": -3.1094958782196045,
+      "logits/rejected": -3.087451696395874,
+      "logps/chosen": -54.83356475830078,
+      "logps/rejected": -56.27382278442383,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.003189521376043558,
+      "rewards/margins": 0.007494642399251461,
+      "rewards/rejected": -0.010684163309633732,
+      "step": 1670
+    },
+    {
+      "epoch": 0.2894555478980014,
+      "grad_norm": 2.241192579269409,
+      "learning_rate": 4.969613579697499e-08,
+      "logits/chosen": -3.066695213317871,
+      "logits/rejected": -3.040623903274536,
+      "logps/chosen": -56.35309982299805,
+      "logps/rejected": -55.74150466918945,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.007447590120136738,
+      "rewards/margins": 0.005884532816708088,
+      "rewards/rejected": -0.013332122936844826,
+      "step": 1680
+    },
+    {
+      "epoch": 0.29117849758787046,
+      "grad_norm": 2.355499267578125,
+      "learning_rate": 4.968433832558549e-08,
+      "logits/chosen": -3.046165943145752,
+      "logits/rejected": -3.0305047035217285,
+      "logps/chosen": -56.28789138793945,
+      "logps/rejected": -53.171592712402344,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.00742595037445426,
+      "rewards/margins": 0.0015562146436423063,
+      "rewards/rejected": -0.008982164785265923,
+      "step": 1690
+    },
+    {
+      "epoch": 0.2929014472777395,
+      "grad_norm": 2.4819881916046143,
+      "learning_rate": 4.967231763182385e-08,
+      "logits/chosen": -2.9544806480407715,
+      "logits/rejected": -2.9593846797943115,
+      "logps/chosen": -50.40542221069336,
+      "logps/rejected": -56.5596809387207,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.008423763327300549,
+      "rewards/margins": 0.004045287612825632,
+      "rewards/rejected": -0.012469050474464893,
+      "step": 1700
+    },
+    {
+      "epoch": 0.2929014472777395,
+      "eval_logits/chosen": -3.1501998901367188,
+      "eval_logits/rejected": -3.1445610523223877,
+      "eval_logps/chosen": -58.50439453125,
+      "eval_logps/rejected": -63.15779113769531,
+      "eval_loss": 0.6922464370727539,
+      "eval_rewards/accuracies": 0.5634293556213379,
+      "eval_rewards/chosen": 0.002075033728033304,
+      "eval_rewards/margins": 0.0018516989657655358,
+      "eval_rewards/rejected": 0.0002233349223388359,
+      "eval_runtime": 384.8754,
+      "eval_samples_per_second": 11.183,
+      "eval_steps_per_second": 1.398,
+      "step": 1700
+    },
+    {
+      "epoch": 0.29462439696760856,
+      "grad_norm": 2.547050952911377,
+      "learning_rate": 4.966007382439414e-08,
+      "logits/chosen": -3.0869879722595215,
+      "logits/rejected": -3.0462372303009033,
+      "logps/chosen": -58.976402282714844,
+      "logps/rejected": -55.634246826171875,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0056662289425730705,
+      "rewards/margins": 0.00584133667871356,
+      "rewards/rejected": -0.011507566086947918,
+      "step": 1710
+    },
+    {
+      "epoch": 0.2963473466574776,
+      "grad_norm": 2.3649723529815674,
+      "learning_rate": 4.964760701401807e-08,
+      "logits/chosen": -3.095816135406494,
+      "logits/rejected": -3.066598892211914,
+      "logps/chosen": -57.116065979003906,
+      "logps/rejected": -52.17816925048828,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.0069018201902508736,
+      "rewards/margins": 0.004321283660829067,
+      "rewards/rejected": -0.011223104782402515,
+      "step": 1720
+    },
+    {
+      "epoch": 0.29807029634734666,
+      "grad_norm": 2.187278985977173,
+      "learning_rate": 4.963491731343395e-08,
+      "logits/chosen": -3.0910511016845703,
+      "logits/rejected": -3.073817729949951,
+      "logps/chosen": -54.72662353515625,
+      "logps/rejected": -53.55133819580078,
+      "loss": 0.692,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.007922997698187828,
+      "rewards/margins": 0.002532970393076539,
+      "rewards/rejected": -0.010455967858433723,
+      "step": 1730
+    },
+    {
+      "epoch": 0.2997932460372157,
+      "grad_norm": 2.326688766479492,
+      "learning_rate": 4.9622004837395725e-08,
+      "logits/chosen": -3.0401599407196045,
+      "logits/rejected": -3.0300910472869873,
+      "logps/chosen": -53.082374572753906,
+      "logps/rejected": -55.3353157043457,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.00981101207435131,
+      "rewards/margins": 0.0029289829544723034,
+      "rewards/rejected": -0.012739995494484901,
+      "step": 1740
+    },
+    {
+      "epoch": 0.30151619572708477,
+      "grad_norm": 2.5052545070648193,
+      "learning_rate": 4.9608869702671903e-08,
+      "logits/chosen": -3.08416748046875,
+      "logits/rejected": -3.0729095935821533,
+      "logps/chosen": -55.7491340637207,
+      "logps/rejected": -56.982154846191406,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.005631032399833202,
+      "rewards/margins": 0.003924180753529072,
+      "rewards/rejected": -0.009555214084684849,
+      "step": 1750
+    },
+    {
+      "epoch": 0.30323914541695385,
+      "grad_norm": 2.386603832244873,
+      "learning_rate": 4.9595512028044526e-08,
+      "logits/chosen": -3.0715887546539307,
+      "logits/rejected": -3.034848928451538,
+      "logps/chosen": -56.62477493286133,
+      "logps/rejected": -54.42699432373047,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.006016246043145657,
+      "rewards/margins": 0.008768090978264809,
+      "rewards/rejected": -0.01478433795273304,
+      "step": 1760
+    },
+    {
+      "epoch": 0.3049620951068229,
+      "grad_norm": 1.9907580614089966,
+      "learning_rate": 4.958193193430807e-08,
+      "logits/chosen": -3.0987541675567627,
+      "logits/rejected": -3.062049150466919,
+      "logps/chosen": -55.8708381652832,
+      "logps/rejected": -50.57661056518555,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.008891335688531399,
+      "rewards/margins": 0.007452656514942646,
+      "rewards/rejected": -0.016343992203474045,
+      "step": 1770
+    },
+    {
+      "epoch": 0.30668504479669195,
+      "grad_norm": 2.6231467723846436,
+      "learning_rate": 4.956812954426837e-08,
+      "logits/chosen": -3.0133068561553955,
+      "logits/rejected": -3.015068769454956,
+      "logps/chosen": -55.39440155029297,
+      "logps/rejected": -59.69197463989258,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.008758464828133583,
+      "rewards/margins": 0.0035373582504689693,
+      "rewards/rejected": -0.012295822612941265,
+      "step": 1780
+    },
+    {
+      "epoch": 0.308407994486561,
+      "grad_norm": 2.586334228515625,
+      "learning_rate": 4.9554104982741504e-08,
+      "logits/chosen": -3.159836769104004,
+      "logits/rejected": -3.1374447345733643,
+      "logps/chosen": -52.631736755371094,
+      "logps/rejected": -53.81390380859375,
+      "loss": 0.69,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.00636354461312294,
+      "rewards/margins": 0.006470109336078167,
+      "rewards/rejected": -0.012833654880523682,
+      "step": 1790
+    },
+    {
+      "epoch": 0.31013094417643006,
+      "grad_norm": 2.200411558151245,
+      "learning_rate": 4.953985837655266e-08,
+      "logits/chosen": -3.123415470123291,
+      "logits/rejected": -3.0996241569519043,
+      "logps/chosen": -52.55958938598633,
+      "logps/rejected": -53.61354446411133,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.007574164308607578,
+      "rewards/margins": 0.008652850054204464,
+      "rewards/rejected": -0.016227014362812042,
+      "step": 1800
+    },
+    {
+      "epoch": 0.31013094417643006,
+      "eval_logits/chosen": -3.148066520690918,
+      "eval_logits/rejected": -3.142429828643799,
+      "eval_logps/chosen": -58.5174674987793,
+      "eval_logps/rejected": -63.1905632019043,
+      "eval_loss": 0.6921526789665222,
+      "eval_rewards/accuracies": 0.5652881264686584,
+      "eval_rewards/chosen": 0.0019442432094365358,
+      "eval_rewards/margins": 0.0020486859139055014,
+      "eval_rewards/rejected": -0.00010444273357279599,
+      "eval_runtime": 385.0778,
+      "eval_samples_per_second": 11.177,
+      "eval_steps_per_second": 1.397,
+      "step": 1800
+    },
+    {
+      "epoch": 0.3118538938662991,
+      "grad_norm": 2.394313097000122,
+      "learning_rate": 4.952538985453499e-08,
+      "logits/chosen": -3.1263439655303955,
+      "logits/rejected": -3.0907680988311768,
+      "logps/chosen": -57.171791076660156,
+      "logps/rejected": -54.928672790527344,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.005546397529542446,
+      "rewards/margins": 0.006834310479462147,
+      "rewards/rejected": -0.012380707077682018,
+      "step": 1810
+    },
+    {
+      "epoch": 0.31357684355616816,
+      "grad_norm": 2.297745943069458,
+      "learning_rate": 4.9510699547528456e-08,
+      "logits/chosen": -3.0971500873565674,
+      "logits/rejected": -3.0625205039978027,
+      "logps/chosen": -55.925254821777344,
+      "logps/rejected": -51.983642578125,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.007631627377122641,
+      "rewards/margins": 0.005384031217545271,
+      "rewards/rejected": -0.013015657663345337,
+      "step": 1820
+    },
+    {
+      "epoch": 0.31529979324603724,
+      "grad_norm": 2.2693684101104736,
+      "learning_rate": 4.949578758837864e-08,
+      "logits/chosen": -2.9917683601379395,
+      "logits/rejected": -2.977911949157715,
+      "logps/chosen": -56.2487678527832,
+      "logps/rejected": -55.882347106933594,
+      "loss": 0.691,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.007054704241454601,
+      "rewards/margins": 0.004440615884959698,
+      "rewards/rejected": -0.011495320126414299,
+      "step": 1830
+    },
+    {
+      "epoch": 0.31702274293590627,
+      "grad_norm": 2.399961233139038,
+      "learning_rate": 4.948065411193554e-08,
+      "logits/chosen": -3.201258420944214,
+      "logits/rejected": -3.1973021030426025,
+      "logps/chosen": -55.89738845825195,
+      "logps/rejected": -56.39849090576172,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.010772759094834328,
+      "rewards/margins": 0.0027497331611812115,
+      "rewards/rejected": -0.013522490859031677,
+      "step": 1840
+    },
+    {
+      "epoch": 0.31874569262577535,
+      "grad_norm": 2.377139091491699,
+      "learning_rate": 4.946529925505233e-08,
+      "logits/chosen": -3.066678524017334,
+      "logits/rejected": -3.066776752471924,
+      "logps/chosen": -54.073997497558594,
+      "logps/rejected": -56.110328674316406,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.008642922155559063,
+      "rewards/margins": 0.005384859628975391,
+      "rewards/rejected": -0.014027783647179604,
+      "step": 1850
+    },
+    {
+      "epoch": 0.32046864231564437,
+      "grad_norm": 2.49811053276062,
+      "learning_rate": 4.9449723156584175e-08,
+      "logits/chosen": -2.999185800552368,
+      "logits/rejected": -2.962897539138794,
+      "logps/chosen": -57.76921463012695,
+      "logps/rejected": -55.38594436645508,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.00477262819185853,
+      "rewards/margins": 0.011732656508684158,
+      "rewards/rejected": -0.0165052842348814,
+      "step": 1860
+    },
+    {
+      "epoch": 0.32219159200551345,
+      "grad_norm": 2.6477980613708496,
+      "learning_rate": 4.943392595738694e-08,
+      "logits/chosen": -3.0453426837921143,
+      "logits/rejected": -3.0198445320129395,
+      "logps/chosen": -56.69841766357422,
+      "logps/rejected": -55.8046760559082,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.007487172726541758,
+      "rewards/margins": 0.012675365433096886,
+      "rewards/rejected": -0.020162541419267654,
+      "step": 1870
+    },
+    {
+      "epoch": 0.3239145416953825,
+      "grad_norm": 2.336548328399658,
+      "learning_rate": 4.9417907800315904e-08,
+      "logits/chosen": -3.0918257236480713,
+      "logits/rejected": -3.053905963897705,
+      "logps/chosen": -53.47248458862305,
+      "logps/rejected": -53.1220703125,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.007566615007817745,
+      "rewards/margins": 0.00894746370613575,
+      "rewards/rejected": -0.01651407778263092,
+      "step": 1880
+    },
+    {
+      "epoch": 0.32563749138525155,
+      "grad_norm": 2.576293468475342,
+      "learning_rate": 4.94016688302245e-08,
+      "logits/chosen": -3.114389657974243,
+      "logits/rejected": -3.1086478233337402,
+      "logps/chosen": -53.63127517700195,
+      "logps/rejected": -56.028892517089844,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.005735150538384914,
+      "rewards/margins": 0.010075276717543602,
+      "rewards/rejected": -0.015810426324605942,
+      "step": 1890
+    },
+    {
+      "epoch": 0.32736044107512063,
+      "grad_norm": 2.517943859100342,
+      "learning_rate": 4.9385209193962974e-08,
+      "logits/chosen": -3.0471818447113037,
+      "logits/rejected": -3.0141072273254395,
+      "logps/chosen": -58.10990524291992,
+      "logps/rejected": -53.577362060546875,
+      "loss": 0.69,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.006701114121824503,
+      "rewards/margins": 0.006454487796872854,
+      "rewards/rejected": -0.013155601918697357,
+      "step": 1900
+    },
+    {
+      "epoch": 0.32736044107512063,
+      "eval_logits/chosen": -3.1463582515716553,
+      "eval_logits/rejected": -3.140700578689575,
+      "eval_logps/chosen": -58.52104187011719,
+      "eval_logps/rejected": -63.24062728881836,
+      "eval_loss": 0.6919280886650085,
+      "eval_rewards/accuracies": 0.5771375298500061,
+      "eval_rewards/chosen": 0.001908516394905746,
+      "eval_rewards/margins": 0.0025135509204119444,
+      "eval_rewards/rejected": -0.0006050347001291811,
+      "eval_runtime": 384.2912,
+      "eval_samples_per_second": 11.2,
+      "eval_steps_per_second": 1.4,
+      "step": 1900
+    },
+    {
+      "epoch": 0.32908339076498966,
+      "grad_norm": 2.596487522125244,
+      "learning_rate": 4.93685290403771e-08,
+      "logits/chosen": -2.9463412761688232,
+      "logits/rejected": -2.9115188121795654,
+      "logps/chosen": -55.4236946105957,
+      "logps/rejected": -55.08970260620117,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.009559987112879753,
+      "rewards/margins": 0.008361767046153545,
+      "rewards/rejected": -0.017921755090355873,
+      "step": 1910
+    },
+    {
+      "epoch": 0.33080634045485874,
+      "grad_norm": 2.362659215927124,
+      "learning_rate": 4.9351628520306774e-08,
+      "logits/chosen": -3.0599961280822754,
+      "logits/rejected": -3.0321669578552246,
+      "logps/chosen": -56.270164489746094,
+      "logps/rejected": -55.482696533203125,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.00901498831808567,
+      "rewards/margins": 0.008792147971689701,
+      "rewards/rejected": -0.017807137221097946,
+      "step": 1920
+    },
+    {
+      "epoch": 0.33252929014472776,
+      "grad_norm": 2.1666100025177,
+      "learning_rate": 4.933450778658472e-08,
+      "logits/chosen": -3.0084261894226074,
+      "logits/rejected": -2.971450090408325,
+      "logps/chosen": -54.70751953125,
+      "logps/rejected": -55.65217208862305,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.008170957677066326,
+      "rewards/margins": 0.009846963919699192,
+      "rewards/rejected": -0.018017921596765518,
+      "step": 1930
+    },
+    {
+      "epoch": 0.33425223983459684,
+      "grad_norm": 2.225851058959961,
+      "learning_rate": 4.9317166994035036e-08,
+      "logits/chosen": -3.032421588897705,
+      "logits/rejected": -3.0139946937561035,
+      "logps/chosen": -52.16240310668945,
+      "logps/rejected": -51.86973190307617,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.012224128469824791,
+      "rewards/margins": 0.006234189961105585,
+      "rewards/rejected": -0.018458319827914238,
+      "step": 1940
+    },
+    {
+      "epoch": 0.33597518952446587,
+      "grad_norm": 2.2780954837799072,
+      "learning_rate": 4.929960629947185e-08,
+      "logits/chosen": -3.038595199584961,
+      "logits/rejected": -3.037022113800049,
+      "logps/chosen": -53.5986442565918,
+      "logps/rejected": -56.46440887451172,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.010910035111010075,
+      "rewards/margins": 0.0059595005586743355,
+      "rewards/rejected": -0.01686953380703926,
+      "step": 1950
+    },
+    {
+      "epoch": 0.33769813921433495,
+      "grad_norm": 2.351255178451538,
+      "learning_rate": 4.928182586169787e-08,
+      "logits/chosen": -3.044883966445923,
+      "logits/rejected": -3.0256218910217285,
+      "logps/chosen": -55.34600067138672,
+      "logps/rejected": -56.22704315185547,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.009864835068583488,
+      "rewards/margins": 0.007436974439769983,
+      "rewards/rejected": -0.017301809042692184,
+      "step": 1960
+    },
+    {
+      "epoch": 0.33942108890420397,
+      "grad_norm": 2.1785995960235596,
+      "learning_rate": 4.926382584150298e-08,
+      "logits/chosen": -3.089529037475586,
+      "logits/rejected": -3.060178279876709,
+      "logps/chosen": -56.95575714111328,
+      "logps/rejected": -53.68779373168945,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.008091448806226254,
+      "rewards/margins": 0.01033073477447033,
+      "rewards/rejected": -0.018422184512019157,
+      "step": 1970
+    },
+    {
+      "epoch": 0.34114403859407305,
+      "grad_norm": 2.4621241092681885,
+      "learning_rate": 4.924560640166273e-08,
+      "logits/chosen": -2.995124578475952,
+      "logits/rejected": -2.9896817207336426,
+      "logps/chosen": -56.81432342529297,
+      "logps/rejected": -55.559043884277344,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.012265205383300781,
+      "rewards/margins": 0.00293142581358552,
+      "rewards/rejected": -0.015196630731225014,
+      "step": 1980
+    },
+    {
+      "epoch": 0.34286698828394213,
+      "grad_norm": 2.3733723163604736,
+      "learning_rate": 4.922716770693692e-08,
+      "logits/chosen": -3.123896360397339,
+      "logits/rejected": -3.0889506340026855,
+      "logps/chosen": -52.01383590698242,
+      "logps/rejected": -50.25537872314453,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.01179430540651083,
+      "rewards/margins": 0.009063664823770523,
+      "rewards/rejected": -0.020857971161603928,
+      "step": 1990
+    },
+    {
+      "epoch": 0.34458993797381116,
+      "grad_norm": 2.5672614574432373,
+      "learning_rate": 4.920850992406809e-08,
+      "logits/chosen": -3.0455069541931152,
+      "logits/rejected": -3.0517501831054688,
+      "logps/chosen": -51.317840576171875,
+      "logps/rejected": -59.641029357910156,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.013569338247179985,
+      "rewards/margins": 0.006717065814882517,
+      "rewards/rejected": -0.020286407321691513,
+      "step": 2000
+    },
+    {
+      "epoch": 0.34458993797381116,
+      "eval_logits/chosen": -3.1432974338531494,
+      "eval_logits/rejected": -3.137617826461792,
+      "eval_logps/chosen": -58.55636215209961,
+      "eval_logps/rejected": -63.291343688964844,
+      "eval_loss": 0.6918609142303467,
+      "eval_rewards/accuracies": 0.5771375298500061,
+      "eval_rewards/chosen": 0.0015553623670712113,
+      "eval_rewards/margins": 0.00266754487529397,
+      "eval_rewards/rejected": -0.001112182391807437,
+      "eval_runtime": 383.0568,
+      "eval_samples_per_second": 11.236,
+      "eval_steps_per_second": 1.404,
+      "step": 2000
+    },
+    {
+      "epoch": 0.34631288766368024,
+      "grad_norm": 2.439398765563965,
+      "learning_rate": 4.918963322178001e-08,
+      "logits/chosen": -3.0514400005340576,
+      "logits/rejected": -3.0230164527893066,
+      "logps/chosen": -56.60076904296875,
+      "logps/rejected": -53.78191375732422,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.009988987818360329,
+      "rewards/margins": 0.009806125424802303,
+      "rewards/rejected": -0.019795114174485207,
+      "step": 2010
+    },
+    {
+      "epoch": 0.34803583735354926,
+      "grad_norm": 2.296095132827759,
+      "learning_rate": 4.917053777077616e-08,
+      "logits/chosen": -3.0394394397735596,
+      "logits/rejected": -3.017590045928955,
+      "logps/chosen": -51.05394744873047,
+      "logps/rejected": -56.84633255004883,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.012246248312294483,
+      "rewards/margins": 0.009431076236069202,
+      "rewards/rejected": -0.021677324548363686,
+      "step": 2020
+    },
+    {
+      "epoch": 0.34975878704341834,
+      "grad_norm": 2.3916471004486084,
+      "learning_rate": 4.915122374373815e-08,
+      "logits/chosen": -3.1270077228546143,
+      "logits/rejected": -3.1054224967956543,
+      "logps/chosen": -58.155540466308594,
+      "logps/rejected": -57.05060958862305,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.011032627895474434,
+      "rewards/margins": 0.007658441551029682,
+      "rewards/rejected": -0.01869107037782669,
+      "step": 2030
+    },
+    {
+      "epoch": 0.35148173673328736,
+      "grad_norm": 2.39408278465271,
+      "learning_rate": 4.9131691315324224e-08,
+      "logits/chosen": -2.9906091690063477,
+      "logits/rejected": -2.9778077602386475,
+      "logps/chosen": -53.980430603027344,
+      "logps/rejected": -55.977874755859375,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.012644794769585133,
+      "rewards/margins": 0.0075990790501236916,
+      "rewards/rejected": -0.020243871957063675,
+      "step": 2040
+    },
+    {
+      "epoch": 0.35320468642315644,
+      "grad_norm": 2.433422565460205,
+      "learning_rate": 4.911194066216765e-08,
+      "logits/chosen": -3.1080446243286133,
+      "logits/rejected": -3.0830893516540527,
+      "logps/chosen": -52.22490310668945,
+      "logps/rejected": -56.18841552734375,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.010028828866779804,
+      "rewards/margins": 0.00775051349774003,
+      "rewards/rejected": -0.017779342830181122,
+      "step": 2050
+    },
+    {
+      "epoch": 0.3549276361130255,
+      "grad_norm": 2.2094593048095703,
+      "learning_rate": 4.909197196287509e-08,
+      "logits/chosen": -3.095278263092041,
+      "logits/rejected": -3.0567901134490967,
+      "logps/chosen": -56.10393142700195,
+      "logps/rejected": -53.3204231262207,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.007744644768536091,
+      "rewards/margins": 0.009637429378926754,
+      "rewards/rejected": -0.017382074147462845,
+      "step": 2060
+    },
+    {
+      "epoch": 0.35665058580289455,
+      "grad_norm": 2.488966226577759,
+      "learning_rate": 4.907178539802503e-08,
+      "logits/chosen": -3.112682580947876,
+      "logits/rejected": -3.089406728744507,
+      "logps/chosen": -56.32941436767578,
+      "logps/rejected": -56.4896240234375,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.010362197645008564,
+      "rewards/margins": 0.006709497421979904,
+      "rewards/rejected": -0.017071694135665894,
+      "step": 2070
+    },
+    {
+      "epoch": 0.35837353549276363,
+      "grad_norm": 2.4468612670898438,
+      "learning_rate": 4.9051381150166136e-08,
+      "logits/chosen": -3.0530130863189697,
+      "logits/rejected": -3.023387908935547,
+      "logps/chosen": -53.25905227661133,
+      "logps/rejected": -52.76204299926758,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.013912415131926537,
+      "rewards/margins": 0.00739931408315897,
+      "rewards/rejected": -0.02131173014640808,
+      "step": 2080
+    },
+    {
+      "epoch": 0.36009648518263265,
+      "grad_norm": 2.2995336055755615,
+      "learning_rate": 4.903075940381559e-08,
+      "logits/chosen": -3.0774669647216797,
+      "logits/rejected": -3.074070453643799,
+      "logps/chosen": -55.41902542114258,
+      "logps/rejected": -56.202720642089844,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.010242661461234093,
+      "rewards/margins": 0.005199112929403782,
+      "rewards/rejected": -0.01544177532196045,
+      "step": 2090
+    },
+    {
+      "epoch": 0.36181943487250173,
+      "grad_norm": 2.673253059387207,
+      "learning_rate": 4.900992034545743e-08,
+      "logits/chosen": -3.02644681930542,
+      "logits/rejected": -3.0042808055877686,
+      "logps/chosen": -56.63521194458008,
+      "logps/rejected": -53.34986114501953,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.012887482531368732,
+      "rewards/margins": 0.008120874874293804,
+      "rewards/rejected": -0.021008357405662537,
+      "step": 2100
+    },
+    {
+      "epoch": 0.36181943487250173,
+      "eval_logits/chosen": -3.141157865524292,
+      "eval_logits/rejected": -3.1355414390563965,
+      "eval_logps/chosen": -58.58728790283203,
+      "eval_logps/rejected": -63.352333068847656,
+      "eval_loss": 0.6917189359664917,
+      "eval_rewards/accuracies": 0.574117124080658,
+      "eval_rewards/chosen": 0.0012460495345294476,
+      "eval_rewards/margins": 0.002968190936371684,
+      "eval_rewards/rejected": -0.0017221415182575583,
+      "eval_runtime": 384.9823,
+      "eval_samples_per_second": 11.18,
+      "eval_steps_per_second": 1.397,
+      "step": 2100
+    },
+    {
+      "epoch": 0.36354238456237076,
+      "grad_norm": 2.3091492652893066,
+      "learning_rate": 4.898886416354088e-08,
+      "logits/chosen": -3.082261323928833,
+      "logits/rejected": -3.0853066444396973,
+      "logps/chosen": -50.565589904785156,
+      "logps/rejected": -57.25511932373047,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.016659824177622795,
+      "rewards/margins": 0.003997630439698696,
+      "rewards/rejected": -0.020657453685998917,
+      "step": 2110
+    },
+    {
+      "epoch": 0.36526533425223984,
+      "grad_norm": 2.391576051712036,
+      "learning_rate": 4.896759104847859e-08,
+      "logits/chosen": -2.947147846221924,
+      "logits/rejected": -2.9198501110076904,
+      "logps/chosen": -54.49988555908203,
+      "logps/rejected": -52.780616760253906,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.009242286905646324,
+      "rewards/margins": 0.009268452413380146,
+      "rewards/rejected": -0.018510738387703896,
+      "step": 2120
+    },
+    {
+      "epoch": 0.3669882839421089,
+      "grad_norm": 2.1581666469573975,
+      "learning_rate": 4.8946101192644994e-08,
+      "logits/chosen": -2.9847073554992676,
+      "logits/rejected": -2.966226577758789,
+      "logps/chosen": -55.905853271484375,
+      "logps/rejected": -54.883575439453125,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.014355327002704144,
+      "rewards/margins": 0.009111381135880947,
+      "rewards/rejected": -0.02346671000123024,
+      "step": 2130
+    },
+    {
+      "epoch": 0.36871123363197794,
+      "grad_norm": 2.36787486076355,
+      "learning_rate": 4.8924394790374505e-08,
+      "logits/chosen": -2.9621386528015137,
+      "logits/rejected": -2.9540748596191406,
+      "logps/chosen": -50.79756546020508,
+      "logps/rejected": -56.081809997558594,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.02142391912639141,
+      "rewards/margins": 0.003870949614793062,
+      "rewards/rejected": -0.025294870138168335,
+      "step": 2140
+    },
+    {
+      "epoch": 0.370434183321847,
+      "grad_norm": 2.312957286834717,
+      "learning_rate": 4.8902472037959796e-08,
+      "logits/chosen": -3.0135490894317627,
+      "logits/rejected": -2.967263698577881,
+      "logps/chosen": -55.557403564453125,
+      "logps/rejected": -50.41822052001953,
+      "loss": 0.6857,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.008688176050782204,
+      "rewards/margins": 0.015405970625579357,
+      "rewards/rejected": -0.024094145745038986,
+      "step": 2150
+    },
+    {
+      "epoch": 0.37215713301171605,
+      "grad_norm": 2.343566656112671,
+      "learning_rate": 4.888033313365001e-08,
+      "logits/chosen": -2.9797677993774414,
+      "logits/rejected": -2.9645633697509766,
+      "logps/chosen": -56.07050704956055,
+      "logps/rejected": -51.643821716308594,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.013134809210896492,
+      "rewards/margins": 0.007764987647533417,
+      "rewards/rejected": -0.020899798721075058,
+      "step": 2160
+    },
+    {
+      "epoch": 0.3738800827015851,
+      "grad_norm": 2.4061942100524902,
+      "learning_rate": 4.885797827764895e-08,
+      "logits/chosen": -3.1028759479522705,
+      "logits/rejected": -3.0869998931884766,
+      "logps/chosen": -55.68096160888672,
+      "logps/rejected": -53.65068817138672,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.013497320003807545,
+      "rewards/margins": 0.008188484236598015,
+      "rewards/rejected": -0.021685805171728134,
+      "step": 2170
+    },
+    {
+      "epoch": 0.37560303239145415,
+      "grad_norm": 2.7323555946350098,
+      "learning_rate": 4.88354076721133e-08,
+      "logits/chosen": -3.15089750289917,
+      "logits/rejected": -3.112945079803467,
+      "logps/chosen": -59.338233947753906,
+      "logps/rejected": -55.4169921875,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.013141061179339886,
+      "rewards/margins": 0.01368576567620039,
+      "rewards/rejected": -0.026826824992895126,
+      "step": 2180
+    },
+    {
+      "epoch": 0.37732598208132323,
+      "grad_norm": 2.622997760772705,
+      "learning_rate": 4.88126215211508e-08,
+      "logits/chosen": -3.21452260017395,
+      "logits/rejected": -3.2080929279327393,
+      "logps/chosen": -54.2584228515625,
+      "logps/rejected": -55.864952087402344,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.014165801927447319,
+      "rewards/margins": 0.005931084044277668,
+      "rewards/rejected": -0.02009688690304756,
+      "step": 2190
+    },
+    {
+      "epoch": 0.37904893177119225,
+      "grad_norm": 2.4621634483337402,
+      "learning_rate": 4.878962003081835e-08,
+      "logits/chosen": -3.044160842895508,
+      "logits/rejected": -3.0212242603302,
+      "logps/chosen": -53.479270935058594,
+      "logps/rejected": -57.32262420654297,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.01170541811734438,
+      "rewards/margins": 0.013583732768893242,
+      "rewards/rejected": -0.025289153680205345,
+      "step": 2200
+    },
+    {
+      "epoch": 0.37904893177119225,
+      "eval_logits/chosen": -3.1380650997161865,
+      "eval_logits/rejected": -3.132411241531372,
+      "eval_logps/chosen": -58.630428314208984,
+      "eval_logps/rejected": -63.43059539794922,
+      "eval_loss": 0.6915519833564758,
+      "eval_rewards/accuracies": 0.5743494629859924,
+      "eval_rewards/chosen": 0.0008146798354573548,
+      "eval_rewards/margins": 0.003319424344226718,
+      "eval_rewards/rejected": -0.002504744566977024,
+      "eval_runtime": 383.4709,
+      "eval_samples_per_second": 11.224,
+      "eval_steps_per_second": 1.403,
+      "step": 2200
+    },
+    {
+      "epoch": 0.38077188146106133,
+      "grad_norm": 2.309136390686035,
+      "learning_rate": 4.87664034091202e-08,
+      "logits/chosen": -3.1016571521759033,
+      "logits/rejected": -3.085378646850586,
+      "logps/chosen": -55.73920822143555,
+      "logps/rejected": -55.83252716064453,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.010128566063940525,
+      "rewards/margins": 0.0111006461083889,
+      "rewards/rejected": -0.021229213103652,
+      "step": 2210
+    },
+    {
+      "epoch": 0.3824948311509304,
+      "grad_norm": 2.396667718887329,
+      "learning_rate": 4.8742971866006064e-08,
+      "logits/chosen": -2.9284088611602783,
+      "logits/rejected": -2.921046257019043,
+      "logps/chosen": -55.0631217956543,
+      "logps/rejected": -56.01151657104492,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.011851219460368156,
+      "rewards/margins": 0.00979273859411478,
+      "rewards/rejected": -0.021643957123160362,
+      "step": 2220
+    },
+    {
+      "epoch": 0.38421778084079944,
+      "grad_norm": 2.337989091873169,
+      "learning_rate": 4.8719325613369177e-08,
+      "logits/chosen": -3.0859460830688477,
+      "logits/rejected": -3.0541107654571533,
+      "logps/chosen": -55.3967399597168,
+      "logps/rejected": -53.51177978515625,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.01719152182340622,
+      "rewards/margins": 0.010506326332688332,
+      "rewards/rejected": -0.0276978500187397,
+      "step": 2230
+    },
+    {
+      "epoch": 0.3859407305306685,
+      "grad_norm": 2.577080488204956,
+      "learning_rate": 4.869546486504443e-08,
+      "logits/chosen": -3.038649797439575,
+      "logits/rejected": -3.002626895904541,
+      "logps/chosen": -58.27519607543945,
+      "logps/rejected": -56.236045837402344,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.014655525796115398,
+      "rewards/margins": 0.011769579723477364,
+      "rewards/rejected": -0.026425102725625038,
+      "step": 2240
+    },
+    {
+      "epoch": 0.38766368022053754,
+      "grad_norm": 2.6772000789642334,
+      "learning_rate": 4.8671389836806395e-08,
+      "logits/chosen": -2.9977569580078125,
+      "logits/rejected": -2.9633584022521973,
+      "logps/chosen": -56.72434616088867,
+      "logps/rejected": -55.922279357910156,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.013744845986366272,
+      "rewards/margins": 0.012024512514472008,
+      "rewards/rejected": -0.02576935850083828,
+      "step": 2250
+    },
+    {
+      "epoch": 0.3893866299104066,
+      "grad_norm": 2.2914111614227295,
+      "learning_rate": 4.864710074636742e-08,
+      "logits/chosen": -2.954263210296631,
+      "logits/rejected": -2.928157329559326,
+      "logps/chosen": -62.249961853027344,
+      "logps/rejected": -59.8015251159668,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.014423278160393238,
+      "rewards/margins": 0.00994022935628891,
+      "rewards/rejected": -0.024363510310649872,
+      "step": 2260
+    },
+    {
+      "epoch": 0.39110957960027565,
+      "grad_norm": 2.5474023818969727,
+      "learning_rate": 4.862259781337561e-08,
+      "logits/chosen": -2.939342737197876,
+      "logits/rejected": -2.9061412811279297,
+      "logps/chosen": -56.060150146484375,
+      "logps/rejected": -54.82807159423828,
+      "loss": 0.6879,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.019987616688013077,
+      "rewards/margins": 0.011128445155918598,
+      "rewards/rejected": -0.03111605904996395,
+      "step": 2270
+    },
+    {
+      "epoch": 0.3928325292901447,
+      "grad_norm": 2.4649059772491455,
+      "learning_rate": 4.8597881259412874e-08,
+      "logits/chosen": -3.0179972648620605,
+      "logits/rejected": -3.0079293251037598,
+      "logps/chosen": -54.802581787109375,
+      "logps/rejected": -57.5006103515625,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.013590415008366108,
+      "rewards/margins": 0.006114205811172724,
+      "rewards/rejected": -0.01970462128520012,
+      "step": 2280
+    },
+    {
+      "epoch": 0.3945554789800138,
+      "grad_norm": 2.567060708999634,
+      "learning_rate": 4.857295130799293e-08,
+      "logits/chosen": -2.9124302864074707,
+      "logits/rejected": -2.8838517665863037,
+      "logps/chosen": -54.587135314941406,
+      "logps/rejected": -56.66890335083008,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.01704244688153267,
+      "rewards/margins": 0.012043480761349201,
+      "rewards/rejected": -0.029085928574204445,
+      "step": 2290
+    },
+    {
+      "epoch": 0.39627842866988283,
+      "grad_norm": 2.3706486225128174,
+      "learning_rate": 4.8547808184559225e-08,
+      "logits/chosen": -3.0982589721679688,
+      "logits/rejected": -3.064117670059204,
+      "logps/chosen": -56.87422561645508,
+      "logps/rejected": -56.080039978027344,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.009291517548263073,
+      "rewards/margins": 0.014950186014175415,
+      "rewards/rejected": -0.024241704493761063,
+      "step": 2300
+    },
+    {
+      "epoch": 0.39627842866988283,
+      "eval_logits/chosen": -3.136117935180664,
+      "eval_logits/rejected": -3.1304562091827393,
+      "eval_logps/chosen": -58.685909271240234,
+      "eval_logps/rejected": -63.52629089355469,
+      "eval_loss": 0.6913630962371826,
+      "eval_rewards/accuracies": 0.5683085322380066,
+      "eval_rewards/chosen": 0.00025985814863815904,
+      "eval_rewards/margins": 0.003721560351550579,
+      "eval_rewards/rejected": -0.003461702261120081,
+      "eval_runtime": 383.5499,
+      "eval_samples_per_second": 11.221,
+      "eval_steps_per_second": 1.403,
+      "step": 2300
+    },
+    {
+      "epoch": 0.3980013783597519,
+      "grad_norm": 2.4941394329071045,
+      "learning_rate": 4.852245211648297e-08,
+      "logits/chosen": -2.9995315074920654,
+      "logits/rejected": -2.983858585357666,
+      "logps/chosen": -52.953948974609375,
+      "logps/rejected": -54.343841552734375,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.01875821128487587,
+      "rewards/margins": 0.01033565029501915,
+      "rewards/rejected": -0.029093865305185318,
+      "step": 2310
+    },
+    {
+      "epoch": 0.39972432804962094,
+      "grad_norm": 2.4663379192352295,
+      "learning_rate": 4.8496883333061044e-08,
+      "logits/chosen": -3.09529447555542,
+      "logits/rejected": -3.0538482666015625,
+      "logps/chosen": -58.8281135559082,
+      "logps/rejected": -54.089324951171875,
+      "loss": 0.6856,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.013867232017219067,
+      "rewards/margins": 0.015552783384919167,
+      "rewards/rejected": -0.02942001447081566,
+      "step": 2320
+    },
+    {
+      "epoch": 0.40144727773949,
+      "grad_norm": 2.278412103652954,
+      "learning_rate": 4.8471102065513926e-08,
+      "logits/chosen": -3.006350040435791,
+      "logits/rejected": -2.9842560291290283,
+      "logps/chosen": -57.26202392578125,
+      "logps/rejected": -55.360809326171875,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.014526404440402985,
+      "rewards/margins": 0.016528811305761337,
+      "rewards/rejected": -0.031055215746164322,
+      "step": 2330
+    },
+    {
+      "epoch": 0.40317022742935904,
+      "grad_norm": 2.3886208534240723,
+      "learning_rate": 4.844510854698359e-08,
+      "logits/chosen": -3.0484840869903564,
+      "logits/rejected": -3.0363705158233643,
+      "logps/chosen": -54.29203414916992,
+      "logps/rejected": -58.19976043701172,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.019904740154743195,
+      "rewards/margins": 0.008316367864608765,
+      "rewards/rejected": -0.02822110988199711,
+      "step": 2340
+    },
+    {
+      "epoch": 0.4048931771192281,
+      "grad_norm": 2.559204578399658,
+      "learning_rate": 4.841890301253144e-08,
+      "logits/chosen": -3.0326600074768066,
+      "logits/rejected": -3.009913206100464,
+      "logps/chosen": -59.752281188964844,
+      "logps/rejected": -58.23968505859375,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.014909838326275349,
+      "rewards/margins": 0.014325137250125408,
+      "rewards/rejected": -0.029234975576400757,
+      "step": 2350
+    },
+    {
+      "epoch": 0.4066161268090972,
+      "grad_norm": 2.2223856449127197,
+      "learning_rate": 4.8392485699136144e-08,
+      "logits/chosen": -3.0519063472747803,
+      "logits/rejected": -3.0387070178985596,
+      "logps/chosen": -52.20775604248047,
+      "logps/rejected": -53.79903030395508,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.018456747755408287,
+      "rewards/margins": 0.008236783556640148,
+      "rewards/rejected": -0.02669353224337101,
+      "step": 2360
+    },
+    {
+      "epoch": 0.4083390764989662,
+      "grad_norm": 2.642369031906128,
+      "learning_rate": 4.836585684569148e-08,
+      "logits/chosen": -3.067197322845459,
+      "logits/rejected": -3.060576915740967,
+      "logps/chosen": -55.74579620361328,
+      "logps/rejected": -56.8653564453125,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.01983823999762535,
+      "rewards/margins": 0.01016821339726448,
+      "rewards/rejected": -0.03000645339488983,
+      "step": 2370
+    },
+    {
+      "epoch": 0.4100620261888353,
+      "grad_norm": 2.277115821838379,
+      "learning_rate": 4.833901669300424e-08,
+      "logits/chosen": -2.9923930168151855,
+      "logits/rejected": -2.967503309249878,
+      "logps/chosen": -55.86818313598633,
+      "logps/rejected": -55.577484130859375,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.015260124579071999,
+      "rewards/margins": 0.011868754401803017,
+      "rewards/rejected": -0.027128878980875015,
+      "step": 2380
+    },
+    {
+      "epoch": 0.41178497587870433,
+      "grad_norm": 2.3972115516662598,
+      "learning_rate": 4.831196548379198e-08,
+      "logits/chosen": -3.0778603553771973,
+      "logits/rejected": -3.0505530834198,
+      "logps/chosen": -57.98725509643555,
+      "logps/rejected": -56.29851531982422,
+      "loss": 0.6862,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.017532609403133392,
+      "rewards/margins": 0.014511296525597572,
+      "rewards/rejected": -0.032043904066085815,
+      "step": 2390
+    },
+    {
+      "epoch": 0.4135079255685734,
+      "grad_norm": 2.2844955921173096,
+      "learning_rate": 4.828470346268089e-08,
+      "logits/chosen": -3.125135898590088,
+      "logits/rejected": -3.1043922901153564,
+      "logps/chosen": -54.28480911254883,
+      "logps/rejected": -56.15095138549805,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.01984257437288761,
+      "rewards/margins": 0.009031950496137142,
+      "rewards/rejected": -0.028874525800347328,
+      "step": 2400
+    },
+    {
+      "epoch": 0.4135079255685734,
+      "eval_logits/chosen": -3.1323635578155518,
+      "eval_logits/rejected": -3.1266722679138184,
+      "eval_logps/chosen": -58.7735595703125,
+      "eval_logps/rejected": -63.654964447021484,
+      "eval_loss": 0.6911728978157043,
+      "eval_rewards/accuracies": 0.578066885471344,
+      "eval_rewards/chosen": -0.0006167090614326298,
+      "eval_rewards/margins": 0.004131661728024483,
+      "eval_rewards/rejected": -0.004748370964080095,
+      "eval_runtime": 383.8141,
+      "eval_samples_per_second": 11.214,
+      "eval_steps_per_second": 1.402,
+      "step": 2400
+    },
+    {
+      "epoch": 0.41523087525844243,
+      "grad_norm": 2.3269927501678467,
+      "learning_rate": 4.825723087620349e-08,
+      "logits/chosen": -3.0994343757629395,
+      "logits/rejected": -3.0597128868103027,
+      "logps/chosen": -55.057212829589844,
+      "logps/rejected": -52.28462600708008,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.01206869538873434,
+      "rewards/margins": 0.018094927072525024,
+      "rewards/rejected": -0.03016362152993679,
+      "step": 2410
+    },
+    {
+      "epoch": 0.4169538249483115,
+      "grad_norm": 2.118802785873413,
+      "learning_rate": 4.822954797279652e-08,
+      "logits/chosen": -3.0960757732391357,
+      "logits/rejected": -3.0733866691589355,
+      "logps/chosen": -57.76322555541992,
+      "logps/rejected": -57.767852783203125,
+      "loss": 0.687,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.015811149030923843,
+      "rewards/margins": 0.012842496857047081,
+      "rewards/rejected": -0.028653645887970924,
+      "step": 2420
+    },
+    {
+      "epoch": 0.41867677463818054,
+      "grad_norm": 2.3563570976257324,
+      "learning_rate": 4.82016550027986e-08,
+      "logits/chosen": -3.0582525730133057,
+      "logits/rejected": -3.0358920097351074,
+      "logps/chosen": -56.497901916503906,
+      "logps/rejected": -54.659690856933594,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.01654912158846855,
+      "rewards/margins": 0.012714418582618237,
+      "rewards/rejected": -0.029263541102409363,
+      "step": 2430
+    },
+    {
+      "epoch": 0.4203997243280496,
+      "grad_norm": 2.5051543712615967,
+      "learning_rate": 4.817355221844801e-08,
+      "logits/chosen": -3.053461790084839,
+      "logits/rejected": -3.0400352478027344,
+      "logps/chosen": -56.637908935546875,
+      "logps/rejected": -57.384132385253906,
+      "loss": 0.687,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.01581026054918766,
+      "rewards/margins": 0.013030016794800758,
+      "rewards/rejected": -0.02884027734398842,
+      "step": 2440
+    },
+    {
+      "epoch": 0.4221226740179187,
+      "grad_norm": 2.4265761375427246,
+      "learning_rate": 4.814523987388038e-08,
+      "logits/chosen": -3.010159730911255,
+      "logits/rejected": -2.9870247840881348,
+      "logps/chosen": -55.32374954223633,
+      "logps/rejected": -55.44095993041992,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.022654101252555847,
+      "rewards/margins": 0.009176109917461872,
+      "rewards/rejected": -0.03183021396398544,
+      "step": 2450
+    },
+    {
+      "epoch": 0.4238456237077877,
+      "grad_norm": 2.45207142829895,
+      "learning_rate": 4.811671822512644e-08,
+      "logits/chosen": -2.9933693408966064,
+      "logits/rejected": -2.960698366165161,
+      "logps/chosen": -58.13505935668945,
+      "logps/rejected": -55.71586227416992,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.019936252385377884,
+      "rewards/margins": 0.011892484501004219,
+      "rewards/rejected": -0.031828735023736954,
+      "step": 2460
+    },
+    {
+      "epoch": 0.4255685733976568,
+      "grad_norm": 2.435755491256714,
+      "learning_rate": 4.808798753010965e-08,
+      "logits/chosen": -3.154000759124756,
+      "logits/rejected": -3.1325736045837402,
+      "logps/chosen": -56.58711624145508,
+      "logps/rejected": -55.6090087890625,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.01542896218597889,
+      "rewards/margins": 0.013578522019088268,
+      "rewards/rejected": -0.029007483273744583,
+      "step": 2470
+    },
+    {
+      "epoch": 0.4272915230875258,
+      "grad_norm": 2.5430045127868652,
+      "learning_rate": 4.805904804864389e-08,
+      "logits/chosen": -3.050870418548584,
+      "logits/rejected": -3.019662380218506,
+      "logps/chosen": -56.7473258972168,
+      "logps/rejected": -53.550865173339844,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.023691657930612564,
+      "rewards/margins": 0.007560255471616983,
+      "rewards/rejected": -0.03125191479921341,
+      "step": 2480
+    },
+    {
+      "epoch": 0.4290144727773949,
+      "grad_norm": 2.2236156463623047,
+      "learning_rate": 4.802990004243112e-08,
+      "logits/chosen": -3.0343146324157715,
+      "logits/rejected": -3.025660276412964,
+      "logps/chosen": -52.4778938293457,
+      "logps/rejected": -54.61433029174805,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0243905670940876,
+      "rewards/margins": 0.005803496576845646,
+      "rewards/rejected": -0.030194062739610672,
+      "step": 2490
+    },
+    {
+      "epoch": 0.43073742246726393,
+      "grad_norm": 2.261009931564331,
+      "learning_rate": 4.800054377505901e-08,
+      "logits/chosen": -3.1122264862060547,
+      "logits/rejected": -3.0881476402282715,
+      "logps/chosen": -59.1455078125,
+      "logps/rejected": -59.477020263671875,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.022548049688339233,
+      "rewards/margins": 0.006525802426040173,
+      "rewards/rejected": -0.02907385304570198,
+      "step": 2500
+    },
+    {
+      "epoch": 0.43073742246726393,
+      "eval_logits/chosen": -3.1292638778686523,
+      "eval_logits/rejected": -3.123566150665283,
+      "eval_logps/chosen": -58.855674743652344,
+      "eval_logps/rejected": -63.77573776245117,
+      "eval_loss": 0.6909918189048767,
+      "eval_rewards/accuracies": 0.578066885471344,
+      "eval_rewards/chosen": -0.0014377871993929148,
+      "eval_rewards/margins": 0.004518397152423859,
+      "eval_rewards/rejected": -0.005956184584647417,
+      "eval_runtime": 383.6212,
+      "eval_samples_per_second": 11.219,
+      "eval_steps_per_second": 1.402,
+      "step": 2500
+    },
+    {
+      "epoch": 0.432460372157133,
+      "grad_norm": 2.4257278442382812,
+      "learning_rate": 4.797097951199854e-08,
+      "logits/chosen": -2.961013078689575,
+      "logits/rejected": -2.9510326385498047,
+      "logps/chosen": -54.5804328918457,
+      "logps/rejected": -55.508544921875,
+      "loss": 0.6867,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.021947655826807022,
+      "rewards/margins": 0.013322239741683006,
+      "rewards/rejected": -0.03526989743113518,
+      "step": 2510
+    },
+    {
+      "epoch": 0.4341833218470021,
+      "grad_norm": 2.391739845275879,
+      "learning_rate": 4.7941207520601625e-08,
+      "logits/chosen": -2.9505152702331543,
+      "logits/rejected": -2.9227962493896484,
+      "logps/chosen": -54.3175048828125,
+      "logps/rejected": -52.03093338012695,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.018492883071303368,
+      "rewards/margins": 0.011797771789133549,
+      "rewards/rejected": -0.030290652066469193,
+      "step": 2520
+    },
+    {
+      "epoch": 0.4359062715368711,
+      "grad_norm": 2.2736146450042725,
+      "learning_rate": 4.791122807009867e-08,
+      "logits/chosen": -3.0387697219848633,
+      "logits/rejected": -3.0342087745666504,
+      "logps/chosen": -53.55059051513672,
+      "logps/rejected": -57.30076217651367,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.02381020598113537,
+      "rewards/margins": 0.007672504987567663,
+      "rewards/rejected": -0.03148271143436432,
+      "step": 2530
+    },
+    {
+      "epoch": 0.4376292212267402,
+      "grad_norm": 2.3488962650299072,
+      "learning_rate": 4.7881041431596156e-08,
+      "logits/chosen": -3.078721284866333,
+      "logits/rejected": -3.067998170852661,
+      "logps/chosen": -56.17926788330078,
+      "logps/rejected": -57.44658660888672,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.026388268917798996,
+      "rewards/margins": 0.009600050747394562,
+      "rewards/rejected": -0.035988323390483856,
+      "step": 2540
+    },
+    {
+      "epoch": 0.4393521709166092,
+      "grad_norm": 2.393836498260498,
+      "learning_rate": 4.7850647878074176e-08,
+      "logits/chosen": -3.0226335525512695,
+      "logits/rejected": -2.990096092224121,
+      "logps/chosen": -60.133636474609375,
+      "logps/rejected": -57.18227005004883,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.014505361206829548,
+      "rewards/margins": 0.01760152354836464,
+      "rewards/rejected": -0.03210688382387161,
+      "step": 2550
+    },
+    {
+      "epoch": 0.4410751206064783,
+      "grad_norm": 2.534972906112671,
+      "learning_rate": 4.782004768438399e-08,
+      "logits/chosen": -3.1110281944274902,
+      "logits/rejected": -3.0864949226379395,
+      "logps/chosen": -56.98291778564453,
+      "logps/rejected": -57.44844436645508,
+      "loss": 0.685,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.01722562685608864,
+      "rewards/margins": 0.017057429999113083,
+      "rewards/rejected": -0.03428305312991142,
+      "step": 2560
+    },
+    {
+      "epoch": 0.4427980702963473,
+      "grad_norm": 2.5366227626800537,
+      "learning_rate": 4.7789241127245484e-08,
+      "logits/chosen": -3.044506311416626,
+      "logits/rejected": -3.0261383056640625,
+      "logps/chosen": -60.37172317504883,
+      "logps/rejected": -59.65507888793945,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.01769999787211418,
+      "rewards/margins": 0.013814535923302174,
+      "rewards/rejected": -0.03151453286409378,
+      "step": 2570
+    },
+    {
+      "epoch": 0.4445210199862164,
+      "grad_norm": 2.656857967376709,
+      "learning_rate": 4.775822848524473e-08,
+      "logits/chosen": -3.0702033042907715,
+      "logits/rejected": -3.0513551235198975,
+      "logps/chosen": -56.100502014160156,
+      "logps/rejected": -56.40464401245117,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.027147063985466957,
+      "rewards/margins": 0.00507187657058239,
+      "rewards/rejected": -0.03221894055604935,
+      "step": 2580
+    },
+    {
+      "epoch": 0.4462439696760855,
+      "grad_norm": 2.7219526767730713,
+      "learning_rate": 4.7727010038831456e-08,
+      "logits/chosen": -3.047976016998291,
+      "logits/rejected": -3.010606050491333,
+      "logps/chosen": -58.72911834716797,
+      "logps/rejected": -55.26195526123047,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.020346758887171745,
+      "rewards/margins": 0.01980571076273918,
+      "rewards/rejected": -0.04015246778726578,
+      "step": 2590
+    },
+    {
+      "epoch": 0.4479669193659545,
+      "grad_norm": 2.5914359092712402,
+      "learning_rate": 4.769558607031646e-08,
+      "logits/chosen": -3.0201575756073,
+      "logits/rejected": -2.9720511436462402,
+      "logps/chosen": -59.4500617980957,
+      "logps/rejected": -52.533180236816406,
+      "loss": 0.685,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.023538867011666298,
+      "rewards/margins": 0.017153343185782433,
+      "rewards/rejected": -0.04069220647215843,
+      "step": 2600
+    },
+    {
+      "epoch": 0.4479669193659545,
+      "eval_logits/chosen": -3.127211332321167,
+      "eval_logits/rejected": -3.1215596199035645,
+      "eval_logps/chosen": -58.99774169921875,
+      "eval_logps/rejected": -63.95884704589844,
+      "eval_loss": 0.6908034682273865,
+      "eval_rewards/accuracies": 0.5824813842773438,
+      "eval_rewards/chosen": -0.0028585141990333796,
+      "eval_rewards/margins": 0.004928695969283581,
+      "eval_rewards/rejected": -0.007787210866808891,
+      "eval_runtime": 383.9054,
+      "eval_samples_per_second": 11.211,
+      "eval_steps_per_second": 1.401,
+      "step": 2600
+    },
+    {
+      "epoch": 0.4496898690558236,
+      "grad_norm": 2.3016531467437744,
+      "learning_rate": 4.766395686386911e-08,
+      "logits/chosen": -2.998870849609375,
+      "logits/rejected": -2.959412097930908,
+      "logps/chosen": -60.63080596923828,
+      "logps/rejected": -55.50718307495117,
+      "loss": 0.6856,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.022517530247569084,
+      "rewards/margins": 0.015829652547836304,
+      "rewards/rejected": -0.03834718465805054,
+      "step": 2610
+    },
+    {
+      "epoch": 0.4514128187456926,
+      "grad_norm": 2.2674720287323,
+      "learning_rate": 4.7632122705514764e-08,
+      "logits/chosen": -3.089473247528076,
+      "logits/rejected": -3.059152126312256,
+      "logps/chosen": -53.82482147216797,
+      "logps/rejected": -54.8328857421875,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.02558869682252407,
+      "rewards/margins": 0.018264109268784523,
+      "rewards/rejected": -0.043852806091308594,
+      "step": 2620
+    },
+    {
+      "epoch": 0.4531357684355617,
+      "grad_norm": 2.6756553649902344,
+      "learning_rate": 4.760008388313216e-08,
+      "logits/chosen": -2.9717869758605957,
+      "logits/rejected": -2.939453601837158,
+      "logps/chosen": -57.078269958496094,
+      "logps/rejected": -57.16656494140625,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.022647377103567123,
+      "rewards/margins": 0.01746378466486931,
+      "rewards/rejected": -0.04011116176843643,
+      "step": 2630
+    },
+    {
+      "epoch": 0.4548587181254307,
+      "grad_norm": 2.1800289154052734,
+      "learning_rate": 4.7567840686450835e-08,
+      "logits/chosen": -2.993943929672241,
+      "logits/rejected": -2.9725537300109863,
+      "logps/chosen": -52.48736572265625,
+      "logps/rejected": -52.89733123779297,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.02391946315765381,
+      "rewards/margins": 0.015178429894149303,
+      "rewards/rejected": -0.039097897708415985,
+      "step": 2640
+    },
+    {
+      "epoch": 0.4565816678152998,
+      "grad_norm": 2.263821840286255,
+      "learning_rate": 4.7535393407048503e-08,
+      "logits/chosen": -2.965701103210449,
+      "logits/rejected": -2.9480319023132324,
+      "logps/chosen": -54.40728759765625,
+      "logps/rejected": -57.77775192260742,
+      "loss": 0.686,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.019764604046940804,
+      "rewards/margins": 0.015169797465205193,
+      "rewards/rejected": -0.0349343977868557,
+      "step": 2650
+    },
+    {
+      "epoch": 0.4583046175051689,
+      "grad_norm": 2.657289743423462,
+      "learning_rate": 4.7502742338348405e-08,
+      "logits/chosen": -2.994309186935425,
+      "logits/rejected": -2.9520974159240723,
+      "logps/chosen": -60.75459671020508,
+      "logps/rejected": -56.639564514160156,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.024647792801260948,
+      "rewards/margins": 0.01700269803404808,
+      "rewards/rejected": -0.041650496423244476,
+      "step": 2660
+    },
+    {
+      "epoch": 0.4600275671950379,
+      "grad_norm": 2.3581361770629883,
+      "learning_rate": 4.746988777561668e-08,
+      "logits/chosen": -2.9663584232330322,
+      "logits/rejected": -2.9378390312194824,
+      "logps/chosen": -56.21480178833008,
+      "logps/rejected": -56.303314208984375,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.022688308730721474,
+      "rewards/margins": 0.01972900703549385,
+      "rewards/rejected": -0.042417313903570175,
+      "step": 2670
+    },
+    {
+      "epoch": 0.461750516884907,
+      "grad_norm": 2.674792766571045,
+      "learning_rate": 4.743683001595965e-08,
+      "logits/chosen": -3.012934923171997,
+      "logits/rejected": -2.9855589866638184,
+      "logps/chosen": -60.850120544433594,
+      "logps/rejected": -57.48209762573242,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.026025820523500443,
+      "rewards/margins": 0.012167487293481827,
+      "rewards/rejected": -0.03819330781698227,
+      "step": 2680
+    },
+    {
+      "epoch": 0.463473466574776,
+      "grad_norm": 2.40645432472229,
+      "learning_rate": 4.7403569358321205e-08,
+      "logits/chosen": -2.98614764213562,
+      "logits/rejected": -2.963613986968994,
+      "logps/chosen": -53.42826461791992,
+      "logps/rejected": -54.72166061401367,
+      "loss": 0.6833,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.027871781960129738,
+      "rewards/margins": 0.020479358732700348,
+      "rewards/rejected": -0.048351138830184937,
+      "step": 2690
+    },
+    {
+      "epoch": 0.4651964162646451,
+      "grad_norm": 2.3990304470062256,
+      "learning_rate": 4.737010610348001e-08,
+      "logits/chosen": -3.014841318130493,
+      "logits/rejected": -2.9882454872131348,
+      "logps/chosen": -55.58037185668945,
+      "logps/rejected": -57.473899841308594,
+      "loss": 0.6852,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.023496050387620926,
+      "rewards/margins": 0.016478871926665306,
+      "rewards/rejected": -0.03997492790222168,
+      "step": 2700
+    },
+    {
+      "epoch": 0.4651964162646451,
+      "eval_logits/chosen": -3.1246092319488525,
+      "eval_logits/rejected": -3.118943452835083,
+      "eval_logps/chosen": -59.19208526611328,
+      "eval_logps/rejected": -64.2020492553711,
+      "eval_loss": 0.6905856728553772,
+      "eval_rewards/accuracies": 0.5834107995033264,
+      "eval_rewards/chosen": -0.004801863804459572,
+      "eval_rewards/margins": 0.005417466629296541,
+      "eval_rewards/rejected": -0.010219329036772251,
+      "eval_runtime": 383.5481,
+      "eval_samples_per_second": 11.222,
+      "eval_steps_per_second": 1.403,
+      "step": 2700
+    },
+    {
+      "epoch": 0.4669193659545141,
+      "grad_norm": 2.6741602420806885,
+      "learning_rate": 4.733644055404687e-08,
+      "logits/chosen": -3.0382771492004395,
+      "logits/rejected": -3.0192952156066895,
+      "logps/chosen": -61.61924362182617,
+      "logps/rejected": -58.47269821166992,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.02172602340579033,
+      "rewards/margins": 0.009493627585470676,
+      "rewards/rejected": -0.03121965005993843,
+      "step": 2710
+    },
+    {
+      "epoch": 0.4686423156443832,
+      "grad_norm": 2.8766732215881348,
+      "learning_rate": 4.730257301446193e-08,
+      "logits/chosen": -3.0275590419769287,
+      "logits/rejected": -3.030806064605713,
+      "logps/chosen": -55.16575241088867,
+      "logps/rejected": -57.712371826171875,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.031185084953904152,
+      "rewards/margins": 0.008310927078127861,
+      "rewards/rejected": -0.03949601203203201,
+      "step": 2720
+    },
+    {
+      "epoch": 0.4703652653342522,
+      "grad_norm": 2.3278653621673584,
+      "learning_rate": 4.726850379099198e-08,
+      "logits/chosen": -3.0274832248687744,
+      "logits/rejected": -3.0075907707214355,
+      "logps/chosen": -60.72504806518555,
+      "logps/rejected": -58.92608642578125,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.021583404392004013,
+      "rewards/margins": 0.015425076708197594,
+      "rewards/rejected": -0.037008482962846756,
+      "step": 2730
+    },
+    {
+      "epoch": 0.4720882150241213,
+      "grad_norm": 2.5836517810821533,
+      "learning_rate": 4.7234233191727604e-08,
+      "logits/chosen": -3.0168375968933105,
+      "logits/rejected": -2.995499849319458,
+      "logps/chosen": -55.875572204589844,
+      "logps/rejected": -53.20180130004883,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.03016316331923008,
+      "rewards/margins": 0.008027995005249977,
+      "rewards/rejected": -0.03819115832448006,
+      "step": 2740
+    },
+    {
+      "epoch": 0.4738111647139904,
+      "grad_norm": 2.2258636951446533,
+      "learning_rate": 4.7199761526580484e-08,
+      "logits/chosen": -2.9475860595703125,
+      "logits/rejected": -2.942593574523926,
+      "logps/chosen": -52.258270263671875,
+      "logps/rejected": -57.7049446105957,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.029945258051156998,
+      "rewards/margins": 0.012834271416068077,
+      "rewards/rejected": -0.042779527604579926,
+      "step": 2750
+    },
+    {
+      "epoch": 0.4755341144038594,
+      "grad_norm": 2.5605227947235107,
+      "learning_rate": 4.716508910728054e-08,
+      "logits/chosen": -2.9973998069763184,
+      "logits/rejected": -2.9798712730407715,
+      "logps/chosen": -57.110862731933594,
+      "logps/rejected": -59.198211669921875,
+      "loss": 0.6857,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.028358791023492813,
+      "rewards/margins": 0.015565874986350536,
+      "rewards/rejected": -0.043924666941165924,
+      "step": 2760
+    },
+    {
+      "epoch": 0.4772570640937285,
+      "grad_norm": 2.7382118701934814,
+      "learning_rate": 4.713021624737312e-08,
+      "logits/chosen": -3.1292343139648438,
+      "logits/rejected": -3.09916615486145,
+      "logps/chosen": -57.5919303894043,
+      "logps/rejected": -56.7816276550293,
+      "loss": 0.6867,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.023771721869707108,
+      "rewards/margins": 0.013658533804118633,
+      "rewards/rejected": -0.037430256605148315,
+      "step": 2770
+    },
+    {
+      "epoch": 0.4789800137835975,
+      "grad_norm": 2.5300447940826416,
+      "learning_rate": 4.70951432622162e-08,
+      "logits/chosen": -2.947597026824951,
+      "logits/rejected": -2.917452573776245,
+      "logps/chosen": -55.9733772277832,
+      "logps/rejected": -54.83564376831055,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.03144543617963791,
+      "rewards/margins": 0.010759610682725906,
+      "rewards/rejected": -0.042205046862363815,
+      "step": 2780
+    },
+    {
+      "epoch": 0.4807029634734666,
+      "grad_norm": 2.6612281799316406,
+      "learning_rate": 4.7059870468977484e-08,
+      "logits/chosen": -3.0657873153686523,
+      "logits/rejected": -3.0320510864257812,
+      "logps/chosen": -54.16001510620117,
+      "logps/rejected": -54.211158752441406,
+      "loss": 0.6849,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.02399759367108345,
+      "rewards/margins": 0.016985418274998665,
+      "rewards/rejected": -0.040983010083436966,
+      "step": 2790
+    },
+    {
+      "epoch": 0.4824259131633356,
+      "grad_norm": 2.8492729663848877,
+      "learning_rate": 4.702439818663153e-08,
+      "logits/chosen": -3.046370267868042,
+      "logits/rejected": -3.021303653717041,
+      "logps/chosen": -62.86781692504883,
+      "logps/rejected": -61.252357482910156,
+      "loss": 0.6857,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.02832144871354103,
+      "rewards/margins": 0.015826255083084106,
+      "rewards/rejected": -0.04414770007133484,
+      "step": 2800
+    },
+    {
+      "epoch": 0.4824259131633356,
+      "eval_logits/chosen": -3.1211092472076416,
+      "eval_logits/rejected": -3.1154263019561768,
+      "eval_logps/chosen": -59.331790924072266,
+      "eval_logps/rejected": -64.37612915039062,
+      "eval_loss": 0.6904300451278687,
+      "eval_rewards/accuracies": 0.5859665274620056,
+      "eval_rewards/chosen": -0.006198993884027004,
+      "eval_rewards/margins": 0.005761031527072191,
+      "eval_rewards/rejected": -0.011960024945437908,
+      "eval_runtime": 383.3187,
+      "eval_samples_per_second": 11.228,
+      "eval_steps_per_second": 1.404,
+      "step": 2800
+    },
+    {
+      "epoch": 0.4841488628532047,
+      "grad_norm": 2.344043254852295,
+      "learning_rate": 4.6988726735956954e-08,
+      "logits/chosen": -2.936617374420166,
+      "logits/rejected": -2.912167549133301,
+      "logps/chosen": -58.06193161010742,
+      "logps/rejected": -60.141136169433594,
+      "loss": 0.6853,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.027261238545179367,
+      "rewards/margins": 0.016514476388692856,
+      "rewards/rejected": -0.04377571493387222,
+      "step": 2810
+    },
+    {
+      "epoch": 0.48587181254307377,
+      "grad_norm": 2.7455015182495117,
+      "learning_rate": 4.69528564395334e-08,
+      "logits/chosen": -3.0858654975891113,
+      "logits/rejected": -3.0735526084899902,
+      "logps/chosen": -57.2254753112793,
+      "logps/rejected": -57.30078125,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.02898283302783966,
+      "rewards/margins": 0.013826819136738777,
+      "rewards/rejected": -0.04280964657664299,
+      "step": 2820
+    },
+    {
+      "epoch": 0.4875947622329428,
+      "grad_norm": 2.3750805854797363,
+      "learning_rate": 4.691678762173874e-08,
+      "logits/chosen": -2.887464761734009,
+      "logits/rejected": -2.8721446990966797,
+      "logps/chosen": -58.10028076171875,
+      "logps/rejected": -59.103538513183594,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.025997614488005638,
+      "rewards/margins": 0.015392879955470562,
+      "rewards/rejected": -0.04139048978686333,
+      "step": 2830
+    },
+    {
+      "epoch": 0.48931771192281187,
+      "grad_norm": 2.3693149089813232,
+      "learning_rate": 4.688052060874606e-08,
+      "logits/chosen": -3.061169385910034,
+      "logits/rejected": -3.049863815307617,
+      "logps/chosen": -55.8497200012207,
+      "logps/rejected": -56.280967712402344,
+      "loss": 0.6852,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.032134395092725754,
+      "rewards/margins": 0.016874976456165314,
+      "rewards/rejected": -0.049009375274181366,
+      "step": 2840
+    },
+    {
+      "epoch": 0.4910406616126809,
+      "grad_norm": 2.6595659255981445,
+      "learning_rate": 4.684405572852077e-08,
+      "logits/chosen": -3.022156238555908,
+      "logits/rejected": -3.002824068069458,
+      "logps/chosen": -54.9843864440918,
+      "logps/rejected": -60.23651123046875,
+      "loss": 0.6804,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.027379611507058144,
+      "rewards/margins": 0.026498600840568542,
+      "rewards/rejected": -0.05387821048498154,
+      "step": 2850
+    },
+    {
+      "epoch": 0.49276361130255,
+      "grad_norm": 2.1085870265960693,
+      "learning_rate": 4.6807393310817575e-08,
+      "logits/chosen": -3.0010194778442383,
+      "logits/rejected": -2.976646661758423,
+      "logps/chosen": -57.82773971557617,
+      "logps/rejected": -56.6772575378418,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.028632182627916336,
+      "rewards/margins": 0.01836460642516613,
+      "rewards/rejected": -0.046996790915727615,
+      "step": 2860
+    },
+    {
+      "epoch": 0.494486560992419,
+      "grad_norm": 2.4450454711914062,
+      "learning_rate": 4.677053368717754e-08,
+      "logits/chosen": -3.042559862136841,
+      "logits/rejected": -3.01857590675354,
+      "logps/chosen": -58.71149826049805,
+      "logps/rejected": -57.6026725769043,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.025957271456718445,
+      "rewards/margins": 0.019699634984135628,
+      "rewards/rejected": -0.04565690457820892,
+      "step": 2870
+    },
+    {
+      "epoch": 0.4962095106822881,
+      "grad_norm": 2.803981065750122,
+      "learning_rate": 4.673347719092507e-08,
+      "logits/chosen": -3.1182825565338135,
+      "logits/rejected": -3.091951847076416,
+      "logps/chosen": -58.57697677612305,
+      "logps/rejected": -55.82576370239258,
+      "loss": 0.6853,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.0322895422577858,
+      "rewards/margins": 0.016705023124814034,
+      "rewards/rejected": -0.04899457469582558,
+      "step": 2880
+    },
+    {
+      "epoch": 0.49793246037215716,
+      "grad_norm": 2.348870038986206,
+      "learning_rate": 4.669622415716494e-08,
+      "logits/chosen": -3.1272709369659424,
+      "logits/rejected": -3.121854305267334,
+      "logps/chosen": -56.19138717651367,
+      "logps/rejected": -57.14445114135742,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.032336968928575516,
+      "rewards/margins": 0.013913976028561592,
+      "rewards/rejected": -0.04625094681978226,
+      "step": 2890
+    },
+    {
+      "epoch": 0.4996554100620262,
+      "grad_norm": 2.56561017036438,
+      "learning_rate": 4.665877492277919e-08,
+      "logits/chosen": -3.045564651489258,
+      "logits/rejected": -3.0358288288116455,
+      "logps/chosen": -55.23905563354492,
+      "logps/rejected": -55.39752197265625,
+      "loss": 0.688,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.027509957551956177,
+      "rewards/margins": 0.011119430884718895,
+      "rewards/rejected": -0.03862939029932022,
+      "step": 2900
+    },
+    {
+      "epoch": 0.4996554100620262,
+      "eval_logits/chosen": -3.1176185607910156,
+      "eval_logits/rejected": -3.111891269683838,
+      "eval_logps/chosen": -59.58071517944336,
+      "eval_logps/rejected": -64.67279815673828,
+      "eval_loss": 0.6902225017547607,
+      "eval_rewards/accuracies": 0.5861988663673401,
+      "eval_rewards/chosen": -0.008688190951943398,
+      "eval_rewards/margins": 0.006238551810383797,
+      "eval_rewards/rejected": -0.014926743693649769,
+      "eval_runtime": 383.252,
+      "eval_samples_per_second": 11.23,
+      "eval_steps_per_second": 1.404,
+      "step": 2900
+    },
+    {
+      "epoch": 0.5013783597518953,
+      "grad_norm": 3.1003239154815674,
+      "learning_rate": 4.6621129826424115e-08,
+      "logits/chosen": -3.0847606658935547,
+      "logits/rejected": -3.0727763175964355,
+      "logps/chosen": -58.9963493347168,
+      "logps/rejected": -60.630828857421875,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.028891712427139282,
+      "rewards/margins": 0.018411526456475258,
+      "rewards/rejected": -0.04730324074625969,
+      "step": 2910
+    },
+    {
+      "epoch": 0.5031013094417643,
+      "grad_norm": 2.446190118789673,
+      "learning_rate": 4.6583289208527247e-08,
+      "logits/chosen": -3.0294718742370605,
+      "logits/rejected": -3.025437831878662,
+      "logps/chosen": -56.81879806518555,
+      "logps/rejected": -60.310523986816406,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.032486725598573685,
+      "rewards/margins": 0.015557927079498768,
+      "rewards/rejected": -0.04804465174674988,
+      "step": 2920
+    },
+    {
+      "epoch": 0.5048242591316333,
+      "grad_norm": 2.5872488021850586,
+      "learning_rate": 4.654525341128418e-08,
+      "logits/chosen": -3.0019631385803223,
+      "logits/rejected": -2.974017381668091,
+      "logps/chosen": -56.43025588989258,
+      "logps/rejected": -56.761260986328125,
+      "loss": 0.6831,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.02979024313390255,
+      "rewards/margins": 0.02114255167543888,
+      "rewards/rejected": -0.05093279480934143,
+      "step": 2930
+    },
+    {
+      "epoch": 0.5065472088215024,
+      "grad_norm": 2.268840789794922,
+      "learning_rate": 4.650702277865558e-08,
+      "logits/chosen": -3.032402515411377,
+      "logits/rejected": -2.995234489440918,
+      "logps/chosen": -54.46013259887695,
+      "logps/rejected": -55.5176887512207,
+      "loss": 0.685,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.029997989535331726,
+      "rewards/margins": 0.017327243462204933,
+      "rewards/rejected": -0.04732522368431091,
+      "step": 2940
+    },
+    {
+      "epoch": 0.5082701585113715,
+      "grad_norm": 2.260488986968994,
+      "learning_rate": 4.6468597656363996e-08,
+      "logits/chosen": -3.0672240257263184,
+      "logits/rejected": -3.048755168914795,
+      "logps/chosen": -57.317970275878906,
+      "logps/rejected": -59.37456512451172,
+      "loss": 0.6845,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.0299685001373291,
+      "rewards/margins": 0.018443303182721138,
+      "rewards/rejected": -0.04841180518269539,
+      "step": 2950
+    },
+    {
+      "epoch": 0.5099931082012406,
+      "grad_norm": 2.6867690086364746,
+      "learning_rate": 4.642997839189076e-08,
+      "logits/chosen": -3.0259053707122803,
+      "logits/rejected": -2.9986701011657715,
+      "logps/chosen": -57.10251998901367,
+      "logps/rejected": -57.13700485229492,
+      "loss": 0.6832,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.031545598059892654,
+      "rewards/margins": 0.0208672396838665,
+      "rewards/rejected": -0.05241284519433975,
+      "step": 2960
+    },
+    {
+      "epoch": 0.5117160578911096,
+      "grad_norm": 3.036938428878784,
+      "learning_rate": 4.639116533447286e-08,
+      "logits/chosen": -2.9585442543029785,
+      "logits/rejected": -2.930492401123047,
+      "logps/chosen": -58.95167922973633,
+      "logps/rejected": -57.681495666503906,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.03714163228869438,
+      "rewards/margins": 0.01553442794829607,
+      "rewards/rejected": -0.05267605930566788,
+      "step": 2970
+    },
+    {
+      "epoch": 0.5134390075809786,
+      "grad_norm": 2.597028970718384,
+      "learning_rate": 4.6352158835099756e-08,
+      "logits/chosen": -2.955645799636841,
+      "logits/rejected": -2.929425001144409,
+      "logps/chosen": -56.53186798095703,
+      "logps/rejected": -57.37433624267578,
+      "loss": 0.6812,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.025926580652594566,
+      "rewards/margins": 0.025008264929056168,
+      "rewards/rejected": -0.050934843719005585,
+      "step": 2980
+    },
+    {
+      "epoch": 0.5151619572708477,
+      "grad_norm": 2.4624998569488525,
+      "learning_rate": 4.6312959246510234e-08,
+      "logits/chosen": -3.0022027492523193,
+      "logits/rejected": -2.9742767810821533,
+      "logps/chosen": -59.90388107299805,
+      "logps/rejected": -59.825225830078125,
+      "loss": 0.6841,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.032940447330474854,
+      "rewards/margins": 0.019164832308888435,
+      "rewards/rejected": -0.05210528522729874,
+      "step": 2990
+    },
+    {
+      "epoch": 0.5168849069607168,
+      "grad_norm": 2.176590919494629,
+      "learning_rate": 4.627356692318919e-08,
+      "logits/chosen": -3.0138776302337646,
+      "logits/rejected": -3.0080418586730957,
+      "logps/chosen": -55.33563232421875,
+      "logps/rejected": -58.42255401611328,
+      "loss": 0.6877,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.03323281183838844,
+      "rewards/margins": 0.012048874981701374,
+      "rewards/rejected": -0.04528168588876724,
+      "step": 3000
+    },
+    {
+      "epoch": 0.5168849069607168,
+      "eval_logits/chosen": -3.114572048187256,
+      "eval_logits/rejected": -3.108858346939087,
+      "eval_logps/chosen": -59.85063934326172,
+      "eval_logps/rejected": -64.97742462158203,
+      "eval_loss": 0.6900810599327087,
+      "eval_rewards/accuracies": 0.5794609785079956,
+      "eval_rewards/chosen": -0.0113874152302742,
+      "eval_rewards/margins": 0.006585672497749329,
+      "eval_rewards/rejected": -0.01797308772802353,
+      "eval_runtime": 383.6668,
+      "eval_samples_per_second": 11.218,
+      "eval_steps_per_second": 1.402,
+      "step": 3000
+    },
+    {
+      "epoch": 0.5186078566505858,
+      "grad_norm": 2.7165989875793457,
+      "learning_rate": 4.6233982221364434e-08,
+      "logits/chosen": -3.03937029838562,
+      "logits/rejected": -3.0160698890686035,
+      "logps/chosen": -58.7923698425293,
+      "logps/rejected": -57.86932373046875,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.03604704886674881,
+      "rewards/margins": 0.01170069444924593,
+      "rewards/rejected": -0.047747738659381866,
+      "step": 3010
+    },
+    {
+      "epoch": 0.5203308063404548,
+      "grad_norm": 2.395535707473755,
+      "learning_rate": 4.6194205499003467e-08,
+      "logits/chosen": -3.1249048709869385,
+      "logits/rejected": -3.093398332595825,
+      "logps/chosen": -58.301673889160156,
+      "logps/rejected": -56.125648498535156,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.04063145071268082,
+      "rewards/margins": 0.017468957230448723,
+      "rewards/rejected": -0.05810040980577469,
+      "step": 3020
+    },
+    {
+      "epoch": 0.5220537560303239,
+      "grad_norm": 2.491513729095459,
+      "learning_rate": 4.6154237115810266e-08,
+      "logits/chosen": -3.0263257026672363,
+      "logits/rejected": -3.0045909881591797,
+      "logps/chosen": -59.91956329345703,
+      "logps/rejected": -59.80186080932617,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.04244648665189743,
+      "rewards/margins": 0.006677904166281223,
+      "rewards/rejected": -0.04912438988685608,
+      "step": 3030
+    },
+    {
+      "epoch": 0.523776705720193,
+      "grad_norm": 2.2903382778167725,
+      "learning_rate": 4.6114077433221995e-08,
+      "logits/chosen": -3.130286693572998,
+      "logits/rejected": -3.123812675476074,
+      "logps/chosen": -55.290557861328125,
+      "logps/rejected": -59.867210388183594,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.041027769446372986,
+      "rewards/margins": 0.01448782254010439,
+      "rewards/rejected": -0.0555155873298645,
+      "step": 3040
+    },
+    {
+      "epoch": 0.525499655410062,
+      "grad_norm": 2.178276777267456,
+      "learning_rate": 4.6073726814405746e-08,
+      "logits/chosen": -2.9891786575317383,
+      "logits/rejected": -2.9757232666015625,
+      "logps/chosen": -57.28485870361328,
+      "logps/rejected": -60.19240188598633,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.037267621606588364,
+      "rewards/margins": 0.011081216856837273,
+      "rewards/rejected": -0.04834883660078049,
+      "step": 3050
+    },
+    {
+      "epoch": 0.5272226050999311,
+      "grad_norm": 2.7194161415100098,
+      "learning_rate": 4.603318562425528e-08,
+      "logits/chosen": -3.0249791145324707,
+      "logits/rejected": -3.0064873695373535,
+      "logps/chosen": -59.319496154785156,
+      "logps/rejected": -58.89420700073242,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.03643704578280449,
+      "rewards/margins": 0.008830266073346138,
+      "rewards/rejected": -0.045267317444086075,
+      "step": 3060
+    },
+    {
+      "epoch": 0.5289455547898001,
+      "grad_norm": 2.5902607440948486,
+      "learning_rate": 4.59924542293877e-08,
+      "logits/chosen": -2.9844601154327393,
+      "logits/rejected": -2.9587693214416504,
+      "logps/chosen": -62.00069046020508,
+      "logps/rejected": -59.24231719970703,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.04058091714978218,
+      "rewards/margins": 0.012460616417229176,
+      "rewards/rejected": -0.05304152891039848,
+      "step": 3070
+    },
+    {
+      "epoch": 0.5306685044796692,
+      "grad_norm": 2.2749767303466797,
+      "learning_rate": 4.5951532998140134e-08,
+      "logits/chosen": -2.89788556098938,
+      "logits/rejected": -2.8808937072753906,
+      "logps/chosen": -55.66410446166992,
+      "logps/rejected": -60.67927932739258,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.04805658757686615,
+      "rewards/margins": 0.010724262334406376,
+      "rewards/rejected": -0.05878084897994995,
+      "step": 3080
+    },
+    {
+      "epoch": 0.5323914541695383,
+      "grad_norm": 2.4029645919799805,
+      "learning_rate": 4.591042230056644e-08,
+      "logits/chosen": -2.982564926147461,
+      "logits/rejected": -2.963219404220581,
+      "logps/chosen": -57.242591857910156,
+      "logps/rejected": -59.30971145629883,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.036305077373981476,
+      "rewards/margins": 0.01974949985742569,
+      "rewards/rejected": -0.056054580956697464,
+      "step": 3090
+    },
+    {
+      "epoch": 0.5341144038594073,
+      "grad_norm": 2.806472063064575,
+      "learning_rate": 4.5869122508433834e-08,
+      "logits/chosen": -3.0192148685455322,
+      "logits/rejected": -2.9897754192352295,
+      "logps/chosen": -57.5784912109375,
+      "logps/rejected": -57.04255294799805,
+      "loss": 0.6846,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.03658083826303482,
+      "rewards/margins": 0.018136832863092422,
+      "rewards/rejected": -0.05471767112612724,
+      "step": 3100
+    },
+    {
+      "epoch": 0.5341144038594073,
+      "eval_logits/chosen": -3.112844705581665,
+      "eval_logits/rejected": -3.1071627140045166,
+      "eval_logps/chosen": -59.9371452331543,
+      "eval_logps/rejected": -65.10148620605469,
+      "eval_loss": 0.6899077296257019,
+      "eval_rewards/accuracies": 0.5822490453720093,
+      "eval_rewards/chosen": -0.012252441607415676,
+      "eval_rewards/margins": 0.006961132865399122,
+      "eval_rewards/rejected": -0.01921357400715351,
+      "eval_runtime": 383.1301,
+      "eval_samples_per_second": 11.234,
+      "eval_steps_per_second": 1.404,
+      "step": 3100
+    },
+    {
+      "epoch": 0.5358373535492763,
+      "grad_norm": 2.829054832458496,
+      "learning_rate": 4.5827633995219486e-08,
+      "logits/chosen": -2.9593725204467773,
+      "logits/rejected": -2.969134569168091,
+      "logps/chosen": -55.128814697265625,
+      "logps/rejected": -60.54915237426758,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.04516337066888809,
+      "rewards/margins": 0.010849391110241413,
+      "rewards/rejected": -0.05601276084780693,
+      "step": 3110
+    },
+    {
+      "epoch": 0.5375603032391454,
+      "grad_norm": 2.5551376342773438,
+      "learning_rate": 4.5785957136107236e-08,
+      "logits/chosen": -3.007732629776001,
+      "logits/rejected": -2.9836857318878174,
+      "logps/chosen": -59.59978103637695,
+      "logps/rejected": -61.34168243408203,
+      "loss": 0.684,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.035889994353055954,
+      "rewards/margins": 0.01947486773133278,
+      "rewards/rejected": -0.055364858359098434,
+      "step": 3120
+    },
+    {
+      "epoch": 0.5392832529290145,
+      "grad_norm": 2.3199799060821533,
+      "learning_rate": 4.574409230798413e-08,
+      "logits/chosen": -2.924534320831299,
+      "logits/rejected": -2.9163551330566406,
+      "logps/chosen": -55.8783073425293,
+      "logps/rejected": -58.179161071777344,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.03797352313995361,
+      "rewards/margins": 0.01041981391608715,
+      "rewards/rejected": -0.048393331468105316,
+      "step": 3130
+    },
+    {
+      "epoch": 0.5410062026188835,
+      "grad_norm": 2.401380777359009,
+      "learning_rate": 4.5702039889437015e-08,
+      "logits/chosen": -3.0412960052490234,
+      "logits/rejected": -3.025895118713379,
+      "logps/chosen": -55.72760009765625,
+      "logps/rejected": -59.47303009033203,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.04030473902821541,
+      "rewards/margins": 0.017010238021612167,
+      "rewards/rejected": -0.05731497332453728,
+      "step": 3140
+    },
+    {
+      "epoch": 0.5427291523087526,
+      "grad_norm": 2.482997179031372,
+      "learning_rate": 4.565980026074917e-08,
+      "logits/chosen": -3.0422937870025635,
+      "logits/rejected": -3.013444423675537,
+      "logps/chosen": -57.84309768676758,
+      "logps/rejected": -59.41813278198242,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.03474130108952522,
+      "rewards/margins": 0.0234179999679327,
+      "rewards/rejected": -0.058159299194812775,
+      "step": 3150
+    },
+    {
+      "epoch": 0.5444521019986216,
+      "grad_norm": 2.523099660873413,
+      "learning_rate": 4.5617373803896796e-08,
+      "logits/chosen": -2.9821133613586426,
+      "logits/rejected": -2.952793598175049,
+      "logps/chosen": -57.093482971191406,
+      "logps/rejected": -58.33715057373047,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.029471775516867638,
+      "rewards/margins": 0.027565976604819298,
+      "rewards/rejected": -0.05703775957226753,
+      "step": 3160
+    },
+    {
+      "epoch": 0.5461750516884907,
+      "grad_norm": 2.708991527557373,
+      "learning_rate": 4.557476090254562e-08,
+      "logits/chosen": -3.0086758136749268,
+      "logits/rejected": -2.975803852081299,
+      "logps/chosen": -59.65704345703125,
+      "logps/rejected": -55.933998107910156,
+      "loss": 0.6803,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.037676259875297546,
+      "rewards/margins": 0.026803482323884964,
+      "rewards/rejected": -0.06447974592447281,
+      "step": 3170
+    },
+    {
+      "epoch": 0.5478980013783598,
+      "grad_norm": 2.3031082153320312,
+      "learning_rate": 4.5531961942047384e-08,
+      "logits/chosen": -3.060319662094116,
+      "logits/rejected": -3.0228328704833984,
+      "logps/chosen": -57.08624267578125,
+      "logps/rejected": -57.317832946777344,
+      "loss": 0.6833,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.03829856216907501,
+      "rewards/margins": 0.020599478855729103,
+      "rewards/rejected": -0.05889805033802986,
+      "step": 3180
+    },
+    {
+      "epoch": 0.5496209510682288,
+      "grad_norm": 2.529620885848999,
+      "learning_rate": 4.548897730943638e-08,
+      "logits/chosen": -3.0590758323669434,
+      "logits/rejected": -3.053528308868408,
+      "logps/chosen": -56.3664436340332,
+      "logps/rejected": -63.008079528808594,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.04068754240870476,
+      "rewards/margins": 0.02274017035961151,
+      "rewards/rejected": -0.06342772394418716,
+      "step": 3190
+    },
+    {
+      "epoch": 0.5513439007580979,
+      "grad_norm": 3.0161774158477783,
+      "learning_rate": 4.544580739342596e-08,
+      "logits/chosen": -2.983168601989746,
+      "logits/rejected": -2.968716859817505,
+      "logps/chosen": -58.88862991333008,
+      "logps/rejected": -57.30841827392578,
+      "loss": 0.6856,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.03559957817196846,
+      "rewards/margins": 0.016162436455488205,
+      "rewards/rejected": -0.051762014627456665,
+      "step": 3200
+    },
+    {
+      "epoch": 0.5513439007580979,
+      "eval_logits/chosen": -3.1091630458831787,
+      "eval_logits/rejected": -3.103480577468872,
+      "eval_logps/chosen": -60.252586364746094,
+      "eval_logps/rejected": -65.4752197265625,
+      "eval_loss": 0.6896567940711975,
+      "eval_rewards/accuracies": 0.5822490453720093,
+      "eval_rewards/chosen": -0.015406901016831398,
+      "eval_rewards/margins": 0.007544039748609066,
+      "eval_rewards/rejected": -0.02295094169676304,
+      "eval_runtime": 383.1558,
+      "eval_samples_per_second": 11.233,
+      "eval_steps_per_second": 1.404,
+      "step": 3200
+    },
+    {
+      "epoch": 0.5530668504479669,
+      "grad_norm": 2.809147596359253,
+      "learning_rate": 4.540245258440499e-08,
+      "logits/chosen": -2.900620698928833,
+      "logits/rejected": -2.876187324523926,
+      "logps/chosen": -57.85747146606445,
+      "logps/rejected": -58.46887969970703,
+      "loss": 0.6835,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.03906077519059181,
+      "rewards/margins": 0.020655179396271706,
+      "rewards/rejected": -0.05971595644950867,
+      "step": 3210
+    },
+    {
+      "epoch": 0.554789800137836,
+      "grad_norm": 2.3990120887756348,
+      "learning_rate": 4.535891327443435e-08,
+      "logits/chosen": -2.9358742237091064,
+      "logits/rejected": -2.930753707885742,
+      "logps/chosen": -54.84906005859375,
+      "logps/rejected": -58.648719787597656,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.045052286237478256,
+      "rewards/margins": 0.01260636281222105,
+      "rewards/rejected": -0.05765865370631218,
+      "step": 3220
+    },
+    {
+      "epoch": 0.556512749827705,
+      "grad_norm": 2.3733887672424316,
+      "learning_rate": 4.531518985724338e-08,
+      "logits/chosen": -2.990746259689331,
+      "logits/rejected": -2.9742918014526367,
+      "logps/chosen": -58.27122116088867,
+      "logps/rejected": -61.59682083129883,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.0411384217441082,
+      "rewards/margins": 0.011042706668376923,
+      "rewards/rejected": -0.05218113213777542,
+      "step": 3230
+    },
+    {
+      "epoch": 0.5582356995175741,
+      "grad_norm": 2.4337801933288574,
+      "learning_rate": 4.527128272822629e-08,
+      "logits/chosen": -3.1114020347595215,
+      "logits/rejected": -3.0879967212677,
+      "logps/chosen": -60.933677673339844,
+      "logps/rejected": -59.181678771972656,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.04697013646364212,
+      "rewards/margins": 0.019410159438848495,
+      "rewards/rejected": -0.06638029962778091,
+      "step": 3240
+    },
+    {
+      "epoch": 0.5599586492074431,
+      "grad_norm": 2.404066562652588,
+      "learning_rate": 4.5227192284438634e-08,
+      "logits/chosen": -3.005336046218872,
+      "logits/rejected": -2.9841012954711914,
+      "logps/chosen": -56.500282287597656,
+      "logps/rejected": -59.909324645996094,
+      "loss": 0.6803,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.03927716612815857,
+      "rewards/margins": 0.026730556041002274,
+      "rewards/rejected": -0.06600771844387054,
+      "step": 3250
+    },
+    {
+      "epoch": 0.5616815988973122,
+      "grad_norm": 2.671459674835205,
+      "learning_rate": 4.5182918924593705e-08,
+      "logits/chosen": -3.017117977142334,
+      "logits/rejected": -2.999938488006592,
+      "logps/chosen": -56.0377197265625,
+      "logps/rejected": -59.20392608642578,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.04502294957637787,
+      "rewards/margins": 0.01962658017873764,
+      "rewards/rejected": -0.06464952975511551,
+      "step": 3260
+    },
+    {
+      "epoch": 0.5634045485871813,
+      "grad_norm": 2.684394359588623,
+      "learning_rate": 4.5138463049058887e-08,
+      "logits/chosen": -3.0742757320404053,
+      "logits/rejected": -3.070272922515869,
+      "logps/chosen": -60.09043502807617,
+      "logps/rejected": -64.94206237792969,
+      "loss": 0.6834,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.04960837960243225,
+      "rewards/margins": 0.020932510495185852,
+      "rewards/rejected": -0.0705408900976181,
+      "step": 3270
+    },
+    {
+      "epoch": 0.5651274982770503,
+      "grad_norm": 2.6779043674468994,
+      "learning_rate": 4.5093825059852096e-08,
+      "logits/chosen": -3.0797085762023926,
+      "logits/rejected": -3.068922519683838,
+      "logps/chosen": -54.30656051635742,
+      "logps/rejected": -58.85213088989258,
+      "loss": 0.686,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.04885853826999664,
+      "rewards/margins": 0.015684762969613075,
+      "rewards/rejected": -0.06454329937696457,
+      "step": 3280
+    },
+    {
+      "epoch": 0.5668504479669194,
+      "grad_norm": 2.370781421661377,
+      "learning_rate": 4.50490053606381e-08,
+      "logits/chosen": -3.1264491081237793,
+      "logits/rejected": -3.095414161682129,
+      "logps/chosen": -57.42754364013672,
+      "logps/rejected": -60.08568572998047,
+      "loss": 0.6813,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.04261315241456032,
+      "rewards/margins": 0.02501532807946205,
+      "rewards/rejected": -0.06762847304344177,
+      "step": 3290
+    },
+    {
+      "epoch": 0.5685733976567884,
+      "grad_norm": 2.8540031909942627,
+      "learning_rate": 4.5004004356724893e-08,
+      "logits/chosen": -3.002164125442505,
+      "logits/rejected": -2.980264186859131,
+      "logps/chosen": -59.46967315673828,
+      "logps/rejected": -60.4211311340332,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.04553186520934105,
+      "rewards/margins": 0.02290916070342064,
+      "rewards/rejected": -0.06844101846218109,
+      "step": 3300
+    },
+    {
+      "epoch": 0.5685733976567884,
+      "eval_logits/chosen": -3.1044232845306396,
+      "eval_logits/rejected": -3.0987377166748047,
+      "eval_logps/chosen": -60.55705642700195,
+      "eval_logps/rejected": -65.8370361328125,
+      "eval_loss": 0.6894074082374573,
+      "eval_rewards/accuracies": 0.5859665274620056,
+      "eval_rewards/chosen": -0.018451597541570663,
+      "eval_rewards/margins": 0.008117567747831345,
+      "eval_rewards/rejected": -0.02656916342675686,
+      "eval_runtime": 383.0108,
+      "eval_samples_per_second": 11.237,
+      "eval_steps_per_second": 1.405,
+      "step": 3300
+    },
+    {
+      "epoch": 0.5702963473466575,
+      "grad_norm": 2.5802247524261475,
+      "learning_rate": 4.495882245506002e-08,
+      "logits/chosen": -2.942958354949951,
+      "logits/rejected": -2.9146721363067627,
+      "logps/chosen": -57.35901641845703,
+      "logps/rejected": -57.24003219604492,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.04260212928056717,
+      "rewards/margins": 0.02280474081635475,
+      "rewards/rejected": -0.06540687382221222,
+      "step": 3310
+    },
+    {
+      "epoch": 0.5720192970365265,
+      "grad_norm": 2.8888728618621826,
+      "learning_rate": 4.4913460064226896e-08,
+      "logits/chosen": -3.00022554397583,
+      "logits/rejected": -2.9643304347991943,
+      "logps/chosen": -58.44237518310547,
+      "logps/rejected": -56.3722038269043,
+      "loss": 0.6803,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.047347474843263626,
+      "rewards/margins": 0.026923689991235733,
+      "rewards/rejected": -0.07427116483449936,
+      "step": 3320
+    },
+    {
+      "epoch": 0.5737422467263956,
+      "grad_norm": 2.6733479499816895,
+      "learning_rate": 4.486791759444111e-08,
+      "logits/chosen": -3.101069688796997,
+      "logits/rejected": -3.0694613456726074,
+      "logps/chosen": -57.95048904418945,
+      "logps/rejected": -60.2997932434082,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.03737090528011322,
+      "rewards/margins": 0.028867101296782494,
+      "rewards/rejected": -0.06623800098896027,
+      "step": 3330
+    },
+    {
+      "epoch": 0.5754651964162646,
+      "grad_norm": 2.4325153827667236,
+      "learning_rate": 4.482219545754672e-08,
+      "logits/chosen": -3.036646842956543,
+      "logits/rejected": -3.018866777420044,
+      "logps/chosen": -60.54935836791992,
+      "logps/rejected": -61.04949188232422,
+      "loss": 0.6818,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.04636218771338463,
+      "rewards/margins": 0.024338793009519577,
+      "rewards/rejected": -0.0707009807229042,
+      "step": 3340
+    },
+    {
+      "epoch": 0.5771881461061337,
+      "grad_norm": 2.5532121658325195,
+      "learning_rate": 4.4776294067012546e-08,
+      "logits/chosen": -2.930751323699951,
+      "logits/rejected": -2.9158504009246826,
+      "logps/chosen": -61.03418731689453,
+      "logps/rejected": -63.93449783325195,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.052555155009031296,
+      "rewards/margins": 0.013547113165259361,
+      "rewards/rejected": -0.06610226631164551,
+      "step": 3350
+    },
+    {
+      "epoch": 0.5789110957960028,
+      "grad_norm": 2.5190041065216064,
+      "learning_rate": 4.473021383792838e-08,
+      "logits/chosen": -3.0526387691497803,
+      "logits/rejected": -3.017991304397583,
+      "logps/chosen": -59.423728942871094,
+      "logps/rejected": -59.21442794799805,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.04608495905995369,
+      "rewards/margins": 0.023744704201817513,
+      "rewards/rejected": -0.06982966512441635,
+      "step": 3360
+    },
+    {
+      "epoch": 0.5806340454858718,
+      "grad_norm": 2.631010055541992,
+      "learning_rate": 4.468395518700129e-08,
+      "logits/chosen": -2.9855659008026123,
+      "logits/rejected": -2.9757871627807617,
+      "logps/chosen": -55.68975830078125,
+      "logps/rejected": -60.87224578857422,
+      "loss": 0.6832,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.04734468832612038,
+      "rewards/margins": 0.021241312846541405,
+      "rewards/rejected": -0.06858599931001663,
+      "step": 3370
+    },
+    {
+      "epoch": 0.5823569951757409,
+      "grad_norm": 2.4844372272491455,
+      "learning_rate": 4.463751853255182e-08,
+      "logits/chosen": -3.0820019245147705,
+      "logits/rejected": -3.0496532917022705,
+      "logps/chosen": -59.5984992980957,
+      "logps/rejected": -57.16900634765625,
+      "loss": 0.6795,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.04137615114450455,
+      "rewards/margins": 0.02887653186917305,
+      "rewards/rejected": -0.0702526792883873,
+      "step": 3380
+    },
+    {
+      "epoch": 0.5840799448656099,
+      "grad_norm": 2.2379589080810547,
+      "learning_rate": 4.45909042945102e-08,
+      "logits/chosen": -3.0041701793670654,
+      "logits/rejected": -2.977856397628784,
+      "logps/chosen": -58.76365280151367,
+      "logps/rejected": -58.33111572265625,
+      "loss": 0.6845,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.0503598228096962,
+      "rewards/margins": 0.018615610897541046,
+      "rewards/rejected": -0.06897543370723724,
+      "step": 3390
+    },
+    {
+      "epoch": 0.585802894555479,
+      "grad_norm": 2.5656955242156982,
+      "learning_rate": 4.454411289441259e-08,
+      "logits/chosen": -3.0565237998962402,
+      "logits/rejected": -3.017136812210083,
+      "logps/chosen": -57.367149353027344,
+      "logps/rejected": -59.04413604736328,
+      "loss": 0.6782,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.04510427266359329,
+      "rewards/margins": 0.031491007655858994,
+      "rewards/rejected": -0.07659528404474258,
+      "step": 3400
+    },
+    {
+      "epoch": 0.585802894555479,
+      "eval_logits/chosen": -3.1006083488464355,
+      "eval_logits/rejected": -3.0949037075042725,
+      "eval_logps/chosen": -60.79747009277344,
+      "eval_logps/rejected": -66.13670349121094,
+      "eval_loss": 0.6891415119171143,
+      "eval_rewards/accuracies": 0.589219331741333,
+      "eval_rewards/chosen": -0.02085571549832821,
+      "eval_rewards/margins": 0.008710107766091824,
+      "eval_rewards/rejected": -0.029565826058387756,
+      "eval_runtime": 382.988,
+      "eval_samples_per_second": 11.238,
+      "eval_steps_per_second": 1.405,
+      "step": 3400
+    },
+    {
+      "epoch": 0.587525844245348,
+      "grad_norm": 2.3698174953460693,
+      "learning_rate": 4.4497144755397215e-08,
+      "logits/chosen": -2.944680690765381,
+      "logits/rejected": -2.9156534671783447,
+      "logps/chosen": -54.545806884765625,
+      "logps/rejected": -54.521392822265625,
+      "loss": 0.6815,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.04972557723522186,
+      "rewards/margins": 0.024806033819913864,
+      "rewards/rejected": -0.07453161478042603,
+      "step": 3410
+    },
+    {
+      "epoch": 0.5892487939352171,
+      "grad_norm": 3.0524845123291016,
+      "learning_rate": 4.4450000302200574e-08,
+      "logits/chosen": -2.965384006500244,
+      "logits/rejected": -2.9409515857696533,
+      "logps/chosen": -57.24824142456055,
+      "logps/rejected": -60.267913818359375,
+      "loss": 0.6815,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.0448092445731163,
+      "rewards/margins": 0.02488141693174839,
+      "rewards/rejected": -0.06969065964221954,
+      "step": 3420
+    },
+    {
+      "epoch": 0.5909717436250862,
+      "grad_norm": 2.2783493995666504,
+      "learning_rate": 4.440267996115359e-08,
+      "logits/chosen": -3.0496408939361572,
+      "logits/rejected": -3.0292272567749023,
+      "logps/chosen": -56.09685134887695,
+      "logps/rejected": -61.353546142578125,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.047559600323438644,
+      "rewards/margins": 0.019659578800201416,
+      "rewards/rejected": -0.06721917539834976,
+      "step": 3430
+    },
+    {
+      "epoch": 0.5926946933149552,
+      "grad_norm": 2.6467339992523193,
+      "learning_rate": 4.435518416017774e-08,
+      "logits/chosen": -2.9709994792938232,
+      "logits/rejected": -2.9549765586853027,
+      "logps/chosen": -58.36515426635742,
+      "logps/rejected": -60.28800582885742,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.057922862470149994,
+      "rewards/margins": 0.016946915537118912,
+      "rewards/rejected": -0.0748697817325592,
+      "step": 3440
+    },
+    {
+      "epoch": 0.5944176430048242,
+      "grad_norm": 2.7503037452697754,
+      "learning_rate": 4.430751332878122e-08,
+      "logits/chosen": -3.2181389331817627,
+      "logits/rejected": -3.1773762702941895,
+      "logps/chosen": -64.21342468261719,
+      "logps/rejected": -62.100074768066406,
+      "loss": 0.6823,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.048339568078517914,
+      "rewards/margins": 0.0231911763548851,
+      "rewards/rejected": -0.07153073698282242,
+      "step": 3450
+    },
+    {
+      "epoch": 0.5961405926946933,
+      "grad_norm": 2.6095528602600098,
+      "learning_rate": 4.425966789805503e-08,
+      "logits/chosen": -2.997051477432251,
+      "logits/rejected": -2.9842872619628906,
+      "logps/chosen": -56.6351318359375,
+      "logps/rejected": -59.5648078918457,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.04455737769603729,
+      "rewards/margins": 0.015089410357177258,
+      "rewards/rejected": -0.05964677780866623,
+      "step": 3460
+    },
+    {
+      "epoch": 0.5978635423845624,
+      "grad_norm": 2.290261745452881,
+      "learning_rate": 4.4211648300669074e-08,
+      "logits/chosen": -3.07570481300354,
+      "logits/rejected": -3.059583902359009,
+      "logps/chosen": -60.6319694519043,
+      "logps/rejected": -61.009071350097656,
+      "loss": 0.6846,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.04821758717298508,
+      "rewards/margins": 0.018515009433031082,
+      "rewards/rejected": -0.06673260033130646,
+      "step": 3470
+    },
+    {
+      "epoch": 0.5995864920744314,
+      "grad_norm": 2.6477210521698,
+      "learning_rate": 4.416345497086827e-08,
+      "logits/chosen": -3.0032341480255127,
+      "logits/rejected": -2.9717013835906982,
+      "logps/chosen": -59.45502471923828,
+      "logps/rejected": -59.22282028198242,
+      "loss": 0.679,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.04340440779924393,
+      "rewards/margins": 0.029992705211043358,
+      "rewards/rejected": -0.07339711487293243,
+      "step": 3480
+    },
+    {
+      "epoch": 0.6013094417643005,
+      "grad_norm": 2.7273612022399902,
+      "learning_rate": 4.411508834446863e-08,
+      "logits/chosen": -3.011401414871216,
+      "logits/rejected": -2.9824185371398926,
+      "logps/chosen": -58.92223358154297,
+      "logps/rejected": -61.866722106933594,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.04652860388159752,
+      "rewards/margins": 0.026909958571195602,
+      "rewards/rejected": -0.07343856990337372,
+      "step": 3490
+    },
+    {
+      "epoch": 0.6030323914541695,
+      "grad_norm": 2.5121710300445557,
+      "learning_rate": 4.406654885885326e-08,
+      "logits/chosen": -2.9573516845703125,
+      "logits/rejected": -2.9494025707244873,
+      "logps/chosen": -57.327293395996094,
+      "logps/rejected": -61.879295349121094,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0532461479306221,
+      "rewards/margins": 0.018929705023765564,
+      "rewards/rejected": -0.07217585295438766,
+      "step": 3500
+    },
+    {
+      "epoch": 0.6030323914541695,
+      "eval_logits/chosen": -3.097958564758301,
+      "eval_logits/rejected": -3.0922303199768066,
+      "eval_logps/chosen": -61.010868072509766,
+      "eval_logps/rejected": -66.39277648925781,
+      "eval_loss": 0.6889605522155762,
+      "eval_rewards/accuracies": 0.5903810262680054,
+      "eval_rewards/chosen": -0.022989705204963684,
+      "eval_rewards/margins": 0.009136790409684181,
+      "eval_rewards/rejected": -0.032126497477293015,
+      "eval_runtime": 383.1973,
+      "eval_samples_per_second": 11.232,
+      "eval_steps_per_second": 1.404,
+      "step": 3500
+    },
+    {
+      "epoch": 0.6047553411440386,
+      "grad_norm": 2.5552871227264404,
+      "learning_rate": 4.401783695296847e-08,
+      "logits/chosen": -2.9002912044525146,
+      "logits/rejected": -2.8752737045288086,
+      "logps/chosen": -59.427833557128906,
+      "logps/rejected": -59.167259216308594,
+      "loss": 0.683,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.058605749160051346,
+      "rewards/margins": 0.02162277139723301,
+      "rewards/rejected": -0.0802285224199295,
+      "step": 3510
+    },
+    {
+      "epoch": 0.6064782908339077,
+      "grad_norm": 2.5069310665130615,
+      "learning_rate": 4.3968953067319766e-08,
+      "logits/chosen": -2.934281826019287,
+      "logits/rejected": -2.914170503616333,
+      "logps/chosen": -60.286041259765625,
+      "logps/rejected": -59.225730895996094,
+      "loss": 0.6828,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.049517445266246796,
+      "rewards/margins": 0.022084731608629227,
+      "rewards/rejected": -0.07160218060016632,
+      "step": 3520
+    },
+    {
+      "epoch": 0.6082012405237767,
+      "grad_norm": 2.6399123668670654,
+      "learning_rate": 4.391989764396792e-08,
+      "logits/chosen": -3.1014933586120605,
+      "logits/rejected": -3.0584206581115723,
+      "logps/chosen": -60.59342575073242,
+      "logps/rejected": -58.522125244140625,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.04635945335030556,
+      "rewards/margins": 0.029721911996603012,
+      "rewards/rejected": -0.07608136534690857,
+      "step": 3530
+    },
+    {
+      "epoch": 0.6099241902136457,
+      "grad_norm": 2.8140361309051514,
+      "learning_rate": 4.387067112652487e-08,
+      "logits/chosen": -2.922349452972412,
+      "logits/rejected": -2.8977274894714355,
+      "logps/chosen": -59.139808654785156,
+      "logps/rejected": -60.839202880859375,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.05227070301771164,
+      "rewards/margins": 0.020450804382562637,
+      "rewards/rejected": -0.07272150367498398,
+      "step": 3540
+    },
+    {
+      "epoch": 0.6116471399035148,
+      "grad_norm": 2.7008562088012695,
+      "learning_rate": 4.382127396014982e-08,
+      "logits/chosen": -3.012406349182129,
+      "logits/rejected": -3.0028626918792725,
+      "logps/chosen": -60.59033966064453,
+      "logps/rejected": -59.45725631713867,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.06027187779545784,
+      "rewards/margins": 0.008788404986262321,
+      "rewards/rejected": -0.06906028091907501,
+      "step": 3550
+    },
+    {
+      "epoch": 0.6133700895933839,
+      "grad_norm": 2.476224184036255,
+      "learning_rate": 4.377170659154514e-08,
+      "logits/chosen": -2.946342706680298,
+      "logits/rejected": -2.9185807704925537,
+      "logps/chosen": -58.499481201171875,
+      "logps/rejected": -60.6409912109375,
+      "loss": 0.6814,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.055995263159275055,
+      "rewards/margins": 0.024932775646448135,
+      "rewards/rejected": -0.0809280276298523,
+      "step": 3560
+    },
+    {
+      "epoch": 0.6150930392832529,
+      "grad_norm": 2.8927676677703857,
+      "learning_rate": 4.372196946895238e-08,
+      "logits/chosen": -3.0909087657928467,
+      "logits/rejected": -3.0562610626220703,
+      "logps/chosen": -62.383270263671875,
+      "logps/rejected": -60.56291580200195,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.055105555802583694,
+      "rewards/margins": 0.018301472067832947,
+      "rewards/rejected": -0.07340702414512634,
+      "step": 3570
+    },
+    {
+      "epoch": 0.616815988973122,
+      "grad_norm": 2.698302984237671,
+      "learning_rate": 4.367206304214815e-08,
+      "logits/chosen": -3.0235342979431152,
+      "logits/rejected": -3.0003914833068848,
+      "logps/chosen": -62.93693923950195,
+      "logps/rejected": -65.2998046875,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.04742145910859108,
+      "rewards/margins": 0.03086162731051445,
+      "rewards/rejected": -0.07828308641910553,
+      "step": 3580
+    },
+    {
+      "epoch": 0.618538938662991,
+      "grad_norm": 2.5029187202453613,
+      "learning_rate": 4.3621987762440115e-08,
+      "logits/chosen": -3.0117130279541016,
+      "logits/rejected": -2.995633602142334,
+      "logps/chosen": -60.524391174316406,
+      "logps/rejected": -62.40476608276367,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.06099390238523483,
+      "rewards/margins": 0.02319389581680298,
+      "rewards/rejected": -0.08418779820203781,
+      "step": 3590
+    },
+    {
+      "epoch": 0.6202618883528601,
+      "grad_norm": 2.8349897861480713,
+      "learning_rate": 4.3571744082662884e-08,
+      "logits/chosen": -2.9916810989379883,
+      "logits/rejected": -2.9671850204467773,
+      "logps/chosen": -60.09454345703125,
+      "logps/rejected": -59.30830764770508,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.054474882781505585,
+      "rewards/margins": 0.02300008200109005,
+      "rewards/rejected": -0.07747496664524078,
+      "step": 3600
+    },
+    {
+      "epoch": 0.6202618883528601,
+      "eval_logits/chosen": -3.0943825244903564,
+      "eval_logits/rejected": -3.088628053665161,
+      "eval_logps/chosen": -61.21992492675781,
+      "eval_logps/rejected": -66.65460968017578,
+      "eval_loss": 0.6887229084968567,
+      "eval_rewards/accuracies": 0.5934014916419983,
+      "eval_rewards/chosen": -0.025080382823944092,
+      "eval_rewards/margins": 0.009664410725235939,
+      "eval_rewards/rejected": -0.03474479913711548,
+      "eval_runtime": 383.2505,
+      "eval_samples_per_second": 11.23,
+      "eval_steps_per_second": 1.404,
+      "step": 3600
+    },
+    {
+      "epoch": 0.6219848380427292,
+      "grad_norm": 2.234377384185791,
+      "learning_rate": 4.352133245717393e-08,
+      "logits/chosen": -2.958303451538086,
+      "logits/rejected": -2.942148208618164,
+      "logps/chosen": -58.13250732421875,
+      "logps/rejected": -57.554290771484375,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.06639299541711807,
+      "rewards/margins": 0.014730398543179035,
+      "rewards/rejected": -0.08112339675426483,
+      "step": 3610
+    },
+    {
+      "epoch": 0.6237077877325982,
+      "grad_norm": 2.570688247680664,
+      "learning_rate": 4.347075334184946e-08,
+      "logits/chosen": -2.9175684452056885,
+      "logits/rejected": -2.892578601837158,
+      "logps/chosen": -60.026031494140625,
+      "logps/rejected": -60.940773010253906,
+      "loss": 0.6798,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.04518166929483414,
+      "rewards/margins": 0.02825523540377617,
+      "rewards/rejected": -0.07343690097332001,
+      "step": 3620
+    },
+    {
+      "epoch": 0.6254307374224672,
+      "grad_norm": 2.4950618743896484,
+      "learning_rate": 4.34200071940803e-08,
+      "logits/chosen": -3.0738365650177,
+      "logits/rejected": -3.067145347595215,
+      "logps/chosen": -56.715431213378906,
+      "logps/rejected": -64.12156677246094,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.05574717000126839,
+      "rewards/margins": 0.028617192059755325,
+      "rewards/rejected": -0.08436436206102371,
+      "step": 3630
+    },
+    {
+      "epoch": 0.6271536871123363,
+      "grad_norm": 2.7924208641052246,
+      "learning_rate": 4.3369094472767786e-08,
+      "logits/chosen": -3.023838758468628,
+      "logits/rejected": -3.0090363025665283,
+      "logps/chosen": -60.09112548828125,
+      "logps/rejected": -63.513458251953125,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.050480134785175323,
+      "rewards/margins": 0.022859184071421623,
+      "rewards/rejected": -0.07333932816982269,
+      "step": 3640
+    },
+    {
+      "epoch": 0.6288766368022054,
+      "grad_norm": 2.284653425216675,
+      "learning_rate": 4.331801563831956e-08,
+      "logits/chosen": -2.974470615386963,
+      "logits/rejected": -2.9797394275665283,
+      "logps/chosen": -56.4565315246582,
+      "logps/rejected": -61.20124053955078,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.05375003069639206,
+      "rewards/margins": 0.012932727113366127,
+      "rewards/rejected": -0.06668275594711304,
+      "step": 3650
+    },
+    {
+      "epoch": 0.6305995864920745,
+      "grad_norm": 2.4951345920562744,
+      "learning_rate": 4.326677115264547e-08,
+      "logits/chosen": -3.032104969024658,
+      "logits/rejected": -3.002105236053467,
+      "logps/chosen": -59.15620040893555,
+      "logps/rejected": -61.78845977783203,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.052974503487348557,
+      "rewards/margins": 0.029516946524381638,
+      "rewards/rejected": -0.0824914500117302,
+      "step": 3660
+    },
+    {
+      "epoch": 0.6323225361819435,
+      "grad_norm": 2.63869309425354,
+      "learning_rate": 4.321536147915334e-08,
+      "logits/chosen": -2.9882678985595703,
+      "logits/rejected": -2.9602417945861816,
+      "logps/chosen": -58.3986930847168,
+      "logps/rejected": -60.359230041503906,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.055475085973739624,
+      "rewards/margins": 0.022640887647867203,
+      "rewards/rejected": -0.07811598479747772,
+      "step": 3670
+    },
+    {
+      "epoch": 0.6340454858718125,
+      "grad_norm": 2.5783205032348633,
+      "learning_rate": 4.3163787082744806e-08,
+      "logits/chosen": -3.0069332122802734,
+      "logits/rejected": -2.9662632942199707,
+      "logps/chosen": -60.43860626220703,
+      "logps/rejected": -58.22229766845703,
+      "loss": 0.6824,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.05762777477502823,
+      "rewards/margins": 0.02358635514974594,
+      "rewards/rejected": -0.08121412992477417,
+      "step": 3680
+    },
+    {
+      "epoch": 0.6357684355616816,
+      "grad_norm": 3.023909568786621,
+      "learning_rate": 4.31120484298111e-08,
+      "logits/chosen": -2.94201397895813,
+      "logits/rejected": -2.950760841369629,
+      "logps/chosen": -57.85539627075195,
+      "logps/rejected": -66.70121002197266,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.0576167032122612,
+      "rewards/margins": 0.018452594056725502,
+      "rewards/rejected": -0.07606930285692215,
+      "step": 3690
+    },
+    {
+      "epoch": 0.6374913852515507,
+      "grad_norm": 2.739835500717163,
+      "learning_rate": 4.306014598822886e-08,
+      "logits/chosen": -2.9856293201446533,
+      "logits/rejected": -2.959155559539795,
+      "logps/chosen": -62.75537872314453,
+      "logps/rejected": -67.74443054199219,
+      "loss": 0.6782,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.051578450947999954,
+      "rewards/margins": 0.03199709951877594,
+      "rewards/rejected": -0.0835755467414856,
+      "step": 3700
+    },
+    {
+      "epoch": 0.6374913852515507,
+      "eval_logits/chosen": -3.0905635356903076,
+      "eval_logits/rejected": -3.084831714630127,
+      "eval_logps/chosen": -61.44447708129883,
+      "eval_logps/rejected": -66.92031860351562,
+      "eval_loss": 0.6885430216789246,
+      "eval_rewards/accuracies": 0.5920074582099915,
+      "eval_rewards/chosen": -0.027325810864567757,
+      "eval_rewards/margins": 0.010076167993247509,
+      "eval_rewards/rejected": -0.03740197792649269,
+      "eval_runtime": 383.1891,
+      "eval_samples_per_second": 11.232,
+      "eval_steps_per_second": 1.404,
+      "step": 3700
+    },
+    {
+      "epoch": 0.6392143349414197,
+      "grad_norm": 2.825364351272583,
+      "learning_rate": 4.300808022735584e-08,
+      "logits/chosen": -3.0198187828063965,
+      "logits/rejected": -2.986969470977783,
+      "logps/chosen": -59.27638626098633,
+      "logps/rejected": -59.07899856567383,
+      "loss": 0.6808,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.054077018052339554,
+      "rewards/margins": 0.026380132883787155,
+      "rewards/rejected": -0.0804571583867073,
+      "step": 3710
+    },
+    {
+      "epoch": 0.6409372846312887,
+      "grad_norm": 2.604074716567993,
+      "learning_rate": 4.295585161802674e-08,
+      "logits/chosen": -3.0332984924316406,
+      "logits/rejected": -3.0157291889190674,
+      "logps/chosen": -57.33430099487305,
+      "logps/rejected": -61.54401779174805,
+      "loss": 0.6804,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.05437540262937546,
+      "rewards/margins": 0.027211666107177734,
+      "rewards/rejected": -0.08158706873655319,
+      "step": 3720
+    },
+    {
+      "epoch": 0.6426602343211578,
+      "grad_norm": 3.0167033672332764,
+      "learning_rate": 4.290346063254889e-08,
+      "logits/chosen": -2.994412899017334,
+      "logits/rejected": -2.971510648727417,
+      "logps/chosen": -59.98322296142578,
+      "logps/rejected": -59.22069549560547,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.054655902087688446,
+      "rewards/margins": 0.0304948091506958,
+      "rewards/rejected": -0.08515071123838425,
+      "step": 3730
+    },
+    {
+      "epoch": 0.6443831840110269,
+      "grad_norm": 2.8925511837005615,
+      "learning_rate": 4.285090774469802e-08,
+      "logits/chosen": -2.9922571182250977,
+      "logits/rejected": -2.967726230621338,
+      "logps/chosen": -59.203651428222656,
+      "logps/rejected": -64.27935028076172,
+      "loss": 0.6835,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.05768605321645737,
+      "rewards/margins": 0.02101275324821472,
+      "rewards/rejected": -0.07869881391525269,
+      "step": 3740
+    },
+    {
+      "epoch": 0.646106133700896,
+      "grad_norm": 2.819869041442871,
+      "learning_rate": 4.279819342971391e-08,
+      "logits/chosen": -3.063446283340454,
+      "logits/rejected": -3.0350348949432373,
+      "logps/chosen": -58.6164665222168,
+      "logps/rejected": -61.5792350769043,
+      "loss": 0.6781,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.0529780387878418,
+      "rewards/margins": 0.03202220797538757,
+      "rewards/rejected": -0.08500026166439056,
+      "step": 3750
+    },
+    {
+      "epoch": 0.647829083390765,
+      "grad_norm": 2.741615056991577,
+      "learning_rate": 4.27453181642962e-08,
+      "logits/chosen": -3.0189037322998047,
+      "logits/rejected": -3.0157222747802734,
+      "logps/chosen": -62.073394775390625,
+      "logps/rejected": -64.53569793701172,
+      "loss": 0.6828,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.061997391283512115,
+      "rewards/margins": 0.022763317450881004,
+      "rewards/rejected": -0.08476071059703827,
+      "step": 3760
+    },
+    {
+      "epoch": 0.649552033080634,
+      "grad_norm": 2.7681660652160645,
+      "learning_rate": 4.269228242659997e-08,
+      "logits/chosen": -2.9845850467681885,
+      "logits/rejected": -2.9641308784484863,
+      "logps/chosen": -59.365478515625,
+      "logps/rejected": -62.1631965637207,
+      "loss": 0.6849,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.058237820863723755,
+      "rewards/margins": 0.01789437234401703,
+      "rewards/rejected": -0.07613219320774078,
+      "step": 3770
+    },
+    {
+      "epoch": 0.6512749827705031,
+      "grad_norm": 2.5837435722351074,
+      "learning_rate": 4.2639086696231486e-08,
+      "logits/chosen": -3.0354232788085938,
+      "logits/rejected": -2.9964494705200195,
+      "logps/chosen": -60.25130081176758,
+      "logps/rejected": -57.72199630737305,
+      "loss": 0.6833,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.06172683835029602,
+      "rewards/margins": 0.02148234099149704,
+      "rewards/rejected": -0.08320917189121246,
+      "step": 3780
+    },
+    {
+      "epoch": 0.6529979324603722,
+      "grad_norm": 2.7079079151153564,
+      "learning_rate": 4.2585731454243836e-08,
+      "logits/chosen": -2.987069845199585,
+      "logits/rejected": -2.957787036895752,
+      "logps/chosen": -59.439537048339844,
+      "logps/rejected": -59.216346740722656,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.05451414734125137,
+      "rewards/margins": 0.027742888778448105,
+      "rewards/rejected": -0.08225704729557037,
+      "step": 3790
+    },
+    {
+      "epoch": 0.6547208821502413,
+      "grad_norm": 3.3162882328033447,
+      "learning_rate": 4.2532217183132566e-08,
+      "logits/chosen": -3.003960609436035,
+      "logits/rejected": -2.972020387649536,
+      "logps/chosen": -63.24890899658203,
+      "logps/rejected": -60.95477294921875,
+      "loss": 0.6814,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.05555642768740654,
+      "rewards/margins": 0.025314921513199806,
+      "rewards/rejected": -0.08087135851383209,
+      "step": 3800
+    },
+    {
+      "epoch": 0.6547208821502413,
+      "eval_logits/chosen": -3.0873570442199707,
+      "eval_logits/rejected": -3.0816380977630615,
+      "eval_logps/chosen": -61.75248336791992,
+      "eval_logps/rejected": -67.2955551147461,
+      "eval_loss": 0.6882451772689819,
+      "eval_rewards/accuracies": 0.5915427803993225,
+      "eval_rewards/chosen": -0.03040589578449726,
+      "eval_rewards/margins": 0.010748391970992088,
+      "eval_rewards/rejected": -0.04115428403019905,
+      "eval_runtime": 382.8067,
+      "eval_samples_per_second": 11.243,
+      "eval_steps_per_second": 1.405,
+      "step": 3800
+    },
+    {
+      "epoch": 0.6564438318401102,
+      "grad_norm": 2.542724132537842,
+      "learning_rate": 4.247854436683137e-08,
+      "logits/chosen": -3.0303971767425537,
+      "logits/rejected": -2.986292600631714,
+      "logps/chosen": -63.115516662597656,
+      "logps/rejected": -61.104637145996094,
+      "loss": 0.6772,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.05647412687540054,
+      "rewards/margins": 0.03393317386507988,
+      "rewards/rejected": -0.09040729701519012,
+      "step": 3810
+    },
+    {
+      "epoch": 0.6581667815299793,
+      "grad_norm": 2.707315683364868,
+      "learning_rate": 4.242471349070765e-08,
+      "logits/chosen": -2.9975123405456543,
+      "logits/rejected": -2.9804158210754395,
+      "logps/chosen": -59.8589973449707,
+      "logps/rejected": -60.76411819458008,
+      "loss": 0.6843,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.05760779231786728,
+      "rewards/margins": 0.01915673352777958,
+      "rewards/rejected": -0.07676452398300171,
+      "step": 3820
+    },
+    {
+      "epoch": 0.6598897312198484,
+      "grad_norm": 2.639753818511963,
+      "learning_rate": 4.237072504155817e-08,
+      "logits/chosen": -3.043461561203003,
+      "logits/rejected": -2.997986078262329,
+      "logps/chosen": -63.53108596801758,
+      "logps/rejected": -59.734130859375,
+      "loss": 0.6766,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.04853980243206024,
+      "rewards/margins": 0.03519828990101814,
+      "rewards/rejected": -0.08373810350894928,
+      "step": 3830
+    },
+    {
+      "epoch": 0.6616126809097175,
+      "grad_norm": 2.8300094604492188,
+      "learning_rate": 4.231657950760461e-08,
+      "logits/chosen": -2.9670801162719727,
+      "logits/rejected": -2.9537301063537598,
+      "logps/chosen": -59.12525177001953,
+      "logps/rejected": -66.98822784423828,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.050388552248477936,
+      "rewards/margins": 0.029177028685808182,
+      "rewards/rejected": -0.07956556975841522,
+      "step": 3840
+    },
+    {
+      "epoch": 0.6633356305995864,
+      "grad_norm": 2.6567018032073975,
+      "learning_rate": 4.2262277378489225e-08,
+      "logits/chosen": -3.093801736831665,
+      "logits/rejected": -3.0786468982696533,
+      "logps/chosen": -63.184783935546875,
+      "logps/rejected": -61.35536575317383,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.05883680656552315,
+      "rewards/margins": 0.01705780252814293,
+      "rewards/rejected": -0.07589460909366608,
+      "step": 3850
+    },
+    {
+      "epoch": 0.6650585802894555,
+      "grad_norm": 2.9890902042388916,
+      "learning_rate": 4.220781914527035e-08,
+      "logits/chosen": -3.103285074234009,
+      "logits/rejected": -3.0767664909362793,
+      "logps/chosen": -61.01404571533203,
+      "logps/rejected": -61.87114334106445,
+      "loss": 0.6838,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.06833615154027939,
+      "rewards/margins": 0.020947633311152458,
+      "rewards/rejected": -0.0892837792634964,
+      "step": 3860
+    },
+    {
+      "epoch": 0.6667815299793246,
+      "grad_norm": 3.654477834701538,
+      "learning_rate": 4.2153205300417966e-08,
+      "logits/chosen": -2.993138313293457,
+      "logits/rejected": -2.968773603439331,
+      "logps/chosen": -64.29295349121094,
+      "logps/rejected": -63.77922821044922,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.05440130829811096,
+      "rewards/margins": 0.026751140132546425,
+      "rewards/rejected": -0.08115243911743164,
+      "step": 3870
+    },
+    {
+      "epoch": 0.6685044796691937,
+      "grad_norm": 2.9977378845214844,
+      "learning_rate": 4.209843633780929e-08,
+      "logits/chosen": -3.023743152618408,
+      "logits/rejected": -3.0283050537109375,
+      "logps/chosen": -57.99518966674805,
+      "logps/rejected": -61.97289276123047,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.07116183638572693,
+      "rewards/margins": 0.013650787994265556,
+      "rewards/rejected": -0.08481262624263763,
+      "step": 3880
+    },
+    {
+      "epoch": 0.6702274293590628,
+      "grad_norm": 2.609907865524292,
+      "learning_rate": 4.2043512752724265e-08,
+      "logits/chosen": -3.071347951889038,
+      "logits/rejected": -3.043524742126465,
+      "logps/chosen": -60.55571746826172,
+      "logps/rejected": -61.212158203125,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.06369321048259735,
+      "rewards/margins": 0.026957347989082336,
+      "rewards/rejected": -0.09065055102109909,
+      "step": 3890
+    },
+    {
+      "epoch": 0.6719503790489317,
+      "grad_norm": 2.9191722869873047,
+      "learning_rate": 4.19884350418411e-08,
+      "logits/chosen": -3.04362154006958,
+      "logits/rejected": -2.9878525733947754,
+      "logps/chosen": -62.60947799682617,
+      "logps/rejected": -59.17048263549805,
+      "loss": 0.6784,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.05780899524688721,
+      "rewards/margins": 0.03127293661236763,
+      "rewards/rejected": -0.08908193558454514,
+      "step": 3900
+    },
+    {
+      "epoch": 0.6719503790489317,
+      "eval_logits/chosen": -3.0841052532196045,
+      "eval_logits/rejected": -3.0783629417419434,
+      "eval_logps/chosen": -62.062828063964844,
+      "eval_logps/rejected": -67.67224884033203,
+      "eval_loss": 0.6879503726959229,
+      "eval_rewards/accuracies": 0.5936338305473328,
+      "eval_rewards/chosen": -0.03350934013724327,
+      "eval_rewards/margins": 0.011411985382437706,
+      "eval_rewards/rejected": -0.044921327382326126,
+      "eval_runtime": 382.946,
+      "eval_samples_per_second": 11.239,
+      "eval_steps_per_second": 1.405,
+      "step": 3900
+    },
+    {
+      "epoch": 0.6736733287388008,
+      "grad_norm": 2.6801702976226807,
+      "learning_rate": 4.1933203703231764e-08,
+      "logits/chosen": -3.019151449203491,
+      "logits/rejected": -3.0033278465270996,
+      "logps/chosen": -62.104248046875,
+      "logps/rejected": -64.01966094970703,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.06462881714105606,
+      "rewards/margins": 0.029508817940950394,
+      "rewards/rejected": -0.09413763135671616,
+      "step": 3910
+    },
+    {
+      "epoch": 0.6753962784286699,
+      "grad_norm": 2.9897258281707764,
+      "learning_rate": 4.187781923635753e-08,
+      "logits/chosen": -3.0900540351867676,
+      "logits/rejected": -3.0438742637634277,
+      "logps/chosen": -64.23818969726562,
+      "logps/rejected": -59.7086296081543,
+      "loss": 0.6761,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.055687807500362396,
+      "rewards/margins": 0.03691873699426651,
+      "rewards/rejected": -0.0926065519452095,
+      "step": 3920
+    },
+    {
+      "epoch": 0.677119228118539,
+      "grad_norm": 2.839127779006958,
+      "learning_rate": 4.182228214206437e-08,
+      "logits/chosen": -3.092264413833618,
+      "logits/rejected": -3.087522506713867,
+      "logps/chosen": -58.1790885925293,
+      "logps/rejected": -61.78032684326172,
+      "loss": 0.6838,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.06921277195215225,
+      "rewards/margins": 0.020769404247403145,
+      "rewards/rejected": -0.08998216688632965,
+      "step": 3930
+    },
+    {
+      "epoch": 0.6788421778084079,
+      "grad_norm": 2.7984611988067627,
+      "learning_rate": 4.176659292257853e-08,
+      "logits/chosen": -2.9133617877960205,
+      "logits/rejected": -2.89428448677063,
+      "logps/chosen": -59.46643829345703,
+      "logps/rejected": -62.433319091796875,
+      "loss": 0.6803,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.059664350003004074,
+      "rewards/margins": 0.02768806181848049,
+      "rewards/rejected": -0.08735241740942001,
+      "step": 3940
+    },
+    {
+      "epoch": 0.680565127498277,
+      "grad_norm": 2.8083808422088623,
+      "learning_rate": 4.1710752081501877e-08,
+      "logits/chosen": -2.916139602661133,
+      "logits/rejected": -2.8688082695007324,
+      "logps/chosen": -61.85334396362305,
+      "logps/rejected": -60.78570556640625,
+      "loss": 0.6768,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.06379992514848709,
+      "rewards/margins": 0.0345899872481823,
+      "rewards/rejected": -0.09838990867137909,
+      "step": 3950
+    },
+    {
+      "epoch": 0.6822880771881461,
+      "grad_norm": 2.9986305236816406,
+      "learning_rate": 4.1654760123807465e-08,
+      "logits/chosen": -3.0221641063690186,
+      "logits/rejected": -3.023681163787842,
+      "logps/chosen": -58.96043014526367,
+      "logps/rejected": -68.14930725097656,
+      "loss": 0.6815,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.06257260590791702,
+      "rewards/margins": 0.024971123784780502,
+      "rewards/rejected": -0.08754372596740723,
+      "step": 3960
+    },
+    {
+      "epoch": 0.6840110268780152,
+      "grad_norm": 2.8469443321228027,
+      "learning_rate": 4.1598617555834866e-08,
+      "logits/chosen": -3.0332300662994385,
+      "logits/rejected": -3.009059429168701,
+      "logps/chosen": -60.48701858520508,
+      "logps/rejected": -62.04472732543945,
+      "loss": 0.6811,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.06414119899272919,
+      "rewards/margins": 0.026268791407346725,
+      "rewards/rejected": -0.09040997922420502,
+      "step": 3970
+    },
+    {
+      "epoch": 0.6857339765678843,
+      "grad_norm": 2.664799928665161,
+      "learning_rate": 4.1542324885285656e-08,
+      "logits/chosen": -2.9106838703155518,
+      "logits/rejected": -2.8912010192871094,
+      "logps/chosen": -60.42401885986328,
+      "logps/rejected": -63.05646896362305,
+      "loss": 0.6803,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.07084504514932632,
+      "rewards/margins": 0.027743428945541382,
+      "rewards/rejected": -0.0985884815454483,
+      "step": 3980
+    },
+    {
+      "epoch": 0.6874569262577532,
+      "grad_norm": 2.6945955753326416,
+      "learning_rate": 4.148588262121877e-08,
+      "logits/chosen": -3.023764133453369,
+      "logits/rejected": -3.009467124938965,
+      "logps/chosen": -60.0223388671875,
+      "logps/rejected": -64.20491027832031,
+      "loss": 0.6833,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.06805658340454102,
+      "rewards/margins": 0.021325495094060898,
+      "rewards/rejected": -0.08938208222389221,
+      "step": 3990
+    },
+    {
+      "epoch": 0.6891798759476223,
+      "grad_norm": 2.903080940246582,
+      "learning_rate": 4.1429291274045966e-08,
+      "logits/chosen": -3.0891542434692383,
+      "logits/rejected": -3.0542166233062744,
+      "logps/chosen": -63.660003662109375,
+      "logps/rejected": -61.81816864013672,
+      "loss": 0.6811,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.06455765664577484,
+      "rewards/margins": 0.025998909026384354,
+      "rewards/rejected": -0.0905565693974495,
+      "step": 4000
+    },
+    {
+      "epoch": 0.6891798759476223,
+      "eval_logits/chosen": -3.0804879665374756,
+      "eval_logits/rejected": -3.0747900009155273,
+      "eval_logps/chosen": -62.41653060913086,
+      "eval_logps/rejected": -68.09286499023438,
+      "eval_loss": 0.6876544952392578,
+      "eval_rewards/accuracies": 0.5950278639793396,
+      "eval_rewards/chosen": -0.03704635426402092,
+      "eval_rewards/margins": 0.01208107452839613,
+      "eval_rewards/rejected": -0.049127425998449326,
+      "eval_runtime": 383.3586,
+      "eval_samples_per_second": 11.227,
+      "eval_steps_per_second": 1.403,
+      "step": 4000
+    },
+    {
+      "epoch": 0.6909028256374914,
+      "grad_norm": 2.6702637672424316,
+      "learning_rate": 4.137255135552714e-08,
+      "logits/chosen": -2.9730191230773926,
+      "logits/rejected": -2.9638609886169434,
+      "logps/chosen": -55.092979431152344,
+      "logps/rejected": -62.62208938598633,
+      "loss": 0.6812,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.07073577493429184,
+      "rewards/margins": 0.025310367345809937,
+      "rewards/rejected": -0.09604613482952118,
+      "step": 4010
+    },
+    {
+      "epoch": 0.6926257753273605,
+      "grad_norm": 2.8805553913116455,
+      "learning_rate": 4.131566337876575e-08,
+      "logits/chosen": -2.9815170764923096,
+      "logits/rejected": -2.9676499366760254,
+      "logps/chosen": -58.787132263183594,
+      "logps/rejected": -62.66908645629883,
+      "loss": 0.6795,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.06601620465517044,
+      "rewards/margins": 0.02941732481122017,
+      "rewards/rejected": -0.0954335406422615,
+      "step": 4020
+    },
+    {
+      "epoch": 0.6943487250172296,
+      "grad_norm": 2.7405943870544434,
+      "learning_rate": 4.1258627858204156e-08,
+      "logits/chosen": -2.9791533946990967,
+      "logits/rejected": -2.957274913787842,
+      "logps/chosen": -62.0950813293457,
+      "logps/rejected": -64.92857360839844,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.06041101738810539,
+      "rewards/margins": 0.030954252928495407,
+      "rewards/rejected": -0.0913652777671814,
+      "step": 4030
+    },
+    {
+      "epoch": 0.6960716747070985,
+      "grad_norm": 3.0750865936279297,
+      "learning_rate": 4.1201445309618957e-08,
+      "logits/chosen": -3.0695481300354004,
+      "logits/rejected": -3.057634115219116,
+      "logps/chosen": -62.894691467285156,
+      "logps/rejected": -63.13087844848633,
+      "loss": 0.6822,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.06702469289302826,
+      "rewards/margins": 0.02379484660923481,
+      "rewards/rejected": -0.09081953763961792,
+      "step": 4040
+    },
+    {
+      "epoch": 0.6977946243969676,
+      "grad_norm": 2.7510950565338135,
+      "learning_rate": 4.114411625011634e-08,
+      "logits/chosen": -2.975109577178955,
+      "logits/rejected": -2.957728147506714,
+      "logps/chosen": -57.39542770385742,
+      "logps/rejected": -62.23594284057617,
+      "loss": 0.6813,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.06830006837844849,
+      "rewards/margins": 0.025825828313827515,
+      "rewards/rejected": -0.094125896692276,
+      "step": 4050
+    },
+    {
+      "epoch": 0.6995175740868367,
+      "grad_norm": 2.8251287937164307,
+      "learning_rate": 4.10866411981274e-08,
+      "logits/chosen": -2.9768476486206055,
+      "logits/rejected": -2.9438469409942627,
+      "logps/chosen": -63.757469177246094,
+      "logps/rejected": -62.3167610168457,
+      "loss": 0.6804,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.07397790253162384,
+      "rewards/margins": 0.02778732217848301,
+      "rewards/rejected": -0.1017652153968811,
+      "step": 4060
+    },
+    {
+      "epoch": 0.7012405237767058,
+      "grad_norm": 2.715132713317871,
+      "learning_rate": 4.102902067340348e-08,
+      "logits/chosen": -2.9819633960723877,
+      "logits/rejected": -2.95967173576355,
+      "logps/chosen": -61.2569580078125,
+      "logps/rejected": -63.06255340576172,
+      "loss": 0.6804,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.07425063848495483,
+      "rewards/margins": 0.027594611048698425,
+      "rewards/rejected": -0.10184524208307266,
+      "step": 4070
+    },
+    {
+      "epoch": 0.7029634734665747,
+      "grad_norm": 2.857219696044922,
+      "learning_rate": 4.0971255197011395e-08,
+      "logits/chosen": -2.9544215202331543,
+      "logits/rejected": -2.947038173675537,
+      "logps/chosen": -58.709022521972656,
+      "logps/rejected": -63.45673370361328,
+      "loss": 0.6812,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.06809200346469879,
+      "rewards/margins": 0.026254136115312576,
+      "rewards/rejected": -0.09434615075588226,
+      "step": 4080
+    },
+    {
+      "epoch": 0.7046864231564438,
+      "grad_norm": 2.758906841278076,
+      "learning_rate": 4.091334529132881e-08,
+      "logits/chosen": -3.031752109527588,
+      "logits/rejected": -3.0011003017425537,
+      "logps/chosen": -61.869049072265625,
+      "logps/rejected": -64.00714874267578,
+      "loss": 0.6788,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0639178603887558,
+      "rewards/margins": 0.030895311385393143,
+      "rewards/rejected": -0.09481316804885864,
+      "step": 4090
+    },
+    {
+      "epoch": 0.7064093728463129,
+      "grad_norm": 2.721099615097046,
+      "learning_rate": 4.085529148003945e-08,
+      "logits/chosen": -2.993710994720459,
+      "logits/rejected": -2.9647717475891113,
+      "logps/chosen": -58.31772994995117,
+      "logps/rejected": -58.77184295654297,
+      "loss": 0.6741,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.05474283546209335,
+      "rewards/margins": 0.0405287928879261,
+      "rewards/rejected": -0.09527162462472916,
+      "step": 4100
+    },
+    {
+      "epoch": 0.7064093728463129,
+      "eval_logits/chosen": -3.0755438804626465,
+      "eval_logits/rejected": -3.0698182582855225,
+      "eval_logps/chosen": -62.49949264526367,
+      "eval_logps/rejected": -68.21248626708984,
+      "eval_loss": 0.6874942183494568,
+      "eval_rewards/accuracies": 0.5922397971153259,
+      "eval_rewards/chosen": -0.03787597268819809,
+      "eval_rewards/margins": 0.012447633780539036,
+      "eval_rewards/rejected": -0.05032360553741455,
+      "eval_runtime": 383.2827,
+      "eval_samples_per_second": 11.229,
+      "eval_steps_per_second": 1.404,
+      "step": 4100
+    },
+    {
+      "epoch": 0.708132322536182,
+      "grad_norm": 2.7906482219696045,
+      "learning_rate": 4.079709428812842e-08,
+      "logits/chosen": -2.9597997665405273,
+      "logits/rejected": -2.948230266571045,
+      "logps/chosen": -63.466209411621094,
+      "logps/rejected": -60.81968307495117,
+      "loss": 0.6845,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.07403786480426788,
+      "rewards/margins": 0.019126856699585915,
+      "rewards/rejected": -0.09316471964120865,
+      "step": 4110
+    },
+    {
+      "epoch": 0.709855272226051,
+      "grad_norm": 3.098349094390869,
+      "learning_rate": 4.073875424187739e-08,
+      "logits/chosen": -2.9166245460510254,
+      "logits/rejected": -2.9196839332580566,
+      "logps/chosen": -58.108375549316406,
+      "logps/rejected": -64.48966979980469,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.08143486082553864,
+      "rewards/margins": 0.015587328001856804,
+      "rewards/rejected": -0.09702219069004059,
+      "step": 4120
+    },
+    {
+      "epoch": 0.71157822191592,
+      "grad_norm": 2.863992929458618,
+      "learning_rate": 4.06802718688599e-08,
+      "logits/chosen": -3.0177383422851562,
+      "logits/rejected": -2.994535446166992,
+      "logps/chosen": -60.27470016479492,
+      "logps/rejected": -62.57825469970703,
+      "loss": 0.6753,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.06306761503219604,
+      "rewards/margins": 0.037384577095508575,
+      "rewards/rejected": -0.10045219957828522,
+      "step": 4130
+    },
+    {
+      "epoch": 0.7133011716057891,
+      "grad_norm": 2.577974557876587,
+      "learning_rate": 4.0621647697936555e-08,
+      "logits/chosen": -3.021245002746582,
+      "logits/rejected": -2.992985248565674,
+      "logps/chosen": -62.58727264404297,
+      "logps/rejected": -59.7284049987793,
+      "loss": 0.6843,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07061640173196793,
+      "rewards/margins": 0.01963687315583229,
+      "rewards/rejected": -0.09025327861309052,
+      "step": 4140
+    },
+    {
+      "epoch": 0.7150241212956582,
+      "grad_norm": 2.7020983695983887,
+      "learning_rate": 4.056288225925023e-08,
+      "logits/chosen": -3.041928768157959,
+      "logits/rejected": -3.0152289867401123,
+      "logps/chosen": -64.84591674804688,
+      "logps/rejected": -62.09287643432617,
+      "loss": 0.6778,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.07004263252019882,
+      "rewards/margins": 0.03264743834733963,
+      "rewards/rejected": -0.10269006341695786,
+      "step": 4150
+    },
+    {
+      "epoch": 0.7167470709855273,
+      "grad_norm": 2.6901121139526367,
+      "learning_rate": 4.050397608422132e-08,
+      "logits/chosen": -2.934506893157959,
+      "logits/rejected": -2.9032063484191895,
+      "logps/chosen": -59.04931640625,
+      "logps/rejected": -59.49287033081055,
+      "loss": 0.6792,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.07371102273464203,
+      "rewards/margins": 0.03009924665093422,
+      "rewards/rejected": -0.10381027311086655,
+      "step": 4160
+    },
+    {
+      "epoch": 0.7184700206753962,
+      "grad_norm": 2.9999139308929443,
+      "learning_rate": 4.044492970554292e-08,
+      "logits/chosen": -2.9868712425231934,
+      "logits/rejected": -2.978951930999756,
+      "logps/chosen": -58.512664794921875,
+      "logps/rejected": -62.911048889160156,
+      "loss": 0.6831,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.07116648554801941,
+      "rewards/margins": 0.022090468555688858,
+      "rewards/rejected": -0.09325695782899857,
+      "step": 4170
+    },
+    {
+      "epoch": 0.7201929703652653,
+      "grad_norm": 2.8827977180480957,
+      "learning_rate": 4.038574365717594e-08,
+      "logits/chosen": -2.986929416656494,
+      "logits/rejected": -2.972083330154419,
+      "logps/chosen": -62.00639724731445,
+      "logps/rejected": -65.6218490600586,
+      "loss": 0.6808,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.06660818308591843,
+      "rewards/margins": 0.02672097086906433,
+      "rewards/rejected": -0.09332916140556335,
+      "step": 4180
+    },
+    {
+      "epoch": 0.7219159200551344,
+      "grad_norm": 2.713106870651245,
+      "learning_rate": 4.0326418474344414e-08,
+      "logits/chosen": -2.96708607673645,
+      "logits/rejected": -2.953481435775757,
+      "logps/chosen": -58.287879943847656,
+      "logps/rejected": -62.633094787597656,
+      "loss": 0.6798,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.07219740003347397,
+      "rewards/margins": 0.02892964519560337,
+      "rewards/rejected": -0.10112704336643219,
+      "step": 4190
+    },
+    {
+      "epoch": 0.7236388697450035,
+      "grad_norm": 2.8900680541992188,
+      "learning_rate": 4.026695469353051e-08,
+      "logits/chosen": -3.018639087677002,
+      "logits/rejected": -3.002345561981201,
+      "logps/chosen": -62.337677001953125,
+      "logps/rejected": -61.29419708251953,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.06631200015544891,
+      "rewards/margins": 0.020741142332553864,
+      "rewards/rejected": -0.08705313503742218,
+      "step": 4200
+    },
+    {
+      "epoch": 0.7236388697450035,
+      "eval_logits/chosen": -3.0720202922821045,
+      "eval_logits/rejected": -3.0662994384765625,
+      "eval_logps/chosen": -62.69786071777344,
+      "eval_logps/rejected": -68.43623352050781,
+      "eval_loss": 0.687391996383667,
+      "eval_rewards/accuracies": 0.5952602028846741,
+      "eval_rewards/chosen": -0.03985972702503204,
+      "eval_rewards/margins": 0.012701462022960186,
+      "eval_rewards/rejected": -0.052561189979314804,
+      "eval_runtime": 382.9172,
+      "eval_samples_per_second": 11.24,
+      "eval_steps_per_second": 1.405,
+      "step": 4200
+    },
+    {
+      "epoch": 0.7253618194348725,
+      "grad_norm": 3.0406978130340576,
+      "learning_rate": 4.020735285246979e-08,
+      "logits/chosen": -2.9998621940612793,
+      "logits/rejected": -2.9769227504730225,
+      "logps/chosen": -62.87953567504883,
+      "logps/rejected": -63.29198455810547,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.07582788914442062,
+      "rewards/margins": 0.02934197150170803,
+      "rewards/rejected": -0.1051698699593544,
+      "step": 4210
+    },
+    {
+      "epoch": 0.7270847691247415,
+      "grad_norm": 2.779994487762451,
+      "learning_rate": 4.0147613490146285e-08,
+      "logits/chosen": -2.854778528213501,
+      "logits/rejected": -2.8360886573791504,
+      "logps/chosen": -59.31269454956055,
+      "logps/rejected": -62.579986572265625,
+      "loss": 0.6803,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.07094010710716248,
+      "rewards/margins": 0.027808677405118942,
+      "rewards/rejected": -0.09874878078699112,
+      "step": 4220
+    },
+    {
+      "epoch": 0.7288077188146106,
+      "grad_norm": 2.6176230907440186,
+      "learning_rate": 4.0087737146787653e-08,
+      "logits/chosen": -3.03908634185791,
+      "logits/rejected": -3.018826723098755,
+      "logps/chosen": -62.5227165222168,
+      "logps/rejected": -64.32291412353516,
+      "loss": 0.6793,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0721665769815445,
+      "rewards/margins": 0.0300659891217947,
+      "rewards/rejected": -0.10223256051540375,
+      "step": 4230
+    },
+    {
+      "epoch": 0.7305306685044797,
+      "grad_norm": 2.754491090774536,
+      "learning_rate": 4.002772436386027e-08,
+      "logits/chosen": -2.9413435459136963,
+      "logits/rejected": -2.9169068336486816,
+      "logps/chosen": -61.72612380981445,
+      "logps/rejected": -66.4214096069336,
+      "loss": 0.6769,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.0698201134800911,
+      "rewards/margins": 0.03485408425331116,
+      "rewards/rejected": -0.10467419773340225,
+      "step": 4240
+    },
+    {
+      "epoch": 0.7322536181943488,
+      "grad_norm": 2.919862985610962,
+      "learning_rate": 3.996757568406437e-08,
+      "logits/chosen": -2.9614784717559814,
+      "logits/rejected": -2.945136547088623,
+      "logps/chosen": -62.63618087768555,
+      "logps/rejected": -62.29045486450195,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.07587381452322006,
+      "rewards/margins": 0.020346181467175484,
+      "rewards/rejected": -0.096220001578331,
+      "step": 4250
+    },
+    {
+      "epoch": 0.7339765678842178,
+      "grad_norm": 2.8628578186035156,
+      "learning_rate": 3.990729165132907e-08,
+      "logits/chosen": -2.924016237258911,
+      "logits/rejected": -2.908646821975708,
+      "logps/chosen": -61.55481719970703,
+      "logps/rejected": -64.23966217041016,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.07026652246713638,
+      "rewards/margins": 0.023592906072735786,
+      "rewards/rejected": -0.09385941922664642,
+      "step": 4260
+    },
+    {
+      "epoch": 0.7356995175740868,
+      "grad_norm": 2.8779003620147705,
+      "learning_rate": 3.9846872810807543e-08,
+      "logits/chosen": -2.9434211254119873,
+      "logits/rejected": -2.9178109169006348,
+      "logps/chosen": -57.59919357299805,
+      "logps/rejected": -60.01405715942383,
+      "loss": 0.6769,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.05990757793188095,
+      "rewards/margins": 0.034761738032102585,
+      "rewards/rejected": -0.09466931968927383,
+      "step": 4270
+    },
+    {
+      "epoch": 0.7374224672639559,
+      "grad_norm": 2.6272737979888916,
+      "learning_rate": 3.978631970887201e-08,
+      "logits/chosen": -2.917940616607666,
+      "logits/rejected": -2.8910393714904785,
+      "logps/chosen": -59.134498596191406,
+      "logps/rejected": -62.644691467285156,
+      "loss": 0.6784,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.07416973263025284,
+      "rewards/margins": 0.03172297403216362,
+      "rewards/rejected": -0.10589270293712616,
+      "step": 4280
+    },
+    {
+      "epoch": 0.739145416953825,
+      "grad_norm": 2.679978609085083,
+      "learning_rate": 3.9725632893108816e-08,
+      "logits/chosen": -2.968236207962036,
+      "logits/rejected": -2.944017171859741,
+      "logps/chosen": -62.491905212402344,
+      "logps/rejected": -61.7115364074707,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.07252033054828644,
+      "rewards/margins": 0.028450122103095055,
+      "rewards/rejected": -0.10097044706344604,
+      "step": 4290
+    },
+    {
+      "epoch": 0.740868366643694,
+      "grad_norm": 2.6929240226745605,
+      "learning_rate": 3.9664812912313536e-08,
+      "logits/chosen": -2.9825527667999268,
+      "logits/rejected": -2.9718177318573,
+      "logps/chosen": -60.1163444519043,
+      "logps/rejected": -62.9154167175293,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0756002813577652,
+      "rewards/margins": 0.02290363982319832,
+      "rewards/rejected": -0.09850391745567322,
+      "step": 4300
+    },
+    {
+      "epoch": 0.740868366643694,
+      "eval_logits/chosen": -3.0688538551330566,
+      "eval_logits/rejected": -3.063112497329712,
+      "eval_logps/chosen": -62.78386306762695,
+      "eval_logps/rejected": -68.57715606689453,
+      "eval_loss": 0.6871354579925537,
+      "eval_rewards/accuracies": 0.5959572196006775,
+      "eval_rewards/chosen": -0.040719665586948395,
+      "eval_rewards/margins": 0.013250667601823807,
+      "eval_rewards/rejected": -0.0539703369140625,
+      "eval_runtime": 383.3046,
+      "eval_samples_per_second": 11.229,
+      "eval_steps_per_second": 1.404,
+      "step": 4300
+    },
+    {
+      "epoch": 0.742591316333563,
+      "grad_norm": 3.0209193229675293,
+      "learning_rate": 3.960386031648592e-08,
+      "logits/chosen": -2.9321722984313965,
+      "logits/rejected": -2.912781238555908,
+      "logps/chosen": -60.87428665161133,
+      "logps/rejected": -61.4201774597168,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.06822346150875092,
+      "rewards/margins": 0.03010530397295952,
+      "rewards/rejected": -0.09832876175642014,
+      "step": 4310
+    },
+    {
+      "epoch": 0.7443142660234321,
+      "grad_norm": 2.5176730155944824,
+      "learning_rate": 3.9542775656825e-08,
+      "logits/chosen": -3.0085463523864746,
+      "logits/rejected": -2.9806668758392334,
+      "logps/chosen": -66.06652069091797,
+      "logps/rejected": -65.59309387207031,
+      "loss": 0.6767,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.0742451399564743,
+      "rewards/margins": 0.034913886338472366,
+      "rewards/rejected": -0.10915901511907578,
+      "step": 4320
+    },
+    {
+      "epoch": 0.7460372157133012,
+      "grad_norm": 2.9192137718200684,
+      "learning_rate": 3.9481559485724046e-08,
+      "logits/chosen": -2.9154465198516846,
+      "logits/rejected": -2.8837738037109375,
+      "logps/chosen": -63.7230224609375,
+      "logps/rejected": -60.057098388671875,
+      "loss": 0.684,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08232223242521286,
+      "rewards/margins": 0.020322374999523163,
+      "rewards/rejected": -0.10264460742473602,
+      "step": 4330
+    },
+    {
+      "epoch": 0.7477601654031703,
+      "grad_norm": 2.6742444038391113,
+      "learning_rate": 3.942021235676561e-08,
+      "logits/chosen": -2.9199399948120117,
+      "logits/rejected": -2.901151657104492,
+      "logps/chosen": -57.373191833496094,
+      "logps/rejected": -62.356788635253906,
+      "loss": 0.6771,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08376865088939667,
+      "rewards/margins": 0.03481664881110191,
+      "rewards/rejected": -0.11858530342578888,
+      "step": 4340
+    },
+    {
+      "epoch": 0.7494831150930393,
+      "grad_norm": 2.821366548538208,
+      "learning_rate": 3.93587348247165e-08,
+      "logits/chosen": -2.931494951248169,
+      "logits/rejected": -2.9172167778015137,
+      "logps/chosen": -57.559600830078125,
+      "logps/rejected": -62.69353103637695,
+      "loss": 0.6795,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.06549771875143051,
+      "rewards/margins": 0.029477929696440697,
+      "rewards/rejected": -0.09497565031051636,
+      "step": 4350
+    },
+    {
+      "epoch": 0.7512060647829083,
+      "grad_norm": 2.872011423110962,
+      "learning_rate": 3.929712744552278e-08,
+      "logits/chosen": -2.980806350708008,
+      "logits/rejected": -2.959137439727783,
+      "logps/chosen": -62.00044631958008,
+      "logps/rejected": -62.60979080200195,
+      "loss": 0.681,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0832880288362503,
+      "rewards/margins": 0.02630927599966526,
+      "rewards/rejected": -0.10959731042385101,
+      "step": 4360
+    },
+    {
+      "epoch": 0.7529290144727774,
+      "grad_norm": 2.5919382572174072,
+      "learning_rate": 3.923539077630471e-08,
+      "logits/chosen": -2.9942774772644043,
+      "logits/rejected": -2.9795749187469482,
+      "logps/chosen": -62.52122116088867,
+      "logps/rejected": -65.0046615600586,
+      "loss": 0.681,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.07562626898288727,
+      "rewards/margins": 0.026432711631059647,
+      "rewards/rejected": -0.10205898433923721,
+      "step": 4370
+    },
+    {
+      "epoch": 0.7546519641626465,
+      "grad_norm": 2.8279430866241455,
+      "learning_rate": 3.917352537535176e-08,
+      "logits/chosen": -3.014645576477051,
+      "logits/rejected": -2.9911980628967285,
+      "logps/chosen": -61.656166076660156,
+      "logps/rejected": -61.64357376098633,
+      "loss": 0.6835,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.07352598756551743,
+      "rewards/margins": 0.021115479990839958,
+      "rewards/rejected": -0.09464148432016373,
+      "step": 4380
+    },
+    {
+      "epoch": 0.7563749138525155,
+      "grad_norm": 2.538715124130249,
+      "learning_rate": 3.91115318021175e-08,
+      "logits/chosen": -2.941464900970459,
+      "logits/rejected": -2.927806854248047,
+      "logps/chosen": -61.001800537109375,
+      "logps/rejected": -64.02394104003906,
+      "loss": 0.6804,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.07765157520771027,
+      "rewards/margins": 0.027613604441285133,
+      "rewards/rejected": -0.10526517778635025,
+      "step": 4390
+    },
+    {
+      "epoch": 0.7580978635423845,
+      "grad_norm": 2.7481465339660645,
+      "learning_rate": 3.9049410617214604e-08,
+      "logits/chosen": -2.9618136882781982,
+      "logits/rejected": -2.944315195083618,
+      "logps/chosen": -63.8530158996582,
+      "logps/rejected": -65.01741790771484,
+      "loss": 0.681,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.06947740912437439,
+      "rewards/margins": 0.026488369330763817,
+      "rewards/rejected": -0.09596577286720276,
+      "step": 4400
+    },
+    {
+      "epoch": 0.7580978635423845,
+      "eval_logits/chosen": -3.0659899711608887,
+      "eval_logits/rejected": -3.0602529048919678,
+      "eval_logps/chosen": -62.9919548034668,
+      "eval_logps/rejected": -68.79925537109375,
+      "eval_loss": 0.687089741230011,
+      "eval_rewards/accuracies": 0.5938661694526672,
+      "eval_rewards/chosen": -0.042800575494766235,
+      "eval_rewards/margins": 0.013390742242336273,
+      "eval_rewards/rejected": -0.05619131401181221,
+      "eval_runtime": 382.891,
+      "eval_samples_per_second": 11.241,
+      "eval_steps_per_second": 1.405,
+      "step": 4400
+    },
+    {
+      "epoch": 0.7598208132322536,
+      "grad_norm": 2.6842169761657715,
+      "learning_rate": 3.898716238240971e-08,
+      "logits/chosen": -2.9264450073242188,
+      "logits/rejected": -2.91013503074646,
+      "logps/chosen": -62.860565185546875,
+      "logps/rejected": -65.85084533691406,
+      "loss": 0.68,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.07968903332948685,
+      "rewards/margins": 0.02851291373372078,
+      "rewards/rejected": -0.10820194333791733,
+      "step": 4410
+    },
+    {
+      "epoch": 0.7615437629221227,
+      "grad_norm": 2.7951512336730957,
+      "learning_rate": 3.892478766061841e-08,
+      "logits/chosen": -3.044710159301758,
+      "logits/rejected": -3.0004498958587646,
+      "logps/chosen": -61.93156051635742,
+      "logps/rejected": -61.21696090698242,
+      "loss": 0.6764,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.07115557044744492,
+      "rewards/margins": 0.03631502017378807,
+      "rewards/rejected": -0.10747058689594269,
+      "step": 4420
+    },
+    {
+      "epoch": 0.7632667126119917,
+      "grad_norm": 2.416335344314575,
+      "learning_rate": 3.886228701590011e-08,
+      "logits/chosen": -2.9367547035217285,
+      "logits/rejected": -2.9052672386169434,
+      "logps/chosen": -61.77679443359375,
+      "logps/rejected": -61.63386154174805,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.0799412876367569,
+      "rewards/margins": 0.023188332095742226,
+      "rewards/rejected": -0.10312961041927338,
+      "step": 4430
+    },
+    {
+      "epoch": 0.7649896623018608,
+      "grad_norm": 2.8418731689453125,
+      "learning_rate": 3.879966101345296e-08,
+      "logits/chosen": -3.002845525741577,
+      "logits/rejected": -2.971587896347046,
+      "logps/chosen": -66.99166107177734,
+      "logps/rejected": -66.14315032958984,
+      "loss": 0.6766,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.075896255671978,
+      "rewards/margins": 0.03530532866716385,
+      "rewards/rejected": -0.11120158433914185,
+      "step": 4440
+    },
+    {
+      "epoch": 0.7667126119917298,
+      "grad_norm": 2.794100284576416,
+      "learning_rate": 3.8736910219608706e-08,
+      "logits/chosen": -2.8756022453308105,
+      "logits/rejected": -2.8632168769836426,
+      "logps/chosen": -63.414825439453125,
+      "logps/rejected": -64.45621490478516,
+      "loss": 0.6815,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.07699565589427948,
+      "rewards/margins": 0.02569589391350746,
+      "rewards/rejected": -0.10269154608249664,
+      "step": 4450
+    },
+    {
+      "epoch": 0.7684355616815989,
+      "grad_norm": 2.966275691986084,
+      "learning_rate": 3.867403520182762e-08,
+      "logits/chosen": -2.988692045211792,
+      "logits/rejected": -2.9812755584716797,
+      "logps/chosen": -60.994110107421875,
+      "logps/rejected": -63.28435134887695,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.08262128382921219,
+      "rewards/margins": 0.02362518385052681,
+      "rewards/rejected": -0.1062464714050293,
+      "step": 4460
+    },
+    {
+      "epoch": 0.770158511371468,
+      "grad_norm": 2.8154702186584473,
+      "learning_rate": 3.861103652869334e-08,
+      "logits/chosen": -3.0256400108337402,
+      "logits/rejected": -2.9956679344177246,
+      "logps/chosen": -63.7596549987793,
+      "logps/rejected": -61.85748291015625,
+      "loss": 0.6755,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.0783766508102417,
+      "rewards/margins": 0.037811528891325,
+      "rewards/rejected": -0.1161881685256958,
+      "step": 4470
+    },
+    {
+      "epoch": 0.771881461061337,
+      "grad_norm": 2.878232479095459,
+      "learning_rate": 3.854791476990771e-08,
+      "logits/chosen": -3.052502155303955,
+      "logits/rejected": -3.0460922718048096,
+      "logps/chosen": -59.557472229003906,
+      "logps/rejected": -62.81451416015625,
+      "loss": 0.6841,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.08546684682369232,
+      "rewards/margins": 0.020212283357977867,
+      "rewards/rejected": -0.10567913204431534,
+      "step": 4480
+    },
+    {
+      "epoch": 0.7736044107512061,
+      "grad_norm": 2.899576187133789,
+      "learning_rate": 3.848467049628564e-08,
+      "logits/chosen": -2.942364454269409,
+      "logits/rejected": -2.914937734603882,
+      "logps/chosen": -59.479469299316406,
+      "logps/rejected": -61.19508743286133,
+      "loss": 0.6793,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0833863765001297,
+      "rewards/margins": 0.03019108809530735,
+      "rewards/rejected": -0.1135774701833725,
+      "step": 4490
+    },
+    {
+      "epoch": 0.7753273604410751,
+      "grad_norm": 3.2685701847076416,
+      "learning_rate": 3.842130427974998e-08,
+      "logits/chosen": -2.957045078277588,
+      "logits/rejected": -2.9423325061798096,
+      "logps/chosen": -64.41265869140625,
+      "logps/rejected": -63.36194610595703,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.08077342808246613,
+      "rewards/margins": 0.023212425410747528,
+      "rewards/rejected": -0.10398584604263306,
+      "step": 4500
+    },
+    {
+      "epoch": 0.7753273604410751,
+      "eval_logits/chosen": -3.062258720397949,
+      "eval_logits/rejected": -3.0565457344055176,
+      "eval_logps/chosen": -63.344627380371094,
+      "eval_logps/rejected": -69.22073364257812,
+      "eval_loss": 0.6867921352386475,
+      "eval_rewards/accuracies": 0.5931691527366638,
+      "eval_rewards/chosen": -0.046327367424964905,
+      "eval_rewards/margins": 0.014078744687139988,
+      "eval_rewards/rejected": -0.06040611490607262,
+      "eval_runtime": 383.1677,
+      "eval_samples_per_second": 11.233,
+      "eval_steps_per_second": 1.404,
+      "step": 4500
+    },
+    {
+      "epoch": 0.7770503101309442,
+      "grad_norm": 3.2590956687927246,
+      "learning_rate": 3.835781669332631e-08,
+      "logits/chosen": -3.071157932281494,
+      "logits/rejected": -3.0434021949768066,
+      "logps/chosen": -60.83068084716797,
+      "logps/rejected": -66.49485778808594,
+      "loss": 0.6768,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.0739826112985611,
+      "rewards/margins": 0.03479046747088432,
+      "rewards/rejected": -0.10877307504415512,
+      "step": 4510
+    },
+    {
+      "epoch": 0.7787732598208132,
+      "grad_norm": 3.092900276184082,
+      "learning_rate": 3.829420831113775e-08,
+      "logits/chosen": -2.9990553855895996,
+      "logits/rejected": -2.977864980697632,
+      "logps/chosen": -61.963409423828125,
+      "logps/rejected": -61.382606506347656,
+      "loss": 0.6846,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.09365353733301163,
+      "rewards/margins": 0.019364597275853157,
+      "rewards/rejected": -0.11301813274621964,
+      "step": 4520
+    },
+    {
+      "epoch": 0.7804962095106823,
+      "grad_norm": 3.0335867404937744,
+      "learning_rate": 3.823047970839981e-08,
+      "logits/chosen": -2.9933671951293945,
+      "logits/rejected": -2.9845235347747803,
+      "logps/chosen": -62.74645233154297,
+      "logps/rejected": -63.59551239013672,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.08972060680389404,
+      "rewards/margins": 0.01141512393951416,
+      "rewards/rejected": -0.1011357456445694,
+      "step": 4530
+    },
+    {
+      "epoch": 0.7822191592005513,
+      "grad_norm": 3.1603808403015137,
+      "learning_rate": 3.816663146141514e-08,
+      "logits/chosen": -2.9051544666290283,
+      "logits/rejected": -2.8932433128356934,
+      "logps/chosen": -60.87034225463867,
+      "logps/rejected": -62.28226852416992,
+      "loss": 0.6783,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.08138547837734222,
+      "rewards/margins": 0.03208800032734871,
+      "rewards/rejected": -0.11347347497940063,
+      "step": 4540
+    },
+    {
+      "epoch": 0.7839421088904204,
+      "grad_norm": 3.228480100631714,
+      "learning_rate": 3.810266414756836e-08,
+      "logits/chosen": -3.0088953971862793,
+      "logits/rejected": -2.985595226287842,
+      "logps/chosen": -64.1148452758789,
+      "logps/rejected": -64.74221801757812,
+      "loss": 0.6763,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.07756370306015015,
+      "rewards/margins": 0.03594299778342247,
+      "rewards/rejected": -0.11350671201944351,
+      "step": 4550
+    },
+    {
+      "epoch": 0.7856650585802895,
+      "grad_norm": 2.9145894050598145,
+      "learning_rate": 3.803857834532081e-08,
+      "logits/chosen": -2.906909465789795,
+      "logits/rejected": -2.869502305984497,
+      "logps/chosen": -62.88597869873047,
+      "logps/rejected": -64.16455841064453,
+      "loss": 0.6723,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.06668932735919952,
+      "rewards/margins": 0.044065989553928375,
+      "rewards/rejected": -0.1107553243637085,
+      "step": 4560
+    },
+    {
+      "epoch": 0.7873880082701585,
+      "grad_norm": 2.750051498413086,
+      "learning_rate": 3.7974374634205344e-08,
+      "logits/chosen": -2.933103084564209,
+      "logits/rejected": -2.9082539081573486,
+      "logps/chosen": -59.66290283203125,
+      "logps/rejected": -63.569908142089844,
+      "loss": 0.678,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.08572540432214737,
+      "rewards/margins": 0.032440848648548126,
+      "rewards/rejected": -0.1181662529706955,
+      "step": 4570
+    },
+    {
+      "epoch": 0.7891109579600276,
+      "grad_norm": 2.9855685234069824,
+      "learning_rate": 3.791005359482106e-08,
+      "logits/chosen": -2.9066781997680664,
+      "logits/rejected": -2.87978196144104,
+      "logps/chosen": -60.9447021484375,
+      "logps/rejected": -61.77702713012695,
+      "loss": 0.6774,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.06685087829828262,
+      "rewards/margins": 0.03330328315496445,
+      "rewards/rejected": -0.10015416145324707,
+      "step": 4580
+    },
+    {
+      "epoch": 0.7908339076498966,
+      "grad_norm": 3.152148962020874,
+      "learning_rate": 3.7845615808828055e-08,
+      "logits/chosen": -3.0044760704040527,
+      "logits/rejected": -2.976330280303955,
+      "logps/chosen": -63.10289764404297,
+      "logps/rejected": -63.216529846191406,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.08218622207641602,
+      "rewards/margins": 0.027914974838495255,
+      "rewards/rejected": -0.11010120064020157,
+      "step": 4590
+    },
+    {
+      "epoch": 0.7925568573397657,
+      "grad_norm": 2.7841618061065674,
+      "learning_rate": 3.7781061858942206e-08,
+      "logits/chosen": -2.8832943439483643,
+      "logits/rejected": -2.8638150691986084,
+      "logps/chosen": -62.109466552734375,
+      "logps/rejected": -67.11309051513672,
+      "loss": 0.6744,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.08744364976882935,
+      "rewards/margins": 0.040177009999752045,
+      "rewards/rejected": -0.127620667219162,
+      "step": 4600
+    },
+    {
+      "epoch": 0.7925568573397657,
+      "eval_logits/chosen": -3.059840440750122,
+      "eval_logits/rejected": -3.054107427597046,
+      "eval_logps/chosen": -63.59987258911133,
+      "eval_logps/rejected": -69.53276062011719,
+      "eval_loss": 0.6865466833114624,
+      "eval_rewards/accuracies": 0.5943308472633362,
+      "eval_rewards/chosen": -0.0488797090947628,
+      "eval_rewards/margins": 0.014646684750914574,
+      "eval_rewards/rejected": -0.06352639943361282,
+      "eval_runtime": 383.2931,
+      "eval_samples_per_second": 11.229,
+      "eval_steps_per_second": 1.404,
+      "step": 4600
+    },
+    {
+      "epoch": 0.7942798070296347,
+      "grad_norm": 2.762080430984497,
+      "learning_rate": 3.7716392328929864e-08,
+      "logits/chosen": -2.9162168502807617,
+      "logits/rejected": -2.9055216312408447,
+      "logps/chosen": -61.060157775878906,
+      "logps/rejected": -62.6396598815918,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.09922213852405548,
+      "rewards/margins": 0.017702821642160416,
+      "rewards/rejected": -0.1169249564409256,
+      "step": 4610
+    },
+    {
+      "epoch": 0.7960027567195038,
+      "grad_norm": 3.070054292678833,
+      "learning_rate": 3.765160780360254e-08,
+      "logits/chosen": -2.9464099407196045,
+      "logits/rejected": -2.923234224319458,
+      "logps/chosen": -63.55975341796875,
+      "logps/rejected": -66.6691665649414,
+      "loss": 0.6778,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.07841027528047562,
+      "rewards/margins": 0.033157579600811005,
+      "rewards/rejected": -0.11156786978244781,
+      "step": 4620
+    },
+    {
+      "epoch": 0.7977257064093728,
+      "grad_norm": 2.701042890548706,
+      "learning_rate": 3.7586708868811703e-08,
+      "logits/chosen": -2.987760066986084,
+      "logits/rejected": -2.953920602798462,
+      "logps/chosen": -64.90353393554688,
+      "logps/rejected": -67.55546569824219,
+      "loss": 0.6775,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.07619847357273102,
+      "rewards/margins": 0.03340502828359604,
+      "rewards/rejected": -0.10960348695516586,
+      "step": 4630
+    },
+    {
+      "epoch": 0.7994486560992419,
+      "grad_norm": 3.0437116622924805,
+      "learning_rate": 3.7521696111443416e-08,
+      "logits/chosen": -2.9908881187438965,
+      "logits/rejected": -2.97601318359375,
+      "logps/chosen": -64.6410903930664,
+      "logps/rejected": -67.6855697631836,
+      "loss": 0.6831,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0988648533821106,
+      "rewards/margins": 0.02248019352555275,
+      "rewards/rejected": -0.12134504318237305,
+      "step": 4640
+    },
+    {
+      "epoch": 0.801171605789111,
+      "grad_norm": 2.8858652114868164,
+      "learning_rate": 3.7456570119413035e-08,
+      "logits/chosen": -2.98907208442688,
+      "logits/rejected": -2.9561123847961426,
+      "logps/chosen": -62.654869079589844,
+      "logps/rejected": -63.3292236328125,
+      "loss": 0.6743,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.08504073321819305,
+      "rewards/margins": 0.04046284034848213,
+      "rewards/rejected": -0.1255035698413849,
+      "step": 4650
+    },
+    {
+      "epoch": 0.80289455547898,
+      "grad_norm": 2.9204647541046143,
+      "learning_rate": 3.739133148165994e-08,
+      "logits/chosen": -2.9786360263824463,
+      "logits/rejected": -2.960247039794922,
+      "logps/chosen": -62.31835174560547,
+      "logps/rejected": -65.58287048339844,
+      "loss": 0.678,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.08011087775230408,
+      "rewards/margins": 0.03293083235621452,
+      "rewards/rejected": -0.1130416989326477,
+      "step": 4660
+    },
+    {
+      "epoch": 0.8046175051688491,
+      "grad_norm": 2.5241215229034424,
+      "learning_rate": 3.732598078814215e-08,
+      "logits/chosen": -2.975822925567627,
+      "logits/rejected": -2.950880527496338,
+      "logps/chosen": -60.61690139770508,
+      "logps/rejected": -60.671913146972656,
+      "loss": 0.6776,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.08189544081687927,
+      "rewards/margins": 0.0338485911488533,
+      "rewards/rejected": -0.11574403196573257,
+      "step": 4670
+    },
+    {
+      "epoch": 0.8063404548587181,
+      "grad_norm": 2.74289608001709,
+      "learning_rate": 3.7260518629831006e-08,
+      "logits/chosen": -2.954606056213379,
+      "logits/rejected": -2.923494338989258,
+      "logps/chosen": -61.05181884765625,
+      "logps/rejected": -63.698219299316406,
+      "loss": 0.6773,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.08561243861913681,
+      "rewards/margins": 0.03447011113166809,
+      "rewards/rejected": -0.1200825423002243,
+      "step": 4680
+    },
+    {
+      "epoch": 0.8080634045485872,
+      "grad_norm": 3.2770135402679443,
+      "learning_rate": 3.7194945598705865e-08,
+      "logits/chosen": -3.0004029273986816,
+      "logits/rejected": -2.978562593460083,
+      "logps/chosen": -65.1228256225586,
+      "logps/rejected": -67.41207122802734,
+      "loss": 0.676,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.09359704703092575,
+      "rewards/margins": 0.037521880120038986,
+      "rewards/rejected": -0.13111893832683563,
+      "step": 4690
+    },
+    {
+      "epoch": 0.8097863542384562,
+      "grad_norm": 3.1083126068115234,
+      "learning_rate": 3.712926228774868e-08,
+      "logits/chosen": -2.974524974822998,
+      "logits/rejected": -2.9666621685028076,
+      "logps/chosen": -61.60593795776367,
+      "logps/rejected": -67.92273712158203,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.09871597588062286,
+      "rewards/margins": 0.023613417521119118,
+      "rewards/rejected": -0.12232939898967743,
+      "step": 4700
+    },
+    {
+      "epoch": 0.8097863542384562,
+      "eval_logits/chosen": -3.0568747520446777,
+      "eval_logits/rejected": -3.0511281490325928,
+      "eval_logps/chosen": -63.95634841918945,
+      "eval_logps/rejected": -69.95233154296875,
+      "eval_loss": 0.6862771511077881,
+      "eval_rewards/accuracies": 0.5989776849746704,
+      "eval_rewards/chosen": -0.052444539964199066,
+      "eval_rewards/margins": 0.015277646481990814,
+      "eval_rewards/rejected": -0.06772218644618988,
+      "eval_runtime": 382.9968,
+      "eval_samples_per_second": 11.238,
+      "eval_steps_per_second": 1.405,
+      "step": 4700
+    },
+    {
+      "epoch": 0.8115093039283253,
+      "grad_norm": 2.9480574131011963,
+      "learning_rate": 3.70634692909387e-08,
+      "logits/chosen": -3.02522611618042,
+      "logits/rejected": -2.9872002601623535,
+      "logps/chosen": -66.51214599609375,
+      "logps/rejected": -64.6211166381836,
+      "loss": 0.6764,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.08564888685941696,
+      "rewards/margins": 0.03673321753740311,
+      "rewards/rejected": -0.12238208949565887,
+      "step": 4710
+    },
+    {
+      "epoch": 0.8132322536181944,
+      "grad_norm": 2.9188199043273926,
+      "learning_rate": 3.699756720324706e-08,
+      "logits/chosen": -2.8988656997680664,
+      "logits/rejected": -2.8683784008026123,
+      "logps/chosen": -64.82237243652344,
+      "logps/rejected": -64.97240447998047,
+      "loss": 0.6722,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.0802006646990776,
+      "rewards/margins": 0.0451851561665535,
+      "rewards/rejected": -0.1253858506679535,
+      "step": 4720
+    },
+    {
+      "epoch": 0.8149552033080634,
+      "grad_norm": 2.9289486408233643,
+      "learning_rate": 3.693155662063141e-08,
+      "logits/chosen": -2.890758514404297,
+      "logits/rejected": -2.8653643131256104,
+      "logps/chosen": -63.50850296020508,
+      "logps/rejected": -63.32905197143555,
+      "loss": 0.6829,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.09762267768383026,
+      "rewards/margins": 0.023435968905687332,
+      "rewards/rejected": -0.12105865776538849,
+      "step": 4730
+    },
+    {
+      "epoch": 0.8166781529979324,
+      "grad_norm": 3.129823923110962,
+      "learning_rate": 3.686543814003053e-08,
+      "logits/chosen": -2.9791624546051025,
+      "logits/rejected": -2.968313694000244,
+      "logps/chosen": -62.63859939575195,
+      "logps/rejected": -65.29605865478516,
+      "loss": 0.6782,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08197047561407089,
+      "rewards/margins": 0.032283343374729156,
+      "rewards/rejected": -0.11425381898880005,
+      "step": 4740
+    },
+    {
+      "epoch": 0.8184011026878015,
+      "grad_norm": 3.0790274143218994,
+      "learning_rate": 3.6799212359358935e-08,
+      "logits/chosen": -2.9491429328918457,
+      "logits/rejected": -2.929957866668701,
+      "logps/chosen": -62.570594787597656,
+      "logps/rejected": -65.6785659790039,
+      "loss": 0.6749,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.07178530097007751,
+      "rewards/margins": 0.03906538337469101,
+      "rewards/rejected": -0.11085069179534912,
+      "step": 4750
+    },
+    {
+      "epoch": 0.8201240523776706,
+      "grad_norm": 3.2847981452941895,
+      "learning_rate": 3.673287987750146e-08,
+      "logits/chosen": -2.9462928771972656,
+      "logits/rejected": -2.924769878387451,
+      "logps/chosen": -62.06188201904297,
+      "logps/rejected": -64.01334381103516,
+      "loss": 0.6799,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.09589815884828568,
+      "rewards/margins": 0.028655197471380234,
+      "rewards/rejected": -0.12455336004495621,
+      "step": 4760
+    },
+    {
+      "epoch": 0.8218470020675396,
+      "grad_norm": 3.3376004695892334,
+      "learning_rate": 3.6666441294307835e-08,
+      "logits/chosen": -3.0105905532836914,
+      "logits/rejected": -2.989311695098877,
+      "logps/chosen": -68.61347961425781,
+      "logps/rejected": -66.96179962158203,
+      "loss": 0.6806,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.09023363888263702,
+      "rewards/margins": 0.027392562478780746,
+      "rewards/rejected": -0.11762620508670807,
+      "step": 4770
+    },
+    {
+      "epoch": 0.8235699517574087,
+      "grad_norm": 3.4045300483703613,
+      "learning_rate": 3.65998972105873e-08,
+      "logits/chosen": -2.884986162185669,
+      "logits/rejected": -2.8665640354156494,
+      "logps/chosen": -60.494529724121094,
+      "logps/rejected": -66.86381530761719,
+      "loss": 0.675,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.09024428576231003,
+      "rewards/margins": 0.03895661607384682,
+      "rewards/rejected": -0.12920090556144714,
+      "step": 4780
+    },
+    {
+      "epoch": 0.8252929014472777,
+      "grad_norm": 2.5735859870910645,
+      "learning_rate": 3.6533248228103114e-08,
+      "logits/chosen": -2.985891819000244,
+      "logits/rejected": -2.960270881652832,
+      "logps/chosen": -67.19239807128906,
+      "logps/rejected": -65.16417694091797,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.09401097893714905,
+      "rewards/margins": 0.02976982668042183,
+      "rewards/rejected": -0.12378080934286118,
+      "step": 4790
+    },
+    {
+      "epoch": 0.8270158511371468,
+      "grad_norm": 3.2274160385131836,
+      "learning_rate": 3.6466494949567175e-08,
+      "logits/chosen": -2.9101932048797607,
+      "logits/rejected": -2.8946692943573,
+      "logps/chosen": -61.12465286254883,
+      "logps/rejected": -61.92424392700195,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.09129075706005096,
+      "rewards/margins": 0.024655131623148918,
+      "rewards/rejected": -0.11594589054584503,
+      "step": 4800
+    },
+    {
+      "epoch": 0.8270158511371468,
+      "eval_logits/chosen": -3.0544254779815674,
+      "eval_logits/rejected": -3.0486838817596436,
+      "eval_logps/chosen": -64.3050308227539,
+      "eval_logps/rejected": -70.34410858154297,
+      "eval_loss": 0.6860989332199097,
+      "eval_rewards/accuracies": 0.5934014916419983,
+      "eval_rewards/chosen": -0.055931344628334045,
+      "eval_rewards/margins": 0.015708543360233307,
+      "eval_rewards/rejected": -0.07163988798856735,
+      "eval_runtime": 383.3104,
+      "eval_samples_per_second": 11.228,
+      "eval_steps_per_second": 1.404,
+      "step": 4800
+    },
+    {
+      "epoch": 0.8287388008270159,
+      "grad_norm": 3.084843873977661,
+      "learning_rate": 3.639963797863449e-08,
+      "logits/chosen": -2.921961545944214,
+      "logits/rejected": -2.8869853019714355,
+      "logps/chosen": -66.07090759277344,
+      "logps/rejected": -65.43196105957031,
+      "loss": 0.6738,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.08854615688323975,
+      "rewards/margins": 0.042032163590192795,
+      "rewards/rejected": -0.13057832419872284,
+      "step": 4810
+    },
+    {
+      "epoch": 0.8304617505168849,
+      "grad_norm": 2.857238531112671,
+      "learning_rate": 3.633267791989782e-08,
+      "logits/chosen": -2.9575302600860596,
+      "logits/rejected": -2.948070764541626,
+      "logps/chosen": -60.68806838989258,
+      "logps/rejected": -64.50486755371094,
+      "loss": 0.6788,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.09530703723430634,
+      "rewards/margins": 0.031566210091114044,
+      "rewards/rejected": -0.12687325477600098,
+      "step": 4820
+    },
+    {
+      "epoch": 0.832184700206754,
+      "grad_norm": 2.7476537227630615,
+      "learning_rate": 3.626561537888214e-08,
+      "logits/chosen": -2.9676928520202637,
+      "logits/rejected": -2.94512677192688,
+      "logps/chosen": -63.49809646606445,
+      "logps/rejected": -64.7082290649414,
+      "loss": 0.6838,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.0992119237780571,
+      "rewards/margins": 0.021593395620584488,
+      "rewards/rejected": -0.12080532312393188,
+      "step": 4830
+    },
+    {
+      "epoch": 0.833907649896623,
+      "grad_norm": 3.282191038131714,
+      "learning_rate": 3.6198450962039146e-08,
+      "logits/chosen": -2.925630569458008,
+      "logits/rejected": -2.8953135013580322,
+      "logps/chosen": -65.4959945678711,
+      "logps/rejected": -66.07198333740234,
+      "loss": 0.6731,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.09325645864009857,
+      "rewards/margins": 0.04333256930112839,
+      "rewards/rejected": -0.13658902049064636,
+      "step": 4840
+    },
+    {
+      "epoch": 0.8356305995864921,
+      "grad_norm": 2.9265682697296143,
+      "learning_rate": 3.613118527674184e-08,
+      "logits/chosen": -3.0008318424224854,
+      "logits/rejected": -2.9832568168640137,
+      "logps/chosen": -63.3024787902832,
+      "logps/rejected": -66.6944351196289,
+      "loss": 0.6751,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.09168411791324615,
+      "rewards/margins": 0.0391385592520237,
+      "rewards/rejected": -0.13082267343997955,
+      "step": 4850
+    },
+    {
+      "epoch": 0.8373535492763611,
+      "grad_norm": 3.0301127433776855,
+      "learning_rate": 3.6063818931279e-08,
+      "logits/chosen": -3.006028413772583,
+      "logits/rejected": -2.972626209259033,
+      "logps/chosen": -67.97903442382812,
+      "logps/rejected": -63.214637756347656,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.09521900862455368,
+      "rewards/margins": 0.031165260821580887,
+      "rewards/rejected": -0.12638425827026367,
+      "step": 4860
+    },
+    {
+      "epoch": 0.8390764989662302,
+      "grad_norm": 3.2389333248138428,
+      "learning_rate": 3.599635253484967e-08,
+      "logits/chosen": -3.02378249168396,
+      "logits/rejected": -2.9980101585388184,
+      "logps/chosen": -67.08110046386719,
+      "logps/rejected": -64.73031616210938,
+      "loss": 0.6763,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.09611519426107407,
+      "rewards/margins": 0.03626709803938866,
+      "rewards/rejected": -0.13238230347633362,
+      "step": 4870
+    },
+    {
+      "epoch": 0.8407994486560992,
+      "grad_norm": 2.6722793579101562,
+      "learning_rate": 3.5928786697557667e-08,
+      "logits/chosen": -2.9082553386688232,
+      "logits/rejected": -2.87309193611145,
+      "logps/chosen": -62.43416213989258,
+      "logps/rejected": -63.674835205078125,
+      "loss": 0.6754,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.09253338724374771,
+      "rewards/margins": 0.038005854934453964,
+      "rewards/rejected": -0.13053925335407257,
+      "step": 4880
+    },
+    {
+      "epoch": 0.8425223983459683,
+      "grad_norm": 3.1033287048339844,
+      "learning_rate": 3.586112203040607e-08,
+      "logits/chosen": -2.964315176010132,
+      "logits/rejected": -2.9426321983337402,
+      "logps/chosen": -64.45318603515625,
+      "logps/rejected": -66.62843322753906,
+      "loss": 0.6745,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.09032586216926575,
+      "rewards/margins": 0.04062148928642273,
+      "rewards/rejected": -0.13094735145568848,
+      "step": 4890
+    },
+    {
+      "epoch": 0.8442453480358374,
+      "grad_norm": 2.787128210067749,
+      "learning_rate": 3.579335914529166e-08,
+      "logits/chosen": -2.9342403411865234,
+      "logits/rejected": -2.9013447761535645,
+      "logps/chosen": -62.97007369995117,
+      "logps/rejected": -63.61249923706055,
+      "loss": 0.677,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.0976768285036087,
+      "rewards/margins": 0.036010511219501495,
+      "rewards/rejected": -0.1336873471736908,
+      "step": 4900
+    },
+    {
+      "epoch": 0.8442453480358374,
+      "eval_logits/chosen": -3.051394462585449,
+      "eval_logits/rejected": -3.04563570022583,
+      "eval_logps/chosen": -64.64353942871094,
+      "eval_logps/rejected": -70.75474548339844,
+      "eval_loss": 0.6857821941375732,
+      "eval_rewards/accuracies": 0.5922397971153259,
+      "eval_rewards/chosen": -0.05931641906499863,
+      "eval_rewards/margins": 0.016429824754595757,
+      "eval_rewards/rejected": -0.07574624568223953,
+      "eval_runtime": 383.0142,
+      "eval_samples_per_second": 11.237,
+      "eval_steps_per_second": 1.405,
+      "step": 4900
+    },
+    {
+      "epoch": 0.8459682977257064,
+      "grad_norm": 3.1621809005737305,
+      "learning_rate": 3.572549865499944e-08,
+      "logits/chosen": -3.0738797187805176,
+      "logits/rejected": -3.0422232151031494,
+      "logps/chosen": -65.54385375976562,
+      "logps/rejected": -65.21018981933594,
+      "loss": 0.6767,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.0920463427901268,
+      "rewards/margins": 0.03571182116866112,
+      "rewards/rejected": -0.1277581751346588,
+      "step": 4910
+    },
+    {
+      "epoch": 0.8476912474155754,
+      "grad_norm": 3.0529861450195312,
+      "learning_rate": 3.5657541173197025e-08,
+      "logits/chosen": -2.902818202972412,
+      "logits/rejected": -2.884861469268799,
+      "logps/chosen": -63.76279830932617,
+      "logps/rejected": -67.92424011230469,
+      "loss": 0.6721,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.0992293655872345,
+      "rewards/margins": 0.045357845723629,
+      "rewards/rejected": -0.1445872038602829,
+      "step": 4920
+    },
+    {
+      "epoch": 0.8494141971054445,
+      "grad_norm": 2.9861624240875244,
+      "learning_rate": 3.558948731442918e-08,
+      "logits/chosen": -3.0807108879089355,
+      "logits/rejected": -3.0689167976379395,
+      "logps/chosen": -63.81813430786133,
+      "logps/rejected": -71.7119140625,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.10675004869699478,
+      "rewards/margins": 0.03033631481230259,
+      "rewards/rejected": -0.13708636164665222,
+      "step": 4930
+    },
+    {
+      "epoch": 0.8511371467953136,
+      "grad_norm": 3.328608274459839,
+      "learning_rate": 3.5521337694112176e-08,
+      "logits/chosen": -3.0066285133361816,
+      "logits/rejected": -2.9769961833953857,
+      "logps/chosen": -66.49190521240234,
+      "logps/rejected": -67.34407806396484,
+      "loss": 0.6767,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.10570058971643448,
+      "rewards/margins": 0.03547225892543793,
+      "rewards/rejected": -0.1411728411912918,
+      "step": 4940
+    },
+    {
+      "epoch": 0.8528600964851827,
+      "grad_norm": 3.3148512840270996,
+      "learning_rate": 3.5453092928528286e-08,
+      "logits/chosen": -2.8109657764434814,
+      "logits/rejected": -2.7961106300354004,
+      "logps/chosen": -63.87224578857422,
+      "logps/rejected": -65.82002258300781,
+      "loss": 0.6811,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.09045035392045975,
+      "rewards/margins": 0.026566555723547935,
+      "rewards/rejected": -0.11701689660549164,
+      "step": 4950
+    },
+    {
+      "epoch": 0.8545830461750517,
+      "grad_norm": 2.719325065612793,
+      "learning_rate": 3.538475363482017e-08,
+      "logits/chosen": -2.9267661571502686,
+      "logits/rejected": -2.928370475769043,
+      "logps/chosen": -62.599456787109375,
+      "logps/rejected": -69.12854766845703,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.10345681756734848,
+      "rewards/margins": 0.02363778091967106,
+      "rewards/rejected": -0.1270945966243744,
+      "step": 4960
+    },
+    {
+      "epoch": 0.8563059958649207,
+      "grad_norm": 3.429021120071411,
+      "learning_rate": 3.531632043098533e-08,
+      "logits/chosen": -2.9128732681274414,
+      "logits/rejected": -2.9022717475891113,
+      "logps/chosen": -61.94645309448242,
+      "logps/rejected": -65.67939758300781,
+      "loss": 0.6784,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.10572998225688934,
+      "rewards/margins": 0.03250650316476822,
+      "rewards/rejected": -0.13823649287223816,
+      "step": 4970
+    },
+    {
+      "epoch": 0.8580289455547898,
+      "grad_norm": 2.654076099395752,
+      "learning_rate": 3.524779393587049e-08,
+      "logits/chosen": -2.9864518642425537,
+      "logits/rejected": -2.980220317840576,
+      "logps/chosen": -62.534149169921875,
+      "logps/rejected": -66.48970794677734,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.09676633775234222,
+      "rewards/margins": 0.019277844578027725,
+      "rewards/rejected": -0.11604418605566025,
+      "step": 4980
+    },
+    {
+      "epoch": 0.8597518952446589,
+      "grad_norm": 3.059110164642334,
+      "learning_rate": 3.517917476916604e-08,
+      "logits/chosen": -2.895920991897583,
+      "logits/rejected": -2.887972593307495,
+      "logps/chosen": -61.4960823059082,
+      "logps/rejected": -67.4173812866211,
+      "loss": 0.6806,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.09711698442697525,
+      "rewards/margins": 0.028186067938804626,
+      "rewards/rejected": -0.12530304491519928,
+      "step": 4990
+    },
+    {
+      "epoch": 0.8614748449345279,
+      "grad_norm": 2.893425226211548,
+      "learning_rate": 3.511046355140036e-08,
+      "logits/chosen": -2.927217960357666,
+      "logits/rejected": -2.9070544242858887,
+      "logps/chosen": -65.15436553955078,
+      "logps/rejected": -65.69318389892578,
+      "loss": 0.6765,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.0935039073228836,
+      "rewards/margins": 0.036374080926179886,
+      "rewards/rejected": -0.1298779845237732,
+      "step": 5000
+    },
+    {
+      "epoch": 0.8614748449345279,
+      "eval_logits/chosen": -3.04819393157959,
+      "eval_logits/rejected": -3.0424447059631348,
+      "eval_logps/chosen": -64.78231048583984,
+      "eval_logps/rejected": -70.91890716552734,
+      "eval_loss": 0.6856780648231506,
+      "eval_rewards/accuracies": 0.5934014916419983,
+      "eval_rewards/chosen": -0.06070411577820778,
+      "eval_rewards/margins": 0.016683679074048996,
+      "eval_rewards/rejected": -0.07738778740167618,
+      "eval_runtime": 382.9468,
+      "eval_samples_per_second": 11.239,
+      "eval_steps_per_second": 1.405,
+      "step": 5000
+    },
+    {
+      "epoch": 0.8631977946243969,
+      "grad_norm": 3.4637153148651123,
+      "learning_rate": 3.5041660903934306e-08,
+      "logits/chosen": -2.9237253665924072,
+      "logits/rejected": -2.8969008922576904,
+      "logps/chosen": -64.21942901611328,
+      "logps/rejected": -65.36909484863281,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.1116863489151001,
+      "rewards/margins": 0.030864771455526352,
+      "rewards/rejected": -0.14255113899707794,
+      "step": 5010
+    },
+    {
+      "epoch": 0.864920744314266,
+      "grad_norm": 3.098551034927368,
+      "learning_rate": 3.4972767448955513e-08,
+      "logits/chosen": -2.920698642730713,
+      "logits/rejected": -2.89512038230896,
+      "logps/chosen": -62.69829559326172,
+      "logps/rejected": -64.89715576171875,
+      "loss": 0.6764,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.09198883175849915,
+      "rewards/margins": 0.0370183028280735,
+      "rewards/rejected": -0.12900713086128235,
+      "step": 5020
+    },
+    {
+      "epoch": 0.8666436940041351,
+      "grad_norm": 3.0203561782836914,
+      "learning_rate": 3.490378380947279e-08,
+      "logits/chosen": -2.8700191974639893,
+      "logits/rejected": -2.8605918884277344,
+      "logps/chosen": -63.085838317871094,
+      "logps/rejected": -66.87300872802734,
+      "loss": 0.6822,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.1046866774559021,
+      "rewards/margins": 0.024456951767206192,
+      "rewards/rejected": -0.129143625497818,
+      "step": 5030
+    },
+    {
+      "epoch": 0.8683666436940042,
+      "grad_norm": 3.223085403442383,
+      "learning_rate": 3.483471060931051e-08,
+      "logits/chosen": -3.0959112644195557,
+      "logits/rejected": -3.054464101791382,
+      "logps/chosen": -66.9554443359375,
+      "logps/rejected": -63.43058395385742,
+      "loss": 0.6788,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.0979798287153244,
+      "rewards/margins": 0.03185773640871048,
+      "rewards/rejected": -0.12983758747577667,
+      "step": 5040
+    },
+    {
+      "epoch": 0.8700895933838731,
+      "grad_norm": 3.136145830154419,
+      "learning_rate": 3.476554847310294e-08,
+      "logits/chosen": -2.9676690101623535,
+      "logits/rejected": -2.9438183307647705,
+      "logps/chosen": -65.78553771972656,
+      "logps/rejected": -68.30848693847656,
+      "loss": 0.6776,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.10181204974651337,
+      "rewards/margins": 0.03420623391866684,
+      "rewards/rejected": -0.1360182762145996,
+      "step": 5050
+    },
+    {
+      "epoch": 0.8718125430737422,
+      "grad_norm": 2.7850277423858643,
+      "learning_rate": 3.4696298026288585e-08,
+      "logits/chosen": -2.8281092643737793,
+      "logits/rejected": -2.8221898078918457,
+      "logps/chosen": -61.24578857421875,
+      "logps/rejected": -65.58992004394531,
+      "loss": 0.6806,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.10773894935846329,
+      "rewards/margins": 0.027806958183646202,
+      "rewards/rejected": -0.13554592430591583,
+      "step": 5060
+    },
+    {
+      "epoch": 0.8735354927636113,
+      "grad_norm": 3.012092351913452,
+      "learning_rate": 3.462695989510459e-08,
+      "logits/chosen": -2.937713146209717,
+      "logits/rejected": -2.911863327026367,
+      "logps/chosen": -62.76781463623047,
+      "logps/rejected": -61.98115158081055,
+      "loss": 0.6753,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.10019499063491821,
+      "rewards/margins": 0.038344841450452805,
+      "rewards/rejected": -0.13853982090950012,
+      "step": 5070
+    },
+    {
+      "epoch": 0.8752584424534804,
+      "grad_norm": 3.0558197498321533,
+      "learning_rate": 3.4557534706580996e-08,
+      "logits/chosen": -3.1279244422912598,
+      "logits/rejected": -3.090226888656616,
+      "logps/chosen": -66.27345275878906,
+      "logps/rejected": -64.3001480102539,
+      "loss": 0.6716,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.08769071847200394,
+      "rewards/margins": 0.04638975113630295,
+      "rewards/rejected": -0.1340804547071457,
+      "step": 5080
+    },
+    {
+      "epoch": 0.8769813921433495,
+      "grad_norm": 3.2655441761016846,
+      "learning_rate": 3.448802308853515e-08,
+      "logits/chosen": -2.99532413482666,
+      "logits/rejected": -2.963634967803955,
+      "logps/chosen": -64.12982177734375,
+      "logps/rejected": -65.93929290771484,
+      "loss": 0.6717,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.0983911082148552,
+      "rewards/margins": 0.04599766805768013,
+      "rewards/rejected": -0.14438876509666443,
+      "step": 5090
+    },
+    {
+      "epoch": 0.8787043418332184,
+      "grad_norm": 3.103555917739868,
+      "learning_rate": 3.441842566956595e-08,
+      "logits/chosen": -2.83174729347229,
+      "logits/rejected": -2.7964928150177,
+      "logps/chosen": -65.63934326171875,
+      "logps/rejected": -66.02079772949219,
+      "loss": 0.6792,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.10572458803653717,
+      "rewards/margins": 0.031257860362529755,
+      "rewards/rejected": -0.13698245584964752,
+      "step": 5100
+    },
+    {
+      "epoch": 0.8787043418332184,
+      "eval_logits/chosen": -3.045055627822876,
+      "eval_logits/rejected": -3.0393154621124268,
+      "eval_logps/chosen": -65.13949584960938,
+      "eval_logps/rejected": -71.34764099121094,
+      "eval_loss": 0.6853744387626648,
+      "eval_rewards/accuracies": 0.5908457040786743,
+      "eval_rewards/chosen": -0.06427601724863052,
+      "eval_rewards/margins": 0.01739918440580368,
+      "eval_rewards/rejected": -0.0816752091050148,
+      "eval_runtime": 383.1786,
+      "eval_samples_per_second": 11.232,
+      "eval_steps_per_second": 1.404,
+      "step": 5100
+    },
+    {
+      "epoch": 0.8804272915230875,
+      "grad_norm": 3.2230236530303955,
+      "learning_rate": 3.434874307904822e-08,
+      "logits/chosen": -2.9752650260925293,
+      "logits/rejected": -2.9393346309661865,
+      "logps/chosen": -68.30015563964844,
+      "logps/rejected": -67.37004089355469,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.10427304357290268,
+      "rewards/margins": 0.031031746417284012,
+      "rewards/rejected": -0.1353047788143158,
+      "step": 5110
+    },
+    {
+      "epoch": 0.8821502412129566,
+      "grad_norm": 2.939256191253662,
+      "learning_rate": 3.427897594712699e-08,
+      "logits/chosen": -3.0306265354156494,
+      "logits/rejected": -2.9998245239257812,
+      "logps/chosen": -65.02655029296875,
+      "logps/rejected": -63.78974151611328,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.11153165251016617,
+      "rewards/margins": 0.023502185940742493,
+      "rewards/rejected": -0.13503384590148926,
+      "step": 5120
+    },
+    {
+      "epoch": 0.8838731909028257,
+      "grad_norm": 3.0386645793914795,
+      "learning_rate": 3.4209124904711807e-08,
+      "logits/chosen": -3.0498805046081543,
+      "logits/rejected": -3.016744613647461,
+      "logps/chosen": -63.69218826293945,
+      "logps/rejected": -63.81365966796875,
+      "loss": 0.6761,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.10126085579395294,
+      "rewards/margins": 0.03710508719086647,
+      "rewards/rejected": -0.1383659541606903,
+      "step": 5130
+    },
+    {
+      "epoch": 0.8855961405926946,
+      "grad_norm": 2.9432573318481445,
+      "learning_rate": 3.413919058347102e-08,
+      "logits/chosen": -2.9604434967041016,
+      "logits/rejected": -2.9238979816436768,
+      "logps/chosen": -66.90623474121094,
+      "logps/rejected": -66.09468078613281,
+      "loss": 0.6746,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.09087949246168137,
+      "rewards/margins": 0.04035644978284836,
+      "rewards/rejected": -0.13123594224452972,
+      "step": 5140
+    },
+    {
+      "epoch": 0.8873190902825637,
+      "grad_norm": 2.904973030090332,
+      "learning_rate": 3.40691736158261e-08,
+      "logits/chosen": -3.0018153190612793,
+      "logits/rejected": -2.9929990768432617,
+      "logps/chosen": -62.939552307128906,
+      "logps/rejected": -68.25997924804688,
+      "loss": 0.6766,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.0980975553393364,
+      "rewards/margins": 0.03576790541410446,
+      "rewards/rejected": -0.13386544585227966,
+      "step": 5150
+    },
+    {
+      "epoch": 0.8890420399724328,
+      "grad_norm": 3.0510683059692383,
+      "learning_rate": 3.399907463494585e-08,
+      "logits/chosen": -2.9710328578948975,
+      "logits/rejected": -2.935215950012207,
+      "logps/chosen": -62.36646270751953,
+      "logps/rejected": -62.82854080200195,
+      "loss": 0.6752,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.10949627310037613,
+      "rewards/margins": 0.03866765648126602,
+      "rewards/rejected": -0.14816391468048096,
+      "step": 5160
+    },
+    {
+      "epoch": 0.8907649896623019,
+      "grad_norm": 2.886368751525879,
+      "learning_rate": 3.392889427474077e-08,
+      "logits/chosen": -2.9627110958099365,
+      "logits/rejected": -2.9393908977508545,
+      "logps/chosen": -64.44051361083984,
+      "logps/rejected": -65.55287170410156,
+      "loss": 0.6721,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.091439388692379,
+      "rewards/margins": 0.04561624675989151,
+      "rewards/rejected": -0.1370556652545929,
+      "step": 5170
+    },
+    {
+      "epoch": 0.892487939352171,
+      "grad_norm": 3.6461808681488037,
+      "learning_rate": 3.385863316985726e-08,
+      "logits/chosen": -3.052621364593506,
+      "logits/rejected": -3.04278564453125,
+      "logps/chosen": -66.69034576416016,
+      "logps/rejected": -68.59880828857422,
+      "loss": 0.6808,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.11287333071231842,
+      "rewards/margins": 0.028028393164277077,
+      "rewards/rejected": -0.14090171456336975,
+      "step": 5180
+    },
+    {
+      "epoch": 0.8942108890420399,
+      "grad_norm": 3.4137489795684814,
+      "learning_rate": 3.3788291955671886e-08,
+      "logits/chosen": -2.9324746131896973,
+      "logits/rejected": -2.9214537143707275,
+      "logps/chosen": -63.736541748046875,
+      "logps/rejected": -66.77010345458984,
+      "loss": 0.6847,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.10896332561969757,
+      "rewards/margins": 0.019834989681839943,
+      "rewards/rejected": -0.12879832088947296,
+      "step": 5190
+    },
+    {
+      "epoch": 0.895933838731909,
+      "grad_norm": 2.8943235874176025,
+      "learning_rate": 3.371787126828568e-08,
+      "logits/chosen": -2.9941670894622803,
+      "logits/rejected": -2.9719367027282715,
+      "logps/chosen": -62.474098205566406,
+      "logps/rejected": -68.3436050415039,
+      "loss": 0.6752,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.10395494848489761,
+      "rewards/margins": 0.03885011747479439,
+      "rewards/rejected": -0.1428050696849823,
+      "step": 5200
+    },
+    {
+      "epoch": 0.895933838731909,
+      "eval_logits/chosen": -3.042752742767334,
+      "eval_logits/rejected": -3.036936044692993,
+      "eval_logps/chosen": -65.38580322265625,
+      "eval_logps/rejected": -71.62882995605469,
+      "eval_loss": 0.6852246522903442,
+      "eval_rewards/accuracies": 0.595724880695343,
+      "eval_rewards/chosen": -0.06673907488584518,
+      "eval_rewards/margins": 0.01774805411696434,
+      "eval_rewards/rejected": -0.08448711782693863,
+      "eval_runtime": 383.4139,
+      "eval_samples_per_second": 11.225,
+      "eval_steps_per_second": 1.403,
+      "step": 5200
+    },
+    {
+      "epoch": 0.8976567884217781,
+      "grad_norm": 2.8745086193084717,
+      "learning_rate": 3.3647371744518336e-08,
+      "logits/chosen": -2.9597458839416504,
+      "logits/rejected": -2.945530891418457,
+      "logps/chosen": -61.87052536010742,
+      "logps/rejected": -63.273406982421875,
+      "loss": 0.6792,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.1023128405213356,
+      "rewards/margins": 0.030416157096624374,
+      "rewards/rejected": -0.13272902369499207,
+      "step": 5210
+    },
+    {
+      "epoch": 0.8993797381116472,
+      "grad_norm": 3.2095768451690674,
+      "learning_rate": 3.3576794021902476e-08,
+      "logits/chosen": -2.986565589904785,
+      "logits/rejected": -2.973409652709961,
+      "logps/chosen": -62.5984001159668,
+      "logps/rejected": -68.31192779541016,
+      "loss": 0.6828,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.09987334907054901,
+      "rewards/margins": 0.023435372859239578,
+      "rewards/rejected": -0.12330871820449829,
+      "step": 5220
+    },
+    {
+      "epoch": 0.9011026878015161,
+      "grad_norm": 3.285916328430176,
+      "learning_rate": 3.350613873867788e-08,
+      "logits/chosen": -2.9264607429504395,
+      "logits/rejected": -2.9153189659118652,
+      "logps/chosen": -65.3592529296875,
+      "logps/rejected": -70.46281433105469,
+      "loss": 0.6739,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.10096422582864761,
+      "rewards/margins": 0.042541682720184326,
+      "rewards/rejected": -0.14350591599941254,
+      "step": 5230
+    },
+    {
+      "epoch": 0.9028256374913852,
+      "grad_norm": 3.246617555618286,
+      "learning_rate": 3.343540653378571e-08,
+      "logits/chosen": -2.9255661964416504,
+      "logits/rejected": -2.903221845626831,
+      "logps/chosen": -62.842430114746094,
+      "logps/rejected": -68.85960388183594,
+      "loss": 0.6676,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.10069449990987778,
+      "rewards/margins": 0.054425548762083054,
+      "rewards/rejected": -0.15512004494667053,
+      "step": 5240
+    },
+    {
+      "epoch": 0.9045485871812543,
+      "grad_norm": 2.739880084991455,
+      "learning_rate": 3.336459804686275e-08,
+      "logits/chosen": -2.8717594146728516,
+      "logits/rejected": -2.8575332164764404,
+      "logps/chosen": -62.514915466308594,
+      "logps/rejected": -65.03268432617188,
+      "loss": 0.6814,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.1153719425201416,
+      "rewards/margins": 0.02615945041179657,
+      "rewards/rejected": -0.14153139293193817,
+      "step": 5250
+    },
+    {
+      "epoch": 0.9062715368711234,
+      "grad_norm": 3.4216339588165283,
+      "learning_rate": 3.3293713918235594e-08,
+      "logits/chosen": -2.965236186981201,
+      "logits/rejected": -2.9239606857299805,
+      "logps/chosen": -66.67221069335938,
+      "logps/rejected": -65.8084487915039,
+      "loss": 0.6726,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.09886843711137772,
+      "rewards/margins": 0.04410909116268158,
+      "rewards/rejected": -0.1429775208234787,
+      "step": 5260
+    },
+    {
+      "epoch": 0.9079944865609925,
+      "grad_norm": 2.965733766555786,
+      "learning_rate": 3.3222754788914874e-08,
+      "logits/chosen": -3.060945510864258,
+      "logits/rejected": -3.046046733856201,
+      "logps/chosen": -62.63567352294922,
+      "logps/rejected": -66.13652801513672,
+      "loss": 0.675,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.11081449687480927,
+      "rewards/margins": 0.039091773331165314,
+      "rewards/rejected": -0.14990627765655518,
+      "step": 5270
+    },
+    {
+      "epoch": 0.9097174362508614,
+      "grad_norm": 3.131254196166992,
+      "learning_rate": 3.315172130058946e-08,
+      "logits/chosen": -2.964064836502075,
+      "logits/rejected": -2.9335336685180664,
+      "logps/chosen": -66.12247467041016,
+      "logps/rejected": -66.18555450439453,
+      "loss": 0.6761,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.11753039062023163,
+      "rewards/margins": 0.03762251138687134,
+      "rewards/rejected": -0.15515288710594177,
+      "step": 5280
+    },
+    {
+      "epoch": 0.9114403859407305,
+      "grad_norm": 3.027679443359375,
+      "learning_rate": 3.308061409562065e-08,
+      "logits/chosen": -2.9082133769989014,
+      "logits/rejected": -2.8638081550598145,
+      "logps/chosen": -65.77743530273438,
+      "logps/rejected": -65.99995422363281,
+      "loss": 0.6747,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.09921703487634659,
+      "rewards/margins": 0.04038939252495766,
+      "rewards/rejected": -0.13960641622543335,
+      "step": 5290
+    },
+    {
+      "epoch": 0.9131633356305996,
+      "grad_norm": 3.310492992401123,
+      "learning_rate": 3.300943381703639e-08,
+      "logits/chosen": -2.9168453216552734,
+      "logits/rejected": -2.9031195640563965,
+      "logps/chosen": -64.75244903564453,
+      "logps/rejected": -68.72442626953125,
+      "loss": 0.6752,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.09654269367456436,
+      "rewards/margins": 0.039225827902555466,
+      "rewards/rejected": -0.13576850295066833,
+      "step": 5300
+    },
+    {
+      "epoch": 0.9131633356305996,
+      "eval_logits/chosen": -3.039010763168335,
+      "eval_logits/rejected": -3.0332562923431396,
+      "eval_logps/chosen": -65.65825653076172,
+      "eval_logps/rejected": -71.9351806640625,
+      "eval_loss": 0.6850875020027161,
+      "eval_rewards/accuracies": 0.5910780429840088,
+      "eval_rewards/chosen": -0.06946365535259247,
+      "eval_rewards/margins": 0.018086979165673256,
+      "eval_rewards/rejected": -0.08755064755678177,
+      "eval_runtime": 383.0451,
+      "eval_samples_per_second": 11.236,
+      "eval_steps_per_second": 1.405,
+      "step": 5300
+    },
+    {
+      "epoch": 0.9148862853204687,
+      "grad_norm": 3.2121903896331787,
+      "learning_rate": 3.293818110852541e-08,
+      "logits/chosen": -3.047006368637085,
+      "logits/rejected": -3.035228967666626,
+      "logps/chosen": -68.60002899169922,
+      "logps/rejected": -66.37408447265625,
+      "loss": 0.6773,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.10466282069683075,
+      "rewards/margins": 0.035110954195261,
+      "rewards/rejected": -0.13977377116680145,
+      "step": 5310
+    },
+    {
+      "epoch": 0.9166092350103378,
+      "grad_norm": 2.828583002090454,
+      "learning_rate": 3.286685661443144e-08,
+      "logits/chosen": -2.9578301906585693,
+      "logits/rejected": -2.9045443534851074,
+      "logps/chosen": -68.4032974243164,
+      "logps/rejected": -65.54502868652344,
+      "loss": 0.6724,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.10849446058273315,
+      "rewards/margins": 0.04436909034848213,
+      "rewards/rejected": -0.152863547205925,
+      "step": 5320
+    },
+    {
+      "epoch": 0.9183321847002067,
+      "grad_norm": 3.070533275604248,
+      "learning_rate": 3.279546097974738e-08,
+      "logits/chosen": -2.8978967666625977,
+      "logits/rejected": -2.901106357574463,
+      "logps/chosen": -62.782936096191406,
+      "logps/rejected": -67.70713806152344,
+      "loss": 0.683,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.11929229646921158,
+      "rewards/margins": 0.023524750024080276,
+      "rewards/rejected": -0.14281703531742096,
+      "step": 5330
+    },
+    {
+      "epoch": 0.9200551343900758,
+      "grad_norm": 3.241379976272583,
+      "learning_rate": 3.272399485010943e-08,
+      "logits/chosen": -2.9366934299468994,
+      "logits/rejected": -2.884451389312744,
+      "logps/chosen": -66.99887084960938,
+      "logps/rejected": -66.48941802978516,
+      "loss": 0.6723,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.10636929422616959,
+      "rewards/margins": 0.04477467015385628,
+      "rewards/rejected": -0.15114395320415497,
+      "step": 5340
+    },
+    {
+      "epoch": 0.9217780840799449,
+      "grad_norm": 3.1132972240448,
+      "learning_rate": 3.265245887179133e-08,
+      "logits/chosen": -2.8886008262634277,
+      "logits/rejected": -2.852480411529541,
+      "logps/chosen": -64.51188659667969,
+      "logps/rejected": -67.63773345947266,
+      "loss": 0.676,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.1186944991350174,
+      "rewards/margins": 0.03796886280179024,
+      "rewards/rejected": -0.15666337311267853,
+      "step": 5350
+    },
+    {
+      "epoch": 0.923501033769814,
+      "grad_norm": 3.308631181716919,
+      "learning_rate": 3.2580853691698416e-08,
+      "logits/chosen": -3.005100727081299,
+      "logits/rejected": -2.998098850250244,
+      "logps/chosen": -63.81513595581055,
+      "logps/rejected": -70.41316223144531,
+      "loss": 0.6781,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.106010302901268,
+      "rewards/margins": 0.033305782824754715,
+      "rewards/rejected": -0.13931608200073242,
+      "step": 5360
+    },
+    {
+      "epoch": 0.9252239834596829,
+      "grad_norm": 3.2489013671875,
+      "learning_rate": 3.2509179957361865e-08,
+      "logits/chosen": -2.9063713550567627,
+      "logits/rejected": -2.8889074325561523,
+      "logps/chosen": -62.87049102783203,
+      "logps/rejected": -66.99803161621094,
+      "loss": 0.6816,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.10945756733417511,
+      "rewards/margins": 0.026481598615646362,
+      "rewards/rejected": -0.13593915104866028,
+      "step": 5370
+    },
+    {
+      "epoch": 0.926946933149552,
+      "grad_norm": 3.0641298294067383,
+      "learning_rate": 3.2437438316932765e-08,
+      "logits/chosen": -3.01530385017395,
+      "logits/rejected": -2.9757161140441895,
+      "logps/chosen": -67.37745666503906,
+      "logps/rejected": -66.40092468261719,
+      "loss": 0.6708,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.09979332983493805,
+      "rewards/margins": 0.04791291430592537,
+      "rewards/rejected": -0.14770624041557312,
+      "step": 5380
+    },
+    {
+      "epoch": 0.9286698828394211,
+      "grad_norm": 3.079801082611084,
+      "learning_rate": 3.2365629419176294e-08,
+      "logits/chosen": -2.9320549964904785,
+      "logits/rejected": -2.89007306098938,
+      "logps/chosen": -69.92964172363281,
+      "logps/rejected": -65.9744873046875,
+      "loss": 0.6798,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.11362340301275253,
+      "rewards/margins": 0.029988756403326988,
+      "rewards/rejected": -0.14361217617988586,
+      "step": 5390
+    },
+    {
+      "epoch": 0.9303928325292902,
+      "grad_norm": 2.838062047958374,
+      "learning_rate": 3.2293753913465856e-08,
+      "logits/chosen": -2.956315517425537,
+      "logits/rejected": -2.9388930797576904,
+      "logps/chosen": -62.12248611450195,
+      "logps/rejected": -68.49189758300781,
+      "loss": 0.6766,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.1101614236831665,
+      "rewards/margins": 0.035839568823575974,
+      "rewards/rejected": -0.1460009664297104,
+      "step": 5400
+    },
+    {
+      "epoch": 0.9303928325292902,
+      "eval_logits/chosen": -3.037017822265625,
+      "eval_logits/rejected": -3.031259775161743,
+      "eval_logps/chosen": -65.77825164794922,
+      "eval_logps/rejected": -72.1090316772461,
+      "eval_loss": 0.6848456859588623,
+      "eval_rewards/accuracies": 0.5973513126373291,
+      "eval_rewards/chosen": -0.07066360861063004,
+      "eval_rewards/margins": 0.01862553134560585,
+      "eval_rewards/rejected": -0.08928914368152618,
+      "eval_runtime": 382.9781,
+      "eval_samples_per_second": 11.238,
+      "eval_steps_per_second": 1.405,
+      "step": 5400
+    },
+    {
+      "epoch": 0.9321157822191593,
+      "grad_norm": 3.2152926921844482,
+      "learning_rate": 3.2221812449777164e-08,
+      "logits/chosen": -2.96891713142395,
+      "logits/rejected": -2.9570252895355225,
+      "logps/chosen": -68.37773132324219,
+      "logps/rejected": -67.91644287109375,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.11738348007202148,
+      "rewards/margins": 0.020904619246721268,
+      "rewards/rejected": -0.13828811049461365,
+      "step": 5410
+    },
+    {
+      "epoch": 0.9338387319090282,
+      "grad_norm": 3.524214029312134,
+      "learning_rate": 3.214980567868242e-08,
+      "logits/chosen": -3.022268772125244,
+      "logits/rejected": -3.0003955364227295,
+      "logps/chosen": -64.8818130493164,
+      "logps/rejected": -68.00125885009766,
+      "loss": 0.6793,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.11163567006587982,
+      "rewards/margins": 0.030855387449264526,
+      "rewards/rejected": -0.14249105751514435,
+      "step": 5420
+    },
+    {
+      "epoch": 0.9355616815988973,
+      "grad_norm": 2.811424493789673,
+      "learning_rate": 3.2077734251344407e-08,
+      "logits/chosen": -2.9444308280944824,
+      "logits/rejected": -2.9333183765411377,
+      "logps/chosen": -66.33551788330078,
+      "logps/rejected": -67.55891418457031,
+      "loss": 0.6793,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.12737420201301575,
+      "rewards/margins": 0.030707884579896927,
+      "rewards/rejected": -0.15808209776878357,
+      "step": 5430
+    },
+    {
+      "epoch": 0.9372846312887664,
+      "grad_norm": 3.1715621948242188,
+      "learning_rate": 3.200559881951059e-08,
+      "logits/chosen": -2.955134391784668,
+      "logits/rejected": -2.938178539276123,
+      "logps/chosen": -66.97594451904297,
+      "logps/rejected": -70.67919921875,
+      "loss": 0.6715,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.10774526745080948,
+      "rewards/margins": 0.046603165566921234,
+      "rewards/rejected": -0.1543484479188919,
+      "step": 5440
+    },
+    {
+      "epoch": 0.9390075809786355,
+      "grad_norm": 3.2779929637908936,
+      "learning_rate": 3.193340003550722e-08,
+      "logits/chosen": -2.8824093341827393,
+      "logits/rejected": -2.8709964752197266,
+      "logps/chosen": -67.60548400878906,
+      "logps/rejected": -69.1977767944336,
+      "loss": 0.6767,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.12026361376047134,
+      "rewards/margins": 0.036938928067684174,
+      "rewards/rejected": -0.15720254182815552,
+      "step": 5450
+    },
+    {
+      "epoch": 0.9407305306685044,
+      "grad_norm": 3.556159019470215,
+      "learning_rate": 3.186113855223348e-08,
+      "logits/chosen": -2.968726873397827,
+      "logits/rejected": -2.953226327896118,
+      "logps/chosen": -64.79072570800781,
+      "logps/rejected": -65.23482513427734,
+      "loss": 0.6831,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.1094733253121376,
+      "rewards/margins": 0.02276560664176941,
+      "rewards/rejected": -0.1322389394044876,
+      "step": 5460
+    },
+    {
+      "epoch": 0.9424534803583735,
+      "grad_norm": 3.377781867980957,
+      "learning_rate": 3.1788815023155517e-08,
+      "logits/chosen": -2.9218828678131104,
+      "logits/rejected": -2.902078628540039,
+      "logps/chosen": -63.620445251464844,
+      "logps/rejected": -65.23646545410156,
+      "loss": 0.6823,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.12261815369129181,
+      "rewards/margins": 0.0249023400247097,
+      "rewards/rejected": -0.147520512342453,
+      "step": 5470
+    },
+    {
+      "epoch": 0.9441764300482426,
+      "grad_norm": 2.96226167678833,
+      "learning_rate": 3.171643010230057e-08,
+      "logits/chosen": -2.948779344558716,
+      "logits/rejected": -2.9241623878479004,
+      "logps/chosen": -64.3949203491211,
+      "logps/rejected": -67.62200164794922,
+      "loss": 0.6724,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1145172119140625,
+      "rewards/margins": 0.04526824504137039,
+      "rewards/rejected": -0.1597854495048523,
+      "step": 5480
+    },
+    {
+      "epoch": 0.9458993797381117,
+      "grad_norm": 3.0583314895629883,
+      "learning_rate": 3.1643984444251056e-08,
+      "logits/chosen": -2.928798198699951,
+      "logits/rejected": -2.901287794113159,
+      "logps/chosen": -67.13384246826172,
+      "logps/rejected": -66.43314361572266,
+      "loss": 0.6817,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.11568702757358551,
+      "rewards/margins": 0.026522953063249588,
+      "rewards/rejected": -0.1422099769115448,
+      "step": 5490
+    },
+    {
+      "epoch": 0.9476223294279807,
+      "grad_norm": 3.267442226409912,
+      "learning_rate": 3.157147870413864e-08,
+      "logits/chosen": -3.0032730102539062,
+      "logits/rejected": -2.987034559249878,
+      "logps/chosen": -64.98506164550781,
+      "logps/rejected": -69.59332275390625,
+      "loss": 0.6761,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.11174207925796509,
+      "rewards/margins": 0.03753441199660301,
+      "rewards/rejected": -0.1492764800786972,
+      "step": 5500
+    },
+    {
+      "epoch": 0.9476223294279807,
+      "eval_logits/chosen": -3.034360408782959,
+      "eval_logits/rejected": -3.028585433959961,
+      "eval_logps/chosen": -65.88710021972656,
+      "eval_logps/rejected": -72.22322082519531,
+      "eval_loss": 0.6848300099372864,
+      "eval_rewards/accuracies": 0.5968866348266602,
+      "eval_rewards/chosen": -0.07175204902887344,
+      "eval_rewards/margins": 0.0186789408326149,
+      "eval_rewards/rejected": -0.09043098986148834,
+      "eval_runtime": 382.7227,
+      "eval_samples_per_second": 11.246,
+      "eval_steps_per_second": 1.406,
+      "step": 5500
+    },
+    {
+      "epoch": 0.9493452791178497,
+      "grad_norm": 3.024010181427002,
+      "learning_rate": 3.149891353763832e-08,
+      "logits/chosen": -2.912994623184204,
+      "logits/rejected": -2.8967056274414062,
+      "logps/chosen": -65.10884094238281,
+      "logps/rejected": -67.06105041503906,
+      "loss": 0.6823,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.11881420761346817,
+      "rewards/margins": 0.024477368220686913,
+      "rewards/rejected": -0.14329156279563904,
+      "step": 5510
+    },
+    {
+      "epoch": 0.9510682288077188,
+      "grad_norm": 3.6382317543029785,
+      "learning_rate": 3.142628960096246e-08,
+      "logits/chosen": -2.907003164291382,
+      "logits/rejected": -2.8814024925231934,
+      "logps/chosen": -66.17145538330078,
+      "logps/rejected": -66.17786407470703,
+      "loss": 0.6733,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.10347002744674683,
+      "rewards/margins": 0.04286672919988632,
+      "rewards/rejected": -0.14633674919605255,
+      "step": 5520
+    },
+    {
+      "epoch": 0.9527911784975879,
+      "grad_norm": 3.2115139961242676,
+      "learning_rate": 3.1353607550854935e-08,
+      "logits/chosen": -2.9616315364837646,
+      "logits/rejected": -2.9268949031829834,
+      "logps/chosen": -70.25215911865234,
+      "logps/rejected": -66.99165344238281,
+      "loss": 0.6769,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.10345442593097687,
+      "rewards/margins": 0.0357283391058445,
+      "rewards/rejected": -0.13918276131153107,
+      "step": 5530
+    },
+    {
+      "epoch": 0.954514128187457,
+      "grad_norm": 3.314854621887207,
+      "learning_rate": 3.12808680445851e-08,
+      "logits/chosen": -2.928626537322998,
+      "logits/rejected": -2.934112787246704,
+      "logps/chosen": -63.8646125793457,
+      "logps/rejected": -71.4774398803711,
+      "loss": 0.6785,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.11517419666051865,
+      "rewards/margins": 0.03258698433637619,
+      "rewards/rejected": -0.14776118099689484,
+      "step": 5540
+    },
+    {
+      "epoch": 0.956237077877326,
+      "grad_norm": 3.25010085105896,
+      "learning_rate": 3.120807173994194e-08,
+      "logits/chosen": -2.8098206520080566,
+      "logits/rejected": -2.79390287399292,
+      "logps/chosen": -64.23780822753906,
+      "logps/rejected": -68.62481689453125,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.11619962751865387,
+      "rewards/margins": 0.029282648116350174,
+      "rewards/rejected": -0.14548227190971375,
+      "step": 5550
+    },
+    {
+      "epoch": 0.957960027567195,
+      "grad_norm": 3.3916916847229004,
+      "learning_rate": 3.1135219295228014e-08,
+      "logits/chosen": -2.935326099395752,
+      "logits/rejected": -2.9091124534606934,
+      "logps/chosen": -63.62070846557617,
+      "logps/rejected": -64.52685546875,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.11733970791101456,
+      "rewards/margins": 0.03302709758281708,
+      "rewards/rejected": -0.15036679804325104,
+      "step": 5560
+    },
+    {
+      "epoch": 0.9596829772570641,
+      "grad_norm": 3.2920241355895996,
+      "learning_rate": 3.1062311369253604e-08,
+      "logits/chosen": -2.9766359329223633,
+      "logits/rejected": -2.967144727706909,
+      "logps/chosen": -61.848060607910156,
+      "logps/rejected": -71.1884536743164,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.11181558668613434,
+      "rewards/margins": 0.030701782554388046,
+      "rewards/rejected": -0.1425173580646515,
+      "step": 5570
+    },
+    {
+      "epoch": 0.9614059269469332,
+      "grad_norm": 2.9465503692626953,
+      "learning_rate": 3.0989348621330694e-08,
+      "logits/chosen": -2.850776195526123,
+      "logits/rejected": -2.8374733924865723,
+      "logps/chosen": -65.46949768066406,
+      "logps/rejected": -69.12590026855469,
+      "loss": 0.6795,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.12291932106018066,
+      "rewards/margins": 0.03047947585582733,
+      "rewards/rejected": -0.153398796916008,
+      "step": 5580
+    },
+    {
+      "epoch": 0.9631288766368022,
+      "grad_norm": 3.268836498260498,
+      "learning_rate": 3.091633171126703e-08,
+      "logits/chosen": -3.0223710536956787,
+      "logits/rejected": -2.9788646697998047,
+      "logps/chosen": -67.64588928222656,
+      "logps/rejected": -69.81709289550781,
+      "loss": 0.6756,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.10814741998910904,
+      "rewards/margins": 0.03850184381008148,
+      "rewards/rejected": -0.14664927124977112,
+      "step": 5590
+    },
+    {
+      "epoch": 0.9648518263266712,
+      "grad_norm": 3.0799717903137207,
+      "learning_rate": 3.0843261299360165e-08,
+      "logits/chosen": -2.9842476844787598,
+      "logits/rejected": -2.9818167686462402,
+      "logps/chosen": -64.10701751708984,
+      "logps/rejected": -71.81382751464844,
+      "loss": 0.68,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.11603100597858429,
+      "rewards/margins": 0.029025936499238014,
+      "rewards/rejected": -0.14505693316459656,
+      "step": 5600
+    },
+    {
+      "epoch": 0.9648518263266712,
+      "eval_logits/chosen": -3.032444715499878,
+      "eval_logits/rejected": -3.0266876220703125,
+      "eval_logps/chosen": -65.86900329589844,
+      "eval_logps/rejected": -72.22492980957031,
+      "eval_loss": 0.6847413182258606,
+      "eval_rewards/accuracies": 0.5992100238800049,
+      "eval_rewards/chosen": -0.07157105207443237,
+      "eval_rewards/margins": 0.018877046182751656,
+      "eval_rewards/rejected": -0.09044808894395828,
+      "eval_runtime": 383.2309,
+      "eval_samples_per_second": 11.231,
+      "eval_steps_per_second": 1.404,
+      "step": 5600
+    },
+    {
+      "epoch": 0.9665747760165403,
+      "grad_norm": 3.286726474761963,
+      "learning_rate": 3.077013804639144e-08,
+      "logits/chosen": -2.9934186935424805,
+      "logits/rejected": -2.975888729095459,
+      "logps/chosen": -63.889930725097656,
+      "logps/rejected": -71.9775161743164,
+      "loss": 0.675,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.11489413678646088,
+      "rewards/margins": 0.03929578512907028,
+      "rewards/rejected": -0.15418991446495056,
+      "step": 5610
+    },
+    {
+      "epoch": 0.9682977257064094,
+      "grad_norm": 3.469292640686035,
+      "learning_rate": 3.069696261362008e-08,
+      "logits/chosen": -2.9224209785461426,
+      "logits/rejected": -2.900421619415283,
+      "logps/chosen": -67.15715789794922,
+      "logps/rejected": -66.35441589355469,
+      "loss": 0.6764,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.12137474119663239,
+      "rewards/margins": 0.036920662969350815,
+      "rewards/rejected": -0.1582954078912735,
+      "step": 5620
+    },
+    {
+      "epoch": 0.9700206753962785,
+      "grad_norm": 3.268543243408203,
+      "learning_rate": 3.062373566277715e-08,
+      "logits/chosen": -2.9750144481658936,
+      "logits/rejected": -2.945526123046875,
+      "logps/chosen": -67.59455108642578,
+      "logps/rejected": -65.2641830444336,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.11686857789754868,
+      "rewards/margins": 0.024227596819400787,
+      "rewards/rejected": -0.14109618961811066,
+      "step": 5630
+    },
+    {
+      "epoch": 0.9717436250861475,
+      "grad_norm": 3.45585560798645,
+      "learning_rate": 3.0550457856059594e-08,
+      "logits/chosen": -2.9457907676696777,
+      "logits/rejected": -2.9322636127471924,
+      "logps/chosen": -63.965599060058594,
+      "logps/rejected": -69.56895446777344,
+      "loss": 0.6774,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.10019659996032715,
+      "rewards/margins": 0.03455501049757004,
+      "rewards/rejected": -0.1347516030073166,
+      "step": 5640
+    },
+    {
+      "epoch": 0.9734665747760165,
+      "grad_norm": 3.33117938041687,
+      "learning_rate": 3.047712985612428e-08,
+      "logits/chosen": -2.856759548187256,
+      "logits/rejected": -2.850398540496826,
+      "logps/chosen": -63.4826545715332,
+      "logps/rejected": -67.79341125488281,
+      "loss": 0.6743,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.11472135782241821,
+      "rewards/margins": 0.04117739945650101,
+      "rewards/rejected": -0.15589874982833862,
+      "step": 5650
+    },
+    {
+      "epoch": 0.9751895244658856,
+      "grad_norm": 3.1903717517852783,
+      "learning_rate": 3.040375232608194e-08,
+      "logits/chosen": -2.86580228805542,
+      "logits/rejected": -2.8627536296844482,
+      "logps/chosen": -63.67845916748047,
+      "logps/rejected": -66.02207946777344,
+      "loss": 0.679,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.11814604699611664,
+      "rewards/margins": 0.032066408544778824,
+      "rewards/rejected": -0.15021245181560516,
+      "step": 5660
+    },
+    {
+      "epoch": 0.9769124741557547,
+      "grad_norm": 3.37758207321167,
+      "learning_rate": 3.033032592949125e-08,
+      "logits/chosen": -2.9100098609924316,
+      "logits/rejected": -2.888782024383545,
+      "logps/chosen": -64.72911834716797,
+      "logps/rejected": -67.49930572509766,
+      "loss": 0.675,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.10686689615249634,
+      "rewards/margins": 0.039780668914318085,
+      "rewards/rejected": -0.14664755761623383,
+      "step": 5670
+    },
+    {
+      "epoch": 0.9786354238456237,
+      "grad_norm": 3.150059461593628,
+      "learning_rate": 3.025685133035275e-08,
+      "logits/chosen": -2.964108943939209,
+      "logits/rejected": -2.923243999481201,
+      "logps/chosen": -66.42132568359375,
+      "logps/rejected": -67.96315002441406,
+      "loss": 0.6734,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.12042836099863052,
+      "rewards/margins": 0.04271755367517471,
+      "rewards/rejected": -0.16314592957496643,
+      "step": 5680
+    },
+    {
+      "epoch": 0.9803583735354927,
+      "grad_norm": 3.4928531646728516,
+      "learning_rate": 3.0183329193102894e-08,
+      "logits/chosen": -2.989520311355591,
+      "logits/rejected": -2.9588470458984375,
+      "logps/chosen": -69.19246673583984,
+      "logps/rejected": -66.87910461425781,
+      "loss": 0.6746,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.11193940788507462,
+      "rewards/margins": 0.04019111394882202,
+      "rewards/rejected": -0.15213051438331604,
+      "step": 5690
+    },
+    {
+      "epoch": 0.9820813232253618,
+      "grad_norm": 3.2812881469726562,
+      "learning_rate": 3.0109760182608054e-08,
+      "logits/chosen": -2.795853614807129,
+      "logits/rejected": -2.777820110321045,
+      "logps/chosen": -68.9146499633789,
+      "logps/rejected": -68.53731536865234,
+      "loss": 0.6744,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1112879291176796,
+      "rewards/margins": 0.041012950241565704,
+      "rewards/rejected": -0.1523008793592453,
+      "step": 5700
+    },
+    {
+      "epoch": 0.9820813232253618,
+      "eval_logits/chosen": -3.029478073120117,
+      "eval_logits/rejected": -3.02372407913208,
+      "eval_logps/chosen": -66.06312561035156,
+      "eval_logps/rejected": -72.46124267578125,
+      "eval_loss": 0.6845579147338867,
+      "eval_rewards/accuracies": 0.598280668258667,
+      "eval_rewards/chosen": -0.07351229339838028,
+      "eval_rewards/margins": 0.01929888129234314,
+      "eval_rewards/rejected": -0.09281118214130402,
+      "eval_runtime": 382.951,
+      "eval_samples_per_second": 11.239,
+      "eval_steps_per_second": 1.405,
+      "step": 5700
+    },
+    {
+      "epoch": 0.9838042729152309,
+      "grad_norm": 2.9453823566436768,
+      "learning_rate": 3.0036144964158425e-08,
+      "logits/chosen": -2.9801430702209473,
+      "logits/rejected": -2.9514033794403076,
+      "logps/chosen": -66.82977294921875,
+      "logps/rejected": -69.03922271728516,
+      "loss": 0.6716,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.11716566234827042,
+      "rewards/margins": 0.04739820957183838,
+      "rewards/rejected": -0.164563849568367,
+      "step": 5710
+    },
+    {
+      "epoch": 0.9855272226051,
+      "grad_norm": 3.220808506011963,
+      "learning_rate": 2.9962484203462114e-08,
+      "logits/chosen": -2.9525985717773438,
+      "logits/rejected": -2.9292001724243164,
+      "logps/chosen": -63.6707763671875,
+      "logps/rejected": -66.572998046875,
+      "loss": 0.6696,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.11658785492181778,
+      "rewards/margins": 0.05078582838177681,
+      "rewards/rejected": -0.1673736870288849,
+      "step": 5720
+    },
+    {
+      "epoch": 0.987250172294969,
+      "grad_norm": 3.125443696975708,
+      "learning_rate": 2.988877856663905e-08,
+      "logits/chosen": -3.0078330039978027,
+      "logits/rejected": -2.9969029426574707,
+      "logps/chosen": -63.35039520263672,
+      "logps/rejected": -66.87187194824219,
+      "loss": 0.6778,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.11625798046588898,
+      "rewards/margins": 0.03362895920872688,
+      "rewards/rejected": -0.14988692104816437,
+      "step": 5730
+    },
+    {
+      "epoch": 0.988973121984838,
+      "grad_norm": 3.114074230194092,
+      "learning_rate": 2.9815028720214984e-08,
+      "logits/chosen": -2.907036304473877,
+      "logits/rejected": -2.875609874725342,
+      "logps/chosen": -71.34327697753906,
+      "logps/rejected": -70.927734375,
+      "loss": 0.673,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.12134728580713272,
+      "rewards/margins": 0.04349363595247269,
+      "rewards/rejected": -0.1648409217596054,
+      "step": 5740
+    },
+    {
+      "epoch": 0.9906960716747071,
+      "grad_norm": 3.259777545928955,
+      "learning_rate": 2.974123533111545e-08,
+      "logits/chosen": -3.04443621635437,
+      "logits/rejected": -3.0146572589874268,
+      "logps/chosen": -68.79096984863281,
+      "logps/rejected": -66.2094955444336,
+      "loss": 0.6758,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.12126721441745758,
+      "rewards/margins": 0.038117434829473495,
+      "rewards/rejected": -0.15938463807106018,
+      "step": 5750
+    },
+    {
+      "epoch": 0.9924190213645762,
+      "grad_norm": 3.076246738433838,
+      "learning_rate": 2.9667399066659754e-08,
+      "logits/chosen": -2.91542387008667,
+      "logits/rejected": -2.889267683029175,
+      "logps/chosen": -67.78711700439453,
+      "logps/rejected": -70.97908020019531,
+      "loss": 0.675,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.11377334594726562,
+      "rewards/margins": 0.040327467024326324,
+      "rewards/rejected": -0.15410080552101135,
+      "step": 5760
+    },
+    {
+      "epoch": 0.9941419710544452,
+      "grad_norm": 3.337702512741089,
+      "learning_rate": 2.959352059455492e-08,
+      "logits/chosen": -2.8798606395721436,
+      "logits/rejected": -2.8607451915740967,
+      "logps/chosen": -62.68259811401367,
+      "logps/rejected": -68.51402282714844,
+      "loss": 0.6762,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.10827942937612534,
+      "rewards/margins": 0.036389876157045364,
+      "rewards/rejected": -0.1446692943572998,
+      "step": 5770
+    },
+    {
+      "epoch": 0.9958649207443143,
+      "grad_norm": 3.527778148651123,
+      "learning_rate": 2.9519600582889654e-08,
+      "logits/chosen": -2.910694122314453,
+      "logits/rejected": -2.8908352851867676,
+      "logps/chosen": -63.43735885620117,
+      "logps/rejected": -69.01725769042969,
+      "loss": 0.6744,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.10973789542913437,
+      "rewards/margins": 0.040754105895757675,
+      "rewards/rejected": -0.15049201250076294,
+      "step": 5780
+    },
+    {
+      "epoch": 0.9975878704341833,
+      "grad_norm": 3.0185470581054688,
+      "learning_rate": 2.944563970012831e-08,
+      "logits/chosen": -2.8491313457489014,
+      "logits/rejected": -2.8152623176574707,
+      "logps/chosen": -62.327613830566406,
+      "logps/rejected": -67.97642517089844,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.10853513330221176,
+      "rewards/margins": 0.03189641982316971,
+      "rewards/rejected": -0.14043155312538147,
+      "step": 5790
+    },
+    {
+      "epoch": 0.9993108201240524,
+      "grad_norm": 3.2220873832702637,
+      "learning_rate": 2.937163861510486e-08,
+      "logits/chosen": -2.9354023933410645,
+      "logits/rejected": -2.9071598052978516,
+      "logps/chosen": -62.23699951171875,
+      "logps/rejected": -68.48313903808594,
+      "loss": 0.6709,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.11486121267080307,
+      "rewards/margins": 0.04809769243001938,
+      "rewards/rejected": -0.16295892000198364,
+      "step": 5800
+    },
+    {
+      "epoch": 0.9993108201240524,
+      "eval_logits/chosen": -3.0261149406433105,
+      "eval_logits/rejected": -3.0203123092651367,
+      "eval_logps/chosen": -66.34803771972656,
+      "eval_logps/rejected": -72.80882263183594,
+      "eval_loss": 0.6842846274375916,
+      "eval_rewards/accuracies": 0.5999070405960083,
+      "eval_rewards/chosen": -0.07636149972677231,
+      "eval_rewards/margins": 0.019925493746995926,
+      "eval_rewards/rejected": -0.09628698974847794,
+      "eval_runtime": 383.0219,
+      "eval_samples_per_second": 11.237,
+      "eval_steps_per_second": 1.405,
+      "step": 5800
+    },
+    {
+      "epoch": 1.0010337698139213,
+      "grad_norm": 3.183600664138794,
+      "learning_rate": 2.92975979970168e-08,
+      "logits/chosen": -2.9936161041259766,
+      "logits/rejected": -2.9777989387512207,
+      "logps/chosen": -66.12782287597656,
+      "logps/rejected": -69.59823608398438,
+      "loss": 0.6749,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.12056756019592285,
+      "rewards/margins": 0.03943460062146187,
+      "rewards/rejected": -0.16000214219093323,
+      "step": 5810
+    },
+    {
+      "epoch": 1.0027567195037905,
+      "grad_norm": 3.094125986099243,
+      "learning_rate": 2.9223518515419147e-08,
+      "logits/chosen": -2.997633695602417,
+      "logits/rejected": -2.963797092437744,
+      "logps/chosen": -64.92887115478516,
+      "logps/rejected": -66.06184387207031,
+      "loss": 0.6708,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.11902846395969391,
+      "rewards/margins": 0.048291053622961044,
+      "rewards/rejected": -0.16731952130794525,
+      "step": 5820
+    },
+    {
+      "epoch": 1.0044796691936595,
+      "grad_norm": 3.196580648422241,
+      "learning_rate": 2.914940084021836e-08,
+      "logits/chosen": -2.8819403648376465,
+      "logits/rejected": -2.850109577178955,
+      "logps/chosen": -64.55899047851562,
+      "logps/rejected": -68.2925033569336,
+      "loss": 0.6733,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.110624298453331,
+      "rewards/margins": 0.043652065098285675,
+      "rewards/rejected": -0.15427637100219727,
+      "step": 5830
+    },
+    {
+      "epoch": 1.0062026188835287,
+      "grad_norm": 3.1817409992218018,
+      "learning_rate": 2.9075245641666278e-08,
+      "logits/chosen": -2.913461685180664,
+      "logits/rejected": -2.8926737308502197,
+      "logps/chosen": -64.00286865234375,
+      "logps/rejected": -70.56645965576172,
+      "loss": 0.6731,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.11614841222763062,
+      "rewards/margins": 0.04381295293569565,
+      "rewards/rejected": -0.15996137261390686,
+      "step": 5840
+    },
+    {
+      "epoch": 1.0079255685733977,
+      "grad_norm": 3.182068109512329,
+      "learning_rate": 2.9001053590354075e-08,
+      "logits/chosen": -2.981210708618164,
+      "logits/rejected": -2.9607720375061035,
+      "logps/chosen": -62.511627197265625,
+      "logps/rejected": -70.78588104248047,
+      "loss": 0.6718,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.11596854776144028,
+      "rewards/margins": 0.046909723430871964,
+      "rewards/rejected": -0.16287824511528015,
+      "step": 5850
+    },
+    {
+      "epoch": 1.0096485182632666,
+      "grad_norm": 2.9458975791931152,
+      "learning_rate": 2.8926825357206174e-08,
+      "logits/chosen": -2.8454506397247314,
+      "logits/rejected": -2.835629463195801,
+      "logps/chosen": -64.00395202636719,
+      "logps/rejected": -69.4656982421875,
+      "loss": 0.675,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.1223059892654419,
+      "rewards/margins": 0.03999713435769081,
+      "rewards/rejected": -0.1623031049966812,
+      "step": 5860
+    },
+    {
+      "epoch": 1.0113714679531358,
+      "grad_norm": 3.5210626125335693,
+      "learning_rate": 2.8852561613474213e-08,
+      "logits/chosen": -2.8781299591064453,
+      "logits/rejected": -2.8573925495147705,
+      "logps/chosen": -67.77476501464844,
+      "logps/rejected": -71.89720153808594,
+      "loss": 0.6688,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.11593930423259735,
+      "rewards/margins": 0.05276470258831978,
+      "rewards/rejected": -0.16870398819446564,
+      "step": 5870
+    },
+    {
+      "epoch": 1.0130944176430048,
+      "grad_norm": 3.3739428520202637,
+      "learning_rate": 2.8778263030730937e-08,
+      "logits/chosen": -2.956064462661743,
+      "logits/rejected": -2.943455219268799,
+      "logps/chosen": -65.3647689819336,
+      "logps/rejected": -66.77729797363281,
+      "loss": 0.6797,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.10622813552618027,
+      "rewards/margins": 0.03024216555058956,
+      "rewards/rejected": -0.13647030293941498,
+      "step": 5880
+    },
+    {
+      "epoch": 1.014817367332874,
+      "grad_norm": 3.1337947845458984,
+      "learning_rate": 2.8703930280864165e-08,
+      "logits/chosen": -2.981327533721924,
+      "logits/rejected": -2.967268705368042,
+      "logps/chosen": -66.04319763183594,
+      "logps/rejected": -72.87442016601562,
+      "loss": 0.6759,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.1087782233953476,
+      "rewards/margins": 0.03787430375814438,
+      "rewards/rejected": -0.14665253460407257,
+      "step": 5890
+    },
+    {
+      "epoch": 1.016540317022743,
+      "grad_norm": 3.2543833255767822,
+      "learning_rate": 2.8629564036070662e-08,
+      "logits/chosen": -2.887153148651123,
+      "logits/rejected": -2.8672187328338623,
+      "logps/chosen": -67.63594818115234,
+      "logps/rejected": -70.91566467285156,
+      "loss": 0.6738,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.11969922482967377,
+      "rewards/margins": 0.04276402294635773,
+      "rewards/rejected": -0.1624632328748703,
+      "step": 5900
+    },
+    {
+      "epoch": 1.016540317022743,
+      "eval_logits/chosen": -3.0225746631622314,
+      "eval_logits/rejected": -3.016803741455078,
+      "eval_logps/chosen": -66.40996551513672,
+      "eval_logps/rejected": -72.89781951904297,
+      "eval_loss": 0.6841657757759094,
+      "eval_rewards/accuracies": 0.6017658114433289,
+      "eval_rewards/chosen": -0.07698071748018265,
+      "eval_rewards/margins": 0.020196298137307167,
+      "eval_rewards/rejected": -0.09717702120542526,
+      "eval_runtime": 383.0372,
+      "eval_samples_per_second": 11.237,
+      "eval_steps_per_second": 1.405,
+      "step": 5900
+    },
+    {
+      "epoch": 1.018263266712612,
+      "grad_norm": 3.394274950027466,
+      "learning_rate": 2.8555164968850108e-08,
+      "logits/chosen": -2.882486581802368,
+      "logits/rejected": -2.8862173557281494,
+      "logps/chosen": -64.24687194824219,
+      "logps/rejected": -71.83860778808594,
+      "loss": 0.6828,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.11734390258789062,
+      "rewards/margins": 0.02387392148375511,
+      "rewards/rejected": -0.14121782779693604,
+      "step": 5910
+    },
+    {
+      "epoch": 1.019986216402481,
+      "grad_norm": 2.979868173599243,
+      "learning_rate": 2.848073375199901e-08,
+      "logits/chosen": -2.9345128536224365,
+      "logits/rejected": -2.9159903526306152,
+      "logps/chosen": -63.35466766357422,
+      "logps/rejected": -68.86671447753906,
+      "loss": 0.6777,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.1187097579240799,
+      "rewards/margins": 0.034884996712207794,
+      "rewards/rejected": -0.1535947620868683,
+      "step": 5920
+    },
+    {
+      "epoch": 1.02170916609235,
+      "grad_norm": 2.977306365966797,
+      "learning_rate": 2.8406271058604575e-08,
+      "logits/chosen": -2.940401554107666,
+      "logits/rejected": -2.9340319633483887,
+      "logps/chosen": -63.742164611816406,
+      "logps/rejected": -68.03939819335938,
+      "loss": 0.678,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.1351395845413208,
+      "rewards/margins": 0.03358253836631775,
+      "rewards/rejected": -0.16872212290763855,
+      "step": 5930
+    },
+    {
+      "epoch": 1.0234321157822193,
+      "grad_norm": 3.3495595455169678,
+      "learning_rate": 2.8331777562038677e-08,
+      "logits/chosen": -2.940746784210205,
+      "logits/rejected": -2.900275468826294,
+      "logps/chosen": -64.98384094238281,
+      "logps/rejected": -69.04167938232422,
+      "loss": 0.6689,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.10746674239635468,
+      "rewards/margins": 0.05228624865412712,
+      "rewards/rejected": -0.1597529798746109,
+      "step": 5940
+    },
+    {
+      "epoch": 1.0251550654720882,
+      "grad_norm": 3.4371237754821777,
+      "learning_rate": 2.8257253935951754e-08,
+      "logits/chosen": -2.8107829093933105,
+      "logits/rejected": -2.8076229095458984,
+      "logps/chosen": -65.1948013305664,
+      "logps/rejected": -68.83119201660156,
+      "loss": 0.6775,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.11489598453044891,
+      "rewards/margins": 0.03370455652475357,
+      "rewards/rejected": -0.14860054850578308,
+      "step": 5950
+    },
+    {
+      "epoch": 1.0268780151619572,
+      "grad_norm": 3.379068613052368,
+      "learning_rate": 2.8182700854266677e-08,
+      "logits/chosen": -2.8695921897888184,
+      "logits/rejected": -2.817808151245117,
+      "logps/chosen": -69.67535400390625,
+      "logps/rejected": -67.23384857177734,
+      "loss": 0.6748,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.1135203018784523,
+      "rewards/margins": 0.040807854384183884,
+      "rewards/rejected": -0.1543281376361847,
+      "step": 5960
+    },
+    {
+      "epoch": 1.0286009648518264,
+      "grad_norm": 4.080209255218506,
+      "learning_rate": 2.8108118991172713e-08,
+      "logits/chosen": -2.8631978034973145,
+      "logits/rejected": -2.8456180095672607,
+      "logps/chosen": -64.48259735107422,
+      "logps/rejected": -66.56556701660156,
+      "loss": 0.6745,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.12585251033306122,
+      "rewards/margins": 0.040656980127096176,
+      "rewards/rejected": -0.1665094792842865,
+      "step": 5970
+    },
+    {
+      "epoch": 1.0303239145416954,
+      "grad_norm": 3.43485164642334,
+      "learning_rate": 2.8033509021119394e-08,
+      "logits/chosen": -2.8660757541656494,
+      "logits/rejected": -2.8661885261535645,
+      "logps/chosen": -64.26313781738281,
+      "logps/rejected": -71.25514221191406,
+      "loss": 0.6797,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.11693141609430313,
+      "rewards/margins": 0.030472928658127785,
+      "rewards/rejected": -0.14740434288978577,
+      "step": 5980
+    },
+    {
+      "epoch": 1.0320468642315643,
+      "grad_norm": 3.1449122428894043,
+      "learning_rate": 2.7958871618810432e-08,
+      "logits/chosen": -2.938018321990967,
+      "logits/rejected": -2.904116630554199,
+      "logps/chosen": -66.7431869506836,
+      "logps/rejected": -67.36903381347656,
+      "loss": 0.6762,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.11670415103435516,
+      "rewards/margins": 0.037033285945653915,
+      "rewards/rejected": -0.15373744070529938,
+      "step": 5990
+    },
+    {
+      "epoch": 1.0337698139214335,
+      "grad_norm": 2.794699192047119,
+      "learning_rate": 2.7884207459197584e-08,
+      "logits/chosen": -2.916213274002075,
+      "logits/rejected": -2.8988471031188965,
+      "logps/chosen": -64.56881713867188,
+      "logps/rejected": -68.6457290649414,
+      "loss": 0.6755,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.12464715540409088,
+      "rewards/margins": 0.03907427564263344,
+      "rewards/rejected": -0.16372142732143402,
+      "step": 6000
+    },
+    {
+      "epoch": 1.0337698139214335,
+      "eval_logits/chosen": -3.020717144012451,
+      "eval_logits/rejected": -3.0149612426757812,
+      "eval_logps/chosen": -66.45557403564453,
+      "eval_logps/rejected": -72.94852447509766,
+      "eval_loss": 0.6841490864753723,
+      "eval_rewards/accuracies": 0.6050186157226562,
+      "eval_rewards/chosen": -0.07743674516677856,
+      "eval_rewards/margins": 0.020247286185622215,
+      "eval_rewards/rejected": -0.09768402576446533,
+      "eval_runtime": 383.1633,
+      "eval_samples_per_second": 11.233,
+      "eval_steps_per_second": 1.404,
+      "step": 6000
+    },
+    {
+      "epoch": 1.0354927636113025,
+      "grad_norm": 3.4271628856658936,
+      "learning_rate": 2.780951721747461e-08,
+      "logits/chosen": -2.9177567958831787,
+      "logits/rejected": -2.900395631790161,
+      "logps/chosen": -66.01614379882812,
+      "logps/rejected": -66.52980041503906,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.12980924546718597,
+      "rewards/margins": 0.02142946422100067,
+      "rewards/rejected": -0.15123872458934784,
+      "step": 6010
+    },
+    {
+      "epoch": 1.0372157133011717,
+      "grad_norm": 2.9992895126342773,
+      "learning_rate": 2.7734801569071104e-08,
+      "logits/chosen": -3.0855090618133545,
+      "logits/rejected": -3.053163766860962,
+      "logps/chosen": -66.14179229736328,
+      "logps/rejected": -66.02214050292969,
+      "loss": 0.6737,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.11754494905471802,
+      "rewards/margins": 0.04368080943822861,
+      "rewards/rejected": -0.16122575104236603,
+      "step": 6020
+    },
+    {
+      "epoch": 1.0389386629910407,
+      "grad_norm": 2.920555830001831,
+      "learning_rate": 2.766006118964644e-08,
+      "logits/chosen": -2.7439918518066406,
+      "logits/rejected": -2.7336184978485107,
+      "logps/chosen": -63.365501403808594,
+      "logps/rejected": -66.21366119384766,
+      "loss": 0.6834,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.13527506589889526,
+      "rewards/margins": 0.02369817905128002,
+      "rewards/rejected": -0.15897324681282043,
+      "step": 6030
+    },
+    {
+      "epoch": 1.0406616126809096,
+      "grad_norm": 2.9166626930236816,
+      "learning_rate": 2.7585296755083613e-08,
+      "logits/chosen": -2.9390835762023926,
+      "logits/rejected": -2.919097661972046,
+      "logps/chosen": -65.650390625,
+      "logps/rejected": -67.72566223144531,
+      "loss": 0.6744,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.11942142248153687,
+      "rewards/margins": 0.0410720631480217,
+      "rewards/rejected": -0.16049346327781677,
+      "step": 6040
+    },
+    {
+      "epoch": 1.0423845623707788,
+      "grad_norm": 3.211782932281494,
+      "learning_rate": 2.751050894148317e-08,
+      "logits/chosen": -2.837831974029541,
+      "logits/rejected": -2.8112006187438965,
+      "logps/chosen": -71.4070816040039,
+      "logps/rejected": -70.95361328125,
+      "loss": 0.669,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.12064974009990692,
+      "rewards/margins": 0.05189179256558418,
+      "rewards/rejected": -0.1725415289402008,
+      "step": 6050
+    },
+    {
+      "epoch": 1.0441075120606478,
+      "grad_norm": 2.9817655086517334,
+      "learning_rate": 2.7435698425157065e-08,
+      "logits/chosen": -2.910189628601074,
+      "logits/rejected": -2.887599468231201,
+      "logps/chosen": -65.72587585449219,
+      "logps/rejected": -67.54866790771484,
+      "loss": 0.6781,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.1223108172416687,
+      "rewards/margins": 0.03353102505207062,
+      "rewards/rejected": -0.1558418571949005,
+      "step": 6060
+    },
+    {
+      "epoch": 1.045830461750517,
+      "grad_norm": 3.1714446544647217,
+      "learning_rate": 2.7360865882622558e-08,
+      "logits/chosen": -2.9184298515319824,
+      "logits/rejected": -2.8969321250915527,
+      "logps/chosen": -66.23518371582031,
+      "logps/rejected": -68.22206115722656,
+      "loss": 0.674,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.12292845547199249,
+      "rewards/margins": 0.042066603899002075,
+      "rewards/rejected": -0.16499504446983337,
+      "step": 6070
+    },
+    {
+      "epoch": 1.047553411440386,
+      "grad_norm": 3.258284091949463,
+      "learning_rate": 2.7286011990596092e-08,
+      "logits/chosen": -2.8854336738586426,
+      "logits/rejected": -2.864877223968506,
+      "logps/chosen": -70.7151107788086,
+      "logps/rejected": -70.31018829345703,
+      "loss": 0.6697,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.11639515310525894,
+      "rewards/margins": 0.050934772938489914,
+      "rewards/rejected": -0.16732993721961975,
+      "step": 6080
+    },
+    {
+      "epoch": 1.049276361130255,
+      "grad_norm": 3.3222267627716064,
+      "learning_rate": 2.7211137425987175e-08,
+      "logits/chosen": -2.954876184463501,
+      "logits/rejected": -2.9371752738952637,
+      "logps/chosen": -62.83771514892578,
+      "logps/rejected": -68.13609313964844,
+      "loss": 0.6684,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1142960637807846,
+      "rewards/margins": 0.0535506010055542,
+      "rewards/rejected": -0.1678466498851776,
+      "step": 6090
+    },
+    {
+      "epoch": 1.050999310820124,
+      "grad_norm": 3.2277133464813232,
+      "learning_rate": 2.7136242865892268e-08,
+      "logits/chosen": -2.921597480773926,
+      "logits/rejected": -2.918447971343994,
+      "logps/chosen": -68.9329605102539,
+      "logps/rejected": -70.5098648071289,
+      "loss": 0.6727,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.1214974895119667,
+      "rewards/margins": 0.04489421471953392,
+      "rewards/rejected": -0.16639170050621033,
+      "step": 6100
+    },
+    {
+      "epoch": 1.050999310820124,
+      "eval_logits/chosen": -3.018160104751587,
+      "eval_logits/rejected": -3.0123844146728516,
+      "eval_logps/chosen": -66.61014556884766,
+      "eval_logps/rejected": -73.14734649658203,
+      "eval_loss": 0.683953046798706,
+      "eval_rewards/accuracies": 0.6043215394020081,
+      "eval_rewards/chosen": -0.07898253202438354,
+      "eval_rewards/margins": 0.020689763128757477,
+      "eval_rewards/rejected": -0.09967228025197983,
+      "eval_runtime": 382.7779,
+      "eval_samples_per_second": 11.244,
+      "eval_steps_per_second": 1.406,
+      "step": 6100
+    },
+    {
+      "epoch": 1.052722260509993,
+      "grad_norm": 3.3152217864990234,
+      "learning_rate": 2.7061328987588627e-08,
+      "logits/chosen": -2.9263315200805664,
+      "logits/rejected": -2.911386251449585,
+      "logps/chosen": -64.42832946777344,
+      "logps/rejected": -70.03648376464844,
+      "loss": 0.6759,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.12729394435882568,
+      "rewards/margins": 0.03818570822477341,
+      "rewards/rejected": -0.1654796600341797,
+      "step": 6110
+    },
+    {
+      "epoch": 1.0544452101998623,
+      "grad_norm": 3.2479255199432373,
+      "learning_rate": 2.698639646852824e-08,
+      "logits/chosen": -3.0323052406311035,
+      "logits/rejected": -2.984510898590088,
+      "logps/chosen": -64.88023376464844,
+      "logps/rejected": -70.47579956054688,
+      "loss": 0.6632,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.10558520257472992,
+      "rewards/margins": 0.06439732760190964,
+      "rewards/rejected": -0.16998253762722015,
+      "step": 6120
+    },
+    {
+      "epoch": 1.0561681598897312,
+      "grad_norm": 3.723346710205078,
+      "learning_rate": 2.6911445986331633e-08,
+      "logits/chosen": -2.930084228515625,
+      "logits/rejected": -2.9049084186553955,
+      "logps/chosen": -66.62183380126953,
+      "logps/rejected": -70.44300842285156,
+      "loss": 0.6735,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.12230199575424194,
+      "rewards/margins": 0.04267255216836929,
+      "rewards/rejected": -0.16497455537319183,
+      "step": 6130
+    },
+    {
+      "epoch": 1.0578911095796002,
+      "grad_norm": 3.7378158569335938,
+      "learning_rate": 2.68364782187818e-08,
+      "logits/chosen": -2.963712453842163,
+      "logits/rejected": -2.951669216156006,
+      "logps/chosen": -65.98356628417969,
+      "logps/rejected": -68.6837158203125,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.12904593348503113,
+      "rewards/margins": 0.03299646079540253,
+      "rewards/rejected": -0.16204240918159485,
+      "step": 6140
+    },
+    {
+      "epoch": 1.0596140592694694,
+      "grad_norm": 3.3553664684295654,
+      "learning_rate": 2.676149384381803e-08,
+      "logits/chosen": -2.869103193283081,
+      "logits/rejected": -2.8543384075164795,
+      "logps/chosen": -66.4356918334961,
+      "logps/rejected": -66.22886657714844,
+      "loss": 0.6806,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.13427793979644775,
+      "rewards/margins": 0.028122087940573692,
+      "rewards/rejected": -0.1624000370502472,
+      "step": 6150
+    },
+    {
+      "epoch": 1.0613370089593384,
+      "grad_norm": 3.358989715576172,
+      "learning_rate": 2.66864935395298e-08,
+      "logits/chosen": -2.8266618251800537,
+      "logits/rejected": -2.816030263900757,
+      "logps/chosen": -61.9078369140625,
+      "logps/rejected": -68.06988525390625,
+      "loss": 0.6719,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.10932044684886932,
+      "rewards/margins": 0.04588836431503296,
+      "rewards/rejected": -0.15520881116390228,
+      "step": 6160
+    },
+    {
+      "epoch": 1.0630599586492075,
+      "grad_norm": 3.5227408409118652,
+      "learning_rate": 2.6611477984150627e-08,
+      "logits/chosen": -2.950550079345703,
+      "logits/rejected": -2.9346868991851807,
+      "logps/chosen": -72.44111633300781,
+      "logps/rejected": -73.00975036621094,
+      "loss": 0.6774,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.12980476021766663,
+      "rewards/margins": 0.03557395190000534,
+      "rewards/rejected": -0.16537870466709137,
+      "step": 6170
+    },
+    {
+      "epoch": 1.0647829083390765,
+      "grad_norm": 3.50789475440979,
+      "learning_rate": 2.6536447856051963e-08,
+      "logits/chosen": -2.964113712310791,
+      "logits/rejected": -2.939772129058838,
+      "logps/chosen": -69.8862533569336,
+      "logps/rejected": -69.29154205322266,
+      "loss": 0.6726,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1173110231757164,
+      "rewards/margins": 0.04580897092819214,
+      "rewards/rejected": -0.16312000155448914,
+      "step": 6180
+    },
+    {
+      "epoch": 1.0665058580289455,
+      "grad_norm": 3.4044222831726074,
+      "learning_rate": 2.646140383373704e-08,
+      "logits/chosen": -2.9300460815429688,
+      "logits/rejected": -2.905643939971924,
+      "logps/chosen": -69.40953063964844,
+      "logps/rejected": -69.94522094726562,
+      "loss": 0.6671,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.11233565956354141,
+      "rewards/margins": 0.05709084868431091,
+      "rewards/rejected": -0.16942648589611053,
+      "step": 6190
+    },
+    {
+      "epoch": 1.0682288077188147,
+      "grad_norm": 3.298741340637207,
+      "learning_rate": 2.638634659583472e-08,
+      "logits/chosen": -2.819237470626831,
+      "logits/rejected": -2.7997632026672363,
+      "logps/chosen": -67.05134582519531,
+      "logps/rejected": -71.38154602050781,
+      "loss": 0.677,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.12536853551864624,
+      "rewards/margins": 0.036074813455343246,
+      "rewards/rejected": -0.16144336760044098,
+      "step": 6200
+    },
+    {
+      "epoch": 1.0682288077188147,
+      "eval_logits/chosen": -3.0157365798950195,
+      "eval_logits/rejected": -3.009998083114624,
+      "eval_logps/chosen": -66.75472259521484,
+      "eval_logps/rejected": -73.32022857666016,
+      "eval_loss": 0.6838310360908508,
+      "eval_rewards/accuracies": 0.6052509546279907,
+      "eval_rewards/chosen": -0.08042816817760468,
+      "eval_rewards/margins": 0.020972857251763344,
+      "eval_rewards/rejected": -0.10140103846788406,
+      "eval_runtime": 383.4757,
+      "eval_samples_per_second": 11.224,
+      "eval_steps_per_second": 1.403,
+      "step": 6200
+    },
+    {
+      "epoch": 1.0699517574086836,
+      "grad_norm": 3.2326667308807373,
+      "learning_rate": 2.6311276821093382e-08,
+      "logits/chosen": -2.934070587158203,
+      "logits/rejected": -2.9046998023986816,
+      "logps/chosen": -67.86701202392578,
+      "logps/rejected": -69.99655151367188,
+      "loss": 0.6748,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.13041749596595764,
+      "rewards/margins": 0.04058767482638359,
+      "rewards/rejected": -0.17100517451763153,
+      "step": 6210
+    },
+    {
+      "epoch": 1.0716747070985528,
+      "grad_norm": 3.1769347190856934,
+      "learning_rate": 2.62361951883748e-08,
+      "logits/chosen": -2.919170379638672,
+      "logits/rejected": -2.898787021636963,
+      "logps/chosen": -64.49347686767578,
+      "logps/rejected": -69.13720703125,
+      "loss": 0.6738,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.12354440987110138,
+      "rewards/margins": 0.04235800355672836,
+      "rewards/rejected": -0.16590240597724915,
+      "step": 6220
+    },
+    {
+      "epoch": 1.0733976567884218,
+      "grad_norm": 3.065797805786133,
+      "learning_rate": 2.616110237664793e-08,
+      "logits/chosen": -3.057398796081543,
+      "logits/rejected": -3.0387120246887207,
+      "logps/chosen": -69.72382354736328,
+      "logps/rejected": -73.53699493408203,
+      "loss": 0.6652,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.10891111195087433,
+      "rewards/margins": 0.06093553453683853,
+      "rewards/rejected": -0.16984662413597107,
+      "step": 6230
+    },
+    {
+      "epoch": 1.0751206064782908,
+      "grad_norm": 3.8365414142608643,
+      "learning_rate": 2.608599906498287e-08,
+      "logits/chosen": -2.871263027191162,
+      "logits/rejected": -2.8398754596710205,
+      "logps/chosen": -68.86366271972656,
+      "logps/rejected": -70.4202651977539,
+      "loss": 0.6742,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.12476854026317596,
+      "rewards/margins": 0.042408645153045654,
+      "rewards/rejected": -0.1671772003173828,
+      "step": 6240
+    },
+    {
+      "epoch": 1.07684355616816,
+      "grad_norm": 3.7265093326568604,
+      "learning_rate": 2.6010885932544646e-08,
+      "logits/chosen": -2.957125663757324,
+      "logits/rejected": -2.9346091747283936,
+      "logps/chosen": -67.26287078857422,
+      "logps/rejected": -68.92646789550781,
+      "loss": 0.6775,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.11535082757472992,
+      "rewards/margins": 0.03496875241398811,
+      "rewards/rejected": -0.15031959116458893,
+      "step": 6250
+    },
+    {
+      "epoch": 1.078566505858029,
+      "grad_norm": 3.3692572116851807,
+      "learning_rate": 2.59357636585871e-08,
+      "logits/chosen": -2.8362362384796143,
+      "logits/rejected": -2.815936803817749,
+      "logps/chosen": -67.42303466796875,
+      "logps/rejected": -68.00535583496094,
+      "loss": 0.6748,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.13498619198799133,
+      "rewards/margins": 0.04125773906707764,
+      "rewards/rejected": -0.17624394595623016,
+      "step": 6260
+    },
+    {
+      "epoch": 1.080289455547898,
+      "grad_norm": 3.199134111404419,
+      "learning_rate": 2.5860632922446733e-08,
+      "logits/chosen": -3.1004369258880615,
+      "logits/rejected": -3.0995078086853027,
+      "logps/chosen": -64.9561996459961,
+      "logps/rejected": -70.83625793457031,
+      "loss": 0.6743,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.12055517733097076,
+      "rewards/margins": 0.04142562299966812,
+      "rewards/rejected": -0.16198080778121948,
+      "step": 6270
+    },
+    {
+      "epoch": 1.082012405237767,
+      "grad_norm": 3.395749092102051,
+      "learning_rate": 2.578549440353659e-08,
+      "logits/chosen": -2.7851722240448,
+      "logits/rejected": -2.7610630989074707,
+      "logps/chosen": -66.57701873779297,
+      "logps/rejected": -69.19154357910156,
+      "loss": 0.6731,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.12164785712957382,
+      "rewards/margins": 0.04383537918329239,
+      "rewards/rejected": -0.16548322141170502,
+      "step": 6280
+    },
+    {
+      "epoch": 1.083735354927636,
+      "grad_norm": 3.2278220653533936,
+      "learning_rate": 2.5710348781340068e-08,
+      "logits/chosen": -2.864137649536133,
+      "logits/rejected": -2.8336355686187744,
+      "logps/chosen": -63.213783264160156,
+      "logps/rejected": -69.06321716308594,
+      "loss": 0.6707,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.11835372447967529,
+      "rewards/margins": 0.0485164038836956,
+      "rewards/rejected": -0.1668701320886612,
+      "step": 6290
+    },
+    {
+      "epoch": 1.0854583046175053,
+      "grad_norm": 3.3059394359588623,
+      "learning_rate": 2.5635196735404818e-08,
+      "logits/chosen": -2.938779354095459,
+      "logits/rejected": -2.9110283851623535,
+      "logps/chosen": -66.49371337890625,
+      "logps/rejected": -69.54383850097656,
+      "loss": 0.6778,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.1405746042728424,
+      "rewards/margins": 0.03441260755062103,
+      "rewards/rejected": -0.17498719692230225,
+      "step": 6300
+    },
+    {
+      "epoch": 1.0854583046175053,
+      "eval_logits/chosen": -3.0138537883758545,
+      "eval_logits/rejected": -3.0081186294555664,
+      "eval_logps/chosen": -66.96984100341797,
+      "eval_logps/rejected": -73.54715728759766,
+      "eval_loss": 0.683800220489502,
+      "eval_rewards/accuracies": 0.6017658114433289,
+      "eval_rewards/chosen": -0.08257945626974106,
+      "eval_rewards/margins": 0.021090904250741005,
+      "eval_rewards/rejected": -0.10367034375667572,
+      "eval_runtime": 383.2007,
+      "eval_samples_per_second": 11.232,
+      "eval_steps_per_second": 1.404,
+      "step": 6300
+    },
+    {
+      "epoch": 1.0871812543073742,
+      "grad_norm": 4.099219799041748,
+      "learning_rate": 2.556003894533658e-08,
+      "logits/chosen": -2.92287015914917,
+      "logits/rejected": -2.8924994468688965,
+      "logps/chosen": -63.35823440551758,
+      "logps/rejected": -67.10684204101562,
+      "loss": 0.6815,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.1290132999420166,
+      "rewards/margins": 0.02653651311993599,
+      "rewards/rejected": -0.1555498093366623,
+      "step": 6310
+    },
+    {
+      "epoch": 1.0889042039972432,
+      "grad_norm": 3.341099977493286,
+      "learning_rate": 2.548487609079305e-08,
+      "logits/chosen": -2.8713512420654297,
+      "logits/rejected": -2.8498892784118652,
+      "logps/chosen": -67.93678283691406,
+      "logps/rejected": -70.1052017211914,
+      "loss": 0.6773,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.13214829564094543,
+      "rewards/margins": 0.035667676478624344,
+      "rewards/rejected": -0.16781596839427948,
+      "step": 6320
+    },
+    {
+      "epoch": 1.0906271536871124,
+      "grad_norm": 3.175612688064575,
+      "learning_rate": 2.5409708851477683e-08,
+      "logits/chosen": -2.8839213848114014,
+      "logits/rejected": -2.8639819622039795,
+      "logps/chosen": -65.91523742675781,
+      "logps/rejected": -72.16932678222656,
+      "loss": 0.6658,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.1262522041797638,
+      "rewards/margins": 0.05935942009091377,
+      "rewards/rejected": -0.18561163544654846,
+      "step": 6330
+    },
+    {
+      "epoch": 1.0923501033769814,
+      "grad_norm": 3.384282350540161,
+      "learning_rate": 2.533453790713363e-08,
+      "logits/chosen": -2.893129348754883,
+      "logits/rejected": -2.8708934783935547,
+      "logps/chosen": -67.58055877685547,
+      "logps/rejected": -68.73002624511719,
+      "loss": 0.6756,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.11993197351694107,
+      "rewards/margins": 0.0387616828083992,
+      "rewards/rejected": -0.15869365632534027,
+      "step": 6340
+    },
+    {
+      "epoch": 1.0940730530668505,
+      "grad_norm": 3.1655397415161133,
+      "learning_rate": 2.5259363937537526e-08,
+      "logits/chosen": -2.8713510036468506,
+      "logits/rejected": -2.8641085624694824,
+      "logps/chosen": -68.17046356201172,
+      "logps/rejected": -72.87901306152344,
+      "loss": 0.6755,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.12655644118785858,
+      "rewards/margins": 0.039223965257406235,
+      "rewards/rejected": -0.1657804250717163,
+      "step": 6350
+    },
+    {
+      "epoch": 1.0957960027567195,
+      "grad_norm": 3.159517526626587,
+      "learning_rate": 2.518418762249336e-08,
+      "logits/chosen": -2.901176691055298,
+      "logits/rejected": -2.898557186126709,
+      "logps/chosen": -65.48442840576172,
+      "logps/rejected": -70.63438415527344,
+      "loss": 0.6746,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.12015831470489502,
+      "rewards/margins": 0.04034766927361488,
+      "rewards/rejected": -0.1605059802532196,
+      "step": 6360
+    },
+    {
+      "epoch": 1.0975189524465885,
+      "grad_norm": 3.4141743183135986,
+      "learning_rate": 2.5109009641826344e-08,
+      "logits/chosen": -2.927901268005371,
+      "logits/rejected": -2.9270637035369873,
+      "logps/chosen": -65.72232818603516,
+      "logps/rejected": -70.13214111328125,
+      "loss": 0.6797,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.12578003108501434,
+      "rewards/margins": 0.031120290979743004,
+      "rewards/rejected": -0.1569003164768219,
+      "step": 6370
+    },
+    {
+      "epoch": 1.0992419021364577,
+      "grad_norm": 4.17726993560791,
+      "learning_rate": 2.5033830675376744e-08,
+      "logits/chosen": -2.9569008350372314,
+      "logits/rejected": -2.945883274078369,
+      "logps/chosen": -65.87435150146484,
+      "logps/rejected": -73.2433090209961,
+      "loss": 0.6769,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.13139815628528595,
+      "rewards/margins": 0.0358317494392395,
+      "rewards/rejected": -0.16722990572452545,
+      "step": 6380
+    },
+    {
+      "epoch": 1.1009648518263266,
+      "grad_norm": 3.2536637783050537,
+      "learning_rate": 2.4958651402993735e-08,
+      "logits/chosen": -2.9957499504089355,
+      "logits/rejected": -2.960660934448242,
+      "logps/chosen": -67.391357421875,
+      "logps/rejected": -68.57846069335938,
+      "loss": 0.6639,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.113075852394104,
+      "rewards/margins": 0.06268735229969025,
+      "rewards/rejected": -0.17576318979263306,
+      "step": 6390
+    },
+    {
+      "epoch": 1.1026878015161956,
+      "grad_norm": 3.5774409770965576,
+      "learning_rate": 2.4883472504529286e-08,
+      "logits/chosen": -2.930328130722046,
+      "logits/rejected": -2.9062066078186035,
+      "logps/chosen": -69.8833236694336,
+      "logps/rejected": -72.28460693359375,
+      "loss": 0.6772,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.12189929187297821,
+      "rewards/margins": 0.03555016964673996,
+      "rewards/rejected": -0.15744945406913757,
+      "step": 6400
+    },
+    {
+      "epoch": 1.1026878015161956,
+      "eval_logits/chosen": -3.0117177963256836,
+      "eval_logits/rejected": -3.0059452056884766,
+      "eval_logps/chosen": -67.13489532470703,
+      "eval_logps/rejected": -73.78321838378906,
+      "eval_loss": 0.6834752559661865,
+      "eval_rewards/accuracies": 0.6043215394020081,
+      "eval_rewards/chosen": -0.08422999829053879,
+      "eval_rewards/margins": 0.02180095948278904,
+      "eval_rewards/rejected": -0.10603094846010208,
+      "eval_runtime": 383.1633,
+      "eval_samples_per_second": 11.233,
+      "eval_steps_per_second": 1.404,
+      "step": 6400
+    },
+    {
+      "epoch": 1.1044107512060648,
+      "grad_norm": 3.334273338317871,
+      "learning_rate": 2.4808294659831937e-08,
+      "logits/chosen": -3.027768611907959,
+      "logits/rejected": -3.0016233921051025,
+      "logps/chosen": -66.3612060546875,
+      "logps/rejected": -68.58610534667969,
+      "loss": 0.671,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.11996792256832123,
+      "rewards/margins": 0.048496346920728683,
+      "rewards/rejected": -0.1684642732143402,
+      "step": 6410
+    },
+    {
+      "epoch": 1.1061337008959338,
+      "grad_norm": 3.4068005084991455,
+      "learning_rate": 2.473311854874075e-08,
+      "logits/chosen": -2.9685797691345215,
+      "logits/rejected": -2.950777292251587,
+      "logps/chosen": -67.968017578125,
+      "logps/rejected": -71.16810607910156,
+      "loss": 0.675,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.13177552819252014,
+      "rewards/margins": 0.04065660387277603,
+      "rewards/rejected": -0.17243213951587677,
+      "step": 6420
+    },
+    {
+      "epoch": 1.107856650585803,
+      "grad_norm": 3.306917667388916,
+      "learning_rate": 2.4657944851079078e-08,
+      "logits/chosen": -2.87811541557312,
+      "logits/rejected": -2.8639068603515625,
+      "logps/chosen": -62.72420120239258,
+      "logps/rejected": -63.69243621826172,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.12332651764154434,
+      "rewards/margins": 0.03124641440808773,
+      "rewards/rejected": -0.15457291901111603,
+      "step": 6430
+    },
+    {
+      "epoch": 1.109579600275672,
+      "grad_norm": 3.293018341064453,
+      "learning_rate": 2.4582774246648447e-08,
+      "logits/chosen": -2.877394437789917,
+      "logits/rejected": -2.8596949577331543,
+      "logps/chosen": -66.6890640258789,
+      "logps/rejected": -69.65760803222656,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.13429997861385345,
+      "rewards/margins": 0.030349001288414,
+      "rewards/rejected": -0.16464899480342865,
+      "step": 6440
+    },
+    {
+      "epoch": 1.111302549965541,
+      "grad_norm": 3.2066895961761475,
+      "learning_rate": 2.4507607415222437e-08,
+      "logits/chosen": -2.8996293544769287,
+      "logits/rejected": -2.8666012287139893,
+      "logps/chosen": -68.61375427246094,
+      "logps/rejected": -69.67857360839844,
+      "loss": 0.6695,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.12289108335971832,
+      "rewards/margins": 0.0516129806637764,
+      "rewards/rejected": -0.17450405657291412,
+      "step": 6450
+    },
+    {
+      "epoch": 1.11302549965541,
+      "grad_norm": 3.406348943710327,
+      "learning_rate": 2.443244503654047e-08,
+      "logits/chosen": -2.8633410930633545,
+      "logits/rejected": -2.8660292625427246,
+      "logps/chosen": -64.74095916748047,
+      "logps/rejected": -75.46595764160156,
+      "loss": 0.6716,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.12865126132965088,
+      "rewards/margins": 0.04677743464708328,
+      "rewards/rejected": -0.17542867362499237,
+      "step": 6460
+    },
+    {
+      "epoch": 1.114748449345279,
+      "grad_norm": 3.229921817779541,
+      "learning_rate": 2.4357287790301757e-08,
+      "logits/chosen": -2.8249242305755615,
+      "logits/rejected": -2.802280902862549,
+      "logps/chosen": -65.44139099121094,
+      "logps/rejected": -68.95939636230469,
+      "loss": 0.6727,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.12678185105323792,
+      "rewards/margins": 0.04521452635526657,
+      "rewards/rejected": -0.17199639976024628,
+      "step": 6470
+    },
+    {
+      "epoch": 1.1164713990351482,
+      "grad_norm": 3.5415754318237305,
+      "learning_rate": 2.4282136356159026e-08,
+      "logits/chosen": -2.912820816040039,
+      "logits/rejected": -2.8763773441314697,
+      "logps/chosen": -67.86854553222656,
+      "logps/rejected": -67.3797836303711,
+      "loss": 0.6671,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.11299704015254974,
+      "rewards/margins": 0.05622441694140434,
+      "rewards/rejected": -0.16922146081924438,
+      "step": 6480
+    },
+    {
+      "epoch": 1.1181943487250172,
+      "grad_norm": 3.7462849617004395,
+      "learning_rate": 2.4206991413712514e-08,
+      "logits/chosen": -3.0905098915100098,
+      "logits/rejected": -3.078146457672119,
+      "logps/chosen": -66.9651870727539,
+      "logps/rejected": -71.22671508789062,
+      "loss": 0.6766,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.13344913721084595,
+      "rewards/margins": 0.03752323240041733,
+      "rewards/rejected": -0.17097237706184387,
+      "step": 6490
+    },
+    {
+      "epoch": 1.1199172984148862,
+      "grad_norm": 3.6202900409698486,
+      "learning_rate": 2.4131853642503697e-08,
+      "logits/chosen": -2.9263319969177246,
+      "logits/rejected": -2.9195685386657715,
+      "logps/chosen": -70.27945709228516,
+      "logps/rejected": -70.02632904052734,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.1407271921634674,
+      "rewards/margins": 0.032417409121990204,
+      "rewards/rejected": -0.1731446087360382,
+      "step": 6500
+    },
+    {
+      "epoch": 1.1199172984148862,
+      "eval_logits/chosen": -3.009025812149048,
+      "eval_logits/rejected": -3.0032548904418945,
+      "eval_logps/chosen": -67.27629089355469,
+      "eval_logps/rejected": -73.949951171875,
+      "eval_loss": 0.6833698749542236,
+      "eval_rewards/accuracies": 0.6054832935333252,
+      "eval_rewards/chosen": -0.08564396947622299,
+      "eval_rewards/margins": 0.02205430157482624,
+      "eval_rewards/rejected": -0.10769825428724289,
+      "eval_runtime": 383.0084,
+      "eval_samples_per_second": 11.237,
+      "eval_steps_per_second": 1.405,
+      "step": 6500
+    },
+    {
+      "epoch": 1.1216402481047554,
+      "grad_norm": 3.3961148262023926,
+      "learning_rate": 2.4056723722009246e-08,
+      "logits/chosen": -2.948745012283325,
+      "logits/rejected": -2.903198719024658,
+      "logps/chosen": -68.98822021484375,
+      "logps/rejected": -70.40615844726562,
+      "loss": 0.6691,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1220393180847168,
+      "rewards/margins": 0.05193446949124336,
+      "rewards/rejected": -0.17397376894950867,
+      "step": 6510
+    },
+    {
+      "epoch": 1.1233631977946243,
+      "grad_norm": 3.9625244140625,
+      "learning_rate": 2.3981602331634804e-08,
+      "logits/chosen": -2.8927390575408936,
+      "logits/rejected": -2.8670058250427246,
+      "logps/chosen": -65.39457702636719,
+      "logps/rejected": -67.83021545410156,
+      "loss": 0.6722,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.11783125251531601,
+      "rewards/margins": 0.04589410871267319,
+      "rewards/rejected": -0.1637253761291504,
+      "step": 6520
+    },
+    {
+      "epoch": 1.1250861474844935,
+      "grad_norm": 3.3377065658569336,
+      "learning_rate": 2.3906490150708893e-08,
+      "logits/chosen": -2.8643105030059814,
+      "logits/rejected": -2.835594654083252,
+      "logps/chosen": -65.65389251708984,
+      "logps/rejected": -69.5113525390625,
+      "loss": 0.6651,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.12029922008514404,
+      "rewards/margins": 0.06037300080060959,
+      "rewards/rejected": -0.18067221343517303,
+      "step": 6530
+    },
+    {
+      "epoch": 1.1268090971743625,
+      "grad_norm": 3.3110873699188232,
+      "learning_rate": 2.383138785847674e-08,
+      "logits/chosen": -2.9202046394348145,
+      "logits/rejected": -2.893101453781128,
+      "logps/chosen": -71.09927368164062,
+      "logps/rejected": -73.46993255615234,
+      "loss": 0.6777,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.13724930584430695,
+      "rewards/margins": 0.03440054878592491,
+      "rewards/rejected": -0.17164985835552216,
+      "step": 6540
+    },
+    {
+      "epoch": 1.1285320468642315,
+      "grad_norm": 3.3169775009155273,
+      "learning_rate": 2.3756296134094176e-08,
+      "logits/chosen": -2.8199639320373535,
+      "logits/rejected": -2.7966244220733643,
+      "logps/chosen": -64.99620056152344,
+      "logps/rejected": -69.32502746582031,
+      "loss": 0.6743,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.13450269401073456,
+      "rewards/margins": 0.041776955127716064,
+      "rewards/rejected": -0.17627964913845062,
+      "step": 6550
+    },
+    {
+      "epoch": 1.1302549965541007,
+      "grad_norm": 3.0068087577819824,
+      "learning_rate": 2.368121565662142e-08,
+      "logits/chosen": -2.9498608112335205,
+      "logits/rejected": -2.9085497856140137,
+      "logps/chosen": -67.95137023925781,
+      "logps/rejected": -70.79926300048828,
+      "loss": 0.6603,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.11947166919708252,
+      "rewards/margins": 0.06988431513309479,
+      "rewards/rejected": -0.1893559992313385,
+      "step": 6560
+    },
+    {
+      "epoch": 1.1319779462439696,
+      "grad_norm": 3.66831111907959,
+      "learning_rate": 2.3606147105017038e-08,
+      "logits/chosen": -2.9311492443084717,
+      "logits/rejected": -2.9017367362976074,
+      "logps/chosen": -66.9018325805664,
+      "logps/rejected": -70.43095397949219,
+      "loss": 0.6732,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.13337817788124084,
+      "rewards/margins": 0.04395810887217522,
+      "rewards/rejected": -0.17733629047870636,
+      "step": 6570
+    },
+    {
+      "epoch": 1.1337008959338388,
+      "grad_norm": 3.3068084716796875,
+      "learning_rate": 2.35310911581317e-08,
+      "logits/chosen": -2.947917938232422,
+      "logits/rejected": -2.9091668128967285,
+      "logps/chosen": -67.493408203125,
+      "logps/rejected": -66.68104553222656,
+      "loss": 0.6738,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.12465085089206696,
+      "rewards/margins": 0.04214128106832504,
+      "rewards/rejected": -0.1667921394109726,
+      "step": 6580
+    },
+    {
+      "epoch": 1.1354238456237078,
+      "grad_norm": 3.4474565982818604,
+      "learning_rate": 2.3456048494702132e-08,
+      "logits/chosen": -2.893240451812744,
+      "logits/rejected": -2.870800733566284,
+      "logps/chosen": -65.51515197753906,
+      "logps/rejected": -70.21784973144531,
+      "loss": 0.6745,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.12675215303897858,
+      "rewards/margins": 0.040616780519485474,
+      "rewards/rejected": -0.16736890375614166,
+      "step": 6590
+    },
+    {
+      "epoch": 1.1371467953135768,
+      "grad_norm": 3.4104769229888916,
+      "learning_rate": 2.3381019793344898e-08,
+      "logits/chosen": -2.9892725944519043,
+      "logits/rejected": -2.9733457565307617,
+      "logps/chosen": -69.09583282470703,
+      "logps/rejected": -70.62470245361328,
+      "loss": 0.6776,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.13463488221168518,
+      "rewards/margins": 0.03573363274335861,
+      "rewards/rejected": -0.1703685224056244,
+      "step": 6600
+    },
+    {
+      "epoch": 1.1371467953135768,
+      "eval_logits/chosen": -3.006772994995117,
+      "eval_logits/rejected": -3.000959634780884,
+      "eval_logps/chosen": -67.5068130493164,
+      "eval_logps/rejected": -74.20049285888672,
+      "eval_loss": 0.6833028793334961,
+      "eval_rewards/accuracies": 0.6036245226860046,
+      "eval_rewards/chosen": -0.08794914931058884,
+      "eval_rewards/margins": 0.022254539653658867,
+      "eval_rewards/rejected": -0.11020368337631226,
+      "eval_runtime": 383.4031,
+      "eval_samples_per_second": 11.226,
+      "eval_steps_per_second": 1.403,
+      "step": 6600
+    },
+    {
+      "epoch": 1.138869745003446,
+      "grad_norm": 3.5051920413970947,
+      "learning_rate": 2.330600573255034e-08,
+      "logits/chosen": -2.9431638717651367,
+      "logits/rejected": -2.9237871170043945,
+      "logps/chosen": -64.68794250488281,
+      "logps/rejected": -69.40861511230469,
+      "loss": 0.6703,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.12721717357635498,
+      "rewards/margins": 0.0493275411427021,
+      "rewards/rejected": -0.17654471099376678,
+      "step": 6610
+    },
+    {
+      "epoch": 1.140592694693315,
+      "grad_norm": 3.718956708908081,
+      "learning_rate": 2.3231006990676365e-08,
+      "logits/chosen": -2.908536911010742,
+      "logits/rejected": -2.8863494396209717,
+      "logps/chosen": -67.34029388427734,
+      "logps/rejected": -69.89668273925781,
+      "loss": 0.6809,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.14949442446231842,
+      "rewards/margins": 0.027884934097528458,
+      "rewards/rejected": -0.17737935483455658,
+      "step": 6620
+    },
+    {
+      "epoch": 1.1423156443831841,
+      "grad_norm": 3.2376489639282227,
+      "learning_rate": 2.3156024245942392e-08,
+      "logits/chosen": -2.888465404510498,
+      "logits/rejected": -2.861661195755005,
+      "logps/chosen": -65.98827362060547,
+      "logps/rejected": -66.35800170898438,
+      "loss": 0.6724,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.12676773965358734,
+      "rewards/margins": 0.04488871246576309,
+      "rewards/rejected": -0.17165645956993103,
+      "step": 6630
+    },
+    {
+      "epoch": 1.144038594073053,
+      "grad_norm": 2.998570442199707,
+      "learning_rate": 2.3081058176423148e-08,
+      "logits/chosen": -2.9391846656799316,
+      "logits/rejected": -2.9015655517578125,
+      "logps/chosen": -67.14237976074219,
+      "logps/rejected": -68.13966369628906,
+      "loss": 0.6727,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.14036546647548676,
+      "rewards/margins": 0.04504992812871933,
+      "rewards/rejected": -0.1854153871536255,
+      "step": 6640
+    },
+    {
+      "epoch": 1.145761543762922,
+      "grad_norm": 3.6166880130767822,
+      "learning_rate": 2.3006109460042562e-08,
+      "logits/chosen": -2.972625255584717,
+      "logits/rejected": -2.9506163597106934,
+      "logps/chosen": -65.093017578125,
+      "logps/rejected": -71.38772583007812,
+      "loss": 0.6703,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.1297948658466339,
+      "rewards/margins": 0.05054716393351555,
+      "rewards/rejected": -0.18034201860427856,
+      "step": 6650
+    },
+    {
+      "epoch": 1.1474844934527912,
+      "grad_norm": 3.320610284805298,
+      "learning_rate": 2.293117877456766e-08,
+      "logits/chosen": -3.0081961154937744,
+      "logits/rejected": -2.9891855716705322,
+      "logps/chosen": -64.41062927246094,
+      "logps/rejected": -71.88462829589844,
+      "loss": 0.6698,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.12649908661842346,
+      "rewards/margins": 0.05070801451802254,
+      "rewards/rejected": -0.1772070825099945,
+      "step": 6660
+    },
+    {
+      "epoch": 1.1492074431426602,
+      "grad_norm": 3.476494789123535,
+      "learning_rate": 2.2856266797602393e-08,
+      "logits/chosen": -2.8907864093780518,
+      "logits/rejected": -2.8946692943573,
+      "logps/chosen": -64.23393249511719,
+      "logps/rejected": -74.11820220947266,
+      "loss": 0.6727,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1432705968618393,
+      "rewards/margins": 0.04499240592122078,
+      "rewards/rejected": -0.18826301395893097,
+      "step": 6670
+    },
+    {
+      "epoch": 1.1509303928325294,
+      "grad_norm": 3.6719088554382324,
+      "learning_rate": 2.2781374206581543e-08,
+      "logits/chosen": -2.9075355529785156,
+      "logits/rejected": -2.876981735229492,
+      "logps/chosen": -66.54055786132812,
+      "logps/rejected": -69.81542205810547,
+      "loss": 0.6699,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.12715336680412292,
+      "rewards/margins": 0.050346117466688156,
+      "rewards/rejected": -0.17749947309494019,
+      "step": 6680
+    },
+    {
+      "epoch": 1.1526533425223984,
+      "grad_norm": 3.3001606464385986,
+      "learning_rate": 2.2706501678764558e-08,
+      "logits/chosen": -2.897528886795044,
+      "logits/rejected": -2.8762929439544678,
+      "logps/chosen": -69.32492065429688,
+      "logps/rejected": -74.5130615234375,
+      "loss": 0.6629,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.12165243923664093,
+      "rewards/margins": 0.0651288703083992,
+      "rewards/rejected": -0.18678131699562073,
+      "step": 6690
+    },
+    {
+      "epoch": 1.1543762922122673,
+      "grad_norm": 3.5741047859191895,
+      "learning_rate": 2.26316498912295e-08,
+      "logits/chosen": -2.9127843379974365,
+      "logits/rejected": -2.9017081260681152,
+      "logps/chosen": -69.35139465332031,
+      "logps/rejected": -73.12579345703125,
+      "loss": 0.6755,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.13223156332969666,
+      "rewards/margins": 0.04005131870508194,
+      "rewards/rejected": -0.17228290438652039,
+      "step": 6700
+    },
+    {
+      "epoch": 1.1543762922122673,
+      "eval_logits/chosen": -3.004514455795288,
+      "eval_logits/rejected": -2.9987897872924805,
+      "eval_logps/chosen": -67.7115478515625,
+      "eval_logps/rejected": -74.44764709472656,
+      "eval_loss": 0.6831239461898804,
+      "eval_rewards/accuracies": 0.6057156324386597,
+      "eval_rewards/chosen": -0.0899965912103653,
+      "eval_rewards/margins": 0.022678693756461143,
+      "eval_rewards/rejected": -0.11267527937889099,
+      "eval_runtime": 383.1148,
+      "eval_samples_per_second": 11.234,
+      "eval_steps_per_second": 1.404,
+      "step": 6700
+    },
+    {
+      "epoch": 1.1560992419021365,
+      "grad_norm": 3.5776305198669434,
+      "learning_rate": 2.2556819520866827e-08,
+      "logits/chosen": -2.904379367828369,
+      "logits/rejected": -2.8792219161987305,
+      "logps/chosen": -69.15966796875,
+      "logps/rejected": -71.14330291748047,
+      "loss": 0.6691,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.1223447322845459,
+      "rewards/margins": 0.05360458046197891,
+      "rewards/rejected": -0.1759493201971054,
+      "step": 6710
+    },
+    {
+      "epoch": 1.1578221915920055,
+      "grad_norm": 3.6955976486206055,
+      "learning_rate": 2.2482011244373356e-08,
+      "logits/chosen": -2.8980813026428223,
+      "logits/rejected": -2.8855602741241455,
+      "logps/chosen": -64.0992660522461,
+      "logps/rejected": -69.89405059814453,
+      "loss": 0.6673,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.12684378027915955,
+      "rewards/margins": 0.05660554766654968,
+      "rewards/rejected": -0.18344931304454803,
+      "step": 6720
+    },
+    {
+      "epoch": 1.1595451412818747,
+      "grad_norm": 3.587991952896118,
+      "learning_rate": 2.2407225738246073e-08,
+      "logits/chosen": -2.851447820663452,
+      "logits/rejected": -2.8356666564941406,
+      "logps/chosen": -69.56204986572266,
+      "logps/rejected": -72.05616760253906,
+      "loss": 0.6756,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.12861190736293793,
+      "rewards/margins": 0.03918178752064705,
+      "rewards/rejected": -0.16779370605945587,
+      "step": 6730
+    },
+    {
+      "epoch": 1.1612680909717437,
+      "grad_norm": 3.624330759048462,
+      "learning_rate": 2.233246367877609e-08,
+      "logits/chosen": -2.9220564365386963,
+      "logits/rejected": -2.9138286113739014,
+      "logps/chosen": -67.34629821777344,
+      "logps/rejected": -73.40733337402344,
+      "loss": 0.6732,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.1291312575340271,
+      "rewards/margins": 0.04403195157647133,
+      "rewards/rejected": -0.17316320538520813,
+      "step": 6740
+    },
+    {
+      "epoch": 1.1629910406616126,
+      "grad_norm": 3.451568365097046,
+      "learning_rate": 2.2257725742042437e-08,
+      "logits/chosen": -2.9607136249542236,
+      "logits/rejected": -2.9462342262268066,
+      "logps/chosen": -67.38319396972656,
+      "logps/rejected": -72.05055236816406,
+      "loss": 0.6782,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.14925019443035126,
+      "rewards/margins": 0.0343804769217968,
+      "rewards/rejected": -0.18363067507743835,
+      "step": 6750
+    },
+    {
+      "epoch": 1.1647139903514818,
+      "grad_norm": 3.6338393688201904,
+      "learning_rate": 2.2183012603906064e-08,
+      "logits/chosen": -2.9064667224884033,
+      "logits/rejected": -2.8790745735168457,
+      "logps/chosen": -65.61817169189453,
+      "logps/rejected": -67.46910095214844,
+      "loss": 0.6696,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.1356475055217743,
+      "rewards/margins": 0.051673442125320435,
+      "rewards/rejected": -0.18732096254825592,
+      "step": 6760
+    },
+    {
+      "epoch": 1.1664369400413508,
+      "grad_norm": 3.337958812713623,
+      "learning_rate": 2.2108324940003607e-08,
+      "logits/chosen": -2.9203834533691406,
+      "logits/rejected": -2.914304733276367,
+      "logps/chosen": -66.03895568847656,
+      "logps/rejected": -73.11302185058594,
+      "loss": 0.6688,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.12204036861658096,
+      "rewards/margins": 0.05278633162379265,
+      "rewards/rejected": -0.1748267114162445,
+      "step": 6770
+    },
+    {
+      "epoch": 1.1681598897312198,
+      "grad_norm": 3.8971357345581055,
+      "learning_rate": 2.2033663425741377e-08,
+      "logits/chosen": -2.9164557456970215,
+      "logits/rejected": -2.8855154514312744,
+      "logps/chosen": -70.88284301757812,
+      "logps/rejected": -71.56888580322266,
+      "loss": 0.6761,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.143809512257576,
+      "rewards/margins": 0.03861542046070099,
+      "rewards/rejected": -0.18242493271827698,
+      "step": 6780
+    },
+    {
+      "epoch": 1.169882839421089,
+      "grad_norm": 3.5548527240753174,
+      "learning_rate": 2.1959028736289184e-08,
+      "logits/chosen": -2.925858974456787,
+      "logits/rejected": -2.910332202911377,
+      "logps/chosen": -64.74264526367188,
+      "logps/rejected": -71.9747314453125,
+      "loss": 0.672,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.14218708872795105,
+      "rewards/margins": 0.04636620730161667,
+      "rewards/rejected": -0.18855327367782593,
+      "step": 6790
+    },
+    {
+      "epoch": 1.171605789110958,
+      "grad_norm": 3.188309669494629,
+      "learning_rate": 2.1884421546574288e-08,
+      "logits/chosen": -2.8431239128112793,
+      "logits/rejected": -2.8163630962371826,
+      "logps/chosen": -66.12577819824219,
+      "logps/rejected": -69.16600036621094,
+      "loss": 0.6688,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.13707414269447327,
+      "rewards/margins": 0.0536266565322876,
+      "rewards/rejected": -0.19070079922676086,
+      "step": 6800
+    },
+    {
+      "epoch": 1.171605789110958,
+      "eval_logits/chosen": -3.0017476081848145,
+      "eval_logits/rejected": -2.9959821701049805,
+      "eval_logps/chosen": -67.97056579589844,
+      "eval_logps/rejected": -74.7659683227539,
+      "eval_loss": 0.6828659176826477,
+      "eval_rewards/accuracies": 0.6089683771133423,
+      "eval_rewards/chosen": -0.09258671849966049,
+      "eval_rewards/margins": 0.023271720856428146,
+      "eval_rewards/rejected": -0.11585843563079834,
+      "eval_runtime": 383.0834,
+      "eval_samples_per_second": 11.235,
+      "eval_steps_per_second": 1.404,
+      "step": 6800
+    },
+    {
+      "epoch": 1.173328738800827,
+      "grad_norm": 3.1243183612823486,
+      "learning_rate": 2.180984253127523e-08,
+      "logits/chosen": -2.8966526985168457,
+      "logits/rejected": -2.87964129447937,
+      "logps/chosen": -67.34037017822266,
+      "logps/rejected": -69.68751525878906,
+      "loss": 0.6641,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.1163669228553772,
+      "rewards/margins": 0.0630408525466919,
+      "rewards/rejected": -0.17940779030323029,
+      "step": 6810
+    },
+    {
+      "epoch": 1.175051688490696,
+      "grad_norm": 3.325390577316284,
+      "learning_rate": 2.173529236481581e-08,
+      "logits/chosen": -2.994617223739624,
+      "logits/rejected": -2.9643521308898926,
+      "logps/chosen": -68.11412048339844,
+      "logps/rejected": -71.9454345703125,
+      "loss": 0.666,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1309063583612442,
+      "rewards/margins": 0.058373190462589264,
+      "rewards/rejected": -0.18927955627441406,
+      "step": 6820
+    },
+    {
+      "epoch": 1.176774638180565,
+      "grad_norm": 3.510547637939453,
+      "learning_rate": 2.1660771721358898e-08,
+      "logits/chosen": -3.0016026496887207,
+      "logits/rejected": -2.9947669506073,
+      "logps/chosen": -67.66027069091797,
+      "logps/rejected": -69.55329895019531,
+      "loss": 0.6815,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.13756820559501648,
+      "rewards/margins": 0.027839258313179016,
+      "rewards/rejected": -0.1654074490070343,
+      "step": 6830
+    },
+    {
+      "epoch": 1.1784975878704342,
+      "grad_norm": 3.824273109436035,
+      "learning_rate": 2.1586281274800433e-08,
+      "logits/chosen": -2.962636947631836,
+      "logits/rejected": -2.937431573867798,
+      "logps/chosen": -70.20149993896484,
+      "logps/rejected": -72.98335266113281,
+      "loss": 0.674,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.13270917534828186,
+      "rewards/margins": 0.043135203421115875,
+      "rewards/rejected": -0.17584435641765594,
+      "step": 6840
+    },
+    {
+      "epoch": 1.1802205375603032,
+      "grad_norm": 3.1507015228271484,
+      "learning_rate": 2.1511821698763248e-08,
+      "logits/chosen": -2.8106720447540283,
+      "logits/rejected": -2.7794477939605713,
+      "logps/chosen": -67.33164978027344,
+      "logps/rejected": -72.02129364013672,
+      "loss": 0.6696,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1316729485988617,
+      "rewards/margins": 0.05213087797164917,
+      "rewards/rejected": -0.18380382657051086,
+      "step": 6850
+    },
+    {
+      "epoch": 1.1819434872501722,
+      "grad_norm": 3.2910120487213135,
+      "learning_rate": 2.143739366659102e-08,
+      "logits/chosen": -2.9226691722869873,
+      "logits/rejected": -2.88592267036438,
+      "logps/chosen": -72.56702423095703,
+      "logps/rejected": -71.2208251953125,
+      "loss": 0.6681,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.13452334702014923,
+      "rewards/margins": 0.054468195885419846,
+      "rewards/rejected": -0.18899156153202057,
+      "step": 6860
+    },
+    {
+      "epoch": 1.1836664369400414,
+      "grad_norm": 3.0284669399261475,
+      "learning_rate": 2.1362997851342184e-08,
+      "logits/chosen": -2.8399906158447266,
+      "logits/rejected": -2.82886004447937,
+      "logps/chosen": -66.54041290283203,
+      "logps/rejected": -70.11848449707031,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.1545497179031372,
+      "rewards/margins": 0.0289381705224514,
+      "rewards/rejected": -0.1834879070520401,
+      "step": 6870
+    },
+    {
+      "epoch": 1.1853893866299103,
+      "grad_norm": 3.463501453399658,
+      "learning_rate": 2.1288634925783816e-08,
+      "logits/chosen": -2.95865535736084,
+      "logits/rejected": -2.928438663482666,
+      "logps/chosen": -68.41751861572266,
+      "logps/rejected": -70.52119445800781,
+      "loss": 0.6696,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.14154596626758575,
+      "rewards/margins": 0.052053045481443405,
+      "rewards/rejected": -0.19359901547431946,
+      "step": 6880
+    },
+    {
+      "epoch": 1.1871123363197795,
+      "grad_norm": 3.155576229095459,
+      "learning_rate": 2.1214305562385588e-08,
+      "logits/chosen": -2.8650355339050293,
+      "logits/rejected": -2.8389101028442383,
+      "logps/chosen": -64.65052795410156,
+      "logps/rejected": -70.58853912353516,
+      "loss": 0.6664,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.12340199947357178,
+      "rewards/margins": 0.058589957654476166,
+      "rewards/rejected": -0.18199196457862854,
+      "step": 6890
+    },
+    {
+      "epoch": 1.1888352860096485,
+      "grad_norm": 3.789573907852173,
+      "learning_rate": 2.1140010433313643e-08,
+      "logits/chosen": -2.936095714569092,
+      "logits/rejected": -2.9149789810180664,
+      "logps/chosen": -67.76901245117188,
+      "logps/rejected": -70.50746154785156,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.14214427769184113,
+      "rewards/margins": 0.028564786538481712,
+      "rewards/rejected": -0.1707090437412262,
+      "step": 6900
+    },
+    {
+      "epoch": 1.1888352860096485,
+      "eval_logits/chosen": -2.999910593032837,
+      "eval_logits/rejected": -2.9941437244415283,
+      "eval_logps/chosen": -68.13446807861328,
+      "eval_logps/rejected": -74.94407653808594,
+      "eval_loss": 0.6828203797340393,
+      "eval_rewards/accuracies": 0.6061803102493286,
+      "eval_rewards/chosen": -0.09422574192285538,
+      "eval_rewards/margins": 0.023413825780153275,
+      "eval_rewards/rejected": -0.11763957887887955,
+      "eval_runtime": 383.1036,
+      "eval_samples_per_second": 11.235,
+      "eval_steps_per_second": 1.404,
+      "step": 6900
+    },
+    {
+      "epoch": 1.1905582356995175,
+      "grad_norm": 3.6955511569976807,
+      "learning_rate": 2.106575021042457e-08,
+      "logits/chosen": -2.9674341678619385,
+      "logits/rejected": -2.9427692890167236,
+      "logps/chosen": -69.85762023925781,
+      "logps/rejected": -73.43937683105469,
+      "loss": 0.669,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.12971599400043488,
+      "rewards/margins": 0.05261234566569328,
+      "rewards/rejected": -0.18232834339141846,
+      "step": 6910
+    },
+    {
+      "epoch": 1.1922811853893867,
+      "grad_norm": 3.612447738647461,
+      "learning_rate": 2.099152556525926e-08,
+      "logits/chosen": -3.0253255367279053,
+      "logits/rejected": -2.994903564453125,
+      "logps/chosen": -69.91947174072266,
+      "logps/rejected": -72.19439697265625,
+      "loss": 0.6724,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.13815486431121826,
+      "rewards/margins": 0.04608771950006485,
+      "rewards/rejected": -0.18424257636070251,
+      "step": 6920
+    },
+    {
+      "epoch": 1.1940041350792556,
+      "grad_norm": 3.5276856422424316,
+      "learning_rate": 2.0917337169036925e-08,
+      "logits/chosen": -2.8367297649383545,
+      "logits/rejected": -2.809800863265991,
+      "logps/chosen": -69.68663787841797,
+      "logps/rejected": -72.41487884521484,
+      "loss": 0.6652,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.12052768468856812,
+      "rewards/margins": 0.06110203266143799,
+      "rewards/rejected": -0.1816297322511673,
+      "step": 6930
+    },
+    {
+      "epoch": 1.1957270847691248,
+      "grad_norm": 3.527961254119873,
+      "learning_rate": 2.0843185692648913e-08,
+      "logits/chosen": -2.8302559852600098,
+      "logits/rejected": -2.7844839096069336,
+      "logps/chosen": -70.59068298339844,
+      "logps/rejected": -69.39749145507812,
+      "loss": 0.6645,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.12627747654914856,
+      "rewards/margins": 0.06178699806332588,
+      "rewards/rejected": -0.18806448578834534,
+      "step": 6940
+    },
+    {
+      "epoch": 1.1974500344589938,
+      "grad_norm": 3.3442578315734863,
+      "learning_rate": 2.076907180665276e-08,
+      "logits/chosen": -2.894475221633911,
+      "logits/rejected": -2.8761637210845947,
+      "logps/chosen": -66.38448333740234,
+      "logps/rejected": -71.7999496459961,
+      "loss": 0.664,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.12527704238891602,
+      "rewards/margins": 0.06334027647972107,
+      "rewards/rejected": -0.18861731886863708,
+      "step": 6950
+    },
+    {
+      "epoch": 1.1991729841488628,
+      "grad_norm": 3.4445013999938965,
+      "learning_rate": 2.0694996181266027e-08,
+      "logits/chosen": -3.061396360397339,
+      "logits/rejected": -3.022015333175659,
+      "logps/chosen": -70.1845703125,
+      "logps/rejected": -66.83232116699219,
+      "loss": 0.6745,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.14866146445274353,
+      "rewards/margins": 0.04185408353805542,
+      "rewards/rejected": -0.19051553308963776,
+      "step": 6960
+    },
+    {
+      "epoch": 1.200895933838732,
+      "grad_norm": 3.206153392791748,
+      "learning_rate": 2.0620959486360313e-08,
+      "logits/chosen": -2.9727296829223633,
+      "logits/rejected": -2.9388582706451416,
+      "logps/chosen": -67.38151550292969,
+      "logps/rejected": -68.72689819335938,
+      "loss": 0.668,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.1283971071243286,
+      "rewards/margins": 0.05497384071350098,
+      "rewards/rejected": -0.1833709478378296,
+      "step": 6970
+    },
+    {
+      "epoch": 1.202618883528601,
+      "grad_norm": 3.3232524394989014,
+      "learning_rate": 2.0546962391455128e-08,
+      "logits/chosen": -2.874843120574951,
+      "logits/rejected": -2.8454675674438477,
+      "logps/chosen": -67.70455932617188,
+      "logps/rejected": -72.18318176269531,
+      "loss": 0.6668,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.13479213416576385,
+      "rewards/margins": 0.05696721002459526,
+      "rewards/rejected": -0.19175934791564941,
+      "step": 6980
+    },
+    {
+      "epoch": 1.20434183321847,
+      "grad_norm": 3.6219253540039062,
+      "learning_rate": 2.0473005565711924e-08,
+      "logits/chosen": -2.7958545684814453,
+      "logits/rejected": -2.7859947681427,
+      "logps/chosen": -71.47138977050781,
+      "logps/rejected": -72.26944732666016,
+      "loss": 0.6775,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.14487969875335693,
+      "rewards/margins": 0.035188473761081696,
+      "rewards/rejected": -0.18006817996501923,
+      "step": 6990
+    },
+    {
+      "epoch": 1.206064782908339,
+      "grad_norm": 3.821153163909912,
+      "learning_rate": 2.039908967792795e-08,
+      "logits/chosen": -3.0873427391052246,
+      "logits/rejected": -3.0599758625030518,
+      "logps/chosen": -71.40188598632812,
+      "logps/rejected": -75.17681884765625,
+      "loss": 0.6691,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.15107527375221252,
+      "rewards/margins": 0.053789108991622925,
+      "rewards/rejected": -0.20486438274383545,
+      "step": 7000
+    },
+    {
+      "epoch": 1.206064782908339,
+      "eval_logits/chosen": -2.997684955596924,
+      "eval_logits/rejected": -2.991889715194702,
+      "eval_logps/chosen": -68.35706329345703,
+      "eval_logps/rejected": -75.20162963867188,
+      "eval_loss": 0.6826760768890381,
+      "eval_rewards/accuracies": 0.6071096658706665,
+      "eval_rewards/chosen": -0.09645169973373413,
+      "eval_rewards/margins": 0.02376333251595497,
+      "eval_rewards/rejected": -0.1202150359749794,
+      "eval_runtime": 383.0887,
+      "eval_samples_per_second": 11.235,
+      "eval_steps_per_second": 1.404,
+      "step": 7000
+    },
+    {
+      "epoch": 1.207787732598208,
+      "grad_norm": 3.63569712638855,
+      "learning_rate": 2.0325215396530286e-08,
+      "logits/chosen": -2.902103900909424,
+      "logits/rejected": -2.875323534011841,
+      "logps/chosen": -70.37360382080078,
+      "logps/rejected": -72.7172622680664,
+      "loss": 0.6662,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.13948360085487366,
+      "rewards/margins": 0.0590650849044323,
+      "rewards/rejected": -0.19854867458343506,
+      "step": 7010
+    },
+    {
+      "epoch": 1.2095106822880772,
+      "grad_norm": 3.4854214191436768,
+      "learning_rate": 2.025138338956974e-08,
+      "logits/chosen": -2.889186382293701,
+      "logits/rejected": -2.8632805347442627,
+      "logps/chosen": -65.40814208984375,
+      "logps/rejected": -71.50416564941406,
+      "loss": 0.6701,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.14073331654071808,
+      "rewards/margins": 0.050722360610961914,
+      "rewards/rejected": -0.1914556622505188,
+      "step": 7020
+    },
+    {
+      "epoch": 1.2112336319779462,
+      "grad_norm": 3.174516439437866,
+      "learning_rate": 2.0177594324714838e-08,
+      "logits/chosen": -2.927429437637329,
+      "logits/rejected": -2.9119021892547607,
+      "logps/chosen": -68.04973602294922,
+      "logps/rejected": -73.51303100585938,
+      "loss": 0.6782,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.14949563145637512,
+      "rewards/margins": 0.034495383501052856,
+      "rewards/rejected": -0.18399102985858917,
+      "step": 7030
+    },
+    {
+      "epoch": 1.2129565816678154,
+      "grad_norm": 3.650618553161621,
+      "learning_rate": 2.0103848869245765e-08,
+      "logits/chosen": -2.869797706604004,
+      "logits/rejected": -2.8489603996276855,
+      "logps/chosen": -69.14947509765625,
+      "logps/rejected": -75.38789367675781,
+      "loss": 0.6666,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1307271271944046,
+      "rewards/margins": 0.05817770957946777,
+      "rewards/rejected": -0.18890480697155,
+      "step": 7040
+    },
+    {
+      "epoch": 1.2146795313576844,
+      "grad_norm": 3.9517900943756104,
+      "learning_rate": 2.0030147690048372e-08,
+      "logits/chosen": -2.843709707260132,
+      "logits/rejected": -2.822296619415283,
+      "logps/chosen": -65.77268981933594,
+      "logps/rejected": -73.33934783935547,
+      "loss": 0.6674,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.1431690752506256,
+      "rewards/margins": 0.05644227936863899,
+      "rewards/rejected": -0.1996113508939743,
+      "step": 7050
+    },
+    {
+      "epoch": 1.2164024810475533,
+      "grad_norm": 4.204028606414795,
+      "learning_rate": 1.995649145360809e-08,
+      "logits/chosen": -2.9585540294647217,
+      "logits/rejected": -2.9441981315612793,
+      "logps/chosen": -69.17874908447266,
+      "logps/rejected": -72.58383178710938,
+      "loss": 0.6767,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.15968480706214905,
+      "rewards/margins": 0.037026528269052505,
+      "rewards/rejected": -0.19671133160591125,
+      "step": 7060
+    },
+    {
+      "epoch": 1.2181254307374225,
+      "grad_norm": 3.537989377975464,
+      "learning_rate": 1.988288082600392e-08,
+      "logits/chosen": -2.9185822010040283,
+      "logits/rejected": -2.8918845653533936,
+      "logps/chosen": -68.83052062988281,
+      "logps/rejected": -70.64710235595703,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.14227482676506042,
+      "rewards/margins": 0.03171732276678085,
+      "rewards/rejected": -0.17399214208126068,
+      "step": 7070
+    },
+    {
+      "epoch": 1.2198483804272915,
+      "grad_norm": 3.6491990089416504,
+      "learning_rate": 1.980931647290246e-08,
+      "logits/chosen": -2.9805405139923096,
+      "logits/rejected": -2.9504735469818115,
+      "logps/chosen": -67.77733612060547,
+      "logps/rejected": -73.85661315917969,
+      "loss": 0.6673,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.14309200644493103,
+      "rewards/margins": 0.05582382157444954,
+      "rewards/rejected": -0.19891582429409027,
+      "step": 7080
+    },
+    {
+      "epoch": 1.2215713301171607,
+      "grad_norm": 3.359250783920288,
+      "learning_rate": 1.97357990595518e-08,
+      "logits/chosen": -3.0099129676818848,
+      "logits/rejected": -2.9990859031677246,
+      "logps/chosen": -68.89093780517578,
+      "logps/rejected": -72.12596130371094,
+      "loss": 0.6726,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.15373072028160095,
+      "rewards/margins": 0.04542947933077812,
+      "rewards/rejected": -0.19916021823883057,
+      "step": 7090
+    },
+    {
+      "epoch": 1.2232942798070296,
+      "grad_norm": 3.2894628047943115,
+      "learning_rate": 1.9662329250775585e-08,
+      "logits/chosen": -2.8732142448425293,
+      "logits/rejected": -2.8523426055908203,
+      "logps/chosen": -70.81654357910156,
+      "logps/rejected": -75.62577819824219,
+      "loss": 0.6704,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.13743259012699127,
+      "rewards/margins": 0.05034102126955986,
+      "rewards/rejected": -0.18777363002300262,
+      "step": 7100
+    },
+    {
+      "epoch": 1.2232942798070296,
+      "eval_logits/chosen": -2.9956254959106445,
+      "eval_logits/rejected": -2.989813804626465,
+      "eval_logps/chosen": -68.40953063964844,
+      "eval_logps/rejected": -75.25898742675781,
+      "eval_loss": 0.6826616525650024,
+      "eval_rewards/accuracies": 0.6029275059700012,
+      "eval_rewards/chosen": -0.09697633236646652,
+      "eval_rewards/margins": 0.02381235733628273,
+      "eval_rewards/rejected": -0.12078870087862015,
+      "eval_runtime": 383.2548,
+      "eval_samples_per_second": 11.23,
+      "eval_steps_per_second": 1.404,
+      "step": 7100
+    },
+    {
+      "epoch": 1.2250172294968986,
+      "grad_norm": 3.161003828048706,
+      "learning_rate": 1.9588907710966942e-08,
+      "logits/chosen": -2.9268698692321777,
+      "logits/rejected": -2.8894760608673096,
+      "logps/chosen": -67.5369873046875,
+      "logps/rejected": -69.0163803100586,
+      "loss": 0.6749,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.14064155519008636,
+      "rewards/margins": 0.039694894105196,
+      "rewards/rejected": -0.18033643066883087,
+      "step": 7110
+    },
+    {
+      "epoch": 1.2267401791867678,
+      "grad_norm": 3.440458059310913,
+      "learning_rate": 1.951553510408252e-08,
+      "logits/chosen": -2.9131410121917725,
+      "logits/rejected": -2.870790481567383,
+      "logps/chosen": -74.72032165527344,
+      "logps/rejected": -70.45793914794922,
+      "loss": 0.6693,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.12734079360961914,
+      "rewards/margins": 0.05164145305752754,
+      "rewards/rejected": -0.17898224294185638,
+      "step": 7120
+    },
+    {
+      "epoch": 1.2284631288766368,
+      "grad_norm": 3.6218011379241943,
+      "learning_rate": 1.9442212093636433e-08,
+      "logits/chosen": -2.79545259475708,
+      "logits/rejected": -2.7689714431762695,
+      "logps/chosen": -69.40068054199219,
+      "logps/rejected": -71.83159637451172,
+      "loss": 0.668,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.13059434294700623,
+      "rewards/margins": 0.05593717843294144,
+      "rewards/rejected": -0.18653151392936707,
+      "step": 7130
+    },
+    {
+      "epoch": 1.230186078566506,
+      "grad_norm": 3.378696918487549,
+      "learning_rate": 1.936893934269433e-08,
+      "logits/chosen": -2.9058754444122314,
+      "logits/rejected": -2.9013783931732178,
+      "logps/chosen": -65.583251953125,
+      "logps/rejected": -70.20563507080078,
+      "loss": 0.6752,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.14265577495098114,
+      "rewards/margins": 0.03995499014854431,
+      "rewards/rejected": -0.18261078000068665,
+      "step": 7140
+    },
+    {
+      "epoch": 1.231909028256375,
+      "grad_norm": 3.7034966945648193,
+      "learning_rate": 1.9295717513867323e-08,
+      "logits/chosen": -2.9814138412475586,
+      "logits/rejected": -2.9581286907196045,
+      "logps/chosen": -71.87617492675781,
+      "logps/rejected": -75.21700286865234,
+      "loss": 0.6728,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.1548866480588913,
+      "rewards/margins": 0.04638432711362839,
+      "rewards/rejected": -0.20127101242542267,
+      "step": 7150
+    },
+    {
+      "epoch": 1.233631977946244,
+      "grad_norm": 3.353004217147827,
+      "learning_rate": 1.922254726930607e-08,
+      "logits/chosen": -2.892482280731201,
+      "logits/rejected": -2.8549489974975586,
+      "logps/chosen": -68.1994400024414,
+      "logps/rejected": -74.1641845703125,
+      "loss": 0.6677,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.14537674188613892,
+      "rewards/margins": 0.055488426238298416,
+      "rewards/rejected": -0.20086519420146942,
+      "step": 7160
+    },
+    {
+      "epoch": 1.235354927636113,
+      "grad_norm": 3.804107904434204,
+      "learning_rate": 1.9149429270694706e-08,
+      "logits/chosen": -2.8557260036468506,
+      "logits/rejected": -2.8288421630859375,
+      "logps/chosen": -70.63117980957031,
+      "logps/rejected": -72.84175872802734,
+      "loss": 0.6759,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.15064874291419983,
+      "rewards/margins": 0.0396803617477417,
+      "rewards/rejected": -0.19032913446426392,
+      "step": 7170
+    },
+    {
+      "epoch": 1.237077877325982,
+      "grad_norm": 3.530073404312134,
+      "learning_rate": 1.9076364179244935e-08,
+      "logits/chosen": -2.985349178314209,
+      "logits/rejected": -2.9716153144836426,
+      "logps/chosen": -68.9859848022461,
+      "logps/rejected": -73.22499084472656,
+      "loss": 0.6697,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.1368376761674881,
+      "rewards/margins": 0.050689779222011566,
+      "rewards/rejected": -0.18752744793891907,
+      "step": 7180
+    },
+    {
+      "epoch": 1.2388008270158513,
+      "grad_norm": 3.4539577960968018,
+      "learning_rate": 1.9003352655689992e-08,
+      "logits/chosen": -2.8350939750671387,
+      "logits/rejected": -2.8111681938171387,
+      "logps/chosen": -67.84815979003906,
+      "logps/rejected": -73.03033447265625,
+      "loss": 0.6746,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.15177632868289948,
+      "rewards/margins": 0.0423094816505909,
+      "rewards/rejected": -0.19408579170703888,
+      "step": 7190
+    },
+    {
+      "epoch": 1.2405237767057202,
+      "grad_norm": 4.251198768615723,
+      "learning_rate": 1.8930395360278723e-08,
+      "logits/chosen": -2.850398063659668,
+      "logits/rejected": -2.8331491947174072,
+      "logps/chosen": -67.74391174316406,
+      "logps/rejected": -76.11796569824219,
+      "loss": 0.6693,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.14646559953689575,
+      "rewards/margins": 0.05297350883483887,
+      "rewards/rejected": -0.19943909347057343,
+      "step": 7200
+    },
+    {
+      "epoch": 1.2405237767057202,
+      "eval_logits/chosen": -2.9932336807250977,
+      "eval_logits/rejected": -2.987468719482422,
+      "eval_logps/chosen": -68.55750274658203,
+      "eval_logps/rejected": -75.44210052490234,
+      "eval_loss": 0.6825125813484192,
+      "eval_rewards/accuracies": 0.607342004776001,
+      "eval_rewards/chosen": -0.09845609962940216,
+      "eval_rewards/margins": 0.024163702502846718,
+      "eval_rewards/rejected": -0.12261981517076492,
+      "eval_runtime": 383.0971,
+      "eval_samples_per_second": 11.235,
+      "eval_steps_per_second": 1.404,
+      "step": 7200
+    },
+    {
+      "epoch": 1.2422467263955892,
+      "grad_norm": 3.5033295154571533,
+      "learning_rate": 1.885749295276955e-08,
+      "logits/chosen": -2.9687461853027344,
+      "logits/rejected": -2.9471702575683594,
+      "logps/chosen": -71.31263732910156,
+      "logps/rejected": -71.05479431152344,
+      "loss": 0.6765,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.14283940196037292,
+      "rewards/margins": 0.03741271793842316,
+      "rewards/rejected": -0.18025213479995728,
+      "step": 7210
+    },
+    {
+      "epoch": 1.2439696760854584,
+      "grad_norm": 3.4362106323242188,
+      "learning_rate": 1.878464609242457e-08,
+      "logits/chosen": -2.835714101791382,
+      "logits/rejected": -2.8010780811309814,
+      "logps/chosen": -68.98710632324219,
+      "logps/rejected": -71.03915405273438,
+      "loss": 0.6713,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.14457109570503235,
+      "rewards/margins": 0.04917687922716141,
+      "rewards/rejected": -0.19374799728393555,
+      "step": 7220
+    },
+    {
+      "epoch": 1.2456926257753274,
+      "grad_norm": 3.6216726303100586,
+      "learning_rate": 1.8711855438003542e-08,
+      "logits/chosen": -2.870424747467041,
+      "logits/rejected": -2.8504726886749268,
+      "logps/chosen": -69.70048522949219,
+      "logps/rejected": -71.2332992553711,
+      "loss": 0.6812,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.15479105710983276,
+      "rewards/margins": 0.027741143479943275,
+      "rewards/rejected": -0.1825322061777115,
+      "step": 7230
+    },
+    {
+      "epoch": 1.2474155754651963,
+      "grad_norm": 3.6008265018463135,
+      "learning_rate": 1.8639121647757975e-08,
+      "logits/chosen": -2.903451919555664,
+      "logits/rejected": -2.8922200202941895,
+      "logps/chosen": -72.42183685302734,
+      "logps/rejected": -74.26264953613281,
+      "loss": 0.6729,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.14975066483020782,
+      "rewards/margins": 0.04515998438000679,
+      "rewards/rejected": -0.1949106603860855,
+      "step": 7240
+    },
+    {
+      "epoch": 1.2491385251550655,
+      "grad_norm": 3.8401882648468018,
+      "learning_rate": 1.8566445379425115e-08,
+      "logits/chosen": -2.9538257122039795,
+      "logits/rejected": -2.9233176708221436,
+      "logps/chosen": -70.17655181884766,
+      "logps/rejected": -72.37318420410156,
+      "loss": 0.6757,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.15608899295330048,
+      "rewards/margins": 0.0390971302986145,
+      "rewards/rejected": -0.19518613815307617,
+      "step": 7250
+    },
+    {
+      "epoch": 1.2508614748449345,
+      "grad_norm": 3.372586727142334,
+      "learning_rate": 1.849382729022207e-08,
+      "logits/chosen": -2.9463207721710205,
+      "logits/rejected": -2.9135968685150146,
+      "logps/chosen": -68.3741683959961,
+      "logps/rejected": -72.347412109375,
+      "loss": 0.669,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.14688396453857422,
+      "rewards/margins": 0.0528654046356678,
+      "rewards/rejected": -0.19974938035011292,
+      "step": 7260
+    },
+    {
+      "epoch": 1.2525844245348035,
+      "grad_norm": 3.5523462295532227,
+      "learning_rate": 1.8421268036839798e-08,
+      "logits/chosen": -2.961359739303589,
+      "logits/rejected": -2.931241512298584,
+      "logps/chosen": -68.33455657958984,
+      "logps/rejected": -72.05867004394531,
+      "loss": 0.6682,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.14018899202346802,
+      "rewards/margins": 0.054046880453825,
+      "rewards/rejected": -0.1942358911037445,
+      "step": 7270
+    },
+    {
+      "epoch": 1.2543073742246726,
+      "grad_norm": 3.7690281867980957,
+      "learning_rate": 1.834876827543721e-08,
+      "logits/chosen": -2.991880416870117,
+      "logits/rejected": -2.9574661254882812,
+      "logps/chosen": -70.6148452758789,
+      "logps/rejected": -72.77406311035156,
+      "loss": 0.6663,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.14045098423957825,
+      "rewards/margins": 0.058186065405607224,
+      "rewards/rejected": -0.19863705337047577,
+      "step": 7280
+    },
+    {
+      "epoch": 1.2560303239145416,
+      "grad_norm": 3.3820347785949707,
+      "learning_rate": 1.827632866163525e-08,
+      "logits/chosen": -2.779134511947632,
+      "logits/rejected": -2.766244411468506,
+      "logps/chosen": -69.89405822753906,
+      "logps/rejected": -74.26659393310547,
+      "loss": 0.6784,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.15102741122245789,
+      "rewards/margins": 0.033007699996232986,
+      "rewards/rejected": -0.18403512239456177,
+      "step": 7290
+    },
+    {
+      "epoch": 1.2577532736044108,
+      "grad_norm": 3.549543619155884,
+      "learning_rate": 1.8203949850510903e-08,
+      "logits/chosen": -2.7366385459899902,
+      "logits/rejected": -2.720031261444092,
+      "logps/chosen": -64.93960571289062,
+      "logps/rejected": -69.50364685058594,
+      "loss": 0.6811,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.14678844809532166,
+      "rewards/margins": 0.02817346155643463,
+      "rewards/rejected": -0.1749618947505951,
+      "step": 7300
+    },
+    {
+      "epoch": 1.2577532736044108,
+      "eval_logits/chosen": -2.9913597106933594,
+      "eval_logits/rejected": -2.9855685234069824,
+      "eval_logps/chosen": -68.66927337646484,
+      "eval_logps/rejected": -75.56369018554688,
+      "eval_loss": 0.6824753880500793,
+      "eval_rewards/accuracies": 0.6045538783073425,
+      "eval_rewards/chosen": -0.09957372397184372,
+      "eval_rewards/margins": 0.024261943995952606,
+      "eval_rewards/rejected": -0.12383568286895752,
+      "eval_runtime": 383.1711,
+      "eval_samples_per_second": 11.233,
+      "eval_steps_per_second": 1.404,
+      "step": 7300
+    },
+    {
+      "epoch": 1.2594762232942798,
+      "grad_norm": 3.630915403366089,
+      "learning_rate": 1.8131632496591348e-08,
+      "logits/chosen": -2.9370474815368652,
+      "logits/rejected": -2.9136781692504883,
+      "logps/chosen": -70.0124282836914,
+      "logps/rejected": -71.20661163330078,
+      "loss": 0.674,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.14953410625457764,
+      "rewards/margins": 0.043674636632204056,
+      "rewards/rejected": -0.1932087391614914,
+      "step": 7310
+    },
+    {
+      "epoch": 1.2611991729841487,
+      "grad_norm": 3.6868808269500732,
+      "learning_rate": 1.8059377253847973e-08,
+      "logits/chosen": -2.9524471759796143,
+      "logits/rejected": -2.926992893218994,
+      "logps/chosen": -69.1163101196289,
+      "logps/rejected": -70.07896423339844,
+      "loss": 0.6755,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.14762753248214722,
+      "rewards/margins": 0.04160289093852043,
+      "rewards/rejected": -0.18923041224479675,
+      "step": 7320
+    },
+    {
+      "epoch": 1.262922122674018,
+      "grad_norm": 3.3020687103271484,
+      "learning_rate": 1.798718477569051e-08,
+      "logits/chosen": -2.8624227046966553,
+      "logits/rejected": -2.8358988761901855,
+      "logps/chosen": -66.63627624511719,
+      "logps/rejected": -71.26499938964844,
+      "loss": 0.6656,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.13573691248893738,
+      "rewards/margins": 0.06119762733578682,
+      "rewards/rejected": -0.1969345360994339,
+      "step": 7330
+    },
+    {
+      "epoch": 1.264645072363887,
+      "grad_norm": 3.743194341659546,
+      "learning_rate": 1.791505571496109e-08,
+      "logits/chosen": -2.925804615020752,
+      "logits/rejected": -2.898747444152832,
+      "logps/chosen": -69.90444946289062,
+      "logps/rejected": -73.09220886230469,
+      "loss": 0.6722,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.1452336311340332,
+      "rewards/margins": 0.046533238142728806,
+      "rewards/rejected": -0.1917668879032135,
+      "step": 7340
+    },
+    {
+      "epoch": 1.266368022053756,
+      "grad_norm": 3.6614413261413574,
+      "learning_rate": 1.7842990723928375e-08,
+      "logits/chosen": -2.974283456802368,
+      "logits/rejected": -2.9456067085266113,
+      "logps/chosen": -71.00596618652344,
+      "logps/rejected": -70.55510711669922,
+      "loss": 0.6719,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.14086617529392242,
+      "rewards/margins": 0.0469948835670948,
+      "rewards/rejected": -0.18786105513572693,
+      "step": 7350
+    },
+    {
+      "epoch": 1.268090971743625,
+      "grad_norm": 3.489405870437622,
+      "learning_rate": 1.7770990454281608e-08,
+      "logits/chosen": -2.8831534385681152,
+      "logits/rejected": -2.862837314605713,
+      "logps/chosen": -69.18780517578125,
+      "logps/rejected": -72.06382751464844,
+      "loss": 0.6707,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.14889904856681824,
+      "rewards/margins": 0.04988787695765495,
+      "rewards/rejected": -0.19878694415092468,
+      "step": 7360
+    },
+    {
+      "epoch": 1.269813921433494,
+      "grad_norm": 3.739339828491211,
+      "learning_rate": 1.7699055557124793e-08,
+      "logits/chosen": -2.777637004852295,
+      "logits/rejected": -2.756638288497925,
+      "logps/chosen": -67.82052612304688,
+      "logps/rejected": -69.82765197753906,
+      "loss": 0.6777,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.15014006197452545,
+      "rewards/margins": 0.03572598099708557,
+      "rewards/rejected": -0.18586604297161102,
+      "step": 7370
+    },
+    {
+      "epoch": 1.2715368711233632,
+      "grad_norm": 3.7095649242401123,
+      "learning_rate": 1.7627186682970725e-08,
+      "logits/chosen": -2.8828864097595215,
+      "logits/rejected": -2.862123966217041,
+      "logps/chosen": -68.27281188964844,
+      "logps/rejected": -73.02098846435547,
+      "loss": 0.674,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.1510503590106964,
+      "rewards/margins": 0.04228951781988144,
+      "rewards/rejected": -0.19333985447883606,
+      "step": 7380
+    },
+    {
+      "epoch": 1.2732598208132322,
+      "grad_norm": 3.908808708190918,
+      "learning_rate": 1.755538448173518e-08,
+      "logits/chosen": -2.8484606742858887,
+      "logits/rejected": -2.828400135040283,
+      "logps/chosen": -66.78551483154297,
+      "logps/rejected": -72.60951232910156,
+      "loss": 0.6704,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.1457177847623825,
+      "rewards/margins": 0.05018613487482071,
+      "rewards/rejected": -0.19590391218662262,
+      "step": 7390
+    },
+    {
+      "epoch": 1.2749827705031014,
+      "grad_norm": 3.412832736968994,
+      "learning_rate": 1.7483649602730987e-08,
+      "logits/chosen": -2.855675458908081,
+      "logits/rejected": -2.817229747772217,
+      "logps/chosen": -70.82234191894531,
+      "logps/rejected": -70.3524169921875,
+      "loss": 0.6731,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.15371280908584595,
+      "rewards/margins": 0.04410374164581299,
+      "rewards/rejected": -0.19781655073165894,
+      "step": 7400
+    },
+    {
+      "epoch": 1.2749827705031014,
+      "eval_logits/chosen": -2.9900894165039062,
+      "eval_logits/rejected": -2.98429274559021,
+      "eval_logps/chosen": -68.7872543334961,
+      "eval_logps/rejected": -75.71012115478516,
+      "eval_loss": 0.682349681854248,
+      "eval_rewards/accuracies": 0.6059479713439941,
+      "eval_rewards/chosen": -0.10075362771749496,
+      "eval_rewards/margins": 0.024546433240175247,
+      "eval_rewards/rejected": -0.12530004978179932,
+      "eval_runtime": 383.3288,
+      "eval_samples_per_second": 11.228,
+      "eval_steps_per_second": 1.403,
+      "step": 7400
+    },
+    {
+      "epoch": 1.2767057201929704,
+      "grad_norm": 3.7379229068756104,
+      "learning_rate": 1.741198269466219e-08,
+      "logits/chosen": -2.823806047439575,
+      "logits/rejected": -2.7933902740478516,
+      "logps/chosen": -68.06888580322266,
+      "logps/rejected": -71.93030548095703,
+      "loss": 0.6725,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.14312517642974854,
+      "rewards/margins": 0.045960646122694016,
+      "rewards/rejected": -0.18908584117889404,
+      "step": 7410
+    },
+    {
+      "epoch": 1.2784286698828393,
+      "grad_norm": 3.566232442855835,
+      "learning_rate": 1.7340384405618133e-08,
+      "logits/chosen": -2.7643885612487793,
+      "logits/rejected": -2.738656997680664,
+      "logps/chosen": -69.32161712646484,
+      "logps/rejected": -70.8636474609375,
+      "loss": 0.6722,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.13153675198554993,
+      "rewards/margins": 0.04642341658473015,
+      "rewards/rejected": -0.17796017229557037,
+      "step": 7420
+    },
+    {
+      "epoch": 1.2801516195727085,
+      "grad_norm": 3.7022876739501953,
+      "learning_rate": 1.7268855383067683e-08,
+      "logits/chosen": -2.820042610168457,
+      "logits/rejected": -2.791541576385498,
+      "logps/chosen": -69.34507751464844,
+      "logps/rejected": -72.44905853271484,
+      "loss": 0.6719,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.15940842032432556,
+      "rewards/margins": 0.046309854835271835,
+      "rewards/rejected": -0.20571830868721008,
+      "step": 7430
+    },
+    {
+      "epoch": 1.2818745692625775,
+      "grad_norm": 3.602687358856201,
+      "learning_rate": 1.7197396273853275e-08,
+      "logits/chosen": -2.9304871559143066,
+      "logits/rejected": -2.9020581245422363,
+      "logps/chosen": -72.24201965332031,
+      "logps/rejected": -75.16839599609375,
+      "loss": 0.6676,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1623719036579132,
+      "rewards/margins": 0.055669285356998444,
+      "rewards/rejected": -0.21804121136665344,
+      "step": 7440
+    },
+    {
+      "epoch": 1.2835975189524467,
+      "grad_norm": 4.144638538360596,
+      "learning_rate": 1.7126007724185164e-08,
+      "logits/chosen": -2.996082305908203,
+      "logits/rejected": -2.9688639640808105,
+      "logps/chosen": -70.65128326416016,
+      "logps/rejected": -72.15754699707031,
+      "loss": 0.6716,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.14725489914417267,
+      "rewards/margins": 0.04825368523597717,
+      "rewards/rejected": -0.19550856947898865,
+      "step": 7450
+    },
+    {
+      "epoch": 1.2853204686423156,
+      "grad_norm": 3.5552494525909424,
+      "learning_rate": 1.705469037963548e-08,
+      "logits/chosen": -2.8015012741088867,
+      "logits/rejected": -2.8049817085266113,
+      "logps/chosen": -64.37635803222656,
+      "logps/rejected": -71.40558624267578,
+      "loss": 0.6742,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.14214113354682922,
+      "rewards/margins": 0.042024023830890656,
+      "rewards/rejected": -0.18416516482830048,
+      "step": 7460
+    },
+    {
+      "epoch": 1.2870434183321846,
+      "grad_norm": 3.7495343685150146,
+      "learning_rate": 1.698344488513247e-08,
+      "logits/chosen": -2.893432140350342,
+      "logits/rejected": -2.8765432834625244,
+      "logps/chosen": -69.4414291381836,
+      "logps/rejected": -69.39237213134766,
+      "loss": 0.6793,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.14786800742149353,
+      "rewards/margins": 0.03292161971330643,
+      "rewards/rejected": -0.18078961968421936,
+      "step": 7470
+    },
+    {
+      "epoch": 1.2887663680220538,
+      "grad_norm": 3.1701583862304688,
+      "learning_rate": 1.691227188495461e-08,
+      "logits/chosen": -2.850665330886841,
+      "logits/rejected": -2.8234128952026367,
+      "logps/chosen": -68.8609848022461,
+      "logps/rejected": -67.68092346191406,
+      "loss": 0.6742,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.14228689670562744,
+      "rewards/margins": 0.04254503175616264,
+      "rewards/rejected": -0.184831902384758,
+      "step": 7480
+    },
+    {
+      "epoch": 1.2904893177119228,
+      "grad_norm": 3.644050121307373,
+      "learning_rate": 1.684117202272485e-08,
+      "logits/chosen": -2.81962251663208,
+      "logits/rejected": -2.80633544921875,
+      "logps/chosen": -67.72660827636719,
+      "logps/rejected": -73.80032348632812,
+      "loss": 0.6678,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.15166117250919342,
+      "rewards/margins": 0.05486512929201126,
+      "rewards/rejected": -0.20652630925178528,
+      "step": 7490
+    },
+    {
+      "epoch": 1.292212267401792,
+      "grad_norm": 3.158080816268921,
+      "learning_rate": 1.6770145941404697e-08,
+      "logits/chosen": -2.8429179191589355,
+      "logits/rejected": -2.818450450897217,
+      "logps/chosen": -66.32331085205078,
+      "logps/rejected": -70.9669189453125,
+      "loss": 0.6746,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.1484905183315277,
+      "rewards/margins": 0.04139673709869385,
+      "rewards/rejected": -0.18988727033138275,
+      "step": 7500
+    },
+    {
+      "epoch": 1.292212267401792,
+      "eval_logits/chosen": -2.988320827484131,
+      "eval_logits/rejected": -2.9825377464294434,
+      "eval_logps/chosen": -68.80451202392578,
+      "eval_logps/rejected": -75.74573516845703,
+      "eval_loss": 0.6822639107704163,
+      "eval_rewards/accuracies": 0.6036245226860046,
+      "eval_rewards/chosen": -0.10092618316411972,
+      "eval_rewards/margins": 0.024729974567890167,
+      "eval_rewards/rejected": -0.1256561577320099,
+      "eval_runtime": 383.2863,
+      "eval_samples_per_second": 11.229,
+      "eval_steps_per_second": 1.404,
+      "step": 7500
+    },
+    {
+      "epoch": 1.293935217091661,
+      "grad_norm": 3.778733491897583,
+      "learning_rate": 1.669919428328847e-08,
+      "logits/chosen": -2.876143217086792,
+      "logits/rejected": -2.843479871749878,
+      "logps/chosen": -71.58126068115234,
+      "logps/rejected": -70.61062622070312,
+      "loss": 0.6701,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.14320191740989685,
+      "rewards/margins": 0.05069415643811226,
+      "rewards/rejected": -0.1938960701227188,
+      "step": 7510
+    },
+    {
+      "epoch": 1.29565816678153,
+      "grad_norm": 3.550499439239502,
+      "learning_rate": 1.66283176899975e-08,
+      "logits/chosen": -2.849177837371826,
+      "logits/rejected": -2.8291099071502686,
+      "logps/chosen": -67.57173156738281,
+      "logps/rejected": -73.38545227050781,
+      "loss": 0.6715,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.14643962681293488,
+      "rewards/margins": 0.04884006828069687,
+      "rewards/rejected": -0.19527968764305115,
+      "step": 7520
+    },
+    {
+      "epoch": 1.297381116471399,
+      "grad_norm": 3.6232194900512695,
+      "learning_rate": 1.6557516802474246e-08,
+      "logits/chosen": -2.7914018630981445,
+      "logits/rejected": -2.789238452911377,
+      "logps/chosen": -66.90400695800781,
+      "logps/rejected": -71.94694519042969,
+      "loss": 0.6764,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.1468668431043625,
+      "rewards/margins": 0.03824695199728012,
+      "rewards/rejected": -0.1851138025522232,
+      "step": 7530
+    },
+    {
+      "epoch": 1.299104066161268,
+      "grad_norm": 3.3013417720794678,
+      "learning_rate": 1.648679226097662e-08,
+      "logits/chosen": -2.965933322906494,
+      "logits/rejected": -2.9589555263519287,
+      "logps/chosen": -65.70637512207031,
+      "logps/rejected": -71.29312896728516,
+      "loss": 0.673,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.14165833592414856,
+      "rewards/margins": 0.044515885412693024,
+      "rewards/rejected": -0.18617422878742218,
+      "step": 7540
+    },
+    {
+      "epoch": 1.3008270158511372,
+      "grad_norm": 3.6552910804748535,
+      "learning_rate": 1.641614470507207e-08,
+      "logits/chosen": -2.881110668182373,
+      "logits/rejected": -2.861151933670044,
+      "logps/chosen": -65.75182342529297,
+      "logps/rejected": -71.94224548339844,
+      "loss": 0.665,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.1396106779575348,
+      "rewards/margins": 0.06167968362569809,
+      "rewards/rejected": -0.20129036903381348,
+      "step": 7550
+    },
+    {
+      "epoch": 1.3025499655410062,
+      "grad_norm": 3.9385621547698975,
+      "learning_rate": 1.6345574773631897e-08,
+      "logits/chosen": -2.9588372707366943,
+      "logits/rejected": -2.9393017292022705,
+      "logps/chosen": -65.11895751953125,
+      "logps/rejected": -74.25119018554688,
+      "loss": 0.6637,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1254400908946991,
+      "rewards/margins": 0.06344287097454071,
+      "rewards/rejected": -0.188882976770401,
+      "step": 7560
+    },
+    {
+      "epoch": 1.3042729152308752,
+      "grad_norm": 3.7301559448242188,
+      "learning_rate": 1.627508310482541e-08,
+      "logits/chosen": -2.956386089324951,
+      "logits/rejected": -2.9388997554779053,
+      "logps/chosen": -71.42212677001953,
+      "logps/rejected": -72.64714050292969,
+      "loss": 0.6672,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.1462506204843521,
+      "rewards/margins": 0.057686250656843185,
+      "rewards/rejected": -0.2039368897676468,
+      "step": 7570
+    },
+    {
+      "epoch": 1.3059958649207444,
+      "grad_norm": 3.5880300998687744,
+      "learning_rate": 1.6204670336114223e-08,
+      "logits/chosen": -2.8970632553100586,
+      "logits/rejected": -2.8785603046417236,
+      "logps/chosen": -68.4981460571289,
+      "logps/rejected": -72.99632263183594,
+      "loss": 0.666,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.12484419345855713,
+      "rewards/margins": 0.05853307247161865,
+      "rewards/rejected": -0.18337729573249817,
+      "step": 7580
+    },
+    {
+      "epoch": 1.3077188146106133,
+      "grad_norm": 3.9595916271209717,
+      "learning_rate": 1.6134337104246395e-08,
+      "logits/chosen": -2.9393930435180664,
+      "logits/rejected": -2.8958966732025146,
+      "logps/chosen": -73.5398941040039,
+      "logps/rejected": -72.91618347167969,
+      "loss": 0.6647,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.15217044949531555,
+      "rewards/margins": 0.06287090480327606,
+      "rewards/rejected": -0.2150413691997528,
+      "step": 7590
+    },
+    {
+      "epoch": 1.3094417643004825,
+      "grad_norm": 3.3716061115264893,
+      "learning_rate": 1.6064084045250787e-08,
+      "logits/chosen": -2.90592885017395,
+      "logits/rejected": -2.8760828971862793,
+      "logps/chosen": -68.79309844970703,
+      "logps/rejected": -68.1576156616211,
+      "loss": 0.6788,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.16255898773670197,
+      "rewards/margins": 0.033053189516067505,
+      "rewards/rejected": -0.19561214745044708,
+      "step": 7600
+    },
+    {
+      "epoch": 1.3094417643004825,
+      "eval_logits/chosen": -2.9859960079193115,
+      "eval_logits/rejected": -2.980189561843872,
+      "eval_logps/chosen": -68.90995025634766,
+      "eval_logps/rejected": -75.84913635253906,
+      "eval_loss": 0.6822887063026428,
+      "eval_rewards/accuracies": 0.607342004776001,
+      "eval_rewards/chosen": -0.1019805297255516,
+      "eval_rewards/margins": 0.024709602817893028,
+      "eval_rewards/rejected": -0.1266901195049286,
+      "eval_runtime": 383.2017,
+      "eval_samples_per_second": 11.232,
+      "eval_steps_per_second": 1.404,
+      "step": 7600
+    },
+    {
+      "epoch": 1.3111647139903515,
+      "grad_norm": 3.1363210678100586,
+      "learning_rate": 1.5993911794431198e-08,
+      "logits/chosen": -2.8590292930603027,
+      "logits/rejected": -2.827662944793701,
+      "logps/chosen": -67.81131744384766,
+      "logps/rejected": -70.76785278320312,
+      "loss": 0.6672,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.14899981021881104,
+      "rewards/margins": 0.05645173788070679,
+      "rewards/rejected": -0.20545153319835663,
+      "step": 7610
+    },
+    {
+      "epoch": 1.3128876636802205,
+      "grad_norm": 3.590625047683716,
+      "learning_rate": 1.59238209863607e-08,
+      "logits/chosen": -2.9025955200195312,
+      "logits/rejected": -2.8750128746032715,
+      "logps/chosen": -71.08467864990234,
+      "logps/rejected": -71.47862243652344,
+      "loss": 0.6747,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.13571318984031677,
+      "rewards/margins": 0.04114279896020889,
+      "rewards/rejected": -0.17685601115226746,
+      "step": 7620
+    },
+    {
+      "epoch": 1.3146106133700897,
+      "grad_norm": 3.7938337326049805,
+      "learning_rate": 1.5853812254875877e-08,
+      "logits/chosen": -2.8352577686309814,
+      "logits/rejected": -2.847367286682129,
+      "logps/chosen": -64.74190521240234,
+      "logps/rejected": -72.19737243652344,
+      "loss": 0.6787,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.14908894896507263,
+      "rewards/margins": 0.032521482557058334,
+      "rewards/rejected": -0.18161042034626007,
+      "step": 7630
+    },
+    {
+      "epoch": 1.3163335630599586,
+      "grad_norm": 3.9987897872924805,
+      "learning_rate": 1.5783886233071076e-08,
+      "logits/chosen": -2.802812099456787,
+      "logits/rejected": -2.780230760574341,
+      "logps/chosen": -72.41722106933594,
+      "logps/rejected": -72.95262145996094,
+      "loss": 0.6654,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.1449960470199585,
+      "rewards/margins": 0.0604391023516655,
+      "rewards/rejected": -0.2054351568222046,
+      "step": 7640
+    },
+    {
+      "epoch": 1.3180565127498278,
+      "grad_norm": 3.425074338912964,
+      "learning_rate": 1.5714043553292683e-08,
+      "logits/chosen": -2.9329230785369873,
+      "logits/rejected": -2.9086971282958984,
+      "logps/chosen": -74.3526840209961,
+      "logps/rejected": -76.86766052246094,
+      "loss": 0.6699,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.15956375002861023,
+      "rewards/margins": 0.0513019934296608,
+      "rewards/rejected": -0.21086573600769043,
+      "step": 7650
+    },
+    {
+      "epoch": 1.3197794624396968,
+      "grad_norm": 4.080209255218506,
+      "learning_rate": 1.564428484713345e-08,
+      "logits/chosen": -2.905999183654785,
+      "logits/rejected": -2.8671061992645264,
+      "logps/chosen": -71.86300659179688,
+      "logps/rejected": -74.92207336425781,
+      "loss": 0.6694,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.1483265906572342,
+      "rewards/margins": 0.05317336320877075,
+      "rewards/rejected": -0.20149996876716614,
+      "step": 7660
+    },
+    {
+      "epoch": 1.3215024121295658,
+      "grad_norm": 3.457108736038208,
+      "learning_rate": 1.5574610745426703e-08,
+      "logits/chosen": -2.8598885536193848,
+      "logits/rejected": -2.83192777633667,
+      "logps/chosen": -71.87262725830078,
+      "logps/rejected": -72.23542022705078,
+      "loss": 0.6707,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.14328305423259735,
+      "rewards/margins": 0.04969315975904465,
+      "rewards/rejected": -0.1929762214422226,
+      "step": 7670
+    },
+    {
+      "epoch": 1.323225361819435,
+      "grad_norm": 3.2307112216949463,
+      "learning_rate": 1.550502187824073e-08,
+      "logits/chosen": -2.9309659004211426,
+      "logits/rejected": -2.912719964981079,
+      "logps/chosen": -65.02218627929688,
+      "logps/rejected": -69.34642791748047,
+      "loss": 0.6691,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.14385822415351868,
+      "rewards/margins": 0.05252811312675476,
+      "rewards/rejected": -0.19638635218143463,
+      "step": 7680
+    },
+    {
+      "epoch": 1.324948311509304,
+      "grad_norm": 3.3792569637298584,
+      "learning_rate": 1.543551887487301e-08,
+      "logits/chosen": -3.0184569358825684,
+      "logits/rejected": -2.9714720249176025,
+      "logps/chosen": -67.96367645263672,
+      "logps/rejected": -67.847900390625,
+      "loss": 0.6667,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.12979218363761902,
+      "rewards/margins": 0.058135371655225754,
+      "rewards/rejected": -0.18792754411697388,
+      "step": 7690
+    },
+    {
+      "epoch": 1.3266712611991731,
+      "grad_norm": 3.644927740097046,
+      "learning_rate": 1.536610236384455e-08,
+      "logits/chosen": -2.8754656314849854,
+      "logits/rejected": -2.8459057807922363,
+      "logps/chosen": -67.76346588134766,
+      "logps/rejected": -72.79844665527344,
+      "loss": 0.6704,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.14053042232990265,
+      "rewards/margins": 0.050431691110134125,
+      "rewards/rejected": -0.19096210598945618,
+      "step": 7700
+    },
+    {
+      "epoch": 1.3266712611991731,
+      "eval_logits/chosen": -2.9837443828582764,
+      "eval_logits/rejected": -2.977949857711792,
+      "eval_logps/chosen": -69.04661560058594,
+      "eval_logps/rejected": -76.04169464111328,
+      "eval_loss": 0.6820386052131653,
+      "eval_rewards/accuracies": 0.6066449880599976,
+      "eval_rewards/chosen": -0.10334714502096176,
+      "eval_rewards/margins": 0.025268645957112312,
+      "eval_rewards/rejected": -0.12861579656600952,
+      "eval_runtime": 383.5922,
+      "eval_samples_per_second": 11.22,
+      "eval_steps_per_second": 1.403,
+      "step": 7700
+    },
+    {
+      "epoch": 1.328394210889042,
+      "grad_norm": 3.6044883728027344,
+      "learning_rate": 1.5296772972894213e-08,
+      "logits/chosen": -2.9211082458496094,
+      "logits/rejected": -2.9063804149627686,
+      "logps/chosen": -69.02720642089844,
+      "logps/rejected": -71.40312957763672,
+      "loss": 0.6808,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.17454668879508972,
+      "rewards/margins": 0.030221164226531982,
+      "rewards/rejected": -0.2047678530216217,
+      "step": 7710
+    },
+    {
+      "epoch": 1.330117160578911,
+      "grad_norm": 3.539374589920044,
+      "learning_rate": 1.5227531328972994e-08,
+      "logits/chosen": -2.894953489303589,
+      "logits/rejected": -2.859501361846924,
+      "logps/chosen": -71.73707580566406,
+      "logps/rejected": -71.6160659790039,
+      "loss": 0.6696,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.15594525635242462,
+      "rewards/margins": 0.0520988292992115,
+      "rewards/rejected": -0.20804409682750702,
+      "step": 7720
+    },
+    {
+      "epoch": 1.33184011026878,
+      "grad_norm": 3.7142114639282227,
+      "learning_rate": 1.5158378058238442e-08,
+      "logits/chosen": -2.8536651134490967,
+      "logits/rejected": -2.8312697410583496,
+      "logps/chosen": -69.4502944946289,
+      "logps/rejected": -72.75386047363281,
+      "loss": 0.6705,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.15527936816215515,
+      "rewards/margins": 0.049563415348529816,
+      "rewards/rejected": -0.20484277606010437,
+      "step": 7730
+    },
+    {
+      "epoch": 1.3335630599586492,
+      "grad_norm": 3.2965762615203857,
+      "learning_rate": 1.5089313786048885e-08,
+      "logits/chosen": -2.8467166423797607,
+      "logits/rejected": -2.8310632705688477,
+      "logps/chosen": -66.77486419677734,
+      "logps/rejected": -74.69255065917969,
+      "loss": 0.6666,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.1506994664669037,
+      "rewards/margins": 0.05842146277427673,
+      "rewards/rejected": -0.2091209441423416,
+      "step": 7740
+    },
+    {
+      "epoch": 1.3352860096485184,
+      "grad_norm": 3.1638777256011963,
+      "learning_rate": 1.5020339136957876e-08,
+      "logits/chosen": -2.8631324768066406,
+      "logits/rejected": -2.830181837081909,
+      "logps/chosen": -68.68546295166016,
+      "logps/rejected": -74.04763793945312,
+      "loss": 0.6606,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.14742670953273773,
+      "rewards/margins": 0.07109382003545761,
+      "rewards/rejected": -0.21852055191993713,
+      "step": 7750
+    },
+    {
+      "epoch": 1.3370089593383874,
+      "grad_norm": 3.482968330383301,
+      "learning_rate": 1.4951454734708456e-08,
+      "logits/chosen": -2.768019199371338,
+      "logits/rejected": -2.751142978668213,
+      "logps/chosen": -65.07926940917969,
+      "logps/rejected": -72.26102447509766,
+      "loss": 0.6657,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.1457255780696869,
+      "rewards/margins": 0.059985846281051636,
+      "rewards/rejected": -0.20571140944957733,
+      "step": 7760
+    },
+    {
+      "epoch": 1.3387319090282563,
+      "grad_norm": 3.509697198867798,
+      "learning_rate": 1.4882661202227597e-08,
+      "logits/chosen": -2.8160955905914307,
+      "logits/rejected": -2.7914600372314453,
+      "logps/chosen": -66.82437896728516,
+      "logps/rejected": -72.2037124633789,
+      "loss": 0.6702,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.14708185195922852,
+      "rewards/margins": 0.049828968942165375,
+      "rewards/rejected": -0.1969108283519745,
+      "step": 7770
+    },
+    {
+      "epoch": 1.3404548587181253,
+      "grad_norm": 3.7050113677978516,
+      "learning_rate": 1.4813959161620502e-08,
+      "logits/chosen": -2.9742326736450195,
+      "logits/rejected": -2.9579920768737793,
+      "logps/chosen": -68.44230651855469,
+      "logps/rejected": -74.46659088134766,
+      "loss": 0.6777,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.16903328895568848,
+      "rewards/margins": 0.036769766360521317,
+      "rewards/rejected": -0.2058030366897583,
+      "step": 7780
+    },
+    {
+      "epoch": 1.3421778084079945,
+      "grad_norm": 3.943408250808716,
+      "learning_rate": 1.4745349234165017e-08,
+      "logits/chosen": -2.9041996002197266,
+      "logits/rejected": -2.889291286468506,
+      "logps/chosen": -67.3814468383789,
+      "logps/rejected": -72.92383575439453,
+      "loss": 0.6692,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1581103503704071,
+      "rewards/margins": 0.052158355712890625,
+      "rewards/rejected": -0.21026869118213654,
+      "step": 7790
+    },
+    {
+      "epoch": 1.3439007580978635,
+      "grad_norm": 3.7811434268951416,
+      "learning_rate": 1.4676832040305984e-08,
+      "logits/chosen": -2.938472032546997,
+      "logits/rejected": -2.932990312576294,
+      "logps/chosen": -69.53761291503906,
+      "logps/rejected": -75.5572280883789,
+      "loss": 0.6694,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.1555713713169098,
+      "rewards/margins": 0.052687495946884155,
+      "rewards/rejected": -0.20825886726379395,
+      "step": 7800
+    },
+    {
+      "epoch": 1.3439007580978635,
+      "eval_logits/chosen": -2.982745885848999,
+      "eval_logits/rejected": -2.9769082069396973,
+      "eval_logps/chosen": -69.25647735595703,
+      "eval_logps/rejected": -76.27448272705078,
+      "eval_loss": 0.681953489780426,
+      "eval_rewards/accuracies": 0.6022304892539978,
+      "eval_rewards/chosen": -0.10544583201408386,
+      "eval_rewards/margins": 0.025497842580080032,
+      "eval_rewards/rejected": -0.1309436708688736,
+      "eval_runtime": 383.1002,
+      "eval_samples_per_second": 11.235,
+      "eval_steps_per_second": 1.404,
+      "step": 7800
+    },
+    {
+      "epoch": 1.3456237077877327,
+      "grad_norm": 4.0939106941223145,
+      "learning_rate": 1.4608408199649686e-08,
+      "logits/chosen": -2.9537672996520996,
+      "logits/rejected": -2.925611972808838,
+      "logps/chosen": -73.13529205322266,
+      "logps/rejected": -74.45106506347656,
+      "loss": 0.6678,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.1411769837141037,
+      "rewards/margins": 0.055773038417100906,
+      "rewards/rejected": -0.1969500035047531,
+      "step": 7810
+    },
+    {
+      "epoch": 1.3473466574776016,
+      "grad_norm": 3.4245619773864746,
+      "learning_rate": 1.4540078330958166e-08,
+      "logits/chosen": -2.9206206798553467,
+      "logits/rejected": -2.889345645904541,
+      "logps/chosen": -69.76282501220703,
+      "logps/rejected": -74.32697296142578,
+      "loss": 0.6677,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.13397334516048431,
+      "rewards/margins": 0.05616996809840202,
+      "rewards/rejected": -0.19014331698417664,
+      "step": 7820
+    },
+    {
+      "epoch": 1.3490696071674706,
+      "grad_norm": 3.9951224327087402,
+      "learning_rate": 1.4471843052143696e-08,
+      "logits/chosen": -2.8263535499572754,
+      "logits/rejected": -2.8167710304260254,
+      "logps/chosen": -65.77522277832031,
+      "logps/rejected": -74.73420715332031,
+      "loss": 0.6688,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.1597517430782318,
+      "rewards/margins": 0.053311027586460114,
+      "rewards/rejected": -0.21306276321411133,
+      "step": 7830
+    },
+    {
+      "epoch": 1.3507925568573398,
+      "grad_norm": 3.130455255508423,
+      "learning_rate": 1.4403702980263149e-08,
+      "logits/chosen": -2.817085027694702,
+      "logits/rejected": -2.7957003116607666,
+      "logps/chosen": -69.14472961425781,
+      "logps/rejected": -74.74988555908203,
+      "loss": 0.6712,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.15577241778373718,
+      "rewards/margins": 0.04939967766404152,
+      "rewards/rejected": -0.2051720917224884,
+      "step": 7840
+    },
+    {
+      "epoch": 1.3525155065472088,
+      "grad_norm": 3.658324718475342,
+      "learning_rate": 1.4335658731512452e-08,
+      "logits/chosen": -2.811091184616089,
+      "logits/rejected": -2.7781901359558105,
+      "logps/chosen": -71.12324523925781,
+      "logps/rejected": -68.07506561279297,
+      "loss": 0.6749,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.15546159446239471,
+      "rewards/margins": 0.04152378812432289,
+      "rewards/rejected": -0.1969853937625885,
+      "step": 7850
+    },
+    {
+      "epoch": 1.354238456237078,
+      "grad_norm": 3.456115245819092,
+      "learning_rate": 1.4267710921220974e-08,
+      "logits/chosen": -2.8459620475769043,
+      "logits/rejected": -2.810361862182617,
+      "logps/chosen": -67.52024841308594,
+      "logps/rejected": -74.30799865722656,
+      "loss": 0.6605,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.14623808860778809,
+      "rewards/margins": 0.07031372934579849,
+      "rewards/rejected": -0.21655182540416718,
+      "step": 7860
+    },
+    {
+      "epoch": 1.355961405926947,
+      "grad_norm": 3.5544679164886475,
+      "learning_rate": 1.4199860163846007e-08,
+      "logits/chosen": -2.868586540222168,
+      "logits/rejected": -2.8513782024383545,
+      "logps/chosen": -71.25041198730469,
+      "logps/rejected": -74.87138366699219,
+      "loss": 0.6706,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.15576469898223877,
+      "rewards/margins": 0.04931577667593956,
+      "rewards/rejected": -0.20508047938346863,
+      "step": 7870
+    },
+    {
+      "epoch": 1.3576843556168159,
+      "grad_norm": 3.6760735511779785,
+      "learning_rate": 1.4132107072967165e-08,
+      "logits/chosen": -2.929291248321533,
+      "logits/rejected": -2.913846015930176,
+      "logps/chosen": -68.70439147949219,
+      "logps/rejected": -73.17445373535156,
+      "loss": 0.6705,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.15590780973434448,
+      "rewards/margins": 0.050708264112472534,
+      "rewards/rejected": -0.2066160887479782,
+      "step": 7880
+    },
+    {
+      "epoch": 1.359407305306685,
+      "grad_norm": 3.849466562271118,
+      "learning_rate": 1.406445226128088e-08,
+      "logits/chosen": -2.86910080909729,
+      "logits/rejected": -2.850379467010498,
+      "logps/chosen": -68.72163391113281,
+      "logps/rejected": -74.83341217041016,
+      "loss": 0.6695,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.1434558480978012,
+      "rewards/margins": 0.052369069308042526,
+      "rewards/rejected": -0.19582490622997284,
+      "step": 7890
+    },
+    {
+      "epoch": 1.361130254996554,
+      "grad_norm": 3.9820239543914795,
+      "learning_rate": 1.3996896340594791e-08,
+      "logits/chosen": -2.8582634925842285,
+      "logits/rejected": -2.8552401065826416,
+      "logps/chosen": -68.76918029785156,
+      "logps/rejected": -73.8875503540039,
+      "loss": 0.6779,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.16592998802661896,
+      "rewards/margins": 0.03645527362823486,
+      "rewards/rejected": -0.20238527655601501,
+      "step": 7900
+    },
+    {
+      "epoch": 1.361130254996554,
+      "eval_logits/chosen": -2.9811806678771973,
+      "eval_logits/rejected": -2.975399971008301,
+      "eval_logps/chosen": -69.37777709960938,
+      "eval_logps/rejected": -76.41007995605469,
+      "eval_loss": 0.6819015145301819,
+      "eval_rewards/accuracies": 0.606877326965332,
+      "eval_rewards/chosen": -0.10665879398584366,
+      "eval_rewards/margins": 0.02564079314470291,
+      "eval_rewards/rejected": -0.13229957222938538,
+      "eval_runtime": 382.8871,
+      "eval_samples_per_second": 11.241,
+      "eval_steps_per_second": 1.405,
+      "step": 7900
+    },
+    {
+      "epoch": 1.3628532046864232,
+      "grad_norm": 3.6145782470703125,
+      "learning_rate": 1.3929439921822333e-08,
+      "logits/chosen": -2.864738941192627,
+      "logits/rejected": -2.8416783809661865,
+      "logps/chosen": -72.06912231445312,
+      "logps/rejected": -73.76692199707031,
+      "loss": 0.6714,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.15245692431926727,
+      "rewards/margins": 0.04871072992682457,
+      "rewards/rejected": -0.20116765797138214,
+      "step": 7910
+    },
+    {
+      "epoch": 1.3645761543762922,
+      "grad_norm": 3.7070534229278564,
+      "learning_rate": 1.3862083614977067e-08,
+      "logits/chosen": -2.860708713531494,
+      "logits/rejected": -2.837212085723877,
+      "logps/chosen": -70.11438751220703,
+      "logps/rejected": -71.44000244140625,
+      "loss": 0.672,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.15293672680854797,
+      "rewards/margins": 0.046852219849824905,
+      "rewards/rejected": -0.19978895783424377,
+      "step": 7920
+    },
+    {
+      "epoch": 1.3662991040661612,
+      "grad_norm": 3.871424436569214,
+      "learning_rate": 1.3794828029167265e-08,
+      "logits/chosen": -2.9238908290863037,
+      "logits/rejected": -2.8955588340759277,
+      "logps/chosen": -73.63279724121094,
+      "logps/rejected": -73.25145721435547,
+      "loss": 0.6679,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.15298232436180115,
+      "rewards/margins": 0.055543702095746994,
+      "rewards/rejected": -0.20852604508399963,
+      "step": 7930
+    },
+    {
+      "epoch": 1.3680220537560304,
+      "grad_norm": 4.201838970184326,
+      "learning_rate": 1.3727673772590375e-08,
+      "logits/chosen": -2.8143470287323,
+      "logits/rejected": -2.798787832260132,
+      "logps/chosen": -73.6925277709961,
+      "logps/rejected": -76.87105560302734,
+      "loss": 0.6639,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.1513223648071289,
+      "rewards/margins": 0.06518540531396866,
+      "rewards/rejected": -0.21650776267051697,
+      "step": 7940
+    },
+    {
+      "epoch": 1.3697450034458993,
+      "grad_norm": 4.1368327140808105,
+      "learning_rate": 1.3660621452527505e-08,
+      "logits/chosen": -2.7767703533172607,
+      "logits/rejected": -2.7682571411132812,
+      "logps/chosen": -66.61457061767578,
+      "logps/rejected": -73.73819732666016,
+      "loss": 0.6687,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.15108530223369598,
+      "rewards/margins": 0.05361195281147957,
+      "rewards/rejected": -0.20469728112220764,
+      "step": 7950
+    },
+    {
+      "epoch": 1.3714679531357685,
+      "grad_norm": 3.5207035541534424,
+      "learning_rate": 1.3593671675337953e-08,
+      "logits/chosen": -2.7937610149383545,
+      "logits/rejected": -2.766061305999756,
+      "logps/chosen": -66.50416564941406,
+      "logps/rejected": -72.04696655273438,
+      "loss": 0.678,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.16025115549564362,
+      "rewards/margins": 0.03536851704120636,
+      "rewards/rejected": -0.19561967253684998,
+      "step": 7960
+    },
+    {
+      "epoch": 1.3731909028256375,
+      "grad_norm": 3.2811360359191895,
+      "learning_rate": 1.3526825046453705e-08,
+      "logits/chosen": -2.8932979106903076,
+      "logits/rejected": -2.862522840499878,
+      "logps/chosen": -72.32527923583984,
+      "logps/rejected": -73.83203125,
+      "loss": 0.6719,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.16927920281887054,
+      "rewards/margins": 0.04832344129681587,
+      "rewards/rejected": -0.21760264039039612,
+      "step": 7970
+    },
+    {
+      "epoch": 1.3749138525155065,
+      "grad_norm": 3.972825050354004,
+      "learning_rate": 1.3460082170373988e-08,
+      "logits/chosen": -2.9369592666625977,
+      "logits/rejected": -2.923105001449585,
+      "logps/chosen": -71.78608703613281,
+      "logps/rejected": -76.59053039550781,
+      "loss": 0.668,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.149064838886261,
+      "rewards/margins": 0.05578247457742691,
+      "rewards/rejected": -0.2048473060131073,
+      "step": 7980
+    },
+    {
+      "epoch": 1.3766368022053757,
+      "grad_norm": 3.5907435417175293,
+      "learning_rate": 1.339344365065973e-08,
+      "logits/chosen": -2.962944984436035,
+      "logits/rejected": -2.9507527351379395,
+      "logps/chosen": -66.12873840332031,
+      "logps/rejected": -71.84226989746094,
+      "loss": 0.6726,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.15461526811122894,
+      "rewards/margins": 0.04721805080771446,
+      "rewards/rejected": -0.2018333375453949,
+      "step": 7990
+    },
+    {
+      "epoch": 1.3783597518952446,
+      "grad_norm": 4.229453086853027,
+      "learning_rate": 1.3326910089928244e-08,
+      "logits/chosen": -2.8236045837402344,
+      "logits/rejected": -2.813786029815674,
+      "logps/chosen": -69.64259338378906,
+      "logps/rejected": -76.20364379882812,
+      "loss": 0.6712,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.15125074982643127,
+      "rewards/margins": 0.048361606895923615,
+      "rewards/rejected": -0.1996123492717743,
+      "step": 8000
+    },
+    {
+      "epoch": 1.3783597518952446,
+      "eval_logits/chosen": -2.9797656536102295,
+      "eval_logits/rejected": -2.9739840030670166,
+      "eval_logps/chosen": -69.53035736083984,
+      "eval_logps/rejected": -76.59691619873047,
+      "eval_loss": 0.6817482709884644,
+      "eval_rewards/accuracies": 0.6061803102493286,
+      "eval_rewards/chosen": -0.10818460583686829,
+      "eval_rewards/margins": 0.025983335450291634,
+      "eval_rewards/rejected": -0.13416795432567596,
+      "eval_runtime": 383.1218,
+      "eval_samples_per_second": 11.234,
+      "eval_steps_per_second": 1.404,
+      "step": 8000
+    },
+    {
+      "epoch": 1.3800827015851138,
+      "grad_norm": 3.8489747047424316,
+      "learning_rate": 1.3260482089847603e-08,
+      "logits/chosen": -2.8497960567474365,
+      "logits/rejected": -2.8283565044403076,
+      "logps/chosen": -72.20866394042969,
+      "logps/rejected": -71.20612335205078,
+      "loss": 0.6766,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.15048834681510925,
+      "rewards/margins": 0.037339311093091965,
+      "rewards/rejected": -0.1878276765346527,
+      "step": 8010
+    },
+    {
+      "epoch": 1.3818056512749828,
+      "grad_norm": 3.4156136512756348,
+      "learning_rate": 1.3194160251131364e-08,
+      "logits/chosen": -2.8691112995147705,
+      "logits/rejected": -2.829939603805542,
+      "logps/chosen": -76.88106536865234,
+      "logps/rejected": -76.79621887207031,
+      "loss": 0.6688,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.16643230617046356,
+      "rewards/margins": 0.05410188436508179,
+      "rewards/rejected": -0.22053417563438416,
+      "step": 8020
+    },
+    {
+      "epoch": 1.3835286009648518,
+      "grad_norm": 3.904705047607422,
+      "learning_rate": 1.3127945173532989e-08,
+      "logits/chosen": -2.863123655319214,
+      "logits/rejected": -2.848050355911255,
+      "logps/chosen": -66.33540344238281,
+      "logps/rejected": -73.9996566772461,
+      "loss": 0.669,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.15192072093486786,
+      "rewards/margins": 0.05390414595603943,
+      "rewards/rejected": -0.20582488179206848,
+      "step": 8030
+    },
+    {
+      "epoch": 1.385251550654721,
+      "grad_norm": 3.4018759727478027,
+      "learning_rate": 1.3061837455840539e-08,
+      "logits/chosen": -2.82468843460083,
+      "logits/rejected": -2.7930550575256348,
+      "logps/chosen": -70.85289001464844,
+      "logps/rejected": -73.41654968261719,
+      "loss": 0.6709,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.16426238417625427,
+      "rewards/margins": 0.04962170496582985,
+      "rewards/rejected": -0.2138841152191162,
+      "step": 8040
+    },
+    {
+      "epoch": 1.38697450034459,
+      "grad_norm": 3.442335844039917,
+      "learning_rate": 1.2995837695871186e-08,
+      "logits/chosen": -2.8906948566436768,
+      "logits/rejected": -2.873434543609619,
+      "logps/chosen": -68.89337158203125,
+      "logps/rejected": -75.48945617675781,
+      "loss": 0.6664,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.13785851001739502,
+      "rewards/margins": 0.05857409909367561,
+      "rewards/rejected": -0.19643262028694153,
+      "step": 8050
+    },
+    {
+      "epoch": 1.388697450034459,
+      "grad_norm": 3.6961185932159424,
+      "learning_rate": 1.2929946490465854e-08,
+      "logits/chosen": -2.9348130226135254,
+      "logits/rejected": -2.896115779876709,
+      "logps/chosen": -71.54668426513672,
+      "logps/rejected": -71.59806823730469,
+      "loss": 0.6694,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.16208091378211975,
+      "rewards/margins": 0.053027164191007614,
+      "rewards/rejected": -0.21510808169841766,
+      "step": 8060
+    },
+    {
+      "epoch": 1.390420399724328,
+      "grad_norm": 3.6304800510406494,
+      "learning_rate": 1.2864164435483777e-08,
+      "logits/chosen": -2.843792676925659,
+      "logits/rejected": -2.8135626316070557,
+      "logps/chosen": -69.33329772949219,
+      "logps/rejected": -71.07044219970703,
+      "loss": 0.6696,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.15875005722045898,
+      "rewards/margins": 0.052306950092315674,
+      "rewards/rejected": -0.21105699241161346,
+      "step": 8070
+    },
+    {
+      "epoch": 1.392143349414197,
+      "grad_norm": 3.78218936920166,
+      "learning_rate": 1.2798492125797144e-08,
+      "logits/chosen": -2.8251264095306396,
+      "logits/rejected": -2.820019245147705,
+      "logps/chosen": -67.84989929199219,
+      "logps/rejected": -77.10699462890625,
+      "loss": 0.6741,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.14863455295562744,
+      "rewards/margins": 0.042812101542949677,
+      "rewards/rejected": -0.19144666194915771,
+      "step": 8080
+    },
+    {
+      "epoch": 1.3938662991040662,
+      "grad_norm": 3.5410895347595215,
+      "learning_rate": 1.273293015528571e-08,
+      "logits/chosen": -2.8161749839782715,
+      "logits/rejected": -2.787935733795166,
+      "logps/chosen": -68.72693634033203,
+      "logps/rejected": -73.06669616699219,
+      "loss": 0.6661,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.15608568489551544,
+      "rewards/margins": 0.060046881437301636,
+      "rewards/rejected": -0.21613255143165588,
+      "step": 8090
+    },
+    {
+      "epoch": 1.3955892487939352,
+      "grad_norm": 3.7731690406799316,
+      "learning_rate": 1.2667479116831437e-08,
+      "logits/chosen": -2.870635747909546,
+      "logits/rejected": -2.863468647003174,
+      "logps/chosen": -69.86956787109375,
+      "logps/rejected": -77.5333480834961,
+      "loss": 0.6768,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.15787693858146667,
+      "rewards/margins": 0.03653046116232872,
+      "rewards/rejected": -0.1944073885679245,
+      "step": 8100
+    },
+    {
+      "epoch": 1.3955892487939352,
+      "eval_logits/chosen": -2.9784140586853027,
+      "eval_logits/rejected": -2.972625255584717,
+      "eval_logps/chosen": -69.67633819580078,
+      "eval_logps/rejected": -76.76518249511719,
+      "eval_loss": 0.6816586256027222,
+      "eval_rewards/accuracies": 0.6006041169166565,
+      "eval_rewards/chosen": -0.10964445024728775,
+      "eval_rewards/margins": 0.026206107810139656,
+      "eval_rewards/rejected": -0.13585057854652405,
+      "eval_runtime": 383.3156,
+      "eval_samples_per_second": 11.228,
+      "eval_steps_per_second": 1.404,
+      "step": 8100
+    },
+    {
+      "epoch": 1.3973121984838044,
+      "grad_norm": 4.153486251831055,
+      "learning_rate": 1.2602139602313067e-08,
+      "logits/chosen": -2.8895528316497803,
+      "logits/rejected": -2.852138042449951,
+      "logps/chosen": -71.78083801269531,
+      "logps/rejected": -71.88075256347656,
+      "loss": 0.6701,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.15719279646873474,
+      "rewards/margins": 0.05117177218198776,
+      "rewards/rejected": -0.2083645761013031,
+      "step": 8110
+    },
+    {
+      "epoch": 1.3990351481736734,
+      "grad_norm": 3.7467994689941406,
+      "learning_rate": 1.2536912202600907e-08,
+      "logits/chosen": -2.8456060886383057,
+      "logits/rejected": -2.827108860015869,
+      "logps/chosen": -69.3144302368164,
+      "logps/rejected": -72.37879943847656,
+      "loss": 0.6742,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.15212032198905945,
+      "rewards/margins": 0.042838118970394135,
+      "rewards/rejected": -0.19495847821235657,
+      "step": 8120
+    },
+    {
+      "epoch": 1.4007580978635423,
+      "grad_norm": 3.4037256240844727,
+      "learning_rate": 1.2471797507551324e-08,
+      "logits/chosen": -2.8593244552612305,
+      "logits/rejected": -2.8423967361450195,
+      "logps/chosen": -69.50029754638672,
+      "logps/rejected": -70.24095153808594,
+      "loss": 0.6709,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.1468019038438797,
+      "rewards/margins": 0.0495857410132885,
+      "rewards/rejected": -0.1963876634836197,
+      "step": 8130
+    },
+    {
+      "epoch": 1.4024810475534115,
+      "grad_norm": 4.105754852294922,
+      "learning_rate": 1.2406796106001527e-08,
+      "logits/chosen": -2.8235363960266113,
+      "logits/rejected": -2.7977235317230225,
+      "logps/chosen": -69.04493713378906,
+      "logps/rejected": -73.33890533447266,
+      "loss": 0.6749,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.16318005323410034,
+      "rewards/margins": 0.04213313013315201,
+      "rewards/rejected": -0.20531317591667175,
+      "step": 8140
+    },
+    {
+      "epoch": 1.4042039972432805,
+      "grad_norm": 3.67876935005188,
+      "learning_rate": 1.2341908585764196e-08,
+      "logits/chosen": -2.904982089996338,
+      "logits/rejected": -2.884549140930176,
+      "logps/chosen": -67.12843322753906,
+      "logps/rejected": -74.39512634277344,
+      "loss": 0.6643,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.14980655908584595,
+      "rewards/margins": 0.06340639293193817,
+      "rewards/rejected": -0.21321293711662292,
+      "step": 8150
+    },
+    {
+      "epoch": 1.4059269469331497,
+      "grad_norm": 3.3534793853759766,
+      "learning_rate": 1.2277135533622174e-08,
+      "logits/chosen": -2.8378894329071045,
+      "logits/rejected": -2.8182387351989746,
+      "logps/chosen": -71.95633697509766,
+      "logps/rejected": -74.68289184570312,
+      "loss": 0.6637,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.15943261981010437,
+      "rewards/margins": 0.06451521813869476,
+      "rewards/rejected": -0.22394785284996033,
+      "step": 8160
+    },
+    {
+      "epoch": 1.4076498966230186,
+      "grad_norm": 3.8170390129089355,
+      "learning_rate": 1.2212477535323157e-08,
+      "logits/chosen": -2.9035658836364746,
+      "logits/rejected": -2.878713607788086,
+      "logps/chosen": -74.52428436279297,
+      "logps/rejected": -71.16970825195312,
+      "loss": 0.6735,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.14813444018363953,
+      "rewards/margins": 0.04404045268893242,
+      "rewards/rejected": -0.19217489659786224,
+      "step": 8170
+    },
+    {
+      "epoch": 1.4093728463128876,
+      "grad_norm": 4.525648593902588,
+      "learning_rate": 1.2147935175574404e-08,
+      "logits/chosen": -2.89094877243042,
+      "logits/rejected": -2.870192527770996,
+      "logps/chosen": -72.2078628540039,
+      "logps/rejected": -75.14228057861328,
+      "loss": 0.6747,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.15915609896183014,
+      "rewards/margins": 0.04215477034449577,
+      "rewards/rejected": -0.2013108730316162,
+      "step": 8180
+    },
+    {
+      "epoch": 1.4110957960027566,
+      "grad_norm": 3.435528516769409,
+      "learning_rate": 1.208350903803745e-08,
+      "logits/chosen": -2.8529930114746094,
+      "logits/rejected": -2.8296115398406982,
+      "logps/chosen": -73.67615509033203,
+      "logps/rejected": -74.6364517211914,
+      "loss": 0.6708,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.15697535872459412,
+      "rewards/margins": 0.05076602101325989,
+      "rewards/rejected": -0.20774134993553162,
+      "step": 8190
+    },
+    {
+      "epoch": 1.4128187456926258,
+      "grad_norm": 3.7466177940368652,
+      "learning_rate": 1.2019199705322794e-08,
+      "logits/chosen": -2.882794141769409,
+      "logits/rejected": -2.8498072624206543,
+      "logps/chosen": -71.3676986694336,
+      "logps/rejected": -73.77598571777344,
+      "loss": 0.6714,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.16372562944889069,
+      "rewards/margins": 0.04852137714624405,
+      "rewards/rejected": -0.21224701404571533,
+      "step": 8200
+    },
+    {
+      "epoch": 1.4128187456926258,
+      "eval_logits/chosen": -2.977246046066284,
+      "eval_logits/rejected": -2.9714057445526123,
+      "eval_logps/chosen": -69.83162689208984,
+      "eval_logps/rejected": -76.95604705810547,
+      "eval_loss": 0.6815012097358704,
+      "eval_rewards/accuracies": 0.6045538783073425,
+      "eval_rewards/chosen": -0.11119724810123444,
+      "eval_rewards/margins": 0.026561936363577843,
+      "eval_rewards/rejected": -0.13775917887687683,
+      "eval_runtime": 383.4903,
+      "eval_samples_per_second": 11.223,
+      "eval_steps_per_second": 1.403,
+      "step": 8200
+    },
+    {
+      "epoch": 1.414541695382495,
+      "grad_norm": 3.8294243812561035,
+      "learning_rate": 1.1955007758984717e-08,
+      "logits/chosen": -2.759054183959961,
+      "logits/rejected": -2.741037368774414,
+      "logps/chosen": -71.48738098144531,
+      "logps/rejected": -77.45023345947266,
+      "loss": 0.671,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.1638246774673462,
+      "rewards/margins": 0.049131304025650024,
+      "rewards/rejected": -0.21295598149299622,
+      "step": 8210
+    },
+    {
+      "epoch": 1.416264645072364,
+      "grad_norm": 3.4738824367523193,
+      "learning_rate": 1.1890933779515897e-08,
+      "logits/chosen": -2.8922982215881348,
+      "logits/rejected": -2.866921901702881,
+      "logps/chosen": -68.87584686279297,
+      "logps/rejected": -73.1611328125,
+      "loss": 0.6738,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.15782752633094788,
+      "rewards/margins": 0.04331156983971596,
+      "rewards/rejected": -0.20113909244537354,
+      "step": 8220
+    },
+    {
+      "epoch": 1.417987594762233,
+      "grad_norm": 3.550295114517212,
+      "learning_rate": 1.1826978346342301e-08,
+      "logits/chosen": -2.8465192317962646,
+      "logits/rejected": -2.8281021118164062,
+      "logps/chosen": -71.17774963378906,
+      "logps/rejected": -72.02894592285156,
+      "loss": 0.6721,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.159291610121727,
+      "rewards/margins": 0.04760519042611122,
+      "rewards/rejected": -0.2068968266248703,
+      "step": 8230
+    },
+    {
+      "epoch": 1.4197105444521019,
+      "grad_norm": 3.844132423400879,
+      "learning_rate": 1.1763142037817806e-08,
+      "logits/chosen": -2.9578728675842285,
+      "logits/rejected": -2.9293525218963623,
+      "logps/chosen": -72.63679504394531,
+      "logps/rejected": -73.30245208740234,
+      "loss": 0.6705,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.1646314114332199,
+      "rewards/margins": 0.04997286573052406,
+      "rewards/rejected": -0.21460425853729248,
+      "step": 8240
+    },
+    {
+      "epoch": 1.421433494141971,
+      "grad_norm": 4.150132656097412,
+      "learning_rate": 1.169942543121908e-08,
+      "logits/chosen": -2.8835697174072266,
+      "logits/rejected": -2.8604342937469482,
+      "logps/chosen": -72.97513580322266,
+      "logps/rejected": -74.92355346679688,
+      "loss": 0.6694,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.15325772762298584,
+      "rewards/margins": 0.052743516862392426,
+      "rewards/rejected": -0.20600123703479767,
+      "step": 8250
+    },
+    {
+      "epoch": 1.42315644383184,
+      "grad_norm": 4.193264961242676,
+      "learning_rate": 1.1635829102740293e-08,
+      "logits/chosen": -2.976393699645996,
+      "logits/rejected": -2.955944538116455,
+      "logps/chosen": -71.80229187011719,
+      "logps/rejected": -77.1074447631836,
+      "loss": 0.6643,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.15387707948684692,
+      "rewards/margins": 0.06437652558088303,
+      "rewards/rejected": -0.21825361251831055,
+      "step": 8260
+    },
+    {
+      "epoch": 1.4248793935217092,
+      "grad_norm": 4.057281017303467,
+      "learning_rate": 1.1572353627487949e-08,
+      "logits/chosen": -2.950138568878174,
+      "logits/rejected": -2.9393036365509033,
+      "logps/chosen": -70.5113296508789,
+      "logps/rejected": -75.03779602050781,
+      "loss": 0.6828,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.1711568385362625,
+      "rewards/margins": 0.02553866244852543,
+      "rewards/rejected": -0.196695476770401,
+      "step": 8270
+    },
+    {
+      "epoch": 1.4266023432115782,
+      "grad_norm": 3.7435097694396973,
+      "learning_rate": 1.1508999579475653e-08,
+      "logits/chosen": -2.844937562942505,
+      "logits/rejected": -2.8341002464294434,
+      "logps/chosen": -70.94270324707031,
+      "logps/rejected": -75.1131591796875,
+      "loss": 0.6774,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.18565645813941956,
+      "rewards/margins": 0.03672509640455246,
+      "rewards/rejected": -0.22238154709339142,
+      "step": 8280
+    },
+    {
+      "epoch": 1.4283252929014472,
+      "grad_norm": 3.8032071590423584,
+      "learning_rate": 1.1445767531618943e-08,
+      "logits/chosen": -2.8330256938934326,
+      "logits/rejected": -2.784752607345581,
+      "logps/chosen": -74.58160400390625,
+      "logps/rejected": -74.22502136230469,
+      "loss": 0.6686,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.14645373821258545,
+      "rewards/margins": 0.053861796855926514,
+      "rewards/rejected": -0.20031552016735077,
+      "step": 8290
+    },
+    {
+      "epoch": 1.4300482425913164,
+      "grad_norm": 3.5934009552001953,
+      "learning_rate": 1.1382658055730096e-08,
+      "logits/chosen": -3.00797700881958,
+      "logits/rejected": -2.9846229553222656,
+      "logps/chosen": -73.99989318847656,
+      "logps/rejected": -76.11830139160156,
+      "loss": 0.6705,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.16420504450798035,
+      "rewards/margins": 0.05167809873819351,
+      "rewards/rejected": -0.21588313579559326,
+      "step": 8300
+    },
+    {
+      "epoch": 1.4300482425913164,
+      "eval_logits/chosen": -2.9757673740386963,
+      "eval_logits/rejected": -2.969881296157837,
+      "eval_logps/chosen": -69.93333435058594,
+      "eval_logps/rejected": -77.0526123046875,
+      "eval_loss": 0.6815303564071655,
+      "eval_rewards/accuracies": 0.6001393795013428,
+      "eval_rewards/chosen": -0.11221436411142349,
+      "eval_rewards/margins": 0.02651044726371765,
+      "eval_rewards/rejected": -0.13872480392456055,
+      "eval_runtime": 383.3157,
+      "eval_samples_per_second": 11.228,
+      "eval_steps_per_second": 1.404,
+      "step": 8300
+    },
+    {
+      "epoch": 1.4317711922811853,
+      "grad_norm": 4.16780948638916,
+      "learning_rate": 1.1319671722512957e-08,
+      "logits/chosen": -2.7828173637390137,
+      "logits/rejected": -2.752716302871704,
+      "logps/chosen": -71.08320617675781,
+      "logps/rejected": -70.18670654296875,
+      "loss": 0.6742,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.14814069867134094,
+      "rewards/margins": 0.04219583421945572,
+      "rewards/rejected": -0.19033654034137726,
+      "step": 8310
+    },
+    {
+      "epoch": 1.4334941419710545,
+      "grad_norm": 3.3126111030578613,
+      "learning_rate": 1.1256809101557793e-08,
+      "logits/chosen": -2.863334894180298,
+      "logits/rejected": -2.8466808795928955,
+      "logps/chosen": -67.10429382324219,
+      "logps/rejected": -73.7756118774414,
+      "loss": 0.67,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.14590856432914734,
+      "rewards/margins": 0.051301706582307816,
+      "rewards/rejected": -0.19721028208732605,
+      "step": 8320
+    },
+    {
+      "epoch": 1.4352170916609235,
+      "grad_norm": 3.9216489791870117,
+      "learning_rate": 1.1194070761336133e-08,
+      "logits/chosen": -2.864744186401367,
+      "logits/rejected": -2.855569362640381,
+      "logps/chosen": -65.58021545410156,
+      "logps/rejected": -75.94027709960938,
+      "loss": 0.6702,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.16965366899967194,
+      "rewards/margins": 0.0511154904961586,
+      "rewards/rejected": -0.22076916694641113,
+      "step": 8330
+    },
+    {
+      "epoch": 1.4369400413507925,
+      "grad_norm": 3.490569829940796,
+      "learning_rate": 1.11314572691956e-08,
+      "logits/chosen": -2.922933578491211,
+      "logits/rejected": -2.907587766647339,
+      "logps/chosen": -70.60150146484375,
+      "logps/rejected": -72.54051971435547,
+      "loss": 0.6725,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.16157957911491394,
+      "rewards/margins": 0.04720939323306084,
+      "rewards/rejected": -0.20878896117210388,
+      "step": 8340
+    },
+    {
+      "epoch": 1.4386629910406616,
+      "grad_norm": 4.3055315017700195,
+      "learning_rate": 1.106896919135483e-08,
+      "logits/chosen": -2.7723186016082764,
+      "logits/rejected": -2.753783941268921,
+      "logps/chosen": -71.75452423095703,
+      "logps/rejected": -74.16422271728516,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.18583495914936066,
+      "rewards/margins": 0.02986169420182705,
+      "rewards/rejected": -0.21569666266441345,
+      "step": 8350
+    },
+    {
+      "epoch": 1.4403859407305306,
+      "grad_norm": 3.852482318878174,
+      "learning_rate": 1.1006607092898326e-08,
+      "logits/chosen": -2.7669973373413086,
+      "logits/rejected": -2.7321600914001465,
+      "logps/chosen": -69.51951599121094,
+      "logps/rejected": -73.9227523803711,
+      "loss": 0.6607,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.1496542990207672,
+      "rewards/margins": 0.0705617219209671,
+      "rewards/rejected": -0.2202160358428955,
+      "step": 8360
+    },
+    {
+      "epoch": 1.4421088904203998,
+      "grad_norm": 3.74485445022583,
+      "learning_rate": 1.0944371537771346e-08,
+      "logits/chosen": -2.8431124687194824,
+      "logits/rejected": -2.8254306316375732,
+      "logps/chosen": -69.08981323242188,
+      "logps/rejected": -75.19486236572266,
+      "loss": 0.6777,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.18033535778522491,
+      "rewards/margins": 0.035481635481119156,
+      "rewards/rejected": -0.21581697463989258,
+      "step": 8370
+    },
+    {
+      "epoch": 1.4438318401102688,
+      "grad_norm": 3.4626200199127197,
+      "learning_rate": 1.0882263088774809e-08,
+      "logits/chosen": -2.9633259773254395,
+      "logits/rejected": -2.944427013397217,
+      "logps/chosen": -66.32066345214844,
+      "logps/rejected": -74.64082336425781,
+      "loss": 0.6685,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1507943719625473,
+      "rewards/margins": 0.05414756014943123,
+      "rewards/rejected": -0.20494194328784943,
+      "step": 8380
+    },
+    {
+      "epoch": 1.4455547898001377,
+      "grad_norm": 3.640671968460083,
+      "learning_rate": 1.0820282307560197e-08,
+      "logits/chosen": -2.9133589267730713,
+      "logits/rejected": -2.8810665607452393,
+      "logps/chosen": -71.62699890136719,
+      "logps/rejected": -79.75428771972656,
+      "loss": 0.6524,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.14930753409862518,
+      "rewards/margins": 0.08714863657951355,
+      "rewards/rejected": -0.23645620048046112,
+      "step": 8390
+    },
+    {
+      "epoch": 1.447277739490007,
+      "grad_norm": 3.8194971084594727,
+      "learning_rate": 1.075842975462449e-08,
+      "logits/chosen": -2.8910470008850098,
+      "logits/rejected": -2.8695387840270996,
+      "logps/chosen": -66.54859924316406,
+      "logps/rejected": -71.95136260986328,
+      "loss": 0.6706,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.16242536902427673,
+      "rewards/margins": 0.049275849014520645,
+      "rewards/rejected": -0.21170122921466827,
+      "step": 8400
+    },
+    {
+      "epoch": 1.447277739490007,
+      "eval_logits/chosen": -2.9748239517211914,
+      "eval_logits/rejected": -2.969017744064331,
+      "eval_logps/chosen": -70.02185821533203,
+      "eval_logps/rejected": -77.17131042480469,
+      "eval_loss": 0.6814039945602417,
+      "eval_rewards/accuracies": 0.6024628281593323,
+      "eval_rewards/chosen": -0.11309970170259476,
+      "eval_rewards/margins": 0.026812180876731873,
+      "eval_rewards/rejected": -0.13991187512874603,
+      "eval_runtime": 383.2049,
+      "eval_samples_per_second": 11.232,
+      "eval_steps_per_second": 1.404,
+      "step": 8400
+    },
+    {
+      "epoch": 1.449000689179876,
+      "grad_norm": 3.552772045135498,
+      "learning_rate": 1.0696705989305086e-08,
+      "logits/chosen": -2.8428866863250732,
+      "logits/rejected": -2.8091816902160645,
+      "logps/chosen": -73.93092346191406,
+      "logps/rejected": -75.81602478027344,
+      "loss": 0.6578,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.16270489990711212,
+      "rewards/margins": 0.0773119255900383,
+      "rewards/rejected": -0.2400168478488922,
+      "step": 8410
+    },
+    {
+      "epoch": 1.450723638869745,
+      "grad_norm": 3.908280849456787,
+      "learning_rate": 1.0635111569774754e-08,
+      "logits/chosen": -2.734311580657959,
+      "logits/rejected": -2.7213516235351562,
+      "logps/chosen": -67.79450988769531,
+      "logps/rejected": -72.66189575195312,
+      "loss": 0.6695,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.1614271104335785,
+      "rewards/margins": 0.052555881440639496,
+      "rewards/rejected": -0.21398301422595978,
+      "step": 8420
+    },
+    {
+      "epoch": 1.452446588559614,
+      "grad_norm": 3.6788618564605713,
+      "learning_rate": 1.0573647053036552e-08,
+      "logits/chosen": -2.861396312713623,
+      "logits/rejected": -2.8459248542785645,
+      "logps/chosen": -69.24427795410156,
+      "logps/rejected": -77.0774154663086,
+      "loss": 0.6725,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.1575288027524948,
+      "rewards/margins": 0.04668857157230377,
+      "rewards/rejected": -0.2042173594236374,
+      "step": 8430
+    },
+    {
+      "epoch": 1.454169538249483,
+      "grad_norm": 3.9173455238342285,
+      "learning_rate": 1.0512312994918865e-08,
+      "logits/chosen": -2.8898818492889404,
+      "logits/rejected": -2.872655153274536,
+      "logps/chosen": -71.72085571289062,
+      "logps/rejected": -73.20518493652344,
+      "loss": 0.6721,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.16928228735923767,
+      "rewards/margins": 0.04746762663125992,
+      "rewards/rejected": -0.21674993634223938,
+      "step": 8440
+    },
+    {
+      "epoch": 1.4558924879393522,
+      "grad_norm": 3.6942381858825684,
+      "learning_rate": 1.0451109950070276e-08,
+      "logits/chosen": -2.776853561401367,
+      "logits/rejected": -2.771615982055664,
+      "logps/chosen": -64.40586853027344,
+      "logps/rejected": -74.91841125488281,
+      "loss": 0.6679,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.16394014656543732,
+      "rewards/margins": 0.056191153824329376,
+      "rewards/rejected": -0.2201313078403473,
+      "step": 8450
+    },
+    {
+      "epoch": 1.4576154376292212,
+      "grad_norm": 3.5368716716766357,
+      "learning_rate": 1.039003847195466e-08,
+      "logits/chosen": -2.891749858856201,
+      "logits/rejected": -2.8645224571228027,
+      "logps/chosen": -68.68827056884766,
+      "logps/rejected": -75.73392486572266,
+      "loss": 0.665,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.15881624817848206,
+      "rewards/margins": 0.06121823936700821,
+      "rewards/rejected": -0.22003448009490967,
+      "step": 8460
+    },
+    {
+      "epoch": 1.4593383873190904,
+      "grad_norm": 3.992208242416382,
+      "learning_rate": 1.0329099112846071e-08,
+      "logits/chosen": -2.8628792762756348,
+      "logits/rejected": -2.8355600833892822,
+      "logps/chosen": -73.63648986816406,
+      "logps/rejected": -75.61341857910156,
+      "loss": 0.6714,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.1757003366947174,
+      "rewards/margins": 0.048976074904203415,
+      "rewards/rejected": -0.22467641532421112,
+      "step": 8470
+    },
+    {
+      "epoch": 1.4610613370089593,
+      "grad_norm": 3.5459401607513428,
+      "learning_rate": 1.0268292423823838e-08,
+      "logits/chosen": -2.8649158477783203,
+      "logits/rejected": -2.834582567214966,
+      "logps/chosen": -69.60197448730469,
+      "logps/rejected": -73.1579360961914,
+      "loss": 0.6709,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1656477302312851,
+      "rewards/margins": 0.050050556659698486,
+      "rewards/rejected": -0.2156982719898224,
+      "step": 8480
+    },
+    {
+      "epoch": 1.4627842866988283,
+      "grad_norm": 3.6094672679901123,
+      "learning_rate": 1.020761895476753e-08,
+      "logits/chosen": -2.9066145420074463,
+      "logits/rejected": -2.89974045753479,
+      "logps/chosen": -71.38563537597656,
+      "logps/rejected": -78.689453125,
+      "loss": 0.6718,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.16645634174346924,
+      "rewards/margins": 0.04842091724276543,
+      "rewards/rejected": -0.21487721800804138,
+      "step": 8490
+    },
+    {
+      "epoch": 1.4645072363886975,
+      "grad_norm": 3.577341318130493,
+      "learning_rate": 1.0147079254352e-08,
+      "logits/chosen": -2.7873785495758057,
+      "logits/rejected": -2.7717418670654297,
+      "logps/chosen": -70.33513641357422,
+      "logps/rejected": -75.69012451171875,
+      "loss": 0.6651,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.16028355062007904,
+      "rewards/margins": 0.062162816524505615,
+      "rewards/rejected": -0.22244635224342346,
+      "step": 8500
+    },
+    {
+      "epoch": 1.4645072363886975,
+      "eval_logits/chosen": -2.973315954208374,
+      "eval_logits/rejected": -2.967517137527466,
+      "eval_logps/chosen": -70.08740234375,
+      "eval_logps/rejected": -77.24677276611328,
+      "eval_loss": 0.6813661456108093,
+      "eval_rewards/accuracies": 0.6064126491546631,
+      "eval_rewards/chosen": -0.11375510692596436,
+      "eval_rewards/margins": 0.02691142074763775,
+      "eval_rewards/rejected": -0.14066651463508606,
+      "eval_runtime": 383.444,
+      "eval_samples_per_second": 11.225,
+      "eval_steps_per_second": 1.403,
+      "step": 8500
+    },
+    {
+      "epoch": 1.4662301860785665,
+      "grad_norm": 4.141621112823486,
+      "learning_rate": 1.008667387004242e-08,
+      "logits/chosen": -2.8531124591827393,
+      "logits/rejected": -2.817919969558716,
+      "logps/chosen": -74.30937957763672,
+      "logps/rejected": -74.14118194580078,
+      "loss": 0.6652,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.16460102796554565,
+      "rewards/margins": 0.06154482811689377,
+      "rewards/rejected": -0.2261458933353424,
+      "step": 8510
+    },
+    {
+      "epoch": 1.4679531357684357,
+      "grad_norm": 3.8335866928100586,
+      "learning_rate": 1.0026403348089329e-08,
+      "logits/chosen": -2.846421241760254,
+      "logits/rejected": -2.8118982315063477,
+      "logps/chosen": -69.76349639892578,
+      "logps/rejected": -71.28311920166016,
+      "loss": 0.6647,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.1619357019662857,
+      "rewards/margins": 0.06394011527299881,
+      "rewards/rejected": -0.22587580978870392,
+      "step": 8520
+    },
+    {
+      "epoch": 1.4696760854583046,
+      "grad_norm": 3.5509908199310303,
+      "learning_rate": 9.9662682335237e-09,
+      "logits/chosen": -2.828833818435669,
+      "logits/rejected": -2.811126947402954,
+      "logps/chosen": -71.44664001464844,
+      "logps/rejected": -72.24824523925781,
+      "loss": 0.6799,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.1755964159965515,
+      "rewards/margins": 0.03261363506317139,
+      "rewards/rejected": -0.2082100659608841,
+      "step": 8530
+    },
+    {
+      "epoch": 1.4713990351481736,
+      "grad_norm": 3.521941661834717,
+      "learning_rate": 9.906269070152004e-09,
+      "logits/chosen": -2.959864616394043,
+      "logits/rejected": -2.944051742553711,
+      "logps/chosen": -70.08544921875,
+      "logps/rejected": -74.45684814453125,
+      "loss": 0.6723,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.15391269326210022,
+      "rewards/margins": 0.04631864279508591,
+      "rewards/rejected": -0.20023134350776672,
+      "step": 8540
+    },
+    {
+      "epoch": 1.4731219848380428,
+      "grad_norm": 3.817625045776367,
+      "learning_rate": 9.846406400551307e-09,
+      "logits/chosen": -2.8796818256378174,
+      "logits/rejected": -2.871462106704712,
+      "logps/chosen": -70.18544006347656,
+      "logps/rejected": -76.52879333496094,
+      "loss": 0.6715,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.16347520053386688,
+      "rewards/margins": 0.048515476286411285,
+      "rewards/rejected": -0.21199066936969757,
+      "step": 8550
+    },
+    {
+      "epoch": 1.4748449345279118,
+      "grad_norm": 3.5637881755828857,
+      "learning_rate": 9.786680766064318e-09,
+      "logits/chosen": -2.9665141105651855,
+      "logits/rejected": -2.9442696571350098,
+      "logps/chosen": -70.62559509277344,
+      "logps/rejected": -74.39879608154297,
+      "loss": 0.6679,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.16418324410915375,
+      "rewards/margins": 0.05646062642335892,
+      "rewards/rejected": -0.22064387798309326,
+      "step": 8560
+    },
+    {
+      "epoch": 1.476567884217781,
+      "grad_norm": 3.442197799682617,
+      "learning_rate": 9.727092706794554e-09,
+      "logits/chosen": -2.824240207672119,
+      "logits/rejected": -2.7997817993164062,
+      "logps/chosen": -71.24483489990234,
+      "logps/rejected": -75.1827392578125,
+      "loss": 0.6649,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.1688777655363083,
+      "rewards/margins": 0.06260024011135101,
+      "rewards/rejected": -0.2314780056476593,
+      "step": 8570
+    },
+    {
+      "epoch": 1.47829083390765,
+      "grad_norm": 4.293013572692871,
+      "learning_rate": 9.667642761601433e-09,
+      "logits/chosen": -2.9104347229003906,
+      "logits/rejected": -2.8887314796447754,
+      "logps/chosen": -68.5352783203125,
+      "logps/rejected": -76.14619445800781,
+      "loss": 0.6597,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.14399009943008423,
+      "rewards/margins": 0.07201644033193588,
+      "rewards/rejected": -0.21600651741027832,
+      "step": 8580
+    },
+    {
+      "epoch": 1.480013783597519,
+      "grad_norm": 3.8704121112823486,
+      "learning_rate": 9.608331468095376e-09,
+      "logits/chosen": -2.920609712600708,
+      "logits/rejected": -2.891613483428955,
+      "logps/chosen": -70.98949432373047,
+      "logps/rejected": -73.88973236083984,
+      "loss": 0.6631,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.13994958996772766,
+      "rewards/margins": 0.06512623280286789,
+      "rewards/rejected": -0.20507581532001495,
+      "step": 8590
+    },
+    {
+      "epoch": 1.481736733287388,
+      "grad_norm": 3.4893014430999756,
+      "learning_rate": 9.549159362632986e-09,
+      "logits/chosen": -2.8388099670410156,
+      "logits/rejected": -2.8080246448516846,
+      "logps/chosen": -69.76191711425781,
+      "logps/rejected": -72.10200500488281,
+      "loss": 0.676,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.16323645412921906,
+      "rewards/margins": 0.040306612849235535,
+      "rewards/rejected": -0.2035430669784546,
+      "step": 8600
+    },
+    {
+      "epoch": 1.481736733287388,
+      "eval_logits/chosen": -2.972205877304077,
+      "eval_logits/rejected": -2.966390609741211,
+      "eval_logps/chosen": -70.1413803100586,
+      "eval_logps/rejected": -77.30847930908203,
+      "eval_loss": 0.6813418865203857,
+      "eval_rewards/accuracies": 0.6031598448753357,
+      "eval_rewards/chosen": -0.1142948567867279,
+      "eval_rewards/margins": 0.026988685131072998,
+      "eval_rewards/rejected": -0.1412835568189621,
+      "eval_runtime": 383.5889,
+      "eval_samples_per_second": 11.22,
+      "eval_steps_per_second": 1.403,
+      "step": 8600
+    },
+    {
+      "epoch": 1.483459682977257,
+      "grad_norm": 3.9705910682678223,
+      "learning_rate": 9.490126980312165e-09,
+      "logits/chosen": -2.863651752471924,
+      "logits/rejected": -2.8416531085968018,
+      "logps/chosen": -70.48187255859375,
+      "logps/rejected": -72.033203125,
+      "loss": 0.6707,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.1633698046207428,
+      "rewards/margins": 0.05140014737844467,
+      "rewards/rejected": -0.21476992964744568,
+      "step": 8610
+    },
+    {
+      "epoch": 1.4851826326671262,
+      "grad_norm": 3.7548787593841553,
+      "learning_rate": 9.43123485496729e-09,
+      "logits/chosen": -2.7886345386505127,
+      "logits/rejected": -2.770447254180908,
+      "logps/chosen": -72.08457946777344,
+      "logps/rejected": -72.7845687866211,
+      "loss": 0.6745,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.17310427129268646,
+      "rewards/margins": 0.04223452880978584,
+      "rewards/rejected": -0.2153387814760208,
+      "step": 8620
+    },
+    {
+      "epoch": 1.4869055823569952,
+      "grad_norm": 4.407747745513916,
+      "learning_rate": 9.372483519164398e-09,
+      "logits/chosen": -2.765721082687378,
+      "logits/rejected": -2.753857374191284,
+      "logps/chosen": -67.11076354980469,
+      "logps/rejected": -72.09493255615234,
+      "loss": 0.6721,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.15405713021755219,
+      "rewards/margins": 0.04631081596016884,
+      "rewards/rejected": -0.20036795735359192,
+      "step": 8630
+    },
+    {
+      "epoch": 1.4886285320468642,
+      "grad_norm": 3.824542999267578,
+      "learning_rate": 9.313873504196313e-09,
+      "logits/chosen": -2.9100513458251953,
+      "logits/rejected": -2.880120277404785,
+      "logps/chosen": -71.49581146240234,
+      "logps/rejected": -72.47753143310547,
+      "loss": 0.6766,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.15754231810569763,
+      "rewards/margins": 0.03943229466676712,
+      "rewards/rejected": -0.19697463512420654,
+      "step": 8640
+    },
+    {
+      "epoch": 1.4903514817367332,
+      "grad_norm": 3.534074306488037,
+      "learning_rate": 9.255405340077949e-09,
+      "logits/chosen": -2.823367118835449,
+      "logits/rejected": -2.8025894165039062,
+      "logps/chosen": -69.89283752441406,
+      "logps/rejected": -75.06532287597656,
+      "loss": 0.6658,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.1493956446647644,
+      "rewards/margins": 0.060528464615345,
+      "rewards/rejected": -0.2099241316318512,
+      "step": 8650
+    },
+    {
+      "epoch": 1.4920744314266023,
+      "grad_norm": 3.72786283493042,
+      "learning_rate": 9.197079555541378e-09,
+      "logits/chosen": -2.8354544639587402,
+      "logits/rejected": -2.820857524871826,
+      "logps/chosen": -69.9422836303711,
+      "logps/rejected": -73.80873107910156,
+      "loss": 0.6689,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.16580349206924438,
+      "rewards/margins": 0.05505651235580444,
+      "rewards/rejected": -0.22086003422737122,
+      "step": 8660
+    },
+    {
+      "epoch": 1.4937973811164715,
+      "grad_norm": 3.4578468799591064,
+      "learning_rate": 9.138896678031201e-09,
+      "logits/chosen": -2.9303667545318604,
+      "logits/rejected": -2.912909984588623,
+      "logps/chosen": -69.91584014892578,
+      "logps/rejected": -75.67341613769531,
+      "loss": 0.6712,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.17919966578483582,
+      "rewards/margins": 0.0496668741106987,
+      "rewards/rejected": -0.2288665473461151,
+      "step": 8670
+    },
+    {
+      "epoch": 1.4955203308063405,
+      "grad_norm": 3.7371597290039062,
+      "learning_rate": 9.080857233699624e-09,
+      "logits/chosen": -2.8748059272766113,
+      "logits/rejected": -2.8681819438934326,
+      "logps/chosen": -67.4271240234375,
+      "logps/rejected": -73.93047332763672,
+      "loss": 0.6738,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.16748839616775513,
+      "rewards/margins": 0.04397358372807503,
+      "rewards/rejected": -0.21146197617053986,
+      "step": 8680
+    },
+    {
+      "epoch": 1.4972432804962095,
+      "grad_norm": 3.7510085105895996,
+      "learning_rate": 9.022961747401842e-09,
+      "logits/chosen": -2.8910326957702637,
+      "logits/rejected": -2.8620617389678955,
+      "logps/chosen": -72.6625747680664,
+      "logps/rejected": -69.26103210449219,
+      "loss": 0.6766,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.17126289010047913,
+      "rewards/margins": 0.03826187178492546,
+      "rewards/rejected": -0.20952478051185608,
+      "step": 8690
+    },
+    {
+      "epoch": 1.4989662301860784,
+      "grad_norm": 3.856527090072632,
+      "learning_rate": 8.96521074269117e-09,
+      "logits/chosen": -2.8949217796325684,
+      "logits/rejected": -2.8619656562805176,
+      "logps/chosen": -72.56687927246094,
+      "logps/rejected": -74.1780776977539,
+      "loss": 0.6682,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.16069389879703522,
+      "rewards/margins": 0.05466220900416374,
+      "rewards/rejected": -0.21535611152648926,
+      "step": 8700
+    },
+    {
+      "epoch": 1.4989662301860784,
+      "eval_logits/chosen": -2.971815586090088,
+      "eval_logits/rejected": -2.9659900665283203,
+      "eval_logps/chosen": -70.12594604492188,
+      "eval_logps/rejected": -77.28851318359375,
+      "eval_loss": 0.6813595294952393,
+      "eval_rewards/accuracies": 0.6050186157226562,
+      "eval_rewards/chosen": -0.11414045095443726,
+      "eval_rewards/margins": 0.026943398639559746,
+      "eval_rewards/rejected": -0.14108383655548096,
+      "eval_runtime": 383.1749,
+      "eval_samples_per_second": 11.232,
+      "eval_steps_per_second": 1.404,
+      "step": 8700
+    },
+    {
+      "epoch": 1.5006891798759476,
+      "grad_norm": 3.785468578338623,
+      "learning_rate": 8.907604741814404e-09,
+      "logits/chosen": -2.860616683959961,
+      "logits/rejected": -2.8465497493743896,
+      "logps/chosen": -69.4238052368164,
+      "logps/rejected": -72.99176025390625,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.1658819168806076,
+      "rewards/margins": 0.03257400542497635,
+      "rewards/rejected": -0.19845592975616455,
+      "step": 8710
+    },
+    {
+      "epoch": 1.5024121295658168,
+      "grad_norm": 3.568876028060913,
+      "learning_rate": 8.850144265707039e-09,
+      "logits/chosen": -2.855849027633667,
+      "logits/rejected": -2.8310704231262207,
+      "logps/chosen": -72.10614013671875,
+      "logps/rejected": -75.85025024414062,
+      "loss": 0.6677,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1575855314731598,
+      "rewards/margins": 0.05635889247059822,
+      "rewards/rejected": -0.21394440531730652,
+      "step": 8720
+    },
+    {
+      "epoch": 1.5041350792556858,
+      "grad_norm": 3.5062997341156006,
+      "learning_rate": 8.792829833988588e-09,
+      "logits/chosen": -2.88454008102417,
+      "logits/rejected": -2.8521807193756104,
+      "logps/chosen": -69.22032165527344,
+      "logps/rejected": -74.32414245605469,
+      "loss": 0.6685,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.16949030756950378,
+      "rewards/margins": 0.05526772141456604,
+      "rewards/rejected": -0.22475802898406982,
+      "step": 8730
+    },
+    {
+      "epoch": 1.5058580289455548,
+      "grad_norm": 3.462279796600342,
+      "learning_rate": 8.73566196495787e-09,
+      "logits/chosen": -2.8448476791381836,
+      "logits/rejected": -2.8344905376434326,
+      "logps/chosen": -69.2294692993164,
+      "logps/rejected": -75.1268539428711,
+      "loss": 0.6742,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.1782539188861847,
+      "rewards/margins": 0.043016739189624786,
+      "rewards/rejected": -0.22127068042755127,
+      "step": 8740
+    },
+    {
+      "epoch": 1.5075809786354237,
+      "grad_norm": 4.500255584716797,
+      "learning_rate": 8.678641175588324e-09,
+      "logits/chosen": -2.8818368911743164,
+      "logits/rejected": -2.858457088470459,
+      "logps/chosen": -69.29254150390625,
+      "logps/rejected": -75.35603332519531,
+      "loss": 0.6633,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.15594448149204254,
+      "rewards/margins": 0.06560229510068893,
+      "rewards/rejected": -0.2215467393398285,
+      "step": 8750
+    },
+    {
+      "epoch": 1.509303928325293,
+      "grad_norm": 3.748511791229248,
+      "learning_rate": 8.621767981523351e-09,
+      "logits/chosen": -2.827807903289795,
+      "logits/rejected": -2.817012310028076,
+      "logps/chosen": -66.80030059814453,
+      "logps/rejected": -76.3193588256836,
+      "loss": 0.6664,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1530340015888214,
+      "rewards/margins": 0.058438099920749664,
+      "rewards/rejected": -0.21147210896015167,
+      "step": 8760
+    },
+    {
+      "epoch": 1.5110268780151621,
+      "grad_norm": 3.4706029891967773,
+      "learning_rate": 8.565042897071607e-09,
+      "logits/chosen": -2.8740592002868652,
+      "logits/rejected": -2.8455843925476074,
+      "logps/chosen": -71.60909271240234,
+      "logps/rejected": -72.85354614257812,
+      "loss": 0.6639,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.16533717513084412,
+      "rewards/margins": 0.06559725850820541,
+      "rewards/rejected": -0.23093441128730774,
+      "step": 8770
+    },
+    {
+      "epoch": 1.512749827705031,
+      "grad_norm": 3.8741345405578613,
+      "learning_rate": 8.508466435202402e-09,
+      "logits/chosen": -2.901150703430176,
+      "logits/rejected": -2.9012014865875244,
+      "logps/chosen": -65.85799407958984,
+      "logps/rejected": -75.48812103271484,
+      "loss": 0.6693,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1676429808139801,
+      "rewards/margins": 0.05274323374032974,
+      "rewards/rejected": -0.22038622200489044,
+      "step": 8780
+    },
+    {
+      "epoch": 1.5144727773949,
+      "grad_norm": 3.9681639671325684,
+      "learning_rate": 8.452039107541043e-09,
+      "logits/chosen": -2.9065804481506348,
+      "logits/rejected": -2.8787524700164795,
+      "logps/chosen": -71.93157958984375,
+      "logps/rejected": -74.06289672851562,
+      "loss": 0.6714,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.1700780689716339,
+      "rewards/margins": 0.050253432244062424,
+      "rewards/rejected": -0.22033150494098663,
+      "step": 8790
+    },
+    {
+      "epoch": 1.516195727084769,
+      "grad_norm": 4.009823799133301,
+      "learning_rate": 8.395761424364193e-09,
+      "logits/chosen": -2.8034870624542236,
+      "logits/rejected": -2.77040433883667,
+      "logps/chosen": -70.93616485595703,
+      "logps/rejected": -72.95118713378906,
+      "loss": 0.6732,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.1593705117702484,
+      "rewards/margins": 0.045195553451776505,
+      "rewards/rejected": -0.204566091299057,
+      "step": 8800
+    },
+    {
+      "epoch": 1.516195727084769,
+      "eval_logits/chosen": -2.970808267593384,
+      "eval_logits/rejected": -2.9649600982666016,
+      "eval_logps/chosen": -70.1772689819336,
+      "eval_logps/rejected": -77.3463363647461,
+      "eval_loss": 0.6813353300094604,
+      "eval_rewards/accuracies": 0.5996747016906738,
+      "eval_rewards/chosen": -0.11465370655059814,
+      "eval_rewards/margins": 0.02700836770236492,
+      "eval_rewards/rejected": -0.14166207611560822,
+      "eval_runtime": 383.1362,
+      "eval_samples_per_second": 11.234,
+      "eval_steps_per_second": 1.404,
+      "step": 8800
+    },
+    {
+      "epoch": 1.5179186767746382,
+      "grad_norm": 3.971342086791992,
+      "learning_rate": 8.33963389459528e-09,
+      "logits/chosen": -2.9465956687927246,
+      "logits/rejected": -2.9238173961639404,
+      "logps/chosen": -67.64794921875,
+      "logps/rejected": -74.66142272949219,
+      "loss": 0.6693,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.16687563061714172,
+      "rewards/margins": 0.05367749184370041,
+      "rewards/rejected": -0.22055311501026154,
+      "step": 8810
+    },
+    {
+      "epoch": 1.5196416264645074,
+      "grad_norm": 3.5265471935272217,
+      "learning_rate": 8.283657025799872e-09,
+      "logits/chosen": -2.89447283744812,
+      "logits/rejected": -2.87432599067688,
+      "logps/chosen": -66.1352310180664,
+      "logps/rejected": -71.23625183105469,
+      "loss": 0.6674,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.15711040794849396,
+      "rewards/margins": 0.05742807313799858,
+      "rewards/rejected": -0.21453848481178284,
+      "step": 8820
+    },
+    {
+      "epoch": 1.5213645761543764,
+      "grad_norm": 3.6461234092712402,
+      "learning_rate": 8.227831324181108e-09,
+      "logits/chosen": -2.752190589904785,
+      "logits/rejected": -2.7303922176361084,
+      "logps/chosen": -70.90214538574219,
+      "logps/rejected": -73.49878692626953,
+      "loss": 0.6735,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.16306154429912567,
+      "rewards/margins": 0.044502392411231995,
+      "rewards/rejected": -0.20756396651268005,
+      "step": 8830
+    },
+    {
+      "epoch": 1.5230875258442453,
+      "grad_norm": 3.9130096435546875,
+      "learning_rate": 8.172157294575107e-09,
+      "logits/chosen": -2.7847962379455566,
+      "logits/rejected": -2.774132490158081,
+      "logps/chosen": -67.13465118408203,
+      "logps/rejected": -74.67637634277344,
+      "loss": 0.6653,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.14786283671855927,
+      "rewards/margins": 0.06093641370534897,
+      "rewards/rejected": -0.20879924297332764,
+      "step": 8840
+    },
+    {
+      "epoch": 1.5248104755341143,
+      "grad_norm": 3.72239089012146,
+      "learning_rate": 8.116635440446401e-09,
+      "logits/chosen": -2.9677894115448,
+      "logits/rejected": -2.949326753616333,
+      "logps/chosen": -66.58848571777344,
+      "logps/rejected": -74.23371124267578,
+      "loss": 0.6677,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.14338089525699615,
+      "rewards/margins": 0.05602588504552841,
+      "rewards/rejected": -0.19940677285194397,
+      "step": 8850
+    },
+    {
+      "epoch": 1.5265334252239835,
+      "grad_norm": 4.039936065673828,
+      "learning_rate": 8.061266263883404e-09,
+      "logits/chosen": -2.872671127319336,
+      "logits/rejected": -2.851142406463623,
+      "logps/chosen": -69.2303466796875,
+      "logps/rejected": -74.68756866455078,
+      "loss": 0.6668,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.16049246490001678,
+      "rewards/margins": 0.05777040868997574,
+      "rewards/rejected": -0.21826288104057312,
+      "step": 8860
+    },
+    {
+      "epoch": 1.5282563749138525,
+      "grad_norm": 3.681178092956543,
+      "learning_rate": 8.006050265593815e-09,
+      "logits/chosen": -3.0146279335021973,
+      "logits/rejected": -2.9816460609436035,
+      "logps/chosen": -72.1039047241211,
+      "logps/rejected": -76.38068389892578,
+      "loss": 0.6659,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.1533397138118744,
+      "rewards/margins": 0.06021096184849739,
+      "rewards/rejected": -0.2135506570339203,
+      "step": 8870
+    },
+    {
+      "epoch": 1.5299793246037217,
+      "grad_norm": 3.680227756500244,
+      "learning_rate": 7.950987944900191e-09,
+      "logits/chosen": -2.8119924068450928,
+      "logits/rejected": -2.7853763103485107,
+      "logps/chosen": -71.73873138427734,
+      "logps/rejected": -74.21075439453125,
+      "loss": 0.6743,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.18287543952465057,
+      "rewards/margins": 0.043742455542087555,
+      "rewards/rejected": -0.22661790251731873,
+      "step": 8880
+    },
+    {
+      "epoch": 1.5317022742935906,
+      "grad_norm": 3.5914480686187744,
+      "learning_rate": 7.896079799735308e-09,
+      "logits/chosen": -2.848123550415039,
+      "logits/rejected": -2.822627067565918,
+      "logps/chosen": -70.66083526611328,
+      "logps/rejected": -72.16704559326172,
+      "loss": 0.6691,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.16166475415229797,
+      "rewards/margins": 0.05231751874089241,
+      "rewards/rejected": -0.21398229897022247,
+      "step": 8890
+    },
+    {
+      "epoch": 1.5334252239834596,
+      "grad_norm": 3.5073280334472656,
+      "learning_rate": 7.841326326637782e-09,
+      "logits/chosen": -2.8968758583068848,
+      "logits/rejected": -2.8655457496643066,
+      "logps/chosen": -71.67398834228516,
+      "logps/rejected": -74.85285949707031,
+      "loss": 0.6706,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.16787873208522797,
+      "rewards/margins": 0.05068850517272949,
+      "rewards/rejected": -0.21856722235679626,
+      "step": 8900
+    },
+    {
+      "epoch": 1.5334252239834596,
+      "eval_logits/chosen": -2.9699740409851074,
+      "eval_logits/rejected": -2.9641449451446533,
+      "eval_logps/chosen": -70.3132553100586,
+      "eval_logps/rejected": -77.52465057373047,
+      "eval_loss": 0.6811440587043762,
+      "eval_rewards/accuracies": 0.6108271479606628,
+      "eval_rewards/chosen": -0.11601359397172928,
+      "eval_rewards/margins": 0.027431726455688477,
+      "eval_rewards/rejected": -0.14344531297683716,
+      "eval_runtime": 383.6616,
+      "eval_samples_per_second": 11.218,
+      "eval_steps_per_second": 1.402,
+      "step": 8900
+    },
+    {
+      "epoch": 1.5351481736733288,
+      "grad_norm": 3.794710636138916,
+      "learning_rate": 7.786728020747463e-09,
+      "logits/chosen": -2.858767032623291,
+      "logits/rejected": -2.8458919525146484,
+      "logps/chosen": -72.8824691772461,
+      "logps/rejected": -75.8398208618164,
+      "loss": 0.668,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.17206808924674988,
+      "rewards/margins": 0.05573774501681328,
+      "rewards/rejected": -0.22780582308769226,
+      "step": 8910
+    },
+    {
+      "epoch": 1.5368711233631978,
+      "grad_norm": 4.242312908172607,
+      "learning_rate": 7.732285375801039e-09,
+      "logits/chosen": -2.94620680809021,
+      "logits/rejected": -2.914452075958252,
+      "logps/chosen": -73.83602905273438,
+      "logps/rejected": -75.88147735595703,
+      "loss": 0.6621,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.1679040640592575,
+      "rewards/margins": 0.06980032473802567,
+      "rewards/rejected": -0.23770436644554138,
+      "step": 8920
+    },
+    {
+      "epoch": 1.538594073053067,
+      "grad_norm": 3.4974377155303955,
+      "learning_rate": 7.677998884127543e-09,
+      "logits/chosen": -2.894538164138794,
+      "logits/rejected": -2.8633735179901123,
+      "logps/chosen": -71.07281494140625,
+      "logps/rejected": -72.61754608154297,
+      "loss": 0.6675,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.16383609175682068,
+      "rewards/margins": 0.05663411691784859,
+      "rewards/rejected": -0.22047021985054016,
+      "step": 8930
+    },
+    {
+      "epoch": 1.540317022742936,
+      "grad_norm": 3.6466331481933594,
+      "learning_rate": 7.623869036643902e-09,
+      "logits/chosen": -2.874589443206787,
+      "logits/rejected": -2.8569836616516113,
+      "logps/chosen": -72.00877380371094,
+      "logps/rejected": -76.20338439941406,
+      "loss": 0.6756,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.16374003887176514,
+      "rewards/margins": 0.04039059206843376,
+      "rewards/rejected": -0.2041306048631668,
+      "step": 8940
+    },
+    {
+      "epoch": 1.5420399724328049,
+      "grad_norm": 4.222042560577393,
+      "learning_rate": 7.569896322850488e-09,
+      "logits/chosen": -2.7846286296844482,
+      "logits/rejected": -2.785799980163574,
+      "logps/chosen": -69.29002380371094,
+      "logps/rejected": -76.82630920410156,
+      "loss": 0.674,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.15188291668891907,
+      "rewards/margins": 0.04464026540517807,
+      "rewards/rejected": -0.19652317464351654,
+      "step": 8950
+    },
+    {
+      "epoch": 1.5437629221226739,
+      "grad_norm": 3.8883285522460938,
+      "learning_rate": 7.516081230826716e-09,
+      "logits/chosen": -2.900395631790161,
+      "logits/rejected": -2.8739254474639893,
+      "logps/chosen": -72.75261688232422,
+      "logps/rejected": -76.04643249511719,
+      "loss": 0.6667,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.18349973857402802,
+      "rewards/margins": 0.059685118496418,
+      "rewards/rejected": -0.2431848794221878,
+      "step": 8960
+    },
+    {
+      "epoch": 1.545485871812543,
+      "grad_norm": 3.927036762237549,
+      "learning_rate": 7.462424247226607e-09,
+      "logits/chosen": -2.8542091846466064,
+      "logits/rejected": -2.8271656036376953,
+      "logps/chosen": -69.92227935791016,
+      "logps/rejected": -72.38612365722656,
+      "loss": 0.6723,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.16638341546058655,
+      "rewards/margins": 0.046493999660015106,
+      "rewards/rejected": -0.21287742257118225,
+      "step": 8970
+    },
+    {
+      "epoch": 1.5472088215024122,
+      "grad_norm": 3.974217653274536,
+      "learning_rate": 7.408925857274373e-09,
+      "logits/chosen": -2.9434852600097656,
+      "logits/rejected": -2.9125571250915527,
+      "logps/chosen": -72.40852355957031,
+      "logps/rejected": -74.55555725097656,
+      "loss": 0.6766,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.17927810549736023,
+      "rewards/margins": 0.03927849605679512,
+      "rewards/rejected": -0.21855656802654266,
+      "step": 8980
+    },
+    {
+      "epoch": 1.5489317711922812,
+      "grad_norm": 4.278979301452637,
+      "learning_rate": 7.355586544760109e-09,
+      "logits/chosen": -2.849220037460327,
+      "logits/rejected": -2.8318448066711426,
+      "logps/chosen": -70.37113952636719,
+      "logps/rejected": -74.50631713867188,
+      "loss": 0.6697,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.154214546084404,
+      "rewards/margins": 0.05064479261636734,
+      "rewards/rejected": -0.20485934615135193,
+      "step": 8990
+    },
+    {
+      "epoch": 1.5506547208821502,
+      "grad_norm": 3.891509532928467,
+      "learning_rate": 7.302406792035298e-09,
+      "logits/chosen": -2.884089946746826,
+      "logits/rejected": -2.8555400371551514,
+      "logps/chosen": -73.0742416381836,
+      "logps/rejected": -75.74182891845703,
+      "loss": 0.6589,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.15659791231155396,
+      "rewards/margins": 0.07500281929969788,
+      "rewards/rejected": -0.23160073161125183,
+      "step": 9000
+    },
+    {
+      "epoch": 1.5506547208821502,
+      "eval_logits/chosen": -2.96890926361084,
+      "eval_logits/rejected": -2.963088274002075,
+      "eval_logps/chosen": -70.39956665039062,
+      "eval_logps/rejected": -77.6093978881836,
+      "eval_loss": 0.6811633110046387,
+      "eval_rewards/accuracies": 0.6052509546279907,
+      "eval_rewards/chosen": -0.11687670648097992,
+      "eval_rewards/margins": 0.0274160485714674,
+      "eval_rewards/rejected": -0.14429275691509247,
+      "eval_runtime": 383.0012,
+      "eval_samples_per_second": 11.238,
+      "eval_steps_per_second": 1.405,
+      "step": 9000
+    },
+    {
+      "epoch": 1.5523776705720191,
+      "grad_norm": 3.356119155883789,
+      "learning_rate": 7.249387080008551e-09,
+      "logits/chosen": -2.867082118988037,
+      "logits/rejected": -2.8417649269104004,
+      "logps/chosen": -71.67340087890625,
+      "logps/rejected": -74.15412902832031,
+      "loss": 0.6691,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.16034123301506042,
+      "rewards/margins": 0.052654944360256195,
+      "rewards/rejected": -0.21299616992473602,
+      "step": 9010
+    },
+    {
+      "epoch": 1.5541006202618883,
+      "grad_norm": 3.5731403827667236,
+      "learning_rate": 7.196527888141199e-09,
+      "logits/chosen": -2.793649196624756,
+      "logits/rejected": -2.77695631980896,
+      "logps/chosen": -70.78572082519531,
+      "logps/rejected": -74.31391906738281,
+      "loss": 0.6633,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.16861680150032043,
+      "rewards/margins": 0.06532196700572968,
+      "rewards/rejected": -0.2339387685060501,
+      "step": 9020
+    },
+    {
+      "epoch": 1.5558235699517575,
+      "grad_norm": 3.3170645236968994,
+      "learning_rate": 7.14382969444299e-09,
+      "logits/chosen": -2.8138012886047363,
+      "logits/rejected": -2.819331645965576,
+      "logps/chosen": -67.64290618896484,
+      "logps/rejected": -75.09062957763672,
+      "loss": 0.672,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.16490910947322845,
+      "rewards/margins": 0.04784511774778366,
+      "rewards/rejected": -0.21275421977043152,
+      "step": 9030
+    },
+    {
+      "epoch": 1.5575465196416265,
+      "grad_norm": 3.843120813369751,
+      "learning_rate": 7.091292975467744e-09,
+      "logits/chosen": -2.828660249710083,
+      "logits/rejected": -2.807074785232544,
+      "logps/chosen": -68.58784484863281,
+      "logps/rejected": -74.06195068359375,
+      "loss": 0.6722,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.17418920993804932,
+      "rewards/margins": 0.04642399400472641,
+      "rewards/rejected": -0.22061319649219513,
+      "step": 9040
+    },
+    {
+      "epoch": 1.5592694693314955,
+      "grad_norm": 3.8218929767608643,
+      "learning_rate": 7.038918206309061e-09,
+      "logits/chosen": -2.877487897872925,
+      "logits/rejected": -2.8585591316223145,
+      "logps/chosen": -71.30154418945312,
+      "logps/rejected": -74.51936340332031,
+      "loss": 0.6729,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.17544016242027283,
+      "rewards/margins": 0.04533000662922859,
+      "rewards/rejected": -0.22077016532421112,
+      "step": 9050
+    },
+    {
+      "epoch": 1.5609924190213644,
+      "grad_norm": 3.8904221057891846,
+      "learning_rate": 6.986705860596004e-09,
+      "logits/chosen": -2.8809316158294678,
+      "logits/rejected": -2.8551480770111084,
+      "logps/chosen": -72.36249542236328,
+      "logps/rejected": -75.11692810058594,
+      "loss": 0.6743,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.17649489641189575,
+      "rewards/margins": 0.04264426976442337,
+      "rewards/rejected": -0.21913917362689972,
+      "step": 9060
+    },
+    {
+      "epoch": 1.5627153687112336,
+      "grad_norm": 3.8687069416046143,
+      "learning_rate": 6.934656410488848e-09,
+      "logits/chosen": -2.845407009124756,
+      "logits/rejected": -2.82771635055542,
+      "logps/chosen": -70.21786499023438,
+      "logps/rejected": -77.31141662597656,
+      "loss": 0.6619,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.15652312338352203,
+      "rewards/margins": 0.06859511137008667,
+      "rewards/rejected": -0.2251182347536087,
+      "step": 9070
+    },
+    {
+      "epoch": 1.5644383184011028,
+      "grad_norm": 3.566767930984497,
+      "learning_rate": 6.882770326674752e-09,
+      "logits/chosen": -2.8376388549804688,
+      "logits/rejected": -2.8341097831726074,
+      "logps/chosen": -68.06311798095703,
+      "logps/rejected": -76.09378051757812,
+      "loss": 0.6727,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.160824254155159,
+      "rewards/margins": 0.04588223993778229,
+      "rewards/rejected": -0.2067064791917801,
+      "step": 9080
+    },
+    {
+      "epoch": 1.5661612680909718,
+      "grad_norm": 4.2194013595581055,
+      "learning_rate": 6.831048078363602e-09,
+      "logits/chosen": -2.8140177726745605,
+      "logits/rejected": -2.779407501220703,
+      "logps/chosen": -72.14134216308594,
+      "logps/rejected": -72.44676971435547,
+      "loss": 0.6694,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.16872745752334595,
+      "rewards/margins": 0.05158630758523941,
+      "rewards/rejected": -0.22031378746032715,
+      "step": 9090
+    },
+    {
+      "epoch": 1.5678842177808407,
+      "grad_norm": 4.045811653137207,
+      "learning_rate": 6.779490133283638e-09,
+      "logits/chosen": -2.8834240436553955,
+      "logits/rejected": -2.8569464683532715,
+      "logps/chosen": -72.9145736694336,
+      "logps/rejected": -74.83987426757812,
+      "loss": 0.6694,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.16117341816425323,
+      "rewards/margins": 0.05212359502911568,
+      "rewards/rejected": -0.21329697966575623,
+      "step": 9100
+    },
+    {
+      "epoch": 1.5678842177808407,
+      "eval_logits/chosen": -2.967963218688965,
+      "eval_logits/rejected": -2.9621081352233887,
+      "eval_logps/chosen": -70.43242645263672,
+      "eval_logps/rejected": -77.64903259277344,
+      "eval_loss": 0.6811376810073853,
+      "eval_rewards/accuracies": 0.6043215394020081,
+      "eval_rewards/chosen": -0.11720532923936844,
+      "eval_rewards/margins": 0.02748374082148075,
+      "eval_rewards/rejected": -0.14468906819820404,
+      "eval_runtime": 382.588,
+      "eval_samples_per_second": 11.25,
+      "eval_steps_per_second": 1.406,
+      "step": 9100
+    },
+    {
+      "epoch": 1.5696071674707097,
+      "grad_norm": 3.6328752040863037,
+      "learning_rate": 6.72809695767736e-09,
+      "logits/chosen": -2.8882861137390137,
+      "logits/rejected": -2.861119031906128,
+      "logps/chosen": -70.92989349365234,
+      "logps/rejected": -74.76731872558594,
+      "loss": 0.6637,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.15681412816047668,
+      "rewards/margins": 0.06490675359964371,
+      "rewards/rejected": -0.221720889210701,
+      "step": 9110
+    },
+    {
+      "epoch": 1.571330117160579,
+      "grad_norm": 4.029331207275391,
+      "learning_rate": 6.676869016297179e-09,
+      "logits/chosen": -2.9040613174438477,
+      "logits/rejected": -2.8693647384643555,
+      "logps/chosen": -75.60948181152344,
+      "logps/rejected": -76.40543365478516,
+      "loss": 0.6659,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.15333883464336395,
+      "rewards/margins": 0.059713780879974365,
+      "rewards/rejected": -0.21305260062217712,
+      "step": 9120
+    },
+    {
+      "epoch": 1.573053066850448,
+      "grad_norm": 3.922640800476074,
+      "learning_rate": 6.625806772401346e-09,
+      "logits/chosen": -2.818856716156006,
+      "logits/rejected": -2.795642137527466,
+      "logps/chosen": -70.77824401855469,
+      "logps/rejected": -74.95015716552734,
+      "loss": 0.6704,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.15759356319904327,
+      "rewards/margins": 0.05089858919382095,
+      "rewards/rejected": -0.20849213004112244,
+      "step": 9130
+    },
+    {
+      "epoch": 1.574776016540317,
+      "grad_norm": 3.4892215728759766,
+      "learning_rate": 6.574910687749641e-09,
+      "logits/chosen": -2.898493528366089,
+      "logits/rejected": -2.8468663692474365,
+      "logps/chosen": -71.40845489501953,
+      "logps/rejected": -72.4461441040039,
+      "loss": 0.6517,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.14126896858215332,
+      "rewards/margins": 0.09145893901586533,
+      "rewards/rejected": -0.23272791504859924,
+      "step": 9140
+    },
+    {
+      "epoch": 1.576498966230186,
+      "grad_norm": 4.021738529205322,
+      "learning_rate": 6.524181222599282e-09,
+      "logits/chosen": -2.8827242851257324,
+      "logits/rejected": -2.8551697731018066,
+      "logps/chosen": -75.06977844238281,
+      "logps/rejected": -77.94156646728516,
+      "loss": 0.6626,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.16743791103363037,
+      "rewards/margins": 0.06720934808254242,
+      "rewards/rejected": -0.23464727401733398,
+      "step": 9150
+    },
+    {
+      "epoch": 1.578221915920055,
+      "grad_norm": 3.029460906982422,
+      "learning_rate": 6.473618835700731e-09,
+      "logits/chosen": -2.892106771469116,
+      "logits/rejected": -2.8879284858703613,
+      "logps/chosen": -65.77980041503906,
+      "logps/rejected": -75.30325317382812,
+      "loss": 0.6678,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.16629895567893982,
+      "rewards/margins": 0.055885035544633865,
+      "rewards/rejected": -0.2221839874982834,
+      "step": 9160
+    },
+    {
+      "epoch": 1.5799448656099242,
+      "grad_norm": 4.185909748077393,
+      "learning_rate": 6.4232239842935434e-09,
+      "logits/chosen": -2.8942768573760986,
+      "logits/rejected": -2.862457513809204,
+      "logps/chosen": -72.95067596435547,
+      "logps/rejected": -73.90100860595703,
+      "loss": 0.6646,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.18109166622161865,
+      "rewards/margins": 0.06319538503885269,
+      "rewards/rejected": -0.24428705871105194,
+      "step": 9170
+    },
+    {
+      "epoch": 1.5816678152997934,
+      "grad_norm": 4.010531902313232,
+      "learning_rate": 6.372997124102245e-09,
+      "logits/chosen": -2.8776092529296875,
+      "logits/rejected": -2.8569729328155518,
+      "logps/chosen": -72.42376708984375,
+      "logps/rejected": -73.94020080566406,
+      "loss": 0.6752,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.17692846059799194,
+      "rewards/margins": 0.04083051159977913,
+      "rewards/rejected": -0.21775896847248077,
+      "step": 9180
+    },
+    {
+      "epoch": 1.5833907649896624,
+      "grad_norm": 3.8330183029174805,
+      "learning_rate": 6.3229387093321955e-09,
+      "logits/chosen": -2.98978590965271,
+      "logits/rejected": -2.983905076980591,
+      "logps/chosen": -71.66443634033203,
+      "logps/rejected": -78.99949645996094,
+      "loss": 0.6729,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.17507201433181763,
+      "rewards/margins": 0.04564880579710007,
+      "rewards/rejected": -0.22072084248065948,
+      "step": 9190
+    },
+    {
+      "epoch": 1.5851137146795313,
+      "grad_norm": 3.9870967864990234,
+      "learning_rate": 6.273049192665503e-09,
+      "logits/chosen": -2.8897652626037598,
+      "logits/rejected": -2.8683674335479736,
+      "logps/chosen": -71.17974090576172,
+      "logps/rejected": -75.3651351928711,
+      "loss": 0.6691,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.183277890086174,
+      "rewards/margins": 0.054264552891254425,
+      "rewards/rejected": -0.23754243552684784,
+      "step": 9200
+    },
+    {
+      "epoch": 1.5851137146795313,
+      "eval_logits/chosen": -2.96746563911438,
+      "eval_logits/rejected": -2.9616546630859375,
+      "eval_logps/chosen": -70.4981460571289,
+      "eval_logps/rejected": -77.73652648925781,
+      "eval_loss": 0.6810349225997925,
+      "eval_rewards/accuracies": 0.6010687947273254,
+      "eval_rewards/chosen": -0.11786249279975891,
+      "eval_rewards/margins": 0.027701441198587418,
+      "eval_rewards/rejected": -0.14556396007537842,
+      "eval_runtime": 382.2372,
+      "eval_samples_per_second": 11.26,
+      "eval_steps_per_second": 1.408,
+      "step": 9200
+    },
+    {
+      "epoch": 1.5868366643694003,
+      "grad_norm": 3.631648540496826,
+      "learning_rate": 6.223329025256896e-09,
+      "logits/chosen": -2.7699246406555176,
+      "logits/rejected": -2.7432992458343506,
+      "logps/chosen": -73.16697692871094,
+      "logps/rejected": -74.9011001586914,
+      "loss": 0.6674,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.17320004105567932,
+      "rewards/margins": 0.057905275374650955,
+      "rewards/rejected": -0.23110529780387878,
+      "step": 9210
+    },
+    {
+      "epoch": 1.5885596140592695,
+      "grad_norm": 4.097886085510254,
+      "learning_rate": 6.173778656729678e-09,
+      "logits/chosen": -2.866349220275879,
+      "logits/rejected": -2.843660354614258,
+      "logps/chosen": -66.9336929321289,
+      "logps/rejected": -71.09359741210938,
+      "loss": 0.6726,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.16920194029808044,
+      "rewards/margins": 0.04624428600072861,
+      "rewards/rejected": -0.21544623374938965,
+      "step": 9220
+    },
+    {
+      "epoch": 1.5902825637491387,
+      "grad_norm": 4.021084785461426,
+      "learning_rate": 6.124398535171654e-09,
+      "logits/chosen": -2.7859976291656494,
+      "logits/rejected": -2.77193546295166,
+      "logps/chosen": -66.0755615234375,
+      "logps/rejected": -73.76950073242188,
+      "loss": 0.6745,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.1676885336637497,
+      "rewards/margins": 0.04146091639995575,
+      "rewards/rejected": -0.20914945006370544,
+      "step": 9230
+    },
+    {
+      "epoch": 1.5920055134390076,
+      "grad_norm": 3.8421661853790283,
+      "learning_rate": 6.075189107131059e-09,
+      "logits/chosen": -2.820019006729126,
+      "logits/rejected": -2.8151967525482178,
+      "logps/chosen": -72.20233154296875,
+      "logps/rejected": -76.58494567871094,
+      "loss": 0.6668,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.16052143275737762,
+      "rewards/margins": 0.05939025804400444,
+      "rewards/rejected": -0.21991169452667236,
+      "step": 9240
+    },
+    {
+      "epoch": 1.5937284631288766,
+      "grad_norm": 3.7846853733062744,
+      "learning_rate": 6.026150817612544e-09,
+      "logits/chosen": -2.8399131298065186,
+      "logits/rejected": -2.814845323562622,
+      "logps/chosen": -70.1598129272461,
+      "logps/rejected": -74.39149475097656,
+      "loss": 0.6736,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.15373827517032623,
+      "rewards/margins": 0.0440959706902504,
+      "rewards/rejected": -0.19783422350883484,
+      "step": 9250
+    },
+    {
+      "epoch": 1.5954514128187456,
+      "grad_norm": 3.5805282592773438,
+      "learning_rate": 5.977284110073136e-09,
+      "logits/chosen": -2.813131332397461,
+      "logits/rejected": -2.7947404384613037,
+      "logps/chosen": -66.49539184570312,
+      "logps/rejected": -72.11285400390625,
+      "loss": 0.6722,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.18432626128196716,
+      "rewards/margins": 0.04689601808786392,
+      "rewards/rejected": -0.2312222719192505,
+      "step": 9260
+    },
+    {
+      "epoch": 1.5971743625086148,
+      "grad_norm": 3.5770912170410156,
+      "learning_rate": 5.928589426418234e-09,
+      "logits/chosen": -2.953868865966797,
+      "logits/rejected": -2.9207611083984375,
+      "logps/chosen": -72.19523620605469,
+      "logps/rejected": -74.2688217163086,
+      "loss": 0.6703,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.17301973700523376,
+      "rewards/margins": 0.051523953676223755,
+      "rewards/rejected": -0.22454366087913513,
+      "step": 9270
+    },
+    {
+      "epoch": 1.598897312198484,
+      "grad_norm": 3.960315704345703,
+      "learning_rate": 5.880067206997611e-09,
+      "logits/chosen": -2.871420383453369,
+      "logits/rejected": -2.8556928634643555,
+      "logps/chosen": -70.1946029663086,
+      "logps/rejected": -76.12310028076172,
+      "loss": 0.668,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.166243314743042,
+      "rewards/margins": 0.057053785771131516,
+      "rewards/rejected": -0.2232970893383026,
+      "step": 9280
+    },
+    {
+      "epoch": 1.600620261888353,
+      "grad_norm": 3.5591444969177246,
+      "learning_rate": 5.831717890601434e-09,
+      "logits/chosen": -2.7665529251098633,
+      "logits/rejected": -2.744448184967041,
+      "logps/chosen": -70.98143005371094,
+      "logps/rejected": -71.2352523803711,
+      "loss": 0.6754,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.16326478123664856,
+      "rewards/margins": 0.04042563587427139,
+      "rewards/rejected": -0.20369040966033936,
+      "step": 9290
+    },
+    {
+      "epoch": 1.602343211578222,
+      "grad_norm": 4.341728210449219,
+      "learning_rate": 5.7835419144563e-09,
+      "logits/chosen": -2.8497815132141113,
+      "logits/rejected": -2.8448171615600586,
+      "logps/chosen": -72.17292785644531,
+      "logps/rejected": -79.54914855957031,
+      "loss": 0.6701,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.18489593267440796,
+      "rewards/margins": 0.051917482167482376,
+      "rewards/rejected": -0.23681339621543884,
+      "step": 9300
+    },
+    {
+      "epoch": 1.602343211578222,
+      "eval_logits/chosen": -2.966949939727783,
+      "eval_logits/rejected": -2.9611239433288574,
+      "eval_logps/chosen": -70.50237274169922,
+      "eval_logps/rejected": -77.72882080078125,
+      "eval_loss": 0.6810948848724365,
+      "eval_rewards/accuracies": 0.6026951670646667,
+      "eval_rewards/chosen": -0.11790473759174347,
+      "eval_rewards/margins": 0.027582230046391487,
+      "eval_rewards/rejected": -0.1454869657754898,
+      "eval_runtime": 383.0627,
+      "eval_samples_per_second": 11.236,
+      "eval_steps_per_second": 1.404,
+      "step": 9300
+    },
+    {
+      "epoch": 1.6040661612680909,
+      "grad_norm": 3.7766687870025635,
+      "learning_rate": 5.7355397142212495e-09,
+      "logits/chosen": -2.8706588745117188,
+      "logits/rejected": -2.841761350631714,
+      "logps/chosen": -71.28263854980469,
+      "logps/rejected": -73.50550842285156,
+      "loss": 0.6678,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.17537018656730652,
+      "rewards/margins": 0.055922459810972214,
+      "rewards/rejected": -0.23129265010356903,
+      "step": 9310
+    },
+    {
+      "epoch": 1.60578911095796,
+      "grad_norm": 3.629901170730591,
+      "learning_rate": 5.687711723983907e-09,
+      "logits/chosen": -2.9317731857299805,
+      "logits/rejected": -2.9119925498962402,
+      "logps/chosen": -69.0033187866211,
+      "logps/rejected": -77.94873809814453,
+      "loss": 0.6689,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.18171173334121704,
+      "rewards/margins": 0.05364919826388359,
+      "rewards/rejected": -0.23536093533039093,
+      "step": 9320
+    },
+    {
+      "epoch": 1.607512060647829,
+      "grad_norm": 4.601132392883301,
+      "learning_rate": 5.640058376256437e-09,
+      "logits/chosen": -2.887226104736328,
+      "logits/rejected": -2.8611021041870117,
+      "logps/chosen": -73.9962158203125,
+      "logps/rejected": -79.05517578125,
+      "loss": 0.6638,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.17418168485164642,
+      "rewards/margins": 0.06576906144618988,
+      "rewards/rejected": -0.2399507462978363,
+      "step": 9330
+    },
+    {
+      "epoch": 1.6092350103376982,
+      "grad_norm": 3.999147653579712,
+      "learning_rate": 5.592580101971764e-09,
+      "logits/chosen": -2.808189868927002,
+      "logits/rejected": -2.7961807250976562,
+      "logps/chosen": -70.0978012084961,
+      "logps/rejected": -75.26704406738281,
+      "loss": 0.6691,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.18288619816303253,
+      "rewards/margins": 0.0538967028260231,
+      "rewards/rejected": -0.23678286373615265,
+      "step": 9340
+    },
+    {
+      "epoch": 1.6109579600275672,
+      "grad_norm": 8.552509307861328,
+      "learning_rate": 5.545277330479558e-09,
+      "logits/chosen": -2.8659329414367676,
+      "logits/rejected": -2.835376739501953,
+      "logps/chosen": -68.70686340332031,
+      "logps/rejected": -74.13843536376953,
+      "loss": 0.6661,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.16156792640686035,
+      "rewards/margins": 0.060629405081272125,
+      "rewards/rejected": -0.22219733893871307,
+      "step": 9350
+    },
+    {
+      "epoch": 1.6126809097174362,
+      "grad_norm": 3.331613302230835,
+      "learning_rate": 5.498150489542428e-09,
+      "logits/chosen": -2.941678285598755,
+      "logits/rejected": -2.903059482574463,
+      "logps/chosen": -72.90933227539062,
+      "logps/rejected": -76.06845092773438,
+      "loss": 0.6599,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.16226966679096222,
+      "rewards/margins": 0.07264933735132217,
+      "rewards/rejected": -0.23491902649402618,
+      "step": 9360
+    },
+    {
+      "epoch": 1.6144038594073054,
+      "grad_norm": 4.387882232666016,
+      "learning_rate": 5.4512000053320264e-09,
+      "logits/chosen": -2.9490227699279785,
+      "logits/rejected": -2.915626049041748,
+      "logps/chosen": -70.8326644897461,
+      "logps/rejected": -73.27078247070312,
+      "loss": 0.6715,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.1699340045452118,
+      "rewards/margins": 0.04876552149653435,
+      "rewards/rejected": -0.21869949996471405,
+      "step": 9370
+    },
+    {
+      "epoch": 1.6161268090971743,
+      "grad_norm": 3.7646243572235107,
+      "learning_rate": 5.4044263024251994e-09,
+      "logits/chosen": -2.952543020248413,
+      "logits/rejected": -2.936002254486084,
+      "logps/chosen": -69.97832489013672,
+      "logps/rejected": -72.88124084472656,
+      "loss": 0.6778,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.15569569170475006,
+      "rewards/margins": 0.03555545583367348,
+      "rewards/rejected": -0.19125112891197205,
+      "step": 9380
+    },
+    {
+      "epoch": 1.6178497587870435,
+      "grad_norm": 3.685889959335327,
+      "learning_rate": 5.3578298038001375e-09,
+      "logits/chosen": -2.720308780670166,
+      "logits/rejected": -2.7059531211853027,
+      "logps/chosen": -70.67179107666016,
+      "logps/rejected": -77.00505065917969,
+      "loss": 0.6723,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.17571790516376495,
+      "rewards/margins": 0.046207744628190994,
+      "rewards/rejected": -0.22192561626434326,
+      "step": 9390
+    },
+    {
+      "epoch": 1.6195727084769125,
+      "grad_norm": 4.016559600830078,
+      "learning_rate": 5.311410930832574e-09,
+      "logits/chosen": -2.8271687030792236,
+      "logits/rejected": -2.8092026710510254,
+      "logps/chosen": -70.81584167480469,
+      "logps/rejected": -76.15065002441406,
+      "loss": 0.6705,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.1674792468547821,
+      "rewards/margins": 0.05131470412015915,
+      "rewards/rejected": -0.21879395842552185,
+      "step": 9400
+    },
+    {
+      "epoch": 1.6195727084769125,
+      "eval_logits/chosen": -2.966122627258301,
+      "eval_logits/rejected": -2.960282564163208,
+      "eval_logps/chosen": -70.532470703125,
+      "eval_logps/rejected": -77.78785705566406,
+      "eval_loss": 0.6809635162353516,
+      "eval_rewards/accuracies": 0.6078066825866699,
+      "eval_rewards/chosen": -0.11820568144321442,
+      "eval_rewards/margins": 0.027871670201420784,
+      "eval_rewards/rejected": -0.14607734978199005,
+      "eval_runtime": 382.7936,
+      "eval_samples_per_second": 11.244,
+      "eval_steps_per_second": 1.405,
+      "step": 9400
+    },
+    {
+      "epoch": 1.6212956581667815,
+      "grad_norm": 4.102982997894287,
+      "learning_rate": 5.265170103291952e-09,
+      "logits/chosen": -2.8423664569854736,
+      "logits/rejected": -2.8262267112731934,
+      "logps/chosen": -70.85324096679688,
+      "logps/rejected": -74.3784408569336,
+      "loss": 0.6696,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.16979512572288513,
+      "rewards/margins": 0.05269570276141167,
+      "rewards/rejected": -0.2224908173084259,
+      "step": 9410
+    },
+    {
+      "epoch": 1.6230186078566504,
+      "grad_norm": 3.972567558288574,
+      "learning_rate": 5.219107739337616e-09,
+      "logits/chosen": -2.8813631534576416,
+      "logits/rejected": -2.862475872039795,
+      "logps/chosen": -72.97261047363281,
+      "logps/rejected": -74.63453674316406,
+      "loss": 0.6729,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.179336816072464,
+      "rewards/margins": 0.045100051909685135,
+      "rewards/rejected": -0.22443684935569763,
+      "step": 9420
+    },
+    {
+      "epoch": 1.6247415575465196,
+      "grad_norm": 3.425323724746704,
+      "learning_rate": 5.173224255515099e-09,
+      "logits/chosen": -2.84765362739563,
+      "logits/rejected": -2.835587978363037,
+      "logps/chosen": -70.63075256347656,
+      "logps/rejected": -72.2029037475586,
+      "loss": 0.6714,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.18198652565479279,
+      "rewards/margins": 0.05070618540048599,
+      "rewards/rejected": -0.232692688703537,
+      "step": 9430
+    },
+    {
+      "epoch": 1.6264645072363888,
+      "grad_norm": 4.106331825256348,
+      "learning_rate": 5.127520066752256e-09,
+      "logits/chosen": -2.8394663333892822,
+      "logits/rejected": -2.8335843086242676,
+      "logps/chosen": -67.78823852539062,
+      "logps/rejected": -73.92683410644531,
+      "loss": 0.679,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.18153499066829681,
+      "rewards/margins": 0.03312157467007637,
+      "rewards/rejected": -0.21465656161308289,
+      "step": 9440
+    },
+    {
+      "epoch": 1.6281874569262578,
+      "grad_norm": 3.853538751602173,
+      "learning_rate": 5.081995586355592e-09,
+      "logits/chosen": -2.982290744781494,
+      "logits/rejected": -2.9742536544799805,
+      "logps/chosen": -73.12635803222656,
+      "logps/rejected": -73.24528503417969,
+      "loss": 0.6816,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.1973741054534912,
+      "rewards/margins": 0.028698790818452835,
+      "rewards/rejected": -0.22607286274433136,
+      "step": 9450
+    },
+    {
+      "epoch": 1.6299104066161267,
+      "grad_norm": 3.968383312225342,
+      "learning_rate": 5.0366512260064885e-09,
+      "logits/chosen": -2.8280904293060303,
+      "logits/rejected": -2.8236067295074463,
+      "logps/chosen": -68.44127655029297,
+      "logps/rejected": -79.30323791503906,
+      "loss": 0.6515,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.16190388798713684,
+      "rewards/margins": 0.09087204933166504,
+      "rewards/rejected": -0.25277596712112427,
+      "step": 9460
+    },
+    {
+      "epoch": 1.6316333563059957,
+      "grad_norm": 3.9391064643859863,
+      "learning_rate": 4.99148739575749e-09,
+      "logits/chosen": -2.6954078674316406,
+      "logits/rejected": -2.6645379066467285,
+      "logps/chosen": -70.33480834960938,
+      "logps/rejected": -72.42179870605469,
+      "loss": 0.675,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.1768968403339386,
+      "rewards/margins": 0.042635273188352585,
+      "rewards/rejected": -0.21953213214874268,
+      "step": 9470
+    },
+    {
+      "epoch": 1.633356305995865,
+      "grad_norm": 3.800852060317993,
+      "learning_rate": 4.94650450402859e-09,
+      "logits/chosen": -2.836413860321045,
+      "logits/rejected": -2.8037123680114746,
+      "logps/chosen": -72.28011322021484,
+      "logps/rejected": -76.6923828125,
+      "loss": 0.6671,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.18287594616413116,
+      "rewards/margins": 0.05748379975557327,
+      "rewards/rejected": -0.24035973846912384,
+      "step": 9480
+    },
+    {
+      "epoch": 1.635079255685734,
+      "grad_norm": 3.556088447570801,
+      "learning_rate": 4.90170295760354e-09,
+      "logits/chosen": -2.829535961151123,
+      "logits/rejected": -2.8108413219451904,
+      "logps/chosen": -71.16184997558594,
+      "logps/rejected": -75.09629821777344,
+      "loss": 0.6671,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.17049381136894226,
+      "rewards/margins": 0.057703666388988495,
+      "rewards/rejected": -0.22819748520851135,
+      "step": 9490
+    },
+    {
+      "epoch": 1.636802205375603,
+      "grad_norm": 3.4479246139526367,
+      "learning_rate": 4.857083161626174e-09,
+      "logits/chosen": -2.865699291229248,
+      "logits/rejected": -2.85237979888916,
+      "logps/chosen": -70.15234375,
+      "logps/rejected": -76.6731185913086,
+      "loss": 0.6699,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.1650591939687729,
+      "rewards/margins": 0.050943780690431595,
+      "rewards/rejected": -0.21600297093391418,
+      "step": 9500
+    },
+    {
+      "epoch": 1.636802205375603,
+      "eval_logits/chosen": -2.9653658866882324,
+      "eval_logits/rejected": -2.959566116333008,
+      "eval_logps/chosen": -70.5706787109375,
+      "eval_logps/rejected": -77.81787109375,
+      "eval_loss": 0.6810044050216675,
+      "eval_rewards/accuracies": 0.607342004776001,
+      "eval_rewards/chosen": -0.11858777701854706,
+      "eval_rewards/margins": 0.02778976783156395,
+      "eval_rewards/rejected": -0.1463775336742401,
+      "eval_runtime": 382.4679,
+      "eval_samples_per_second": 11.253,
+      "eval_steps_per_second": 1.407,
+      "step": 9500
+    },
+    {
+      "epoch": 1.638525155065472,
+      "grad_norm": 4.313417911529541,
+      "learning_rate": 4.812645519596748e-09,
+      "logits/chosen": -2.780649185180664,
+      "logits/rejected": -2.7642836570739746,
+      "logps/chosen": -67.84613037109375,
+      "logps/rejected": -77.24888610839844,
+      "loss": 0.6643,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.16963109374046326,
+      "rewards/margins": 0.06343833357095718,
+      "rewards/rejected": -0.23306944966316223,
+      "step": 9510
+    },
+    {
+      "epoch": 1.640248104755341,
+      "grad_norm": 3.477318525314331,
+      "learning_rate": 4.768390433368272e-09,
+      "logits/chosen": -2.989579439163208,
+      "logits/rejected": -2.9753360748291016,
+      "logps/chosen": -69.60972595214844,
+      "logps/rejected": -80.99372863769531,
+      "loss": 0.667,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.17271924018859863,
+      "rewards/margins": 0.05871378630399704,
+      "rewards/rejected": -0.23143303394317627,
+      "step": 9520
+    },
+    {
+      "epoch": 1.6419710544452102,
+      "grad_norm": 3.8098816871643066,
+      "learning_rate": 4.72431830314291e-09,
+      "logits/chosen": -2.893848180770874,
+      "logits/rejected": -2.871277093887329,
+      "logps/chosen": -70.4249496459961,
+      "logps/rejected": -75.01579284667969,
+      "loss": 0.6634,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.17528972029685974,
+      "rewards/margins": 0.06601680815219879,
+      "rewards/rejected": -0.24130654335021973,
+      "step": 9530
+    },
+    {
+      "epoch": 1.6436940041350794,
+      "grad_norm": 3.7073323726654053,
+      "learning_rate": 4.680429527468311e-09,
+      "logits/chosen": -2.8330495357513428,
+      "logits/rejected": -2.8102149963378906,
+      "logps/chosen": -71.7137451171875,
+      "logps/rejected": -76.85325622558594,
+      "loss": 0.6671,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.17595164477825165,
+      "rewards/margins": 0.05851873755455017,
+      "rewards/rejected": -0.23447036743164062,
+      "step": 9540
+    },
+    {
+      "epoch": 1.6454169538249483,
+      "grad_norm": 3.8477959632873535,
+      "learning_rate": 4.636724503234074e-09,
+      "logits/chosen": -2.876939535140991,
+      "logits/rejected": -2.878736972808838,
+      "logps/chosen": -70.28003692626953,
+      "logps/rejected": -78.5749740600586,
+      "loss": 0.6722,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.170567125082016,
+      "rewards/margins": 0.04722961038351059,
+      "rewards/rejected": -0.21779675781726837,
+      "step": 9550
+    },
+    {
+      "epoch": 1.6471399035148173,
+      "grad_norm": 4.014000415802002,
+      "learning_rate": 4.593203625668077e-09,
+      "logits/chosen": -2.937992811203003,
+      "logits/rejected": -2.9308300018310547,
+      "logps/chosen": -70.22956085205078,
+      "logps/rejected": -75.47201538085938,
+      "loss": 0.6673,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.1660676896572113,
+      "rewards/margins": 0.05808206647634506,
+      "rewards/rejected": -0.22414974868297577,
+      "step": 9560
+    },
+    {
+      "epoch": 1.6488628532046863,
+      "grad_norm": 3.749941825866699,
+      "learning_rate": 4.549867288332987e-09,
+      "logits/chosen": -2.8480618000030518,
+      "logits/rejected": -2.8335087299346924,
+      "logps/chosen": -68.06394958496094,
+      "logps/rejected": -73.62196350097656,
+      "loss": 0.6701,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.1781192570924759,
+      "rewards/margins": 0.05149886757135391,
+      "rewards/rejected": -0.2296181172132492,
+      "step": 9570
+    },
+    {
+      "epoch": 1.6505858028945555,
+      "grad_norm": 3.875450611114502,
+      "learning_rate": 4.506715883122628e-09,
+      "logits/chosen": -2.9222967624664307,
+      "logits/rejected": -2.904524326324463,
+      "logps/chosen": -71.25375366210938,
+      "logps/rejected": -79.62515258789062,
+      "loss": 0.6665,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.16765882074832916,
+      "rewards/margins": 0.05914496257901192,
+      "rewards/rejected": -0.22680377960205078,
+      "step": 9580
+    },
+    {
+      "epoch": 1.6523087525844247,
+      "grad_norm": 3.816660165786743,
+      "learning_rate": 4.463749800258479e-09,
+      "logits/chosen": -3.0026345252990723,
+      "logits/rejected": -2.9840052127838135,
+      "logps/chosen": -73.01414489746094,
+      "logps/rejected": -77.6004638671875,
+      "loss": 0.6747,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.16917413473129272,
+      "rewards/margins": 0.04371969401836395,
+      "rewards/rejected": -0.21289381384849548,
+      "step": 9590
+    },
+    {
+      "epoch": 1.6540317022742936,
+      "grad_norm": 4.095786094665527,
+      "learning_rate": 4.420969428286139e-09,
+      "logits/chosen": -2.826476573944092,
+      "logits/rejected": -2.7938411235809326,
+      "logps/chosen": -71.62020111083984,
+      "logps/rejected": -74.24677276611328,
+      "loss": 0.6699,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.16559767723083496,
+      "rewards/margins": 0.05324871465563774,
+      "rewards/rejected": -0.2188464105129242,
+      "step": 9600
+    },
+    {
+      "epoch": 1.6540317022742936,
+      "eval_logits/chosen": -2.9648587703704834,
+      "eval_logits/rejected": -2.9590563774108887,
+      "eval_logps/chosen": -70.62537384033203,
+      "eval_logps/rejected": -77.88689422607422,
+      "eval_loss": 0.6809461116790771,
+      "eval_rewards/accuracies": 0.6092007160186768,
+      "eval_rewards/chosen": -0.11913486570119858,
+      "eval_rewards/margins": 0.027932781726121902,
+      "eval_rewards/rejected": -0.14706765115261078,
+      "eval_runtime": 383.0052,
+      "eval_samples_per_second": 11.237,
+      "eval_steps_per_second": 1.405,
+      "step": 9600
+    },
+    {
+      "epoch": 1.6557546519641626,
+      "grad_norm": 4.119799613952637,
+      "learning_rate": 4.3783751540718065e-09,
+      "logits/chosen": -2.8534164428710938,
+      "logits/rejected": -2.8336071968078613,
+      "logps/chosen": -71.8329086303711,
+      "logps/rejected": -75.6262435913086,
+      "loss": 0.67,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.15524590015411377,
+      "rewards/margins": 0.051429133862257004,
+      "rewards/rejected": -0.20667505264282227,
+      "step": 9610
+    },
+    {
+      "epoch": 1.6574776016540316,
+      "grad_norm": 3.4090576171875,
+      "learning_rate": 4.335967362798787e-09,
+      "logits/chosen": -2.9712417125701904,
+      "logits/rejected": -2.9634149074554443,
+      "logps/chosen": -71.32904815673828,
+      "logps/rejected": -75.81230163574219,
+      "loss": 0.6772,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.1674264520406723,
+      "rewards/margins": 0.036689918488264084,
+      "rewards/rejected": -0.20411637425422668,
+      "step": 9620
+    },
+    {
+      "epoch": 1.6592005513439008,
+      "grad_norm": 4.009415626525879,
+      "learning_rate": 4.2937464379639824e-09,
+      "logits/chosen": -2.8842902183532715,
+      "logits/rejected": -2.857496500015259,
+      "logps/chosen": -73.74797821044922,
+      "logps/rejected": -76.10332489013672,
+      "loss": 0.6717,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.18289715051651,
+      "rewards/margins": 0.047768689692020416,
+      "rewards/rejected": -0.23066584765911102,
+      "step": 9630
+    },
+    {
+      "epoch": 1.66092350103377,
+      "grad_norm": 3.7246079444885254,
+      "learning_rate": 4.251712761374499e-09,
+      "logits/chosen": -2.9415066242218018,
+      "logits/rejected": -2.9111008644104004,
+      "logps/chosen": -73.59918975830078,
+      "logps/rejected": -73.96611022949219,
+      "loss": 0.6667,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.17602315545082092,
+      "rewards/margins": 0.05835791304707527,
+      "rewards/rejected": -0.2343810796737671,
+      "step": 9640
+    },
+    {
+      "epoch": 1.662646450723639,
+      "grad_norm": 3.695028305053711,
+      "learning_rate": 4.209866713144078e-09,
+      "logits/chosen": -2.8103384971618652,
+      "logits/rejected": -2.7894585132598877,
+      "logps/chosen": -73.53229522705078,
+      "logps/rejected": -75.52281188964844,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.18054452538490295,
+      "rewards/margins": 0.03476036339998245,
+      "rewards/rejected": -0.215304896235466,
+      "step": 9650
+    },
+    {
+      "epoch": 1.664369400413508,
+      "grad_norm": 5.097640037536621,
+      "learning_rate": 4.1682086716897824e-09,
+      "logits/chosen": -2.836087703704834,
+      "logits/rejected": -2.8303141593933105,
+      "logps/chosen": -67.35543823242188,
+      "logps/rejected": -75.42122650146484,
+      "loss": 0.6711,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.16770949959754944,
+      "rewards/margins": 0.0500066876411438,
+      "rewards/rejected": -0.21771618723869324,
+      "step": 9660
+    },
+    {
+      "epoch": 1.6660923501033769,
+      "grad_norm": 3.517241954803467,
+      "learning_rate": 4.1267390137284725e-09,
+      "logits/chosen": -2.9226136207580566,
+      "logits/rejected": -2.8937320709228516,
+      "logps/chosen": -72.1032943725586,
+      "logps/rejected": -76.14041900634766,
+      "loss": 0.6627,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.15986700356006622,
+      "rewards/margins": 0.06810684502124786,
+      "rewards/rejected": -0.2279738485813141,
+      "step": 9670
+    },
+    {
+      "epoch": 1.667815299793246,
+      "grad_norm": 3.6247615814208984,
+      "learning_rate": 4.0854581142734625e-09,
+      "logits/chosen": -2.8844375610351562,
+      "logits/rejected": -2.8690502643585205,
+      "logps/chosen": -71.11817932128906,
+      "logps/rejected": -72.27885437011719,
+      "loss": 0.6787,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.16649232804775238,
+      "rewards/margins": 0.03433884680271149,
+      "rewards/rejected": -0.20083117485046387,
+      "step": 9680
+    },
+    {
+      "epoch": 1.6695382494831152,
+      "grad_norm": 4.157865047454834,
+      "learning_rate": 4.044366346631107e-09,
+      "logits/chosen": -2.7747912406921387,
+      "logits/rejected": -2.7562317848205566,
+      "logps/chosen": -73.37675476074219,
+      "logps/rejected": -76.99559783935547,
+      "loss": 0.6719,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.17548641562461853,
+      "rewards/margins": 0.04835302382707596,
+      "rewards/rejected": -0.2238394320011139,
+      "step": 9690
+    },
+    {
+      "epoch": 1.6712611991729842,
+      "grad_norm": 3.6431965827941895,
+      "learning_rate": 4.003464082397421e-09,
+      "logits/chosen": -2.842743396759033,
+      "logits/rejected": -2.811091899871826,
+      "logps/chosen": -70.03892517089844,
+      "logps/rejected": -75.41410064697266,
+      "loss": 0.6675,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.1733238399028778,
+      "rewards/margins": 0.058085449039936066,
+      "rewards/rejected": -0.23140931129455566,
+      "step": 9700
+    },
+    {
+      "epoch": 1.6712611991729842,
+      "eval_logits/chosen": -2.9642741680145264,
+      "eval_logits/rejected": -2.958444356918335,
+      "eval_logps/chosen": -70.66960144042969,
+      "eval_logps/rejected": -77.94719696044922,
+      "eval_loss": 0.6808727383613586,
+      "eval_rewards/accuracies": 0.6015334725379944,
+      "eval_rewards/chosen": -0.1195770725607872,
+      "eval_rewards/margins": 0.028093617409467697,
+      "eval_rewards/rejected": -0.1476706862449646,
+      "eval_runtime": 382.2085,
+      "eval_samples_per_second": 11.261,
+      "eval_steps_per_second": 1.408,
+      "step": 9700
+    },
+    {
+      "epoch": 1.6729841488628532,
+      "grad_norm": 3.716322660446167,
+      "learning_rate": 3.9627516914547295e-09,
+      "logits/chosen": -2.708674669265747,
+      "logits/rejected": -2.6939029693603516,
+      "logps/chosen": -70.75727844238281,
+      "logps/rejected": -73.68192291259766,
+      "loss": 0.6732,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.1712871938943863,
+      "rewards/margins": 0.04632136970758438,
+      "rewards/rejected": -0.21760857105255127,
+      "step": 9710
+    },
+    {
+      "epoch": 1.6747070985527222,
+      "grad_norm": 4.471319198608398,
+      "learning_rate": 3.922229541968322e-09,
+      "logits/chosen": -2.879821538925171,
+      "logits/rejected": -2.8662455081939697,
+      "logps/chosen": -70.99128723144531,
+      "logps/rejected": -74.05103302001953,
+      "loss": 0.6707,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.17632760107517242,
+      "rewards/margins": 0.05069394037127495,
+      "rewards/rejected": -0.22702153027057648,
+      "step": 9720
+    },
+    {
+      "epoch": 1.6764300482425913,
+      "grad_norm": 4.097490310668945,
+      "learning_rate": 3.8818980003831155e-09,
+      "logits/chosen": -2.8493313789367676,
+      "logits/rejected": -2.835684061050415,
+      "logps/chosen": -70.39976501464844,
+      "logps/rejected": -77.43048095703125,
+      "loss": 0.665,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.16663111746311188,
+      "rewards/margins": 0.06328977644443512,
+      "rewards/rejected": -0.229920893907547,
+      "step": 9730
+    },
+    {
+      "epoch": 1.6781529979324605,
+      "grad_norm": 3.5712475776672363,
+      "learning_rate": 3.841757431420351e-09,
+      "logits/chosen": -2.886239528656006,
+      "logits/rejected": -2.8678431510925293,
+      "logps/chosen": -71.83845520019531,
+      "logps/rejected": -74.11939239501953,
+      "loss": 0.6783,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.1736433058977127,
+      "rewards/margins": 0.034900881350040436,
+      "rewards/rejected": -0.20854417979717255,
+      "step": 9740
+    },
+    {
+      "epoch": 1.6798759476223295,
+      "grad_norm": 4.063652992248535,
+      "learning_rate": 3.8018081980742664e-09,
+      "logits/chosen": -2.8808934688568115,
+      "logits/rejected": -2.839453935623169,
+      "logps/chosen": -72.46504211425781,
+      "logps/rejected": -72.09400939941406,
+      "loss": 0.6688,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.17337141931056976,
+      "rewards/margins": 0.05403054878115654,
+      "rewards/rejected": -0.2274019718170166,
+      "step": 9750
+    },
+    {
+      "epoch": 1.6815988973121985,
+      "grad_norm": 3.716475248336792,
+      "learning_rate": 3.7620506616088815e-09,
+      "logits/chosen": -2.8993422985076904,
+      "logits/rejected": -2.875225782394409,
+      "logps/chosen": -69.80843353271484,
+      "logps/rejected": -73.61235046386719,
+      "loss": 0.6678,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.16625088453292847,
+      "rewards/margins": 0.05705966427922249,
+      "rewards/rejected": -0.22331054508686066,
+      "step": 9760
+    },
+    {
+      "epoch": 1.6833218470020674,
+      "grad_norm": 3.7014029026031494,
+      "learning_rate": 3.7224851815546298e-09,
+      "logits/chosen": -2.8099617958068848,
+      "logits/rejected": -2.7909657955169678,
+      "logps/chosen": -70.91781616210938,
+      "logps/rejected": -75.9351577758789,
+      "loss": 0.6704,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.16809794306755066,
+      "rewards/margins": 0.051731228828430176,
+      "rewards/rejected": -0.21982917189598083,
+      "step": 9770
+    },
+    {
+      "epoch": 1.6850447966919366,
+      "grad_norm": 3.723214864730835,
+      "learning_rate": 3.6831121157052254e-09,
+      "logits/chosen": -2.8869619369506836,
+      "logits/rejected": -2.84010648727417,
+      "logps/chosen": -77.27418518066406,
+      "logps/rejected": -75.48441314697266,
+      "loss": 0.6591,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.16391608119010925,
+      "rewards/margins": 0.07485713064670563,
+      "rewards/rejected": -0.23877322673797607,
+      "step": 9780
+    },
+    {
+      "epoch": 1.6867677463818056,
+      "grad_norm": 3.748608350753784,
+      "learning_rate": 3.64393182011431e-09,
+      "logits/chosen": -2.8572869300842285,
+      "logits/rejected": -2.857247829437256,
+      "logps/chosen": -69.39112854003906,
+      "logps/rejected": -81.21896362304688,
+      "loss": 0.6588,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.15979528427124023,
+      "rewards/margins": 0.07655854523181915,
+      "rewards/rejected": -0.23635384440422058,
+      "step": 9790
+    },
+    {
+      "epoch": 1.6884906960716748,
+      "grad_norm": 3.9616074562072754,
+      "learning_rate": 3.604944649092323e-09,
+      "logits/chosen": -2.8877270221710205,
+      "logits/rejected": -2.8524415493011475,
+      "logps/chosen": -72.82737731933594,
+      "logps/rejected": -76.26792907714844,
+      "loss": 0.6639,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.16025106608867645,
+      "rewards/margins": 0.06542577594518661,
+      "rewards/rejected": -0.22567686438560486,
+      "step": 9800
+    },
+    {
+      "epoch": 1.6884906960716748,
+      "eval_logits/chosen": -2.9643194675445557,
+      "eval_logits/rejected": -2.958489179611206,
+      "eval_logps/chosen": -70.690185546875,
+      "eval_logps/rejected": -77.96759033203125,
+      "eval_loss": 0.6808727383613586,
+      "eval_rewards/accuracies": 0.6082713603973389,
+      "eval_rewards/chosen": -0.11978290975093842,
+      "eval_rewards/margins": 0.028091706335544586,
+      "eval_rewards/rejected": -0.1478746086359024,
+      "eval_runtime": 382.6304,
+      "eval_samples_per_second": 11.248,
+      "eval_steps_per_second": 1.406,
+      "step": 9800
+    },
+    {
+      "epoch": 1.6902136457615438,
+      "grad_norm": 3.7881956100463867,
+      "learning_rate": 3.566150955203251e-09,
+      "logits/chosen": -2.819148302078247,
+      "logits/rejected": -2.785698890686035,
+      "logps/chosen": -72.33556365966797,
+      "logps/rejected": -77.89657592773438,
+      "loss": 0.661,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.16916406154632568,
+      "rewards/margins": 0.07089757919311523,
+      "rewards/rejected": -0.24006164073944092,
+      "step": 9810
+    },
+    {
+      "epoch": 1.6919365954514127,
+      "grad_norm": 3.683380603790283,
+      "learning_rate": 3.52755108926146e-09,
+      "logits/chosen": -2.8804218769073486,
+      "logits/rejected": -2.8556015491485596,
+      "logps/chosen": -70.12922668457031,
+      "logps/rejected": -76.98387145996094,
+      "loss": 0.6614,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1600746363401413,
+      "rewards/margins": 0.07011716067790985,
+      "rewards/rejected": -0.23019179701805115,
+      "step": 9820
+    },
+    {
+      "epoch": 1.693659545141282,
+      "grad_norm": 3.688957929611206,
+      "learning_rate": 3.489145400328511e-09,
+      "logits/chosen": -2.912524461746216,
+      "logits/rejected": -2.907432794570923,
+      "logps/chosen": -75.81372833251953,
+      "logps/rejected": -79.09293365478516,
+      "loss": 0.6768,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.17825815081596375,
+      "rewards/margins": 0.03921232372522354,
+      "rewards/rejected": -0.21747048199176788,
+      "step": 9830
+    },
+    {
+      "epoch": 1.6953824948311509,
+      "grad_norm": 3.685058832168579,
+      "learning_rate": 3.4509342357099904e-09,
+      "logits/chosen": -2.8437867164611816,
+      "logits/rejected": -2.8128018379211426,
+      "logps/chosen": -74.21378326416016,
+      "logps/rejected": -77.3313980102539,
+      "loss": 0.6717,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.18335919082164764,
+      "rewards/margins": 0.04958488419651985,
+      "rewards/rejected": -0.2329440861940384,
+      "step": 9840
+    },
+    {
+      "epoch": 1.69710544452102,
+      "grad_norm": 3.822871685028076,
+      "learning_rate": 3.412917940952423e-09,
+      "logits/chosen": -2.862717390060425,
+      "logits/rejected": -2.851430892944336,
+      "logps/chosen": -70.2558364868164,
+      "logps/rejected": -71.82099914550781,
+      "loss": 0.6733,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.15654197335243225,
+      "rewards/margins": 0.045024700462818146,
+      "rewards/rejected": -0.2015666663646698,
+      "step": 9850
+    },
+    {
+      "epoch": 1.698828394210889,
+      "grad_norm": 4.254611015319824,
+      "learning_rate": 3.375096859840071e-09,
+      "logits/chosen": -2.9296274185180664,
+      "logits/rejected": -2.919512987136841,
+      "logps/chosen": -75.64163208007812,
+      "logps/rejected": -76.4888916015625,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.19315288960933685,
+      "rewards/margins": 0.032892294228076935,
+      "rewards/rejected": -0.226045161485672,
+      "step": 9860
+    },
+    {
+      "epoch": 1.700551343900758,
+      "grad_norm": 3.810955286026001,
+      "learning_rate": 3.337471334391903e-09,
+      "logits/chosen": -2.9020638465881348,
+      "logits/rejected": -2.8757150173187256,
+      "logps/chosen": -69.56945037841797,
+      "logps/rejected": -73.5923843383789,
+      "loss": 0.6663,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.16838295757770538,
+      "rewards/margins": 0.05868646502494812,
+      "rewards/rejected": -0.2270694226026535,
+      "step": 9870
+    },
+    {
+      "epoch": 1.7022742935906272,
+      "grad_norm": 3.6049232482910156,
+      "learning_rate": 3.300041704858425e-09,
+      "logits/chosen": -2.7955679893493652,
+      "logits/rejected": -2.7915472984313965,
+      "logps/chosen": -69.15766906738281,
+      "logps/rejected": -77.57084655761719,
+      "loss": 0.6707,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.17092104256153107,
+      "rewards/margins": 0.05027003213763237,
+      "rewards/rejected": -0.22119107842445374,
+      "step": 9880
+    },
+    {
+      "epoch": 1.7039972432804962,
+      "grad_norm": 3.9352307319641113,
+      "learning_rate": 3.2628083097186675e-09,
+      "logits/chosen": -2.7613677978515625,
+      "logits/rejected": -2.755209445953369,
+      "logps/chosen": -73.49481201171875,
+      "logps/rejected": -77.52919006347656,
+      "loss": 0.6638,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.17077158391475677,
+      "rewards/margins": 0.06572885811328888,
+      "rewards/rejected": -0.23650045692920685,
+      "step": 9890
+    },
+    {
+      "epoch": 1.7057201929703654,
+      "grad_norm": 4.055057048797607,
+      "learning_rate": 3.2257714856770866e-09,
+      "logits/chosen": -2.8598074913024902,
+      "logits/rejected": -2.8309202194213867,
+      "logps/chosen": -67.30292510986328,
+      "logps/rejected": -73.12186431884766,
+      "loss": 0.6578,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.1559639722108841,
+      "rewards/margins": 0.0773005411028862,
+      "rewards/rejected": -0.2332645207643509,
+      "step": 9900
+    },
+    {
+      "epoch": 1.7057201929703654,
+      "eval_logits/chosen": -2.964097261428833,
+      "eval_logits/rejected": -2.9582717418670654,
+      "eval_logps/chosen": -70.71080780029297,
+      "eval_logps/rejected": -77.99824523925781,
+      "eval_loss": 0.6808318495750427,
+      "eval_rewards/accuracies": 0.6043215394020081,
+      "eval_rewards/chosen": -0.11998917162418365,
+      "eval_rewards/margins": 0.0281920675188303,
+      "eval_rewards/rejected": -0.1481812298297882,
+      "eval_runtime": 383.031,
+      "eval_samples_per_second": 11.237,
+      "eval_steps_per_second": 1.405,
+      "step": 9900
+    },
+    {
+      "epoch": 1.7074431426602343,
+      "grad_norm": 3.8326563835144043,
+      "learning_rate": 3.188931567660533e-09,
+      "logits/chosen": -2.918703079223633,
+      "logits/rejected": -2.874882698059082,
+      "logps/chosen": -74.7436752319336,
+      "logps/rejected": -73.4123764038086,
+      "loss": 0.6684,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.17175398766994476,
+      "rewards/margins": 0.055866263806819916,
+      "rewards/rejected": -0.22762027382850647,
+      "step": 9910
+    },
+    {
+      "epoch": 1.7091660923501033,
+      "grad_norm": 3.7598774433135986,
+      "learning_rate": 3.152288888815227e-09,
+      "logits/chosen": -2.935502529144287,
+      "logits/rejected": -2.907062530517578,
+      "logps/chosen": -69.5457992553711,
+      "logps/rejected": -72.77131652832031,
+      "loss": 0.6631,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1685691624879837,
+      "rewards/margins": 0.06584379822015762,
+      "rewards/rejected": -0.23441298305988312,
+      "step": 9920
+    },
+    {
+      "epoch": 1.7108890420399723,
+      "grad_norm": 3.992805004119873,
+      "learning_rate": 3.1158437805037296e-09,
+      "logits/chosen": -2.869058132171631,
+      "logits/rejected": -2.866360902786255,
+      "logps/chosen": -68.98038482666016,
+      "logps/rejected": -77.19906616210938,
+      "loss": 0.6698,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.17073717713356018,
+      "rewards/margins": 0.0526517853140831,
+      "rewards/rejected": -0.22338895499706268,
+      "step": 9930
+    },
+    {
+      "epoch": 1.7126119917298415,
+      "grad_norm": 3.6274828910827637,
+      "learning_rate": 3.0795965723019653e-09,
+      "logits/chosen": -2.9381473064422607,
+      "logits/rejected": -2.93322491645813,
+      "logps/chosen": -71.323974609375,
+      "logps/rejected": -80.96036529541016,
+      "loss": 0.6758,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.16262264549732208,
+      "rewards/margins": 0.03957369551062584,
+      "rewards/rejected": -0.20219635963439941,
+      "step": 9940
+    },
+    {
+      "epoch": 1.7143349414197107,
+      "grad_norm": 4.280511856079102,
+      "learning_rate": 3.043547591996226e-09,
+      "logits/chosen": -2.9030919075012207,
+      "logits/rejected": -2.8677384853363037,
+      "logps/chosen": -71.38412475585938,
+      "logps/rejected": -74.15007019042969,
+      "loss": 0.669,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.1741980016231537,
+      "rewards/margins": 0.054007090628147125,
+      "rewards/rejected": -0.22820508480072021,
+      "step": 9950
+    },
+    {
+      "epoch": 1.7160578911095796,
+      "grad_norm": 3.831204891204834,
+      "learning_rate": 3.0076971655802196e-09,
+      "logits/chosen": -2.9348716735839844,
+      "logits/rejected": -2.918351173400879,
+      "logps/chosen": -72.17022705078125,
+      "logps/rejected": -76.02115631103516,
+      "loss": 0.6831,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.1873314082622528,
+      "rewards/margins": 0.026789620518684387,
+      "rewards/rejected": -0.21412105858325958,
+      "step": 9960
+    },
+    {
+      "epoch": 1.7177808407994486,
+      "grad_norm": 4.162856101989746,
+      "learning_rate": 2.972045617252114e-09,
+      "logits/chosen": -2.8762049674987793,
+      "logits/rejected": -2.8567683696746826,
+      "logps/chosen": -68.46476745605469,
+      "logps/rejected": -74.4446792602539,
+      "loss": 0.6679,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.17104646563529968,
+      "rewards/margins": 0.05570066720247269,
+      "rewards/rejected": -0.22674712538719177,
+      "step": 9970
+    },
+    {
+      "epoch": 1.7195037904893176,
+      "grad_norm": 3.7854368686676025,
+      "learning_rate": 2.9365932694115913e-09,
+      "logits/chosen": -2.825007438659668,
+      "logits/rejected": -2.8123536109924316,
+      "logps/chosen": -70.79936218261719,
+      "logps/rejected": -76.53886413574219,
+      "loss": 0.6693,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.18003466725349426,
+      "rewards/margins": 0.05360063910484314,
+      "rewards/rejected": -0.2336353063583374,
+      "step": 9980
+    },
+    {
+      "epoch": 1.7212267401791868,
+      "grad_norm": 3.9185843467712402,
+      "learning_rate": 2.9013404426569853e-09,
+      "logits/chosen": -2.915039539337158,
+      "logits/rejected": -2.866765022277832,
+      "logps/chosen": -71.15538024902344,
+      "logps/rejected": -71.94966125488281,
+      "loss": 0.6729,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.17422670125961304,
+      "rewards/margins": 0.046107299625873566,
+      "rewards/rejected": -0.2203340083360672,
+      "step": 9990
+    },
+    {
+      "epoch": 1.722949689869056,
+      "grad_norm": 3.8135972023010254,
+      "learning_rate": 2.8662874557823015e-09,
+      "logits/chosen": -2.919663190841675,
+      "logits/rejected": -2.8996212482452393,
+      "logps/chosen": -71.70952606201172,
+      "logps/rejected": -79.14985656738281,
+      "loss": 0.6647,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.15960988402366638,
+      "rewards/margins": 0.06221207231283188,
+      "rewards/rejected": -0.22182197868824005,
+      "step": 10000
+    },
+    {
+      "epoch": 1.722949689869056,
+      "eval_logits/chosen": -2.963688373565674,
+      "eval_logits/rejected": -2.9578399658203125,
+      "eval_logps/chosen": -70.74726867675781,
+      "eval_logps/rejected": -78.02753448486328,
+      "eval_loss": 0.6808710098266602,
+      "eval_rewards/accuracies": 0.604786217212677,
+      "eval_rewards/chosen": -0.12035379558801651,
+      "eval_rewards/margins": 0.028120284900069237,
+      "eval_rewards/rejected": -0.1484740525484085,
+      "eval_runtime": 382.7055,
+      "eval_samples_per_second": 11.246,
+      "eval_steps_per_second": 1.406,
+      "step": 10000
+    },
+    {
+      "epoch": 1.724672639558925,
+      "grad_norm": 4.249887943267822,
+      "learning_rate": 2.8314346257744175e-09,
+      "logits/chosen": -2.8962366580963135,
+      "logits/rejected": -2.875702142715454,
+      "logps/chosen": -68.9428939819336,
+      "logps/rejected": -76.77128601074219,
+      "loss": 0.659,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.16563533246517181,
+      "rewards/margins": 0.0743037685751915,
+      "rewards/rejected": -0.2399391233921051,
+      "step": 10010
+    },
+    {
+      "epoch": 1.7263955892487939,
+      "grad_norm": 3.3708910942077637,
+      "learning_rate": 2.7967822678101468e-09,
+      "logits/chosen": -2.8379855155944824,
+      "logits/rejected": -2.8061249256134033,
+      "logps/chosen": -71.79359436035156,
+      "logps/rejected": -73.92970275878906,
+      "loss": 0.6717,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.18120987713336945,
+      "rewards/margins": 0.0482921302318573,
+      "rewards/rejected": -0.22950199246406555,
+      "step": 10020
+    },
+    {
+      "epoch": 1.7281185389386629,
+      "grad_norm": 4.198774337768555,
+      "learning_rate": 2.7623306952534314e-09,
+      "logits/chosen": -2.880358934402466,
+      "logits/rejected": -2.848755121231079,
+      "logps/chosen": -73.47486877441406,
+      "logps/rejected": -72.7856674194336,
+      "loss": 0.6724,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.16083672642707825,
+      "rewards/margins": 0.04571079835295677,
+      "rewards/rejected": -0.20654752850532532,
+      "step": 10030
+    },
+    {
+      "epoch": 1.729841488628532,
+      "grad_norm": 3.9818735122680664,
+      "learning_rate": 2.7280802196525036e-09,
+      "logits/chosen": -3.009675979614258,
+      "logits/rejected": -2.9948902130126953,
+      "logps/chosen": -71.70025634765625,
+      "logps/rejected": -76.58061981201172,
+      "loss": 0.6699,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.16859248280525208,
+      "rewards/margins": 0.0528547577559948,
+      "rewards/rejected": -0.22144722938537598,
+      "step": 10040
+    },
+    {
+      "epoch": 1.7315644383184012,
+      "grad_norm": 4.2449259757995605,
+      "learning_rate": 2.694031150737036e-09,
+      "logits/chosen": -2.8549702167510986,
+      "logits/rejected": -2.851288318634033,
+      "logps/chosen": -73.53884887695312,
+      "logps/rejected": -77.0943603515625,
+      "loss": 0.689,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.19567081332206726,
+      "rewards/margins": 0.01291059423238039,
+      "rewards/rejected": -0.20858140289783478,
+      "step": 10050
+    },
+    {
+      "epoch": 1.7332873880082702,
+      "grad_norm": 3.6095335483551025,
+      "learning_rate": 2.6601837964153994e-09,
+      "logits/chosen": -2.81109356880188,
+      "logits/rejected": -2.8057665824890137,
+      "logps/chosen": -66.6965560913086,
+      "logps/rejected": -73.97227478027344,
+      "loss": 0.6781,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.17959430813789368,
+      "rewards/margins": 0.03626516833901405,
+      "rewards/rejected": -0.21585945785045624,
+      "step": 10060
+    },
+    {
+      "epoch": 1.7350103376981392,
+      "grad_norm": 5.101108074188232,
+      "learning_rate": 2.6265384627718046e-09,
+      "logits/chosen": -2.7846710681915283,
+      "logits/rejected": -2.774519443511963,
+      "logps/chosen": -69.79252624511719,
+      "logps/rejected": -76.56681823730469,
+      "loss": 0.6629,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.17337627708911896,
+      "rewards/margins": 0.06749327480792999,
+      "rewards/rejected": -0.24086956679821014,
+      "step": 10070
+    },
+    {
+      "epoch": 1.7367332873880081,
+      "grad_norm": 4.413517475128174,
+      "learning_rate": 2.593095454063615e-09,
+      "logits/chosen": -2.9150795936584473,
+      "logits/rejected": -2.8999037742614746,
+      "logps/chosen": -74.34882354736328,
+      "logps/rejected": -76.0543212890625,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.178108811378479,
+      "rewards/margins": 0.031575895845890045,
+      "rewards/rejected": -0.20968468487262726,
+      "step": 10080
+    },
+    {
+      "epoch": 1.7384562370778773,
+      "grad_norm": 3.7781460285186768,
+      "learning_rate": 2.5598550727185142e-09,
+      "logits/chosen": -2.946135997772217,
+      "logits/rejected": -2.926077365875244,
+      "logps/chosen": -70.86009216308594,
+      "logps/rejected": -79.28380584716797,
+      "loss": 0.6701,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.1648683398962021,
+      "rewards/margins": 0.050810687243938446,
+      "rewards/rejected": -0.21567900478839874,
+      "step": 10090
+    },
+    {
+      "epoch": 1.7401791867677465,
+      "grad_norm": 4.160381317138672,
+      "learning_rate": 2.5268176193318473e-09,
+      "logits/chosen": -2.889782428741455,
+      "logits/rejected": -2.8805079460144043,
+      "logps/chosen": -71.83940887451172,
+      "logps/rejected": -77.52182006835938,
+      "loss": 0.6655,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.16853490471839905,
+      "rewards/margins": 0.06162790209054947,
+      "rewards/rejected": -0.23016276955604553,
+      "step": 10100
+    },
+    {
+      "epoch": 1.7401791867677465,
+      "eval_logits/chosen": -2.9637393951416016,
+      "eval_logits/rejected": -2.9579427242279053,
+      "eval_logps/chosen": -70.75068664550781,
+      "eval_logps/rejected": -78.03936767578125,
+      "eval_loss": 0.6808256506919861,
+      "eval_rewards/accuracies": 0.6071096658706665,
+      "eval_rewards/chosen": -0.12038794904947281,
+      "eval_rewards/margins": 0.02820446901023388,
+      "eval_rewards/rejected": -0.14859241247177124,
+      "eval_runtime": 382.5619,
+      "eval_samples_per_second": 11.25,
+      "eval_steps_per_second": 1.406,
+      "step": 10100
+    },
+    {
+      "epoch": 1.7419021364576155,
+      "grad_norm": 3.9755020141601562,
+      "learning_rate": 2.4939833926638397e-09,
+      "logits/chosen": -2.937535047531128,
+      "logits/rejected": -2.9455933570861816,
+      "logps/chosen": -68.93180084228516,
+      "logps/rejected": -77.62749481201172,
+      "loss": 0.673,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.18358638882637024,
+      "rewards/margins": 0.04623475298285484,
+      "rewards/rejected": -0.22982116043567657,
+      "step": 10110
+    },
+    {
+      "epoch": 1.7436250861474845,
+      "grad_norm": 3.9231276512145996,
+      "learning_rate": 2.4613526896369308e-09,
+      "logits/chosen": -2.889739990234375,
+      "logits/rejected": -2.8676419258117676,
+      "logps/chosen": -71.18447875976562,
+      "logps/rejected": -73.41014099121094,
+      "loss": 0.6691,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.17460809648036957,
+      "rewards/margins": 0.05342432111501694,
+      "rewards/rejected": -0.2280324250459671,
+      "step": 10120
+    },
+    {
+      "epoch": 1.7453480358373534,
+      "grad_norm": 4.129141807556152,
+      "learning_rate": 2.428925805333082e-09,
+      "logits/chosen": -2.925279140472412,
+      "logits/rejected": -2.9049384593963623,
+      "logps/chosen": -68.45442962646484,
+      "logps/rejected": -75.52873229980469,
+      "loss": 0.6625,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.1510809063911438,
+      "rewards/margins": 0.06770742684602737,
+      "rewards/rejected": -0.21878834068775177,
+      "step": 10130
+    },
+    {
+      "epoch": 1.7470709855272226,
+      "grad_norm": 4.374783992767334,
+      "learning_rate": 2.396703032991107e-09,
+      "logits/chosen": -2.894760847091675,
+      "logits/rejected": -2.852609872817993,
+      "logps/chosen": -75.83023071289062,
+      "logps/rejected": -76.44752502441406,
+      "loss": 0.6612,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1540442407131195,
+      "rewards/margins": 0.07076205313205719,
+      "rewards/rejected": -0.2248062640428543,
+      "step": 10140
+    },
+    {
+      "epoch": 1.7487939352170918,
+      "grad_norm": 3.6009159088134766,
+      "learning_rate": 2.364684664004016e-09,
+      "logits/chosen": -2.8377366065979004,
+      "logits/rejected": -2.81890869140625,
+      "logps/chosen": -72.79371643066406,
+      "logps/rejected": -77.93059539794922,
+      "loss": 0.6706,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.1742021143436432,
+      "rewards/margins": 0.05106892064213753,
+      "rewards/rejected": -0.22527103126049042,
+      "step": 10150
+    },
+    {
+      "epoch": 1.7505168849069608,
+      "grad_norm": 3.669557571411133,
+      "learning_rate": 2.3328709879163826e-09,
+      "logits/chosen": -2.8641180992126465,
+      "logits/rejected": -2.8491604328155518,
+      "logps/chosen": -69.2738037109375,
+      "logps/rejected": -76.1553955078125,
+      "loss": 0.663,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.17201951146125793,
+      "rewards/margins": 0.06748448312282562,
+      "rewards/rejected": -0.23950400948524475,
+      "step": 10160
+    },
+    {
+      "epoch": 1.7522398345968297,
+      "grad_norm": 3.6047472953796387,
+      "learning_rate": 2.301262292421732e-09,
+      "logits/chosen": -2.86650013923645,
+      "logits/rejected": -2.856301784515381,
+      "logps/chosen": -71.25776672363281,
+      "logps/rejected": -75.60033416748047,
+      "loss": 0.6718,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.16347889602184296,
+      "rewards/margins": 0.048476915806531906,
+      "rewards/rejected": -0.21195581555366516,
+      "step": 10170
+    },
+    {
+      "epoch": 1.7539627842866987,
+      "grad_norm": 3.8775806427001953,
+      "learning_rate": 2.269858863359936e-09,
+      "logits/chosen": -2.805027484893799,
+      "logits/rejected": -2.7773194313049316,
+      "logps/chosen": -69.18348693847656,
+      "logps/rejected": -76.39485168457031,
+      "loss": 0.6643,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.16741517186164856,
+      "rewards/margins": 0.06343530118465424,
+      "rewards/rejected": -0.2308504581451416,
+      "step": 10180
+    },
+    {
+      "epoch": 1.755685733976568,
+      "grad_norm": 4.161981105804443,
+      "learning_rate": 2.2386609847146077e-09,
+      "logits/chosen": -2.844789505004883,
+      "logits/rejected": -2.8204150199890137,
+      "logps/chosen": -67.22627258300781,
+      "logps/rejected": -73.14070129394531,
+      "loss": 0.6688,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.1626812368631363,
+      "rewards/margins": 0.05465472862124443,
+      "rewards/rejected": -0.21733596920967102,
+      "step": 10190
+    },
+    {
+      "epoch": 1.757408683666437,
+      "grad_norm": 4.200125694274902,
+      "learning_rate": 2.207668938610582e-09,
+      "logits/chosen": -2.9119696617126465,
+      "logits/rejected": -2.892543315887451,
+      "logps/chosen": -70.0393295288086,
+      "logps/rejected": -74.30731201171875,
+      "loss": 0.6671,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.17618602514266968,
+      "rewards/margins": 0.0576285794377327,
+      "rewards/rejected": -0.23381459712982178,
+      "step": 10200
+    },
+    {
+      "epoch": 1.757408683666437,
+      "eval_logits/chosen": -2.9632084369659424,
+      "eval_logits/rejected": -2.957366704940796,
+      "eval_logps/chosen": -70.77372741699219,
+      "eval_logps/rejected": -78.06083679199219,
+      "eval_loss": 0.6808416843414307,
+      "eval_rewards/accuracies": 0.6059479713439941,
+      "eval_rewards/chosen": -0.12061834335327148,
+      "eval_rewards/margins": 0.0281887948513031,
+      "eval_rewards/rejected": -0.14880715310573578,
+      "eval_runtime": 382.8502,
+      "eval_samples_per_second": 11.242,
+      "eval_steps_per_second": 1.405,
+      "step": 10200
+    },
+    {
+      "epoch": 1.759131633356306,
+      "grad_norm": 4.217611312866211,
+      "learning_rate": 2.176883005311303e-09,
+      "logits/chosen": -2.909919261932373,
+      "logits/rejected": -2.905447483062744,
+      "logps/chosen": -72.11750030517578,
+      "logps/rejected": -81.77589416503906,
+      "loss": 0.6641,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.18104566633701324,
+      "rewards/margins": 0.06545436382293701,
+      "rewards/rejected": -0.24650004506111145,
+      "step": 10210
+    },
+    {
+      "epoch": 1.760854583046175,
+      "grad_norm": 3.4456098079681396,
+      "learning_rate": 2.1463034632163533e-09,
+      "logits/chosen": -2.8593690395355225,
+      "logits/rejected": -2.854443311691284,
+      "logps/chosen": -65.67058563232422,
+      "logps/rejected": -72.94866180419922,
+      "loss": 0.6726,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.17285093665122986,
+      "rewards/margins": 0.04540492966771126,
+      "rewards/rejected": -0.218255877494812,
+      "step": 10220
+    },
+    {
+      "epoch": 1.762577532736044,
+      "grad_norm": 3.368154287338257,
+      "learning_rate": 2.1159305888588664e-09,
+      "logits/chosen": -2.7855865955352783,
+      "logits/rejected": -2.7559919357299805,
+      "logps/chosen": -70.97809600830078,
+      "logps/rejected": -73.81156158447266,
+      "loss": 0.674,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.1753716915845871,
+      "rewards/margins": 0.04415439814329147,
+      "rewards/rejected": -0.21952608227729797,
+      "step": 10230
+    },
+    {
+      "epoch": 1.7643004824259132,
+      "grad_norm": 3.614197015762329,
+      "learning_rate": 2.085764656903105e-09,
+      "logits/chosen": -2.837266445159912,
+      "logits/rejected": -2.815450429916382,
+      "logps/chosen": -69.06104278564453,
+      "logps/rejected": -76.53809356689453,
+      "loss": 0.6638,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.17604005336761475,
+      "rewards/margins": 0.06568211317062378,
+      "rewards/rejected": -0.24172218143939972,
+      "step": 10240
+    },
+    {
+      "epoch": 1.7660234321157822,
+      "grad_norm": 3.6950478553771973,
+      "learning_rate": 2.055805940141897e-09,
+      "logits/chosen": -2.9089982509613037,
+      "logits/rejected": -2.868102788925171,
+      "logps/chosen": -73.52598571777344,
+      "logps/rejected": -73.79502868652344,
+      "loss": 0.6596,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.16750463843345642,
+      "rewards/margins": 0.07365141808986664,
+      "rewards/rejected": -0.24115605652332306,
+      "step": 10250
+    },
+    {
+      "epoch": 1.7677463818056514,
+      "grad_norm": 3.8028886318206787,
+      "learning_rate": 2.026054709494235e-09,
+      "logits/chosen": -2.8465054035186768,
+      "logits/rejected": -2.850390911102295,
+      "logps/chosen": -70.68544006347656,
+      "logps/rejected": -78.37113952636719,
+      "loss": 0.6749,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.17420677840709686,
+      "rewards/margins": 0.04078082740306854,
+      "rewards/rejected": -0.2149876058101654,
+      "step": 10260
+    },
+    {
+      "epoch": 1.7694693314955203,
+      "grad_norm": 5.158822536468506,
+      "learning_rate": 1.9965112340027874e-09,
+      "logits/chosen": -2.84818959236145,
+      "logits/rejected": -2.8380677700042725,
+      "logps/chosen": -72.91519165039062,
+      "logps/rejected": -79.8048324584961,
+      "loss": 0.663,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.17484435439109802,
+      "rewards/margins": 0.066738560795784,
+      "rewards/rejected": -0.24158291518688202,
+      "step": 10270
+    },
+    {
+      "epoch": 1.7711922811853893,
+      "grad_norm": 4.081360816955566,
+      "learning_rate": 1.9671757808314675e-09,
+      "logits/chosen": -2.8360214233398438,
+      "logits/rejected": -2.8178062438964844,
+      "logps/chosen": -72.91610717773438,
+      "logps/rejected": -75.32176208496094,
+      "loss": 0.6768,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.1858697533607483,
+      "rewards/margins": 0.0369776114821434,
+      "rewards/rejected": -0.2228473722934723,
+      "step": 10280
+    },
+    {
+      "epoch": 1.7729152308752585,
+      "grad_norm": 3.681887149810791,
+      "learning_rate": 1.9380486152630547e-09,
+      "logits/chosen": -2.7909083366394043,
+      "logits/rejected": -2.786146879196167,
+      "logps/chosen": -67.0426254272461,
+      "logps/rejected": -78.82637023925781,
+      "loss": 0.6579,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.15606342256069183,
+      "rewards/margins": 0.07736193388700485,
+      "rewards/rejected": -0.2334253489971161,
+      "step": 10290
+    },
+    {
+      "epoch": 1.7746381805651275,
+      "grad_norm": 3.9091060161590576,
+      "learning_rate": 1.909130000696732e-09,
+      "logits/chosen": -2.8685708045959473,
+      "logits/rejected": -2.85327410697937,
+      "logps/chosen": -72.94339752197266,
+      "logps/rejected": -76.65965270996094,
+      "loss": 0.6774,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.18295586109161377,
+      "rewards/margins": 0.038772594183683395,
+      "rewards/rejected": -0.22172844409942627,
+      "step": 10300
+    },
+    {
+      "epoch": 1.7746381805651275,
+      "eval_logits/chosen": -2.962756395339966,
+      "eval_logits/rejected": -2.956923723220825,
+      "eval_logps/chosen": -70.78290557861328,
+      "eval_logps/rejected": -78.08394622802734,
+      "eval_loss": 0.6807774901390076,
+      "eval_rewards/accuracies": 0.6054832935333252,
+      "eval_rewards/chosen": -0.12071008235216141,
+      "eval_rewards/margins": 0.028328191488981247,
+      "eval_rewards/rejected": -0.14903828501701355,
+      "eval_runtime": 382.6937,
+      "eval_samples_per_second": 11.247,
+      "eval_steps_per_second": 1.406,
+      "step": 10300
+    },
+    {
+      "epoch": 1.7763611302549966,
+      "grad_norm": 3.669783592224121,
+      "learning_rate": 1.880420198645774e-09,
+      "logits/chosen": -2.791095733642578,
+      "logits/rejected": -2.766355037689209,
+      "logps/chosen": -74.05281829833984,
+      "logps/rejected": -75.95161437988281,
+      "loss": 0.6625,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.17790111899375916,
+      "rewards/margins": 0.06707983464002609,
+      "rewards/rejected": -0.24498097598552704,
+      "step": 10310
+    },
+    {
+      "epoch": 1.7780840799448656,
+      "grad_norm": 3.6750380992889404,
+      "learning_rate": 1.8519194687351191e-09,
+      "logits/chosen": -2.866307258605957,
+      "logits/rejected": -2.8344671726226807,
+      "logps/chosen": -73.96429443359375,
+      "logps/rejected": -76.40363311767578,
+      "loss": 0.6679,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.17202650010585785,
+      "rewards/margins": 0.05679994076490402,
+      "rewards/rejected": -0.22882644832134247,
+      "step": 10320
+    },
+    {
+      "epoch": 1.7798070296347346,
+      "grad_norm": 3.4580092430114746,
+      "learning_rate": 1.8236280686990653e-09,
+      "logits/chosen": -2.88166880607605,
+      "logits/rejected": -2.864823579788208,
+      "logps/chosen": -70.29212951660156,
+      "logps/rejected": -77.10099029541016,
+      "loss": 0.6734,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.18147775530815125,
+      "rewards/margins": 0.04437021166086197,
+      "rewards/rejected": -0.22584795951843262,
+      "step": 10330
+    },
+    {
+      "epoch": 1.7815299793246038,
+      "grad_norm": 4.283066749572754,
+      "learning_rate": 1.7955462543789268e-09,
+      "logits/chosen": -2.889761447906494,
+      "logits/rejected": -2.8688511848449707,
+      "logps/chosen": -72.72980499267578,
+      "logps/rejected": -77.96984100341797,
+      "loss": 0.6625,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.16642308235168457,
+      "rewards/margins": 0.0675925612449646,
+      "rewards/rejected": -0.23401565849781036,
+      "step": 10340
+    },
+    {
+      "epoch": 1.7832529290144727,
+      "grad_norm": 3.428448438644409,
+      "learning_rate": 1.7676742797207045e-09,
+      "logits/chosen": -2.979663372039795,
+      "logits/rejected": -2.9547200202941895,
+      "logps/chosen": -69.99241638183594,
+      "logps/rejected": -73.25730895996094,
+      "loss": 0.6646,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.15910452604293823,
+      "rewards/margins": 0.06292103230953217,
+      "rewards/rejected": -0.2220255434513092,
+      "step": 10350
+    },
+    {
+      "epoch": 1.784975878704342,
+      "grad_norm": 3.8993866443634033,
+      "learning_rate": 1.7400123967728192e-09,
+      "logits/chosen": -2.826450824737549,
+      "logits/rejected": -2.7919113636016846,
+      "logps/chosen": -72.7823486328125,
+      "logps/rejected": -74.42078399658203,
+      "loss": 0.6718,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.1683550924062729,
+      "rewards/margins": 0.04780002683401108,
+      "rewards/rejected": -0.21615514159202576,
+      "step": 10360
+    },
+    {
+      "epoch": 1.786698828394211,
+      "grad_norm": 3.6110901832580566,
+      "learning_rate": 1.7125608556838034e-09,
+      "logits/chosen": -2.7002077102661133,
+      "logits/rejected": -2.6696345806121826,
+      "logps/chosen": -68.55941772460938,
+      "logps/rejected": -76.7696762084961,
+      "loss": 0.6596,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.16582003235816956,
+      "rewards/margins": 0.07307492941617966,
+      "rewards/rejected": -0.238894984126091,
+      "step": 10370
+    },
+    {
+      "epoch": 1.7884217780840799,
+      "grad_norm": 3.697478771209717,
+      "learning_rate": 1.6853199047000583e-09,
+      "logits/chosen": -2.8547871112823486,
+      "logits/rejected": -2.843250274658203,
+      "logps/chosen": -71.00952911376953,
+      "logps/rejected": -73.87296295166016,
+      "loss": 0.6838,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.19130629301071167,
+      "rewards/margins": 0.02437610551714897,
+      "rewards/rejected": -0.21568241715431213,
+      "step": 10380
+    },
+    {
+      "epoch": 1.7901447277739488,
+      "grad_norm": 3.8814244270324707,
+      "learning_rate": 1.6582897901636028e-09,
+      "logits/chosen": -2.8919217586517334,
+      "logits/rejected": -2.8629393577575684,
+      "logps/chosen": -72.35261535644531,
+      "logps/rejected": -76.77022552490234,
+      "loss": 0.6657,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.16543419659137726,
+      "rewards/margins": 0.061357736587524414,
+      "rewards/rejected": -0.22679194808006287,
+      "step": 10390
+    },
+    {
+      "epoch": 1.791867677463818,
+      "grad_norm": 3.966329336166382,
+      "learning_rate": 1.6314707565098396e-09,
+      "logits/chosen": -2.8499722480773926,
+      "logits/rejected": -2.8321492671966553,
+      "logps/chosen": -69.59270477294922,
+      "logps/rejected": -79.35921478271484,
+      "loss": 0.6629,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.1745244860649109,
+      "rewards/margins": 0.06597074121236801,
+      "rewards/rejected": -0.2404952496290207,
+      "step": 10400
+    },
+    {
+      "epoch": 1.791867677463818,
+      "eval_logits/chosen": -2.9626080989837646,
+      "eval_logits/rejected": -2.956780433654785,
+      "eval_logps/chosen": -70.79251861572266,
+      "eval_logps/rejected": -78.10979461669922,
+      "eval_loss": 0.6806948184967041,
+      "eval_rewards/accuracies": 0.6075743436813354,
+      "eval_rewards/chosen": -0.12080621719360352,
+      "eval_rewards/margins": 0.02849056012928486,
+      "eval_rewards/rejected": -0.14929679036140442,
+      "eval_runtime": 382.5925,
+      "eval_samples_per_second": 11.25,
+      "eval_steps_per_second": 1.406,
+      "step": 10400
+    },
+    {
+      "epoch": 1.7935906271536872,
+      "grad_norm": 3.5720443725585938,
+      "learning_rate": 1.6048630462653618e-09,
+      "logits/chosen": -2.850275993347168,
+      "logits/rejected": -2.8123936653137207,
+      "logps/chosen": -73.47718811035156,
+      "logps/rejected": -75.9654541015625,
+      "loss": 0.6661,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.16959655284881592,
+      "rewards/margins": 0.05929601937532425,
+      "rewards/rejected": -0.22889256477355957,
+      "step": 10410
+    },
+    {
+      "epoch": 1.7953135768435562,
+      "grad_norm": 3.665773391723633,
+      "learning_rate": 1.5784669000457328e-09,
+      "logits/chosen": -2.874826192855835,
+      "logits/rejected": -2.8476502895355225,
+      "logps/chosen": -71.03950500488281,
+      "logps/rejected": -76.36225891113281,
+      "loss": 0.6693,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.1691894382238388,
+      "rewards/margins": 0.053178779780864716,
+      "rewards/rejected": -0.2223682403564453,
+      "step": 10420
+    },
+    {
+      "epoch": 1.7970365265334252,
+      "grad_norm": 3.998572826385498,
+      "learning_rate": 1.5522825565533443e-09,
+      "logits/chosen": -2.9320762157440186,
+      "logits/rejected": -2.9155282974243164,
+      "logps/chosen": -71.90483856201172,
+      "logps/rejected": -74.3819580078125,
+      "loss": 0.6743,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.17034272849559784,
+      "rewards/margins": 0.04292534291744232,
+      "rewards/rejected": -0.21326807141304016,
+      "step": 10430
+    },
+    {
+      "epoch": 1.7987594762232941,
+      "grad_norm": 3.644744873046875,
+      "learning_rate": 1.5263102525752219e-09,
+      "logits/chosen": -2.9383254051208496,
+      "logits/rejected": -2.9280219078063965,
+      "logps/chosen": -75.635498046875,
+      "logps/rejected": -78.91709899902344,
+      "loss": 0.6765,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.17765475809574127,
+      "rewards/margins": 0.03858640417456627,
+      "rewards/rejected": -0.21624115109443665,
+      "step": 10440
+    },
+    {
+      "epoch": 1.8004824259131633,
+      "grad_norm": 3.378493547439575,
+      "learning_rate": 1.500550222980923e-09,
+      "logits/chosen": -2.890306234359741,
+      "logits/rejected": -2.886087656021118,
+      "logps/chosen": -70.95532989501953,
+      "logps/rejected": -81.1356430053711,
+      "loss": 0.6649,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.17333310842514038,
+      "rewards/margins": 0.061642199754714966,
+      "rewards/rejected": -0.23497530817985535,
+      "step": 10450
+    },
+    {
+      "epoch": 1.8022053756030325,
+      "grad_norm": 3.955036163330078,
+      "learning_rate": 1.4750027007203653e-09,
+      "logits/chosen": -2.8868396282196045,
+      "logits/rejected": -2.860283613204956,
+      "logps/chosen": -71.43733978271484,
+      "logps/rejected": -75.09507751464844,
+      "loss": 0.67,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.16862361133098602,
+      "rewards/margins": 0.05204921215772629,
+      "rewards/rejected": -0.2206728458404541,
+      "step": 10460
+    },
+    {
+      "epoch": 1.8039283252929015,
+      "grad_norm": 3.876396417617798,
+      "learning_rate": 1.4496679168217645e-09,
+      "logits/chosen": -2.7342419624328613,
+      "logits/rejected": -2.711254596710205,
+      "logps/chosen": -68.5736312866211,
+      "logps/rejected": -73.13326263427734,
+      "loss": 0.6675,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.166672021150589,
+      "rewards/margins": 0.056859634816646576,
+      "rewards/rejected": -0.22353163361549377,
+      "step": 10470
+    },
+    {
+      "epoch": 1.8056512749827704,
+      "grad_norm": 4.06005859375,
+      "learning_rate": 1.424546100389523e-09,
+      "logits/chosen": -2.887176752090454,
+      "logits/rejected": -2.8506431579589844,
+      "logps/chosen": -72.84221649169922,
+      "logps/rejected": -75.02110290527344,
+      "loss": 0.6675,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.16871331632137299,
+      "rewards/margins": 0.05747460573911667,
+      "rewards/rejected": -0.22618794441223145,
+      "step": 10480
+    },
+    {
+      "epoch": 1.8073742246726394,
+      "grad_norm": 3.682004928588867,
+      "learning_rate": 1.3996374786021641e-09,
+      "logits/chosen": -2.8760416507720947,
+      "logits/rejected": -2.845892906188965,
+      "logps/chosen": -69.3520736694336,
+      "logps/rejected": -74.26506805419922,
+      "loss": 0.6653,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.15541638433933258,
+      "rewards/margins": 0.0617084726691246,
+      "rewards/rejected": -0.2171248495578766,
+      "step": 10490
+    },
+    {
+      "epoch": 1.8090971743625086,
+      "grad_norm": 3.8918299674987793,
+      "learning_rate": 1.3749422767102697e-09,
+      "logits/chosen": -2.870562791824341,
+      "logits/rejected": -2.857217311859131,
+      "logps/chosen": -69.58777618408203,
+      "logps/rejected": -78.36985778808594,
+      "loss": 0.6648,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.16888299584388733,
+      "rewards/margins": 0.06370017677545547,
+      "rewards/rejected": -0.2325831949710846,
+      "step": 10500
+    },
+    {
+      "epoch": 1.8090971743625086,
+      "eval_logits/chosen": -2.962520122528076,
+      "eval_logits/rejected": -2.956692934036255,
+      "eval_logps/chosen": -70.82084655761719,
+      "eval_logps/rejected": -78.12088775634766,
+      "eval_loss": 0.6807798147201538,
+      "eval_rewards/accuracies": 0.6092007160186768,
+      "eval_rewards/chosen": -0.12108955532312393,
+      "eval_rewards/margins": 0.028318092226982117,
+      "eval_rewards/rejected": -0.14940764009952545,
+      "eval_runtime": 382.5547,
+      "eval_samples_per_second": 11.251,
+      "eval_steps_per_second": 1.406,
+      "step": 10500
+    },
+    {
+      "epoch": 1.8108201240523778,
+      "grad_norm": 3.951345682144165,
+      "learning_rate": 1.3504607180344462e-09,
+      "logits/chosen": -2.8572044372558594,
+      "logits/rejected": -2.8333327770233154,
+      "logps/chosen": -72.73995971679688,
+      "logps/rejected": -76.60794830322266,
+      "loss": 0.6645,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.17151859402656555,
+      "rewards/margins": 0.06281152367591858,
+      "rewards/rejected": -0.23433010280132294,
+      "step": 10510
+    },
+    {
+      "epoch": 1.8125430737422468,
+      "grad_norm": 3.574965715408325,
+      "learning_rate": 1.3261930239633263e-09,
+      "logits/chosen": -2.924222707748413,
+      "logits/rejected": -2.927717685699463,
+      "logps/chosen": -67.39115905761719,
+      "logps/rejected": -78.22112274169922,
+      "loss": 0.6622,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.16905637085437775,
+      "rewards/margins": 0.06775665283203125,
+      "rewards/rejected": -0.236813023686409,
+      "step": 10520
+    },
+    {
+      "epoch": 1.8142660234321157,
+      "grad_norm": 3.7543129920959473,
+      "learning_rate": 1.3021394139515196e-09,
+      "logits/chosen": -2.833270311355591,
+      "logits/rejected": -2.8111801147460938,
+      "logps/chosen": -70.82547760009766,
+      "logps/rejected": -74.15909576416016,
+      "loss": 0.673,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.17484574019908905,
+      "rewards/margins": 0.04549097269773483,
+      "rewards/rejected": -0.22033675014972687,
+      "step": 10530
+    },
+    {
+      "epoch": 1.8159889731219847,
+      "grad_norm": 3.745802402496338,
+      "learning_rate": 1.2783001055176905e-09,
+      "logits/chosen": -2.812009334564209,
+      "logits/rejected": -2.786550998687744,
+      "logps/chosen": -69.57952880859375,
+      "logps/rejected": -72.56072998046875,
+      "loss": 0.6669,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.17248830199241638,
+      "rewards/margins": 0.059173740446567535,
+      "rewards/rejected": -0.23166203498840332,
+      "step": 10540
+    },
+    {
+      "epoch": 1.817711922811854,
+      "grad_norm": 4.233473300933838,
+      "learning_rate": 1.2546753142425314e-09,
+      "logits/chosen": -2.970397710800171,
+      "logits/rejected": -2.9690561294555664,
+      "logps/chosen": -71.99134063720703,
+      "logps/rejected": -78.69660949707031,
+      "loss": 0.6729,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.1824040412902832,
+      "rewards/margins": 0.04674986004829407,
+      "rewards/rejected": -0.22915391623973846,
+      "step": 10550
+    },
+    {
+      "epoch": 1.819434872501723,
+      "grad_norm": 3.990586042404175,
+      "learning_rate": 1.23126525376685e-09,
+      "logits/chosen": -2.8297266960144043,
+      "logits/rejected": -2.810699224472046,
+      "logps/chosen": -72.87188720703125,
+      "logps/rejected": -77.64937591552734,
+      "loss": 0.6675,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.17239682376384735,
+      "rewards/margins": 0.057883985340595245,
+      "rewards/rejected": -0.230280801653862,
+      "step": 10560
+    },
+    {
+      "epoch": 1.821157822191592,
+      "grad_norm": 3.458730697631836,
+      "learning_rate": 1.2080701357896266e-09,
+      "logits/chosen": -2.908543825149536,
+      "logits/rejected": -2.89131760597229,
+      "logps/chosen": -72.95198059082031,
+      "logps/rejected": -80.17982482910156,
+      "loss": 0.664,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.17005160450935364,
+      "rewards/margins": 0.06410513073205948,
+      "rewards/rejected": -0.2341567575931549,
+      "step": 10570
+    },
+    {
+      "epoch": 1.822880771881461,
+      "grad_norm": 3.599823474884033,
+      "learning_rate": 1.185090170066097e-09,
+      "logits/chosen": -2.909520387649536,
+      "logits/rejected": -2.8927040100097656,
+      "logps/chosen": -70.51603698730469,
+      "logps/rejected": -75.52699279785156,
+      "loss": 0.6681,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.17154043912887573,
+      "rewards/margins": 0.05559182167053223,
+      "rewards/rejected": -0.22713224589824677,
+      "step": 10580
+    },
+    {
+      "epoch": 1.82460372157133,
+      "grad_norm": 3.347785472869873,
+      "learning_rate": 1.1623255644058637e-09,
+      "logits/chosen": -2.850966691970825,
+      "logits/rejected": -2.810495138168335,
+      "logps/chosen": -73.90401458740234,
+      "logps/rejected": -74.43180084228516,
+      "loss": 0.6639,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.17999489605426788,
+      "rewards/margins": 0.06497448682785034,
+      "rewards/rejected": -0.24496936798095703,
+      "step": 10590
+    },
+    {
+      "epoch": 1.8263266712611992,
+      "grad_norm": 3.513667345046997,
+      "learning_rate": 1.1397765246710072e-09,
+      "logits/chosen": -2.9486608505249023,
+      "logits/rejected": -2.938467502593994,
+      "logps/chosen": -69.7696762084961,
+      "logps/rejected": -75.84690856933594,
+      "loss": 0.6745,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.1688709855079651,
+      "rewards/margins": 0.042718805372714996,
+      "rewards/rejected": -0.21158981323242188,
+      "step": 10600
+    },
+    {
+      "epoch": 1.8263266712611992,
+      "eval_logits/chosen": -2.96266770362854,
+      "eval_logits/rejected": -2.956784725189209,
+      "eval_logps/chosen": -70.82789611816406,
+      "eval_logps/rejected": -78.13334655761719,
+      "eval_loss": 0.6807528138160706,
+      "eval_rewards/accuracies": 0.6082713603973389,
+      "eval_rewards/chosen": -0.12116001546382904,
+      "eval_rewards/margins": 0.028372230008244514,
+      "eval_rewards/rejected": -0.1495322436094284,
+      "eval_runtime": 382.6432,
+      "eval_samples_per_second": 11.248,
+      "eval_steps_per_second": 1.406,
+      "step": 10600
+    },
+    {
+      "epoch": 1.8280496209510684,
+      "grad_norm": 4.013094425201416,
+      "learning_rate": 1.1174432547742307e-09,
+      "logits/chosen": -2.844428062438965,
+      "logits/rejected": -2.8343491554260254,
+      "logps/chosen": -68.98588562011719,
+      "logps/rejected": -75.021728515625,
+      "loss": 0.6695,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1792183220386505,
+      "rewards/margins": 0.05233689397573471,
+      "rewards/rejected": -0.23155522346496582,
+      "step": 10610
+    },
+    {
+      "epoch": 1.8297725706409373,
+      "grad_norm": 3.9664597511291504,
+      "learning_rate": 1.095325956677015e-09,
+      "logits/chosen": -2.7978720664978027,
+      "logits/rejected": -2.7752254009246826,
+      "logps/chosen": -71.32487487792969,
+      "logps/rejected": -74.47241973876953,
+      "loss": 0.6729,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.1752103865146637,
+      "rewards/margins": 0.04622713476419449,
+      "rewards/rejected": -0.22143752872943878,
+      "step": 10620
+    },
+    {
+      "epoch": 1.8314955203308063,
+      "grad_norm": 3.80210018157959,
+      "learning_rate": 1.0734248303877812e-09,
+      "logits/chosen": -2.8827273845672607,
+      "logits/rejected": -2.8644754886627197,
+      "logps/chosen": -71.40470886230469,
+      "logps/rejected": -77.18016052246094,
+      "loss": 0.6674,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.17146174609661102,
+      "rewards/margins": 0.05864910036325455,
+      "rewards/rejected": -0.23011083900928497,
+      "step": 10630
+    },
+    {
+      "epoch": 1.8332184700206753,
+      "grad_norm": 4.384250164031982,
+      "learning_rate": 1.051740073960114e-09,
+      "logits/chosen": -2.847252607345581,
+      "logits/rejected": -2.8244431018829346,
+      "logps/chosen": -72.31382751464844,
+      "logps/rejected": -75.85406494140625,
+      "loss": 0.6665,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.17247872054576874,
+      "rewards/margins": 0.06027457118034363,
+      "rewards/rejected": -0.23275330662727356,
+      "step": 10640
+    },
+    {
+      "epoch": 1.8349414197105445,
+      "grad_norm": 3.846719980239868,
+      "learning_rate": 1.0302718834909213e-09,
+      "logits/chosen": -2.9262871742248535,
+      "logits/rejected": -2.904855489730835,
+      "logps/chosen": -70.89274597167969,
+      "logps/rejected": -78.53726959228516,
+      "loss": 0.6592,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.17271259427070618,
+      "rewards/margins": 0.07448399811983109,
+      "rewards/rejected": -0.24719662964344025,
+      "step": 10650
+    },
+    {
+      "epoch": 1.8366643694004137,
+      "grad_norm": 3.7579784393310547,
+      "learning_rate": 1.0090204531187168e-09,
+      "logits/chosen": -2.832010269165039,
+      "logits/rejected": -2.819505453109741,
+      "logps/chosen": -73.2978286743164,
+      "logps/rejected": -77.49386596679688,
+      "loss": 0.6772,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.18599703907966614,
+      "rewards/margins": 0.03714095428586006,
+      "rewards/rejected": -0.2231380045413971,
+      "step": 10660
+    },
+    {
+      "epoch": 1.8383873190902826,
+      "grad_norm": 4.046555519104004,
+      "learning_rate": 9.8798597502181e-10,
+      "logits/chosen": -2.852919816970825,
+      "logits/rejected": -2.834721803665161,
+      "logps/chosen": -71.24771881103516,
+      "logps/rejected": -76.40409088134766,
+      "loss": 0.6679,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.1910954862833023,
+      "rewards/margins": 0.05730627849698067,
+      "rewards/rejected": -0.24840179085731506,
+      "step": 10670
+    },
+    {
+      "epoch": 1.8401102687801516,
+      "grad_norm": 3.5693230628967285,
+      "learning_rate": 9.671686394166156e-10,
+      "logits/chosen": -2.887040615081787,
+      "logits/rejected": -2.8431241512298584,
+      "logps/chosen": -73.22282409667969,
+      "logps/rejected": -74.20548248291016,
+      "loss": 0.6642,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.16933730244636536,
+      "rewards/margins": 0.06490884721279144,
+      "rewards/rejected": -0.2342461347579956,
+      "step": 10680
+    },
+    {
+      "epoch": 1.8418332184700206,
+      "grad_norm": 3.5955333709716797,
+      "learning_rate": 9.465686345558944e-10,
+      "logits/chosen": -2.8702359199523926,
+      "logits/rejected": -2.864204168319702,
+      "logps/chosen": -70.88008880615234,
+      "logps/rejected": -78.17689514160156,
+      "loss": 0.6726,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.17130103707313538,
+      "rewards/margins": 0.04670166224241257,
+      "rewards/rejected": -0.21800275146961212,
+      "step": 10690
+    },
+    {
+      "epoch": 1.8435561681598898,
+      "grad_norm": 3.4594194889068604,
+      "learning_rate": 9.261861467270788e-10,
+      "logits/chosen": -2.928183078765869,
+      "logits/rejected": -2.888151168823242,
+      "logps/chosen": -72.86530303955078,
+      "logps/rejected": -72.958740234375,
+      "loss": 0.6665,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.1669774204492569,
+      "rewards/margins": 0.059218309819698334,
+      "rewards/rejected": -0.22619576752185822,
+      "step": 10700
+    },
+    {
+      "epoch": 1.8435561681598898,
+      "eval_logits/chosen": -2.9624197483062744,
+      "eval_logits/rejected": -2.956575393676758,
+      "eval_logps/chosen": -70.82567596435547,
+      "eval_logps/rejected": -78.12754821777344,
+      "eval_loss": 0.6807761788368225,
+      "eval_rewards/accuracies": 0.6052509546279907,
+      "eval_rewards/chosen": -0.12113778293132782,
+      "eval_rewards/margins": 0.02833644114434719,
+      "eval_rewards/rejected": -0.14947423338890076,
+      "eval_runtime": 382.9111,
+      "eval_samples_per_second": 11.24,
+      "eval_steps_per_second": 1.405,
+      "step": 10700
+    },
+    {
+      "epoch": 1.8452791178497587,
+      "grad_norm": 4.013017177581787,
+      "learning_rate": 9.060213602505778e-10,
+      "logits/chosen": -2.838757038116455,
+      "logits/rejected": -2.809877395629883,
+      "logps/chosen": -70.12583923339844,
+      "logps/rejected": -71.8887939453125,
+      "loss": 0.6672,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.1668120175600052,
+      "rewards/margins": 0.05769949406385422,
+      "rewards/rejected": -0.2245115041732788,
+      "step": 10710
+    },
+    {
+      "epoch": 1.847002067539628,
+      "grad_norm": 4.000687599182129,
+      "learning_rate": 8.860744574781032e-10,
+      "logits/chosen": -2.864497184753418,
+      "logits/rejected": -2.8338265419006348,
+      "logps/chosen": -71.42401885986328,
+      "logps/rejected": -75.85304260253906,
+      "loss": 0.6726,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1883702278137207,
+      "rewards/margins": 0.0473480224609375,
+      "rewards/rejected": -0.2357182800769806,
+      "step": 10720
+    },
+    {
+      "epoch": 1.848725017229497,
+      "grad_norm": 4.42160701751709,
+      "learning_rate": 8.663456187910423e-10,
+      "logits/chosen": -2.9521403312683105,
+      "logits/rejected": -2.9164040088653564,
+      "logps/chosen": -70.5853500366211,
+      "logps/rejected": -73.77149963378906,
+      "loss": 0.6597,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.16626228392124176,
+      "rewards/margins": 0.07266949862241745,
+      "rewards/rejected": -0.2389317750930786,
+      "step": 10730
+    },
+    {
+      "epoch": 1.8504479669193659,
+      "grad_norm": 5.024820804595947,
+      "learning_rate": 8.468350225987909e-10,
+      "logits/chosen": -2.8210701942443848,
+      "logits/rejected": -2.811014413833618,
+      "logps/chosen": -73.56663513183594,
+      "logps/rejected": -76.52776336669922,
+      "loss": 0.6832,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.18876317143440247,
+      "rewards/margins": 0.02501165308058262,
+      "rewards/rejected": -0.21377484500408173,
+      "step": 10740
+    },
+    {
+      "epoch": 1.852170916609235,
+      "grad_norm": 3.44173002243042,
+      "learning_rate": 8.275428453371813e-10,
+      "logits/chosen": -2.825425624847412,
+      "logits/rejected": -2.796748161315918,
+      "logps/chosen": -68.54912567138672,
+      "logps/rejected": -76.44398498535156,
+      "loss": 0.6597,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.16490864753723145,
+      "rewards/margins": 0.0728837326169014,
+      "rewards/rejected": -0.23779237270355225,
+      "step": 10750
+    },
+    {
+      "epoch": 1.853893866299104,
+      "grad_norm": 4.073663234710693,
+      "learning_rate": 8.084692614668543e-10,
+      "logits/chosen": -2.8366236686706543,
+      "logits/rejected": -2.8169126510620117,
+      "logps/chosen": -70.49452209472656,
+      "logps/rejected": -74.8795166015625,
+      "loss": 0.6695,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.16813725233078003,
+      "rewards/margins": 0.052797652781009674,
+      "rewards/rejected": -0.2209349125623703,
+      "step": 10760
+    },
+    {
+      "epoch": 1.8556168159889732,
+      "grad_norm": 4.0448317527771,
+      "learning_rate": 7.896144434716951e-10,
+      "logits/chosen": -2.8195300102233887,
+      "logits/rejected": -2.8038220405578613,
+      "logps/chosen": -72.86800384521484,
+      "logps/rejected": -76.54383850097656,
+      "loss": 0.6655,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.15796320140361786,
+      "rewards/margins": 0.061583906412124634,
+      "rewards/rejected": -0.2195471227169037,
+      "step": 10770
+    },
+    {
+      "epoch": 1.8573397656788422,
+      "grad_norm": 3.5834341049194336,
+      "learning_rate": 7.709785618572801e-10,
+      "logits/chosen": -2.9493727684020996,
+      "logits/rejected": -2.932036876678467,
+      "logps/chosen": -70.44261169433594,
+      "logps/rejected": -76.95472717285156,
+      "loss": 0.6729,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.1727297455072403,
+      "rewards/margins": 0.04596462845802307,
+      "rewards/rejected": -0.21869435906410217,
+      "step": 10780
+    },
+    {
+      "epoch": 1.8590627153687111,
+      "grad_norm": 3.887505292892456,
+      "learning_rate": 7.525617851493166e-10,
+      "logits/chosen": -2.9606025218963623,
+      "logits/rejected": -2.9302608966827393,
+      "logps/chosen": -67.64434814453125,
+      "logps/rejected": -75.49552917480469,
+      "loss": 0.6625,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.15303254127502441,
+      "rewards/margins": 0.067867711186409,
+      "rewards/rejected": -0.2209002524614334,
+      "step": 10790
+    },
+    {
+      "epoch": 1.8607856650585803,
+      "grad_norm": 3.9976820945739746,
+      "learning_rate": 7.343642798921384e-10,
+      "logits/chosen": -2.956207513809204,
+      "logits/rejected": -2.9348456859588623,
+      "logps/chosen": -69.45646667480469,
+      "logps/rejected": -77.43666076660156,
+      "loss": 0.6663,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.17497113347053528,
+      "rewards/margins": 0.06019216775894165,
+      "rewards/rejected": -0.23516333103179932,
+      "step": 10800
+    },
+    {
+      "epoch": 1.8607856650585803,
+      "eval_logits/chosen": -2.96236515045166,
+      "eval_logits/rejected": -2.956552028656006,
+      "eval_logps/chosen": -70.83243560791016,
+      "eval_logps/rejected": -78.13817596435547,
+      "eval_loss": 0.6807573437690735,
+      "eval_rewards/accuracies": 0.6078066825866699,
+      "eval_rewards/chosen": -0.12120549380779266,
+      "eval_rewards/margins": 0.02837507613003254,
+      "eval_rewards/rejected": -0.14958056807518005,
+      "eval_runtime": 382.5068,
+      "eval_samples_per_second": 11.252,
+      "eval_steps_per_second": 1.407,
+      "step": 10800
+    },
+    {
+      "epoch": 1.8625086147484493,
+      "grad_norm": 3.966688394546509,
+      "learning_rate": 7.163862106471852e-10,
+      "logits/chosen": -2.8858401775360107,
+      "logits/rejected": -2.8436789512634277,
+      "logps/chosen": -74.41349029541016,
+      "logps/rejected": -73.30751037597656,
+      "loss": 0.6654,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.1703367531299591,
+      "rewards/margins": 0.06163492053747177,
+      "rewards/rejected": -0.23197169601917267,
+      "step": 10810
+    },
+    {
+      "epoch": 1.8642315644383185,
+      "grad_norm": 3.734957218170166,
+      "learning_rate": 6.986277399915197e-10,
+      "logits/chosen": -2.8294825553894043,
+      "logits/rejected": -2.8054709434509277,
+      "logps/chosen": -68.5473403930664,
+      "logps/rejected": -72.60321044921875,
+      "loss": 0.6672,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1545238196849823,
+      "rewards/margins": 0.0574672594666481,
+      "rewards/rejected": -0.2119910717010498,
+      "step": 10820
+    },
+    {
+      "epoch": 1.8659545141281875,
+      "grad_norm": 3.868393898010254,
+      "learning_rate": 6.810890285163628e-10,
+      "logits/chosen": -2.848698377609253,
+      "logits/rejected": -2.8219423294067383,
+      "logps/chosen": -69.58952331542969,
+      "logps/rejected": -75.9024887084961,
+      "loss": 0.6718,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.17773368954658508,
+      "rewards/margins": 0.04823298379778862,
+      "rewards/rejected": -0.2259666621685028,
+      "step": 10830
+    },
+    {
+      "epoch": 1.8676774638180564,
+      "grad_norm": 3.7370858192443848,
+      "learning_rate": 6.637702348256307e-10,
+      "logits/chosen": -2.8949930667877197,
+      "logits/rejected": -2.865039110183716,
+      "logps/chosen": -69.01568603515625,
+      "logps/rejected": -72.116455078125,
+      "loss": 0.6703,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.1602485477924347,
+      "rewards/margins": 0.051848627626895905,
+      "rewards/rejected": -0.21209721267223358,
+      "step": 10840
+    },
+    {
+      "epoch": 1.8694004135079254,
+      "grad_norm": 3.7863426208496094,
+      "learning_rate": 6.466715155345109e-10,
+      "logits/chosen": -2.8188366889953613,
+      "logits/rejected": -2.8034310340881348,
+      "logps/chosen": -66.44449615478516,
+      "logps/rejected": -76.65794372558594,
+      "loss": 0.6667,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.16283705830574036,
+      "rewards/margins": 0.05919041112065315,
+      "rewards/rejected": -0.2220274657011032,
+      "step": 10850
+    },
+    {
+      "epoch": 1.8711233631977946,
+      "grad_norm": 4.034375190734863,
+      "learning_rate": 6.2979302526803e-10,
+      "logits/chosen": -2.940924882888794,
+      "logits/rejected": -2.908414363861084,
+      "logps/chosen": -72.17329406738281,
+      "logps/rejected": -75.18521881103516,
+      "loss": 0.6668,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.16914477944374084,
+      "rewards/margins": 0.05857177451252937,
+      "rewards/rejected": -0.2277165651321411,
+      "step": 10860
+    },
+    {
+      "epoch": 1.8728463128876638,
+      "grad_norm": 3.3708362579345703,
+      "learning_rate": 6.131349166596883e-10,
+      "logits/chosen": -2.7760252952575684,
+      "logits/rejected": -2.7716622352600098,
+      "logps/chosen": -67.44125366210938,
+      "logps/rejected": -79.22506713867188,
+      "loss": 0.6653,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.16955524682998657,
+      "rewards/margins": 0.06207970529794693,
+      "rewards/rejected": -0.2316349446773529,
+      "step": 10870
+    },
+    {
+      "epoch": 1.8745692625775328,
+      "grad_norm": 3.8947677612304688,
+      "learning_rate": 5.966973403500303e-10,
+      "logits/chosen": -2.8660035133361816,
+      "logits/rejected": -2.8458571434020996,
+      "logps/chosen": -70.62030792236328,
+      "logps/rejected": -75.12186431884766,
+      "loss": 0.6653,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.1641118973493576,
+      "rewards/margins": 0.06016100198030472,
+      "rewards/rejected": -0.2242729216814041,
+      "step": 10880
+    },
+    {
+      "epoch": 1.8762922122674017,
+      "grad_norm": 3.604186773300171,
+      "learning_rate": 5.804804449853401e-10,
+      "logits/chosen": -2.9161853790283203,
+      "logits/rejected": -2.898859977722168,
+      "logps/chosen": -70.59284973144531,
+      "logps/rejected": -77.46681213378906,
+      "loss": 0.6697,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1743243932723999,
+      "rewards/margins": 0.05209631472826004,
+      "rewards/rejected": -0.22642068564891815,
+      "step": 10890
+    },
+    {
+      "epoch": 1.8780151619572707,
+      "grad_norm": 3.339538097381592,
+      "learning_rate": 5.644843772162373e-10,
+      "logits/chosen": -2.9663305282592773,
+      "logits/rejected": -2.93340802192688,
+      "logps/chosen": -70.63031005859375,
+      "logps/rejected": -71.8335189819336,
+      "loss": 0.6674,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.15637972950935364,
+      "rewards/margins": 0.05591698735952377,
+      "rewards/rejected": -0.2122967541217804,
+      "step": 10900
+    },
+    {
+      "epoch": 1.8780151619572707,
+      "eval_logits/chosen": -2.962559700012207,
+      "eval_logits/rejected": -2.9567580223083496,
+      "eval_logps/chosen": -70.84233093261719,
+      "eval_logps/rejected": -78.1541748046875,
+      "eval_loss": 0.6807303428649902,
+      "eval_rewards/accuracies": 0.6082713603973389,
+      "eval_rewards/chosen": -0.12130436301231384,
+      "eval_rewards/margins": 0.028436101973056793,
+      "eval_rewards/rejected": -0.14974050223827362,
+      "eval_runtime": 382.9413,
+      "eval_samples_per_second": 11.239,
+      "eval_steps_per_second": 1.405,
+      "step": 10900
+    },
+    {
+      "epoch": 1.8797381116471399,
+      "grad_norm": 4.056522369384766,
+      "learning_rate": 5.487092816963995e-10,
+      "logits/chosen": -2.864166736602783,
+      "logits/rejected": -2.831540584564209,
+      "logps/chosen": -70.78670501708984,
+      "logps/rejected": -76.51332092285156,
+      "loss": 0.6602,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.1512027084827423,
+      "rewards/margins": 0.07212905585765839,
+      "rewards/rejected": -0.2233317643404007,
+      "step": 10910
+    },
+    {
+      "epoch": 1.881461061337009,
+      "grad_norm": 3.823746919631958,
+      "learning_rate": 5.331553010812311e-10,
+      "logits/chosen": -2.832442283630371,
+      "logits/rejected": -2.8082435131073,
+      "logps/chosen": -73.90846252441406,
+      "logps/rejected": -76.31095886230469,
+      "loss": 0.6689,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.18070828914642334,
+      "rewards/margins": 0.05319540575146675,
+      "rewards/rejected": -0.2339036911725998,
+      "step": 10920
+    },
+    {
+      "epoch": 1.883184011026878,
+      "grad_norm": 4.18733024597168,
+      "learning_rate": 5.178225760265775e-10,
+      "logits/chosen": -2.8075039386749268,
+      "logits/rejected": -2.7775464057922363,
+      "logps/chosen": -71.64750671386719,
+      "logps/rejected": -73.6923828125,
+      "loss": 0.6721,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.17435511946678162,
+      "rewards/margins": 0.04769337177276611,
+      "rewards/rejected": -0.22204849123954773,
+      "step": 10930
+    },
+    {
+      "epoch": 1.884906960716747,
+      "grad_norm": 4.253951549530029,
+      "learning_rate": 5.027112451874482e-10,
+      "logits/chosen": -2.7650978565216064,
+      "logits/rejected": -2.7459352016448975,
+      "logps/chosen": -73.86014556884766,
+      "logps/rejected": -77.244140625,
+      "loss": 0.6725,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.17400948703289032,
+      "rewards/margins": 0.04671402648091316,
+      "rewards/rejected": -0.22072353959083557,
+      "step": 10940
+    },
+    {
+      "epoch": 1.886629910406616,
+      "grad_norm": 3.775698184967041,
+      "learning_rate": 4.87821445216774e-10,
+      "logits/chosen": -2.8180079460144043,
+      "logits/rejected": -2.800579071044922,
+      "logps/chosen": -72.72742462158203,
+      "logps/rejected": -77.5301742553711,
+      "loss": 0.672,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.18120987713336945,
+      "rewards/margins": 0.04714659973978996,
+      "rewards/rejected": -0.22835645079612732,
+      "step": 10950
+    },
+    {
+      "epoch": 1.8883528600964852,
+      "grad_norm": 4.0146894454956055,
+      "learning_rate": 4.731533107641627e-10,
+      "logits/chosen": -2.8881704807281494,
+      "logits/rejected": -2.86869740486145,
+      "logps/chosen": -71.45487976074219,
+      "logps/rejected": -76.72061920166016,
+      "loss": 0.6642,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1764443814754486,
+      "rewards/margins": 0.06502439081668854,
+      "rewards/rejected": -0.24146878719329834,
+      "step": 10960
+    },
+    {
+      "epoch": 1.8900758097863544,
+      "grad_norm": 4.310637950897217,
+      "learning_rate": 4.587069744746791e-10,
+      "logits/chosen": -2.904383420944214,
+      "logits/rejected": -2.874455690383911,
+      "logps/chosen": -72.63307189941406,
+      "logps/rejected": -76.18873596191406,
+      "loss": 0.6651,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.15770134329795837,
+      "rewards/margins": 0.06307530403137207,
+      "rewards/rejected": -0.22077664732933044,
+      "step": 10970
+    },
+    {
+      "epoch": 1.8917987594762233,
+      "grad_norm": 3.9744670391082764,
+      "learning_rate": 4.4448256698766393e-10,
+      "logits/chosen": -2.8859124183654785,
+      "logits/rejected": -2.8715624809265137,
+      "logps/chosen": -72.58234405517578,
+      "logps/rejected": -79.98782348632812,
+      "loss": 0.6678,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.18009212613105774,
+      "rewards/margins": 0.057999808341264725,
+      "rewards/rejected": -0.23809191584587097,
+      "step": 10980
+    },
+    {
+      "epoch": 1.8935217091660923,
+      "grad_norm": 4.373286247253418,
+      "learning_rate": 4.3048021693552206e-10,
+      "logits/chosen": -2.806406259536743,
+      "logits/rejected": -2.7946393489837646,
+      "logps/chosen": -70.21060943603516,
+      "logps/rejected": -75.15233612060547,
+      "loss": 0.6709,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.1772306263446808,
+      "rewards/margins": 0.049336228519678116,
+      "rewards/rejected": -0.2265668660402298,
+      "step": 10990
+    },
+    {
+      "epoch": 1.8952446588559613,
+      "grad_norm": 4.192987442016602,
+      "learning_rate": 4.167000509425811e-10,
+      "logits/chosen": -2.9980900287628174,
+      "logits/rejected": -2.9941141605377197,
+      "logps/chosen": -71.83065032958984,
+      "logps/rejected": -79.85494232177734,
+      "loss": 0.6767,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.18941548466682434,
+      "rewards/margins": 0.039289992302656174,
+      "rewards/rejected": -0.22870545089244843,
+      "step": 11000
+    },
+    {
+      "epoch": 1.8952446588559613,
+      "eval_logits/chosen": -2.9626047611236572,
+      "eval_logits/rejected": -2.956747531890869,
+      "eval_logps/chosen": -70.82953643798828,
+      "eval_logps/rejected": -78.12945556640625,
+      "eval_loss": 0.6807882785797119,
+      "eval_rewards/accuracies": 0.6078066825866699,
+      "eval_rewards/chosen": -0.1211763545870781,
+      "eval_rewards/margins": 0.028317056596279144,
+      "eval_rewards/rejected": -0.14949342608451843,
+      "eval_runtime": 382.4795,
+      "eval_samples_per_second": 11.253,
+      "eval_steps_per_second": 1.407,
+      "step": 11000
+    },
+    {
+      "epoch": 1.8969676085458305,
+      "grad_norm": 3.8019895553588867,
+      "learning_rate": 4.0314219362395095e-10,
+      "logits/chosen": -2.925306797027588,
+      "logits/rejected": -2.893857955932617,
+      "logps/chosen": -68.93820190429688,
+      "logps/rejected": -74.15766143798828,
+      "loss": 0.6612,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.1573188751935959,
+      "rewards/margins": 0.06955505162477493,
+      "rewards/rejected": -0.22687391936779022,
+      "step": 11010
+    },
+    {
+      "epoch": 1.8986905582356997,
+      "grad_norm": 4.183621406555176,
+      "learning_rate": 3.898067675843747e-10,
+      "logits/chosen": -2.9294357299804688,
+      "logits/rejected": -2.9109275341033936,
+      "logps/chosen": -71.27839660644531,
+      "logps/rejected": -75.10275268554688,
+      "loss": 0.6672,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.16274376213550568,
+      "rewards/margins": 0.05788187310099602,
+      "rewards/rejected": -0.220625638961792,
+      "step": 11020
+    },
+    {
+      "epoch": 1.9004135079255686,
+      "grad_norm": 3.97092342376709,
+      "learning_rate": 3.766938934171349e-10,
+      "logits/chosen": -2.902945041656494,
+      "logits/rejected": -2.8947877883911133,
+      "logps/chosen": -72.25243377685547,
+      "logps/rejected": -78.0600357055664,
+      "loss": 0.6655,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1746421754360199,
+      "rewards/margins": 0.061062950640916824,
+      "rewards/rejected": -0.23570513725280762,
+      "step": 11030
+    },
+    {
+      "epoch": 1.9021364576154376,
+      "grad_norm": 3.779587984085083,
+      "learning_rate": 3.6380368970296836e-10,
+      "logits/chosen": -2.931520938873291,
+      "logits/rejected": -2.916541576385498,
+      "logps/chosen": -74.1861343383789,
+      "logps/rejected": -78.86256408691406,
+      "loss": 0.6684,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.1771935522556305,
+      "rewards/margins": 0.056019581854343414,
+      "rewards/rejected": -0.23321311175823212,
+      "step": 11040
+    },
+    {
+      "epoch": 1.9038594073053066,
+      "grad_norm": 3.927412509918213,
+      "learning_rate": 3.5113627300897284e-10,
+      "logits/chosen": -2.855600118637085,
+      "logits/rejected": -2.8361144065856934,
+      "logps/chosen": -67.69244384765625,
+      "logps/rejected": -76.83377075195312,
+      "loss": 0.6669,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.15728303790092468,
+      "rewards/margins": 0.058397598564624786,
+      "rewards/rejected": -0.21568067371845245,
+      "step": 11050
+    },
+    {
+      "epoch": 1.9055823569951758,
+      "grad_norm": 3.9805715084075928,
+      "learning_rate": 3.38691757887577e-10,
+      "logits/chosen": -2.8994479179382324,
+      "logits/rejected": -2.8570735454559326,
+      "logps/chosen": -71.16365051269531,
+      "logps/rejected": -75.70503234863281,
+      "loss": 0.6613,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.16888776421546936,
+      "rewards/margins": 0.07078926265239716,
+      "rewards/rejected": -0.23967702686786652,
+      "step": 11060
+    },
+    {
+      "epoch": 1.907305306685045,
+      "grad_norm": 3.7986254692077637,
+      "learning_rate": 3.264702568754912e-10,
+      "logits/chosen": -2.9236810207366943,
+      "logits/rejected": -2.8767077922821045,
+      "logps/chosen": -68.6697006225586,
+      "logps/rejected": -73.80796813964844,
+      "loss": 0.6529,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.15494240820407867,
+      "rewards/margins": 0.08748442679643631,
+      "rewards/rejected": -0.2424268275499344,
+      "step": 11070
+    },
+    {
+      "epoch": 1.909028256374914,
+      "grad_norm": 4.023408889770508,
+      "learning_rate": 3.1447188049268656e-10,
+      "logits/chosen": -2.905372142791748,
+      "logits/rejected": -2.879030704498291,
+      "logps/chosen": -73.01191711425781,
+      "logps/rejected": -76.07627868652344,
+      "loss": 0.6657,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.1750846654176712,
+      "rewards/margins": 0.060736656188964844,
+      "rewards/rejected": -0.23582132160663605,
+      "step": 11080
+    },
+    {
+      "epoch": 1.9107512060647829,
+      "grad_norm": 3.9777402877807617,
+      "learning_rate": 3.0269673724140353e-10,
+      "logits/chosen": -2.8732471466064453,
+      "logits/rejected": -2.853816270828247,
+      "logps/chosen": -75.87340545654297,
+      "logps/rejected": -75.41868591308594,
+      "loss": 0.673,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.18049900233745575,
+      "rewards/margins": 0.04642079770565033,
+      "rewards/rejected": -0.22691981494426727,
+      "step": 11090
+    },
+    {
+      "epoch": 1.9124741557546519,
+      "grad_norm": 3.3175225257873535,
+      "learning_rate": 2.9114493360517243e-10,
+      "logits/chosen": -2.7630672454833984,
+      "logits/rejected": -2.7479026317596436,
+      "logps/chosen": -67.33642578125,
+      "logps/rejected": -75.93243408203125,
+      "loss": 0.6683,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1645837426185608,
+      "rewards/margins": 0.05451236292719841,
+      "rewards/rejected": -0.2190960943698883,
+      "step": 11100
+    },
+    {
+      "epoch": 1.9124741557546519,
+      "eval_logits/chosen": -2.962803363800049,
+      "eval_logits/rejected": -2.956941604614258,
+      "eval_logps/chosen": -70.83159637451172,
+      "eval_logps/rejected": -78.1378402709961,
+      "eval_loss": 0.6807635426521301,
+      "eval_rewards/accuracies": 0.6087360382080078,
+      "eval_rewards/chosen": -0.12119705975055695,
+      "eval_rewards/margins": 0.02838001400232315,
+      "eval_rewards/rejected": -0.1495770663022995,
+      "eval_runtime": 383.208,
+      "eval_samples_per_second": 11.231,
+      "eval_steps_per_second": 1.404,
+      "step": 11100
+    },
+    {
+      "epoch": 1.914197105444521,
+      "grad_norm": 3.7980966567993164,
+      "learning_rate": 2.79816574047842e-10,
+      "logits/chosen": -2.901625394821167,
+      "logits/rejected": -2.8771677017211914,
+      "logps/chosen": -72.34051513671875,
+      "logps/rejected": -79.36370086669922,
+      "loss": 0.6596,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.1843886822462082,
+      "rewards/margins": 0.07403866946697235,
+      "rewards/rejected": -0.25842735171318054,
+      "step": 11110
+    },
+    {
+      "epoch": 1.9159200551343902,
+      "grad_norm": 3.338250160217285,
+      "learning_rate": 2.6871176101263826e-10,
+      "logits/chosen": -2.991860866546631,
+      "logits/rejected": -2.962674379348755,
+      "logps/chosen": -77.74948120117188,
+      "logps/rejected": -77.08177185058594,
+      "loss": 0.6704,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.18234574794769287,
+      "rewards/margins": 0.05172035098075867,
+      "rewards/rejected": -0.23406608402729034,
+      "step": 11120
+    },
+    {
+      "epoch": 1.9176430048242592,
+      "grad_norm": 4.406763076782227,
+      "learning_rate": 2.5783059492124335e-10,
+      "logits/chosen": -2.8212103843688965,
+      "logits/rejected": -2.799124240875244,
+      "logps/chosen": -75.0366439819336,
+      "logps/rejected": -75.9485092163086,
+      "loss": 0.6776,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.19160573184490204,
+      "rewards/margins": 0.03602327033877373,
+      "rewards/rejected": -0.22762899100780487,
+      "step": 11130
+    },
+    {
+      "epoch": 1.9193659545141282,
+      "grad_norm": 3.755013942718506,
+      "learning_rate": 2.471731741728794e-10,
+      "logits/chosen": -2.7939274311065674,
+      "logits/rejected": -2.7739617824554443,
+      "logps/chosen": -70.43766021728516,
+      "logps/rejected": -73.73213195800781,
+      "loss": 0.6683,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.16624291241168976,
+      "rewards/margins": 0.055305201560258865,
+      "rewards/rejected": -0.22154811024665833,
+      "step": 11140
+    },
+    {
+      "epoch": 1.9210889042039971,
+      "grad_norm": 4.383630752563477,
+      "learning_rate": 2.367395951434231e-10,
+      "logits/chosen": -2.872030735015869,
+      "logits/rejected": -2.859677314758301,
+      "logps/chosen": -76.06889343261719,
+      "logps/rejected": -77.54962921142578,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.1797533631324768,
+      "rewards/margins": 0.03063751757144928,
+      "rewards/rejected": -0.2103908807039261,
+      "step": 11150
+    },
+    {
+      "epoch": 1.9228118538938663,
+      "grad_norm": 3.5709264278411865,
+      "learning_rate": 2.2652995218452876e-10,
+      "logits/chosen": -2.960582733154297,
+      "logits/rejected": -2.9416518211364746,
+      "logps/chosen": -67.99903106689453,
+      "logps/rejected": -72.40933227539062,
+      "loss": 0.6684,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.15880367159843445,
+      "rewards/margins": 0.05513704940676689,
+      "rewards/rejected": -0.21394070982933044,
+      "step": 11160
+    },
+    {
+      "epoch": 1.9245348035837355,
+      "grad_norm": 3.6263105869293213,
+      "learning_rate": 2.1654433762278713e-10,
+      "logits/chosen": -2.7908620834350586,
+      "logits/rejected": -2.762441635131836,
+      "logps/chosen": -73.19970703125,
+      "logps/rejected": -70.05372619628906,
+      "loss": 0.671,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.16037556529045105,
+      "rewards/margins": 0.049938809126615524,
+      "rewards/rejected": -0.21031439304351807,
+      "step": 11170
+    },
+    {
+      "epoch": 1.9262577532736045,
+      "grad_norm": 3.9126534461975098,
+      "learning_rate": 2.0678284175887906e-10,
+      "logits/chosen": -2.919954776763916,
+      "logits/rejected": -2.8970367908477783,
+      "logps/chosen": -72.05885314941406,
+      "logps/rejected": -74.23884582519531,
+      "loss": 0.6694,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.1809300184249878,
+      "rewards/margins": 0.0532182939350605,
+      "rewards/rejected": -0.234148308634758,
+      "step": 11180
+    },
+    {
+      "epoch": 1.9279807029634735,
+      "grad_norm": 4.053760528564453,
+      "learning_rate": 1.972455528667677e-10,
+      "logits/chosen": -2.920574188232422,
+      "logits/rejected": -2.891080141067505,
+      "logps/chosen": -71.44400787353516,
+      "logps/rejected": -75.93632507324219,
+      "loss": 0.6575,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.16562137007713318,
+      "rewards/margins": 0.0773579552769661,
+      "rewards/rejected": -0.24297931790351868,
+      "step": 11190
+    },
+    {
+      "epoch": 1.9297036526533424,
+      "grad_norm": 3.9351112842559814,
+      "learning_rate": 1.8793255719288248e-10,
+      "logits/chosen": -2.9200241565704346,
+      "logits/rejected": -2.899113178253174,
+      "logps/chosen": -68.9150161743164,
+      "logps/rejected": -74.97122192382812,
+      "loss": 0.6673,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.15601924061775208,
+      "rewards/margins": 0.05622955411672592,
+      "rewards/rejected": -0.2122488021850586,
+      "step": 11200
+    },
+    {
+      "epoch": 1.9297036526533424,
+      "eval_logits/chosen": -2.9624381065368652,
+      "eval_logits/rejected": -2.956598997116089,
+      "eval_logps/chosen": -70.82904815673828,
+      "eval_logps/rejected": -78.1369857788086,
+      "eval_loss": 0.6807476878166199,
+      "eval_rewards/accuracies": 0.6089683771133423,
+      "eval_rewards/chosen": -0.12117151916027069,
+      "eval_rewards/margins": 0.028397098183631897,
+      "eval_rewards/rejected": -0.1495686173439026,
+      "eval_runtime": 383.4214,
+      "eval_samples_per_second": 11.225,
+      "eval_steps_per_second": 1.403,
+      "step": 11200
+    },
+    {
+      "epoch": 1.9314266023432116,
+      "grad_norm": 3.7449188232421875,
+      "learning_rate": 1.7884393895536697e-10,
+      "logits/chosen": -2.772278308868408,
+      "logits/rejected": -2.7558465003967285,
+      "logps/chosen": -67.96211242675781,
+      "logps/rejected": -77.23825073242188,
+      "loss": 0.668,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.17551326751708984,
+      "rewards/margins": 0.057414185255765915,
+      "rewards/rejected": -0.23292744159698486,
+      "step": 11210
+    },
+    {
+      "epoch": 1.9331495520330806,
+      "grad_norm": 3.853837251663208,
+      "learning_rate": 1.6997978034329342e-10,
+      "logits/chosen": -2.882115364074707,
+      "logits/rejected": -2.868640184402466,
+      "logps/chosen": -68.49288940429688,
+      "logps/rejected": -77.95133972167969,
+      "loss": 0.6672,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.1644320785999298,
+      "rewards/margins": 0.05823678523302078,
+      "rewards/rejected": -0.2226688414812088,
+      "step": 11220
+    },
+    {
+      "epoch": 1.9348725017229498,
+      "grad_norm": 3.7204480171203613,
+      "learning_rate": 1.6134016151592988e-10,
+      "logits/chosen": -2.8454439640045166,
+      "logits/rejected": -2.832601547241211,
+      "logps/chosen": -75.22699737548828,
+      "logps/rejected": -74.13967895507812,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.1945803016424179,
+      "rewards/margins": 0.031563881784677505,
+      "rewards/rejected": -0.22614416480064392,
+      "step": 11230
+    },
+    {
+      "epoch": 1.9365954514128187,
+      "grad_norm": 3.689436435699463,
+      "learning_rate": 1.5292516060201598e-10,
+      "logits/chosen": -2.8353400230407715,
+      "logits/rejected": -2.8199429512023926,
+      "logps/chosen": -71.64716339111328,
+      "logps/rejected": -74.92933654785156,
+      "loss": 0.6733,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.1699708253145218,
+      "rewards/margins": 0.04488908126950264,
+      "rewards/rejected": -0.21485991775989532,
+      "step": 11240
+    },
+    {
+      "epoch": 1.9383184011026877,
+      "grad_norm": 4.465695381164551,
+      "learning_rate": 1.4473485369905225e-10,
+      "logits/chosen": -2.8656206130981445,
+      "logits/rejected": -2.8391621112823486,
+      "logps/chosen": -70.72211456298828,
+      "logps/rejected": -76.77116394042969,
+      "loss": 0.6622,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.16433927416801453,
+      "rewards/margins": 0.06710951775312424,
+      "rewards/rejected": -0.23144879937171936,
+      "step": 11250
+    },
+    {
+      "epoch": 1.940041350792557,
+      "grad_norm": 4.292695999145508,
+      "learning_rate": 1.3676931487261456e-10,
+      "logits/chosen": -2.812873363494873,
+      "logits/rejected": -2.779399871826172,
+      "logps/chosen": -74.55575561523438,
+      "logps/rejected": -73.16273498535156,
+      "loss": 0.6716,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.1672225147485733,
+      "rewards/margins": 0.04880499094724655,
+      "rewards/rejected": -0.21602746844291687,
+      "step": 11260
+    },
+    {
+      "epoch": 1.9417643004824259,
+      "grad_norm": 3.9187545776367188,
+      "learning_rate": 1.2902861615568529e-10,
+      "logits/chosen": -2.864241123199463,
+      "logits/rejected": -2.8340325355529785,
+      "logps/chosen": -72.13081359863281,
+      "logps/rejected": -73.9896011352539,
+      "loss": 0.6642,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.16564273834228516,
+      "rewards/margins": 0.06385232508182526,
+      "rewards/rejected": -0.2294950783252716,
+      "step": 11270
+    },
+    {
+      "epoch": 1.943487250172295,
+      "grad_norm": 3.635704755783081,
+      "learning_rate": 1.215128275479954e-10,
+      "logits/chosen": -2.9202098846435547,
+      "logits/rejected": -2.890702724456787,
+      "logps/chosen": -73.21311950683594,
+      "logps/rejected": -74.48944091796875,
+      "loss": 0.6696,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.16900990903377533,
+      "rewards/margins": 0.051773883402347565,
+      "rewards/rejected": -0.2207837998867035,
+      "step": 11280
+    },
+    {
+      "epoch": 1.945210199862164,
+      "grad_norm": 4.021252632141113,
+      "learning_rate": 1.1422201701540569e-10,
+      "logits/chosen": -2.877960205078125,
+      "logits/rejected": -2.8624284267425537,
+      "logps/chosen": -75.56314849853516,
+      "logps/rejected": -78.49247741699219,
+      "loss": 0.6627,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.15604360401630402,
+      "rewards/margins": 0.06746500730514526,
+      "rewards/rejected": -0.22350859642028809,
+      "step": 11290
+    },
+    {
+      "epoch": 1.946933149552033,
+      "grad_norm": 3.8260245323181152,
+      "learning_rate": 1.0715625048927091e-10,
+      "logits/chosen": -2.8497841358184814,
+      "logits/rejected": -2.8242392539978027,
+      "logps/chosen": -76.53433227539062,
+      "logps/rejected": -74.91511535644531,
+      "loss": 0.6781,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.17610198259353638,
+      "rewards/margins": 0.035939160734415054,
+      "rewards/rejected": -0.21204113960266113,
+      "step": 11300
+    },
+    {
+      "epoch": 1.946933149552033,
+      "eval_logits/chosen": -2.9626305103302,
+      "eval_logits/rejected": -2.956784963607788,
+      "eval_logps/chosen": -70.81900024414062,
+      "eval_logps/rejected": -78.13628387451172,
+      "eval_loss": 0.6807034015655518,
+      "eval_rewards/accuracies": 0.6096654534339905,
+      "eval_rewards/chosen": -0.12107101827859879,
+      "eval_rewards/margins": 0.028490470722317696,
+      "eval_rewards/rejected": -0.14956150949001312,
+      "eval_runtime": 383.2956,
+      "eval_samples_per_second": 11.229,
+      "eval_steps_per_second": 1.404,
+      "step": 11300
+    },
+    {
+      "epoch": 1.948656099241902,
+      "grad_norm": 4.015454292297363,
+      "learning_rate": 1.0031559186586824e-10,
+      "logits/chosen": -2.959744453430176,
+      "logits/rejected": -2.9537577629089355,
+      "logps/chosen": -67.63035583496094,
+      "logps/rejected": -75.73067474365234,
+      "loss": 0.6611,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.17167212069034576,
+      "rewards/margins": 0.07040779292583466,
+      "rewards/rejected": -0.24207989871501923,
+      "step": 11310
+    },
+    {
+      "epoch": 1.9503790489317712,
+      "grad_norm": 3.534909248352051,
+      "learning_rate": 9.370010300579212e-11,
+      "logits/chosen": -2.878385305404663,
+      "logits/rejected": -2.848980665206909,
+      "logps/chosen": -70.15480041503906,
+      "logps/rejected": -76.25794982910156,
+      "loss": 0.661,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.15233257412910461,
+      "rewards/margins": 0.07115460187196732,
+      "rewards/rejected": -0.22348718345165253,
+      "step": 11320
+    },
+    {
+      "epoch": 1.9521019986216404,
+      "grad_norm": 4.1883440017700195,
+      "learning_rate": 8.73098437334241e-11,
+      "logits/chosen": -2.8965351581573486,
+      "logits/rejected": -2.860743999481201,
+      "logps/chosen": -69.93683624267578,
+      "logps/rejected": -72.91194915771484,
+      "loss": 0.6592,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.17378143966197968,
+      "rewards/margins": 0.07399272918701172,
+      "rewards/rejected": -0.24777419865131378,
+      "step": 11330
+    },
+    {
+      "epoch": 1.9538249483115093,
+      "grad_norm": 3.9219887256622314,
+      "learning_rate": 8.114487183636942e-11,
+      "logits/chosen": -2.7759623527526855,
+      "logits/rejected": -2.756114959716797,
+      "logps/chosen": -73.11310577392578,
+      "logps/rejected": -78.029052734375,
+      "loss": 0.6688,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.17733654379844666,
+      "rewards/margins": 0.055362798273563385,
+      "rewards/rejected": -0.23269934952259064,
+      "step": 11340
+    },
+    {
+      "epoch": 1.9555478980013783,
+      "grad_norm": 3.7523226737976074,
+      "learning_rate": 7.520524306494358e-11,
+      "logits/chosen": -2.9297935962677,
+      "logits/rejected": -2.918485164642334,
+      "logps/chosen": -75.0234146118164,
+      "logps/rejected": -78.57817840576172,
+      "loss": 0.6737,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.1811675727367401,
+      "rewards/margins": 0.04412878304719925,
+      "rewards/rejected": -0.22529634833335876,
+      "step": 11350
+    },
+    {
+      "epoch": 1.9572708476912473,
+      "grad_norm": 3.7307770252227783,
+      "learning_rate": 6.949101113166711e-11,
+      "logits/chosen": -2.8915507793426514,
+      "logits/rejected": -2.8621668815612793,
+      "logps/chosen": -70.44617462158203,
+      "logps/rejected": -72.85865783691406,
+      "loss": 0.6657,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.18191471695899963,
+      "rewards/margins": 0.06108817458152771,
+      "rewards/rejected": -0.24300289154052734,
+      "step": 11360
+    },
+    {
+      "epoch": 1.9589937973811165,
+      "grad_norm": 4.060991287231445,
+      "learning_rate": 6.40022277107799e-11,
+      "logits/chosen": -2.848566770553589,
+      "logits/rejected": -2.829324722290039,
+      "logps/chosen": -73.57774353027344,
+      "logps/rejected": -76.95945739746094,
+      "loss": 0.666,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.17302419245243073,
+      "rewards/margins": 0.06029297783970833,
+      "rewards/rejected": -0.23331718146800995,
+      "step": 11370
+    },
+    {
+      "epoch": 1.9607167470709856,
+      "grad_norm": 3.899641275405884,
+      "learning_rate": 5.873894243776933e-11,
+      "logits/chosen": -2.8242900371551514,
+      "logits/rejected": -2.805497169494629,
+      "logps/chosen": -70.15582275390625,
+      "logps/rejected": -73.19532775878906,
+      "loss": 0.6722,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.1832256317138672,
+      "rewards/margins": 0.04804827645421028,
+      "rewards/rejected": -0.23127388954162598,
+      "step": 11380
+    },
+    {
+      "epoch": 1.9624396967608546,
+      "grad_norm": 3.8247265815734863,
+      "learning_rate": 5.3701202908931766e-11,
+      "logits/chosen": -2.938483476638794,
+      "logits/rejected": -2.9180026054382324,
+      "logps/chosen": -68.40145874023438,
+      "logps/rejected": -78.46292877197266,
+      "loss": 0.6607,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.1607675701379776,
+      "rewards/margins": 0.0707019492983818,
+      "rewards/rejected": -0.2314695417881012,
+      "step": 11390
+    },
+    {
+      "epoch": 1.9641626464507236,
+      "grad_norm": 3.8907339572906494,
+      "learning_rate": 4.8889054680936736e-11,
+      "logits/chosen": -2.892984628677368,
+      "logits/rejected": -2.8598499298095703,
+      "logps/chosen": -71.83979797363281,
+      "logps/rejected": -74.864501953125,
+      "loss": 0.6682,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.1592736840248108,
+      "rewards/margins": 0.05626269429922104,
+      "rewards/rejected": -0.21553640067577362,
+      "step": 11400
+    },
+    {
+      "epoch": 1.9641626464507236,
+      "eval_logits/chosen": -2.9625651836395264,
+      "eval_logits/rejected": -2.956709384918213,
+      "eval_logps/chosen": -70.84457397460938,
+      "eval_logps/rejected": -78.16127014160156,
+      "eval_loss": 0.6807080507278442,
+      "eval_rewards/accuracies": 0.6085036993026733,
+      "eval_rewards/chosen": -0.12132680416107178,
+      "eval_rewards/margins": 0.028484534472227097,
+      "eval_rewards/rejected": -0.14981132745742798,
+      "eval_runtime": 383.0514,
+      "eval_samples_per_second": 11.236,
+      "eval_steps_per_second": 1.405,
+      "step": 11400
+    },
+    {
+      "epoch": 1.9658855961405926,
+      "grad_norm": 3.946910858154297,
+      "learning_rate": 4.4302541270407887e-11,
+      "logits/chosen": -2.897346019744873,
+      "logits/rejected": -2.8696742057800293,
+      "logps/chosen": -71.43907165527344,
+      "logps/rejected": -75.54390716552734,
+      "loss": 0.6724,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.1575334370136261,
+      "rewards/margins": 0.04721371829509735,
+      "rewards/rejected": -0.20474717020988464,
+      "step": 11410
+    },
+    {
+      "epoch": 1.9676085458304617,
+      "grad_norm": 4.062943935394287,
+      "learning_rate": 3.994170415353715e-11,
+      "logits/chosen": -2.879944324493408,
+      "logits/rejected": -2.848836660385132,
+      "logps/chosen": -70.45439147949219,
+      "logps/rejected": -72.85489654541016,
+      "loss": 0.667,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.1676943600177765,
+      "rewards/margins": 0.05826814845204353,
+      "rewards/rejected": -0.22596248984336853,
+      "step": 11420
+    },
+    {
+      "epoch": 1.969331495520331,
+      "grad_norm": 3.783891439437866,
+      "learning_rate": 3.5806582765715576e-11,
+      "logits/chosen": -2.8017635345458984,
+      "logits/rejected": -2.7758994102478027,
+      "logps/chosen": -71.9343032836914,
+      "logps/rejected": -74.35984802246094,
+      "loss": 0.6723,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.16963975131511688,
+      "rewards/margins": 0.047324612736701965,
+      "rewards/rejected": -0.21696436405181885,
+      "step": 11430
+    },
+    {
+      "epoch": 1.9710544452102,
+      "grad_norm": 3.800581216812134,
+      "learning_rate": 3.189721450116145e-11,
+      "logits/chosen": -2.8798656463623047,
+      "logits/rejected": -2.868277072906494,
+      "logps/chosen": -70.92523193359375,
+      "logps/rejected": -74.1414794921875,
+      "loss": 0.6797,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.18375080823898315,
+      "rewards/margins": 0.03151247650384903,
+      "rewards/rejected": -0.21526332199573517,
+      "step": 11440
+    },
+    {
+      "epoch": 1.9727773949000689,
+      "grad_norm": 4.323183536529541,
+      "learning_rate": 2.821363471259275e-11,
+      "logits/chosen": -2.803658962249756,
+      "logits/rejected": -2.7815325260162354,
+      "logps/chosen": -70.53925323486328,
+      "logps/rejected": -75.36064910888672,
+      "loss": 0.6667,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.17831102013587952,
+      "rewards/margins": 0.05829024314880371,
+      "rewards/rejected": -0.23660127818584442,
+      "step": 11450
+    },
+    {
+      "epoch": 1.9745003445899378,
+      "grad_norm": 3.5089046955108643,
+      "learning_rate": 2.4755876710905176e-11,
+      "logits/chosen": -2.846515655517578,
+      "logits/rejected": -2.826266050338745,
+      "logps/chosen": -71.52725982666016,
+      "logps/rejected": -76.92378234863281,
+      "loss": 0.6637,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1598942130804062,
+      "rewards/margins": 0.06598251312971115,
+      "rewards/rejected": -0.22587671875953674,
+      "step": 11460
+    },
+    {
+      "epoch": 1.976223294279807,
+      "grad_norm": 4.123968124389648,
+      "learning_rate": 2.1523971764869642e-11,
+      "logits/chosen": -2.8938114643096924,
+      "logits/rejected": -2.8641016483306885,
+      "logps/chosen": -74.74168395996094,
+      "logps/rejected": -75.82394409179688,
+      "loss": 0.6618,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.17899183928966522,
+      "rewards/margins": 0.06946253776550293,
+      "rewards/rejected": -0.24845437705516815,
+      "step": 11470
+    },
+    {
+      "epoch": 1.9779462439696762,
+      "grad_norm": 3.5717272758483887,
+      "learning_rate": 1.851794910085469e-11,
+      "logits/chosen": -2.9332807064056396,
+      "logits/rejected": -2.900256633758545,
+      "logps/chosen": -71.8976058959961,
+      "logps/rejected": -72.39883422851562,
+      "loss": 0.6655,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.15803134441375732,
+      "rewards/margins": 0.06166013330221176,
+      "rewards/rejected": -0.2196914702653885,
+      "step": 11480
+    },
+    {
+      "epoch": 1.9796691936595452,
+      "grad_norm": 3.6061160564422607,
+      "learning_rate": 1.5737835902551733e-11,
+      "logits/chosen": -2.8529391288757324,
+      "logits/rejected": -2.8254666328430176,
+      "logps/chosen": -72.53865814208984,
+      "logps/rejected": -72.86511993408203,
+      "loss": 0.676,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.17847374081611633,
+      "rewards/margins": 0.03899824619293213,
+      "rewards/rejected": -0.21747198700904846,
+      "step": 11490
+    },
+    {
+      "epoch": 1.9813921433494142,
+      "grad_norm": 3.1397452354431152,
+      "learning_rate": 1.3183657310741891e-11,
+      "logits/chosen": -2.9005250930786133,
+      "logits/rejected": -2.8793203830718994,
+      "logps/chosen": -67.98265838623047,
+      "logps/rejected": -72.17521667480469,
+      "loss": 0.6775,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.17694365978240967,
+      "rewards/margins": 0.03691878542304039,
+      "rewards/rejected": -0.21386241912841797,
+      "step": 11500
+    },
+    {
+      "epoch": 1.9813921433494142,
+      "eval_logits/chosen": -2.9624252319335938,
+      "eval_logits/rejected": -2.956608772277832,
+      "eval_logps/chosen": -70.83636474609375,
+      "eval_logps/rejected": -78.12662506103516,
+      "eval_loss": 0.6808305382728577,
+      "eval_rewards/accuracies": 0.6082713603973389,
+      "eval_rewards/chosen": -0.1212446540594101,
+      "eval_rewards/margins": 0.028220284730196,
+      "eval_rewards/rejected": -0.149464949965477,
+      "eval_runtime": 383.0304,
+      "eval_samples_per_second": 11.237,
+      "eval_steps_per_second": 1.405,
+      "step": 11500
+    },
+    {
+      "epoch": 1.9831150930392831,
+      "grad_norm": 4.2021026611328125,
+      "learning_rate": 1.0855436423054531e-11,
+      "logits/chosen": -2.8023393154144287,
+      "logits/rejected": -2.7929940223693848,
+      "logps/chosen": -72.55413055419922,
+      "logps/rejected": -75.46101379394531,
+      "loss": 0.6711,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.17073677480220795,
+      "rewards/margins": 0.05118516832590103,
+      "rewards/rejected": -0.22192195057868958,
+      "step": 11510
+    },
+    {
+      "epoch": 1.9848380427291523,
+      "grad_norm": 3.3203485012054443,
+      "learning_rate": 8.753194293770194e-12,
+      "logits/chosen": -2.9113240242004395,
+      "logits/rejected": -2.85616397857666,
+      "logps/chosen": -74.67838287353516,
+      "logps/rejected": -73.07746887207031,
+      "loss": 0.6555,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.1487974375486374,
+      "rewards/margins": 0.08221492171287537,
+      "rewards/rejected": -0.23101234436035156,
+      "step": 11520
+    },
+    {
+      "epoch": 1.9865609924190215,
+      "grad_norm": 3.6697351932525635,
+      "learning_rate": 6.876949933631859e-12,
+      "logits/chosen": -2.8749098777770996,
+      "logits/rejected": -2.840366840362549,
+      "logps/chosen": -70.08689880371094,
+      "logps/rejected": -71.27586364746094,
+      "loss": 0.6674,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.16625139117240906,
+      "rewards/margins": 0.056402742862701416,
+      "rewards/rejected": -0.22265414893627167,
+      "step": 11530
+    },
+    {
+      "epoch": 1.9882839421088905,
+      "grad_norm": 3.597338914871216,
+      "learning_rate": 5.226720309656207e-12,
+      "logits/chosen": -2.9159908294677734,
+      "logits/rejected": -2.9044508934020996,
+      "logps/chosen": -67.4108657836914,
+      "logps/rejected": -78.33393859863281,
+      "loss": 0.6638,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.16663439571857452,
+      "rewards/margins": 0.06443803012371063,
+      "rewards/rejected": -0.23107245564460754,
+      "step": 11540
+    },
+    {
+      "epoch": 1.9900068917987594,
+      "grad_norm": 3.7232704162597656,
+      "learning_rate": 3.802520345000393e-12,
+      "logits/chosen": -2.8454906940460205,
+      "logits/rejected": -2.8252408504486084,
+      "logps/chosen": -67.67414855957031,
+      "logps/rejected": -74.90187072753906,
+      "loss": 0.6677,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.16344700753688812,
+      "rewards/margins": 0.057278893887996674,
+      "rewards/rejected": -0.2207259237766266,
+      "step": 11550
+    },
+    {
+      "epoch": 1.9917298414886284,
+      "grad_norm": 3.721557378768921,
+      "learning_rate": 2.604362918812164e-12,
+      "logits/chosen": -2.875913619995117,
+      "logits/rejected": -2.8471953868865967,
+      "logps/chosen": -73.41490173339844,
+      "logps/rejected": -74.85926055908203,
+      "loss": 0.6727,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.1843193769454956,
+      "rewards/margins": 0.04646942764520645,
+      "rewards/rejected": -0.23078879714012146,
+      "step": 11560
+    },
+    {
+      "epoch": 1.9934527911784976,
+      "grad_norm": 3.9034640789031982,
+      "learning_rate": 1.6322588661216163e-12,
+      "logits/chosen": -2.8795297145843506,
+      "logits/rejected": -2.8602633476257324,
+      "logps/chosen": -73.44380187988281,
+      "logps/rejected": -79.3753433227539,
+      "loss": 0.6743,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.193327859044075,
+      "rewards/margins": 0.04342865198850632,
+      "rewards/rejected": -0.23675648868083954,
+      "step": 11570
+    },
+    {
+      "epoch": 1.9951757408683668,
+      "grad_norm": 4.1552734375,
+      "learning_rate": 8.862169777440476e-13,
+      "logits/chosen": -2.9075911045074463,
+      "logits/rejected": -2.88792085647583,
+      "logps/chosen": -74.33565521240234,
+      "logps/rejected": -77.86045837402344,
+      "loss": 0.6751,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.1807744801044464,
+      "rewards/margins": 0.04108527675271034,
+      "rewards/rejected": -0.22185973823070526,
+      "step": 11580
+    },
+    {
+      "epoch": 1.9968986905582358,
+      "grad_norm": 4.192768573760986,
+      "learning_rate": 3.662440001883649e-13,
+      "logits/chosen": -2.809661388397217,
+      "logits/rejected": -2.77116060256958,
+      "logps/chosen": -75.36515808105469,
+      "logps/rejected": -72.06199645996094,
+      "loss": 0.6697,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.1586371213197708,
+      "rewards/margins": 0.05176359415054321,
+      "rewards/rejected": -0.21040070056915283,
+      "step": 11590
+    },
+    {
+      "epoch": 1.9986216402481047,
+      "grad_norm": 4.017699718475342,
+      "learning_rate": 7.234463561267556e-14,
+      "logits/chosen": -2.850468158721924,
+      "logits/rejected": -2.8381567001342773,
+      "logps/chosen": -67.05789947509766,
+      "logps/rejected": -75.74360656738281,
+      "loss": 0.6688,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.16608582437038422,
+      "rewards/margins": 0.0535251684486866,
+      "rewards/rejected": -0.2196110039949417,
+      "step": 11600
+    },
+    {
+      "epoch": 1.9986216402481047,
+      "eval_logits/chosen": -2.962392568588257,
+      "eval_logits/rejected": -2.956583023071289,
+      "eval_logps/chosen": -70.84989929199219,
+      "eval_logps/rejected": -78.1531753540039,
+      "eval_loss": 0.6807694435119629,
+      "eval_rewards/accuracies": 0.6089683771133423,
+      "eval_rewards/chosen": -0.12137996405363083,
+      "eval_rewards/margins": 0.028350669890642166,
+      "eval_rewards/rejected": -0.1497306227684021,
+      "eval_runtime": 383.293,
+      "eval_samples_per_second": 11.229,
+      "eval_steps_per_second": 1.404,
+      "step": 11600
+    },
+    {
+      "epoch": 2.0,
+      "step": 11608,
+      "total_flos": 0.0,
+      "train_loss": 0.6777890619945723,
+      "train_runtime": 94641.0715,
+      "train_samples_per_second": 1.962,
+      "train_steps_per_second": 0.123
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 11608,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}