diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,3102 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.994495412844037,
+  "eval_steps": 500,
+  "global_step": 408,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.014678899082568808,
+      "grad_norm": 1.6221877336502075,
+      "learning_rate": 2.439024390243903e-07,
+      "logits/chosen": -0.7747717499732971,
+      "logits/rejected": -2.1629748344421387,
+      "logps/chosen": -248.1531524658203,
+      "logps/rejected": -193.47401428222656,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 2
+    },
+    {
+      "epoch": 0.029357798165137616,
+      "grad_norm": 1.7869738340377808,
+      "learning_rate": 4.878048780487805e-07,
+      "logits/chosen": -0.8566749095916748,
+      "logits/rejected": -2.0849714279174805,
+      "logps/chosen": -272.6571960449219,
+      "logps/rejected": -212.95504760742188,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.004582214169204235,
+      "rewards/margins": 0.005821239203214645,
+      "rewards/rejected": -0.001239025266841054,
+      "step": 4
+    },
+    {
+      "epoch": 0.044036697247706424,
+      "grad_norm": 1.8637501001358032,
+      "learning_rate": 7.317073170731707e-07,
+      "logits/chosen": -1.0140094757080078,
+      "logits/rejected": -2.308026075363159,
+      "logps/chosen": -277.5355529785156,
+      "logps/rejected": -211.09579467773438,
+      "loss": 0.6967,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.0007174371276050806,
+      "rewards/margins": -0.005946946796029806,
+      "rewards/rejected": 0.005229509435594082,
+      "step": 6
+    },
+    {
+      "epoch": 0.05871559633027523,
+      "grad_norm": 1.6607173681259155,
+      "learning_rate": 9.75609756097561e-07,
+      "logits/chosen": -1.1254106760025024,
+      "logits/rejected": -2.3103413581848145,
+      "logps/chosen": -236.63705444335938,
+      "logps/rejected": -172.72019958496094,
+      "loss": 0.6966,
+      "rewards/accuracies": 0.453125,
+      "rewards/chosen": -0.003201799700036645,
+      "rewards/margins": -0.005821935832500458,
+      "rewards/rejected": 0.002620136830955744,
+      "step": 8
+    },
+    {
+      "epoch": 0.07339449541284404,
+      "grad_norm": 1.9114854335784912,
+      "learning_rate": 1.2195121951219514e-06,
+      "logits/chosen": -0.8396108746528625,
+      "logits/rejected": -2.356397867202759,
+      "logps/chosen": -285.3742980957031,
+      "logps/rejected": -167.82122802734375,
+      "loss": 0.6939,
+      "rewards/accuracies": 0.453125,
+      "rewards/chosen": -0.00033334456384181976,
+      "rewards/margins": -0.00014529703184962273,
+      "rewards/rejected": -0.00018804636783897877,
+      "step": 10
+    },
+    {
+      "epoch": 0.08807339449541285,
+      "grad_norm": 2.11832332611084,
+      "learning_rate": 1.4634146341463414e-06,
+      "logits/chosen": -0.8920382857322693,
+      "logits/rejected": -2.228362798690796,
+      "logps/chosen": -321.5919189453125,
+      "logps/rejected": -234.6510467529297,
+      "loss": 0.6939,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.007709708996117115,
+      "rewards/margins": 0.00042582384776324034,
+      "rewards/rejected": -0.008135532028973103,
+      "step": 12
+    },
+    {
+      "epoch": 0.10275229357798166,
+      "grad_norm": 1.8611751794815063,
+      "learning_rate": 1.707317073170732e-06,
+      "logits/chosen": -0.7588200569152832,
+      "logits/rejected": -2.1696934700012207,
+      "logps/chosen": -239.06419372558594,
+      "logps/rejected": -176.63113403320312,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.004243845120072365,
+      "rewards/margins": 0.0036368640139698982,
+      "rewards/rejected": 0.0006069811061024666,
+      "step": 14
+    },
+    {
+      "epoch": 0.11743119266055047,
+      "grad_norm": 1.886559009552002,
+      "learning_rate": 1.951219512195122e-06,
+      "logits/chosen": -0.9555227756500244,
+      "logits/rejected": -2.175183057785034,
+      "logps/chosen": -252.0240478515625,
+      "logps/rejected": -196.8078155517578,
+      "loss": 0.6971,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.009129801765084267,
+      "rewards/margins": -0.006077417638152838,
+      "rewards/rejected": -0.0030523836612701416,
+      "step": 16
+    },
+    {
+      "epoch": 0.13211009174311927,
+      "grad_norm": 2.0312628746032715,
+      "learning_rate": 2.1951219512195125e-06,
+      "logits/chosen": -0.8100669384002686,
+      "logits/rejected": -2.175964832305908,
+      "logps/chosen": -294.4822692871094,
+      "logps/rejected": -208.19273376464844,
+      "loss": 0.6959,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.003589821048080921,
+      "rewards/margins": -0.0039059999398887157,
+      "rewards/rejected": 0.007495820987969637,
+      "step": 18
+    },
+    {
+      "epoch": 0.14678899082568808,
+      "grad_norm": 1.7934964895248413,
+      "learning_rate": 2.4390243902439027e-06,
+      "logits/chosen": -0.8831109404563904,
+      "logits/rejected": -2.3040294647216797,
+      "logps/chosen": -289.34765625,
+      "logps/rejected": -165.73974609375,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.004587325267493725,
+      "rewards/margins": 0.0008857820648699999,
+      "rewards/rejected": 0.003701543901115656,
+      "step": 20
+    },
+    {
+      "epoch": 0.1614678899082569,
+      "grad_norm": 1.853503942489624,
+      "learning_rate": 2.682926829268293e-06,
+      "logits/chosen": -1.0103861093521118,
+      "logits/rejected": -2.400630474090576,
+      "logps/chosen": -279.53070068359375,
+      "logps/rejected": -195.76055908203125,
+      "loss": 0.6847,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.006311780773103237,
+      "rewards/margins": 0.01845557615160942,
+      "rewards/rejected": -0.012143796309828758,
+      "step": 22
+    },
+    {
+      "epoch": 0.1761467889908257,
+      "grad_norm": 1.876775860786438,
+      "learning_rate": 2.926829268292683e-06,
+      "logits/chosen": -0.8252619504928589,
+      "logits/rejected": -2.4449005126953125,
+      "logps/chosen": -349.03436279296875,
+      "logps/rejected": -165.7700958251953,
+      "loss": 0.6828,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": 0.009145505726337433,
+      "rewards/margins": 0.021766357123851776,
+      "rewards/rejected": -0.012620854191482067,
+      "step": 24
+    },
+    {
+      "epoch": 0.1908256880733945,
+      "grad_norm": 1.8004379272460938,
+      "learning_rate": 3.1707317073170736e-06,
+      "logits/chosen": -1.0986889600753784,
+      "logits/rejected": -2.3428730964660645,
+      "logps/chosen": -307.81927490234375,
+      "logps/rejected": -190.91653442382812,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.0025536715984344482,
+      "rewards/margins": 0.014828955754637718,
+      "rewards/rejected": -0.012275285087525845,
+      "step": 26
+    },
+    {
+      "epoch": 0.20550458715596331,
+      "grad_norm": 1.6710703372955322,
+      "learning_rate": 3.414634146341464e-06,
+      "logits/chosen": -1.0080599784851074,
+      "logits/rejected": -2.3728153705596924,
+      "logps/chosen": -268.09893798828125,
+      "logps/rejected": -173.4761199951172,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": 0.009534865617752075,
+      "rewards/margins": 0.010362524539232254,
+      "rewards/rejected": -0.000827658106572926,
+      "step": 28
+    },
+    {
+      "epoch": 0.22018348623853212,
+      "grad_norm": 1.6752780675888062,
+      "learning_rate": 3.6585365853658537e-06,
+      "logits/chosen": -0.8720824718475342,
+      "logits/rejected": -2.2563316822052,
+      "logps/chosen": -272.8302001953125,
+      "logps/rejected": -207.05535888671875,
+      "loss": 0.6955,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.008826537057757378,
+      "rewards/margins": -0.0032597517129033804,
+      "rewards/rejected": -0.005566785112023354,
+      "step": 30
+    },
+    {
+      "epoch": 0.23486238532110093,
+      "grad_norm": 1.8804222345352173,
+      "learning_rate": 3.902439024390244e-06,
+      "logits/chosen": -0.9830515384674072,
+      "logits/rejected": -2.316348075866699,
+      "logps/chosen": -269.2813415527344,
+      "logps/rejected": -172.68609619140625,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.004377659410238266,
+      "rewards/margins": 0.005325704347342253,
+      "rewards/rejected": -0.009703361429274082,
+      "step": 32
+    },
+    {
+      "epoch": 0.24954128440366974,
+      "grad_norm": 2.0692217350006104,
+      "learning_rate": 4.146341463414634e-06,
+      "logits/chosen": -0.969600260257721,
+      "logits/rejected": -2.3533740043640137,
+      "logps/chosen": -307.40386962890625,
+      "logps/rejected": -173.79537963867188,
+      "loss": 0.6841,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": 0.010469496250152588,
+      "rewards/margins": 0.020308587700128555,
+      "rewards/rejected": -0.009839091449975967,
+      "step": 34
+    },
+    {
+      "epoch": 0.26422018348623855,
+      "grad_norm": 1.9420565366744995,
+      "learning_rate": 4.390243902439025e-06,
+      "logits/chosen": -1.0136321783065796,
+      "logits/rejected": -2.2236032485961914,
+      "logps/chosen": -293.4641418457031,
+      "logps/rejected": -213.1688232421875,
+      "loss": 0.68,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": 0.019286828115582466,
+      "rewards/margins": 0.028108475729823112,
+      "rewards/rejected": -0.008821647614240646,
+      "step": 36
+    },
+    {
+      "epoch": 0.27889908256880735,
+      "grad_norm": 1.8155481815338135,
+      "learning_rate": 4.634146341463416e-06,
+      "logits/chosen": -1.0221481323242188,
+      "logits/rejected": -2.349212884902954,
+      "logps/chosen": -273.8066101074219,
+      "logps/rejected": -209.1209259033203,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.546875,
+      "rewards/chosen": -0.0073402514681220055,
+      "rewards/margins": 0.016111046075820923,
+      "rewards/rejected": -0.023451298475265503,
+      "step": 38
+    },
+    {
+      "epoch": 0.29357798165137616,
+      "grad_norm": 1.9104901552200317,
+      "learning_rate": 4.8780487804878055e-06,
+      "logits/chosen": -0.8466785550117493,
+      "logits/rejected": -2.214355945587158,
+      "logps/chosen": -325.6095275878906,
+      "logps/rejected": -189.9873046875,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.00729791447520256,
+      "rewards/margins": 0.011406856589019299,
+      "rewards/rejected": -0.018704771995544434,
+      "step": 40
+    },
+    {
+      "epoch": 0.30825688073394497,
+      "grad_norm": 1.8166546821594238,
+      "learning_rate": 4.999908404322799e-06,
+      "logits/chosen": -0.8763958215713501,
+      "logits/rejected": -2.4433987140655518,
+      "logps/chosen": -300.2408752441406,
+      "logps/rejected": -182.14207458496094,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.0032976572401821613,
+      "rewards/margins": 0.028552744537591934,
+      "rewards/rejected": -0.02525508403778076,
+      "step": 42
+    },
+    {
+      "epoch": 0.3229357798165138,
+      "grad_norm": 1.839516043663025,
+      "learning_rate": 4.999175679175577e-06,
+      "logits/chosen": -0.9340443015098572,
+      "logits/rejected": -2.2553768157958984,
+      "logps/chosen": -234.7823486328125,
+      "logps/rejected": -172.17637634277344,
+      "loss": 0.673,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": 0.006214863620698452,
+      "rewards/margins": 0.04187798872590065,
+      "rewards/rejected": -0.03566312417387962,
+      "step": 44
+    },
+    {
+      "epoch": 0.3376146788990826,
+      "grad_norm": 1.9286494255065918,
+      "learning_rate": 4.997710443643461e-06,
+      "logits/chosen": -0.9604765772819519,
+      "logits/rejected": -2.187485694885254,
+      "logps/chosen": -242.36959838867188,
+      "logps/rejected": -215.9381103515625,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.011644757352769375,
+      "rewards/margins": 0.019570371136069298,
+      "rewards/rejected": -0.0312151238322258,
+      "step": 46
+    },
+    {
+      "epoch": 0.3522935779816514,
+      "grad_norm": 2.0474560260772705,
+      "learning_rate": 4.995513127188151e-06,
+      "logits/chosen": -0.9198620319366455,
+      "logits/rejected": -2.353731155395508,
+      "logps/chosen": -342.00421142578125,
+      "logps/rejected": -190.65626525878906,
+      "loss": 0.6615,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.014831364154815674,
+      "rewards/margins": 0.0657879188656807,
+      "rewards/rejected": -0.05095655843615532,
+      "step": 48
+    },
+    {
+      "epoch": 0.3669724770642202,
+      "grad_norm": 2.0733015537261963,
+      "learning_rate": 4.992584373844853e-06,
+      "logits/chosen": -1.0313482284545898,
+      "logits/rejected": -2.189700126647949,
+      "logps/chosen": -327.1130065917969,
+      "logps/rejected": -193.40577697753906,
+      "loss": 0.6693,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": 0.002555199433118105,
+      "rewards/margins": 0.04951968789100647,
+      "rewards/rejected": -0.04696448892354965,
+      "step": 50
+    },
+    {
+      "epoch": 0.381651376146789,
+      "grad_norm": 1.9505176544189453,
+      "learning_rate": 4.98892504203351e-06,
+      "logits/chosen": -1.071111798286438,
+      "logits/rejected": -2.2670626640319824,
+      "logps/chosen": -263.16058349609375,
+      "logps/rejected": -167.900390625,
+      "loss": 0.6679,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": 0.0007026728708297014,
+      "rewards/margins": 0.052561890333890915,
+      "rewards/rejected": -0.05185921862721443,
+      "step": 52
+    },
+    {
+      "epoch": 0.3963302752293578,
+      "grad_norm": 1.940078854560852,
+      "learning_rate": 4.9845362043071925e-06,
+      "logits/chosen": -0.8662849068641663,
+      "logits/rejected": -2.2801122665405273,
+      "logps/chosen": -272.940673828125,
+      "logps/rejected": -171.84495544433594,
+      "loss": 0.6581,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.007315092254430056,
+      "rewards/margins": 0.07306656986474991,
+      "rewards/rejected": -0.08038166165351868,
+      "step": 54
+    },
+    {
+      "epoch": 0.41100917431192663,
+      "grad_norm": 2.0381312370300293,
+      "learning_rate": 4.97941914703774e-06,
+      "logits/chosen": -0.9563165903091431,
+      "logits/rejected": -2.2946839332580566,
+      "logps/chosen": -269.4028015136719,
+      "logps/rejected": -211.50933837890625,
+      "loss": 0.6426,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": 0.006978507619351149,
+      "rewards/margins": 0.10549122840166092,
+      "rewards/rejected": -0.09851271659135818,
+      "step": 56
+    },
+    {
+      "epoch": 0.42568807339449544,
+      "grad_norm": 2.180021286010742,
+      "learning_rate": 4.973575370038718e-06,
+      "logits/chosen": -0.8351713418960571,
+      "logits/rejected": -2.1809074878692627,
+      "logps/chosen": -285.6255798339844,
+      "logps/rejected": -202.8754119873047,
+      "loss": 0.6367,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.02219582349061966,
+      "rewards/margins": 0.11899632215499878,
+      "rewards/rejected": -0.09680050611495972,
+      "step": 58
+    },
+    {
+      "epoch": 0.44036697247706424,
+      "grad_norm": 2.023611545562744,
+      "learning_rate": 4.967006586125827e-06,
+      "logits/chosen": -1.099950909614563,
+      "logits/rejected": -2.2148141860961914,
+      "logps/chosen": -283.2462158203125,
+      "logps/rejected": -195.86944580078125,
+      "loss": 0.6195,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": 0.03062710538506508,
+      "rewards/margins": 0.15735232830047607,
+      "rewards/rejected": -0.1267252117395401,
+      "step": 60
+    },
+    {
+      "epoch": 0.45504587155963305,
+      "grad_norm": 2.1152608394622803,
+      "learning_rate": 4.959714720614871e-06,
+      "logits/chosen": -0.9920430183410645,
+      "logits/rejected": -2.3323516845703125,
+      "logps/chosen": -300.185791015625,
+      "logps/rejected": -194.13824462890625,
+      "loss": 0.6116,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.019140686839818954,
+      "rewards/margins": 0.1737564355134964,
+      "rewards/rejected": -0.15461575984954834,
+      "step": 62
+    },
+    {
+      "epoch": 0.46972477064220186,
+      "grad_norm": 2.0386292934417725,
+      "learning_rate": 4.951701910757446e-06,
+      "logits/chosen": -0.9583941102027893,
+      "logits/rejected": -2.1204147338867188,
+      "logps/chosen": -236.59213256835938,
+      "logps/rejected": -199.07980346679688,
+      "loss": 0.6071,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.02290705032646656,
+      "rewards/margins": 0.18515917658805847,
+      "rewards/rejected": -0.16225211322307587,
+      "step": 64
+    },
+    {
+      "epoch": 0.48440366972477067,
+      "grad_norm": 2.033529043197632,
+      "learning_rate": 4.942970505114514e-06,
+      "logits/chosen": -0.8373759388923645,
+      "logits/rejected": -2.231731414794922,
+      "logps/chosen": -290.7356262207031,
+      "logps/rejected": -188.03863525390625,
+      "loss": 0.5823,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.026615679264068604,
+      "rewards/margins": 0.24059633910655975,
+      "rewards/rejected": -0.21398067474365234,
+      "step": 66
+    },
+    {
+      "epoch": 0.4990825688073395,
+      "grad_norm": 1.8862334489822388,
+      "learning_rate": 4.933523062868033e-06,
+      "logits/chosen": -0.8499428629875183,
+      "logits/rejected": -2.3141591548919678,
+      "logps/chosen": -253.00765991210938,
+      "logps/rejected": -174.625732421875,
+      "loss": 0.5955,
+      "rewards/accuracies": 0.921875,
+      "rewards/chosen": -0.006930825300514698,
+      "rewards/margins": 0.21197618544101715,
+      "rewards/rejected": -0.21890701353549957,
+      "step": 68
+    },
+    {
+      "epoch": 0.5137614678899083,
+      "grad_norm": 2.032834053039551,
+      "learning_rate": 4.923362353070859e-06,
+      "logits/chosen": -0.6541658639907837,
+      "logits/rejected": -2.2294771671295166,
+      "logps/chosen": -271.1413879394531,
+      "logps/rejected": -170.8126220703125,
+      "loss": 0.5739,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.015698038041591644,
+      "rewards/margins": 0.261942982673645,
+      "rewards/rejected": -0.2462449073791504,
+      "step": 70
+    },
+    {
+      "epoch": 0.5284403669724771,
+      "grad_norm": 1.8714652061462402,
+      "learning_rate": 4.912491353835138e-06,
+      "logits/chosen": -0.9249717593193054,
+      "logits/rejected": -2.170142650604248,
+      "logps/chosen": -245.40841674804688,
+      "logps/rejected": -196.6009521484375,
+      "loss": 0.5615,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": 0.004103909246623516,
+      "rewards/margins": 0.29537737369537354,
+      "rewards/rejected": -0.29127344489097595,
+      "step": 72
+    },
+    {
+      "epoch": 0.5431192660550459,
+      "grad_norm": 1.8837171792984009,
+      "learning_rate": 4.900913251459418e-06,
+      "logits/chosen": -0.8780805468559265,
+      "logits/rejected": -2.149808406829834,
+      "logps/chosen": -252.37844848632812,
+      "logps/rejected": -185.75148010253906,
+      "loss": 0.5552,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": -0.003393257036805153,
+      "rewards/margins": 0.3080388009548187,
+      "rewards/rejected": -0.3114320635795593,
+      "step": 74
+    },
+    {
+      "epoch": 0.5577981651376147,
+      "grad_norm": 1.8990751504898071,
+      "learning_rate": 4.8886314394947396e-06,
+      "logits/chosen": -0.7858492732048035,
+      "logits/rejected": -2.1859428882598877,
+      "logps/chosen": -260.1080322265625,
+      "logps/rejected": -198.69061279296875,
+      "loss": 0.5215,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.011569229885935783,
+      "rewards/margins": 0.3970867097377777,
+      "rewards/rejected": -0.3855174779891968,
+      "step": 76
+    },
+    {
+      "epoch": 0.5724770642201835,
+      "grad_norm": 1.9948140382766724,
+      "learning_rate": 4.875649517749985e-06,
+      "logits/chosen": -0.8377081751823425,
+      "logits/rejected": -2.2524642944335938,
+      "logps/chosen": -267.073486328125,
+      "logps/rejected": -202.4425811767578,
+      "loss": 0.5145,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.0006968197412788868,
+      "rewards/margins": 0.4172001779079437,
+      "rewards/rejected": -0.41650334000587463,
+      "step": 78
+    },
+    {
+      "epoch": 0.5871559633027523,
+      "grad_norm": 1.8098971843719482,
+      "learning_rate": 4.861971291236772e-06,
+      "logits/chosen": -0.957794725894928,
+      "logits/rejected": -2.141266345977783,
+      "logps/chosen": -311.6953430175781,
+      "logps/rejected": -200.5380859375,
+      "loss": 0.53,
+      "rewards/accuracies": 0.890625,
+      "rewards/chosen": 0.013315271586179733,
+      "rewards/margins": 0.3864455223083496,
+      "rewards/rejected": -0.3731302320957184,
+      "step": 80
+    },
+    {
+      "epoch": 0.6018348623853211,
+      "grad_norm": 1.9890069961547852,
+      "learning_rate": 4.847600769054201e-06,
+      "logits/chosen": -0.8957269191741943,
+      "logits/rejected": -2.102496385574341,
+      "logps/chosen": -346.88726806640625,
+      "logps/rejected": -232.16412353515625,
+      "loss": 0.5022,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": 0.019277315586805344,
+      "rewards/margins": 0.4478000998497009,
+      "rewards/rejected": -0.4285227954387665,
+      "step": 82
+    },
+    {
+      "epoch": 0.6165137614678899,
+      "grad_norm": 1.801570177078247,
+      "learning_rate": 4.832542163213787e-06,
+      "logits/chosen": -0.7643452882766724,
+      "logits/rejected": -2.177706718444824,
+      "logps/chosen": -246.15560913085938,
+      "logps/rejected": -167.4416046142578,
+      "loss": 0.4934,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.00458180159330368,
+      "rewards/margins": 0.48400723934173584,
+      "rewards/rejected": -0.4885890483856201,
+      "step": 84
+    },
+    {
+      "epoch": 0.6311926605504588,
+      "grad_norm": 1.8030428886413574,
+      "learning_rate": 4.816799887404911e-06,
+      "logits/chosen": -0.9738877415657043,
+      "logits/rejected": -2.146649122238159,
+      "logps/chosen": -286.7640686035156,
+      "logps/rejected": -197.47161865234375,
+      "loss": 0.4836,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 0.009418713860213757,
+      "rewards/margins": 0.5085384845733643,
+      "rewards/rejected": -0.4991198182106018,
+      "step": 86
+    },
+    {
+      "epoch": 0.6458715596330276,
+      "grad_norm": 1.814665675163269,
+      "learning_rate": 4.800378555701168e-06,
+      "logits/chosen": -0.7987996339797974,
+      "logits/rejected": -2.0137877464294434,
+      "logps/chosen": -339.0395202636719,
+      "logps/rejected": -201.66848754882812,
+      "loss": 0.4407,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.0017917649820446968,
+      "rewards/margins": 0.6508389115333557,
+      "rewards/rejected": -0.6526306867599487,
+      "step": 88
+    },
+    {
+      "epoch": 0.6605504587155964,
+      "grad_norm": 1.9332906007766724,
+      "learning_rate": 4.783282981207979e-06,
+      "logits/chosen": -0.8255445957183838,
+      "logits/rejected": -2.3335933685302734,
+      "logps/chosen": -282.8710632324219,
+      "logps/rejected": -184.0906524658203,
+      "loss": 0.4509,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.0369749516248703,
+      "rewards/margins": 0.6082916855812073,
+      "rewards/rejected": -0.6452666521072388,
+      "step": 90
+    },
+    {
+      "epoch": 0.6752293577981652,
+      "grad_norm": 1.6867598295211792,
+      "learning_rate": 4.765518174651864e-06,
+      "logits/chosen": -0.8639607429504395,
+      "logits/rejected": -2.074816942214966,
+      "logps/chosen": -271.90521240234375,
+      "logps/rejected": -205.10400390625,
+      "loss": 0.4227,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.017826007679104805,
+      "rewards/margins": 0.6985148787498474,
+      "rewards/rejected": -0.7163408398628235,
+      "step": 92
+    },
+    {
+      "epoch": 0.689908256880734,
+      "grad_norm": 1.8225523233413696,
+      "learning_rate": 4.747089342911793e-06,
+      "logits/chosen": -0.652176022529602,
+      "logits/rejected": -2.121016502380371,
+      "logps/chosen": -276.7890930175781,
+      "logps/rejected": -188.45567321777344,
+      "loss": 0.421,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.014648550190031528,
+      "rewards/margins": 0.7090162038803101,
+      "rewards/rejected": -0.7236647605895996,
+      "step": 94
+    },
+    {
+      "epoch": 0.7045871559633028,
+      "grad_norm": 1.730716586112976,
+      "learning_rate": 4.728001887493048e-06,
+      "logits/chosen": -0.7071309685707092,
+      "logits/rejected": -2.1385228633880615,
+      "logps/chosen": -284.6495056152344,
+      "logps/rejected": -207.17828369140625,
+      "loss": 0.4037,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.011353883892297745,
+      "rewards/margins": 0.7462868690490723,
+      "rewards/rejected": -0.7576407790184021,
+      "step": 96
+    },
+    {
+      "epoch": 0.7192660550458716,
+      "grad_norm": 1.7052290439605713,
+      "learning_rate": 4.708261402944036e-06,
+      "logits/chosen": -0.8458466529846191,
+      "logits/rejected": -2.128801107406616,
+      "logps/chosen": -300.57489013671875,
+      "logps/rejected": -204.04148864746094,
+      "loss": 0.3953,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": -0.1128288209438324,
+      "rewards/margins": 0.834511399269104,
+      "rewards/rejected": -0.9473402500152588,
+      "step": 98
+    },
+    {
+      "epoch": 0.7339449541284404,
+      "grad_norm": 1.7650856971740723,
+      "learning_rate": 4.687873675216522e-06,
+      "logits/chosen": -0.694564700126648,
+      "logits/rejected": -1.9271796941757202,
+      "logps/chosen": -287.0938415527344,
+      "logps/rejected": -212.9856414794922,
+      "loss": 0.3808,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.029861731454730034,
+      "rewards/margins": 0.8324739933013916,
+      "rewards/rejected": -0.862335741519928,
+      "step": 100
+    },
+    {
+      "epoch": 0.7486238532110092,
+      "grad_norm": 1.5149580240249634,
+      "learning_rate": 4.666844679969765e-06,
+      "logits/chosen": -0.9843853116035461,
+      "logits/rejected": -2.171410083770752,
+      "logps/chosen": -286.80181884765625,
+      "logps/rejected": -227.1826629638672,
+      "loss": 0.3233,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.03178291767835617,
+      "rewards/margins": 1.0707846879959106,
+      "rewards/rejected": -1.1025675535202026,
+      "step": 102
+    },
+    {
+      "epoch": 0.763302752293578,
+      "grad_norm": 1.7084800004959106,
+      "learning_rate": 4.6451805808190464e-06,
+      "logits/chosen": -0.7646867632865906,
+      "logits/rejected": -2.0860140323638916,
+      "logps/chosen": -272.03631591796875,
+      "logps/rejected": -193.9704132080078,
+      "loss": 0.3502,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.09451958537101746,
+      "rewards/margins": 0.9896395206451416,
+      "rewards/rejected": -1.0841591358184814,
+      "step": 104
+    },
+    {
+      "epoch": 0.7779816513761468,
+      "grad_norm": 1.6702779531478882,
+      "learning_rate": 4.622887727529104e-06,
+      "logits/chosen": -0.7461971044540405,
+      "logits/rejected": -2.0296335220336914,
+      "logps/chosen": -245.9069061279297,
+      "logps/rejected": -227.5930633544922,
+      "loss": 0.3127,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": -0.1017722487449646,
+      "rewards/margins": 1.1482524871826172,
+      "rewards/rejected": -1.2500247955322266,
+      "step": 106
+    },
+    {
+      "epoch": 0.7926605504587156,
+      "grad_norm": 1.4205806255340576,
+      "learning_rate": 4.599972654153018e-06,
+      "logits/chosen": -0.6252501606941223,
+      "logits/rejected": -2.0300350189208984,
+      "logps/chosen": -287.1758728027344,
+      "logps/rejected": -191.18710327148438,
+      "loss": 0.303,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.03701897710561752,
+      "rewards/margins": 1.180790901184082,
+      "rewards/rejected": -1.217809796333313,
+      "step": 108
+    },
+    {
+      "epoch": 0.8073394495412844,
+      "grad_norm": 1.5258698463439941,
+      "learning_rate": 4.5764420771170735e-06,
+      "logits/chosen": -0.6961276531219482,
+      "logits/rejected": -2.080204725265503,
+      "logps/chosen": -265.47674560546875,
+      "logps/rejected": -210.6199493408203,
+      "loss": 0.317,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.10400635749101639,
+      "rewards/margins": 1.1012389659881592,
+      "rewards/rejected": -1.2052453756332397,
+      "step": 110
+    },
+    {
+      "epoch": 0.8220183486238533,
+      "grad_norm": 1.397694706916809,
+      "learning_rate": 4.552302893252166e-06,
+      "logits/chosen": -0.9548951387405396,
+      "logits/rejected": -2.0966246128082275,
+      "logps/chosen": -294.2589111328125,
+      "logps/rejected": -223.81103515625,
+      "loss": 0.2781,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0296751968562603,
+      "rewards/margins": 1.2581570148468018,
+      "rewards/rejected": -1.287832260131836,
+      "step": 112
+    },
+    {
+      "epoch": 0.8366972477064221,
+      "grad_norm": 1.4720474481582642,
+      "learning_rate": 4.52756217777234e-06,
+      "logits/chosen": -0.9968761801719666,
+      "logits/rejected": -2.1294007301330566,
+      "logps/chosen": -298.44085693359375,
+      "logps/rejected": -224.59336853027344,
+      "loss": 0.3122,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.12370166927576065,
+      "rewards/margins": 1.1629935503005981,
+      "rewards/rejected": -1.286695122718811,
+      "step": 114
+    },
+    {
+      "epoch": 0.8513761467889909,
+      "grad_norm": 1.354582667350769,
+      "learning_rate": 4.502227182201035e-06,
+      "logits/chosen": -0.6440415382385254,
+      "logits/rejected": -1.8898322582244873,
+      "logps/chosen": -253.26058959960938,
+      "logps/rejected": -196.10592651367188,
+      "loss": 0.2555,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.07564966380596161,
+      "rewards/margins": 1.4416813850402832,
+      "rewards/rejected": -1.5173311233520508,
+      "step": 116
+    },
+    {
+      "epoch": 0.8660550458715597,
+      "grad_norm": 1.3083575963974,
+      "learning_rate": 4.476305332245662e-06,
+      "logits/chosen": -0.7695977687835693,
+      "logits/rejected": -2.1613001823425293,
+      "logps/chosen": -302.2278137207031,
+      "logps/rejected": -171.18521118164062,
+      "loss": 0.2852,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.09552773833274841,
+      "rewards/margins": 1.2843879461288452,
+      "rewards/rejected": -1.379915475845337,
+      "step": 118
+    },
+    {
+      "epoch": 0.8807339449541285,
+      "grad_norm": 1.4233521223068237,
+      "learning_rate": 4.449804225621116e-06,
+      "logits/chosen": -0.769046425819397,
+      "logits/rejected": -2.0579347610473633,
+      "logps/chosen": -268.5230407714844,
+      "logps/rejected": -199.9818572998047,
+      "loss": 0.273,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": -0.09396351873874664,
+      "rewards/margins": 1.3788070678710938,
+      "rewards/rejected": -1.4727706909179688,
+      "step": 120
+    },
+    {
+      "epoch": 0.8954128440366973,
+      "grad_norm": 1.5535210371017456,
+      "learning_rate": 4.422731629822887e-06,
+      "logits/chosen": -0.6471429467201233,
+      "logits/rejected": -1.874241828918457,
+      "logps/chosen": -303.5356750488281,
+      "logps/rejected": -214.80975341796875,
+      "loss": 0.2744,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.19331903755664825,
+      "rewards/margins": 1.402610182762146,
+      "rewards/rejected": -1.5959291458129883,
+      "step": 122
+    },
+    {
+      "epoch": 0.9100917431192661,
+      "grad_norm": 1.4141969680786133,
+      "learning_rate": 4.395095479850396e-06,
+      "logits/chosen": -0.7038341760635376,
+      "logits/rejected": -1.7795767784118652,
+      "logps/chosen": -278.2030334472656,
+      "logps/rejected": -206.88665771484375,
+      "loss": 0.298,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.1966189742088318,
+      "rewards/margins": 1.306958794593811,
+      "rewards/rejected": -1.503577709197998,
+      "step": 124
+    },
+    {
+      "epoch": 0.9247706422018349,
+      "grad_norm": 1.2042831182479858,
+      "learning_rate": 4.366903875881243e-06,
+      "logits/chosen": -0.7927061319351196,
+      "logits/rejected": -2.196439743041992,
+      "logps/chosen": -264.9695129394531,
+      "logps/rejected": -187.66848754882812,
+      "loss": 0.2327,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.16811293363571167,
+      "rewards/margins": 1.5934040546417236,
+      "rewards/rejected": -1.76151704788208,
+      "step": 126
+    },
+    {
+      "epoch": 0.9394495412844037,
+      "grad_norm": 1.2989264726638794,
+      "learning_rate": 4.3381650808970365e-06,
+      "logits/chosen": -0.7110509872436523,
+      "logits/rejected": -1.7662936449050903,
+      "logps/chosen": -244.37979125976562,
+      "logps/rejected": -208.79306030273438,
+      "loss": 0.2276,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.14624029397964478,
+      "rewards/margins": 1.6342490911483765,
+      "rewards/rejected": -1.7804893255233765,
+      "step": 128
+    },
+    {
+      "epoch": 0.9541284403669725,
+      "grad_norm": 1.248761773109436,
+      "learning_rate": 4.308887518261507e-06,
+      "logits/chosen": -0.5622550249099731,
+      "logits/rejected": -1.7699952125549316,
+      "logps/chosen": -268.9392395019531,
+      "logps/rejected": -218.73167419433594,
+      "loss": 0.2147,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.11277558654546738,
+      "rewards/margins": 1.7473864555358887,
+      "rewards/rejected": -1.8601620197296143,
+      "step": 130
+    },
+    {
+      "epoch": 0.9688073394495413,
+      "grad_norm": 1.3512104749679565,
+      "learning_rate": 4.279079769251617e-06,
+      "logits/chosen": -0.8375473618507385,
+      "logits/rejected": -2.039938449859619,
+      "logps/chosen": -341.29595947265625,
+      "logps/rejected": -234.00616455078125,
+      "loss": 0.2101,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.11729223281145096,
+      "rewards/margins": 1.6870366334915161,
+      "rewards/rejected": -1.8043287992477417,
+      "step": 132
+    },
+    {
+      "epoch": 0.9834862385321101,
+      "grad_norm": 1.2423762083053589,
+      "learning_rate": 4.248750570542373e-06,
+      "logits/chosen": -0.7437604069709778,
+      "logits/rejected": -1.9573638439178467,
+      "logps/chosen": -260.789794921875,
+      "logps/rejected": -202.69984436035156,
+      "loss": 0.2094,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.1173069104552269,
+      "rewards/margins": 1.8479876518249512,
+      "rewards/rejected": -1.965294599533081,
+      "step": 134
+    },
+    {
+      "epoch": 0.998165137614679,
+      "grad_norm": 1.075205683708191,
+      "learning_rate": 4.21790881164611e-06,
+      "logits/chosen": -0.6165802478790283,
+      "logits/rejected": -1.9370815753936768,
+      "logps/chosen": -272.431884765625,
+      "logps/rejected": -222.2723388671875,
+      "loss": 0.1783,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.1736324280500412,
+      "rewards/margins": 2.0683982372283936,
+      "rewards/rejected": -2.2420308589935303,
+      "step": 136
+    },
+    {
+      "epoch": 1.0128440366972478,
+      "grad_norm": 1.2893800735473633,
+      "learning_rate": 4.186563532306957e-06,
+      "logits/chosen": -0.6493566632270813,
+      "logits/rejected": -1.9289031028747559,
+      "logps/chosen": -281.0344543457031,
+      "logps/rejected": -191.85787963867188,
+      "loss": 0.2379,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.31633660197257996,
+      "rewards/margins": 1.6654047966003418,
+      "rewards/rejected": -1.9817413091659546,
+      "step": 138
+    },
+    {
+      "epoch": 1.0275229357798166,
+      "grad_norm": 1.2571147680282593,
+      "learning_rate": 4.154723919851291e-06,
+      "logits/chosen": -0.8061354160308838,
+      "logits/rejected": -1.9505449533462524,
+      "logps/chosen": -283.486083984375,
+      "logps/rejected": -197.98060607910156,
+      "loss": 0.2287,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.296590119600296,
+      "rewards/margins": 1.6940876245498657,
+      "rewards/rejected": -1.990677833557129,
+      "step": 140
+    },
+    {
+      "epoch": 1.0422018348623854,
+      "grad_norm": 1.1618239879608154,
+      "learning_rate": 4.122399306494918e-06,
+      "logits/chosen": -0.8136368989944458,
+      "logits/rejected": -2.0863683223724365,
+      "logps/chosen": -327.2650146484375,
+      "logps/rejected": -223.3490753173828,
+      "loss": 0.2024,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.327208012342453,
+      "rewards/margins": 1.8999422788619995,
+      "rewards/rejected": -2.2271502017974854,
+      "step": 142
+    },
+    {
+      "epoch": 1.0568807339449542,
+      "grad_norm": 1.1636958122253418,
+      "learning_rate": 4.089599166607794e-06,
+      "logits/chosen": -0.7669267058372498,
+      "logits/rejected": -1.9041523933410645,
+      "logps/chosen": -284.34210205078125,
+      "logps/rejected": -217.7493133544922,
+      "loss": 0.1682,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3707123100757599,
+      "rewards/margins": 2.189767360687256,
+      "rewards/rejected": -2.5604796409606934,
+      "step": 144
+    },
+    {
+      "epoch": 1.071559633027523,
+      "grad_norm": 1.1475207805633545,
+      "learning_rate": 4.05633311393708e-06,
+      "logits/chosen": -0.60993492603302,
+      "logits/rejected": -1.7652596235275269,
+      "logps/chosen": -250.69619750976562,
+      "logps/rejected": -198.53050231933594,
+      "loss": 0.1732,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.399620920419693,
+      "rewards/margins": 2.0180652141571045,
+      "rewards/rejected": -2.4176859855651855,
+      "step": 146
+    },
+    {
+      "epoch": 1.0862385321100918,
+      "grad_norm": 1.0832470655441284,
+      "learning_rate": 4.022610898789349e-06,
+      "logits/chosen": -0.6742774844169617,
+      "logits/rejected": -1.9231276512145996,
+      "logps/chosen": -258.0038757324219,
+      "logps/rejected": -214.44337463378906,
+      "loss": 0.1719,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.2988199293613434,
+      "rewards/margins": 2.191661834716797,
+      "rewards/rejected": -2.4904818534851074,
+      "step": 148
+    },
+    {
+      "epoch": 1.1009174311926606,
+      "grad_norm": 1.4428913593292236,
+      "learning_rate": 3.988442405172755e-06,
+      "logits/chosen": -0.49406272172927856,
+      "logits/rejected": -1.771551489830017,
+      "logps/chosen": -272.45654296875,
+      "logps/rejected": -232.3162841796875,
+      "loss": 0.1474,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.2635681927204132,
+      "rewards/margins": 2.3189291954040527,
+      "rewards/rejected": -2.5824971199035645,
+      "step": 150
+    },
+    {
+      "epoch": 1.1155963302752294,
+      "grad_norm": 0.7493584752082825,
+      "learning_rate": 3.953837647900031e-06,
+      "logits/chosen": -0.6438045501708984,
+      "logits/rejected": -1.8923343420028687,
+      "logps/chosen": -263.40106201171875,
+      "logps/rejected": -226.0013427734375,
+      "loss": 0.1388,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.19538956880569458,
+      "rewards/margins": 2.50761342048645,
+      "rewards/rejected": -2.703003168106079,
+      "step": 152
+    },
+    {
+      "epoch": 1.1302752293577982,
+      "grad_norm": 1.5492652654647827,
+      "learning_rate": 3.918806769653135e-06,
+      "logits/chosen": -0.5117900371551514,
+      "logits/rejected": -1.775586724281311,
+      "logps/chosen": -309.75885009765625,
+      "logps/rejected": -225.79420471191406,
+      "loss": 0.2011,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.4026896357536316,
+      "rewards/margins": 2.218205451965332,
+      "rewards/rejected": -2.6208949089050293,
+      "step": 154
+    },
+    {
+      "epoch": 1.144954128440367,
+      "grad_norm": 1.2843326330184937,
+      "learning_rate": 3.88336003801042e-06,
+      "logits/chosen": -0.6652198433876038,
+      "logits/rejected": -1.7710597515106201,
+      "logps/chosen": -246.24229431152344,
+      "logps/rejected": -204.2578125,
+      "loss": 0.1966,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": -0.3051142990589142,
+      "rewards/margins": 2.153491497039795,
+      "rewards/rejected": -2.4586057662963867,
+      "step": 156
+    },
+    {
+      "epoch": 1.1596330275229358,
+      "grad_norm": 0.8037837147712708,
+      "learning_rate": 3.847507842437205e-06,
+      "logits/chosen": -0.5658106803894043,
+      "logits/rejected": -1.8227128982543945,
+      "logps/chosen": -290.1756286621094,
+      "logps/rejected": -201.77011108398438,
+      "loss": 0.1418,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3708067536354065,
+      "rewards/margins": 2.446641206741333,
+      "rewards/rejected": -2.8174479007720947,
+      "step": 158
+    },
+    {
+      "epoch": 1.1743119266055047,
+      "grad_norm": 1.2949639558792114,
+      "learning_rate": 3.811260691240604e-06,
+      "logits/chosen": -0.5217835903167725,
+      "logits/rejected": -1.8691177368164062,
+      "logps/chosen": -331.89788818359375,
+      "logps/rejected": -220.96426391601562,
+      "loss": 0.1652,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.45729154348373413,
+      "rewards/margins": 2.46223783493042,
+      "rewards/rejected": -2.9195291996002197,
+      "step": 160
+    },
+    {
+      "epoch": 1.1889908256880735,
+      "grad_norm": 1.2344180345535278,
+      "learning_rate": 3.774629208489547e-06,
+      "logits/chosen": -0.631909966468811,
+      "logits/rejected": -1.9337414503097534,
+      "logps/chosen": -235.50601196289062,
+      "logps/rejected": -202.8614044189453,
+      "loss": 0.1554,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.4534618854522705,
+      "rewards/margins": 2.3637375831604004,
+      "rewards/rejected": -2.817199468612671,
+      "step": 162
+    },
+    {
+      "epoch": 1.2036697247706423,
+      "grad_norm": 0.9420382976531982,
+      "learning_rate": 3.7376241309008433e-06,
+      "logits/chosen": -0.780289888381958,
+      "logits/rejected": -1.9408084154129028,
+      "logps/chosen": -317.29364013671875,
+      "logps/rejected": -214.83853149414062,
+      "loss": 0.1339,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.27944618463516235,
+      "rewards/margins": 2.5412490367889404,
+      "rewards/rejected": -2.820695400238037,
+      "step": 164
+    },
+    {
+      "epoch": 1.218348623853211,
+      "grad_norm": 0.9693976640701294,
+      "learning_rate": 3.7002563046922502e-06,
+      "logits/chosen": -0.7415852546691895,
+      "logits/rejected": -1.9916393756866455,
+      "logps/chosen": -319.46490478515625,
+      "logps/rejected": -206.1062774658203,
+      "loss": 0.1372,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.4460511803627014,
+      "rewards/margins": 2.6280391216278076,
+      "rewards/rejected": -3.0740902423858643,
+      "step": 166
+    },
+    {
+      "epoch": 1.2330275229357799,
+      "grad_norm": 0.6352249383926392,
+      "learning_rate": 3.6625366824034337e-06,
+      "logits/chosen": -0.4777001142501831,
+      "logits/rejected": -1.8029272556304932,
+      "logps/chosen": -270.57958984375,
+      "logps/rejected": -244.18014526367188,
+      "loss": 0.1086,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.3066992461681366,
+      "rewards/margins": 3.132423162460327,
+      "rewards/rejected": -3.439122438430786,
+      "step": 168
+    },
+    {
+      "epoch": 1.2477064220183487,
+      "grad_norm": 1.2337560653686523,
+      "learning_rate": 3.6244763196857714e-06,
+      "logits/chosen": -0.5872082710266113,
+      "logits/rejected": -1.7992435693740845,
+      "logps/chosen": -289.64788818359375,
+      "logps/rejected": -216.86793518066406,
+      "loss": 0.141,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.5070605874061584,
+      "rewards/margins": 2.9018495082855225,
+      "rewards/rejected": -3.408910036087036,
+      "step": 170
+    },
+    {
+      "epoch": 1.2623853211009175,
+      "grad_norm": 0.9028024673461914,
+      "learning_rate": 3.5860863720619333e-06,
+      "logits/chosen": -0.6659822463989258,
+      "logits/rejected": -1.8413385152816772,
+      "logps/chosen": -280.3963623046875,
+      "logps/rejected": -216.70907592773438,
+      "loss": 0.1117,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3429238796234131,
+      "rewards/margins": 2.7253475189208984,
+      "rewards/rejected": -3.0682713985443115,
+      "step": 172
+    },
+    {
+      "epoch": 1.2770642201834863,
+      "grad_norm": 1.1501646041870117,
+      "learning_rate": 3.547378091656186e-06,
+      "logits/chosen": -0.5201318264007568,
+      "logits/rejected": -1.8148276805877686,
+      "logps/chosen": -287.64886474609375,
+      "logps/rejected": -212.76695251464844,
+      "loss": 0.114,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.4755076467990875,
+      "rewards/margins": 3.0462517738342285,
+      "rewards/rejected": -3.521759033203125,
+      "step": 174
+    },
+    {
+      "epoch": 1.2917431192660551,
+      "grad_norm": 1.032481074333191,
+      "learning_rate": 3.5083628238963913e-06,
+      "logits/chosen": -0.7503701448440552,
+      "logits/rejected": -1.6679210662841797,
+      "logps/chosen": -228.3303985595703,
+      "logps/rejected": -207.5117645263672,
+      "loss": 0.1515,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.3945405185222626,
+      "rewards/margins": 2.6488091945648193,
+      "rewards/rejected": -3.0433499813079834,
+      "step": 176
+    },
+    {
+      "epoch": 1.306422018348624,
+      "grad_norm": 0.6138296723365784,
+      "learning_rate": 3.4690520041886473e-06,
+      "logits/chosen": -0.5367707014083862,
+      "logits/rejected": -1.7213959693908691,
+      "logps/chosen": -267.9410705566406,
+      "logps/rejected": -249.54847717285156,
+      "loss": 0.0978,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.42533883452415466,
+      "rewards/margins": 3.054936408996582,
+      "rewards/rejected": -3.4802753925323486,
+      "step": 178
+    },
+    {
+      "epoch": 1.3211009174311927,
+      "grad_norm": 0.6739828586578369,
+      "learning_rate": 3.4294571545655653e-06,
+      "logits/chosen": -0.5461620688438416,
+      "logits/rejected": -1.905317783355713,
+      "logps/chosen": -284.7737731933594,
+      "logps/rejected": -219.20343017578125,
+      "loss": 0.0795,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4114089012145996,
+      "rewards/margins": 3.219048023223877,
+      "rewards/rejected": -3.6304564476013184,
+      "step": 180
+    },
+    {
+      "epoch": 1.3357798165137615,
+      "grad_norm": 0.8482432961463928,
+      "learning_rate": 3.38958988030915e-06,
+      "logits/chosen": -0.8901345729827881,
+      "logits/rejected": -1.8333852291107178,
+      "logps/chosen": -267.26361083984375,
+      "logps/rejected": -262.9141845703125,
+      "loss": 0.1353,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.4545396566390991,
+      "rewards/margins": 3.137350082397461,
+      "rewards/rejected": -3.5918896198272705,
+      "step": 182
+    },
+    {
+      "epoch": 1.3504587155963304,
+      "grad_norm": 1.3089014291763306,
+      "learning_rate": 3.3494618665492833e-06,
+      "logits/chosen": -0.755707859992981,
+      "logits/rejected": -1.7024749517440796,
+      "logps/chosen": -250.79086303710938,
+      "logps/rejected": -227.2308349609375,
+      "loss": 0.1522,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.7349607348442078,
+      "rewards/margins": 2.6569156646728516,
+      "rewards/rejected": -3.391876459121704,
+      "step": 184
+    },
+    {
+      "epoch": 1.3651376146788992,
+      "grad_norm": 1.3347550630569458,
+      "learning_rate": 3.3090848748388042e-06,
+      "logits/chosen": -0.6413676738739014,
+      "logits/rejected": -1.825758457183838,
+      "logps/chosen": -347.5032653808594,
+      "logps/rejected": -237.12234497070312,
+      "loss": 0.1274,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": -0.6479396820068359,
+      "rewards/margins": 3.553611993789673,
+      "rewards/rejected": -4.201551914215088,
+      "step": 186
+    },
+    {
+      "epoch": 1.379816513761468,
+      "grad_norm": 1.3518805503845215,
+      "learning_rate": 3.2684707397061887e-06,
+      "logits/chosen": -0.6914655566215515,
+      "logits/rejected": -1.7866692543029785,
+      "logps/chosen": -285.770751953125,
+      "logps/rejected": -212.2018585205078,
+      "loss": 0.1099,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.5680760741233826,
+      "rewards/margins": 3.106520652770996,
+      "rewards/rejected": -3.6745967864990234,
+      "step": 188
+    },
+    {
+      "epoch": 1.3944954128440368,
+      "grad_norm": 0.8207036256790161,
+      "learning_rate": 3.2276313651868364e-06,
+      "logits/chosen": -0.6406370997428894,
+      "logits/rejected": -1.84329354763031,
+      "logps/chosen": -293.601806640625,
+      "logps/rejected": -199.5387420654297,
+      "loss": 0.1137,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.6545761823654175,
+      "rewards/margins": 2.952256917953491,
+      "rewards/rejected": -3.606832981109619,
+      "step": 190
+    },
+    {
+      "epoch": 1.4091743119266056,
+      "grad_norm": 0.5552340149879456,
+      "learning_rate": 3.1865787213339926e-06,
+      "logits/chosen": -0.6593654155731201,
+      "logits/rejected": -1.8798208236694336,
+      "logps/chosen": -274.5489501953125,
+      "logps/rejected": -228.21701049804688,
+      "loss": 0.1071,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5877947211265564,
+      "rewards/margins": 3.3616068363189697,
+      "rewards/rejected": -3.94940185546875,
+      "step": 192
+    },
+    {
+      "epoch": 1.4238532110091744,
+      "grad_norm": 1.2562384605407715,
+      "learning_rate": 3.1453248407103156e-06,
+      "logits/chosen": -0.7084455490112305,
+      "logits/rejected": -1.8440113067626953,
+      "logps/chosen": -284.3964538574219,
+      "logps/rejected": -209.3497314453125,
+      "loss": 0.1227,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.7787793278694153,
+      "rewards/margins": 2.995060443878174,
+      "rewards/rejected": -3.7738397121429443,
+      "step": 194
+    },
+    {
+      "epoch": 1.4385321100917432,
+      "grad_norm": 1.6725304126739502,
+      "learning_rate": 3.1038818148611178e-06,
+      "logits/chosen": -0.6947388648986816,
+      "logits/rejected": -1.848347544670105,
+      "logps/chosen": -306.3591003417969,
+      "logps/rejected": -223.3616943359375,
+      "loss": 0.135,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": -0.6587986946105957,
+      "rewards/margins": 3.338867664337158,
+      "rewards/rejected": -3.997666120529175,
+      "step": 196
+    },
+    {
+      "epoch": 1.453211009174312,
+      "grad_norm": 0.9124309420585632,
+      "learning_rate": 3.062261790770331e-06,
+      "logits/chosen": -0.5535175800323486,
+      "logits/rejected": -1.734500527381897,
+      "logps/chosen": -254.01608276367188,
+      "logps/rejected": -220.9300994873047,
+      "loss": 0.1372,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": -0.5250685214996338,
+      "rewards/margins": 3.1529276371002197,
+      "rewards/rejected": -3.6779963970184326,
+      "step": 198
+    },
+    {
+      "epoch": 1.4678899082568808,
+      "grad_norm": 0.8558027744293213,
+      "learning_rate": 3.0204769673002123e-06,
+      "logits/chosen": -0.6569163203239441,
+      "logits/rejected": -1.8613836765289307,
+      "logps/chosen": -328.30047607421875,
+      "logps/rejected": -240.08651733398438,
+      "loss": 0.1041,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.5923442840576172,
+      "rewards/margins": 3.3542637825012207,
+      "rewards/rejected": -3.946608543395996,
+      "step": 200
+    },
+    {
+      "epoch": 1.4825688073394496,
+      "grad_norm": 0.998695969581604,
+      "learning_rate": 2.978539591615848e-06,
+      "logits/chosen": -0.7506656050682068,
+      "logits/rejected": -1.6931097507476807,
+      "logps/chosen": -295.73895263671875,
+      "logps/rejected": -241.44549560546875,
+      "loss": 0.0991,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8281759023666382,
+      "rewards/margins": 3.4153995513916016,
+      "rewards/rejected": -4.243575572967529,
+      "step": 202
+    },
+    {
+      "epoch": 1.4972477064220184,
+      "grad_norm": 1.2060972452163696,
+      "learning_rate": 2.936461955595501e-06,
+      "logits/chosen": -0.7403116226196289,
+      "logits/rejected": -1.9083077907562256,
+      "logps/chosen": -292.3081970214844,
+      "logps/rejected": -228.6778106689453,
+      "loss": 0.1185,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.5762312412261963,
+      "rewards/margins": 3.2148995399475098,
+      "rewards/rejected": -3.7911314964294434,
+      "step": 204
+    },
+    {
+      "epoch": 1.5119266055045872,
+      "grad_norm": 0.8591629862785339,
+      "learning_rate": 2.8942563922278487e-06,
+      "logits/chosen": -0.732769787311554,
+      "logits/rejected": -1.8991410732269287,
+      "logps/chosen": -282.7076721191406,
+      "logps/rejected": -240.99298095703125,
+      "loss": 0.0849,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6992833018302917,
+      "rewards/margins": 3.560328722000122,
+      "rewards/rejected": -4.259612083435059,
+      "step": 206
+    },
+    {
+      "epoch": 1.526605504587156,
+      "grad_norm": 2.0930187702178955,
+      "learning_rate": 2.8519352719971783e-06,
+      "logits/chosen": -0.7622525691986084,
+      "logits/rejected": -1.8798342943191528,
+      "logps/chosen": -311.9097900390625,
+      "logps/rejected": -244.85189819335938,
+      "loss": 0.1431,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.6880066394805908,
+      "rewards/margins": 3.4552805423736572,
+      "rewards/rejected": -4.143287181854248,
+      "step": 208
+    },
+    {
+      "epoch": 1.5412844036697249,
+      "grad_norm": 0.9879018068313599,
+      "learning_rate": 2.8095109992575824e-06,
+      "logits/chosen": -0.6440776586532593,
+      "logits/rejected": -1.8101108074188232,
+      "logps/chosen": -319.416748046875,
+      "logps/rejected": -243.45770263671875,
+      "loss": 0.0775,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.561879575252533,
+      "rewards/margins": 3.422084093093872,
+      "rewards/rejected": -3.98396372795105,
+      "step": 210
+    },
+    {
+      "epoch": 1.5559633027522937,
+      "grad_norm": 0.8792589902877808,
+      "learning_rate": 2.7669960085972407e-06,
+      "logits/chosen": -0.6799444556236267,
+      "logits/rejected": -1.9857302904129028,
+      "logps/chosen": -347.2530212402344,
+      "logps/rejected": -264.5433044433594,
+      "loss": 0.0919,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.8007718324661255,
+      "rewards/margins": 3.4677510261535645,
+      "rewards/rejected": -4.2685227394104,
+      "step": 212
+    },
+    {
+      "epoch": 1.5706422018348625,
+      "grad_norm": 0.6059939861297607,
+      "learning_rate": 2.7244027611938247e-06,
+      "logits/chosen": -0.48384833335876465,
+      "logits/rejected": -1.7149556875228882,
+      "logps/chosen": -247.654296875,
+      "logps/rejected": -265.72039794921875,
+      "loss": 0.0837,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6536756157875061,
+      "rewards/margins": 3.7947700023651123,
+      "rewards/rejected": -4.4484453201293945,
+      "step": 214
+    },
+    {
+      "epoch": 1.5853211009174313,
+      "grad_norm": 1.9957736730575562,
+      "learning_rate": 2.6817437411621194e-06,
+      "logits/chosen": -0.6645992994308472,
+      "logits/rejected": -1.8484892845153809,
+      "logps/chosen": -338.16943359375,
+      "logps/rejected": -281.8222351074219,
+      "loss": 0.1234,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": -0.8013834953308105,
+      "rewards/margins": 3.5191164016723633,
+      "rewards/rejected": -4.320499897003174,
+      "step": 216
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 1.6058154106140137,
+      "learning_rate": 2.639031451894923e-06,
+      "logits/chosen": -0.7305362224578857,
+      "logits/rejected": -1.720422625541687,
+      "logps/chosen": -322.87603759765625,
+      "logps/rejected": -265.8577880859375,
+      "loss": 0.0921,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.6010589599609375,
+      "rewards/margins": 3.54419207572937,
+      "rewards/rejected": -4.1452507972717285,
+      "step": 218
+    },
+    {
+      "epoch": 1.614678899082569,
+      "grad_norm": 1.0787535905838013,
+      "learning_rate": 2.5962784123982843e-06,
+      "logits/chosen": -0.815077006816864,
+      "logits/rejected": -1.973067283630371,
+      "logps/chosen": -302.24835205078125,
+      "logps/rejected": -244.04307556152344,
+      "loss": 0.1152,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": -0.7761982083320618,
+      "rewards/margins": 3.647855520248413,
+      "rewards/rejected": -4.42405366897583,
+      "step": 220
+    },
+    {
+      "epoch": 1.6293577981651377,
+      "grad_norm": 0.6867660880088806,
+      "learning_rate": 2.5534971536221804e-06,
+      "logits/chosen": -0.5493606925010681,
+      "logits/rejected": -1.746401071548462,
+      "logps/chosen": -265.7420349121094,
+      "logps/rejected": -235.35208129882812,
+      "loss": 0.0729,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.7543711066246033,
+      "rewards/margins": 3.5390028953552246,
+      "rewards/rejected": -4.2933735847473145,
+      "step": 222
+    },
+    {
+      "epoch": 1.6440366972477065,
+      "grad_norm": 0.44982749223709106,
+      "learning_rate": 2.5107002147876814e-06,
+      "logits/chosen": -0.6843823194503784,
+      "logits/rejected": -1.5824543237686157,
+      "logps/chosen": -257.94091796875,
+      "logps/rejected": -254.91258239746094,
+      "loss": 0.0712,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6873564124107361,
+      "rewards/margins": 3.931499481201172,
+      "rewards/rejected": -4.618855953216553,
+      "step": 224
+    },
+    {
+      "epoch": 1.6587155963302753,
+      "grad_norm": 1.7579375505447388,
+      "learning_rate": 2.467900139711693e-06,
+      "logits/chosen": -0.7247440814971924,
+      "logits/rejected": -1.7569100856781006,
+      "logps/chosen": -272.9012145996094,
+      "logps/rejected": -242.78591918945312,
+      "loss": 0.1394,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": -0.8232018947601318,
+      "rewards/margins": 3.6535580158233643,
+      "rewards/rejected": -4.476759910583496,
+      "step": 226
+    },
+    {
+      "epoch": 1.6733944954128441,
+      "grad_norm": 0.9240662455558777,
+      "learning_rate": 2.4251094731303586e-06,
+      "logits/chosen": -0.641674280166626,
+      "logits/rejected": -1.8788131475448608,
+      "logps/chosen": -286.55889892578125,
+      "logps/rejected": -223.2461395263672,
+      "loss": 0.0863,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.6447327136993408,
+      "rewards/margins": 3.462599754333496,
+      "rewards/rejected": -4.107332229614258,
+      "step": 228
+    },
+    {
+      "epoch": 1.688073394495413,
+      "grad_norm": 0.6030136346817017,
+      "learning_rate": 2.3823407570221812e-06,
+      "logits/chosen": -0.5796603560447693,
+      "logits/rejected": -1.8217395544052124,
+      "logps/chosen": -295.2866516113281,
+      "logps/rejected": -220.7737274169922,
+      "loss": 0.0741,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6853188276290894,
+      "rewards/margins": 3.6318960189819336,
+      "rewards/rejected": -4.3172149658203125,
+      "step": 230
+    },
+    {
+      "epoch": 1.7027522935779817,
+      "grad_norm": 0.568932294845581,
+      "learning_rate": 2.3396065269319655e-06,
+      "logits/chosen": -0.676604151725769,
+      "logits/rejected": -1.9570648670196533,
+      "logps/chosen": -296.9806823730469,
+      "logps/rejected": -219.16986083984375,
+      "loss": 0.0842,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.6757611036300659,
+      "rewards/margins": 3.6589701175689697,
+      "rewards/rejected": -4.334731101989746,
+      "step": 232
+    },
+    {
+      "epoch": 1.7174311926605506,
+      "grad_norm": 0.6324030756950378,
+      "learning_rate": 2.2969193082966353e-06,
+      "logits/chosen": -0.5455877780914307,
+      "logits/rejected": -1.8318365812301636,
+      "logps/chosen": -281.55133056640625,
+      "logps/rejected": -238.96044921875,
+      "loss": 0.0526,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8087357878684998,
+      "rewards/margins": 3.9599123001098633,
+      "rewards/rejected": -4.76864767074585,
+      "step": 234
+    },
+    {
+      "epoch": 1.7321100917431194,
+      "grad_norm": 0.5559871196746826,
+      "learning_rate": 2.2542916127740194e-06,
+      "logits/chosen": -0.5671895146369934,
+      "logits/rejected": -1.5007973909378052,
+      "logps/chosen": -308.0108642578125,
+      "logps/rejected": -282.2166748046875,
+      "loss": 0.0559,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6229729652404785,
+      "rewards/margins": 4.207201957702637,
+      "rewards/rejected": -4.830175399780273,
+      "step": 236
+    },
+    {
+      "epoch": 1.7467889908256882,
+      "grad_norm": 0.7297521233558655,
+      "learning_rate": 2.211735934575674e-06,
+      "logits/chosen": -0.5655158162117004,
+      "logits/rejected": -1.8047404289245605,
+      "logps/chosen": -280.84136962890625,
+      "logps/rejected": -210.9255828857422,
+      "loss": 0.117,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.9368782043457031,
+      "rewards/margins": 3.448589324951172,
+      "rewards/rejected": -4.385467529296875,
+      "step": 238
+    },
+    {
+      "epoch": 1.761467889908257,
+      "grad_norm": 0.5144051909446716,
+      "learning_rate": 2.1692647468048235e-06,
+      "logits/chosen": -0.8172545433044434,
+      "logits/rejected": -1.7720967531204224,
+      "logps/chosen": -303.94207763671875,
+      "logps/rejected": -259.8736572265625,
+      "loss": 0.0538,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8221498727798462,
+      "rewards/margins": 4.422685623168945,
+      "rewards/rejected": -5.244835376739502,
+      "step": 240
+    },
+    {
+      "epoch": 1.7761467889908258,
+      "grad_norm": 0.9769014120101929,
+      "learning_rate": 2.126890497800477e-06,
+      "logits/chosen": -0.7060015797615051,
+      "logits/rejected": -1.8099745512008667,
+      "logps/chosen": -294.390869140625,
+      "logps/rejected": -243.9739227294922,
+      "loss": 0.116,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8548272848129272,
+      "rewards/margins": 3.3701939582824707,
+      "rewards/rejected": -4.2250213623046875,
+      "step": 242
+    },
+    {
+      "epoch": 1.7908256880733946,
+      "grad_norm": 0.70880126953125,
+      "learning_rate": 2.084625607488816e-06,
+      "logits/chosen": -0.597684919834137,
+      "logits/rejected": -1.7976946830749512,
+      "logps/chosen": -271.812255859375,
+      "logps/rejected": -240.36590576171875,
+      "loss": 0.0667,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.7783107757568359,
+      "rewards/margins": 4.291589260101318,
+      "rewards/rejected": -5.0699005126953125,
+      "step": 244
+    },
+    {
+      "epoch": 1.8055045871559634,
+      "grad_norm": 0.4248654842376709,
+      "learning_rate": 2.0424824637428995e-06,
+      "logits/chosen": -0.5851184725761414,
+      "logits/rejected": -1.9045813083648682,
+      "logps/chosen": -265.13494873046875,
+      "logps/rejected": -221.10482788085938,
+      "loss": 0.053,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8591616749763489,
+      "rewards/margins": 4.024895668029785,
+      "rewards/rejected": -4.88405704498291,
+      "step": 246
+    },
+    {
+      "epoch": 1.8201834862385322,
+      "grad_norm": 0.7956784963607788,
+      "learning_rate": 2.0004734187517744e-06,
+      "logits/chosen": -0.7580417394638062,
+      "logits/rejected": -1.747441053390503,
+      "logps/chosen": -317.2771301269531,
+      "logps/rejected": -223.77203369140625,
+      "loss": 0.0986,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.8614228963851929,
+      "rewards/margins": 3.6957850456237793,
+      "rewards/rejected": -4.557208061218262,
+      "step": 248
+    },
+    {
+      "epoch": 1.834862385321101,
+      "grad_norm": 0.5855574607849121,
+      "learning_rate": 1.9586107854000327e-06,
+      "logits/chosen": -0.8148155808448792,
+      "logits/rejected": -1.9937602281570435,
+      "logps/chosen": -292.9911804199219,
+      "logps/rejected": -214.19070434570312,
+      "loss": 0.0871,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.7950844764709473,
+      "rewards/margins": 3.577152729034424,
+      "rewards/rejected": -4.372237205505371,
+      "step": 250
+    },
+    {
+      "epoch": 1.8495412844036698,
+      "grad_norm": 0.8180505037307739,
+      "learning_rate": 1.916906833658899e-06,
+      "logits/chosen": -0.6161235570907593,
+      "logits/rejected": -1.8707021474838257,
+      "logps/chosen": -320.8139953613281,
+      "logps/rejected": -268.6239929199219,
+      "loss": 0.0714,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.8487557768821716,
+      "rewards/margins": 4.137137413024902,
+      "rewards/rejected": -4.985893249511719,
+      "step": 252
+    },
+    {
+      "epoch": 1.8642201834862386,
+      "grad_norm": 0.4743252992630005,
+      "learning_rate": 1.8753737869898921e-06,
+      "logits/chosen": -0.6861028671264648,
+      "logits/rejected": -1.8028208017349243,
+      "logps/chosen": -246.8392791748047,
+      "logps/rejected": -235.8388214111328,
+      "loss": 0.051,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8036968111991882,
+      "rewards/margins": 4.426541805267334,
+      "rewards/rejected": -5.230238437652588,
+      "step": 254
+    },
+    {
+      "epoch": 1.8788990825688074,
+      "grad_norm": 0.49114933609962463,
+      "learning_rate": 1.8340238187621185e-06,
+      "logits/chosen": -0.5393966436386108,
+      "logits/rejected": -1.8631210327148438,
+      "logps/chosen": -259.17840576171875,
+      "logps/rejected": -220.71517944335938,
+      "loss": 0.1047,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.7826576232910156,
+      "rewards/margins": 3.6857450008392334,
+      "rewards/rejected": -4.468402862548828,
+      "step": 256
+    },
+    {
+      "epoch": 1.8935779816513763,
+      "grad_norm": 0.9561637043952942,
+      "learning_rate": 1.7928690486842438e-06,
+      "logits/chosen": -0.6505446434020996,
+      "logits/rejected": -1.810225248336792,
+      "logps/chosen": -251.15536499023438,
+      "logps/rejected": -207.4978790283203,
+      "loss": 0.0772,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8502838611602783,
+      "rewards/margins": 3.7762393951416016,
+      "rewards/rejected": -4.626523017883301,
+      "step": 258
+    },
+    {
+      "epoch": 1.908256880733945,
+      "grad_norm": 1.0296114683151245,
+      "learning_rate": 1.7519215392522026e-06,
+      "logits/chosen": -0.7036296725273132,
+      "logits/rejected": -1.9768420457839966,
+      "logps/chosen": -280.53668212890625,
+      "logps/rejected": -218.00341796875,
+      "loss": 0.0681,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.8542452454566956,
+      "rewards/margins": 4.108546733856201,
+      "rewards/rejected": -4.96279239654541,
+      "step": 260
+    },
+    {
+      "epoch": 1.9229357798165139,
+      "grad_norm": 0.5742794275283813,
+      "learning_rate": 1.7111932922136715e-06,
+      "logits/chosen": -0.7176038026809692,
+      "logits/rejected": -1.6738024950027466,
+      "logps/chosen": -250.91835021972656,
+      "logps/rejected": -257.7945556640625,
+      "loss": 0.062,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8737991452217102,
+      "rewards/margins": 4.347530841827393,
+      "rewards/rejected": -5.221330165863037,
+      "step": 262
+    },
+    {
+      "epoch": 1.9376146788990827,
+      "grad_norm": 0.4386001229286194,
+      "learning_rate": 1.6706962450503408e-06,
+      "logits/chosen": -0.5108492374420166,
+      "logits/rejected": -1.8909492492675781,
+      "logps/chosen": -278.94171142578125,
+      "logps/rejected": -244.6939697265625,
+      "loss": 0.0459,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.7861737608909607,
+      "rewards/margins": 4.580504417419434,
+      "rewards/rejected": -5.366678714752197,
+      "step": 264
+    },
+    {
+      "epoch": 1.9522935779816515,
+      "grad_norm": 0.8080554604530334,
+      "learning_rate": 1.630442267479034e-06,
+      "logits/chosen": -0.4731358587741852,
+      "logits/rejected": -1.8190574645996094,
+      "logps/chosen": -261.9861145019531,
+      "logps/rejected": -251.31875610351562,
+      "loss": 0.061,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.7401555180549622,
+      "rewards/margins": 4.329522609710693,
+      "rewards/rejected": -5.06967830657959,
+      "step": 266
+    },
+    {
+      "epoch": 1.9669724770642203,
+      "grad_norm": 0.4730006754398346,
+      "learning_rate": 1.5904431579726837e-06,
+      "logits/chosen": -0.6266091465950012,
+      "logits/rejected": -1.9366010427474976,
+      "logps/chosen": -292.2561340332031,
+      "logps/rejected": -217.6514129638672,
+      "loss": 0.0583,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8204957842826843,
+      "rewards/margins": 4.180571556091309,
+      "rewards/rejected": -5.0010666847229,
+      "step": 268
+    },
+    {
+      "epoch": 1.981651376146789,
+      "grad_norm": 0.9025997519493103,
+      "learning_rate": 1.5507106403021897e-06,
+      "logits/chosen": -0.603181004524231,
+      "logits/rejected": -1.930855393409729,
+      "logps/chosen": -323.4551696777344,
+      "logps/rejected": -252.92965698242188,
+      "loss": 0.074,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6537671685218811,
+      "rewards/margins": 4.156303405761719,
+      "rewards/rejected": -4.810070991516113,
+      "step": 270
+    },
+    {
+      "epoch": 1.996330275229358,
+      "grad_norm": 0.5256826877593994,
+      "learning_rate": 1.511256360100171e-06,
+      "logits/chosen": -0.5993788242340088,
+      "logits/rejected": -1.7707700729370117,
+      "logps/chosen": -290.8062744140625,
+      "logps/rejected": -243.2689971923828,
+      "loss": 0.0755,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.7740217447280884,
+      "rewards/margins": 4.305649757385254,
+      "rewards/rejected": -5.079671382904053,
+      "step": 272
+    },
+    {
+      "epoch": 2.0110091743119267,
+      "grad_norm": 0.8649439215660095,
+      "learning_rate": 1.4720918814476234e-06,
+      "logits/chosen": -0.8103897571563721,
+      "logits/rejected": -1.9931031465530396,
+      "logps/chosen": -253.79019165039062,
+      "logps/rejected": -234.64596557617188,
+      "loss": 0.0833,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.7869517207145691,
+      "rewards/margins": 4.74979829788208,
+      "rewards/rejected": -5.536749839782715,
+      "step": 274
+    },
+    {
+      "epoch": 2.0256880733944955,
+      "grad_norm": 1.317905068397522,
+      "learning_rate": 1.4332286834844792e-06,
+      "logits/chosen": -0.7999382019042969,
+      "logits/rejected": -1.9115333557128906,
+      "logps/chosen": -285.6707458496094,
+      "logps/rejected": -239.1488037109375,
+      "loss": 0.0923,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.9859182834625244,
+      "rewards/margins": 4.015196800231934,
+      "rewards/rejected": -5.001114845275879,
+      "step": 276
+    },
+    {
+      "epoch": 2.0403669724770643,
+      "grad_norm": 0.6509101986885071,
+      "learning_rate": 1.3946781570450563e-06,
+      "logits/chosen": -0.701880156993866,
+      "logits/rejected": -1.9084625244140625,
+      "logps/chosen": -299.8097839355469,
+      "logps/rejected": -249.23460388183594,
+      "loss": 0.0472,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8493328094482422,
+      "rewards/margins": 4.347517490386963,
+      "rewards/rejected": -5.196850299835205,
+      "step": 278
+    },
+    {
+      "epoch": 2.055045871559633,
+      "grad_norm": 0.4122222065925598,
+      "learning_rate": 1.3564516013194023e-06,
+      "logits/chosen": -0.4526348114013672,
+      "logits/rejected": -1.7306469678878784,
+      "logps/chosen": -264.879150390625,
+      "logps/rejected": -240.08486938476562,
+      "loss": 0.0461,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8477758169174194,
+      "rewards/margins": 4.5718865394592285,
+      "rewards/rejected": -5.4196624755859375,
+      "step": 280
+    },
+    {
+      "epoch": 2.069724770642202,
+      "grad_norm": 0.6205916404724121,
+      "learning_rate": 1.3185602205414894e-06,
+      "logits/chosen": -0.7086562514305115,
+      "logits/rejected": -1.7690290212631226,
+      "logps/chosen": -270.14947509765625,
+      "logps/rejected": -225.78253173828125,
+      "loss": 0.0774,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.8577483892440796,
+      "rewards/margins": 4.2042083740234375,
+      "rewards/rejected": -5.061956882476807,
+      "step": 282
+    },
+    {
+      "epoch": 2.0844036697247708,
+      "grad_norm": 0.44248032569885254,
+      "learning_rate": 1.2810151207052465e-06,
+      "logits/chosen": -0.7329645156860352,
+      "logits/rejected": -1.7238507270812988,
+      "logps/chosen": -336.89178466796875,
+      "logps/rejected": -278.45166015625,
+      "loss": 0.0692,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1881827116012573,
+      "rewards/margins": 4.469137191772461,
+      "rewards/rejected": -5.657320022583008,
+      "step": 284
+    },
+    {
+      "epoch": 2.0990825688073396,
+      "grad_norm": 0.5951194763183594,
+      "learning_rate": 1.2438273063093811e-06,
+      "logits/chosen": -0.5224804282188416,
+      "logits/rejected": -1.7726545333862305,
+      "logps/chosen": -276.5761413574219,
+      "logps/rejected": -220.84750366210938,
+      "loss": 0.0851,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -1.0991380214691162,
+      "rewards/margins": 3.8999533653259277,
+      "rewards/rejected": -4.999091148376465,
+      "step": 286
+    },
+    {
+      "epoch": 2.1137614678899084,
+      "grad_norm": 1.2658673524856567,
+      "learning_rate": 1.2070076771319536e-06,
+      "logits/chosen": -0.8442491292953491,
+      "logits/rejected": -1.6956205368041992,
+      "logps/chosen": -353.8062744140625,
+      "logps/rejected": -247.41055297851562,
+      "loss": 0.1044,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.956983208656311,
+      "rewards/margins": 3.6315712928771973,
+      "rewards/rejected": -4.588554859161377,
+      "step": 288
+    },
+    {
+      "epoch": 2.128440366972477,
+      "grad_norm": 0.564155101776123,
+      "learning_rate": 1.1705670250356417e-06,
+      "logits/chosen": -0.5238651633262634,
+      "logits/rejected": -1.782932996749878,
+      "logps/chosen": -307.71112060546875,
+      "logps/rejected": -247.0592803955078,
+      "loss": 0.0566,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6434872150421143,
+      "rewards/margins": 4.430055618286133,
+      "rewards/rejected": -5.073543548583984,
+      "step": 290
+    },
+    {
+      "epoch": 2.143119266055046,
+      "grad_norm": 0.506363570690155,
+      "learning_rate": 1.1345160308046413e-06,
+      "logits/chosen": -0.6401405334472656,
+      "logits/rejected": -2.098745822906494,
+      "logps/chosen": -381.30963134765625,
+      "logps/rejected": -261.1710205078125,
+      "loss": 0.0635,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -1.0125696659088135,
+      "rewards/margins": 4.525051593780518,
+      "rewards/rejected": -5.537620544433594,
+      "step": 292
+    },
+    {
+      "epoch": 2.157798165137615,
+      "grad_norm": 0.584295392036438,
+      "learning_rate": 1.0988652610141154e-06,
+      "logits/chosen": -0.6318463683128357,
+      "logits/rejected": -1.676975965499878,
+      "logps/chosen": -274.4164733886719,
+      "logps/rejected": -269.0091247558594,
+      "loss": 0.06,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8393394947052002,
+      "rewards/margins": 4.4933929443359375,
+      "rewards/rejected": -5.3327317237854,
+      "step": 294
+    },
+    {
+      "epoch": 2.1724770642201836,
+      "grad_norm": 0.2857895791530609,
+      "learning_rate": 1.063625164933124e-06,
+      "logits/chosen": -0.6415454149246216,
+      "logits/rejected": -1.8679744005203247,
+      "logps/chosen": -327.9555969238281,
+      "logps/rejected": -269.9202880859375,
+      "loss": 0.0498,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.8799739480018616,
+      "rewards/margins": 5.01440954208374,
+      "rewards/rejected": -5.894383907318115,
+      "step": 296
+    },
+    {
+      "epoch": 2.1871559633027524,
+      "grad_norm": 0.7591383457183838,
+      "learning_rate": 1.0288060714619359e-06,
+      "logits/chosen": -0.8029307126998901,
+      "logits/rejected": -1.9948023557662964,
+      "logps/chosen": -312.4205627441406,
+      "logps/rejected": -219.84292602539062,
+      "loss": 0.0694,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8505744338035583,
+      "rewards/margins": 4.322221279144287,
+      "rewards/rejected": -5.172795295715332,
+      "step": 298
+    },
+    {
+      "epoch": 2.2018348623853212,
+      "grad_norm": 0.6418690085411072,
+      "learning_rate": 9.944181861046188e-07,
+      "logits/chosen": -0.6180175542831421,
+      "logits/rejected": -1.778718113899231,
+      "logps/chosen": -333.8564758300781,
+      "logps/rejected": -259.21697998046875,
+      "loss": 0.1021,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -1.1542073488235474,
+      "rewards/margins": 4.588565826416016,
+      "rewards/rejected": -5.742773056030273,
+      "step": 300
+    },
+    {
+      "epoch": 2.21651376146789,
+      "grad_norm": 0.8044218420982361,
+      "learning_rate": 9.604715879777986e-07,
+      "logits/chosen": -0.6669445037841797,
+      "logits/rejected": -1.950725793838501,
+      "logps/chosen": -277.66876220703125,
+      "logps/rejected": -205.39151000976562,
+      "loss": 0.076,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.0598458051681519,
+      "rewards/margins": 4.117983341217041,
+      "rewards/rejected": -5.177828788757324,
+      "step": 302
+    },
+    {
+      "epoch": 2.231192660550459,
+      "grad_norm": 0.6962106823921204,
+      "learning_rate": 9.269762268564616e-07,
+      "logits/chosen": -0.7753230929374695,
+      "logits/rejected": -1.840043306350708,
+      "logps/chosen": -252.92071533203125,
+      "logps/rejected": -211.79078674316406,
+      "loss": 0.0652,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.7847457528114319,
+      "rewards/margins": 4.182995319366455,
+      "rewards/rejected": -4.967741012573242,
+      "step": 304
+    },
+    {
+      "epoch": 2.2458715596330276,
+      "grad_norm": 0.66120445728302,
+      "learning_rate": 8.939419202576694e-07,
+      "logits/chosen": -0.4923960864543915,
+      "logits/rejected": -1.4844017028808594,
+      "logps/chosen": -257.7115173339844,
+      "logps/rejected": -228.1881561279297,
+      "loss": 0.0903,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.871938943862915,
+      "rewards/margins": 3.5912954807281494,
+      "rewards/rejected": -4.4632344245910645,
+      "step": 306
+    },
+    {
+      "epoch": 2.2605504587155965,
+      "grad_norm": 0.7900298237800598,
+      "learning_rate": 8.61378350563033e-07,
+      "logits/chosen": -0.626375675201416,
+      "logits/rejected": -1.8259055614471436,
+      "logps/chosen": -248.50527954101562,
+      "logps/rejected": -247.7064208984375,
+      "loss": 0.0611,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8818211555480957,
+      "rewards/margins": 4.562689781188965,
+      "rewards/rejected": -5.4445109367370605,
+      "step": 308
+    },
+    {
+      "epoch": 2.2752293577981653,
+      "grad_norm": 0.6607710123062134,
+      "learning_rate": 8.292950621808022e-07,
+      "logits/chosen": -0.7079221606254578,
+      "logits/rejected": -1.9055808782577515,
+      "logps/chosen": -281.2016296386719,
+      "logps/rejected": -247.4744415283203,
+      "loss": 0.0573,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8795925378799438,
+      "rewards/margins": 4.619358539581299,
+      "rewards/rejected": -5.498951435089111,
+      "step": 310
+    },
+    {
+      "epoch": 2.289908256880734,
+      "grad_norm": 0.6514431238174438,
+      "learning_rate": 7.977014587483925e-07,
+      "logits/chosen": -0.7440314888954163,
+      "logits/rejected": -1.7653181552886963,
+      "logps/chosen": -271.65496826171875,
+      "logps/rejected": -283.68035888671875,
+      "loss": 0.0607,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.8735625147819519,
+      "rewards/margins": 4.6589860916137695,
+      "rewards/rejected": -5.532548904418945,
+      "step": 312
+    },
+    {
+      "epoch": 2.304587155963303,
+      "grad_norm": 0.9269477128982544,
+      "learning_rate": 7.666068003761684e-07,
+      "logits/chosen": -0.6487716436386108,
+      "logits/rejected": -1.9153821468353271,
+      "logps/chosen": -295.9359436035156,
+      "logps/rejected": -225.51235961914062,
+      "loss": 0.06,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.9806410670280457,
+      "rewards/margins": 4.522603988647461,
+      "rewards/rejected": -5.503244876861572,
+      "step": 314
+    },
+    {
+      "epoch": 2.3192660550458717,
+      "grad_norm": 0.8910574316978455,
+      "learning_rate": 7.360202009332993e-07,
+      "logits/chosen": -0.7844710350036621,
+      "logits/rejected": -1.9116116762161255,
+      "logps/chosen": -295.5791015625,
+      "logps/rejected": -242.4950714111328,
+      "loss": 0.0489,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8433384299278259,
+      "rewards/margins": 4.688782691955566,
+      "rewards/rejected": -5.532121658325195,
+      "step": 316
+    },
+    {
+      "epoch": 2.3339449541284405,
+      "grad_norm": 0.6180950403213501,
+      "learning_rate": 7.059506253764773e-07,
+      "logits/chosen": -0.6881747245788574,
+      "logits/rejected": -1.776421070098877,
+      "logps/chosen": -311.3552551269531,
+      "logps/rejected": -251.24072265625,
+      "loss": 0.0501,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9155367016792297,
+      "rewards/margins": 4.762095928192139,
+      "rewards/rejected": -5.6776323318481445,
+      "step": 318
+    },
+    {
+      "epoch": 2.3486238532110093,
+      "grad_norm": 0.5022120475769043,
+      "learning_rate": 6.764068871222825e-07,
+      "logits/chosen": -0.46809208393096924,
+      "logits/rejected": -1.7341670989990234,
+      "logps/chosen": -285.6492919921875,
+      "logps/rejected": -240.606689453125,
+      "loss": 0.0592,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8136368989944458,
+      "rewards/margins": 4.417351722717285,
+      "rewards/rejected": -5.230988025665283,
+      "step": 320
+    },
+    {
+      "epoch": 2.363302752293578,
+      "grad_norm": 0.5374433994293213,
+      "learning_rate": 6.473976454639608e-07,
+      "logits/chosen": -0.6733486652374268,
+      "logits/rejected": -1.9388556480407715,
+      "logps/chosen": -291.48248291015625,
+      "logps/rejected": -225.9014129638672,
+      "loss": 0.055,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9106884598731995,
+      "rewards/margins": 4.666688442230225,
+      "rewards/rejected": -5.577376842498779,
+      "step": 322
+    },
+    {
+      "epoch": 2.377981651376147,
+      "grad_norm": 0.9357534646987915,
+      "learning_rate": 6.189314030333796e-07,
+      "logits/chosen": -0.49531441926956177,
+      "logits/rejected": -1.6569920778274536,
+      "logps/chosen": -278.2015686035156,
+      "logps/rejected": -277.6153564453125,
+      "loss": 0.0458,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8881136178970337,
+      "rewards/margins": 4.91811466217041,
+      "rewards/rejected": -5.806228160858154,
+      "step": 324
+    },
+    {
+      "epoch": 2.3926605504587157,
+      "grad_norm": 1.5413628816604614,
+      "learning_rate": 5.910165033089e-07,
+      "logits/chosen": -0.5966196656227112,
+      "logits/rejected": -1.830047845840454,
+      "logps/chosen": -315.2392883300781,
+      "logps/rejected": -256.3215026855469,
+      "loss": 0.058,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.8759629726409912,
+      "rewards/margins": 4.466618537902832,
+      "rewards/rejected": -5.342581748962402,
+      "step": 326
+    },
+    {
+      "epoch": 2.4073394495412845,
+      "grad_norm": 0.5090343356132507,
+      "learning_rate": 5.636611281698956e-07,
+      "logits/chosen": -0.641990602016449,
+      "logits/rejected": -1.7657126188278198,
+      "logps/chosen": -262.68450927734375,
+      "logps/rejected": -237.4891815185547,
+      "loss": 0.055,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9292994737625122,
+      "rewards/margins": 4.346236228942871,
+      "rewards/rejected": -5.275536060333252,
+      "step": 328
+    },
+    {
+      "epoch": 2.4220183486238533,
+      "grad_norm": 0.9838851094245911,
+      "learning_rate": 5.368732954986389e-07,
+      "logits/chosen": -0.7656599879264832,
+      "logits/rejected": -1.8343071937561035,
+      "logps/chosen": -278.4713134765625,
+      "logps/rejected": -249.50767517089844,
+      "loss": 0.0729,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9321765899658203,
+      "rewards/margins": 4.340839385986328,
+      "rewards/rejected": -5.27301549911499,
+      "step": 330
+    },
+    {
+      "epoch": 2.436697247706422,
+      "grad_norm": 0.7453677654266357,
+      "learning_rate": 5.106608568302504e-07,
+      "logits/chosen": -0.6978477239608765,
+      "logits/rejected": -1.8032922744750977,
+      "logps/chosen": -256.7540283203125,
+      "logps/rejected": -254.54901123046875,
+      "loss": 0.0686,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.9668062329292297,
+      "rewards/margins": 4.748128890991211,
+      "rewards/rejected": -5.714934349060059,
+      "step": 332
+    },
+    {
+      "epoch": 2.451376146788991,
+      "grad_norm": 0.6675239205360413,
+      "learning_rate": 4.850314950514124e-07,
+      "logits/chosen": -0.5538415908813477,
+      "logits/rejected": -1.6941921710968018,
+      "logps/chosen": -279.3875732421875,
+      "logps/rejected": -249.05357360839844,
+      "loss": 0.0605,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.9551371335983276,
+      "rewards/margins": 4.742038249969482,
+      "rewards/rejected": -5.6971755027771,
+      "step": 334
+    },
+    {
+      "epoch": 2.4660550458715598,
+      "grad_norm": 1.0032905340194702,
+      "learning_rate": 4.599927221485034e-07,
+      "logits/chosen": -0.5769209265708923,
+      "logits/rejected": -1.8620692491531372,
+      "logps/chosen": -276.2666931152344,
+      "logps/rejected": -226.5665283203125,
+      "loss": 0.0726,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.9197105169296265,
+      "rewards/margins": 4.4358601570129395,
+      "rewards/rejected": -5.355570316314697,
+      "step": 336
+    },
+    {
+      "epoch": 2.4807339449541286,
+      "grad_norm": 0.46133214235305786,
+      "learning_rate": 4.3555187700583175e-07,
+      "logits/chosen": -0.5237099528312683,
+      "logits/rejected": -1.8421297073364258,
+      "logps/chosen": -263.33160400390625,
+      "logps/rejected": -245.607177734375,
+      "loss": 0.0352,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8943026661872864,
+      "rewards/margins": 4.7068657875061035,
+      "rewards/rejected": -5.601168155670166,
+      "step": 338
+    },
+    {
+      "epoch": 2.4954128440366974,
+      "grad_norm": 0.41858726739883423,
+      "learning_rate": 4.1171612325460244e-07,
+      "logits/chosen": -0.6014917492866516,
+      "logits/rejected": -1.6842682361602783,
+      "logps/chosen": -279.3096923828125,
+      "logps/rejected": -236.51959228515625,
+      "loss": 0.0792,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.9955892562866211,
+      "rewards/margins": 4.224085807800293,
+      "rewards/rejected": -5.219675064086914,
+      "step": 340
+    },
+    {
+      "epoch": 2.510091743119266,
+      "grad_norm": 0.3274426758289337,
+      "learning_rate": 3.8849244717325206e-07,
+      "logits/chosen": -0.639855682849884,
+      "logits/rejected": -1.704068899154663,
+      "logps/chosen": -266.26715087890625,
+      "logps/rejected": -260.8788146972656,
+      "loss": 0.049,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8671805262565613,
+      "rewards/margins": 4.942291736602783,
+      "rewards/rejected": -5.80947208404541,
+      "step": 342
+    },
+    {
+      "epoch": 2.524770642201835,
+      "grad_norm": 1.3120359182357788,
+      "learning_rate": 3.658876556397628e-07,
+      "logits/chosen": -0.8555523753166199,
+      "logits/rejected": -1.9403518438339233,
+      "logps/chosen": -254.93899536132812,
+      "logps/rejected": -226.33721923828125,
+      "loss": 0.0831,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -1.1139277219772339,
+      "rewards/margins": 4.305876731872559,
+      "rewards/rejected": -5.419804096221924,
+      "step": 344
+    },
+    {
+      "epoch": 2.539449541284404,
+      "grad_norm": 0.6721531748771667,
+      "learning_rate": 3.4390837413656256e-07,
+      "logits/chosen": -0.6973453164100647,
+      "logits/rejected": -1.883147120475769,
+      "logps/chosen": -277.8430480957031,
+      "logps/rejected": -262.0320739746094,
+      "loss": 0.0622,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.942565381526947,
+      "rewards/margins": 4.684934139251709,
+      "rewards/rejected": -5.627499580383301,
+      "step": 346
+    },
+    {
+      "epoch": 2.5541284403669726,
+      "grad_norm": 0.9806049466133118,
+      "learning_rate": 3.225610448085903e-07,
+      "logits/chosen": -0.6397227048873901,
+      "logits/rejected": -1.8119735717773438,
+      "logps/chosen": -270.50787353515625,
+      "logps/rejected": -238.79200744628906,
+      "loss": 0.08,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": -1.1652170419692993,
+      "rewards/margins": 4.398360252380371,
+      "rewards/rejected": -5.563577651977539,
+      "step": 348
+    },
+    {
+      "epoch": 2.5688073394495414,
+      "grad_norm": 0.6500720381736755,
+      "learning_rate": 3.018519245750989e-07,
+      "logits/chosen": -0.6613521575927734,
+      "logits/rejected": -1.692039132118225,
+      "logps/chosen": -319.7649230957031,
+      "logps/rejected": -278.594482421875,
+      "loss": 0.0602,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8958297371864319,
+      "rewards/margins": 4.4838547706604,
+      "rewards/rejected": -5.3796844482421875,
+      "step": 350
+    },
+    {
+      "epoch": 2.5834862385321102,
+      "grad_norm": 1.183739423751831,
+      "learning_rate": 2.817870832957459e-07,
+      "logits/chosen": -0.611799418926239,
+      "logits/rejected": -1.9181246757507324,
+      "logps/chosen": -258.9988098144531,
+      "logps/rejected": -234.6461639404297,
+      "loss": 0.0814,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.0336565971374512,
+      "rewards/margins": 4.347400665283203,
+      "rewards/rejected": -5.381057262420654,
+      "step": 352
+    },
+    {
+      "epoch": 2.598165137614679,
+      "grad_norm": 1.2900700569152832,
+      "learning_rate": 2.6237240199151386e-07,
+      "logits/chosen": -0.6546847820281982,
+      "logits/rejected": -1.7483494281768799,
+      "logps/chosen": -262.4722595214844,
+      "logps/rejected": -225.11770629882812,
+      "loss": 0.0855,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -1.0690879821777344,
+      "rewards/margins": 4.176665782928467,
+      "rewards/rejected": -5.245754241943359,
+      "step": 354
+    },
+    {
+      "epoch": 2.612844036697248,
+      "grad_norm": 0.7886202335357666,
+      "learning_rate": 2.436135711209786e-07,
+      "logits/chosen": -0.9137476682662964,
+      "logits/rejected": -1.9900403022766113,
+      "logps/chosen": -279.0731201171875,
+      "logps/rejected": -215.946533203125,
+      "loss": 0.0888,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -1.0406084060668945,
+      "rewards/margins": 3.958219528198242,
+      "rewards/rejected": -4.998827934265137,
+      "step": 356
+    },
+    {
+      "epoch": 2.6275229357798167,
+      "grad_norm": 0.7906404137611389,
+      "learning_rate": 2.2551608891243026e-07,
+      "logits/chosen": -0.9193136096000671,
+      "logits/rejected": -1.8110696077346802,
+      "logps/chosen": -351.8749694824219,
+      "logps/rejected": -266.59405517578125,
+      "loss": 0.064,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1595898866653442,
+      "rewards/margins": 4.084352493286133,
+      "rewards/rejected": -5.243941783905029,
+      "step": 358
+    },
+    {
+      "epoch": 2.6422018348623855,
+      "grad_norm": 0.37772616744041443,
+      "learning_rate": 2.0808525975233807e-07,
+      "logits/chosen": -0.5619891881942749,
+      "logits/rejected": -1.8202831745147705,
+      "logps/chosen": -285.0718078613281,
+      "logps/rejected": -255.26644897460938,
+      "loss": 0.0537,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.134505033493042,
+      "rewards/margins": 4.348145484924316,
+      "rewards/rejected": -5.482650279998779,
+      "step": 360
+    },
+    {
+      "epoch": 2.6568807339449543,
+      "grad_norm": 0.7285224795341492,
+      "learning_rate": 1.9132619263063144e-07,
+      "logits/chosen": -0.5652111768722534,
+      "logits/rejected": -1.8313266038894653,
+      "logps/chosen": -340.82025146484375,
+      "logps/rejected": -269.52020263671875,
+      "loss": 0.0568,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.871527373790741,
+      "rewards/margins": 4.85342264175415,
+      "rewards/rejected": -5.724949836730957,
+      "step": 362
+    },
+    {
+      "epoch": 2.671559633027523,
+      "grad_norm": 0.4651119112968445,
+      "learning_rate": 1.7524379964325155e-07,
+      "logits/chosen": -0.6477389931678772,
+      "logits/rejected": -1.8673548698425293,
+      "logps/chosen": -327.2629699707031,
+      "logps/rejected": -260.98095703125,
+      "loss": 0.0737,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -1.1671123504638672,
+      "rewards/margins": 4.629344940185547,
+      "rewards/rejected": -5.796456813812256,
+      "step": 364
+    },
+    {
+      "epoch": 2.686238532110092,
+      "grad_norm": 0.469937264919281,
+      "learning_rate": 1.5984279455240975e-07,
+      "logits/chosen": -0.6893140077590942,
+      "logits/rejected": -1.7950717210769653,
+      "logps/chosen": -279.4891662597656,
+      "logps/rejected": -248.5086669921875,
+      "loss": 0.0408,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8348989486694336,
+      "rewards/margins": 4.693355560302734,
+      "rewards/rejected": -5.52825403213501,
+      "step": 366
+    },
+    {
+      "epoch": 2.7009174311926607,
+      "grad_norm": 0.3594943881034851,
+      "learning_rate": 1.451276914049818e-07,
+      "logits/chosen": -0.6432421207427979,
+      "logits/rejected": -1.7735950946807861,
+      "logps/chosen": -256.6174011230469,
+      "logps/rejected": -232.81739807128906,
+      "loss": 0.0514,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8760130405426025,
+      "rewards/margins": 4.517703056335449,
+      "rewards/rejected": -5.393716335296631,
+      "step": 368
+    },
+    {
+      "epoch": 2.7155963302752295,
+      "grad_norm": 0.8683236241340637,
+      "learning_rate": 1.3110280320943692e-07,
+      "logits/chosen": -0.6071776151657104,
+      "logits/rejected": -1.878743290901184,
+      "logps/chosen": -268.89874267578125,
+      "logps/rejected": -227.52093505859375,
+      "loss": 0.045,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9803122282028198,
+      "rewards/margins": 4.631489276885986,
+      "rewards/rejected": -5.611801624298096,
+      "step": 370
+    },
+    {
+      "epoch": 2.7302752293577983,
+      "grad_norm": 0.3329331874847412,
+      "learning_rate": 1.1777224067169218e-07,
+      "logits/chosen": -0.6055042147636414,
+      "logits/rejected": -1.763778805732727,
+      "logps/chosen": -276.3477478027344,
+      "logps/rejected": -251.5518798828125,
+      "loss": 0.0493,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8497152924537659,
+      "rewards/margins": 4.952919960021973,
+      "rewards/rejected": -5.8026347160339355,
+      "step": 372
+    },
+    {
+      "epoch": 2.744954128440367,
+      "grad_norm": 0.6363049149513245,
+      "learning_rate": 1.0513991099025872e-07,
+      "logits/chosen": -0.7439496517181396,
+      "logits/rejected": -1.894633412361145,
+      "logps/chosen": -322.8822326660156,
+      "logps/rejected": -242.20285034179688,
+      "loss": 0.0739,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.9003086090087891,
+      "rewards/margins": 3.9627842903137207,
+      "rewards/rejected": -4.863093376159668,
+      "step": 374
+    },
+    {
+      "epoch": 2.759633027522936,
+      "grad_norm": 0.7462967038154602,
+      "learning_rate": 9.320951671104194e-08,
+      "logits/chosen": -0.6117614507675171,
+      "logits/rejected": -1.8900599479675293,
+      "logps/chosen": -307.9691467285156,
+      "logps/rejected": -243.08108520507812,
+      "loss": 0.0622,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6807048916816711,
+      "rewards/margins": 4.540770053863525,
+      "rewards/rejected": -5.221475124359131,
+      "step": 376
+    },
+    {
+      "epoch": 2.7743119266055047,
+      "grad_norm": 0.5611811876296997,
+      "learning_rate": 8.198455464212108e-08,
+      "logits/chosen": -0.7264785766601562,
+      "logits/rejected": -1.8659650087356567,
+      "logps/chosen": -290.5928955078125,
+      "logps/rejected": -231.83981323242188,
+      "loss": 0.0447,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.7871606349945068,
+      "rewards/margins": 4.830132484436035,
+      "rewards/rejected": -5.617292881011963,
+      "step": 378
+    },
+    {
+      "epoch": 2.7889908256880735,
+      "grad_norm": 1.062226414680481,
+      "learning_rate": 7.146831482883115e-08,
+      "logits/chosen": -0.41257715225219727,
+      "logits/rejected": -1.8329212665557861,
+      "logps/chosen": -295.2815856933594,
+      "logps/rejected": -231.45794677734375,
+      "loss": 0.0673,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.9870278239250183,
+      "rewards/margins": 4.97043514251709,
+      "rewards/rejected": -5.957463264465332,
+      "step": 380
+    },
+    {
+      "epoch": 2.8036697247706424,
+      "grad_norm": 0.9914811253547668,
+      "learning_rate": 6.16638795894492e-08,
+      "logits/chosen": -0.5121580958366394,
+      "logits/rejected": -1.6407995223999023,
+      "logps/chosen": -260.80078125,
+      "logps/rejected": -255.26918029785156,
+      "loss": 0.0655,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9135465025901794,
+      "rewards/margins": 4.523343563079834,
+      "rewards/rejected": -5.4368896484375,
+      "step": 382
+    },
+    {
+      "epoch": 2.818348623853211,
+      "grad_norm": 0.7881761193275452,
+      "learning_rate": 5.257412261176375e-08,
+      "logits/chosen": -0.7487360835075378,
+      "logits/rejected": -1.7608070373535156,
+      "logps/chosen": -270.4583740234375,
+      "logps/rejected": -244.21591186523438,
+      "loss": 0.0516,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9595422744750977,
+      "rewards/margins": 4.419933319091797,
+      "rewards/rejected": -5.3794755935668945,
+      "step": 384
+    },
+    {
+      "epoch": 2.83302752293578,
+      "grad_norm": 1.0176548957824707,
+      "learning_rate": 4.4201708110795384e-08,
+      "logits/chosen": -0.635594367980957,
+      "logits/rejected": -1.7197755575180054,
+      "logps/chosen": -291.9073791503906,
+      "logps/rejected": -257.1024169921875,
+      "loss": 0.0643,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9439809322357178,
+      "rewards/margins": 4.196859359741211,
+      "rewards/rejected": -5.140840530395508,
+      "step": 386
+    },
+    {
+      "epoch": 2.847706422018349,
+      "grad_norm": 0.39054185152053833,
+      "learning_rate": 3.654909004791152e-08,
+      "logits/chosen": -0.704308807849884,
+      "logits/rejected": -1.9551911354064941,
+      "logps/chosen": -291.7530517578125,
+      "logps/rejected": -240.93592834472656,
+      "loss": 0.0604,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.8366247415542603,
+      "rewards/margins": 4.685898780822754,
+      "rewards/rejected": -5.522522926330566,
+      "step": 388
+    },
+    {
+      "epoch": 2.8623853211009176,
+      "grad_norm": 0.536412239074707,
+      "learning_rate": 2.9618511411570462e-08,
+      "logits/chosen": -0.7265235185623169,
+      "logits/rejected": -1.952091097831726,
+      "logps/chosen": -284.6298522949219,
+      "logps/rejected": -226.05140686035156,
+      "loss": 0.0692,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -1.0189356803894043,
+      "rewards/margins": 4.247507572174072,
+      "rewards/rejected": -5.266443252563477,
+      "step": 390
+    },
+    {
+      "epoch": 2.8770642201834864,
+      "grad_norm": 0.7545129060745239,
+      "learning_rate": 2.3412003559898088e-08,
+      "logits/chosen": -0.6549080610275269,
+      "logits/rejected": -1.6496838331222534,
+      "logps/chosen": -270.34552001953125,
+      "logps/rejected": -260.4422302246094,
+      "loss": 0.0697,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.8946044445037842,
+      "rewards/margins": 4.255610466003418,
+      "rewards/rejected": -5.150214672088623,
+      "step": 392
+    },
+    {
+      "epoch": 2.891743119266055,
+      "grad_norm": 2.283723831176758,
+      "learning_rate": 1.793138562529634e-08,
+      "logits/chosen": -0.6721558570861816,
+      "logits/rejected": -1.893051266670227,
+      "logps/chosen": -344.69683837890625,
+      "logps/rejected": -235.408447265625,
+      "loss": 0.0654,
+      "rewards/accuracies": 0.984375,
+      "rewards/chosen": -0.9549738764762878,
+      "rewards/margins": 4.040391445159912,
+      "rewards/rejected": -4.995365142822266,
+      "step": 394
+    },
+    {
+      "epoch": 2.906422018348624,
+      "grad_norm": 0.4293775260448456,
+      "learning_rate": 1.317826398125277e-08,
+      "logits/chosen": -0.7664704322814941,
+      "logits/rejected": -1.8490607738494873,
+      "logps/chosen": -290.996337890625,
+      "logps/rejected": -263.40814208984375,
+      "loss": 0.0492,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9369643926620483,
+      "rewards/margins": 5.010738849639893,
+      "rewards/rejected": -5.9477033615112305,
+      "step": 396
+    },
+    {
+      "epoch": 2.921100917431193,
+      "grad_norm": 0.5313416123390198,
+      "learning_rate": 9.15403177151275e-09,
+      "logits/chosen": -0.6522485017776489,
+      "logits/rejected": -1.8088821172714233,
+      "logps/chosen": -272.3266296386719,
+      "logps/rejected": -275.0253601074219,
+      "loss": 0.0367,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9389075040817261,
+      "rewards/margins": 4.837636470794678,
+      "rewards/rejected": -5.776544094085693,
+      "step": 398
+    },
+    {
+      "epoch": 2.9357798165137616,
+      "grad_norm": 0.7444632053375244,
+      "learning_rate": 5.85986850174608e-09,
+      "logits/chosen": -0.6238083839416504,
+      "logits/rejected": -2.0871429443359375,
+      "logps/chosen": -310.4813232421875,
+      "logps/rejected": -242.80917358398438,
+      "loss": 0.0519,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9147568345069885,
+      "rewards/margins": 4.864949703216553,
+      "rewards/rejected": -5.779706001281738,
+      "step": 400
+    },
+    {
+      "epoch": 2.9504587155963304,
+      "grad_norm": 1.1092686653137207,
+      "learning_rate": 3.296739693834927e-09,
+      "logits/chosen": -0.82442706823349,
+      "logits/rejected": -1.7480018138885498,
+      "logps/chosen": -306.08526611328125,
+      "logps/rejected": -235.5576171875,
+      "loss": 0.0999,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -1.1727603673934937,
+      "rewards/margins": 3.904568672180176,
+      "rewards/rejected": -5.077329158782959,
+      "step": 402
+    },
+    {
+      "epoch": 2.9651376146788992,
+      "grad_norm": 0.5375072360038757,
+      "learning_rate": 1.4653966028774225e-09,
+      "logits/chosen": -0.6952610611915588,
+      "logits/rejected": -1.7414966821670532,
+      "logps/chosen": -313.39166259765625,
+      "logps/rejected": -276.1308288574219,
+      "loss": 0.0371,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1172269582748413,
+      "rewards/margins": 4.964841842651367,
+      "rewards/rejected": -6.0820698738098145,
+      "step": 404
+    },
+    {
+      "epoch": 2.979816513761468,
+      "grad_norm": 0.6843758225440979,
+      "learning_rate": 3.6637599699351766e-10,
+      "logits/chosen": -0.6138722896575928,
+      "logits/rejected": -1.8517141342163086,
+      "logps/chosen": -289.3631591796875,
+      "logps/rejected": -234.05596923828125,
+      "loss": 0.0817,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.0136228799819946,
+      "rewards/margins": 4.223280906677246,
+      "rewards/rejected": -5.236903667449951,
+      "step": 406
+    },
+    {
+      "epoch": 2.994495412844037,
+      "grad_norm": 0.888327956199646,
+      "learning_rate": 0.0,
+      "logits/chosen": -0.7368099093437195,
+      "logits/rejected": -1.7805346250534058,
+      "logps/chosen": -316.90399169921875,
+      "logps/rejected": -268.7693176269531,
+      "loss": 0.065,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.9527171850204468,
+      "rewards/margins": 4.548114776611328,
+      "rewards/rejected": -5.5008320808410645,
+      "step": 408
+    },
+    {
+      "epoch": 2.994495412844037,
+      "step": 408,
+      "total_flos": 7.837376281021809e+17,
+      "train_loss": 0.23205441671113172,
+      "train_runtime": 8090.8922,
+      "train_samples_per_second": 1.616,
+      "train_steps_per_second": 0.05
+    }
+  ],
+  "logging_steps": 2,
+  "max_steps": 408,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7.837376281021809e+17,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}