diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -1,29 +1,2939 @@
 {
-  "best_metric": null,
-  "best_model_checkpoint": null,
-  "epoch": 1.0,
-  "eval_steps": 10,
-  "global_step": 1,
+  "best_metric": 1.347064733505249,
+  "best_model_checkpoint": "saves/Gemma-7B-It/lora/orpo/checkpoint-1500",
+  "epoch": 2.997999555456768,
+  "eval_steps": 500,
+  "global_step": 1686,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 1.0,
-      "step": 1,
-      "total_flos": 748013124648960.0,
-      "train_loss": 2.0597352981567383,
-      "train_runtime": 12.1534,
-      "train_samples_per_second": 0.741,
-      "train_steps_per_second": 0.082
+      "epoch": 0.017781729273171815,
+      "grad_norm": 2.0664310455322266,
+      "learning_rate": 4.9995745934141085e-06,
+      "logits/chosen": 207.18429565429688,
+      "logits/rejected": 208.7433624267578,
+      "logps/chosen": -2.1626429557800293,
+      "logps/rejected": -2.4889369010925293,
+      "loss": 2.2441,
+      "odds_ratio_loss": 0.8149965405464172,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.2162642925977707,
+      "rewards/margins": 0.03262939304113388,
+      "rewards/rejected": -0.24889369308948517,
+      "sft_loss": 2.1626429557800293,
+      "step": 10
+    },
+    {
+      "epoch": 0.03556345854634363,
+      "grad_norm": 3.5887179374694824,
+      "learning_rate": 4.9982812903243405e-06,
+      "logits/chosen": 207.42489624023438,
+      "logits/rejected": 210.30307006835938,
+      "logps/chosen": -2.3814258575439453,
+      "logps/rejected": -2.279829740524292,
+      "loss": 2.4841,
+      "odds_ratio_loss": 1.0264532566070557,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.23814257979393005,
+      "rewards/margins": -0.01015959121286869,
+      "rewards/rejected": -0.2279830276966095,
+      "sft_loss": 2.3814258575439453,
+      "step": 20
+    },
+    {
+      "epoch": 0.05334518781951545,
+      "grad_norm": 7.692424297332764,
+      "learning_rate": 4.996120496405222e-06,
+      "logits/chosen": 210.8332977294922,
+      "logits/rejected": 211.61099243164062,
+      "logps/chosen": -2.3326590061187744,
+      "logps/rejected": -2.6421866416931152,
+      "loss": 2.4103,
+      "odds_ratio_loss": 0.7764064073562622,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.23326590657234192,
+      "rewards/margins": 0.03095281682908535,
+      "rewards/rejected": -0.2642187178134918,
+      "sft_loss": 2.3326590061187744,
+      "step": 30
+    },
+    {
+      "epoch": 0.07112691709268726,
+      "grad_norm": 2.3221993446350098,
+      "learning_rate": 4.99309296196014e-06,
+      "logits/chosen": 213.6151123046875,
+      "logits/rejected": 215.0422821044922,
+      "logps/chosen": -2.1157479286193848,
+      "logps/rejected": -2.377347469329834,
+      "loss": 2.2037,
+      "odds_ratio_loss": 0.8793656229972839,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.2115747630596161,
+      "rewards/margins": 0.026159971952438354,
+      "rewards/rejected": -0.23773476481437683,
+      "sft_loss": 2.1157479286193848,
+      "step": 40
+    },
+    {
+      "epoch": 0.08890864636585907,
+      "grad_norm": 3.239240884780884,
+      "learning_rate": 4.989199738255166e-06,
+      "logits/chosen": 215.62277221679688,
+      "logits/rejected": 217.9526824951172,
+      "logps/chosen": -2.122495174407959,
+      "logps/rejected": -2.3831381797790527,
+      "loss": 2.2102,
+      "odds_ratio_loss": 0.876750648021698,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.2122495174407959,
+      "rewards/margins": 0.0260643120855093,
+      "rewards/rejected": -0.23831383883953094,
+      "sft_loss": 2.122495174407959,
+      "step": 50
+    },
+    {
+      "epoch": 0.1066903756390309,
+      "grad_norm": 2.045614719390869,
+      "learning_rate": 4.984442177154031e-06,
+      "logits/chosen": 218.19265747070312,
+      "logits/rejected": 219.45907592773438,
+      "logps/chosen": -2.157195568084717,
+      "logps/rejected": -2.426860809326172,
+      "loss": 2.2519,
+      "odds_ratio_loss": 0.9465675354003906,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.215719535946846,
+      "rewards/margins": 0.02696654573082924,
+      "rewards/rejected": -0.24268610775470734,
+      "sft_loss": 2.157195568084717,
+      "step": 60
+    },
+    {
+      "epoch": 0.12447210491220272,
+      "grad_norm": 2.5059540271759033,
+      "learning_rate": 4.978821930648704e-06,
+      "logits/chosen": 222.6106414794922,
+      "logits/rejected": 222.9716339111328,
+      "logps/chosen": -2.108616590499878,
+      "logps/rejected": -2.0369954109191895,
+      "loss": 2.2226,
+      "odds_ratio_loss": 1.1399357318878174,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.21086165308952332,
+      "rewards/margins": -0.007162079215049744,
+      "rewards/rejected": -0.20369958877563477,
+      "sft_loss": 2.108616590499878,
+      "step": 70
+    },
+    {
+      "epoch": 0.14225383418537452,
+      "grad_norm": 3.193876028060913,
+      "learning_rate": 4.97234095028576e-06,
+      "logits/chosen": 226.2631378173828,
+      "logits/rejected": 227.37841796875,
+      "logps/chosen": -2.011748790740967,
+      "logps/rejected": -2.0834341049194336,
+      "loss": 2.0967,
+      "odds_ratio_loss": 0.849190354347229,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.20117488503456116,
+      "rewards/margins": 0.0071685537695884705,
+      "rewards/rejected": -0.20834341645240784,
+      "sft_loss": 2.011748790740967,
+      "step": 80
+    },
+    {
+      "epoch": 0.16003556345854633,
+      "grad_norm": 1.5080177783966064,
+      "learning_rate": 4.965001486488743e-06,
+      "logits/chosen": 227.5520477294922,
+      "logits/rejected": 228.8264923095703,
+      "logps/chosen": -1.7941173315048218,
+      "logps/rejected": -2.0014424324035645,
+      "loss": 1.8703,
+      "odds_ratio_loss": 0.7616177797317505,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.17941176891326904,
+      "rewards/margins": 0.020732494071125984,
+      "rewards/rejected": -0.20014424622058868,
+      "sft_loss": 1.7941173315048218,
+      "step": 90
+    },
+    {
+      "epoch": 0.17781729273171815,
+      "grad_norm": 1.543541669845581,
+      "learning_rate": 4.956806087776732e-06,
+      "logits/chosen": 230.88284301757812,
+      "logits/rejected": 231.593017578125,
+      "logps/chosen": -1.7711349725723267,
+      "logps/rejected": -2.1635921001434326,
+      "loss": 1.8365,
+      "odds_ratio_loss": 0.6534941792488098,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.17711350321769714,
+      "rewards/margins": 0.039245713502168655,
+      "rewards/rejected": -0.2163592129945755,
+      "sft_loss": 1.7711349725723267,
+      "step": 100
+    },
+    {
+      "epoch": 0.19559902200489,
+      "grad_norm": 2.5298948287963867,
+      "learning_rate": 4.947757599879411e-06,
+      "logits/chosen": 235.8949737548828,
+      "logits/rejected": 235.8369140625,
+      "logps/chosen": -1.6983522176742554,
+      "logps/rejected": -1.9385782480239868,
+      "loss": 1.7686,
+      "odds_ratio_loss": 0.7024893760681152,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.16983522474765778,
+      "rewards/margins": 0.024022620171308517,
+      "rewards/rejected": -0.1938578337430954,
+      "sft_loss": 1.6983522176742554,
+      "step": 110
+    },
+    {
+      "epoch": 0.2133807512780618,
+      "grad_norm": 3.6936049461364746,
+      "learning_rate": 4.937859164748931e-06,
+      "logits/chosen": 239.0133514404297,
+      "logits/rejected": 239.0744171142578,
+      "logps/chosen": -1.5454410314559937,
+      "logps/rejected": -1.6620423793792725,
+      "loss": 1.6182,
+      "odds_ratio_loss": 0.727584183216095,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.15454408526420593,
+      "rewards/margins": 0.011660170741379261,
+      "rewards/rejected": -0.16620425879955292,
+      "sft_loss": 1.5454410314559937,
+      "step": 120
+    },
+    {
+      "epoch": 0.23116248055123362,
+      "grad_norm": 2.5647566318511963,
+      "learning_rate": 4.92711421946891e-06,
+      "logits/chosen": 239.533203125,
+      "logits/rejected": 240.0679931640625,
+      "logps/chosen": -1.6095718145370483,
+      "logps/rejected": -1.8200355768203735,
+      "loss": 1.6813,
+      "odds_ratio_loss": 0.7172408103942871,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.16095717251300812,
+      "rewards/margins": 0.021046385169029236,
+      "rewards/rejected": -0.18200358748435974,
+      "sft_loss": 1.6095718145370483,
+      "step": 130
+    },
+    {
+      "epoch": 0.24894420982440543,
+      "grad_norm": 7.456451416015625,
+      "learning_rate": 4.915526495060961e-06,
+      "logits/chosen": 242.9502410888672,
+      "logits/rejected": 243.6327362060547,
+      "logps/chosen": -1.5086013078689575,
+      "logps/rejected": -1.7959527969360352,
+      "loss": 1.5772,
+      "odds_ratio_loss": 0.6855502128601074,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.15086011588573456,
+      "rewards/margins": 0.028735166415572166,
+      "rewards/rejected": -0.17959527671337128,
+      "sft_loss": 1.5086013078689575,
+      "step": 140
+    },
+    {
+      "epoch": 0.26672593909757725,
+      "grad_norm": 3.7102017402648926,
+      "learning_rate": 4.903100015189153e-06,
+      "logits/chosen": 243.65646362304688,
+      "logits/rejected": 243.8603973388672,
+      "logps/chosen": -1.5454883575439453,
+      "logps/rejected": -1.7633510828018188,
+      "loss": 1.6165,
+      "odds_ratio_loss": 0.7103039026260376,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.15454885363578796,
+      "rewards/margins": 0.021786261349916458,
+      "rewards/rejected": -0.17633512616157532,
+      "sft_loss": 1.5454883575439453,
+      "step": 150
+    },
+    {
+      "epoch": 0.28450766837074903,
+      "grad_norm": 1.9714837074279785,
+      "learning_rate": 4.889839094762848e-06,
+      "logits/chosen": 244.4144744873047,
+      "logits/rejected": 244.4231414794922,
+      "logps/chosen": -1.6150789260864258,
+      "logps/rejected": -1.7520729303359985,
+      "loss": 1.6882,
+      "odds_ratio_loss": 0.7308647036552429,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.16150791943073273,
+      "rewards/margins": 0.013699382543563843,
+      "rewards/rejected": -0.17520728707313538,
+      "sft_loss": 1.6150789260864258,
+      "step": 160
+    },
+    {
+      "epoch": 0.3022893976439209,
+      "grad_norm": 1.7966150045394897,
+      "learning_rate": 4.875748338438416e-06,
+      "logits/chosen": 245.6141815185547,
+      "logits/rejected": 245.50735473632812,
+      "logps/chosen": -1.5294561386108398,
+      "logps/rejected": -1.7359291315078735,
+      "loss": 1.5978,
+      "odds_ratio_loss": 0.683633029460907,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.1529456079006195,
+      "rewards/margins": 0.020647313445806503,
+      "rewards/rejected": -0.1735929250717163,
+      "sft_loss": 1.5294561386108398,
+      "step": 170
+    },
+    {
+      "epoch": 0.32007112691709266,
+      "grad_norm": 4.217123031616211,
+      "learning_rate": 4.8608326390203386e-06,
+      "logits/chosen": 245.0269317626953,
+      "logits/rejected": 245.72494506835938,
+      "logps/chosen": -1.4982770681381226,
+      "logps/rejected": -1.7273998260498047,
+      "loss": 1.5646,
+      "odds_ratio_loss": 0.663575291633606,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.1498277187347412,
+      "rewards/margins": 0.022912275046110153,
+      "rewards/rejected": -0.17273999750614166,
+      "sft_loss": 1.4982770681381226,
+      "step": 180
+    },
+    {
+      "epoch": 0.3378528561902645,
+      "grad_norm": 2.1345717906951904,
+      "learning_rate": 4.845097175762251e-06,
+      "logits/chosen": 246.3583984375,
+      "logits/rejected": 246.6023712158203,
+      "logps/chosen": -1.4693564176559448,
+      "logps/rejected": -1.6270701885223389,
+      "loss": 1.5391,
+      "odds_ratio_loss": 0.6974608898162842,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.1469356268644333,
+      "rewards/margins": 0.01577138900756836,
+      "rewards/rejected": -0.16270704567432404,
+      "sft_loss": 1.4693564176559448,
+      "step": 190
+    },
+    {
+      "epoch": 0.3556345854634363,
+      "grad_norm": 4.533120632171631,
+      "learning_rate": 4.8285474125685286e-06,
+      "logits/chosen": 245.734130859375,
+      "logits/rejected": 246.148193359375,
+      "logps/chosen": -1.5207107067108154,
+      "logps/rejected": -1.6708800792694092,
+      "loss": 1.5928,
+      "odds_ratio_loss": 0.7206315994262695,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.15207107365131378,
+      "rewards/margins": 0.015016931109130383,
+      "rewards/rejected": -0.16708801686763763,
+      "sft_loss": 1.5207107067108154,
+      "step": 200
+    },
+    {
+      "epoch": 0.37341631473660813,
+      "grad_norm": 2.590592861175537,
+      "learning_rate": 4.811189096097025e-06,
+      "logits/chosen": 244.85757446289062,
+      "logits/rejected": 245.19387817382812,
+      "logps/chosen": -1.4962936639785767,
+      "logps/rejected": -1.7369601726531982,
+      "loss": 1.5627,
+      "odds_ratio_loss": 0.6638623476028442,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.1496293693780899,
+      "rewards/margins": 0.024066651239991188,
+      "rewards/rejected": -0.17369601130485535,
+      "sft_loss": 1.4962936639785767,
+      "step": 210
+    },
+    {
+      "epoch": 0.39119804400978,
+      "grad_norm": 2.847151279449463,
+      "learning_rate": 4.793028253763633e-06,
+      "logits/chosen": 247.3143310546875,
+      "logits/rejected": 247.9826202392578,
+      "logps/chosen": -1.4769468307495117,
+      "logps/rejected": -1.6658426523208618,
+      "loss": 1.5539,
+      "odds_ratio_loss": 0.769631028175354,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.14769470691680908,
+      "rewards/margins": 0.018889565020799637,
+      "rewards/rejected": -0.16658425331115723,
+      "sft_loss": 1.4769468307495117,
+      "step": 220
+    },
+    {
+      "epoch": 0.40897977328295176,
+      "grad_norm": 2.361198902130127,
+      "learning_rate": 4.774071191649352e-06,
+      "logits/chosen": 247.5041961669922,
+      "logits/rejected": 247.5510711669922,
+      "logps/chosen": -1.3598966598510742,
+      "logps/rejected": -1.6928675174713135,
+      "loss": 1.421,
+      "odds_ratio_loss": 0.6115107536315918,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.13598966598510742,
+      "rewards/margins": 0.033297087997198105,
+      "rewards/rejected": -0.16928674280643463,
+      "sft_loss": 1.3598966598510742,
+      "step": 230
+    },
+    {
+      "epoch": 0.4267615025561236,
+      "grad_norm": 2.774188280105591,
+      "learning_rate": 4.7543244923105975e-06,
+      "logits/chosen": 248.81741333007812,
+      "logits/rejected": 248.56063842773438,
+      "logps/chosen": -1.479175090789795,
+      "logps/rejected": -1.5817492008209229,
+      "loss": 1.5553,
+      "odds_ratio_loss": 0.7613282799720764,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.14791752398014069,
+      "rewards/margins": 0.010257410816848278,
+      "rewards/rejected": -0.15817493200302124,
+      "sft_loss": 1.479175090789795,
+      "step": 240
+    },
+    {
+      "epoch": 0.4445432318292954,
+      "grad_norm": 1.5498669147491455,
+      "learning_rate": 4.733795012493506e-06,
+      "logits/chosen": 249.3889923095703,
+      "logits/rejected": 249.21475219726562,
+      "logps/chosen": -1.544840693473816,
+      "logps/rejected": -1.6548751592636108,
+      "loss": 1.6181,
+      "odds_ratio_loss": 0.7324239611625671,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.15448406338691711,
+      "rewards/margins": 0.011003440245985985,
+      "rewards/rejected": -0.16548751294612885,
+      "sft_loss": 1.544840693473816,
+      "step": 250
+    },
+    {
+      "epoch": 0.46232496110246724,
+      "grad_norm": 1.862833857536316,
+      "learning_rate": 4.712489880753035e-06,
+      "logits/chosen": 245.6899871826172,
+      "logits/rejected": 246.0701446533203,
+      "logps/chosen": -1.316460132598877,
+      "logps/rejected": -1.5619014501571655,
+      "loss": 1.3796,
+      "odds_ratio_loss": 0.631010890007019,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.13164600729942322,
+      "rewards/margins": 0.02454412914812565,
+      "rewards/rejected": -0.15619014203548431,
+      "sft_loss": 1.316460132598877,
+      "step": 260
+    },
+    {
+      "epoch": 0.480106690375639,
+      "grad_norm": 7.502230644226074,
+      "learning_rate": 4.690416494977673e-06,
+      "logits/chosen": 248.2025146484375,
+      "logits/rejected": 249.02481079101562,
+      "logps/chosen": -1.4416104555130005,
+      "logps/rejected": -1.7268083095550537,
+      "loss": 1.5056,
+      "odds_ratio_loss": 0.6399883031845093,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.14416104555130005,
+      "rewards/margins": 0.028519803658127785,
+      "rewards/rejected": -0.1726808398962021,
+      "sft_loss": 1.4416104555130005,
+      "step": 270
+    },
+    {
+      "epoch": 0.49788841964881086,
+      "grad_norm": 1.7004640102386475,
+      "learning_rate": 4.667582519820639e-06,
+      "logits/chosen": 248.4412841796875,
+      "logits/rejected": 248.5986328125,
+      "logps/chosen": -1.4644906520843506,
+      "logps/rejected": -1.6034975051879883,
+      "loss": 1.5385,
+      "odds_ratio_loss": 0.7402389645576477,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.14644907414913177,
+      "rewards/margins": 0.013900673016905785,
+      "rewards/rejected": -0.1603497415781021,
+      "sft_loss": 1.4644906520843506,
+      "step": 280
+    },
+    {
+      "epoch": 0.5156701489219827,
+      "grad_norm": 1.9778798818588257,
+      "learning_rate": 4.643995884038443e-06,
+      "logits/chosen": 249.9197540283203,
+      "logits/rejected": 249.89920043945312,
+      "logps/chosen": -1.390932559967041,
+      "logps/rejected": -1.5962976217269897,
+      "loss": 1.4582,
+      "odds_ratio_loss": 0.6727808713912964,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.13909325003623962,
+      "rewards/margins": 0.020536506548523903,
+      "rewards/rejected": -0.15962976217269897,
+      "sft_loss": 1.390932559967041,
+      "step": 290
+    },
+    {
+      "epoch": 0.5334518781951545,
+      "grad_norm": 1.796635627746582,
+      "learning_rate": 4.6196647777377475e-06,
+      "logits/chosen": 248.5753936767578,
+      "logits/rejected": 248.55288696289062,
+      "logps/chosen": -1.3909634351730347,
+      "logps/rejected": -1.4929519891738892,
+      "loss": 1.4621,
+      "odds_ratio_loss": 0.7113169431686401,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.13909634947776794,
+      "rewards/margins": 0.010198858566582203,
+      "rewards/rejected": -0.14929521083831787,
+      "sft_loss": 1.3909634351730347,
+      "step": 300
+    },
+    {
+      "epoch": 0.5512336074683263,
+      "grad_norm": 1.5784835815429688,
+      "learning_rate": 4.59459764953147e-06,
+      "logits/chosen": 247.842529296875,
+      "logits/rejected": 248.6540069580078,
+      "logps/chosen": -1.449033498764038,
+      "logps/rejected": -1.587956190109253,
+      "loss": 1.5181,
+      "odds_ratio_loss": 0.6905879974365234,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.14490333199501038,
+      "rewards/margins": 0.013892298564314842,
+      "rewards/rejected": -0.15879563987255096,
+      "sft_loss": 1.449033498764038,
+      "step": 310
+    },
+    {
+      "epoch": 0.5690153367414981,
+      "grad_norm": 2.8430497646331787,
+      "learning_rate": 4.568803203605133e-06,
+      "logits/chosen": 247.75045776367188,
+      "logits/rejected": 247.95272827148438,
+      "logps/chosen": -1.3736222982406616,
+      "logps/rejected": -1.5858898162841797,
+      "loss": 1.4444,
+      "odds_ratio_loss": 0.7075997591018677,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.13736224174499512,
+      "rewards/margins": 0.02122673951089382,
+      "rewards/rejected": -0.15858899056911469,
+      "sft_loss": 1.3736222982406616,
+      "step": 320
+    },
+    {
+      "epoch": 0.58679706601467,
+      "grad_norm": 8.24238395690918,
+      "learning_rate": 4.542290396694462e-06,
+      "logits/chosen": 250.15707397460938,
+      "logits/rejected": 249.8236541748047,
+      "logps/chosen": -1.4158143997192383,
+      "logps/rejected": -1.5936088562011719,
+      "loss": 1.4861,
+      "odds_ratio_loss": 0.7032537460327148,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.1415814310312271,
+      "rewards/margins": 0.017779454588890076,
+      "rewards/rejected": -0.15936090052127838,
+      "sft_loss": 1.4158143997192383,
+      "step": 330
+    },
+    {
+      "epoch": 0.6045787952878418,
+      "grad_norm": 8.322469711303711,
+      "learning_rate": 4.515068434975298e-06,
+      "logits/chosen": 247.47579956054688,
+      "logits/rejected": 248.01553344726562,
+      "logps/chosen": -1.413419246673584,
+      "logps/rejected": -1.6688693761825562,
+      "loss": 1.4802,
+      "odds_ratio_loss": 0.6680575013160706,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.1413419246673584,
+      "rewards/margins": 0.02554500475525856,
+      "rewards/rejected": -0.16688695549964905,
+      "sft_loss": 1.413419246673584,
+      "step": 340
+    },
+    {
+      "epoch": 0.6223605245610135,
+      "grad_norm": 1.7108134031295776,
+      "learning_rate": 4.487146770866887e-06,
+      "logits/chosen": 249.2637939453125,
+      "logits/rejected": 249.53549194335938,
+      "logps/chosen": -1.3779569864273071,
+      "logps/rejected": -1.5148077011108398,
+      "loss": 1.4475,
+      "odds_ratio_loss": 0.6950393915176392,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.13779568672180176,
+      "rewards/margins": 0.013685077428817749,
+      "rewards/rejected": -0.1514807641506195,
+      "sft_loss": 1.3779569864273071,
+      "step": 350
+    },
+    {
+      "epoch": 0.6401422538341853,
+      "grad_norm": 1.5160853862762451,
+      "learning_rate": 4.458535099749666e-06,
+      "logits/chosen": 251.33633422851562,
+      "logits/rejected": 251.1912841796875,
+      "logps/chosen": -1.4455270767211914,
+      "logps/rejected": -1.5792427062988281,
+      "loss": 1.5213,
+      "odds_ratio_loss": 0.7574425935745239,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.14455269277095795,
+      "rewards/margins": 0.013371584005653858,
+      "rewards/rejected": -0.15792429447174072,
+      "sft_loss": 1.4455270767211914,
+      "step": 360
+    },
+    {
+      "epoch": 0.6579239831073572,
+      "grad_norm": 2.7603657245635986,
+      "learning_rate": 4.429243356598694e-06,
+      "logits/chosen": 249.0806121826172,
+      "logits/rejected": 249.3708038330078,
+      "logps/chosen": -1.4384644031524658,
+      "logps/rejected": -1.6943752765655518,
+      "loss": 1.504,
+      "odds_ratio_loss": 0.655007004737854,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.14384643733501434,
+      "rewards/margins": 0.025591086596250534,
+      "rewards/rejected": -0.16943752765655518,
+      "sft_loss": 1.4384644031524658,
+      "step": 370
+    },
+    {
+      "epoch": 0.675705712380529,
+      "grad_norm": 1.7461826801300049,
+      "learning_rate": 4.399281712533875e-06,
+      "logits/chosen": 249.9875030517578,
+      "logits/rejected": 250.59976196289062,
+      "logps/chosen": -1.385258674621582,
+      "logps/rejected": -1.5310487747192383,
+      "loss": 1.4575,
+      "odds_ratio_loss": 0.7227787375450134,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.13852587342262268,
+      "rewards/margins": 0.014579012989997864,
+      "rewards/rejected": -0.15310488641262054,
+      "sft_loss": 1.385258674621582,
+      "step": 380
+    },
+    {
+      "epoch": 0.6934874416537008,
+      "grad_norm": 2.5713703632354736,
+      "learning_rate": 4.368660571288192e-06,
+      "logits/chosen": 249.9259490966797,
+      "logits/rejected": 250.7145538330078,
+      "logps/chosen": -1.3335117101669312,
+      "logps/rejected": -1.4699729681015015,
+      "loss": 1.404,
+      "odds_ratio_loss": 0.7047192454338074,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.1333511769771576,
+      "rewards/margins": 0.013646131381392479,
+      "rewards/rejected": -0.14699730277061462,
+      "sft_loss": 1.3335117101669312,
+      "step": 390
+    },
+    {
+      "epoch": 0.7112691709268726,
+      "grad_norm": 1.9281286001205444,
+      "learning_rate": 4.337390565595163e-06,
+      "logits/chosen": 251.0037384033203,
+      "logits/rejected": 250.80783081054688,
+      "logps/chosen": -1.4610103368759155,
+      "logps/rejected": -1.5656871795654297,
+      "loss": 1.5334,
+      "odds_ratio_loss": 0.7237542271614075,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.14610104262828827,
+      "rewards/margins": 0.010467682033777237,
+      "rewards/rejected": -0.1565687209367752,
+      "sft_loss": 1.4610103368759155,
+      "step": 400
+    },
+    {
+      "epoch": 0.7290509002000445,
+      "grad_norm": 2.2883145809173584,
+      "learning_rate": 4.305482553496786e-06,
+      "logits/chosen": 249.4977569580078,
+      "logits/rejected": 249.899658203125,
+      "logps/chosen": -1.3607113361358643,
+      "logps/rejected": -1.5497020483016968,
+      "loss": 1.4299,
+      "odds_ratio_loss": 0.692295491695404,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.1360711306333542,
+      "rewards/margins": 0.018899066373705864,
+      "rewards/rejected": -0.1549702137708664,
+      "sft_loss": 1.3607113361358643,
+      "step": 410
+    },
+    {
+      "epoch": 0.7468326294732163,
+      "grad_norm": 1.6453748941421509,
+      "learning_rate": 4.272947614573244e-06,
+      "logits/chosen": 251.3318328857422,
+      "logits/rejected": 251.18276977539062,
+      "logps/chosen": -1.3767774105072021,
+      "logps/rejected": -1.5617191791534424,
+      "loss": 1.4431,
+      "odds_ratio_loss": 0.662979006767273,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.13767775893211365,
+      "rewards/margins": 0.01849415898323059,
+      "rewards/rejected": -0.15617190301418304,
+      "sft_loss": 1.3767774105072021,
+      "step": 420
+    },
+    {
+      "epoch": 0.7646143587463881,
+      "grad_norm": 1.49653959274292,
+      "learning_rate": 4.23979704609569e-06,
+      "logits/chosen": 251.7825469970703,
+      "logits/rejected": 251.8596954345703,
+      "logps/chosen": -1.360251545906067,
+      "logps/rejected": -1.4845999479293823,
+      "loss": 1.43,
+      "odds_ratio_loss": 0.6971336603164673,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.13602514564990997,
+      "rewards/margins": 0.012434848584234715,
+      "rewards/rejected": -0.1484600007534027,
+      "sft_loss": 1.360251545906067,
+      "step": 430
+    },
+    {
+      "epoch": 0.78239608801956,
+      "grad_norm": 2.6121129989624023,
+      "learning_rate": 4.206042359103435e-06,
+      "logits/chosen": 249.96774291992188,
+      "logits/rejected": 249.6472930908203,
+      "logps/chosen": -1.451643705368042,
+      "logps/rejected": -1.6453460454940796,
+      "loss": 1.5216,
+      "odds_ratio_loss": 0.6996283531188965,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.145164355635643,
+      "rewards/margins": 0.01937025412917137,
+      "rewards/rejected": -0.16453461349010468,
+      "sft_loss": 1.451643705368042,
+      "step": 440
+    },
+    {
+      "epoch": 0.8001778172927317,
+      "grad_norm": 4.463571548461914,
+      "learning_rate": 4.17169527440691e-06,
+      "logits/chosen": 249.04443359375,
+      "logits/rejected": 249.074951171875,
+      "logps/chosen": -1.3891347646713257,
+      "logps/rejected": -1.5458732843399048,
+      "loss": 1.4601,
+      "odds_ratio_loss": 0.7098666429519653,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.13891348242759705,
+      "rewards/margins": 0.015673857182264328,
+      "rewards/rejected": -0.15458734333515167,
+      "sft_loss": 1.3891347646713257,
+      "step": 450
+    },
+    {
+      "epoch": 0.8179595465659035,
+      "grad_norm": 3.0045888423919678,
+      "learning_rate": 4.136767718517797e-06,
+      "logits/chosen": 250.2001495361328,
+      "logits/rejected": 250.69189453125,
+      "logps/chosen": -1.2655476331710815,
+      "logps/rejected": -1.5482288599014282,
+      "loss": 1.3289,
+      "odds_ratio_loss": 0.6339131593704224,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.12655475735664368,
+      "rewards/margins": 0.02826813980937004,
+      "rewards/rejected": -0.15482288599014282,
+      "sft_loss": 1.2655476331710815,
+      "step": 460
+    },
+    {
+      "epoch": 0.8357412758390753,
+      "grad_norm": 5.639819145202637,
+      "learning_rate": 4.1012718195077196e-06,
+      "logits/chosen": 250.90878295898438,
+      "logits/rejected": 251.2098388671875,
+      "logps/chosen": -1.3541600704193115,
+      "logps/rejected": -1.4638829231262207,
+      "loss": 1.4238,
+      "odds_ratio_loss": 0.6962955594062805,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.13541600108146667,
+      "rewards/margins": 0.01097229402512312,
+      "rewards/rejected": -0.14638829231262207,
+      "sft_loss": 1.3541600704193115,
+      "step": 470
+    },
+    {
+      "epoch": 0.8535230051122472,
+      "grad_norm": 2.0918374061584473,
+      "learning_rate": 4.065219902796953e-06,
+      "logits/chosen": 250.6866455078125,
+      "logits/rejected": 250.68875122070312,
+      "logps/chosen": -1.3266746997833252,
+      "logps/rejected": -1.5351797342300415,
+      "loss": 1.3953,
+      "odds_ratio_loss": 0.6859580278396606,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.13266746699810028,
+      "rewards/margins": 0.02085050381720066,
+      "rewards/rejected": -0.1535179764032364,
+      "sft_loss": 1.3266746997833252,
+      "step": 480
+    },
+    {
+      "epoch": 0.871304734385419,
+      "grad_norm": 1.7283604145050049,
+      "learning_rate": 4.028624486874608e-06,
+      "logits/chosen": 250.7782745361328,
+      "logits/rejected": 250.49868774414062,
+      "logps/chosen": -1.3391790390014648,
+      "logps/rejected": -1.5556788444519043,
+      "loss": 1.4069,
+      "odds_ratio_loss": 0.6768004894256592,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.133917897939682,
+      "rewards/margins": 0.021649986505508423,
+      "rewards/rejected": -0.15556788444519043,
+      "sft_loss": 1.3391790390014648,
+      "step": 490
+    },
+    {
+      "epoch": 0.8890864636585908,
+      "grad_norm": 1.839385747909546,
+      "learning_rate": 3.99149827895177e-06,
+      "logits/chosen": 251.3027801513672,
+      "logits/rejected": 250.90676879882812,
+      "logps/chosen": -1.4337760210037231,
+      "logps/rejected": -1.565932273864746,
+      "loss": 1.5041,
+      "odds_ratio_loss": 0.7032449841499329,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.14337760210037231,
+      "rewards/margins": 0.01321563869714737,
+      "rewards/rejected": -0.1565932184457779,
+      "sft_loss": 1.4337760210037231,
+      "step": 500
+    },
+    {
+      "epoch": 0.8890864636585908,
+      "eval_logits/chosen": 250.64625549316406,
+      "eval_logits/rejected": 250.75489807128906,
+      "eval_logps/chosen": -1.352203369140625,
+      "eval_logps/rejected": -1.5643647909164429,
+      "eval_loss": 1.418463945388794,
+      "eval_odds_ratio_loss": 0.6626060605049133,
+      "eval_rewards/accuracies": 0.5529999732971191,
+      "eval_rewards/chosen": -0.13522033393383026,
+      "eval_rewards/margins": 0.021216128021478653,
+      "eval_rewards/rejected": -0.15643645823001862,
+      "eval_runtime": 217.284,
+      "eval_samples_per_second": 4.602,
+      "eval_sft_loss": 1.352203369140625,
+      "eval_steps_per_second": 2.301,
+      "step": 500
+    },
+    {
+      "epoch": 0.9068681929317626,
+      "grad_norm": 4.217110633850098,
+      "learning_rate": 3.953854170549114e-06,
+      "logits/chosen": 251.8994598388672,
+      "logits/rejected": 251.8975067138672,
+      "logps/chosen": -1.34084153175354,
+      "logps/rejected": -1.4915145635604858,
+      "loss": 1.4078,
+      "odds_ratio_loss": 0.6700819730758667,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.13408413529396057,
+      "rewards/margins": 0.015067322179675102,
+      "rewards/rejected": -0.14915145933628082,
+      "sft_loss": 1.34084153175354,
+      "step": 510
+    },
+    {
+      "epoch": 0.9246499222049345,
+      "grad_norm": 2.6186540126800537,
+      "learning_rate": 3.91570523302051e-06,
+      "logits/chosen": 250.59036254882812,
+      "logits/rejected": 250.75003051757812,
+      "logps/chosen": -1.3541263341903687,
+      "logps/rejected": -1.4764378070831299,
+      "loss": 1.427,
+      "odds_ratio_loss": 0.728742241859436,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.13541266322135925,
+      "rewards/margins": 0.012231158092617989,
+      "rewards/rejected": -0.1476438045501709,
+      "sft_loss": 1.3541263341903687,
+      "step": 520
+    },
+    {
+      "epoch": 0.9424316514781063,
+      "grad_norm": 2.1178462505340576,
+      "learning_rate": 3.8770647130141996e-06,
+      "logits/chosen": 250.88150024414062,
+      "logits/rejected": 251.63052368164062,
+      "logps/chosen": -1.3341526985168457,
+      "logps/rejected": -1.5821571350097656,
+      "loss": 1.4029,
+      "odds_ratio_loss": 0.6879525780677795,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.13341526687145233,
+      "rewards/margins": 0.02480044774711132,
+      "rewards/rejected": -0.1582157164812088,
+      "sft_loss": 1.3341526985168457,
+      "step": 530
+    },
+    {
+      "epoch": 0.960213380751278,
+      "grad_norm": 2.5758562088012695,
+      "learning_rate": 3.837946027873086e-06,
+      "logits/chosen": 249.52285766601562,
+      "logits/rejected": 250.219970703125,
+      "logps/chosen": -1.4206160306930542,
+      "logps/rejected": -1.6295530796051025,
+      "loss": 1.4917,
+      "odds_ratio_loss": 0.7105556726455688,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.14206160604953766,
+      "rewards/margins": 0.020893706008791924,
+      "rewards/rejected": -0.16295531392097473,
+      "sft_loss": 1.4206160306930542,
+      "step": 540
+    },
+    {
+      "epoch": 0.9779951100244498,
+      "grad_norm": 2.2665178775787354,
+      "learning_rate": 3.7983627609757713e-06,
+      "logits/chosen": 250.9306640625,
+      "logits/rejected": 251.22189331054688,
+      "logps/chosen": -1.3822170495986938,
+      "logps/rejected": -1.51272714138031,
+      "loss": 1.4502,
+      "odds_ratio_loss": 0.679997444152832,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.13822171092033386,
+      "rewards/margins": 0.01305101066827774,
+      "rewards/rejected": -0.1512727290391922,
+      "sft_loss": 1.3822170495986938,
+      "step": 550
+    },
+    {
+      "epoch": 0.9957768392976217,
+      "grad_norm": 9.529622077941895,
+      "learning_rate": 3.758328657019924e-06,
+      "logits/chosen": 252.7086639404297,
+      "logits/rejected": 252.86575317382812,
+      "logps/chosen": -1.3466806411743164,
+      "logps/rejected": -1.4755653142929077,
+      "loss": 1.4192,
+      "odds_ratio_loss": 0.7249861359596252,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.13466808199882507,
+      "rewards/margins": 0.012888476252555847,
+      "rewards/rejected": -0.14755654335021973,
+      "sft_loss": 1.3466806411743164,
+      "step": 560
+    },
+    {
+      "epoch": 1.0135585685707935,
+      "grad_norm": 8.696991920471191,
+      "learning_rate": 3.717857617249642e-06,
+      "logits/chosen": 250.4936981201172,
+      "logits/rejected": 250.49435424804688,
+      "logps/chosen": -1.4108977317810059,
+      "logps/rejected": -1.591428279876709,
+      "loss": 1.4822,
+      "odds_ratio_loss": 0.7131791710853577,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.1410897672176361,
+      "rewards/margins": 0.018053067848086357,
+      "rewards/rejected": -0.1591428518295288,
+      "sft_loss": 1.4108977317810059,
+      "step": 570
+    },
+    {
+      "epoch": 1.0313402978439654,
+      "grad_norm": 1.4178659915924072,
+      "learning_rate": 3.6769636946284543e-06,
+      "logits/chosen": 252.4293975830078,
+      "logits/rejected": 252.90542602539062,
+      "logps/chosen": -1.2866795063018799,
+      "logps/rejected": -1.4288690090179443,
+      "loss": 1.3561,
+      "odds_ratio_loss": 0.694588840007782,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.12866798043251038,
+      "rewards/margins": 0.01421893946826458,
+      "rewards/rejected": -0.1428869068622589,
+      "sft_loss": 1.2866795063018799,
+      "step": 580
+    },
+    {
+      "epoch": 1.049122027117137,
+      "grad_norm": 2.9459097385406494,
+      "learning_rate": 3.6356610889596355e-06,
+      "logits/chosen": 251.82577514648438,
+      "logits/rejected": 252.0190887451172,
+      "logps/chosen": -1.3364391326904297,
+      "logps/rejected": -1.4779332876205444,
+      "loss": 1.4062,
+      "odds_ratio_loss": 0.6974235773086548,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.13364391028881073,
+      "rewards/margins": 0.014149405062198639,
+      "rewards/rejected": -0.14779332280158997,
+      "sft_loss": 1.3364391326904297,
+      "step": 590
+    },
+    {
+      "epoch": 1.066903756390309,
+      "grad_norm": 2.037022590637207,
+      "learning_rate": 3.593964141955541e-06,
+      "logits/chosen": 251.60153198242188,
+      "logits/rejected": 252.28738403320312,
+      "logps/chosen": -1.3261809349060059,
+      "logps/rejected": -1.4387245178222656,
+      "loss": 1.3975,
+      "odds_ratio_loss": 0.7132095098495483,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.13261809945106506,
+      "rewards/margins": 0.011254330165684223,
+      "rewards/rejected": -0.1438724398612976,
+      "sft_loss": 1.3261809349060059,
+      "step": 600
+    },
+    {
+      "epoch": 1.0846854856634809,
+      "grad_norm": 1.648507833480835,
+      "learning_rate": 3.5518873322576573e-06,
+      "logits/chosen": 251.1070556640625,
+      "logits/rejected": 251.15121459960938,
+      "logps/chosen": -1.3120733499526978,
+      "logps/rejected": -1.5062081813812256,
+      "loss": 1.3823,
+      "odds_ratio_loss": 0.7022742033004761,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.13120731711387634,
+      "rewards/margins": 0.019413486123085022,
+      "rewards/rejected": -0.15062081813812256,
+      "sft_loss": 1.3120733499526978,
+      "step": 610
+    },
+    {
+      "epoch": 1.1024672149366526,
+      "grad_norm": 1.7841641902923584,
+      "learning_rate": 3.5094452704091143e-06,
+      "logits/chosen": 253.28305053710938,
+      "logits/rejected": 253.123046875,
+      "logps/chosen": -1.3503539562225342,
+      "logps/rejected": -1.5104092359542847,
+      "loss": 1.4195,
+      "odds_ratio_loss": 0.6916577219963074,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.13503538072109222,
+      "rewards/margins": 0.016005536541342735,
+      "rewards/rejected": -0.1510409265756607,
+      "sft_loss": 1.3503539562225342,
+      "step": 620
+    },
+    {
+      "epoch": 1.1202489442098245,
+      "grad_norm": 4.146026611328125,
+      "learning_rate": 3.46665269378139e-06,
+      "logits/chosen": 251.23709106445312,
+      "logits/rejected": 251.81686401367188,
+      "logps/chosen": -1.3730881214141846,
+      "logps/rejected": -1.4982974529266357,
+      "loss": 1.4452,
+      "odds_ratio_loss": 0.7208770513534546,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.13730882108211517,
+      "rewards/margins": 0.012520933523774147,
+      "rewards/rejected": -0.14982974529266357,
+      "sft_loss": 1.3730881214141846,
+      "step": 630
+    },
+    {
+      "epoch": 1.1380306734829961,
+      "grad_norm": 3.3884165287017822,
+      "learning_rate": 3.4235244614569794e-06,
+      "logits/chosen": 250.8208770751953,
+      "logits/rejected": 251.1712646484375,
+      "logps/chosen": -1.3837242126464844,
+      "logps/rejected": -1.5583369731903076,
+      "loss": 1.4557,
+      "odds_ratio_loss": 0.7200697660446167,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.13837242126464844,
+      "rewards/margins": 0.017461290583014488,
+      "rewards/rejected": -0.15583372116088867,
+      "sft_loss": 1.3837242126464844,
+      "step": 640
+    },
+    {
+      "epoch": 1.155812402756168,
+      "grad_norm": 1.377685546875,
+      "learning_rate": 3.3800755490698008e-06,
+      "logits/chosen": 252.3297576904297,
+      "logits/rejected": 252.5504150390625,
+      "logps/chosen": -1.310136079788208,
+      "logps/rejected": -1.5744563341140747,
+      "loss": 1.3736,
+      "odds_ratio_loss": 0.6345242857933044,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.13101361691951752,
+      "rewards/margins": 0.026432007551193237,
+      "rewards/rejected": -0.15744563937187195,
+      "sft_loss": 1.310136079788208,
+      "step": 650
+    },
+    {
+      "epoch": 1.17359413202934,
+      "grad_norm": 2.116525411605835,
+      "learning_rate": 3.3363210436051287e-06,
+      "logits/chosen": 252.10305786132812,
+      "logits/rejected": 252.2167205810547,
+      "logps/chosen": -1.348901629447937,
+      "logps/rejected": -1.5241731405258179,
+      "loss": 1.4188,
+      "odds_ratio_loss": 0.6987627148628235,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.13489016890525818,
+      "rewards/margins": 0.017527148127555847,
+      "rewards/rejected": -0.15241730213165283,
+      "sft_loss": 1.348901629447937,
+      "step": 660
+    },
+    {
+      "epoch": 1.1913758613025116,
+      "grad_norm": 1.8333208560943604,
+      "learning_rate": 3.292276138160867e-06,
+      "logits/chosen": 251.9469757080078,
+      "logits/rejected": 252.2017059326172,
+      "logps/chosen": -1.331968069076538,
+      "logps/rejected": -1.551393747329712,
+      "loss": 1.4003,
+      "odds_ratio_loss": 0.6828904151916504,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -0.13319680094718933,
+      "rewards/margins": 0.021942555904388428,
+      "rewards/rejected": -0.15513937175273895,
+      "sft_loss": 1.331968069076538,
+      "step": 670
+    },
+    {
+      "epoch": 1.2091575905756835,
+      "grad_norm": 2.233184576034546,
+      "learning_rate": 3.2479561266719694e-06,
+      "logits/chosen": 252.11386108398438,
+      "logits/rejected": 252.0362548828125,
+      "logps/chosen": -1.346163272857666,
+      "logps/rejected": -1.5309796333312988,
+      "loss": 1.4146,
+      "odds_ratio_loss": 0.6838955283164978,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.13461633026599884,
+      "rewards/margins": 0.01848164014518261,
+      "rewards/rejected": -0.1530979573726654,
+      "sft_loss": 1.346163272857666,
+      "step": 680
+    },
+    {
+      "epoch": 1.2269393198488552,
+      "grad_norm": 4.622295379638672,
+      "learning_rate": 3.2033763985998533e-06,
+      "logits/chosen": 253.1451416015625,
+      "logits/rejected": 253.5478973388672,
+      "logps/chosen": -1.2959039211273193,
+      "logps/rejected": -1.597389578819275,
+      "loss": 1.3583,
+      "odds_ratio_loss": 0.6243518590927124,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.12959036231040955,
+      "rewards/margins": 0.03014858439564705,
+      "rewards/rejected": -0.15973897278308868,
+      "sft_loss": 1.2959039211273193,
+      "step": 690
+    },
+    {
+      "epoch": 1.244721049122027,
+      "grad_norm": 2.6373684406280518,
+      "learning_rate": 3.1585524335886335e-06,
+      "logits/chosen": 252.57833862304688,
+      "logits/rejected": 252.81259155273438,
+      "logps/chosen": -1.2123761177062988,
+      "logps/rejected": -1.4513623714447021,
+      "loss": 1.2775,
+      "odds_ratio_loss": 0.6515756845474243,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.121237613260746,
+      "rewards/margins": 0.023898636922240257,
+      "rewards/rejected": -0.1451362520456314,
+      "sft_loss": 1.2123761177062988,
+      "step": 700
+    },
+    {
+      "epoch": 1.262502778395199,
+      "grad_norm": 2.7742574214935303,
+      "learning_rate": 3.1134997960900536e-06,
+      "logits/chosen": 250.22891235351562,
+      "logits/rejected": 251.8517608642578,
+      "logps/chosen": -1.2099064588546753,
+      "logps/rejected": -1.5609335899353027,
+      "loss": 1.2689,
+      "odds_ratio_loss": 0.5895074009895325,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.12099063396453857,
+      "rewards/margins": 0.0351027250289917,
+      "rewards/rejected": -0.15609335899353027,
+      "sft_loss": 1.2099064588546753,
+      "step": 710
+    },
+    {
+      "epoch": 1.2802845076683709,
+      "grad_norm": 1.7436786890029907,
+      "learning_rate": 3.0682341299589583e-06,
+      "logits/chosen": 252.24365234375,
+      "logits/rejected": 252.2627410888672,
+      "logps/chosen": -1.2945445775985718,
+      "logps/rejected": -1.419683814048767,
+      "loss": 1.3646,
+      "odds_ratio_loss": 0.7010321617126465,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -0.12945446372032166,
+      "rewards/margins": 0.012513910420238972,
+      "rewards/rejected": -0.14196836948394775,
+      "sft_loss": 1.2945445775985718,
+      "step": 720
+    },
+    {
+      "epoch": 1.2980662369415426,
+      "grad_norm": 1.6442652940750122,
+      "learning_rate": 3.022771153021201e-06,
+      "logits/chosen": 252.22427368164062,
+      "logits/rejected": 252.268310546875,
+      "logps/chosen": -1.2626879215240479,
+      "logps/rejected": -1.530016541481018,
+      "loss": 1.3271,
+      "odds_ratio_loss": 0.644324779510498,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.12626877427101135,
+      "rewards/margins": 0.026732871308922768,
+      "rewards/rejected": -0.15300165116786957,
+      "sft_loss": 1.2626879215240479,
+      "step": 730
+    },
+    {
+      "epoch": 1.3158479662147144,
+      "grad_norm": 2.3948917388916016,
+      "learning_rate": 2.9771266516158625e-06,
+      "logits/chosen": 252.8442840576172,
+      "logits/rejected": 253.3193359375,
+      "logps/chosen": -1.2903095483779907,
+      "logps/rejected": -1.4887399673461914,
+      "loss": 1.3586,
+      "odds_ratio_loss": 0.682421088218689,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.12903094291687012,
+      "rewards/margins": 0.019843045622110367,
+      "rewards/rejected": -0.14887399971485138,
+      "sft_loss": 1.2903095483779907,
+      "step": 740
+    },
+    {
+      "epoch": 1.3336296954878861,
+      "grad_norm": 1.7583057880401611,
+      "learning_rate": 2.9313164751136802e-06,
+      "logits/chosen": 251.74462890625,
+      "logits/rejected": 252.4144287109375,
+      "logps/chosen": -1.3303029537200928,
+      "logps/rejected": -1.4901565313339233,
+      "loss": 1.3975,
+      "odds_ratio_loss": 0.6716746091842651,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.13303029537200928,
+      "rewards/margins": 0.015985365957021713,
+      "rewards/rejected": -0.1490156650543213,
+      "sft_loss": 1.3303029537200928,
+      "step": 750
+    },
+    {
+      "epoch": 1.351411424761058,
+      "grad_norm": 2.620116710662842,
+      "learning_rate": 2.8853565304135956e-06,
+      "logits/chosen": 254.57931518554688,
+      "logits/rejected": 254.56857299804688,
+      "logps/chosen": -1.3150755167007446,
+      "logps/rejected": -1.3985707759857178,
+      "loss": 1.3882,
+      "odds_ratio_loss": 0.730746865272522,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.13150756061077118,
+      "rewards/margins": 0.008349532261490822,
+      "rewards/rejected": -0.13985709846019745,
+      "sft_loss": 1.3150755167007446,
+      "step": 760
+    },
+    {
+      "epoch": 1.36919315403423,
+      "grad_norm": 6.955394744873047,
+      "learning_rate": 2.839262776419313e-06,
+      "logits/chosen": 253.0518035888672,
+      "logits/rejected": 253.3127899169922,
+      "logps/chosen": -1.2819963693618774,
+      "logps/rejected": -1.5402740240097046,
+      "loss": 1.346,
+      "odds_ratio_loss": 0.6400705575942993,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.12819963693618774,
+      "rewards/margins": 0.02582777664065361,
+      "rewards/rejected": -0.15402741730213165,
+      "sft_loss": 1.2819963693618774,
+      "step": 770
+    },
+    {
+      "epoch": 1.3869748833074016,
+      "grad_norm": 2.157224416732788,
+      "learning_rate": 2.793051218497817e-06,
+      "logits/chosen": 253.73080444335938,
+      "logits/rejected": 253.3892822265625,
+      "logps/chosen": -1.324377179145813,
+      "logps/rejected": -1.411966323852539,
+      "loss": 1.3963,
+      "odds_ratio_loss": 0.7193102240562439,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.13243773579597473,
+      "rewards/margins": 0.00875892210751772,
+      "rewards/rejected": -0.14119663834571838,
+      "sft_loss": 1.324377179145813,
+      "step": 780
+    },
+    {
+      "epoch": 1.4047566125805735,
+      "grad_norm": 1.813340663909912,
+      "learning_rate": 2.7467379029217437e-06,
+      "logits/chosen": 253.36312866210938,
+      "logits/rejected": 254.00588989257812,
+      "logps/chosen": -1.2761633396148682,
+      "logps/rejected": -1.5060415267944336,
+      "loss": 1.343,
+      "odds_ratio_loss": 0.6682227849960327,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.12761631608009338,
+      "rewards/margins": 0.022987816482782364,
+      "rewards/rejected": -0.15060414373874664,
+      "sft_loss": 1.2761633396148682,
+      "step": 790
+    },
+    {
+      "epoch": 1.4225383418537452,
+      "grad_norm": 1.6623824834823608,
+      "learning_rate": 2.7003389112975546e-06,
+      "logits/chosen": 253.5782470703125,
+      "logits/rejected": 253.32089233398438,
+      "logps/chosen": -1.3095474243164062,
+      "logps/rejected": -1.491077184677124,
+      "loss": 1.3757,
+      "odds_ratio_loss": 0.6613184213638306,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.13095474243164062,
+      "rewards/margins": 0.018152965232729912,
+      "rewards/rejected": -0.14910772442817688,
+      "sft_loss": 1.3095474243164062,
+      "step": 800
+    },
+    {
+      "epoch": 1.440320071126917,
+      "grad_norm": 5.644952774047852,
+      "learning_rate": 2.653870354981437e-06,
+      "logits/chosen": 252.7183380126953,
+      "logits/rejected": 253.2029571533203,
+      "logps/chosen": -1.2087516784667969,
+      "logps/rejected": -1.411168098449707,
+      "loss": 1.2748,
+      "odds_ratio_loss": 0.6601918935775757,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.12087517976760864,
+      "rewards/margins": 0.020241642370820045,
+      "rewards/rejected": -0.14111682772636414,
+      "sft_loss": 1.2087516784667969,
+      "step": 810
+    },
+    {
+      "epoch": 1.458101800400089,
+      "grad_norm": 4.477853298187256,
+      "learning_rate": 2.6073483694848777e-06,
+      "logits/chosen": 253.3741912841797,
+      "logits/rejected": 253.51382446289062,
+      "logps/chosen": -1.2232033014297485,
+      "logps/rejected": -1.4460740089416504,
+      "loss": 1.2895,
+      "odds_ratio_loss": 0.663042426109314,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.12232033163309097,
+      "rewards/margins": 0.02228708192706108,
+      "rewards/rejected": -0.14460742473602295,
+      "sft_loss": 1.2232033014297485,
+      "step": 820
+    },
+    {
+      "epoch": 1.4758835296732609,
+      "grad_norm": 1.6548304557800293,
+      "learning_rate": 2.560789108871847e-06,
+      "logits/chosen": 253.0503692626953,
+      "logits/rejected": 252.9963836669922,
+      "logps/chosen": -1.3039335012435913,
+      "logps/rejected": -1.57505202293396,
+      "loss": 1.3696,
+      "odds_ratio_loss": 0.6571396589279175,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.1303933560848236,
+      "rewards/margins": 0.027111848816275597,
+      "rewards/rejected": -0.15750519931316376,
+      "sft_loss": 1.3039335012435913,
+      "step": 830
+    },
+    {
+      "epoch": 1.4936652589464325,
+      "grad_norm": 5.2365007400512695,
+      "learning_rate": 2.514208740149544e-06,
+      "logits/chosen": 253.81640625,
+      "logits/rejected": 253.57717895507812,
+      "logps/chosen": -1.3866493701934814,
+      "logps/rejected": -1.5284301042556763,
+      "loss": 1.4596,
+      "odds_ratio_loss": 0.7293277978897095,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.13866494596004486,
+      "rewards/margins": 0.014178059995174408,
+      "rewards/rejected": -0.15284302830696106,
+      "sft_loss": 1.3866493701934814,
+      "step": 840
+    },
+    {
+      "epoch": 1.5114469882196042,
+      "grad_norm": 2.916973114013672,
+      "learning_rate": 2.46762343765464e-06,
+      "logits/chosen": 254.6901397705078,
+      "logits/rejected": 254.9998016357422,
+      "logps/chosen": -1.3546521663665771,
+      "logps/rejected": -1.587242841720581,
+      "loss": 1.4197,
+      "odds_ratio_loss": 0.6507903933525085,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.13546521961688995,
+      "rewards/margins": 0.02325906977057457,
+      "rewards/rejected": -0.15872427821159363,
+      "sft_loss": 1.3546521663665771,
+      "step": 850
+    },
+    {
+      "epoch": 1.5292287174927761,
+      "grad_norm": 2.888098955154419,
+      "learning_rate": 2.4210493774369903e-06,
+      "logits/chosen": 253.61563110351562,
+      "logits/rejected": 253.7008056640625,
+      "logps/chosen": -1.3398011922836304,
+      "logps/rejected": -1.5939433574676514,
+      "loss": 1.4092,
+      "odds_ratio_loss": 0.6944610476493835,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.13398012518882751,
+      "rewards/margins": 0.02541421353816986,
+      "rewards/rejected": -0.15939433872699738,
+      "sft_loss": 1.3398011922836304,
+      "step": 860
+    },
+    {
+      "epoch": 1.547010446765948,
+      "grad_norm": 1.7133105993270874,
+      "learning_rate": 2.374502731642732e-06,
+      "logits/chosen": 253.13046264648438,
+      "logits/rejected": 253.28274536132812,
+      "logps/chosen": -1.2972921133041382,
+      "logps/rejected": -1.5040593147277832,
+      "loss": 1.3657,
+      "odds_ratio_loss": 0.6844374537467957,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.12972919642925262,
+      "rewards/margins": 0.02067672833800316,
+      "rewards/rejected": -0.15040592849254608,
+      "sft_loss": 1.2972921133041382,
+      "step": 870
+    },
+    {
+      "epoch": 1.56479217603912,
+      "grad_norm": 1.869072437286377,
+      "learning_rate": 2.3279996628987556e-06,
+      "logits/chosen": 253.9283447265625,
+      "logits/rejected": 254.41073608398438,
+      "logps/chosen": -1.2677370309829712,
+      "logps/rejected": -1.4599510431289673,
+      "loss": 1.3369,
+      "odds_ratio_loss": 0.6917437314987183,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.1267736852169037,
+      "rewards/margins": 0.01922140084207058,
+      "rewards/rejected": -0.1459950953722,
+      "sft_loss": 1.2677370309829712,
+      "step": 880
+    },
+    {
+      "epoch": 1.5825739053122916,
+      "grad_norm": 2.387537717819214,
+      "learning_rate": 2.281556318700474e-06,
+      "logits/chosen": 253.6074676513672,
+      "logits/rejected": 253.7229461669922,
+      "logps/chosen": -1.230033278465271,
+      "logps/rejected": -1.369767189025879,
+      "loss": 1.3007,
+      "odds_ratio_loss": 0.7063087224960327,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.12300334125757217,
+      "rewards/margins": 0.013973374851047993,
+      "rewards/rejected": -0.1369767189025879,
+      "sft_loss": 1.230033278465271,
+      "step": 890
+    },
+    {
+      "epoch": 1.6003556345854635,
+      "grad_norm": 7.490704536437988,
+      "learning_rate": 2.2351888258048408e-06,
+      "logits/chosen": 251.65274047851562,
+      "logits/rejected": 251.5640106201172,
+      "logps/chosen": -1.2508928775787354,
+      "logps/rejected": -1.4577651023864746,
+      "loss": 1.3169,
+      "odds_ratio_loss": 0.6601670980453491,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.12508928775787354,
+      "rewards/margins": 0.020687231793999672,
+      "rewards/rejected": -0.14577652513980865,
+      "sft_loss": 1.2508928775787354,
+      "step": 900
+    },
+    {
+      "epoch": 1.6181373638586352,
+      "grad_norm": 1.7122284173965454,
+      "learning_rate": 2.188913284630584e-06,
+      "logits/chosen": 253.48501586914062,
+      "logits/rejected": 253.5773468017578,
+      "logps/chosen": -1.3153185844421387,
+      "logps/rejected": -1.4097468852996826,
+      "loss": 1.3867,
+      "odds_ratio_loss": 0.7141562700271606,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.13153186440467834,
+      "rewards/margins": 0.009442826732993126,
+      "rewards/rejected": -0.1409747153520584,
+      "sft_loss": 1.3153185844421387,
+      "step": 910
+    },
+    {
+      "epoch": 1.635919093131807,
+      "grad_norm": 3.5988399982452393,
+      "learning_rate": 2.1427457636675652e-06,
+      "logits/chosen": 254.33633422851562,
+      "logits/rejected": 253.96981811523438,
+      "logps/chosen": -1.2676795721054077,
+      "logps/rejected": -1.4219448566436768,
+      "loss": 1.3387,
+      "odds_ratio_loss": 0.7098127007484436,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.12676796317100525,
+      "rewards/margins": 0.015426510944962502,
+      "rewards/rejected": -0.1421944797039032,
+      "sft_loss": 1.2676795721054077,
+      "step": 920
+    },
+    {
+      "epoch": 1.653700822404979,
+      "grad_norm": 1.8244489431381226,
+      "learning_rate": 2.096702293897247e-06,
+      "logits/chosen": 254.08718872070312,
+      "logits/rejected": 254.5830535888672,
+      "logps/chosen": -1.283644437789917,
+      "logps/rejected": -1.5567032098770142,
+      "loss": 1.35,
+      "odds_ratio_loss": 0.6635793447494507,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.1283644437789917,
+      "rewards/margins": 0.027305880561470985,
+      "rewards/rejected": -0.15567031502723694,
+      "sft_loss": 1.283644437789917,
+      "step": 930
+    },
+    {
+      "epoch": 1.6714825516781509,
+      "grad_norm": 3.096039056777954,
+      "learning_rate": 2.0507988632261672e-06,
+      "logits/chosen": 252.3622589111328,
+      "logits/rejected": 252.93716430664062,
+      "logps/chosen": -1.2625596523284912,
+      "logps/rejected": -1.5456993579864502,
+      "loss": 1.3258,
+      "odds_ratio_loss": 0.6320444345474243,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.12625595927238464,
+      "rewards/margins": 0.028313983231782913,
+      "rewards/rejected": -0.15456993877887726,
+      "sft_loss": 1.2625596523284912,
+      "step": 940
+    },
+    {
+      "epoch": 1.6892642809513225,
+      "grad_norm": 3.4061944484710693,
+      "learning_rate": 2.005051410934382e-06,
+      "logits/chosen": 253.2592010498047,
+      "logits/rejected": 253.57846069335938,
+      "logps/chosen": -1.3725305795669556,
+      "logps/rejected": -1.5077531337738037,
+      "loss": 1.4437,
+      "odds_ratio_loss": 0.7114490866661072,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.137253075838089,
+      "rewards/margins": 0.013522250577807426,
+      "rewards/rejected": -0.15077531337738037,
+      "sft_loss": 1.3725305795669556,
+      "step": 950
+    },
+    {
+      "epoch": 1.7070460102244942,
+      "grad_norm": 2.6905410289764404,
+      "learning_rate": 1.9594758221407843e-06,
+      "logits/chosen": 254.19900512695312,
+      "logits/rejected": 254.83987426757812,
+      "logps/chosen": -1.2452795505523682,
+      "logps/rejected": -1.5318635702133179,
+      "loss": 1.3081,
+      "odds_ratio_loss": 0.6286849975585938,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.1245279535651207,
+      "rewards/margins": 0.02865840494632721,
+      "rewards/rejected": -0.1531863659620285,
+      "sft_loss": 1.2452795505523682,
+      "step": 960
+    },
+    {
+      "epoch": 1.724827739497666,
+      "grad_norm": 2.9211478233337402,
+      "learning_rate": 1.9140879222872408e-06,
+      "logits/chosen": 254.0661163330078,
+      "logits/rejected": 254.55819702148438,
+      "logps/chosen": -1.274641752243042,
+      "logps/rejected": -1.3871382474899292,
+      "loss": 1.3478,
+      "odds_ratio_loss": 0.7320677042007446,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.1274641752243042,
+      "rewards/margins": 0.011249655857682228,
+      "rewards/rejected": -0.13871383666992188,
+      "sft_loss": 1.274641752243042,
+      "step": 970
+    },
+    {
+      "epoch": 1.742609468770838,
+      "grad_norm": 2.0924360752105713,
+      "learning_rate": 1.8689034716434346e-06,
+      "logits/chosen": 253.67953491210938,
+      "logits/rejected": 254.02993774414062,
+      "logps/chosen": -1.334486722946167,
+      "logps/rejected": -1.4547626972198486,
+      "loss": 1.4054,
+      "odds_ratio_loss": 0.7087211012840271,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.13344867527484894,
+      "rewards/margins": 0.01202760823071003,
+      "rewards/rejected": -0.14547628164291382,
+      "sft_loss": 1.334486722946167,
+      "step": 980
+    },
+    {
+      "epoch": 1.76039119804401,
+      "grad_norm": 2.326310634613037,
+      "learning_rate": 1.8239381598343576e-06,
+      "logits/chosen": 255.02243041992188,
+      "logits/rejected": 255.69680786132812,
+      "logps/chosen": -1.28658127784729,
+      "logps/rejected": -1.4356472492218018,
+      "loss": 1.3565,
+      "odds_ratio_loss": 0.6996346712112427,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.12865814566612244,
+      "rewards/margins": 0.014906583353877068,
+      "rewards/rejected": -0.14356473088264465,
+      "sft_loss": 1.28658127784729,
+      "step": 990
+    },
+    {
+      "epoch": 1.7781729273171816,
+      "grad_norm": 5.255044460296631,
+      "learning_rate": 1.779207600392312e-06,
+      "logits/chosen": 253.11575317382812,
+      "logits/rejected": 253.3959197998047,
+      "logps/chosen": -1.3602896928787231,
+      "logps/rejected": -1.5267302989959717,
+      "loss": 1.428,
+      "odds_ratio_loss": 0.6775819063186646,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.1360289752483368,
+      "rewards/margins": 0.016644055023789406,
+      "rewards/rejected": -0.15267305076122284,
+      "sft_loss": 1.3602896928787231,
+      "step": 1000
+    },
+    {
+      "epoch": 1.7781729273171816,
+      "eval_logits/chosen": 253.95806884765625,
+      "eval_logits/rejected": 254.13504028320312,
+      "eval_logps/chosen": -1.2936838865280151,
+      "eval_logps/rejected": -1.5090892314910889,
+      "eval_loss": 1.3595393896102905,
+      "eval_odds_ratio_loss": 0.6585569381713867,
+      "eval_rewards/accuracies": 0.5600000023841858,
+      "eval_rewards/chosen": -0.1293683797121048,
+      "eval_rewards/margins": 0.02154054492712021,
+      "eval_rewards/rejected": -0.1509089171886444,
+      "eval_runtime": 217.2988,
+      "eval_samples_per_second": 4.602,
+      "eval_sft_loss": 1.2936838865280151,
+      "eval_steps_per_second": 2.301,
+      "step": 1000
+    },
+    {
+      "epoch": 1.7959546565903532,
+      "grad_norm": 2.5635573863983154,
+      "learning_rate": 1.7347273253353552e-06,
+      "logits/chosen": 254.41915893554688,
+      "logits/rejected": 254.5049285888672,
+      "logps/chosen": -1.2720413208007812,
+      "logps/rejected": -1.4322283267974854,
+      "loss": 1.3422,
+      "odds_ratio_loss": 0.7016333341598511,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.12720413506031036,
+      "rewards/margins": 0.01601869985461235,
+      "rewards/rejected": -0.14322282373905182,
+      "sft_loss": 1.2720413208007812,
+      "step": 1010
+    },
+    {
+      "epoch": 1.8137363858635251,
+      "grad_norm": 5.554447650909424,
+      "learning_rate": 1.690512779774029e-06,
+      "logits/chosen": 253.4269561767578,
+      "logits/rejected": 253.6225128173828,
+      "logps/chosen": -1.3230665922164917,
+      "logps/rejected": -1.678013563156128,
+      "loss": 1.3849,
+      "odds_ratio_loss": 0.6180821657180786,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.13230668008327484,
+      "rewards/margins": 0.035494692623615265,
+      "rewards/rejected": -0.16780135035514832,
+      "sft_loss": 1.3230665922164917,
+      "step": 1020
+    },
+    {
+      "epoch": 1.831518115136697,
+      "grad_norm": 2.634599447250366,
+      "learning_rate": 1.6465793165482838e-06,
+      "logits/chosen": 253.33926391601562,
+      "logits/rejected": 253.8477020263672,
+      "logps/chosen": -1.2152363061904907,
+      "logps/rejected": -1.465906023979187,
+      "loss": 1.2779,
+      "odds_ratio_loss": 0.6263755559921265,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.12152364104986191,
+      "rewards/margins": 0.025066960602998734,
+      "rewards/rejected": -0.14659060537815094,
+      "sft_loss": 1.2152363061904907,
+      "step": 1030
+    },
+    {
+      "epoch": 1.849299844409869,
+      "grad_norm": 3.051729917526245,
+      "learning_rate": 1.6029421908964305e-06,
+      "logits/chosen": 252.26016235351562,
+      "logits/rejected": 253.5745086669922,
+      "logps/chosen": -1.2302608489990234,
+      "logps/rejected": -1.62982976436615,
+      "loss": 1.2943,
+      "odds_ratio_loss": 0.6404699683189392,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.12302607297897339,
+      "rewards/margins": 0.03995689004659653,
+      "rewards/rejected": -0.1629829704761505,
+      "sft_loss": 1.2302608489990234,
+      "step": 1040
+    },
+    {
+      "epoch": 1.8670815736830408,
+      "grad_norm": 5.069088459014893,
+      "learning_rate": 1.559616555157985e-06,
+      "logits/chosen": 253.4644775390625,
+      "logits/rejected": 253.29727172851562,
+      "logps/chosen": -1.2889888286590576,
+      "logps/rejected": -1.4968655109405518,
+      "loss": 1.3589,
+      "odds_ratio_loss": 0.6988304853439331,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.12889888882637024,
+      "rewards/margins": 0.020787667483091354,
+      "rewards/rejected": -0.1496865451335907,
+      "sft_loss": 1.2889888286590576,
+      "step": 1050
+    },
+    {
+      "epoch": 1.8848633029562125,
+      "grad_norm": 2.464268207550049,
+      "learning_rate": 1.516617453512252e-06,
+      "logits/chosen": 254.87197875976562,
+      "logits/rejected": 254.7659454345703,
+      "logps/chosen": -1.351043462753296,
+      "logps/rejected": -1.4525654315948486,
+      "loss": 1.4237,
+      "odds_ratio_loss": 0.7263092398643494,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.13510434329509735,
+      "rewards/margins": 0.010152206756174564,
+      "rewards/rejected": -0.14525654911994934,
+      "sft_loss": 1.351043462753296,
+      "step": 1060
+    },
+    {
+      "epoch": 1.9026450322293842,
+      "grad_norm": 10.498387336730957,
+      "learning_rate": 1.473959816754449e-06,
+      "logits/chosen": 253.4375457763672,
+      "logits/rejected": 254.0106201171875,
+      "logps/chosen": -1.2346054315567017,
+      "logps/rejected": -1.3691545724868774,
+      "loss": 1.3067,
+      "odds_ratio_loss": 0.721213161945343,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.123460553586483,
+      "rewards/margins": 0.013454906642436981,
+      "rewards/rejected": -0.13691547513008118,
+      "sft_loss": 1.2346054315567017,
+      "step": 1070
+    },
+    {
+      "epoch": 1.920426761502556,
+      "grad_norm": 4.5994696617126465,
+      "learning_rate": 1.4316584571112213e-06,
+      "logits/chosen": 256.06500244140625,
+      "logits/rejected": 256.2422790527344,
+      "logps/chosen": -1.3240019083023071,
+      "logps/rejected": -1.4346423149108887,
+      "loss": 1.3965,
+      "odds_ratio_loss": 0.7249402403831482,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.13240018486976624,
+      "rewards/margins": 0.011064060032367706,
+      "rewards/rejected": -0.14346425235271454,
+      "sft_loss": 1.3240019083023071,
+      "step": 1080
+    },
+    {
+      "epoch": 1.938208490775728,
+      "grad_norm": 3.5848639011383057,
+      "learning_rate": 1.389728063097306e-06,
+      "logits/chosen": 255.7421112060547,
+      "logits/rejected": 255.6949005126953,
+      "logps/chosen": -1.3223278522491455,
+      "logps/rejected": -1.6033687591552734,
+      "loss": 1.3875,
+      "odds_ratio_loss": 0.6514922976493835,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.13223278522491455,
+      "rewards/margins": 0.028104085475206375,
+      "rewards/rejected": -0.16033688187599182,
+      "sft_loss": 1.3223278522491455,
+      "step": 1090
+    },
+    {
+      "epoch": 1.9559902200488999,
+      "grad_norm": 2.085165500640869,
+      "learning_rate": 1.348183194415179e-06,
+      "logits/chosen": 254.6555938720703,
+      "logits/rejected": 255.1551513671875,
+      "logps/chosen": -1.2908974885940552,
+      "logps/rejected": -1.5859901905059814,
+      "loss": 1.3544,
+      "odds_ratio_loss": 0.6346372365951538,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.12908974289894104,
+      "rewards/margins": 0.029509279876947403,
+      "rewards/rejected": -0.15859903395175934,
+      "sft_loss": 1.2908974885940552,
+      "step": 1100
+    },
+    {
+      "epoch": 1.9737719493220716,
+      "grad_norm": 2.3228931427001953,
+      "learning_rate": 1.3070382768994015e-06,
+      "logits/chosen": 254.2451171875,
+      "logits/rejected": 254.3721160888672,
+      "logps/chosen": -1.2544920444488525,
+      "logps/rejected": -1.4442470073699951,
+      "loss": 1.3195,
+      "odds_ratio_loss": 0.6503337621688843,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.12544921040534973,
+      "rewards/margins": 0.018975483253598213,
+      "rewards/rejected": -0.1444246917963028,
+      "sft_loss": 1.2544920444488525,
+      "step": 1110
+    },
+    {
+      "epoch": 1.9915536785952432,
+      "grad_norm": 2.0866916179656982,
+      "learning_rate": 1.2663075975074746e-06,
+      "logits/chosen": 253.80899047851562,
+      "logits/rejected": 254.31460571289062,
+      "logps/chosen": -1.2685939073562622,
+      "logps/rejected": -1.4882584810256958,
+      "loss": 1.3361,
+      "odds_ratio_loss": 0.6750525236129761,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.1268593817949295,
+      "rewards/margins": 0.021966462954878807,
+      "rewards/rejected": -0.14882585406303406,
+      "sft_loss": 1.2685939073562622,
+      "step": 1120
+    },
+    {
+      "epoch": 2.009335407868415,
+      "grad_norm": 5.936447620391846,
+      "learning_rate": 1.2260052993589034e-06,
+      "logits/chosen": 253.43161010742188,
+      "logits/rejected": 253.4159698486328,
+      "logps/chosen": -1.3823201656341553,
+      "logps/rejected": -1.4824841022491455,
+      "loss": 1.4548,
+      "odds_ratio_loss": 0.7252174615859985,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.1382320076227188,
+      "rewards/margins": 0.010016398504376411,
+      "rewards/rejected": -0.14824840426445007,
+      "sft_loss": 1.3823201656341553,
+      "step": 1130
+    },
+    {
+      "epoch": 2.027117137141587,
+      "grad_norm": 1.858097791671753,
+      "learning_rate": 1.1861453768242099e-06,
+      "logits/chosen": 255.3394317626953,
+      "logits/rejected": 256.12109375,
+      "logps/chosen": -1.223772644996643,
+      "logps/rejected": -1.5006752014160156,
+      "loss": 1.2835,
+      "odds_ratio_loss": 0.5976046323776245,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.12237726151943207,
+      "rewards/margins": 0.027690261602401733,
+      "rewards/rejected": -0.1500675231218338,
+      "sft_loss": 1.223772644996643,
+      "step": 1140
+    },
+    {
+      "epoch": 2.044898866414759,
+      "grad_norm": 9.576284408569336,
+      "learning_rate": 1.1467416706655982e-06,
+      "logits/chosen": 255.4516143798828,
+      "logits/rejected": 255.5124969482422,
+      "logps/chosen": -1.33961021900177,
+      "logps/rejected": -1.5424822568893433,
+      "loss": 1.4101,
+      "odds_ratio_loss": 0.7047215700149536,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.13396099209785461,
+      "rewards/margins": 0.020287221297621727,
+      "rewards/rejected": -0.1542482227087021,
+      "sft_loss": 1.33961021900177,
+      "step": 1150
+    },
+    {
+      "epoch": 2.062680595687931,
+      "grad_norm": 2.312715768814087,
+      "learning_rate": 1.1078078632309559e-06,
+      "logits/chosen": 255.4163360595703,
+      "logits/rejected": 256.0242004394531,
+      "logps/chosen": -1.2566642761230469,
+      "logps/rejected": -1.4380619525909424,
+      "loss": 1.3236,
+      "odds_ratio_loss": 0.6697817444801331,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.12566642463207245,
+      "rewards/margins": 0.018139759078621864,
+      "rewards/rejected": -0.14380618929862976,
+      "sft_loss": 1.2566642761230469,
+      "step": 1160
+    },
+    {
+      "epoch": 2.0804623249611023,
+      "grad_norm": 2.0991952419281006,
+      "learning_rate": 1.0693574737028627e-06,
+      "logits/chosen": 254.43576049804688,
+      "logits/rejected": 255.13711547851562,
+      "logps/chosen": -1.2939661741256714,
+      "logps/rejected": -1.4546544551849365,
+      "loss": 1.3642,
+      "odds_ratio_loss": 0.7022632360458374,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.12939661741256714,
+      "rewards/margins": 0.01606883481144905,
+      "rewards/rejected": -0.14546546339988708,
+      "sft_loss": 1.2939661741256714,
+      "step": 1170
+    },
+    {
+      "epoch": 2.098244054234274,
+      "grad_norm": 4.281792640686035,
+      "learning_rate": 1.0314038534042586e-06,
+      "logits/chosen": 254.57168579101562,
+      "logits/rejected": 255.064453125,
+      "logps/chosen": -1.1770070791244507,
+      "logps/rejected": -1.3855504989624023,
+      "loss": 1.2441,
+      "odds_ratio_loss": 0.6709107756614685,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.1177007183432579,
+      "rewards/margins": 0.020854344591498375,
+      "rewards/rejected": -0.13855504989624023,
+      "sft_loss": 1.1770070791244507,
+      "step": 1180
+    },
+    {
+      "epoch": 2.116025783507446,
+      "grad_norm": 1.9617539644241333,
+      "learning_rate": 9.939601811623946e-07,
+      "logits/chosen": 253.7810516357422,
+      "logits/rejected": 254.1466827392578,
+      "logps/chosen": -1.2411329746246338,
+      "logps/rejected": -1.4748810529708862,
+      "loss": 1.3104,
+      "odds_ratio_loss": 0.692661702632904,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.1241132840514183,
+      "rewards/margins": 0.02337482199072838,
+      "rewards/rejected": -0.14748811721801758,
+      "sft_loss": 1.2411329746246338,
+      "step": 1190
+    },
+    {
+      "epoch": 2.133807512780618,
+      "grad_norm": 2.653927803039551,
+      "learning_rate": 9.570394587326825e-07,
+      "logits/chosen": 253.3402099609375,
+      "logits/rejected": 253.52310180664062,
+      "logps/chosen": -1.2719714641571045,
+      "logps/rejected": -1.571843147277832,
+      "loss": 1.3354,
+      "odds_ratio_loss": 0.6337886452674866,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.12719716131687164,
+      "rewards/margins": 0.02998717501759529,
+      "rewards/rejected": -0.15718431770801544,
+      "sft_loss": 1.2719714641571045,
+      "step": 1200
+    },
+    {
+      "epoch": 2.15158924205379,
+      "grad_norm": 1.7235380411148071,
+      "learning_rate": 9.206545062840302e-07,
+      "logits/chosen": 255.1512908935547,
+      "logits/rejected": 255.9069366455078,
+      "logps/chosen": -1.2449194192886353,
+      "logps/rejected": -1.4648315906524658,
+      "loss": 1.3088,
+      "odds_ratio_loss": 0.6390582323074341,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.12449195235967636,
+      "rewards/margins": 0.021991217508912086,
+      "rewards/rejected": -0.1464831531047821,
+      "sft_loss": 1.2449194192886353,
+      "step": 1210
+    },
+    {
+      "epoch": 2.1693709713269618,
+      "grad_norm": 2.0629489421844482,
+      "learning_rate": 8.848179579472285e-07,
+      "logits/chosen": 255.0786590576172,
+      "logits/rejected": 255.07534790039062,
+      "logps/chosen": -1.2237968444824219,
+      "logps/rejected": -1.3279929161071777,
+      "loss": 1.2941,
+      "odds_ratio_loss": 0.7028827667236328,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.12237968295812607,
+      "rewards/margins": 0.010419609025120735,
+      "rewards/rejected": -0.13279929757118225,
+      "sft_loss": 1.2237968444824219,
+      "step": 1220
+    },
+    {
+      "epoch": 2.1871527006001332,
+      "grad_norm": 4.134257793426514,
+      "learning_rate": 8.495422574279403e-07,
+      "logits/chosen": 253.44869995117188,
+      "logits/rejected": 253.70126342773438,
+      "logps/chosen": -1.201407551765442,
+      "logps/rejected": -1.4721620082855225,
+      "loss": 1.2647,
+      "odds_ratio_loss": 0.633193850517273,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.12014075368642807,
+      "rewards/margins": 0.02707546018064022,
+      "rewards/rejected": -0.14721623063087463,
+      "sft_loss": 1.201407551765442,
+      "step": 1230
+    },
+    {
+      "epoch": 2.204934429873305,
+      "grad_norm": 2.6145901679992676,
+      "learning_rate": 8.148396536858063e-07,
+      "logits/chosen": 255.00973510742188,
+      "logits/rejected": 255.087646484375,
+      "logps/chosen": -1.311307668685913,
+      "logps/rejected": -1.5930455923080444,
+      "loss": 1.3784,
+      "odds_ratio_loss": 0.670753538608551,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.13113076984882355,
+      "rewards/margins": 0.028173794969916344,
+      "rewards/rejected": -0.15930457413196564,
+      "sft_loss": 1.311307668685913,
+      "step": 1240
+    },
+    {
+      "epoch": 2.222716159146477,
+      "grad_norm": 1.5492827892303467,
+      "learning_rate": 7.807221966811815e-07,
+      "logits/chosen": 255.0123291015625,
+      "logits/rejected": 256.08282470703125,
+      "logps/chosen": -1.222886323928833,
+      "logps/rejected": -1.411444067955017,
+      "loss": 1.2915,
+      "odds_ratio_loss": 0.6862080097198486,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.12228864431381226,
+      "rewards/margins": 0.01885577104985714,
+      "rewards/rejected": -0.14114442467689514,
+      "sft_loss": 1.222886323928833,
+      "step": 1250
+    },
+    {
+      "epoch": 2.240497888419649,
+      "grad_norm": 1.9974987506866455,
+      "learning_rate": 7.47201733190962e-07,
+      "logits/chosen": 254.1110382080078,
+      "logits/rejected": 253.8502197265625,
+      "logps/chosen": -1.2319315671920776,
+      "logps/rejected": -1.3752238750457764,
+      "loss": 1.2993,
+      "odds_ratio_loss": 0.6734907627105713,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.12319314479827881,
+      "rewards/margins": 0.01432921551167965,
+      "rewards/rejected": -0.1375223696231842,
+      "sft_loss": 1.2319315671920776,
+      "step": 1260
+    },
+    {
+      "epoch": 2.258279617692821,
+      "grad_norm": 2.013181209564209,
+      "learning_rate": 7.142899026949721e-07,
+      "logits/chosen": 255.99240112304688,
+      "logits/rejected": 255.9341278076172,
+      "logps/chosen": -1.2691768407821655,
+      "logps/rejected": -1.4378575086593628,
+      "loss": 1.3351,
+      "odds_ratio_loss": 0.6596958637237549,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.12691769003868103,
+      "rewards/margins": 0.016868067905306816,
+      "rewards/rejected": -0.143785759806633,
+      "sft_loss": 1.2691768407821655,
+      "step": 1270
+    },
+    {
+      "epoch": 2.2760613469659923,
+      "grad_norm": 5.814607620239258,
+      "learning_rate": 6.819981333343273e-07,
+      "logits/chosen": 253.1020965576172,
+      "logits/rejected": 253.30661010742188,
+      "logps/chosen": -1.2039464712142944,
+      "logps/rejected": -1.4708151817321777,
+      "loss": 1.2691,
+      "odds_ratio_loss": 0.6517861485481262,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.12039466202259064,
+      "rewards/margins": 0.026686882600188255,
+      "rewards/rejected": -0.14708155393600464,
+      "sft_loss": 1.2039464712142944,
+      "step": 1280
+    },
+    {
+      "epoch": 2.293843076239164,
+      "grad_norm": 2.745338201522827,
+      "learning_rate": 6.503376379431839e-07,
+      "logits/chosen": 253.5093536376953,
+      "logits/rejected": 253.670654296875,
+      "logps/chosen": -1.3510358333587646,
+      "logps/rejected": -1.4214909076690674,
+      "loss": 1.423,
+      "odds_ratio_loss": 0.7199310660362244,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.13510356843471527,
+      "rewards/margins": 0.007045503705739975,
+      "rewards/rejected": -0.14214907586574554,
+      "sft_loss": 1.3510358333587646,
+      "step": 1290
+    },
+    {
+      "epoch": 2.311624805512336,
+      "grad_norm": 2.5681943893432617,
+      "learning_rate": 6.193194101552502e-07,
+      "logits/chosen": 256.16900634765625,
+      "logits/rejected": 256.3623962402344,
+      "logps/chosen": -1.24690580368042,
+      "logps/rejected": -1.4972703456878662,
+      "loss": 1.3081,
+      "odds_ratio_loss": 0.6117135286331177,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.12469057738780975,
+      "rewards/margins": 0.02503645047545433,
+      "rewards/rejected": -0.14972704648971558,
+      "sft_loss": 1.24690580368042,
+      "step": 1300
+    },
+    {
+      "epoch": 2.329406534785508,
+      "grad_norm": 3.2992701530456543,
+      "learning_rate": 5.889542205864083e-07,
+      "logits/chosen": 254.740966796875,
+      "logits/rejected": 255.06307983398438,
+      "logps/chosen": -1.2395174503326416,
+      "logps/rejected": -1.4891048669815063,
+      "loss": 1.303,
+      "odds_ratio_loss": 0.6343892812728882,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.1239517480134964,
+      "rewards/margins": 0.024958748370409012,
+      "rewards/rejected": -0.1489104926586151,
+      "sft_loss": 1.2395174503326416,
+      "step": 1310
+    },
+    {
+      "epoch": 2.34718826405868,
+      "grad_norm": 2.295304298400879,
+      "learning_rate": 5.592526130947862e-07,
+      "logits/chosen": 253.2501220703125,
+      "logits/rejected": 253.6666717529297,
+      "logps/chosen": -1.3131468296051025,
+      "logps/rejected": -1.5006693601608276,
+      "loss": 1.3841,
+      "odds_ratio_loss": 0.709243893623352,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.13131467998027802,
+      "rewards/margins": 0.018752261996269226,
+      "rewards/rejected": -0.15006692707538605,
+      "sft_loss": 1.3131468296051025,
+      "step": 1320
+    },
+    {
+      "epoch": 2.3649699933318518,
+      "grad_norm": 3.0129616260528564,
+      "learning_rate": 5.302249011195507e-07,
+      "logits/chosen": 254.51284790039062,
+      "logits/rejected": 254.08145141601562,
+      "logps/chosen": -1.2322486639022827,
+      "logps/rejected": -1.3891911506652832,
+      "loss": 1.3008,
+      "odds_ratio_loss": 0.6851211786270142,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.12322486937046051,
+      "rewards/margins": 0.015694255009293556,
+      "rewards/rejected": -0.13891912996768951,
+      "sft_loss": 1.2322486639022827,
+      "step": 1330
+    },
+    {
+      "epoch": 2.382751722605023,
+      "grad_norm": 4.828993797302246,
+      "learning_rate": 5.018811640997307e-07,
+      "logits/chosen": 255.24087524414062,
+      "logits/rejected": 255.8361358642578,
+      "logps/chosen": -1.3192299604415894,
+      "logps/rejected": -1.6515556573867798,
+      "loss": 1.3833,
+      "odds_ratio_loss": 0.6403645873069763,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.13192300498485565,
+      "rewards/margins": 0.03323255851864815,
+      "rewards/rejected": -0.1651555597782135,
+      "sft_loss": 1.3192299604415894,
+      "step": 1340
+    },
+    {
+      "epoch": 2.400533451878195,
+      "grad_norm": 1.868326187133789,
+      "learning_rate": 4.7423124397427105e-07,
+      "logits/chosen": 253.5162811279297,
+      "logits/rejected": 253.6891632080078,
+      "logps/chosen": -1.2553585767745972,
+      "logps/rejected": -1.4340143203735352,
+      "loss": 1.3209,
+      "odds_ratio_loss": 0.6558007001876831,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.12553584575653076,
+      "rewards/margins": 0.017865588888525963,
+      "rewards/rejected": -0.14340144395828247,
+      "sft_loss": 1.2553585767745972,
+      "step": 1350
+    },
+    {
+      "epoch": 2.418315181151367,
+      "grad_norm": 3.549006938934326,
+      "learning_rate": 4.472847417645787e-07,
+      "logits/chosen": 255.4973602294922,
+      "logits/rejected": 255.242431640625,
+      "logps/chosen": -1.3243919610977173,
+      "logps/rejected": -1.665623426437378,
+      "loss": 1.3842,
+      "odds_ratio_loss": 0.5979620814323425,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.13243919610977173,
+      "rewards/margins": 0.03412316367030144,
+      "rewards/rejected": -0.16656234860420227,
+      "sft_loss": 1.3243919610977173,
+      "step": 1360
+    },
+    {
+      "epoch": 2.436096910424539,
+      "grad_norm": 1.720616340637207,
+      "learning_rate": 4.210510142406993e-07,
+      "logits/chosen": 254.47842407226562,
+      "logits/rejected": 254.3418731689453,
+      "logps/chosen": -1.2948418855667114,
+      "logps/rejected": -1.6191186904907227,
+      "loss": 1.3596,
+      "odds_ratio_loss": 0.6471132040023804,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.1294841766357422,
+      "rewards/margins": 0.03242768719792366,
+      "rewards/rejected": -0.16191187500953674,
+      "sft_loss": 1.2948418855667114,
+      "step": 1370
+    },
+    {
+      "epoch": 2.4538786396977104,
+      "grad_norm": 2.2009122371673584,
+      "learning_rate": 3.9553917067232966e-07,
+      "logits/chosen": 255.0864715576172,
+      "logits/rejected": 255.9404754638672,
+      "logps/chosen": -1.2902610301971436,
+      "logps/rejected": -1.4941017627716064,
+      "loss": 1.3585,
+      "odds_ratio_loss": 0.6828353404998779,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.12902610003948212,
+      "rewards/margins": 0.02038406766951084,
+      "rewards/rejected": -0.1494101732969284,
+      "sft_loss": 1.2902610301971436,
+      "step": 1380
+    },
+    {
+      "epoch": 2.4716603689708823,
+      "grad_norm": 2.2108042240142822,
+      "learning_rate": 3.707580696657509e-07,
+      "logits/chosen": 253.1954345703125,
+      "logits/rejected": 253.47250366210938,
+      "logps/chosen": -1.2349462509155273,
+      "logps/rejected": -1.409597396850586,
+      "loss": 1.3025,
+      "odds_ratio_loss": 0.6752232313156128,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.12349464744329453,
+      "rewards/margins": 0.017465103417634964,
+      "rewards/rejected": -0.1409597247838974,
+      "sft_loss": 1.2349462509155273,
+      "step": 1390
+    },
+    {
+      "epoch": 2.489442098244054,
+      "grad_norm": 2.4644668102264404,
+      "learning_rate": 3.4671631608781815e-07,
+      "logits/chosen": 254.1881561279297,
+      "logits/rejected": 254.3625030517578,
+      "logps/chosen": -1.3036715984344482,
+      "logps/rejected": -1.4527599811553955,
+      "loss": 1.3758,
+      "odds_ratio_loss": 0.721402108669281,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.13036717474460602,
+      "rewards/margins": 0.014908820390701294,
+      "rewards/rejected": -0.14527598023414612,
+      "sft_loss": 1.3036715984344482,
+      "step": 1400
+    },
+    {
+      "epoch": 2.507223827517226,
+      "grad_norm": 3.9182987213134766,
+      "learning_rate": 3.234222580780405e-07,
+      "logits/chosen": 254.8416290283203,
+      "logits/rejected": 254.9809112548828,
+      "logps/chosen": -1.2807973623275757,
+      "logps/rejected": -1.4203259944915771,
+      "loss": 1.3485,
+      "odds_ratio_loss": 0.6770311594009399,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.12807972729206085,
+      "rewards/margins": 0.013952854089438915,
+      "rewards/rejected": -0.14203259348869324,
+      "sft_loss": 1.2807973623275757,
+      "step": 1410
+    },
+    {
+      "epoch": 2.525005556790398,
+      "grad_norm": 8.136197090148926,
+      "learning_rate": 3.0088398414982375e-07,
+      "logits/chosen": 255.15200805664062,
+      "logits/rejected": 255.2108612060547,
+      "logps/chosen": -1.269274115562439,
+      "logps/rejected": -1.5304492712020874,
+      "loss": 1.3364,
+      "odds_ratio_loss": 0.6713584661483765,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.12692740559577942,
+      "rewards/margins": 0.026117529720067978,
+      "rewards/rejected": -0.1530449539422989,
+      "sft_loss": 1.269274115562439,
+      "step": 1420
+    },
+    {
+      "epoch": 2.54278728606357,
+      "grad_norm": 3.0494956970214844,
+      "learning_rate": 2.7910932038184487e-07,
+      "logits/chosen": 252.93734741210938,
+      "logits/rejected": 254.24777221679688,
+      "logps/chosen": -1.24881112575531,
+      "logps/rejected": -1.4509551525115967,
+      "loss": 1.3164,
+      "odds_ratio_loss": 0.675870418548584,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.12488112598657608,
+      "rewards/margins": 0.020214393734931946,
+      "rewards/rejected": -0.14509549736976624,
+      "sft_loss": 1.24881112575531,
+      "step": 1430
+    },
+    {
+      "epoch": 2.5605690153367417,
+      "grad_norm": 3.582563638687134,
+      "learning_rate": 2.5810582770057325e-07,
+      "logits/chosen": 255.56686401367188,
+      "logits/rejected": 255.5573272705078,
+      "logps/chosen": -1.2086716890335083,
+      "logps/rejected": -1.3686209917068481,
+      "loss": 1.2763,
+      "odds_ratio_loss": 0.6763256788253784,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.12086717784404755,
+      "rewards/margins": 0.01599491387605667,
+      "rewards/rejected": -0.13686209917068481,
+      "sft_loss": 1.2086716890335083,
+      "step": 1440
+    },
+    {
+      "epoch": 2.578350744609913,
+      "grad_norm": 2.657351016998291,
+      "learning_rate": 2.3788079925484402e-07,
+      "logits/chosen": 254.91213989257812,
+      "logits/rejected": 254.9799346923828,
+      "logps/chosen": -1.3017337322235107,
+      "logps/rejected": -1.4446247816085815,
+      "loss": 1.3706,
+      "odds_ratio_loss": 0.6881921887397766,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.13017335534095764,
+      "rewards/margins": 0.01428910344839096,
+      "rewards/rejected": -0.1444624662399292,
+      "sft_loss": 1.3017337322235107,
+      "step": 1450
+    },
+    {
+      "epoch": 2.596132473883085,
+      "grad_norm": 3.423999309539795,
+      "learning_rate": 2.1844125788342661e-07,
+      "logits/chosen": 253.8058624267578,
+      "logits/rejected": 254.35348510742188,
+      "logps/chosen": -1.220453143119812,
+      "logps/rejected": -1.6302309036254883,
+      "loss": 1.284,
+      "odds_ratio_loss": 0.6357576251029968,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.12204531580209732,
+      "rewards/margins": 0.04097776859998703,
+      "rewards/rejected": -0.16302308440208435,
+      "sft_loss": 1.220453143119812,
+      "step": 1460
+    },
+    {
+      "epoch": 2.613914203156257,
+      "grad_norm": 1.957571029663086,
+      "learning_rate": 1.9979395367644428e-07,
+      "logits/chosen": 254.90676879882812,
+      "logits/rejected": 254.95101928710938,
+      "logps/chosen": -1.2191929817199707,
+      "logps/rejected": -1.4415347576141357,
+      "loss": 1.2838,
+      "odds_ratio_loss": 0.645682156085968,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.12191929668188095,
+      "rewards/margins": 0.022234182804822922,
+      "rewards/rejected": -0.14415349066257477,
+      "sft_loss": 1.2191929817199707,
+      "step": 1470
+    },
+    {
+      "epoch": 2.631695932429429,
+      "grad_norm": 4.140540599822998,
+      "learning_rate": 1.81945361631512e-07,
+      "logits/chosen": 254.05709838867188,
+      "logits/rejected": 253.84130859375,
+      "logps/chosen": -1.2751600742340088,
+      "logps/rejected": -1.4497580528259277,
+      "loss": 1.3463,
+      "odds_ratio_loss": 0.711692214012146,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.1275160014629364,
+      "rewards/margins": 0.017459798604249954,
+      "rewards/rejected": -0.14497579634189606,
+      "sft_loss": 1.2751600742340088,
+      "step": 1480
+    },
+    {
+      "epoch": 2.6494776617026004,
+      "grad_norm": 3.2640628814697266,
+      "learning_rate": 1.6490167940538343e-07,
+      "logits/chosen": 254.5946807861328,
+      "logits/rejected": 254.5460662841797,
+      "logps/chosen": -1.2259750366210938,
+      "logps/rejected": -1.4610532522201538,
+      "loss": 1.289,
+      "odds_ratio_loss": 0.6301438212394714,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.12259751558303833,
+      "rewards/margins": 0.023507816717028618,
+      "rewards/rejected": -0.1461053192615509,
+      "sft_loss": 1.2259750366210938,
+      "step": 1490
+    },
+    {
+      "epoch": 2.6672593909757722,
+      "grad_norm": 2.282480001449585,
+      "learning_rate": 1.4866882516191339e-07,
+      "logits/chosen": 253.92684936523438,
+      "logits/rejected": 254.4107208251953,
+      "logps/chosen": -1.259511113166809,
+      "logps/rejected": -1.3850682973861694,
+      "loss": 1.3302,
+      "odds_ratio_loss": 0.7073375582695007,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.1259511113166809,
+      "rewards/margins": 0.012555733323097229,
+      "rewards/rejected": -0.13850684463977814,
+      "sft_loss": 1.259511113166809,
+      "step": 1500
+    },
+    {
+      "epoch": 2.6672593909757722,
+      "eval_logits/chosen": 254.46791076660156,
+      "eval_logits/rejected": 254.66136169433594,
+      "eval_logps/chosen": -1.2813518047332764,
+      "eval_logps/rejected": -1.5003557205200195,
+      "eval_loss": 1.347064733505249,
+      "eval_odds_ratio_loss": 0.6571269631385803,
+      "eval_rewards/accuracies": 0.5609999895095825,
+      "eval_rewards/chosen": -0.12813518941402435,
+      "eval_rewards/margins": 0.021900387480854988,
+      "eval_rewards/rejected": -0.15003558993339539,
+      "eval_runtime": 217.1974,
+      "eval_samples_per_second": 4.604,
+      "eval_sft_loss": 1.2813518047332764,
+      "eval_steps_per_second": 2.302,
+      "step": 1500
+    },
+    {
+      "epoch": 2.685041120248944,
+      "grad_norm": 6.244467258453369,
+      "learning_rate": 1.3325243551706057e-07,
+      "logits/chosen": 254.00894165039062,
+      "logits/rejected": 254.4985809326172,
+      "logps/chosen": -1.272092580795288,
+      "logps/rejected": -1.6640291213989258,
+      "loss": 1.3335,
+      "odds_ratio_loss": 0.6145035624504089,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.12720926105976105,
+      "rewards/margins": 0.03919365257024765,
+      "rewards/rejected": -0.1664029359817505,
+      "sft_loss": 1.272092580795288,
+      "step": 1510
+    },
+    {
+      "epoch": 2.702822849522116,
+      "grad_norm": 6.613005638122559,
+      "learning_rate": 1.1865786358165737e-07,
+      "logits/chosen": 254.2620086669922,
+      "logits/rejected": 254.74526977539062,
+      "logps/chosen": -1.2863306999206543,
+      "logps/rejected": -1.474635124206543,
+      "loss": 1.3537,
+      "odds_ratio_loss": 0.6741579174995422,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.12863309681415558,
+      "rewards/margins": 0.0188304390758276,
+      "rewards/rejected": -0.14746353030204773,
+      "sft_loss": 1.2863306999206543,
+      "step": 1520
+    },
+    {
+      "epoch": 2.720604578795288,
+      "grad_norm": 6.351752758026123,
+      "learning_rate": 1.0489017710262311e-07,
+      "logits/chosen": 255.4362030029297,
+      "logits/rejected": 256.20306396484375,
+      "logps/chosen": -1.331672191619873,
+      "logps/rejected": -1.618596076965332,
+      "loss": 1.4013,
+      "odds_ratio_loss": 0.6964471340179443,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.13316722214221954,
+      "rewards/margins": 0.028692390769720078,
+      "rewards/rejected": -0.16185960173606873,
+      "sft_loss": 1.331672191619873,
+      "step": 1530
+    },
+    {
+      "epoch": 2.73838630806846,
+      "grad_norm": 2.5552542209625244,
+      "learning_rate": 9.195415670326446e-08,
+      "logits/chosen": 256.7084655761719,
+      "logits/rejected": 256.827392578125,
+      "logps/chosen": -1.285346269607544,
+      "logps/rejected": -1.4887077808380127,
+      "loss": 1.3524,
+      "odds_ratio_loss": 0.6702226400375366,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.12853462994098663,
+      "rewards/margins": 0.020336147397756577,
+      "rewards/rejected": -0.1488707810640335,
+      "sft_loss": 1.285346269607544,
+      "step": 1540
+    },
+    {
+      "epoch": 2.7561680373416317,
+      "grad_norm": 4.229678630828857,
+      "learning_rate": 7.985429422327384e-08,
+      "logits/chosen": 254.65280151367188,
+      "logits/rejected": 255.42001342773438,
+      "logps/chosen": -1.2554762363433838,
+      "logps/rejected": -1.3682177066802979,
+      "loss": 1.3267,
+      "odds_ratio_loss": 0.7126489877700806,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.1255476176738739,
+      "rewards/margins": 0.011274145916104317,
+      "rewards/rejected": -0.13682177662849426,
+      "sft_loss": 1.2554762363433838,
+      "step": 1550
+    },
+    {
+      "epoch": 2.773949766614803,
+      "grad_norm": 3.981313705444336,
+      "learning_rate": 6.859479115900818e-08,
+      "logits/chosen": 254.1326446533203,
+      "logits/rejected": 254.6916961669922,
+      "logps/chosen": -1.359695553779602,
+      "logps/rejected": -1.6282466650009155,
+      "loss": 1.425,
+      "odds_ratio_loss": 0.6527189016342163,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.13596956431865692,
+      "rewards/margins": 0.026855114847421646,
+      "rewards/rejected": -0.16282466053962708,
+      "sft_loss": 1.359695553779602,
+      "step": 1560
+    },
+    {
+      "epoch": 2.791731495887975,
+      "grad_norm": 3.927825450897217,
+      "learning_rate": 5.817955720457902e-08,
+      "logits/chosen": 253.6094970703125,
+      "logits/rejected": 253.5572967529297,
+      "logps/chosen": -1.1961861848831177,
+      "logps/rejected": -1.3642635345458984,
+      "loss": 1.262,
+      "odds_ratio_loss": 0.6578332185745239,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.11961860954761505,
+      "rewards/margins": 0.016807744279503822,
+      "rewards/rejected": -0.13642635941505432,
+      "sft_loss": 1.1961861848831177,
+      "step": 1570
+    },
+    {
+      "epoch": 2.809513225161147,
+      "grad_norm": 3.0377912521362305,
+      "learning_rate": 4.861220889427199e-08,
+      "logits/chosen": 254.80587768554688,
+      "logits/rejected": 254.9072265625,
+      "logps/chosen": -1.2765783071517944,
+      "logps/rejected": -1.4059991836547852,
+      "loss": 1.3469,
+      "odds_ratio_loss": 0.7033384442329407,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.12765783071517944,
+      "rewards/margins": 0.012942099943757057,
+      "rewards/rejected": -0.14059992134571075,
+      "sft_loss": 1.2765783071517944,
+      "step": 1580
+    },
+    {
+      "epoch": 2.827294954434319,
+      "grad_norm": 1.8500066995620728,
+      "learning_rate": 3.9896068346758074e-08,
+      "logits/chosen": 254.78158569335938,
+      "logits/rejected": 254.95730590820312,
+      "logps/chosen": -1.2893197536468506,
+      "logps/rejected": -1.4526543617248535,
+      "loss": 1.3588,
+      "odds_ratio_loss": 0.6947753429412842,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.12893196940422058,
+      "rewards/margins": 0.01633346639573574,
+      "rewards/rejected": -0.14526543021202087,
+      "sft_loss": 1.2893197536468506,
+      "step": 1590
+    },
+    {
+      "epoch": 2.8450766837074903,
+      "grad_norm": 5.4357829093933105,
+      "learning_rate": 3.203416211153832e-08,
+      "logits/chosen": 254.9698944091797,
+      "logits/rejected": 256.3062438964844,
+      "logps/chosen": -1.2634626626968384,
+      "logps/rejected": -1.4395668506622314,
+      "loss": 1.3344,
+      "odds_ratio_loss": 0.7093390822410583,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.12634627521038055,
+      "rewards/margins": 0.017610404640436172,
+      "rewards/rejected": -0.14395669102668762,
+      "sft_loss": 1.2634626626968384,
+      "step": 1600
+    },
+    {
+      "epoch": 2.8628584129806622,
+      "grad_norm": 3.7756481170654297,
+      "learning_rate": 2.5029220118019393e-08,
+      "logits/chosen": 253.7996826171875,
+      "logits/rejected": 254.91201782226562,
+      "logps/chosen": -1.3131462335586548,
+      "logps/rejected": -1.434832215309143,
+      "loss": 1.3818,
+      "odds_ratio_loss": 0.6866922974586487,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.13131462037563324,
+      "rewards/margins": 0.012168603017926216,
+      "rewards/rejected": -0.1434832215309143,
+      "sft_loss": 1.3131462335586548,
+      "step": 1610
+    },
+    {
+      "epoch": 2.880640142253834,
+      "grad_norm": 3.6297013759613037,
+      "learning_rate": 1.8883674727586122e-08,
+      "logits/chosen": 254.3390350341797,
+      "logits/rejected": 254.14718627929688,
+      "logps/chosen": -1.1930474042892456,
+      "logps/rejected": -1.5245510339736938,
+      "loss": 1.2541,
+      "odds_ratio_loss": 0.6109124422073364,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.11930473893880844,
+      "rewards/margins": 0.03315035253763199,
+      "rewards/rejected": -0.15245509147644043,
+      "sft_loss": 1.1930474042892456,
+      "step": 1620
+    },
+    {
+      "epoch": 2.898421871527006,
+      "grad_norm": 1.5044124126434326,
+      "learning_rate": 1.3599659889000639e-08,
+      "logits/chosen": 255.17379760742188,
+      "logits/rejected": 254.8286895751953,
+      "logps/chosen": -1.3174787759780884,
+      "logps/rejected": -1.4666922092437744,
+      "loss": 1.3872,
+      "odds_ratio_loss": 0.6974038481712341,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.13174788653850555,
+      "rewards/margins": 0.01492134016007185,
+      "rewards/rejected": -0.14666922390460968,
+      "sft_loss": 1.3174787759780884,
+      "step": 1630
+    },
+    {
+      "epoch": 2.916203600800178,
+      "grad_norm": 2.1138768196105957,
+      "learning_rate": 9.179010397421528e-09,
+      "logits/chosen": 256.32757568359375,
+      "logits/rejected": 255.87295532226562,
+      "logps/chosen": -1.26983642578125,
+      "logps/rejected": -1.4480781555175781,
+      "loss": 1.3386,
+      "odds_ratio_loss": 0.6873185038566589,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.1269836574792862,
+      "rewards/margins": 0.017824167385697365,
+      "rewards/rejected": -0.144807830452919,
+      "sft_loss": 1.26983642578125,
+      "step": 1640
+    },
+    {
+      "epoch": 2.93398533007335,
+      "grad_norm": 1.8579224348068237,
+      "learning_rate": 5.623261257296509e-09,
+      "logits/chosen": 253.3066864013672,
+      "logits/rejected": 253.75753784179688,
+      "logps/chosen": -1.1514933109283447,
+      "logps/rejected": -1.3553564548492432,
+      "loss": 1.2157,
+      "odds_ratio_loss": 0.6422222256660461,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.11514933407306671,
+      "rewards/margins": 0.020386312156915665,
+      "rewards/rejected": -0.13553564250469208,
+      "sft_loss": 1.1514933109283447,
+      "step": 1650
+    },
+    {
+      "epoch": 2.9517670593465217,
+      "grad_norm": 2.8349151611328125,
+      "learning_rate": 2.933647149357122e-09,
+      "logits/chosen": 254.45968627929688,
+      "logits/rejected": 254.57400512695312,
+      "logps/chosen": -1.300631046295166,
+      "logps/rejected": -1.4870007038116455,
+      "loss": 1.3705,
+      "odds_ratio_loss": 0.6987138390541077,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.13006310164928436,
+      "rewards/margins": 0.018636973574757576,
+      "rewards/rejected": -0.14870008826255798,
+      "sft_loss": 1.300631046295166,
+      "step": 1660
+    },
+    {
+      "epoch": 2.969548788619693,
+      "grad_norm": 2.5957581996917725,
+      "learning_rate": 1.1111020018930717e-09,
+      "logits/chosen": 255.29000854492188,
+      "logits/rejected": 255.3350830078125,
+      "logps/chosen": -1.2563540935516357,
+      "logps/rejected": -1.3871452808380127,
+      "loss": 1.3264,
+      "odds_ratio_loss": 0.7003098726272583,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.12563541531562805,
+      "rewards/margins": 0.013079119846224785,
+      "rewards/rejected": -0.1387145221233368,
+      "sft_loss": 1.2563540935516357,
+      "step": 1670
+    },
+    {
+      "epoch": 2.987330517892865,
+      "grad_norm": 1.7800966501235962,
+      "learning_rate": 1.5625866646051813e-10,
+      "logits/chosen": 255.032470703125,
+      "logits/rejected": 255.3948974609375,
+      "logps/chosen": -1.2115042209625244,
+      "logps/rejected": -1.4695723056793213,
+      "loss": 1.2721,
+      "odds_ratio_loss": 0.6058895587921143,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.1211504191160202,
+      "rewards/margins": 0.025806818157434464,
+      "rewards/rejected": -0.14695724844932556,
+      "sft_loss": 1.2115042209625244,
+      "step": 1680
+    },
+    {
+      "epoch": 2.997999555456768,
+      "step": 1686,
+      "total_flos": 2.0747345909474918e+18,
+      "train_loss": 1.4404786094086308,
+      "train_runtime": 22862.0738,
+      "train_samples_per_second": 1.181,
+      "train_steps_per_second": 0.074
     }
   ],
   "logging_steps": 10,
-  "max_steps": 1,
+  "max_steps": 1686,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
-  "save_steps": 10,
-  "total_flos": 748013124648960.0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "total_flos": 2.0747345909474918e+18,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null