diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -1,301 +1,2939 @@
 {
-  "best_metric": null,
-  "best_model_checkpoint": null,
-  "epoch": 2.986666666666667,
+  "best_metric": 0.8975116014480591,
+  "best_model_checkpoint": "saves/Mistral-7B-Instruct-v0.2/lora/orpo/checkpoint-1500",
+  "epoch": 2.997999555456768,
   "eval_steps": 500,
-  "global_step": 168,
+  "global_step": 1686,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.17777777777777778,
-      "grad_norm": 1.5870693922042847,
-      "learning_rate": 4.957230266673969e-06,
-      "logits/chosen": -2.5848517417907715,
-      "logits/rejected": -2.592822313308716,
-      "logps/chosen": -1.2300995588302612,
-      "logps/rejected": -1.2750391960144043,
-      "loss": 1.3004,
-      "odds_ratio_loss": 0.7031511068344116,
-      "rewards/accuracies": 0.45625001192092896,
-      "rewards/chosen": -0.12300996482372284,
-      "rewards/margins": 0.004493957385420799,
-      "rewards/rejected": -0.1275039166212082,
-      "sft_loss": 1.2300995588302612,
+      "epoch": 0.017781729273171815,
+      "grad_norm": 3.822810173034668,
+      "learning_rate": 4.9995745934141085e-06,
+      "logits/chosen": -2.5146775245666504,
+      "logits/rejected": -2.5178749561309814,
+      "logps/chosen": -1.456717848777771,
+      "logps/rejected": -2.2312474250793457,
+      "loss": 1.542,
+      "odds_ratio_loss": 0.853233814239502,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.1456717997789383,
+      "rewards/margins": 0.07745292782783508,
+      "rewards/rejected": -0.22312471270561218,
+      "sft_loss": 1.456717848777771,
       "step": 10
     },
     {
-      "epoch": 0.35555555555555557,
-      "grad_norm": 2.5478012561798096,
-      "learning_rate": 4.84529614391025e-06,
-      "logits/chosen": -2.568402051925659,
-      "logits/rejected": -2.5743260383605957,
-      "logps/chosen": -1.1148786544799805,
-      "logps/rejected": -1.2647678852081299,
-      "loss": 1.1815,
-      "odds_ratio_loss": 0.6664617657661438,
-      "rewards/accuracies": 0.48124998807907104,
-      "rewards/chosen": -0.11148785054683685,
-      "rewards/margins": 0.014988938346505165,
-      "rewards/rejected": -0.12647680938243866,
-      "sft_loss": 1.1148786544799805,
+      "epoch": 0.03556345854634363,
+      "grad_norm": 37.87593078613281,
+      "learning_rate": 4.9982812903243405e-06,
+      "logits/chosen": -2.5083491802215576,
+      "logits/rejected": -2.5308327674865723,
+      "logps/chosen": -1.7510416507720947,
+      "logps/rejected": -1.991346001625061,
+      "loss": 1.8501,
+      "odds_ratio_loss": 0.9901398420333862,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.17510415613651276,
+      "rewards/margins": 0.024030480533838272,
+      "rewards/rejected": -0.19913461804389954,
+      "sft_loss": 1.7510416507720947,
       "step": 20
     },
     {
-      "epoch": 0.5333333333333333,
-      "grad_norm": 1.738682746887207,
-      "learning_rate": 4.667009949002349e-06,
-      "logits/chosen": -2.5901498794555664,
-      "logits/rejected": -2.6051435470581055,
-      "logps/chosen": -1.2092516422271729,
-      "logps/rejected": -1.3586838245391846,
-      "loss": 1.2768,
-      "odds_ratio_loss": 0.6759374737739563,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.120925173163414,
-      "rewards/margins": 0.01494320947676897,
-      "rewards/rejected": -0.1358683854341507,
-      "sft_loss": 1.2092516422271729,
+      "epoch": 0.05334518781951545,
+      "grad_norm": 4.208436012268066,
+      "learning_rate": 4.996120496405222e-06,
+      "logits/chosen": -2.520888090133667,
+      "logits/rejected": -2.4970860481262207,
+      "logps/chosen": -1.4711406230926514,
+      "logps/rejected": -2.380532741546631,
+      "loss": 1.5428,
+      "odds_ratio_loss": 0.7161206603050232,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.14711406826972961,
+      "rewards/margins": 0.09093920141458511,
+      "rewards/rejected": -0.23805327713489532,
+      "sft_loss": 1.4711406230926514,
       "step": 30
     },
     {
-      "epoch": 0.7111111111111111,
-      "grad_norm": 10.248797416687012,
-      "learning_rate": 4.397288409237892e-06,
-      "logits/chosen": -2.4943628311157227,
-      "logits/rejected": -2.5201773643493652,
-      "logps/chosen": -1.1567463874816895,
-      "logps/rejected": -1.3316763639450073,
-      "loss": 1.2213,
-      "odds_ratio_loss": 0.6450860500335693,
-      "rewards/accuracies": 0.48124998807907104,
-      "rewards/chosen": -0.11567463725805283,
-      "rewards/margins": 0.017492998391389847,
-      "rewards/rejected": -0.13316763937473297,
-      "sft_loss": 1.1567463874816895,
+      "epoch": 0.07112691709268726,
+      "grad_norm": 5.997638702392578,
+      "learning_rate": 4.99309296196014e-06,
+      "logits/chosen": -2.5280556678771973,
+      "logits/rejected": -2.552898645401001,
+      "logps/chosen": -1.3411036729812622,
+      "logps/rejected": -1.9365333318710327,
+      "loss": 1.4122,
+      "odds_ratio_loss": 0.711218535900116,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.13411036133766174,
+      "rewards/margins": 0.059542976319789886,
+      "rewards/rejected": -0.19365334510803223,
+      "sft_loss": 1.3411036729812622,
       "step": 40
     },
     {
-      "epoch": 0.8888888888888888,
-      "grad_norm": 6.022939682006836,
-      "learning_rate": 4.097593272948218e-06,
-      "logits/chosen": -2.6041746139526367,
-      "logits/rejected": -2.6204562187194824,
-      "logps/chosen": -1.2715274095535278,
-      "logps/rejected": -1.6485040187835693,
-      "loss": 1.3385,
-      "odds_ratio_loss": 0.6699766516685486,
-      "rewards/accuracies": 0.48124998807907104,
-      "rewards/chosen": -0.12715274095535278,
-      "rewards/margins": 0.03769766539335251,
-      "rewards/rejected": -0.1648503988981247,
-      "sft_loss": 1.2715274095535278,
+      "epoch": 0.08890864636585907,
+      "grad_norm": 6.682967185974121,
+      "learning_rate": 4.989199738255166e-06,
+      "logits/chosen": -2.5629172325134277,
+      "logits/rejected": -2.5831732749938965,
+      "logps/chosen": -1.3399975299835205,
+      "logps/rejected": -1.9187812805175781,
+      "loss": 1.4155,
+      "odds_ratio_loss": 0.7554818391799927,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.133999764919281,
+      "rewards/margins": 0.05787837505340576,
+      "rewards/rejected": -0.19187815487384796,
+      "sft_loss": 1.3399975299835205,
       "step": 50
     },
     {
-      "epoch": 1.0666666666666667,
-      "grad_norm": 1.5987056493759155,
-      "learning_rate": 3.712001321256744e-06,
-      "logits/chosen": -2.6027374267578125,
-      "logits/rejected": -2.627061128616333,
-      "logps/chosen": -1.0649330615997314,
-      "logps/rejected": -1.3192559480667114,
-      "loss": 1.1266,
-      "odds_ratio_loss": 0.616315484046936,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.10649331659078598,
-      "rewards/margins": 0.025432268157601357,
-      "rewards/rejected": -0.13192559778690338,
-      "sft_loss": 1.0649330615997314,
+      "epoch": 0.1066903756390309,
+      "grad_norm": 3.520189046859741,
+      "learning_rate": 4.984442177154031e-06,
+      "logits/chosen": -2.5552244186401367,
+      "logits/rejected": -2.5440051555633545,
+      "logps/chosen": -1.3248951435089111,
+      "logps/rejected": -1.8020858764648438,
+      "loss": 1.4047,
+      "odds_ratio_loss": 0.7983237504959106,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.13248953223228455,
+      "rewards/margins": 0.047719065099954605,
+      "rewards/rejected": -0.18020859360694885,
+      "sft_loss": 1.3248951435089111,
       "step": 60
     },
     {
-      "epoch": 1.2444444444444445,
-      "grad_norm": 9.42609977722168,
-      "learning_rate": 3.284100248421622e-06,
-      "logits/chosen": -2.592658758163452,
-      "logits/rejected": -2.612031936645508,
-      "logps/chosen": -1.0701837539672852,
-      "logps/rejected": -1.2456352710723877,
-      "loss": 1.1365,
-      "odds_ratio_loss": 0.6634659767150879,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.10701838880777359,
-      "rewards/margins": 0.017545154318213463,
-      "rewards/rejected": -0.1245635375380516,
-      "sft_loss": 1.0701837539672852,
+      "epoch": 0.12447210491220272,
+      "grad_norm": 5.902307987213135,
+      "learning_rate": 4.978821930648704e-06,
+      "logits/chosen": -2.6206307411193848,
+      "logits/rejected": -2.6300582885742188,
+      "logps/chosen": -1.4133307933807373,
+      "logps/rejected": -1.5712898969650269,
+      "loss": 1.5087,
+      "odds_ratio_loss": 0.9536682963371277,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.14133310317993164,
+      "rewards/margins": 0.01579589582979679,
+      "rewards/rejected": -0.15712900459766388,
+      "sft_loss": 1.4133307933807373,
       "step": 70
     },
     {
-      "epoch": 1.4222222222222223,
-      "grad_norm": 1.1247073411941528,
-      "learning_rate": 2.8288274295187397e-06,
-      "logits/chosen": -2.6690917015075684,
-      "logits/rejected": -2.6884384155273438,
-      "logps/chosen": -1.0463426113128662,
-      "logps/rejected": -1.2134628295898438,
-      "loss": 1.1088,
-      "odds_ratio_loss": 0.6244794130325317,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.10463427007198334,
-      "rewards/margins": 0.016712015494704247,
-      "rewards/rejected": -0.12134629487991333,
-      "sft_loss": 1.0463426113128662,
+      "epoch": 0.14225383418537452,
+      "grad_norm": 6.316813945770264,
+      "learning_rate": 4.97234095028576e-06,
+      "logits/chosen": -2.6114909648895264,
+      "logits/rejected": -2.6050069332122803,
+      "logps/chosen": -1.1831285953521729,
+      "logps/rejected": -1.6871837377548218,
+      "loss": 1.2502,
+      "odds_ratio_loss": 0.670314371585846,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.11831285059452057,
+      "rewards/margins": 0.050405532121658325,
+      "rewards/rejected": -0.1687183678150177,
+      "sft_loss": 1.1831285953521729,
       "step": 80
     },
     {
-      "epoch": 1.6,
-      "grad_norm": 8.079381942749023,
-      "learning_rate": 2.3620757456122797e-06,
-      "logits/chosen": -2.638796329498291,
-      "logits/rejected": -2.6489083766937256,
-      "logps/chosen": -1.0774247646331787,
-      "logps/rejected": -1.465980887413025,
-      "loss": 1.1411,
-      "odds_ratio_loss": 0.6369243860244751,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.10774248838424683,
-      "rewards/margins": 0.03885561600327492,
-      "rewards/rejected": -0.14659810066223145,
-      "sft_loss": 1.0774247646331787,
+      "epoch": 0.16003556345854633,
+      "grad_norm": 2.3047008514404297,
+      "learning_rate": 4.965001486488743e-06,
+      "logits/chosen": -2.5930087566375732,
+      "logits/rejected": -2.589054584503174,
+      "logps/chosen": -1.0415079593658447,
+      "logps/rejected": -1.5394554138183594,
+      "loss": 1.1006,
+      "odds_ratio_loss": 0.5907732248306274,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.10415079444646835,
+      "rewards/margins": 0.049794748425483704,
+      "rewards/rejected": -0.15394553542137146,
+      "sft_loss": 1.0415079593658447,
       "step": 90
     },
     {
-      "epoch": 1.7777777777777777,
-      "grad_norm": 3.1488540172576904,
-      "learning_rate": 1.9001387874698865e-06,
-      "logits/chosen": -2.659306049346924,
-      "logits/rejected": -2.67085599899292,
-      "logps/chosen": -1.0682704448699951,
-      "logps/rejected": -1.2112770080566406,
-      "loss": 1.1326,
-      "odds_ratio_loss": 0.6432015299797058,
+      "epoch": 0.17781729273171815,
+      "grad_norm": 3.8059961795806885,
+      "learning_rate": 4.956806087776732e-06,
+      "logits/chosen": -2.6263413429260254,
+      "logits/rejected": -2.6199755668640137,
+      "logps/chosen": -1.0670406818389893,
+      "logps/rejected": -2.051734447479248,
+      "loss": 1.1262,
+      "odds_ratio_loss": 0.5915437340736389,
       "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.10682705789804459,
-      "rewards/margins": 0.01430064719170332,
-      "rewards/rejected": -0.12112770229578018,
-      "sft_loss": 1.0682704448699951,
+      "rewards/chosen": -0.10670407116413116,
+      "rewards/margins": 0.09846936166286469,
+      "rewards/rejected": -0.20517341792583466,
+      "sft_loss": 1.0670406818389893,
       "step": 100
     },
     {
-      "epoch": 1.9555555555555557,
-      "grad_norm": 2.191756010055542,
-      "learning_rate": 1.4591420711120425e-06,
-      "logits/chosen": -2.685157299041748,
-      "logits/rejected": -2.695525646209717,
-      "logps/chosen": -1.0902485847473145,
-      "logps/rejected": -1.2659653425216675,
-      "loss": 1.1523,
-      "odds_ratio_loss": 0.6206409335136414,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": -0.10902485996484756,
-      "rewards/margins": 0.017571674659848213,
-      "rewards/rejected": -0.12659654021263123,
-      "sft_loss": 1.0902485847473145,
+      "epoch": 0.19559902200489,
+      "grad_norm": 13.373092651367188,
+      "learning_rate": 4.947757599879411e-06,
+      "logits/chosen": -2.6753718852996826,
+      "logits/rejected": -2.694586992263794,
+      "logps/chosen": -1.2387793064117432,
+      "logps/rejected": -1.7975523471832275,
+      "loss": 1.3076,
+      "odds_ratio_loss": 0.6879797577857971,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.12387792766094208,
+      "rewards/margins": 0.05587731674313545,
+      "rewards/rejected": -0.17975525557994843,
+      "sft_loss": 1.2387793064117432,
       "step": 110
     },
     {
-      "epoch": 2.1333333333333333,
-      "grad_norm": 1.3168927431106567,
-      "learning_rate": 1.0544801205950327e-06,
-      "logits/chosen": -2.6532204151153564,
-      "logits/rejected": -2.6720328330993652,
-      "logps/chosen": -1.0528920888900757,
-      "logps/rejected": -1.2378777265548706,
-      "loss": 1.1173,
-      "odds_ratio_loss": 0.6442069411277771,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.10528920590877533,
-      "rewards/margins": 0.018498573452234268,
-      "rewards/rejected": -0.1237877756357193,
-      "sft_loss": 1.0528920888900757,
+      "epoch": 0.2133807512780618,
+      "grad_norm": 3.2554187774658203,
+      "learning_rate": 4.937859164748931e-06,
+      "logits/chosen": -2.7004992961883545,
+      "logits/rejected": -2.7187867164611816,
+      "logps/chosen": -0.9789896011352539,
+      "logps/rejected": -1.210654854774475,
+      "loss": 1.0426,
+      "odds_ratio_loss": 0.6358731985092163,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.09789897501468658,
+      "rewards/margins": 0.023166518658399582,
+      "rewards/rejected": -0.12106549739837646,
+      "sft_loss": 0.9789896011352539,
       "step": 120
     },
     {
-      "epoch": 2.311111111111111,
-      "grad_norm": 0.9034324288368225,
-      "learning_rate": 7.002790686106056e-07,
-      "logits/chosen": -2.6588597297668457,
-      "logits/rejected": -2.6591033935546875,
-      "logps/chosen": -0.9490475654602051,
-      "logps/rejected": -1.0968741178512573,
-      "loss": 1.0107,
-      "odds_ratio_loss": 0.616373598575592,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.09490474313497543,
-      "rewards/margins": 0.01478266529738903,
-      "rewards/rejected": -0.1096874251961708,
-      "sft_loss": 0.9490475654602051,
+      "epoch": 0.23116248055123362,
+      "grad_norm": 3.08980393409729,
+      "learning_rate": 4.92711421946891e-06,
+      "logits/chosen": -2.654874563217163,
+      "logits/rejected": -2.6488099098205566,
+      "logps/chosen": -1.0345842838287354,
+      "logps/rejected": -1.5608960390090942,
+      "loss": 1.0981,
+      "odds_ratio_loss": 0.6348952054977417,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.10345842689275742,
+      "rewards/margins": 0.052631180733442307,
+      "rewards/rejected": -0.15608961880207062,
+      "sft_loss": 1.0345842838287354,
       "step": 130
     },
     {
-      "epoch": 2.488888888888889,
-      "grad_norm": 1.7593573331832886,
-      "learning_rate": 4.0890353469735717e-07,
-      "logits/chosen": -2.686373472213745,
-      "logits/rejected": -2.7087228298187256,
-      "logps/chosen": -1.0427912473678589,
-      "logps/rejected": -1.486315131187439,
-      "loss": 1.1034,
-      "odds_ratio_loss": 0.6059055328369141,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.10427911579608917,
-      "rewards/margins": 0.04435240477323532,
-      "rewards/rejected": -0.1486315280199051,
-      "sft_loss": 1.0427912473678589,
+      "epoch": 0.24894420982440543,
+      "grad_norm": 2.4850211143493652,
+      "learning_rate": 4.915526495060961e-06,
+      "logits/chosen": -2.7222416400909424,
+      "logits/rejected": -2.716777801513672,
+      "logps/chosen": -0.9182409048080444,
+      "logps/rejected": -1.3934853076934814,
+      "loss": 0.9758,
+      "odds_ratio_loss": 0.5755457878112793,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.09182409942150116,
+      "rewards/margins": 0.047524429857730865,
+      "rewards/rejected": -0.13934853672981262,
+      "sft_loss": 0.9182409048080444,
       "step": 140
     },
     {
-      "epoch": 2.6666666666666665,
-      "grad_norm": 5.990176200866699,
-      "learning_rate": 1.905249951948071e-07,
-      "logits/chosen": -2.715557813644409,
-      "logits/rejected": -2.731482982635498,
-      "logps/chosen": -1.144046425819397,
-      "logps/rejected": -1.2883504629135132,
-      "loss": 1.2074,
-      "odds_ratio_loss": 0.6331667304039001,
-      "rewards/accuracies": 0.48124998807907104,
-      "rewards/chosen": -0.11440464109182358,
-      "rewards/margins": 0.014430420473217964,
-      "rewards/rejected": -0.1288350522518158,
-      "sft_loss": 1.144046425819397,
+      "epoch": 0.26672593909757725,
+      "grad_norm": 2.675926446914673,
+      "learning_rate": 4.903100015189153e-06,
+      "logits/chosen": -2.693967342376709,
+      "logits/rejected": -2.7088241577148438,
+      "logps/chosen": -0.9448526501655579,
+      "logps/rejected": -1.2143518924713135,
+      "loss": 1.0096,
+      "odds_ratio_loss": 0.6471182107925415,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.09448526054620743,
+      "rewards/margins": 0.026949917897582054,
+      "rewards/rejected": -0.12143518775701523,
+      "sft_loss": 0.9448526501655579,
       "step": 150
     },
     {
-      "epoch": 2.8444444444444446,
-      "grad_norm": 2.793104887008667,
-      "learning_rate": 5.2766712488021566e-08,
-      "logits/chosen": -2.695949077606201,
-      "logits/rejected": -2.7109360694885254,
-      "logps/chosen": -0.9868586659431458,
-      "logps/rejected": -1.2109596729278564,
-      "loss": 1.0456,
-      "odds_ratio_loss": 0.5878169536590576,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.0986858680844307,
-      "rewards/margins": 0.022410111501812935,
-      "rewards/rejected": -0.12109597772359848,
-      "sft_loss": 0.9868586659431458,
+      "epoch": 0.28450766837074903,
+      "grad_norm": 2.688988208770752,
+      "learning_rate": 4.889839094762848e-06,
+      "logits/chosen": -2.6985411643981934,
+      "logits/rejected": -2.6904616355895996,
+      "logps/chosen": -0.9669437408447266,
+      "logps/rejected": -1.2702093124389648,
+      "loss": 1.0307,
+      "odds_ratio_loss": 0.6373674869537354,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.09669436514377594,
+      "rewards/margins": 0.030326563864946365,
+      "rewards/rejected": -0.1270209401845932,
+      "sft_loss": 0.9669437408447266,
       "step": 160
     },
     {
-      "epoch": 2.986666666666667,
-      "step": 168,
-      "total_flos": 2.303993975490478e+17,
-      "train_loss": 1.1586682626179285,
-      "train_runtime": 5352.1206,
-      "train_samples_per_second": 0.504,
-      "train_steps_per_second": 0.031
+      "epoch": 0.3022893976439209,
+      "grad_norm": 2.4917891025543213,
+      "learning_rate": 4.875748338438416e-06,
+      "logits/chosen": -2.7611794471740723,
+      "logits/rejected": -2.761101722717285,
+      "logps/chosen": -1.059195876121521,
+      "logps/rejected": -1.361277461051941,
+      "loss": 1.1294,
+      "odds_ratio_loss": 0.7024275064468384,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.10591959953308105,
+      "rewards/margins": 0.03020814061164856,
+      "rewards/rejected": -0.13612774014472961,
+      "sft_loss": 1.059195876121521,
+      "step": 170
+    },
+    {
+      "epoch": 0.32007112691709266,
+      "grad_norm": 1.744297981262207,
+      "learning_rate": 4.8608326390203386e-06,
+      "logits/chosen": -2.774660348892212,
+      "logits/rejected": -2.7589707374572754,
+      "logps/chosen": -0.9749459028244019,
+      "logps/rejected": -1.2964599132537842,
+      "loss": 1.0378,
+      "odds_ratio_loss": 0.6283431053161621,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.09749459475278854,
+      "rewards/margins": 0.03215140849351883,
+      "rewards/rejected": -0.12964601814746857,
+      "sft_loss": 0.9749459028244019,
+      "step": 180
+    },
+    {
+      "epoch": 0.3378528561902645,
+      "grad_norm": 3.190150022506714,
+      "learning_rate": 4.845097175762251e-06,
+      "logits/chosen": -2.794633626937866,
+      "logits/rejected": -2.8166775703430176,
+      "logps/chosen": -1.0003185272216797,
+      "logps/rejected": -1.2995705604553223,
+      "loss": 1.0657,
+      "odds_ratio_loss": 0.6539334058761597,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.10003185272216797,
+      "rewards/margins": 0.029925208538770676,
+      "rewards/rejected": -0.12995705008506775,
+      "sft_loss": 1.0003185272216797,
+      "step": 190
+    },
+    {
+      "epoch": 0.3556345854634363,
+      "grad_norm": 11.579193115234375,
+      "learning_rate": 4.8285474125685286e-06,
+      "logits/chosen": -2.8008620738983154,
+      "logits/rejected": -2.8179948329925537,
+      "logps/chosen": -0.9980157017707825,
+      "logps/rejected": -1.1520756483078003,
+      "loss": 1.0677,
+      "odds_ratio_loss": 0.6971891522407532,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.09980158507823944,
+      "rewards/margins": 0.015405992977321148,
+      "rewards/rejected": -0.11520756781101227,
+      "sft_loss": 0.9980157017707825,
+      "step": 200
+    },
+    {
+      "epoch": 0.37341631473660813,
+      "grad_norm": 1.7257640361785889,
+      "learning_rate": 4.811189096097025e-06,
+      "logits/chosen": -2.795761823654175,
+      "logits/rejected": -2.815319776535034,
+      "logps/chosen": -0.9524224996566772,
+      "logps/rejected": -1.2786835432052612,
+      "loss": 1.0165,
+      "odds_ratio_loss": 0.6408416032791138,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.09524224698543549,
+      "rewards/margins": 0.032626111060380936,
+      "rewards/rejected": -0.1278683841228485,
+      "sft_loss": 0.9524224996566772,
+      "step": 210
+    },
+    {
+      "epoch": 0.39119804400978,
+      "grad_norm": 3.106210231781006,
+      "learning_rate": 4.793028253763633e-06,
+      "logits/chosen": -2.8339433670043945,
+      "logits/rejected": -2.840125799179077,
+      "logps/chosen": -0.9467460513114929,
+      "logps/rejected": -1.1728994846343994,
+      "loss": 1.0181,
+      "odds_ratio_loss": 0.7131101489067078,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.09467460960149765,
+      "rewards/margins": 0.02261533960700035,
+      "rewards/rejected": -0.1172899454832077,
+      "sft_loss": 0.9467460513114929,
+      "step": 220
+    },
+    {
+      "epoch": 0.40897977328295176,
+      "grad_norm": 4.078444957733154,
+      "learning_rate": 4.774071191649352e-06,
+      "logits/chosen": -2.811408519744873,
+      "logits/rejected": -2.8089513778686523,
+      "logps/chosen": -0.9588619470596313,
+      "logps/rejected": -1.2887790203094482,
+      "loss": 1.019,
+      "odds_ratio_loss": 0.6018227934837341,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.09588618576526642,
+      "rewards/margins": 0.032991714775562286,
+      "rewards/rejected": -0.1288779079914093,
+      "sft_loss": 0.9588619470596313,
+      "step": 230
+    },
+    {
+      "epoch": 0.4267615025561236,
+      "grad_norm": 3.046936273574829,
+      "learning_rate": 4.7543244923105975e-06,
+      "logits/chosen": -2.8249118328094482,
+      "logits/rejected": -2.851069927215576,
+      "logps/chosen": -0.9376036524772644,
+      "logps/rejected": -1.0390188694000244,
+      "loss": 1.0087,
+      "odds_ratio_loss": 0.7111011147499084,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.09376037865877151,
+      "rewards/margins": 0.010141508653759956,
+      "rewards/rejected": -0.10390187799930573,
+      "sft_loss": 0.9376036524772644,
+      "step": 240
+    },
+    {
+      "epoch": 0.4445432318292954,
+      "grad_norm": 1.7991715669631958,
+      "learning_rate": 4.733795012493506e-06,
+      "logits/chosen": -2.8495593070983887,
+      "logits/rejected": -2.889249086380005,
+      "logps/chosen": -1.0128796100616455,
+      "logps/rejected": -1.202759861946106,
+      "loss": 1.0815,
+      "odds_ratio_loss": 0.6860855221748352,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.10128796100616455,
+      "rewards/margins": 0.018988018855452538,
+      "rewards/rejected": -0.12027599662542343,
+      "sft_loss": 1.0128796100616455,
+      "step": 250
+    },
+    {
+      "epoch": 0.46232496110246724,
+      "grad_norm": 1.7875176668167114,
+      "learning_rate": 4.712489880753035e-06,
+      "logits/chosen": -2.814908742904663,
+      "logits/rejected": -2.801401138305664,
+      "logps/chosen": -0.8547848463058472,
+      "logps/rejected": -1.1043689250946045,
+      "loss": 0.9165,
+      "odds_ratio_loss": 0.6170121431350708,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.08547849208116531,
+      "rewards/margins": 0.02495841309428215,
+      "rewards/rejected": -0.11043689399957657,
+      "sft_loss": 0.8547848463058472,
+      "step": 260
+    },
+    {
+      "epoch": 0.480106690375639,
+      "grad_norm": 1.5513663291931152,
+      "learning_rate": 4.690416494977673e-06,
+      "logits/chosen": -2.804611921310425,
+      "logits/rejected": -2.835052251815796,
+      "logps/chosen": -0.8691908717155457,
+      "logps/rejected": -1.2830617427825928,
+      "loss": 0.9283,
+      "odds_ratio_loss": 0.5908734202384949,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.08691909164190292,
+      "rewards/margins": 0.04138708487153053,
+      "rewards/rejected": -0.12830618023872375,
+      "sft_loss": 0.8691908717155457,
+      "step": 270
+    },
+    {
+      "epoch": 0.49788841964881086,
+      "grad_norm": 1.2906347513198853,
+      "learning_rate": 4.667582519820639e-06,
+      "logits/chosen": -2.8521058559417725,
+      "logits/rejected": -2.8685240745544434,
+      "logps/chosen": -0.9744144678115845,
+      "logps/rejected": -1.0906678438186646,
+      "loss": 1.0455,
+      "odds_ratio_loss": 0.7107566595077515,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.09744144976139069,
+      "rewards/margins": 0.011625329963862896,
+      "rewards/rejected": -0.10906676948070526,
+      "sft_loss": 0.9744144678115845,
+      "step": 280
+    },
+    {
+      "epoch": 0.5156701489219827,
+      "grad_norm": 3.3145501613616943,
+      "learning_rate": 4.643995884038443e-06,
+      "logits/chosen": -2.861464023590088,
+      "logits/rejected": -2.889545440673828,
+      "logps/chosen": -0.930392861366272,
+      "logps/rejected": -1.140458345413208,
+      "loss": 0.9957,
+      "odds_ratio_loss": 0.6526821255683899,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.09303927421569824,
+      "rewards/margins": 0.021006565541028976,
+      "rewards/rejected": -0.11404584348201752,
+      "sft_loss": 0.930392861366272,
+      "step": 290
+    },
+    {
+      "epoch": 0.5334518781951545,
+      "grad_norm": 1.5059245824813843,
+      "learning_rate": 4.6196647777377475e-06,
+      "logits/chosen": -2.825852870941162,
+      "logits/rejected": -2.8473992347717285,
+      "logps/chosen": -0.9365053176879883,
+      "logps/rejected": -1.1387995481491089,
+      "loss": 1.0047,
+      "odds_ratio_loss": 0.6818416714668274,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.09365053474903107,
+      "rewards/margins": 0.02022942714393139,
+      "rewards/rejected": -0.1138799637556076,
+      "sft_loss": 0.9365053176879883,
+      "step": 300
+    },
+    {
+      "epoch": 0.5512336074683263,
+      "grad_norm": 1.2869160175323486,
+      "learning_rate": 4.59459764953147e-06,
+      "logits/chosen": -2.8529794216156006,
+      "logits/rejected": -2.84968900680542,
+      "logps/chosen": -0.9238570928573608,
+      "logps/rejected": -1.1744383573532104,
+      "loss": 0.9884,
+      "odds_ratio_loss": 0.6458603739738464,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.09238572418689728,
+      "rewards/margins": 0.025058116763830185,
+      "rewards/rejected": -0.11744382232427597,
+      "sft_loss": 0.9238570928573608,
+      "step": 310
+    },
+    {
+      "epoch": 0.5690153367414981,
+      "grad_norm": 2.306725025177002,
+      "learning_rate": 4.568803203605133e-06,
+      "logits/chosen": -2.8887312412261963,
+      "logits/rejected": -2.8795981407165527,
+      "logps/chosen": -0.8577371835708618,
+      "logps/rejected": -1.1274560689926147,
+      "loss": 0.9223,
+      "odds_ratio_loss": 0.6451677680015564,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.08577371388673782,
+      "rewards/margins": 0.02697189524769783,
+      "rewards/rejected": -0.11274560540914536,
+      "sft_loss": 0.8577371835708618,
+      "step": 320
+    },
+    {
+      "epoch": 0.58679706601467,
+      "grad_norm": 1.8801417350769043,
+      "learning_rate": 4.542290396694462e-06,
+      "logits/chosen": -2.875821828842163,
+      "logits/rejected": -2.8778319358825684,
+      "logps/chosen": -0.9018943905830383,
+      "logps/rejected": -1.1100924015045166,
+      "loss": 0.9709,
+      "odds_ratio_loss": 0.6898372769355774,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.09018944203853607,
+      "rewards/margins": 0.020819801837205887,
+      "rewards/rejected": -0.11100924015045166,
+      "sft_loss": 0.9018943905830383,
+      "step": 330
+    },
+    {
+      "epoch": 0.6045787952878418,
+      "grad_norm": 11.704841613769531,
+      "learning_rate": 4.515068434975298e-06,
+      "logits/chosen": -2.8479015827178955,
+      "logits/rejected": -2.862330675125122,
+      "logps/chosen": -0.975223183631897,
+      "logps/rejected": -1.3013463020324707,
+      "loss": 1.0369,
+      "odds_ratio_loss": 0.6164206266403198,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0975223109126091,
+      "rewards/margins": 0.03261232003569603,
+      "rewards/rejected": -0.13013465702533722,
+      "sft_loss": 0.975223183631897,
+      "step": 340
+    },
+    {
+      "epoch": 0.6223605245610135,
+      "grad_norm": 1.487052321434021,
+      "learning_rate": 4.487146770866887e-06,
+      "logits/chosen": -2.872251510620117,
+      "logits/rejected": -2.896749973297119,
+      "logps/chosen": -0.9074058532714844,
+      "logps/rejected": -1.0781655311584473,
+      "loss": 0.9736,
+      "odds_ratio_loss": 0.6616368293762207,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.09074059128761292,
+      "rewards/margins": 0.01707596145570278,
+      "rewards/rejected": -0.10781653970479965,
+      "sft_loss": 0.9074058532714844,
+      "step": 350
+    },
+    {
+      "epoch": 0.6401422538341853,
+      "grad_norm": 2.0436906814575195,
+      "learning_rate": 4.458535099749666e-06,
+      "logits/chosen": -2.8917770385742188,
+      "logits/rejected": -2.9039359092712402,
+      "logps/chosen": -0.9911856651306152,
+      "logps/rejected": -1.1147364377975464,
+      "loss": 1.0655,
+      "odds_ratio_loss": 0.7427235245704651,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.09911856800317764,
+      "rewards/margins": 0.012355070561170578,
+      "rewards/rejected": -0.11147364228963852,
+      "sft_loss": 0.9911856651306152,
+      "step": 360
+    },
+    {
+      "epoch": 0.6579239831073572,
+      "grad_norm": 3.131418228149414,
+      "learning_rate": 4.429243356598694e-06,
+      "logits/chosen": -2.861708641052246,
+      "logits/rejected": -2.865683078765869,
+      "logps/chosen": -0.9219058156013489,
+      "logps/rejected": -1.2925573587417603,
+      "loss": 0.9872,
+      "odds_ratio_loss": 0.6528664231300354,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.09219057857990265,
+      "rewards/margins": 0.03706515580415726,
+      "rewards/rejected": -0.1292557418346405,
+      "sft_loss": 0.9219058156013489,
+      "step": 370
+    },
+    {
+      "epoch": 0.675705712380529,
+      "grad_norm": 2.098020076751709,
+      "learning_rate": 4.399281712533875e-06,
+      "logits/chosen": -2.8784518241882324,
+      "logits/rejected": -2.8920931816101074,
+      "logps/chosen": -0.857374370098114,
+      "logps/rejected": -1.0407134294509888,
+      "loss": 0.9257,
+      "odds_ratio_loss": 0.6832669377326965,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.0857374519109726,
+      "rewards/margins": 0.018333889544010162,
+      "rewards/rejected": -0.10407133400440216,
+      "sft_loss": 0.857374370098114,
+      "step": 380
+    },
+    {
+      "epoch": 0.6934874416537008,
+      "grad_norm": 2.2306644916534424,
+      "learning_rate": 4.368660571288192e-06,
+      "logits/chosen": -2.8683114051818848,
+      "logits/rejected": -2.8982186317443848,
+      "logps/chosen": -0.902807891368866,
+      "logps/rejected": -1.0363978147506714,
+      "loss": 0.9727,
+      "odds_ratio_loss": 0.6994079947471619,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.09028079360723495,
+      "rewards/margins": 0.013359000906348228,
+      "rewards/rejected": -0.10363979637622833,
+      "sft_loss": 0.902807891368866,
+      "step": 390
+    },
+    {
+      "epoch": 0.7112691709268726,
+      "grad_norm": 1.5744704008102417,
+      "learning_rate": 4.337390565595163e-06,
+      "logits/chosen": -2.857945680618286,
+      "logits/rejected": -2.8643996715545654,
+      "logps/chosen": -0.9818795919418335,
+      "logps/rejected": -1.0570474863052368,
+      "loss": 1.0562,
+      "odds_ratio_loss": 0.743161678314209,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.09818795323371887,
+      "rewards/margins": 0.007516802754253149,
+      "rewards/rejected": -0.10570474714040756,
+      "sft_loss": 0.9818795919418335,
+      "step": 400
+    },
+    {
+      "epoch": 0.7290509002000445,
+      "grad_norm": 2.6117560863494873,
+      "learning_rate": 4.305482553496786e-06,
+      "logits/chosen": -2.798802137374878,
+      "logits/rejected": -2.823525905609131,
+      "logps/chosen": -0.8416363596916199,
+      "logps/rejected": -1.0808361768722534,
+      "loss": 0.9042,
+      "odds_ratio_loss": 0.6256502270698547,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.08416363596916199,
+      "rewards/margins": 0.023919973522424698,
+      "rewards/rejected": -0.10808360576629639,
+      "sft_loss": 0.8416363596916199,
+      "step": 410
+    },
+    {
+      "epoch": 0.7468326294732163,
+      "grad_norm": 2.1309070587158203,
+      "learning_rate": 4.272947614573244e-06,
+      "logits/chosen": -2.8514039516448975,
+      "logits/rejected": -2.8706977367401123,
+      "logps/chosen": -0.935457706451416,
+      "logps/rejected": -1.069616436958313,
+      "loss": 1.0031,
+      "odds_ratio_loss": 0.6768142580986023,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.09354577958583832,
+      "rewards/margins": 0.013415870256721973,
+      "rewards/rejected": -0.10696164518594742,
+      "sft_loss": 0.935457706451416,
+      "step": 420
+    },
+    {
+      "epoch": 0.7646143587463881,
+      "grad_norm": 1.1241884231567383,
+      "learning_rate": 4.23979704609569e-06,
+      "logits/chosen": -2.858297824859619,
+      "logits/rejected": -2.8802642822265625,
+      "logps/chosen": -0.8784758448600769,
+      "logps/rejected": -1.0237103700637817,
+      "loss": 0.9426,
+      "odds_ratio_loss": 0.641250491142273,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.08784758299589157,
+      "rewards/margins": 0.01452344935387373,
+      "rewards/rejected": -0.10237103700637817,
+      "sft_loss": 0.8784758448600769,
+      "step": 430
+    },
+    {
+      "epoch": 0.78239608801956,
+      "grad_norm": 1.7693474292755127,
+      "learning_rate": 4.206042359103435e-06,
+      "logits/chosen": -2.8127472400665283,
+      "logits/rejected": -2.8374340534210205,
+      "logps/chosen": -0.933555006980896,
+      "logps/rejected": -1.194852352142334,
+      "loss": 0.9991,
+      "odds_ratio_loss": 0.6553630828857422,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.09335552155971527,
+      "rewards/margins": 0.02612972818315029,
+      "rewards/rejected": -0.11948523670434952,
+      "sft_loss": 0.933555006980896,
+      "step": 440
+    },
+    {
+      "epoch": 0.8001778172927317,
+      "grad_norm": 1.5750656127929688,
+      "learning_rate": 4.17169527440691e-06,
+      "logits/chosen": -2.851017475128174,
+      "logits/rejected": -2.85324764251709,
+      "logps/chosen": -0.8872865438461304,
+      "logps/rejected": -1.0435672998428345,
+      "loss": 0.9571,
+      "odds_ratio_loss": 0.6983404755592346,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.0887286514043808,
+      "rewards/margins": 0.015628065913915634,
+      "rewards/rejected": -0.10435672104358673,
+      "sft_loss": 0.8872865438461304,
+      "step": 450
+    },
+    {
+      "epoch": 0.8179595465659035,
+      "grad_norm": 2.6487419605255127,
+      "learning_rate": 4.136767718517797e-06,
+      "logits/chosen": -2.8638827800750732,
+      "logits/rejected": -2.885394811630249,
+      "logps/chosen": -0.8147961497306824,
+      "logps/rejected": -1.0451968908309937,
+      "loss": 0.8763,
+      "odds_ratio_loss": 0.6151142120361328,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.08147960901260376,
+      "rewards/margins": 0.023040082305669785,
+      "rewards/rejected": -0.10451970249414444,
+      "sft_loss": 0.8147961497306824,
+      "step": 460
+    },
+    {
+      "epoch": 0.8357412758390753,
+      "grad_norm": 8.826119422912598,
+      "learning_rate": 4.1012718195077196e-06,
+      "logits/chosen": -2.907290458679199,
+      "logits/rejected": -2.9450554847717285,
+      "logps/chosen": -0.8985522985458374,
+      "logps/rejected": -1.0472660064697266,
+      "loss": 0.9656,
+      "odds_ratio_loss": 0.6701352000236511,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.08985523879528046,
+      "rewards/margins": 0.014871363528072834,
+      "rewards/rejected": -0.10472659766674042,
+      "sft_loss": 0.8985522985458374,
+      "step": 470
+    },
+    {
+      "epoch": 0.8535230051122472,
+      "grad_norm": 1.3586628437042236,
+      "learning_rate": 4.065219902796953e-06,
+      "logits/chosen": -2.864061117172241,
+      "logits/rejected": -2.8628530502319336,
+      "logps/chosen": -0.8511291742324829,
+      "logps/rejected": -1.0970637798309326,
+      "loss": 0.9158,
+      "odds_ratio_loss": 0.646445631980896,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.08511291444301605,
+      "rewards/margins": 0.024593474343419075,
+      "rewards/rejected": -0.10970638692378998,
+      "sft_loss": 0.8511291742324829,
+      "step": 480
+    },
+    {
+      "epoch": 0.871304734385419,
+      "grad_norm": 1.2425549030303955,
+      "learning_rate": 4.028624486874608e-06,
+      "logits/chosen": -2.8780291080474854,
+      "logits/rejected": -2.901134729385376,
+      "logps/chosen": -0.851506233215332,
+      "logps/rejected": -1.129828691482544,
+      "loss": 0.9181,
+      "odds_ratio_loss": 0.6657174825668335,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.08515061438083649,
+      "rewards/margins": 0.02783224917948246,
+      "rewards/rejected": -0.1129828691482544,
+      "sft_loss": 0.851506233215332,
+      "step": 490
+    },
+    {
+      "epoch": 0.8890864636585908,
+      "grad_norm": 1.8243030309677124,
+      "learning_rate": 3.99149827895177e-06,
+      "logits/chosen": -2.9000747203826904,
+      "logits/rejected": -2.9058332443237305,
+      "logps/chosen": -0.9306007623672485,
+      "logps/rejected": -1.053107500076294,
+      "loss": 1.0001,
+      "odds_ratio_loss": 0.6953399777412415,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.09306007623672485,
+      "rewards/margins": 0.012250673025846481,
+      "rewards/rejected": -0.10531075298786163,
+      "sft_loss": 0.9306007623672485,
+      "step": 500
+    },
+    {
+      "epoch": 0.8890864636585908,
+      "eval_logits/chosen": -2.8713040351867676,
+      "eval_logits/rejected": -2.8936164379119873,
+      "eval_logps/chosen": -0.8689887523651123,
+      "eval_logps/rejected": -1.1122756004333496,
+      "eval_loss": 0.9318326711654663,
+      "eval_odds_ratio_loss": 0.6284381151199341,
+      "eval_rewards/accuracies": 0.5920000076293945,
+      "eval_rewards/chosen": -0.08689887821674347,
+      "eval_rewards/margins": 0.024328680709004402,
+      "eval_rewards/rejected": -0.11122756451368332,
+      "eval_runtime": 189.293,
+      "eval_samples_per_second": 5.283,
+      "eval_sft_loss": 0.8689887523651123,
+      "eval_steps_per_second": 2.641,
+      "step": 500
+    },
+    {
+      "epoch": 0.9068681929317626,
+      "grad_norm": 1.853105902671814,
+      "learning_rate": 3.953854170549114e-06,
+      "logits/chosen": -2.8680295944213867,
+      "logits/rejected": -2.8688206672668457,
+      "logps/chosen": -0.9001534581184387,
+      "logps/rejected": -1.0128238201141357,
+      "loss": 0.9674,
+      "odds_ratio_loss": 0.6725192070007324,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.09001535177230835,
+      "rewards/margins": 0.011267038993537426,
+      "rewards/rejected": -0.10128238052129745,
+      "sft_loss": 0.9001534581184387,
+      "step": 510
+    },
+    {
+      "epoch": 0.9246499222049345,
+      "grad_norm": 1.9707605838775635,
+      "learning_rate": 3.91570523302051e-06,
+      "logits/chosen": -2.8937981128692627,
+      "logits/rejected": -2.9022488594055176,
+      "logps/chosen": -0.8274223208427429,
+      "logps/rejected": -1.105006217956543,
+      "loss": 0.8928,
+      "odds_ratio_loss": 0.6533216834068298,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.08274223655462265,
+      "rewards/margins": 0.027758393436670303,
+      "rewards/rejected": -0.11050061881542206,
+      "sft_loss": 0.8274223208427429,
+      "step": 520
+    },
+    {
+      "epoch": 0.9424316514781063,
+      "grad_norm": 1.4241600036621094,
+      "learning_rate": 3.8770647130141996e-06,
+      "logits/chosen": -2.892141342163086,
+      "logits/rejected": -2.8914198875427246,
+      "logps/chosen": -0.875231921672821,
+      "logps/rejected": -1.0876398086547852,
+      "loss": 0.9415,
+      "odds_ratio_loss": 0.662711501121521,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.08752318471670151,
+      "rewards/margins": 0.02124079503118992,
+      "rewards/rejected": -0.10876397788524628,
+      "sft_loss": 0.875231921672821,
+      "step": 530
+    },
+    {
+      "epoch": 0.960213380751278,
+      "grad_norm": 2.250497341156006,
+      "learning_rate": 3.837946027873086e-06,
+      "logits/chosen": -2.8456499576568604,
+      "logits/rejected": -2.8566787242889404,
+      "logps/chosen": -0.9233220815658569,
+      "logps/rejected": -1.1583213806152344,
+      "loss": 0.9895,
+      "odds_ratio_loss": 0.6621237993240356,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.09233220666646957,
+      "rewards/margins": 0.02349993959069252,
+      "rewards/rejected": -0.11583214998245239,
+      "sft_loss": 0.9233220815658569,
+      "step": 540
+    },
+    {
+      "epoch": 0.9779951100244498,
+      "grad_norm": 5.65519905090332,
+      "learning_rate": 3.7983627609757713e-06,
+      "logits/chosen": -2.8913745880126953,
+      "logits/rejected": -2.901867389678955,
+      "logps/chosen": -0.9292505383491516,
+      "logps/rejected": -1.059091329574585,
+      "loss": 0.996,
+      "odds_ratio_loss": 0.6679803133010864,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.0929250493645668,
+      "rewards/margins": 0.012984082102775574,
+      "rewards/rejected": -0.10590913146734238,
+      "sft_loss": 0.9292505383491516,
+      "step": 550
+    },
+    {
+      "epoch": 0.9957768392976217,
+      "grad_norm": 1.4165409803390503,
+      "learning_rate": 3.758328657019924e-06,
+      "logits/chosen": -2.8648407459259033,
+      "logits/rejected": -2.8767480850219727,
+      "logps/chosen": -0.8623329997062683,
+      "logps/rejected": -1.1223938465118408,
+      "loss": 0.9301,
+      "odds_ratio_loss": 0.6778011322021484,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.08623330295085907,
+      "rewards/margins": 0.026006082072854042,
+      "rewards/rejected": -0.11223938316106796,
+      "sft_loss": 0.8623329997062683,
+      "step": 560
+    },
+    {
+      "epoch": 1.0135585685707935,
+      "grad_norm": 6.480414867401123,
+      "learning_rate": 3.717857617249642e-06,
+      "logits/chosen": -2.8424503803253174,
+      "logits/rejected": -2.8727316856384277,
+      "logps/chosen": -0.9302489161491394,
+      "logps/rejected": -1.159745693206787,
+      "loss": 0.9995,
+      "odds_ratio_loss": 0.6924289464950562,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.09302489459514618,
+      "rewards/margins": 0.02294967696070671,
+      "rewards/rejected": -0.11597456783056259,
+      "sft_loss": 0.9302489161491394,
+      "step": 570
+    },
+    {
+      "epoch": 1.0313402978439654,
+      "grad_norm": 1.2129720449447632,
+      "learning_rate": 3.6769636946284543e-06,
+      "logits/chosen": -2.8574068546295166,
+      "logits/rejected": -2.8658154010772705,
+      "logps/chosen": -0.8263272047042847,
+      "logps/rejected": -1.041471242904663,
+      "loss": 0.8894,
+      "odds_ratio_loss": 0.6303838491439819,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.08263272792100906,
+      "rewards/margins": 0.02151440642774105,
+      "rewards/rejected": -0.10414713621139526,
+      "sft_loss": 0.8263272047042847,
+      "step": 580
+    },
+    {
+      "epoch": 1.049122027117137,
+      "grad_norm": 1.3920116424560547,
+      "learning_rate": 3.6356610889596355e-06,
+      "logits/chosen": -2.8504767417907715,
+      "logits/rejected": -2.8750224113464355,
+      "logps/chosen": -0.891506016254425,
+      "logps/rejected": -1.043367624282837,
+      "loss": 0.9578,
+      "odds_ratio_loss": 0.6629586815834045,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.08915059268474579,
+      "rewards/margins": 0.01518617570400238,
+      "rewards/rejected": -0.10433676093816757,
+      "sft_loss": 0.891506016254425,
+      "step": 590
+    },
+    {
+      "epoch": 1.066903756390309,
+      "grad_norm": 1.6321930885314941,
+      "learning_rate": 3.593964141955541e-06,
+      "logits/chosen": -2.851910352706909,
+      "logits/rejected": -2.856790065765381,
+      "logps/chosen": -0.849289059638977,
+      "logps/rejected": -0.9652984738349915,
+      "loss": 0.9171,
+      "odds_ratio_loss": 0.6785116195678711,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.08492890745401382,
+      "rewards/margins": 0.011600947007536888,
+      "rewards/rejected": -0.09652985632419586,
+      "sft_loss": 0.849289059638977,
+      "step": 600
+    },
+    {
+      "epoch": 1.0846854856634809,
+      "grad_norm": 1.705749273300171,
+      "learning_rate": 3.5518873322576573e-06,
+      "logits/chosen": -2.8073315620422363,
+      "logits/rejected": -2.8466951847076416,
+      "logps/chosen": -0.8435554504394531,
+      "logps/rejected": -1.0204975605010986,
+      "loss": 0.9069,
+      "odds_ratio_loss": 0.6330701112747192,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.08435554802417755,
+      "rewards/margins": 0.017694219946861267,
+      "rewards/rejected": -0.10204975306987762,
+      "sft_loss": 0.8435554504394531,
+      "step": 610
+    },
+    {
+      "epoch": 1.1024672149366526,
+      "grad_norm": 1.2050608396530151,
+      "learning_rate": 3.5094452704091143e-06,
+      "logits/chosen": -2.8757572174072266,
+      "logits/rejected": -2.8627264499664307,
+      "logps/chosen": -0.8311964273452759,
+      "logps/rejected": -1.010647177696228,
+      "loss": 0.8957,
+      "odds_ratio_loss": 0.6448286175727844,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.08311963826417923,
+      "rewards/margins": 0.01794508285820484,
+      "rewards/rejected": -0.10106471925973892,
+      "sft_loss": 0.8311964273452759,
+      "step": 620
+    },
+    {
+      "epoch": 1.1202489442098245,
+      "grad_norm": 2.8746261596679688,
+      "learning_rate": 3.46665269378139e-06,
+      "logits/chosen": -2.834296941757202,
+      "logits/rejected": -2.8471195697784424,
+      "logps/chosen": -0.8721610903739929,
+      "logps/rejected": -1.0536631345748901,
+      "loss": 0.9388,
+      "odds_ratio_loss": 0.6662768125534058,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.08721610903739929,
+      "rewards/margins": 0.018150202929973602,
+      "rewards/rejected": -0.1053663119673729,
+      "sft_loss": 0.8721610903739929,
+      "step": 630
+    },
+    {
+      "epoch": 1.1380306734829961,
+      "grad_norm": 2.6889796257019043,
+      "learning_rate": 3.4235244614569794e-06,
+      "logits/chosen": -2.8430144786834717,
+      "logits/rejected": -2.841323137283325,
+      "logps/chosen": -0.9479130506515503,
+      "logps/rejected": -1.0536433458328247,
+      "loss": 1.0213,
+      "odds_ratio_loss": 0.7333841919898987,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.09479130804538727,
+      "rewards/margins": 0.010573023930191994,
+      "rewards/rejected": -0.10536432266235352,
+      "sft_loss": 0.9479130506515503,
+      "step": 640
+    },
+    {
+      "epoch": 1.155812402756168,
+      "grad_norm": 1.1239742040634155,
+      "learning_rate": 3.3800755490698008e-06,
+      "logits/chosen": -2.867356777191162,
+      "logits/rejected": -2.8678269386291504,
+      "logps/chosen": -0.8813873529434204,
+      "logps/rejected": -1.146475076675415,
+      "loss": 0.9416,
+      "odds_ratio_loss": 0.6019629240036011,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.08813872933387756,
+      "rewards/margins": 0.02650878205895424,
+      "rewards/rejected": -0.1146475300192833,
+      "sft_loss": 0.8813873529434204,
+      "step": 650
+    },
+    {
+      "epoch": 1.17359413202934,
+      "grad_norm": 1.5449692010879517,
+      "learning_rate": 3.3363210436051287e-06,
+      "logits/chosen": -2.8728928565979004,
+      "logits/rejected": -2.8769795894622803,
+      "logps/chosen": -0.9058928489685059,
+      "logps/rejected": -1.1068971157073975,
+      "loss": 0.975,
+      "odds_ratio_loss": 0.6907466650009155,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.09058928489685059,
+      "rewards/margins": 0.0201004259288311,
+      "rewards/rejected": -0.11068971455097198,
+      "sft_loss": 0.9058928489685059,
+      "step": 660
+    },
+    {
+      "epoch": 1.1913758613025116,
+      "grad_norm": 1.958606243133545,
+      "learning_rate": 3.292276138160867e-06,
+      "logits/chosen": -2.8424530029296875,
+      "logits/rejected": -2.854606866836548,
+      "logps/chosen": -0.8394180536270142,
+      "logps/rejected": -1.0364609956741333,
+      "loss": 0.9055,
+      "odds_ratio_loss": 0.6611912250518799,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.08394180238246918,
+      "rewards/margins": 0.019704295322299004,
+      "rewards/rejected": -0.10364609956741333,
+      "sft_loss": 0.8394180536270142,
+      "step": 670
+    },
+    {
+      "epoch": 1.2091575905756835,
+      "grad_norm": 3.3439667224884033,
+      "learning_rate": 3.2479561266719694e-06,
+      "logits/chosen": -2.8539648056030273,
+      "logits/rejected": -2.8596293926239014,
+      "logps/chosen": -0.89495450258255,
+      "logps/rejected": -1.0723049640655518,
+      "loss": 0.9604,
+      "odds_ratio_loss": 0.6547910571098328,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.0894954577088356,
+      "rewards/margins": 0.017735039815306664,
+      "rewards/rejected": -0.10723049938678741,
+      "sft_loss": 0.89495450258255,
+      "step": 680
+    },
+    {
+      "epoch": 1.2269393198488552,
+      "grad_norm": 3.391028881072998,
+      "learning_rate": 3.2033763985998533e-06,
+      "logits/chosen": -2.8784637451171875,
+      "logits/rejected": -2.878801107406616,
+      "logps/chosen": -0.8193685412406921,
+      "logps/rejected": -1.303346037864685,
+      "loss": 0.8782,
+      "odds_ratio_loss": 0.5878696441650391,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.08193685114383698,
+      "rewards/margins": 0.048397745937108994,
+      "rewards/rejected": -0.13033458590507507,
+      "sft_loss": 0.8193685412406921,
+      "step": 690
+    },
+    {
+      "epoch": 1.244721049122027,
+      "grad_norm": 1.8017568588256836,
+      "learning_rate": 3.1585524335886335e-06,
+      "logits/chosen": -2.8997411727905273,
+      "logits/rejected": -2.891542911529541,
+      "logps/chosen": -0.8149515986442566,
+      "logps/rejected": -1.062873125076294,
+      "loss": 0.8781,
+      "odds_ratio_loss": 0.6314008831977844,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.08149515837430954,
+      "rewards/margins": 0.024792145937681198,
+      "rewards/rejected": -0.10628731548786163,
+      "sft_loss": 0.8149515986442566,
+      "step": 700
+    },
+    {
+      "epoch": 1.262502778395199,
+      "grad_norm": 2.5622735023498535,
+      "learning_rate": 3.1134997960900536e-06,
+      "logits/chosen": -2.832627296447754,
+      "logits/rejected": -2.8471617698669434,
+      "logps/chosen": -0.8054940104484558,
+      "logps/rejected": -1.1920253038406372,
+      "loss": 0.864,
+      "odds_ratio_loss": 0.58550626039505,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.08054940402507782,
+      "rewards/margins": 0.03865312412381172,
+      "rewards/rejected": -0.11920253187417984,
+      "sft_loss": 0.8054940104484558,
+      "step": 710
+    },
+    {
+      "epoch": 1.2802845076683709,
+      "grad_norm": 1.4196934700012207,
+      "learning_rate": 3.0682341299589583e-06,
+      "logits/chosen": -2.857084274291992,
+      "logits/rejected": -2.8681209087371826,
+      "logps/chosen": -0.8212624788284302,
+      "logps/rejected": -0.9836801290512085,
+      "loss": 0.887,
+      "odds_ratio_loss": 0.6570355296134949,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.08212624490261078,
+      "rewards/margins": 0.016241777688264847,
+      "rewards/rejected": -0.09836802631616592,
+      "sft_loss": 0.8212624788284302,
+      "step": 720
+    },
+    {
+      "epoch": 1.2980662369415426,
+      "grad_norm": 1.5716747045516968,
+      "learning_rate": 3.022771153021201e-06,
+      "logits/chosen": -2.860788583755493,
+      "logits/rejected": -2.8880984783172607,
+      "logps/chosen": -0.8381561040878296,
+      "logps/rejected": -1.052847981452942,
+      "loss": 0.9036,
+      "odds_ratio_loss": 0.6544219255447388,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.08381561934947968,
+      "rewards/margins": 0.021469179540872574,
+      "rewards/rejected": -0.10528478771448135,
+      "sft_loss": 0.8381561040878296,
+      "step": 730
+    },
+    {
+      "epoch": 1.3158479662147144,
+      "grad_norm": 1.4270474910736084,
+      "learning_rate": 2.9771266516158625e-06,
+      "logits/chosen": -2.8542871475219727,
+      "logits/rejected": -2.877516508102417,
+      "logps/chosen": -0.8428620100021362,
+      "logps/rejected": -1.029301404953003,
+      "loss": 0.9101,
+      "odds_ratio_loss": 0.6720696687698364,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.08428619801998138,
+      "rewards/margins": 0.0186439398676157,
+      "rewards/rejected": -0.10293014347553253,
+      "sft_loss": 0.8428620100021362,
+      "step": 740
+    },
+    {
+      "epoch": 1.3336296954878861,
+      "grad_norm": 1.401987910270691,
+      "learning_rate": 2.9313164751136802e-06,
+      "logits/chosen": -2.840547561645508,
+      "logits/rejected": -2.872194766998291,
+      "logps/chosen": -0.8261799812316895,
+      "logps/rejected": -1.0598170757293701,
+      "loss": 0.8865,
+      "odds_ratio_loss": 0.6030489802360535,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08261799812316895,
+      "rewards/margins": 0.023363707587122917,
+      "rewards/rejected": -0.1059817224740982,
+      "sft_loss": 0.8261799812316895,
+      "step": 750
+    },
+    {
+      "epoch": 1.351411424761058,
+      "grad_norm": 2.474881410598755,
+      "learning_rate": 2.8853565304135956e-06,
+      "logits/chosen": -2.8949756622314453,
+      "logits/rejected": -2.898646593093872,
+      "logps/chosen": -0.918350338935852,
+      "logps/rejected": -1.0580472946166992,
+      "loss": 0.9899,
+      "odds_ratio_loss": 0.7158701419830322,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.09183504432439804,
+      "rewards/margins": 0.013969694264233112,
+      "rewards/rejected": -0.10580474138259888,
+      "sft_loss": 0.918350338935852,
+      "step": 760
+    },
+    {
+      "epoch": 1.36919315403423,
+      "grad_norm": 2.682598829269409,
+      "learning_rate": 2.839262776419313e-06,
+      "logits/chosen": -2.8501102924346924,
+      "logits/rejected": -2.85284686088562,
+      "logps/chosen": -0.8370243906974792,
+      "logps/rejected": -1.3541486263275146,
+      "loss": 0.8958,
+      "odds_ratio_loss": 0.5878289937973022,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.0837024375796318,
+      "rewards/margins": 0.05171241611242294,
+      "rewards/rejected": -0.13541486859321594,
+      "sft_loss": 0.8370243906974792,
+      "step": 770
+    },
+    {
+      "epoch": 1.3869748833074016,
+      "grad_norm": 2.036529064178467,
+      "learning_rate": 2.793051218497817e-06,
+      "logits/chosen": -2.871901035308838,
+      "logits/rejected": -2.880750894546509,
+      "logps/chosen": -0.8279402852058411,
+      "logps/rejected": -0.9436777830123901,
+      "loss": 0.8955,
+      "odds_ratio_loss": 0.6755216717720032,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.08279402554035187,
+      "rewards/margins": 0.011573752388358116,
+      "rewards/rejected": -0.09436777979135513,
+      "sft_loss": 0.8279402852058411,
+      "step": 780
+    },
+    {
+      "epoch": 1.4047566125805735,
+      "grad_norm": 1.4822628498077393,
+      "learning_rate": 2.7467379029217437e-06,
+      "logits/chosen": -2.848276138305664,
+      "logits/rejected": -2.8498966693878174,
+      "logps/chosen": -0.8329153060913086,
+      "logps/rejected": -1.1663061380386353,
+      "loss": 0.8957,
+      "odds_ratio_loss": 0.6282148957252502,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.08329152315855026,
+      "rewards/margins": 0.033339083194732666,
+      "rewards/rejected": -0.11663061380386353,
+      "sft_loss": 0.8329153060913086,
+      "step": 790
+    },
+    {
+      "epoch": 1.4225383418537452,
+      "grad_norm": 1.2180672883987427,
+      "learning_rate": 2.7003389112975546e-06,
+      "logits/chosen": -2.8609681129455566,
+      "logits/rejected": -2.8844683170318604,
+      "logps/chosen": -0.8761266469955444,
+      "logps/rejected": -1.083616018295288,
+      "loss": 0.9409,
+      "odds_ratio_loss": 0.6474456787109375,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.08761266618967056,
+      "rewards/margins": 0.020748943090438843,
+      "rewards/rejected": -0.10836161673069,
+      "sft_loss": 0.8761266469955444,
+      "step": 800
+    },
+    {
+      "epoch": 1.440320071126917,
+      "grad_norm": 3.231017589569092,
+      "learning_rate": 2.653870354981437e-06,
+      "logits/chosen": -2.858649969100952,
+      "logits/rejected": -2.870346784591675,
+      "logps/chosen": -0.7923532128334045,
+      "logps/rejected": -1.0410521030426025,
+      "loss": 0.8559,
+      "odds_ratio_loss": 0.6351376175880432,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.07923531532287598,
+      "rewards/margins": 0.024869883432984352,
+      "rewards/rejected": -0.10410521179437637,
+      "sft_loss": 0.7923532128334045,
+      "step": 810
+    },
+    {
+      "epoch": 1.458101800400089,
+      "grad_norm": 4.5185065269470215,
+      "learning_rate": 2.6073483694848777e-06,
+      "logits/chosen": -2.827204465866089,
+      "logits/rejected": -2.8571865558624268,
+      "logps/chosen": -0.8361181020736694,
+      "logps/rejected": -1.0542662143707275,
+      "loss": 0.9018,
+      "odds_ratio_loss": 0.6572374701499939,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.08361181616783142,
+      "rewards/margins": 0.02181481197476387,
+      "rewards/rejected": -0.10542663186788559,
+      "sft_loss": 0.8361181020736694,
+      "step": 820
+    },
+    {
+      "epoch": 1.4758835296732609,
+      "grad_norm": 1.3891066312789917,
+      "learning_rate": 2.560789108871847e-06,
+      "logits/chosen": -2.8242173194885254,
+      "logits/rejected": -2.8416523933410645,
+      "logps/chosen": -0.8545220494270325,
+      "logps/rejected": -1.1704602241516113,
+      "loss": 0.9191,
+      "odds_ratio_loss": 0.6462253928184509,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.08545219898223877,
+      "rewards/margins": 0.03159382566809654,
+      "rewards/rejected": -0.11704603582620621,
+      "sft_loss": 0.8545220494270325,
+      "step": 830
+    },
+    {
+      "epoch": 1.4936652589464325,
+      "grad_norm": 3.542637348175049,
+      "learning_rate": 2.514208740149544e-06,
+      "logits/chosen": -2.8630993366241455,
+      "logits/rejected": -2.892916440963745,
+      "logps/chosen": -0.8821213841438293,
+      "logps/rejected": -1.1250375509262085,
+      "loss": 0.9496,
+      "odds_ratio_loss": 0.6751636862754822,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.08821214735507965,
+      "rewards/margins": 0.024291623383760452,
+      "rewards/rejected": -0.11250375211238861,
+      "sft_loss": 0.8821213841438293,
+      "step": 840
+    },
+    {
+      "epoch": 1.5114469882196042,
+      "grad_norm": 1.9266990423202515,
+      "learning_rate": 2.46762343765464e-06,
+      "logits/chosen": -2.8725075721740723,
+      "logits/rejected": -2.877906084060669,
+      "logps/chosen": -0.8815126419067383,
+      "logps/rejected": -1.12534499168396,
+      "loss": 0.9432,
+      "odds_ratio_loss": 0.6172918081283569,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.08815126121044159,
+      "rewards/margins": 0.02438322827219963,
+      "rewards/rejected": -0.11253450065851212,
+      "sft_loss": 0.8815126419067383,
+      "step": 850
+    },
+    {
+      "epoch": 1.5292287174927761,
+      "grad_norm": 1.9089457988739014,
+      "learning_rate": 2.4210493774369903e-06,
+      "logits/chosen": -2.843306064605713,
+      "logits/rejected": -2.844432830810547,
+      "logps/chosen": -0.8710628747940063,
+      "logps/rejected": -1.0414961576461792,
+      "loss": 0.9387,
+      "odds_ratio_loss": 0.6761898398399353,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.08710628747940063,
+      "rewards/margins": 0.01704334281384945,
+      "rewards/rejected": -0.10414961725473404,
+      "sft_loss": 0.8710628747940063,
+      "step": 860
+    },
+    {
+      "epoch": 1.547010446765948,
+      "grad_norm": 1.2754076719284058,
+      "learning_rate": 2.374502731642732e-06,
+      "logits/chosen": -2.855776309967041,
+      "logits/rejected": -2.8679866790771484,
+      "logps/chosen": -0.926120400428772,
+      "logps/rejected": -1.1001455783843994,
+      "loss": 0.9908,
+      "odds_ratio_loss": 0.6463494896888733,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.09261203557252884,
+      "rewards/margins": 0.01740253157913685,
+      "rewards/rejected": -0.11001457273960114,
+      "sft_loss": 0.926120400428772,
+      "step": 870
+    },
+    {
+      "epoch": 1.56479217603912,
+      "grad_norm": 1.6846286058425903,
+      "learning_rate": 2.3279996628987556e-06,
+      "logits/chosen": -2.844256639480591,
+      "logits/rejected": -2.8760454654693604,
+      "logps/chosen": -0.8887776136398315,
+      "logps/rejected": -1.0579562187194824,
+      "loss": 0.9585,
+      "odds_ratio_loss": 0.6975380778312683,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.08887775987386703,
+      "rewards/margins": 0.01691785827279091,
+      "rewards/rejected": -0.10579562187194824,
+      "sft_loss": 0.8887776136398315,
+      "step": 880
+    },
+    {
+      "epoch": 1.5825739053122916,
+      "grad_norm": 1.5345253944396973,
+      "learning_rate": 2.281556318700474e-06,
+      "logits/chosen": -2.851228952407837,
+      "logits/rejected": -2.8857483863830566,
+      "logps/chosen": -0.82933509349823,
+      "logps/rejected": -0.9570034742355347,
+      "loss": 0.8991,
+      "odds_ratio_loss": 0.697514533996582,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.08293350785970688,
+      "rewards/margins": 0.012766830623149872,
+      "rewards/rejected": -0.09570035338401794,
+      "sft_loss": 0.82933509349823,
+      "step": 890
+    },
+    {
+      "epoch": 1.6003556345854635,
+      "grad_norm": 3.0883193016052246,
+      "learning_rate": 2.2351888258048408e-06,
+      "logits/chosen": -2.7954063415527344,
+      "logits/rejected": -2.8361153602600098,
+      "logps/chosen": -0.8643034100532532,
+      "logps/rejected": -1.0512335300445557,
+      "loss": 0.928,
+      "odds_ratio_loss": 0.6372779011726379,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.08643033355474472,
+      "rewards/margins": 0.018693022429943085,
+      "rewards/rejected": -0.1051233559846878,
+      "sft_loss": 0.8643034100532532,
+      "step": 900
+    },
+    {
+      "epoch": 1.6181373638586352,
+      "grad_norm": 2.0690836906433105,
+      "learning_rate": 2.188913284630584e-06,
+      "logits/chosen": -2.8729634284973145,
+      "logits/rejected": -2.8929648399353027,
+      "logps/chosen": -0.9121950268745422,
+      "logps/rejected": -1.029121994972229,
+      "loss": 0.9828,
+      "odds_ratio_loss": 0.7060950398445129,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.09121949970722198,
+      "rewards/margins": 0.011692697182297707,
+      "rewards/rejected": -0.10291220247745514,
+      "sft_loss": 0.9121950268745422,
+      "step": 910
+    },
+    {
+      "epoch": 1.635919093131807,
+      "grad_norm": 3.8324978351593018,
+      "learning_rate": 2.1427457636675652e-06,
+      "logits/chosen": -2.862370014190674,
+      "logits/rejected": -2.88765025138855,
+      "logps/chosen": -0.8752270936965942,
+      "logps/rejected": -1.0570242404937744,
+      "loss": 0.944,
+      "odds_ratio_loss": 0.6881860494613647,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.0875227078795433,
+      "rewards/margins": 0.018179720267653465,
+      "rewards/rejected": -0.10570243746042252,
+      "sft_loss": 0.8752270936965942,
+      "step": 920
+    },
+    {
+      "epoch": 1.653700822404979,
+      "grad_norm": 1.9543755054473877,
+      "learning_rate": 2.096702293897247e-06,
+      "logits/chosen": -2.8620665073394775,
+      "logits/rejected": -2.876176357269287,
+      "logps/chosen": -0.8246392011642456,
+      "logps/rejected": -1.1381771564483643,
+      "loss": 0.8887,
+      "odds_ratio_loss": 0.6403511166572571,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.08246392011642456,
+      "rewards/margins": 0.03135378286242485,
+      "rewards/rejected": -0.1138177141547203,
+      "sft_loss": 0.8246392011642456,
+      "step": 930
+    },
+    {
+      "epoch": 1.6714825516781509,
+      "grad_norm": 2.1185944080352783,
+      "learning_rate": 2.0507988632261672e-06,
+      "logits/chosen": -2.834115505218506,
+      "logits/rejected": -2.8819518089294434,
+      "logps/chosen": -0.8675654530525208,
+      "logps/rejected": -1.0815333127975464,
+      "loss": 0.9294,
+      "odds_ratio_loss": 0.6182416677474976,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08675654232501984,
+      "rewards/margins": 0.02139679528772831,
+      "rewards/rejected": -0.1081533432006836,
+      "sft_loss": 0.8675654530525208,
+      "step": 940
+    },
+    {
+      "epoch": 1.6892642809513225,
+      "grad_norm": 3.2840750217437744,
+      "learning_rate": 2.005051410934382e-06,
+      "logits/chosen": -2.8422374725341797,
+      "logits/rejected": -2.8813586235046387,
+      "logps/chosen": -0.9204028248786926,
+      "logps/rejected": -1.0668128728866577,
+      "loss": 0.9896,
+      "odds_ratio_loss": 0.69184410572052,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.0920402854681015,
+      "rewards/margins": 0.01464100182056427,
+      "rewards/rejected": -0.10668128728866577,
+      "sft_loss": 0.9204028248786926,
+      "step": 950
+    },
+    {
+      "epoch": 1.7070460102244942,
+      "grad_norm": 1.923535704612732,
+      "learning_rate": 1.9594758221407843e-06,
+      "logits/chosen": -2.8552603721618652,
+      "logits/rejected": -2.8695271015167236,
+      "logps/chosen": -0.8000820279121399,
+      "logps/rejected": -1.0352107286453247,
+      "loss": 0.8598,
+      "odds_ratio_loss": 0.5971243977546692,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.08000819385051727,
+      "rewards/margins": 0.02351287379860878,
+      "rewards/rejected": -0.10352107137441635,
+      "sft_loss": 0.8000820279121399,
+      "step": 960
+    },
+    {
+      "epoch": 1.724827739497666,
+      "grad_norm": 3.057365894317627,
+      "learning_rate": 1.9140879222872408e-06,
+      "logits/chosen": -2.873203992843628,
+      "logits/rejected": -2.894540309906006,
+      "logps/chosen": -0.813400149345398,
+      "logps/rejected": -0.9625688791275024,
+      "loss": 0.8825,
+      "odds_ratio_loss": 0.6909626722335815,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.08134002983570099,
+      "rewards/margins": 0.014916857704520226,
+      "rewards/rejected": -0.09625686705112457,
+      "sft_loss": 0.813400149345398,
+      "step": 970
+    },
+    {
+      "epoch": 1.742609468770838,
+      "grad_norm": 1.2360864877700806,
+      "learning_rate": 1.8689034716434346e-06,
+      "logits/chosen": -2.8773324489593506,
+      "logits/rejected": -2.8978989124298096,
+      "logps/chosen": -0.8950971364974976,
+      "logps/rejected": -1.0365970134735107,
+      "loss": 0.9651,
+      "odds_ratio_loss": 0.7001620531082153,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.08950972557067871,
+      "rewards/margins": 0.014149991795420647,
+      "rewards/rejected": -0.1036597266793251,
+      "sft_loss": 0.8950971364974976,
+      "step": 980
+    },
+    {
+      "epoch": 1.76039119804401,
+      "grad_norm": 1.4812334775924683,
+      "learning_rate": 1.8239381598343576e-06,
+      "logits/chosen": -2.8683552742004395,
+      "logits/rejected": -2.890810012817383,
+      "logps/chosen": -0.8186439275741577,
+      "logps/rejected": -1.0320905447006226,
+      "loss": 0.8854,
+      "odds_ratio_loss": 0.6674761772155762,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.08186439424753189,
+      "rewards/margins": 0.021344667300581932,
+      "rewards/rejected": -0.10320906341075897,
+      "sft_loss": 0.8186439275741577,
+      "step": 990
+    },
+    {
+      "epoch": 1.7781729273171816,
+      "grad_norm": 5.4584126472473145,
+      "learning_rate": 1.779207600392312e-06,
+      "logits/chosen": -2.8556861877441406,
+      "logits/rejected": -2.859384298324585,
+      "logps/chosen": -0.840672492980957,
+      "logps/rejected": -1.0127156972885132,
+      "loss": 0.906,
+      "odds_ratio_loss": 0.6529561877250671,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.08406725525856018,
+      "rewards/margins": 0.01720432937145233,
+      "rewards/rejected": -0.10127158463001251,
+      "sft_loss": 0.840672492980957,
+      "step": 1000
+    },
+    {
+      "epoch": 1.7781729273171816,
+      "eval_logits/chosen": -2.8532934188842773,
+      "eval_logits/rejected": -2.8782994747161865,
+      "eval_logps/chosen": -0.8414809107780457,
+      "eval_logps/rejected": -1.0811069011688232,
+      "eval_loss": 0.903913676738739,
+      "eval_odds_ratio_loss": 0.6243272423744202,
+      "eval_rewards/accuracies": 0.578000009059906,
+      "eval_rewards/chosen": -0.0841480940580368,
+      "eval_rewards/margins": 0.023962605744600296,
+      "eval_rewards/rejected": -0.1081107035279274,
+      "eval_runtime": 189.1709,
+      "eval_samples_per_second": 5.286,
+      "eval_sft_loss": 0.8414809107780457,
+      "eval_steps_per_second": 2.643,
+      "step": 1000
+    },
+    {
+      "epoch": 1.7959546565903532,
+      "grad_norm": 1.9621540307998657,
+      "learning_rate": 1.7347273253353552e-06,
+      "logits/chosen": -2.8437576293945312,
+      "logits/rejected": -2.8630166053771973,
+      "logps/chosen": -0.8406645655632019,
+      "logps/rejected": -1.0365639925003052,
+      "loss": 0.9076,
+      "odds_ratio_loss": 0.669218897819519,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.08406645804643631,
+      "rewards/margins": 0.019589943811297417,
+      "rewards/rejected": -0.10365639626979828,
+      "sft_loss": 0.8406645655632019,
+      "step": 1010
+    },
+    {
+      "epoch": 1.8137363858635251,
+      "grad_norm": 4.011701583862305,
+      "learning_rate": 1.690512779774029e-06,
+      "logits/chosen": -2.848282814025879,
+      "logits/rejected": -2.8511271476745605,
+      "logps/chosen": -0.8997753858566284,
+      "logps/rejected": -1.1389718055725098,
+      "loss": 0.9632,
+      "odds_ratio_loss": 0.6343892812728882,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.08997754007577896,
+      "rewards/margins": 0.023919641971588135,
+      "rewards/rejected": -0.11389718949794769,
+      "sft_loss": 0.8997753858566284,
+      "step": 1020
+    },
+    {
+      "epoch": 1.831518115136697,
+      "grad_norm": 2.2862370014190674,
+      "learning_rate": 1.6465793165482838e-06,
+      "logits/chosen": -2.8141562938690186,
+      "logits/rejected": -2.8219082355499268,
+      "logps/chosen": -0.8186345100402832,
+      "logps/rejected": -1.0088984966278076,
+      "loss": 0.8807,
+      "odds_ratio_loss": 0.6206359267234802,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.08186344802379608,
+      "rewards/margins": 0.019026407971978188,
+      "rewards/rejected": -0.10088986158370972,
+      "sft_loss": 0.8186345100402832,
+      "step": 1030
+    },
+    {
+      "epoch": 1.849299844409869,
+      "grad_norm": 2.5129692554473877,
+      "learning_rate": 1.6029421908964305e-06,
+      "logits/chosen": -2.821484088897705,
+      "logits/rejected": -2.8394651412963867,
+      "logps/chosen": -0.8219555020332336,
+      "logps/rejected": -1.3423779010772705,
+      "loss": 0.8817,
+      "odds_ratio_loss": 0.5971612334251404,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.08219555765390396,
+      "rewards/margins": 0.052042216062545776,
+      "rewards/rejected": -0.13423778116703033,
+      "sft_loss": 0.8219555020332336,
+      "step": 1040
+    },
+    {
+      "epoch": 1.8670815736830408,
+      "grad_norm": 2.674842596054077,
+      "learning_rate": 1.559616555157985e-06,
+      "logits/chosen": -2.8907628059387207,
+      "logits/rejected": -2.8810207843780518,
+      "logps/chosen": -0.8370941281318665,
+      "logps/rejected": -1.0264050960540771,
+      "loss": 0.9034,
+      "odds_ratio_loss": 0.6629946231842041,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.08370941877365112,
+      "rewards/margins": 0.018931085243821144,
+      "rewards/rejected": -0.10264050960540771,
+      "sft_loss": 0.8370941281318665,
+      "step": 1050
+    },
+    {
+      "epoch": 1.8848633029562125,
+      "grad_norm": 2.72856068611145,
+      "learning_rate": 1.516617453512252e-06,
+      "logits/chosen": -2.8834056854248047,
+      "logits/rejected": -2.8996243476867676,
+      "logps/chosen": -0.8767944574356079,
+      "logps/rejected": -1.0179784297943115,
+      "loss": 0.9475,
+      "odds_ratio_loss": 0.7070103883743286,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.08767944574356079,
+      "rewards/margins": 0.014118405990302563,
+      "rewards/rejected": -0.10179785639047623,
+      "sft_loss": 0.8767944574356079,
+      "step": 1060
+    },
+    {
+      "epoch": 1.9026450322293842,
+      "grad_norm": 3.5679309368133545,
+      "learning_rate": 1.473959816754449e-06,
+      "logits/chosen": -2.857292890548706,
+      "logits/rejected": -2.8642122745513916,
+      "logps/chosen": -0.8274821043014526,
+      "logps/rejected": -0.9466173052787781,
+      "loss": 0.8975,
+      "odds_ratio_loss": 0.6997173428535461,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.08274821937084198,
+      "rewards/margins": 0.011913511902093887,
+      "rewards/rejected": -0.09466172009706497,
+      "sft_loss": 0.8274821043014526,
+      "step": 1070
+    },
+    {
+      "epoch": 1.920426761502556,
+      "grad_norm": 1.369982361793518,
+      "learning_rate": 1.4316584571112213e-06,
+      "logits/chosen": -2.8926608562469482,
+      "logits/rejected": -2.9016175270080566,
+      "logps/chosen": -0.878426194190979,
+      "logps/rejected": -1.022551417350769,
+      "loss": 0.9463,
+      "odds_ratio_loss": 0.6782765984535217,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.08784262835979462,
+      "rewards/margins": 0.014412516728043556,
+      "rewards/rejected": -0.10225514322519302,
+      "sft_loss": 0.878426194190979,
+      "step": 1080
+    },
+    {
+      "epoch": 1.938208490775728,
+      "grad_norm": 2.1955406665802,
+      "learning_rate": 1.389728063097306e-06,
+      "logits/chosen": -2.859020471572876,
+      "logits/rejected": -2.871717929840088,
+      "logps/chosen": -0.8529576063156128,
+      "logps/rejected": -1.0697009563446045,
+      "loss": 0.9165,
+      "odds_ratio_loss": 0.6351792216300964,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.08529576659202576,
+      "rewards/margins": 0.021674351766705513,
+      "rewards/rejected": -0.10697011649608612,
+      "sft_loss": 0.8529576063156128,
+      "step": 1090
+    },
+    {
+      "epoch": 1.9559902200488999,
+      "grad_norm": 1.758565068244934,
+      "learning_rate": 1.348183194415179e-06,
+      "logits/chosen": -2.853273868560791,
+      "logits/rejected": -2.826066493988037,
+      "logps/chosen": -0.8589800000190735,
+      "logps/rejected": -1.1160128116607666,
+      "loss": 0.9192,
+      "odds_ratio_loss": 0.6018035411834717,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0858980119228363,
+      "rewards/margins": 0.025703275576233864,
+      "rewards/rejected": -0.11160127818584442,
+      "sft_loss": 0.8589800000190735,
+      "step": 1100
+    },
+    {
+      "epoch": 1.9737719493220716,
+      "grad_norm": 2.2166192531585693,
+      "learning_rate": 1.3070382768994015e-06,
+      "logits/chosen": -2.8573784828186035,
+      "logits/rejected": -2.87496018409729,
+      "logps/chosen": -0.8593405485153198,
+      "logps/rejected": -1.0153789520263672,
+      "loss": 0.9242,
+      "odds_ratio_loss": 0.6489418148994446,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.08593406528234482,
+      "rewards/margins": 0.015603835694491863,
+      "rewards/rejected": -0.10153790563344955,
+      "sft_loss": 0.8593405485153198,
+      "step": 1110
+    },
+    {
+      "epoch": 1.9915536785952432,
+      "grad_norm": 2.127784013748169,
+      "learning_rate": 1.2663075975074746e-06,
+      "logits/chosen": -2.8475844860076904,
+      "logits/rejected": -2.8674192428588867,
+      "logps/chosen": -0.830818772315979,
+      "logps/rejected": -1.1525778770446777,
+      "loss": 0.8944,
+      "odds_ratio_loss": 0.635840654373169,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.08308187872171402,
+      "rewards/margins": 0.032175906002521515,
+      "rewards/rejected": -0.11525777727365494,
+      "sft_loss": 0.830818772315979,
+      "step": 1120
+    },
+    {
+      "epoch": 2.009335407868415,
+      "grad_norm": 2.4992928504943848,
+      "learning_rate": 1.2260052993589034e-06,
+      "logits/chosen": -2.8519034385681152,
+      "logits/rejected": -2.8721909523010254,
+      "logps/chosen": -0.9101880192756653,
+      "logps/rejected": -1.0249983072280884,
+      "loss": 0.9847,
+      "odds_ratio_loss": 0.7451602220535278,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.09101879596710205,
+      "rewards/margins": 0.011481017805635929,
+      "rewards/rejected": -0.102499820291996,
+      "sft_loss": 0.9101880192756653,
+      "step": 1130
+    },
+    {
+      "epoch": 2.027117137141587,
+      "grad_norm": 1.3270055055618286,
+      "learning_rate": 1.1861453768242099e-06,
+      "logits/chosen": -2.8831446170806885,
+      "logits/rejected": -2.8843555450439453,
+      "logps/chosen": -0.7847625613212585,
+      "logps/rejected": -1.0357749462127686,
+      "loss": 0.8449,
+      "odds_ratio_loss": 0.6014257669448853,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.07847625762224197,
+      "rewards/margins": 0.025101233273744583,
+      "rewards/rejected": -0.10357747972011566,
+      "sft_loss": 0.7847625613212585,
+      "step": 1140
+    },
+    {
+      "epoch": 2.044898866414759,
+      "grad_norm": 11.792732238769531,
+      "learning_rate": 1.1467416706655982e-06,
+      "logits/chosen": -2.8700387477874756,
+      "logits/rejected": -2.8984532356262207,
+      "logps/chosen": -0.9160982370376587,
+      "logps/rejected": -1.110647439956665,
+      "loss": 0.9851,
+      "odds_ratio_loss": 0.6902655363082886,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.09160983562469482,
+      "rewards/margins": 0.019454922527074814,
+      "rewards/rejected": -0.11106475442647934,
+      "sft_loss": 0.9160982370376587,
+      "step": 1150
+    },
+    {
+      "epoch": 2.062680595687931,
+      "grad_norm": 2.206909418106079,
+      "learning_rate": 1.1078078632309559e-06,
+      "logits/chosen": -2.8493118286132812,
+      "logits/rejected": -2.8856310844421387,
+      "logps/chosen": -0.8342447280883789,
+      "logps/rejected": -1.0583864450454712,
+      "loss": 0.8987,
+      "odds_ratio_loss": 0.644468367099762,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.08342447131872177,
+      "rewards/margins": 0.022414181381464005,
+      "rewards/rejected": -0.10583865642547607,
+      "sft_loss": 0.8342447280883789,
+      "step": 1160
+    },
+    {
+      "epoch": 2.0804623249611023,
+      "grad_norm": 4.428743362426758,
+      "learning_rate": 1.0693574737028627e-06,
+      "logits/chosen": -2.860917329788208,
+      "logits/rejected": -2.8824262619018555,
+      "logps/chosen": -0.859830379486084,
+      "logps/rejected": -1.011566162109375,
+      "loss": 0.9292,
+      "odds_ratio_loss": 0.6934677362442017,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.08598305284976959,
+      "rewards/margins": 0.015173576772212982,
+      "rewards/rejected": -0.10115662962198257,
+      "sft_loss": 0.859830379486084,
+      "step": 1170
+    },
+    {
+      "epoch": 2.098244054234274,
+      "grad_norm": 2.3405988216400146,
+      "learning_rate": 1.0314038534042586e-06,
+      "logits/chosen": -2.88313364982605,
+      "logits/rejected": -2.89017915725708,
+      "logps/chosen": -0.8049169778823853,
+      "logps/rejected": -1.0722546577453613,
+      "loss": 0.8704,
+      "odds_ratio_loss": 0.6548857688903809,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.08049169927835464,
+      "rewards/margins": 0.026733767241239548,
+      "rewards/rejected": -0.1072254627943039,
+      "sft_loss": 0.8049169778823853,
+      "step": 1180
+    },
+    {
+      "epoch": 2.116025783507446,
+      "grad_norm": 3.3044447898864746,
+      "learning_rate": 9.939601811623946e-07,
+      "logits/chosen": -2.8709566593170166,
+      "logits/rejected": -2.8734612464904785,
+      "logps/chosen": -0.84669029712677,
+      "logps/rejected": -1.037172555923462,
+      "loss": 0.9141,
+      "odds_ratio_loss": 0.6745220422744751,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.08466903120279312,
+      "rewards/margins": 0.019048208370804787,
+      "rewards/rejected": -0.10371723026037216,
+      "sft_loss": 0.84669029712677,
+      "step": 1190
+    },
+    {
+      "epoch": 2.133807512780618,
+      "grad_norm": 1.8569538593292236,
+      "learning_rate": 9.570394587326825e-07,
+      "logits/chosen": -2.8537509441375732,
+      "logits/rejected": -2.8589584827423096,
+      "logps/chosen": -0.8301420211791992,
+      "logps/rejected": -1.0919809341430664,
+      "loss": 0.8916,
+      "odds_ratio_loss": 0.6140956878662109,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.08301420509815216,
+      "rewards/margins": 0.02618388459086418,
+      "rewards/rejected": -0.10919810831546783,
+      "sft_loss": 0.8301420211791992,
+      "step": 1200
+    },
+    {
+      "epoch": 2.15158924205379,
+      "grad_norm": 1.6108454465866089,
+      "learning_rate": 9.206545062840302e-07,
+      "logits/chosen": -2.9044604301452637,
+      "logits/rejected": -2.876079559326172,
+      "logps/chosen": -0.7979404926300049,
+      "logps/rejected": -1.1050665378570557,
+      "loss": 0.8573,
+      "odds_ratio_loss": 0.5938332676887512,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.07979404926300049,
+      "rewards/margins": 0.03071259893476963,
+      "rewards/rejected": -0.11050665378570557,
+      "sft_loss": 0.7979404926300049,
+      "step": 1210
+    },
+    {
+      "epoch": 2.1693709713269618,
+      "grad_norm": 1.6135528087615967,
+      "learning_rate": 8.848179579472285e-07,
+      "logits/chosen": -2.8942551612854004,
+      "logits/rejected": -2.9058589935302734,
+      "logps/chosen": -0.8136836886405945,
+      "logps/rejected": -0.9423009157180786,
+      "loss": 0.8803,
+      "odds_ratio_loss": 0.666199803352356,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.08136837184429169,
+      "rewards/margins": 0.012861723080277443,
+      "rewards/rejected": -0.09423010051250458,
+      "sft_loss": 0.8136836886405945,
+      "step": 1220
+    },
+    {
+      "epoch": 2.1871527006001332,
+      "grad_norm": 4.099541664123535,
+      "learning_rate": 8.495422574279403e-07,
+      "logits/chosen": -2.8292388916015625,
+      "logits/rejected": -2.825455904006958,
+      "logps/chosen": -0.7480133175849915,
+      "logps/rejected": -1.0116589069366455,
+      "loss": 0.8073,
+      "odds_ratio_loss": 0.5924416780471802,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.07480133324861526,
+      "rewards/margins": 0.026364561170339584,
+      "rewards/rejected": -0.10116589069366455,
+      "sft_loss": 0.7480133175849915,
+      "step": 1230
+    },
+    {
+      "epoch": 2.204934429873305,
+      "grad_norm": 2.086594581604004,
+      "learning_rate": 8.148396536858063e-07,
+      "logits/chosen": -2.863175630569458,
+      "logits/rejected": -2.8580069541931152,
+      "logps/chosen": -0.8519015312194824,
+      "logps/rejected": -1.1402422189712524,
+      "loss": 0.9162,
+      "odds_ratio_loss": 0.6432971954345703,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.08519015461206436,
+      "rewards/margins": 0.028834056109189987,
+      "rewards/rejected": -0.11402420699596405,
+      "sft_loss": 0.8519015312194824,
+      "step": 1240
+    },
+    {
+      "epoch": 2.222716159146477,
+      "grad_norm": 2.096405506134033,
+      "learning_rate": 7.807221966811815e-07,
+      "logits/chosen": -2.8480238914489746,
+      "logits/rejected": -2.8696630001068115,
+      "logps/chosen": -0.8801048994064331,
+      "logps/rejected": -1.0339150428771973,
+      "loss": 0.9478,
+      "odds_ratio_loss": 0.6773165464401245,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.08801049739122391,
+      "rewards/margins": 0.015381010249257088,
+      "rewards/rejected": -0.10339150577783585,
+      "sft_loss": 0.8801048994064331,
+      "step": 1250
+    },
+    {
+      "epoch": 2.240497888419649,
+      "grad_norm": 1.47752845287323,
+      "learning_rate": 7.47201733190962e-07,
+      "logits/chosen": -2.831873655319214,
+      "logits/rejected": -2.850604772567749,
+      "logps/chosen": -0.798612117767334,
+      "logps/rejected": -1.0020263195037842,
+      "loss": 0.8614,
+      "odds_ratio_loss": 0.6277336478233337,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.07986120879650116,
+      "rewards/margins": 0.020341429859399796,
+      "rewards/rejected": -0.10020263493061066,
+      "sft_loss": 0.798612117767334,
+      "step": 1260
+    },
+    {
+      "epoch": 2.258279617692821,
+      "grad_norm": 1.7826944589614868,
+      "learning_rate": 7.142899026949721e-07,
+      "logits/chosen": -2.8982224464416504,
+      "logits/rejected": -2.9022057056427,
+      "logps/chosen": -0.817043662071228,
+      "logps/rejected": -0.9819319844245911,
+      "loss": 0.8802,
+      "odds_ratio_loss": 0.6313419938087463,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.08170437067747116,
+      "rewards/margins": 0.016488831490278244,
+      "rewards/rejected": -0.0981932133436203,
+      "sft_loss": 0.817043662071228,
+      "step": 1270
+    },
+    {
+      "epoch": 2.2760613469659923,
+      "grad_norm": 3.5314130783081055,
+      "learning_rate": 6.819981333343273e-07,
+      "logits/chosen": -2.81345796585083,
+      "logits/rejected": -2.830575704574585,
+      "logps/chosen": -0.8146475553512573,
+      "logps/rejected": -1.0825046300888062,
+      "loss": 0.8761,
+      "odds_ratio_loss": 0.6145650744438171,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.08146476000547409,
+      "rewards/margins": 0.026785722002387047,
+      "rewards/rejected": -0.10825047641992569,
+      "sft_loss": 0.8146475553512573,
+      "step": 1280
+    },
+    {
+      "epoch": 2.293843076239164,
+      "grad_norm": 2.047950267791748,
+      "learning_rate": 6.503376379431839e-07,
+      "logits/chosen": -2.8456294536590576,
+      "logits/rejected": -2.8468470573425293,
+      "logps/chosen": -0.8984563946723938,
+      "logps/rejected": -0.965488612651825,
+      "loss": 0.9673,
+      "odds_ratio_loss": 0.68804532289505,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.08984563499689102,
+      "rewards/margins": 0.006703221704810858,
+      "rewards/rejected": -0.09654886275529861,
+      "sft_loss": 0.8984563946723938,
+      "step": 1290
+    },
+    {
+      "epoch": 2.311624805512336,
+      "grad_norm": 1.6956673860549927,
+      "learning_rate": 6.193194101552502e-07,
+      "logits/chosen": -2.8512775897979736,
+      "logits/rejected": -2.8425419330596924,
+      "logps/chosen": -0.793393611907959,
+      "logps/rejected": -1.0585198402404785,
+      "loss": 0.8516,
+      "odds_ratio_loss": 0.5823057293891907,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.07933936268091202,
+      "rewards/margins": 0.026512619107961655,
+      "rewards/rejected": -0.10585198551416397,
+      "sft_loss": 0.793393611907959,
+      "step": 1300
+    },
+    {
+      "epoch": 2.329406534785508,
+      "grad_norm": 2.3841018676757812,
+      "learning_rate": 5.889542205864083e-07,
+      "logits/chosen": -2.848145008087158,
+      "logits/rejected": -2.8572099208831787,
+      "logps/chosen": -0.853177547454834,
+      "logps/rejected": -1.0807201862335205,
+      "loss": 0.9165,
+      "odds_ratio_loss": 0.6334924697875977,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08531775325536728,
+      "rewards/margins": 0.022754264995455742,
+      "rewards/rejected": -0.10807202011346817,
+      "sft_loss": 0.853177547454834,
+      "step": 1310
+    },
+    {
+      "epoch": 2.34718826405868,
+      "grad_norm": 1.8613510131835938,
+      "learning_rate": 5.592526130947862e-07,
+      "logits/chosen": -2.81400203704834,
+      "logits/rejected": -2.84773325920105,
+      "logps/chosen": -0.8714796304702759,
+      "logps/rejected": -1.0702935457229614,
+      "loss": 0.9403,
+      "odds_ratio_loss": 0.6886882781982422,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.08714796602725983,
+      "rewards/margins": 0.019881393760442734,
+      "rewards/rejected": -0.10702935606241226,
+      "sft_loss": 0.8714796304702759,
+      "step": 1320
+    },
+    {
+      "epoch": 2.3649699933318518,
+      "grad_norm": 1.9149771928787231,
+      "learning_rate": 5.302249011195507e-07,
+      "logits/chosen": -2.82387375831604,
+      "logits/rejected": -2.8486294746398926,
+      "logps/chosen": -0.8160147666931152,
+      "logps/rejected": -0.9580420255661011,
+      "loss": 0.8828,
+      "odds_ratio_loss": 0.6679019927978516,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.08160146325826645,
+      "rewards/margins": 0.014202730730175972,
+      "rewards/rejected": -0.09580419957637787,
+      "sft_loss": 0.8160147666931152,
+      "step": 1330
+    },
+    {
+      "epoch": 2.382751722605023,
+      "grad_norm": 2.322023630142212,
+      "learning_rate": 5.018811640997307e-07,
+      "logits/chosen": -2.833300828933716,
+      "logits/rejected": -2.8586716651916504,
+      "logps/chosen": -0.8773719072341919,
+      "logps/rejected": -1.2085764408111572,
+      "loss": 0.9397,
+      "odds_ratio_loss": 0.6232184171676636,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.08773718029260635,
+      "rewards/margins": 0.033120471984148026,
+      "rewards/rejected": -0.12085764110088348,
+      "sft_loss": 0.8773719072341919,
+      "step": 1340
+    },
+    {
+      "epoch": 2.400533451878195,
+      "grad_norm": 1.6597644090652466,
+      "learning_rate": 4.7423124397427105e-07,
+      "logits/chosen": -2.8043930530548096,
+      "logits/rejected": -2.8492674827575684,
+      "logps/chosen": -0.8693526983261108,
+      "logps/rejected": -1.019430160522461,
+      "loss": 0.9355,
+      "odds_ratio_loss": 0.6619070768356323,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.08693526685237885,
+      "rewards/margins": 0.015007746405899525,
+      "rewards/rejected": -0.1019430160522461,
+      "sft_loss": 0.8693526983261108,
+      "step": 1350
+    },
+    {
+      "epoch": 2.418315181151367,
+      "grad_norm": 2.013772487640381,
+      "learning_rate": 4.472847417645787e-07,
+      "logits/chosen": -2.856095552444458,
+      "logits/rejected": -2.853999614715576,
+      "logps/chosen": -0.8434507250785828,
+      "logps/rejected": -1.1554135084152222,
+      "loss": 0.9027,
+      "odds_ratio_loss": 0.5929771661758423,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.08434506505727768,
+      "rewards/margins": 0.031196290627121925,
+      "rewards/rejected": -0.11554136127233505,
+      "sft_loss": 0.8434507250785828,
+      "step": 1360
+    },
+    {
+      "epoch": 2.436096910424539,
+      "grad_norm": 1.1766934394836426,
+      "learning_rate": 4.210510142406993e-07,
+      "logits/chosen": -2.826666831970215,
+      "logits/rejected": -2.8096094131469727,
+      "logps/chosen": -0.8101630210876465,
+      "logps/rejected": -1.061128854751587,
+      "loss": 0.8711,
+      "odds_ratio_loss": 0.6095449924468994,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.08101630210876465,
+      "rewards/margins": 0.0250965915620327,
+      "rewards/rejected": -0.10611288249492645,
+      "sft_loss": 0.8101630210876465,
+      "step": 1370
+    },
+    {
+      "epoch": 2.4538786396977104,
+      "grad_norm": 1.6385358572006226,
+      "learning_rate": 3.9553917067232966e-07,
+      "logits/chosen": -2.8439018726348877,
+      "logits/rejected": -2.869356155395508,
+      "logps/chosen": -0.8570329546928406,
+      "logps/rejected": -1.0249860286712646,
+      "loss": 0.9248,
+      "odds_ratio_loss": 0.6779541373252869,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0857032984495163,
+      "rewards/margins": 0.016795307397842407,
+      "rewards/rejected": -0.1024986058473587,
+      "sft_loss": 0.8570329546928406,
+      "step": 1380
+    },
+    {
+      "epoch": 2.4716603689708823,
+      "grad_norm": 1.9620130062103271,
+      "learning_rate": 3.707580696657509e-07,
+      "logits/chosen": -2.8321003913879395,
+      "logits/rejected": -2.8363351821899414,
+      "logps/chosen": -0.8065320253372192,
+      "logps/rejected": -0.9883378744125366,
+      "loss": 0.8702,
+      "odds_ratio_loss": 0.6370663046836853,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.08065320551395416,
+      "rewards/margins": 0.018180575221776962,
+      "rewards/rejected": -0.09883377701044083,
+      "sft_loss": 0.8065320253372192,
+      "step": 1390
+    },
+    {
+      "epoch": 2.489442098244054,
+      "grad_norm": 1.9136977195739746,
+      "learning_rate": 3.4671631608781815e-07,
+      "logits/chosen": -2.8458282947540283,
+      "logits/rejected": -2.858309745788574,
+      "logps/chosen": -0.8568046689033508,
+      "logps/rejected": -1.1048234701156616,
+      "loss": 0.9252,
+      "odds_ratio_loss": 0.6840890645980835,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.08568046987056732,
+      "rewards/margins": 0.02480187825858593,
+      "rewards/rejected": -0.110482357442379,
+      "sft_loss": 0.8568046689033508,
+      "step": 1400
+    },
+    {
+      "epoch": 2.507223827517226,
+      "grad_norm": 2.3483951091766357,
+      "learning_rate": 3.234222580780405e-07,
+      "logits/chosen": -2.8452916145324707,
+      "logits/rejected": -2.8620214462280273,
+      "logps/chosen": -0.8229707479476929,
+      "logps/rejected": -1.0171488523483276,
+      "loss": 0.8863,
+      "odds_ratio_loss": 0.6332994699478149,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.08229707181453705,
+      "rewards/margins": 0.01941780559718609,
+      "rewards/rejected": -0.10171488672494888,
+      "sft_loss": 0.8229707479476929,
+      "step": 1410
+    },
+    {
+      "epoch": 2.525005556790398,
+      "grad_norm": 1.985844612121582,
+      "learning_rate": 3.0088398414982375e-07,
+      "logits/chosen": -2.810109853744507,
+      "logits/rejected": -2.8480405807495117,
+      "logps/chosen": -0.8503270149230957,
+      "logps/rejected": -1.14946711063385,
+      "loss": 0.9167,
+      "odds_ratio_loss": 0.6633514165878296,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.08503270149230957,
+      "rewards/margins": 0.02991401217877865,
+      "rewards/rejected": -0.11494670808315277,
+      "sft_loss": 0.8503270149230957,
+      "step": 1420
+    },
+    {
+      "epoch": 2.54278728606357,
+      "grad_norm": 1.8843451738357544,
+      "learning_rate": 2.7910932038184487e-07,
+      "logits/chosen": -2.828174352645874,
+      "logits/rejected": -2.8114073276519775,
+      "logps/chosen": -0.8486431241035461,
+      "logps/rejected": -1.0430662631988525,
+      "loss": 0.9131,
+      "odds_ratio_loss": 0.6441913843154907,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.08486433327198029,
+      "rewards/margins": 0.019442306831479073,
+      "rewards/rejected": -0.10430662333965302,
+      "sft_loss": 0.8486431241035461,
+      "step": 1430
+    },
+    {
+      "epoch": 2.5605690153367417,
+      "grad_norm": 1.9914612770080566,
+      "learning_rate": 2.5810582770057325e-07,
+      "logits/chosen": -2.873000383377075,
+      "logits/rejected": -2.902294635772705,
+      "logps/chosen": -0.8076226115226746,
+      "logps/rejected": -1.0467280149459839,
+      "loss": 0.8714,
+      "odds_ratio_loss": 0.6380000114440918,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.08076226711273193,
+      "rewards/margins": 0.023910541087388992,
+      "rewards/rejected": -0.10467280447483063,
+      "sft_loss": 0.8076226115226746,
+      "step": 1440
+    },
+    {
+      "epoch": 2.578350744609913,
+      "grad_norm": 2.0563340187072754,
+      "learning_rate": 2.3788079925484402e-07,
+      "logits/chosen": -2.849846601486206,
+      "logits/rejected": -2.8575031757354736,
+      "logps/chosen": -0.8711313009262085,
+      "logps/rejected": -0.9900692105293274,
+      "loss": 0.9391,
+      "odds_ratio_loss": 0.6800366044044495,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.08711313456296921,
+      "rewards/margins": 0.01189377624541521,
+      "rewards/rejected": -0.09900690615177155,
+      "sft_loss": 0.8711313009262085,
+      "step": 1450
+    },
+    {
+      "epoch": 2.596132473883085,
+      "grad_norm": 2.145470142364502,
+      "learning_rate": 2.1844125788342661e-07,
+      "logits/chosen": -2.840655565261841,
+      "logits/rejected": -2.859443187713623,
+      "logps/chosen": -0.7942542433738708,
+      "logps/rejected": -1.2094497680664062,
+      "loss": 0.8548,
+      "odds_ratio_loss": 0.6057204008102417,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.07942543178796768,
+      "rewards/margins": 0.04151954501867294,
+      "rewards/rejected": -0.12094497680664062,
+      "sft_loss": 0.7942542433738708,
+      "step": 1460
+    },
+    {
+      "epoch": 2.613914203156257,
+      "grad_norm": 1.8150006532669067,
+      "learning_rate": 1.9979395367644428e-07,
+      "logits/chosen": -2.865077495574951,
+      "logits/rejected": -2.8804574012756348,
+      "logps/chosen": -0.8026032447814941,
+      "logps/rejected": -1.044508457183838,
+      "loss": 0.8624,
+      "odds_ratio_loss": 0.5979448556900024,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.08026032149791718,
+      "rewards/margins": 0.024190524592995644,
+      "rewards/rejected": -0.10445085912942886,
+      "sft_loss": 0.8026032447814941,
+      "step": 1470
+    },
+    {
+      "epoch": 2.631695932429429,
+      "grad_norm": 3.1679821014404297,
+      "learning_rate": 1.81945361631512e-07,
+      "logits/chosen": -2.8732008934020996,
+      "logits/rejected": -2.8841676712036133,
+      "logps/chosen": -0.8442559242248535,
+      "logps/rejected": -1.0102087259292603,
+      "loss": 0.9134,
+      "odds_ratio_loss": 0.6910725235939026,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.08442559093236923,
+      "rewards/margins": 0.016595274209976196,
+      "rewards/rejected": -0.10102085769176483,
+      "sft_loss": 0.8442559242248535,
+      "step": 1480
+    },
+    {
+      "epoch": 2.6494776617026004,
+      "grad_norm": 2.5551412105560303,
+      "learning_rate": 1.6490167940538343e-07,
+      "logits/chosen": -2.8778164386749268,
+      "logits/rejected": -2.8786826133728027,
+      "logps/chosen": -0.8008125424385071,
+      "logps/rejected": -1.0858347415924072,
+      "loss": 0.8593,
+      "odds_ratio_loss": 0.5846473574638367,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.08008125424385071,
+      "rewards/margins": 0.028502214699983597,
+      "rewards/rejected": -0.1085834726691246,
+      "sft_loss": 0.8008125424385071,
+      "step": 1490
+    },
+    {
+      "epoch": 2.6672593909757722,
+      "grad_norm": 1.7907795906066895,
+      "learning_rate": 1.4866882516191339e-07,
+      "logits/chosen": -2.795592784881592,
+      "logits/rejected": -2.8337063789367676,
+      "logps/chosen": -0.8366915583610535,
+      "logps/rejected": -1.0835788249969482,
+      "loss": 0.9019,
+      "odds_ratio_loss": 0.652554988861084,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.08366917073726654,
+      "rewards/margins": 0.0246887169778347,
+      "rewards/rejected": -0.10835788398981094,
+      "sft_loss": 0.8366915583610535,
+      "step": 1500
+    },
+    {
+      "epoch": 2.6672593909757722,
+      "eval_logits/chosen": -2.8460819721221924,
+      "eval_logits/rejected": -2.8720545768737793,
+      "eval_logps/chosen": -0.8352006673812866,
+      "eval_logps/rejected": -1.0736578702926636,
+      "eval_loss": 0.8975116014480591,
+      "eval_odds_ratio_loss": 0.6231085658073425,
+      "eval_rewards/accuracies": 0.5899999737739563,
+      "eval_rewards/chosen": -0.0835200697183609,
+      "eval_rewards/margins": 0.023845719173550606,
+      "eval_rewards/rejected": -0.10736579447984695,
+      "eval_runtime": 189.1942,
+      "eval_samples_per_second": 5.286,
+      "eval_sft_loss": 0.8352006673812866,
+      "eval_steps_per_second": 2.643,
+      "step": 1500
+    },
+    {
+      "epoch": 2.685041120248944,
+      "grad_norm": 5.948815822601318,
+      "learning_rate": 1.3325243551706057e-07,
+      "logits/chosen": -2.823535442352295,
+      "logits/rejected": -2.8570055961608887,
+      "logps/chosen": -0.8226287961006165,
+      "logps/rejected": -1.2481344938278198,
+      "loss": 0.8823,
+      "odds_ratio_loss": 0.5963335633277893,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.08226287364959717,
+      "rewards/margins": 0.04255058616399765,
+      "rewards/rejected": -0.12481345981359482,
+      "sft_loss": 0.8226287961006165,
+      "step": 1510
+    },
+    {
+      "epoch": 2.702822849522116,
+      "grad_norm": 2.1956028938293457,
+      "learning_rate": 1.1865786358165737e-07,
+      "logits/chosen": -2.841465473175049,
+      "logits/rejected": -2.881883382797241,
+      "logps/chosen": -0.8048586845397949,
+      "logps/rejected": -0.9951068162918091,
+      "loss": 0.8691,
+      "odds_ratio_loss": 0.6420662999153137,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.08048586547374725,
+      "rewards/margins": 0.019024807959794998,
+      "rewards/rejected": -0.09951067715883255,
+      "sft_loss": 0.8048586845397949,
+      "step": 1520
+    },
+    {
+      "epoch": 2.720604578795288,
+      "grad_norm": 4.919430255889893,
+      "learning_rate": 1.0489017710262311e-07,
+      "logits/chosen": -2.8700149059295654,
+      "logits/rejected": -2.9017858505249023,
+      "logps/chosen": -0.8849034309387207,
+      "logps/rejected": -1.1535307168960571,
+      "loss": 0.9517,
+      "odds_ratio_loss": 0.667965292930603,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08849034458398819,
+      "rewards/margins": 0.026862725615501404,
+      "rewards/rejected": -0.11535308510065079,
+      "sft_loss": 0.8849034309387207,
+      "step": 1530
+    },
+    {
+      "epoch": 2.73838630806846,
+      "grad_norm": 2.982793092727661,
+      "learning_rate": 9.195415670326446e-08,
+      "logits/chosen": -2.874678611755371,
+      "logits/rejected": -2.887955904006958,
+      "logps/chosen": -0.828321099281311,
+      "logps/rejected": -1.0579955577850342,
+      "loss": 0.8916,
+      "odds_ratio_loss": 0.6329658031463623,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.08283210545778275,
+      "rewards/margins": 0.02296745963394642,
+      "rewards/rejected": -0.10579957067966461,
+      "sft_loss": 0.828321099281311,
+      "step": 1540
+    },
+    {
+      "epoch": 2.7561680373416317,
+      "grad_norm": 2.7303407192230225,
+      "learning_rate": 7.985429422327384e-08,
+      "logits/chosen": -2.8821816444396973,
+      "logits/rejected": -2.9013893604278564,
+      "logps/chosen": -0.8389023542404175,
+      "logps/rejected": -0.9915286302566528,
+      "loss": 0.9067,
+      "odds_ratio_loss": 0.6778423190116882,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.08389024436473846,
+      "rewards/margins": 0.015262606553733349,
+      "rewards/rejected": -0.09915284812450409,
+      "sft_loss": 0.8389023542404175,
+      "step": 1550
+    },
+    {
+      "epoch": 2.773949766614803,
+      "grad_norm": 1.570169448852539,
+      "learning_rate": 6.859479115900818e-08,
+      "logits/chosen": -2.848161458969116,
+      "logits/rejected": -2.8777124881744385,
+      "logps/chosen": -0.8677851557731628,
+      "logps/rejected": -1.1111342906951904,
+      "loss": 0.9293,
+      "odds_ratio_loss": 0.6150751709938049,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.08677852153778076,
+      "rewards/margins": 0.024334916844964027,
+      "rewards/rejected": -0.11111342906951904,
+      "sft_loss": 0.8677851557731628,
+      "step": 1560
+    },
+    {
+      "epoch": 2.791731495887975,
+      "grad_norm": 6.0747294425964355,
+      "learning_rate": 5.817955720457902e-08,
+      "logits/chosen": -2.8411552906036377,
+      "logits/rejected": -2.8624863624572754,
+      "logps/chosen": -0.881393313407898,
+      "logps/rejected": -1.1098216772079468,
+      "loss": 0.9455,
+      "odds_ratio_loss": 0.641205370426178,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.08813934028148651,
+      "rewards/margins": 0.02284284308552742,
+      "rewards/rejected": -0.11098217964172363,
+      "sft_loss": 0.881393313407898,
+      "step": 1570
+    },
+    {
+      "epoch": 2.809513225161147,
+      "grad_norm": 3.2441201210021973,
+      "learning_rate": 4.861220889427199e-08,
+      "logits/chosen": -2.8530640602111816,
+      "logits/rejected": -2.865347385406494,
+      "logps/chosen": -0.8542373776435852,
+      "logps/rejected": -1.0531483888626099,
+      "loss": 0.9195,
+      "odds_ratio_loss": 0.6527055501937866,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.08542371541261673,
+      "rewards/margins": 0.019891122356057167,
+      "rewards/rejected": -0.10531485080718994,
+      "sft_loss": 0.8542373776435852,
+      "step": 1580
+    },
+    {
+      "epoch": 2.827294954434319,
+      "grad_norm": 2.0825612545013428,
+      "learning_rate": 3.9896068346758074e-08,
+      "logits/chosen": -2.8629543781280518,
+      "logits/rejected": -2.887873649597168,
+      "logps/chosen": -0.8457245826721191,
+      "logps/rejected": -1.0121568441390991,
+      "loss": 0.9115,
+      "odds_ratio_loss": 0.6578742265701294,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.0845724567770958,
+      "rewards/margins": 0.016643229871988297,
+      "rewards/rejected": -0.10121568292379379,
+      "sft_loss": 0.8457245826721191,
+      "step": 1590
+    },
+    {
+      "epoch": 2.8450766837074903,
+      "grad_norm": 5.224942207336426,
+      "learning_rate": 3.203416211153832e-08,
+      "logits/chosen": -2.8476452827453613,
+      "logits/rejected": -2.893502712249756,
+      "logps/chosen": -0.843380331993103,
+      "logps/rejected": -1.0194734334945679,
+      "loss": 0.9131,
+      "odds_ratio_loss": 0.6969292163848877,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.08433803915977478,
+      "rewards/margins": 0.017609301954507828,
+      "rewards/rejected": -0.1019473448395729,
+      "sft_loss": 0.843380331993103,
+      "step": 1600
+    },
+    {
+      "epoch": 2.8628584129806622,
+      "grad_norm": 2.344214677810669,
+      "learning_rate": 2.5029220118019393e-08,
+      "logits/chosen": -2.8379952907562256,
+      "logits/rejected": -2.8665690422058105,
+      "logps/chosen": -0.8621314167976379,
+      "logps/rejected": -0.9904851913452148,
+      "loss": 0.929,
+      "odds_ratio_loss": 0.6684004664421082,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.08621314913034439,
+      "rewards/margins": 0.01283537782728672,
+      "rewards/rejected": -0.09904851764440536,
+      "sft_loss": 0.8621314167976379,
+      "step": 1610
+    },
+    {
+      "epoch": 2.880640142253834,
+      "grad_norm": 3.038189649581909,
+      "learning_rate": 1.8883674727586122e-08,
+      "logits/chosen": -2.868648052215576,
+      "logits/rejected": -2.876978635787964,
+      "logps/chosen": -0.8205513954162598,
+      "logps/rejected": -1.1327790021896362,
+      "loss": 0.8816,
+      "odds_ratio_loss": 0.6100881099700928,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.08205513656139374,
+      "rewards/margins": 0.031222760677337646,
+      "rewards/rejected": -0.11327788978815079,
+      "sft_loss": 0.8205513954162598,
+      "step": 1620
+    },
+    {
+      "epoch": 2.898421871527006,
+      "grad_norm": 1.2839456796646118,
+      "learning_rate": 1.3599659889000639e-08,
+      "logits/chosen": -2.8628990650177,
+      "logits/rejected": -2.881967067718506,
+      "logps/chosen": -0.7933380007743835,
+      "logps/rejected": -0.9434422254562378,
+      "loss": 0.8593,
+      "odds_ratio_loss": 0.6599572896957397,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.07933380454778671,
+      "rewards/margins": 0.015010423958301544,
+      "rewards/rejected": -0.09434423595666885,
+      "sft_loss": 0.7933380007743835,
+      "step": 1630
+    },
+    {
+      "epoch": 2.916203600800178,
+      "grad_norm": 1.2355436086654663,
+      "learning_rate": 9.179010397421528e-09,
+      "logits/chosen": -2.864149808883667,
+      "logits/rejected": -2.8793797492980957,
+      "logps/chosen": -0.8156276941299438,
+      "logps/rejected": -1.0415749549865723,
+      "loss": 0.8816,
+      "odds_ratio_loss": 0.659904956817627,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.08156277239322662,
+      "rewards/margins": 0.022594723850488663,
+      "rewards/rejected": -0.10415749251842499,
+      "sft_loss": 0.8156276941299438,
+      "step": 1640
+    },
+    {
+      "epoch": 2.93398533007335,
+      "grad_norm": 1.7059450149536133,
+      "learning_rate": 5.623261257296509e-09,
+      "logits/chosen": -2.8292007446289062,
+      "logits/rejected": -2.865652561187744,
+      "logps/chosen": -0.7724882364273071,
+      "logps/rejected": -0.9592376947402954,
+      "loss": 0.8358,
+      "odds_ratio_loss": 0.6333457231521606,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.07724882662296295,
+      "rewards/margins": 0.01867493987083435,
+      "rewards/rejected": -0.0959237664937973,
+      "sft_loss": 0.7724882364273071,
+      "step": 1650
+    },
+    {
+      "epoch": 2.9517670593465217,
+      "grad_norm": 2.2234692573547363,
+      "learning_rate": 2.933647149357122e-09,
+      "logits/chosen": -2.8548574447631836,
+      "logits/rejected": -2.8638224601745605,
+      "logps/chosen": -0.8041044473648071,
+      "logps/rejected": -1.1135035753250122,
+      "loss": 0.8679,
+      "odds_ratio_loss": 0.6383166909217834,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.0804104432463646,
+      "rewards/margins": 0.030939901247620583,
+      "rewards/rejected": -0.11135034263134003,
+      "sft_loss": 0.8041044473648071,
+      "step": 1660
+    },
+    {
+      "epoch": 2.969548788619693,
+      "grad_norm": 2.0396206378936768,
+      "learning_rate": 1.1111020018930717e-09,
+      "logits/chosen": -2.863762617111206,
+      "logits/rejected": -2.8730015754699707,
+      "logps/chosen": -0.8626953363418579,
+      "logps/rejected": -0.9779960513114929,
+      "loss": 0.9301,
+      "odds_ratio_loss": 0.6735736131668091,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.08626954257488251,
+      "rewards/margins": 0.011530078016221523,
+      "rewards/rejected": -0.09779961407184601,
+      "sft_loss": 0.8626953363418579,
+      "step": 1670
+    },
+    {
+      "epoch": 2.987330517892865,
+      "grad_norm": 1.7169232368469238,
+      "learning_rate": 1.5625866646051813e-10,
+      "logits/chosen": -2.8806686401367188,
+      "logits/rejected": -2.874525547027588,
+      "logps/chosen": -0.8173590898513794,
+      "logps/rejected": -1.051163911819458,
+      "loss": 0.8775,
+      "odds_ratio_loss": 0.600918173789978,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08173591643571854,
+      "rewards/margins": 0.0233804639428854,
+      "rewards/rejected": -0.10511638224124908,
+      "sft_loss": 0.8173590898513794,
+      "step": 1680
+    },
+    {
+      "epoch": 2.997999555456768,
+      "step": 1686,
+      "total_flos": 1.985199772705751e+18,
+      "train_loss": 0.9645641791862949,
+      "train_runtime": 17170.3123,
+      "train_samples_per_second": 1.572,
+      "train_steps_per_second": 0.098
     }
   ],
   "logging_steps": 10,
-  "max_steps": 168,
+  "max_steps": 1686,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 3,
   "save_steps": 500,
-  "total_flos": 2.303993975490478e+17,
+  "total_flos": 1.985199772705751e+18,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null