diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 4.0,
+  "epoch": 1.0,
   "eval_steps": 100,
-  "global_step": 1540,
+  "global_step": 385,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.5,
+      "loss": 0.5102,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,2537 +25,597 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.8667322397232056,
-      "logits/rejected": -1.8710299730300903,
-      "logps/chosen": -36.98433303833008,
-      "logps/rejected": -33.65640640258789,
-      "loss": 0.4948,
+      "logits/chosen": -1.866772174835205,
+      "logits/rejected": -1.8710780143737793,
+      "logps/chosen": -36.99943161010742,
+      "logps/rejected": -33.657470703125,
+      "loss": 0.4858,
       "rewards/accuracies": 0.5277777910232544,
-      "rewards/chosen": 0.015585740096867085,
-      "rewards/margins": 0.034019216895103455,
-      "rewards/rejected": -0.018433474004268646,
+      "rewards/chosen": 0.005016062408685684,
+      "rewards/margins": 0.02419126406311989,
+      "rewards/rejected": -0.019175197929143906,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9972655773162842,
-      "logits/rejected": -1.9999065399169922,
-      "logps/chosen": -29.6416072845459,
-      "logps/rejected": -29.046981811523438,
-      "loss": 0.5013,
+      "logits/chosen": -1.9975645542144775,
+      "logits/rejected": -2.0002217292785645,
+      "logps/chosen": -29.651592254638672,
+      "logps/rejected": -29.066898345947266,
+      "loss": 0.5542,
       "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.00041136163054034114,
-      "rewards/margins": -0.011113673448562622,
-      "rewards/rejected": 0.011525033973157406,
+      "rewards/chosen": -0.006579822860658169,
+      "rewards/margins": -0.004162783268839121,
+      "rewards/rejected": -0.0024170405231416225,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.9205191135406494,
-      "logits/rejected": -1.9178133010864258,
-      "logps/chosen": -31.385412216186523,
-      "logps/rejected": -33.23365020751953,
-      "loss": 0.4942,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.02147640287876129,
-      "rewards/margins": 0.03211076930165291,
-      "rewards/rejected": -0.010634368285536766,
+      "logits/chosen": -1.9212992191314697,
+      "logits/rejected": -1.918600082397461,
+      "logps/chosen": -31.42719078063965,
+      "logps/rejected": -33.23390579223633,
+      "loss": 0.5492,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.00776728754863143,
+      "rewards/margins": 0.0030429032631218433,
+      "rewards/rejected": -0.010810190811753273,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.017784833908081,
-      "logits/rejected": -2.009059429168701,
-      "logps/chosen": -32.547088623046875,
-      "logps/rejected": -32.473243713378906,
-      "loss": 0.5024,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.020800117403268814,
-      "rewards/margins": -0.006797379348427057,
-      "rewards/rejected": 0.027597496286034584,
+      "logits/chosen": -2.0178468227386475,
+      "logits/rejected": -2.009111166000366,
+      "logps/chosen": -32.579471588134766,
+      "logps/rejected": -32.50147247314453,
+      "loss": 0.5878,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.001865685684606433,
+      "rewards/margins": -0.009705344215035439,
+      "rewards/rejected": 0.007839656434953213,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.863996148109436,
-      "logits/rejected": -1.8532311916351318,
-      "logps/chosen": -33.490657806396484,
-      "logps/rejected": -35.415367126464844,
-      "loss": 0.4957,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.046776313334703445,
-      "rewards/margins": 0.02089320495724678,
-      "rewards/rejected": 0.025883108377456665,
+      "logits/chosen": -1.8617979288101196,
+      "logits/rejected": -1.8510282039642334,
+      "logps/chosen": -33.562862396240234,
+      "logps/rejected": -35.4472770690918,
+      "loss": 0.5775,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.0037666684947907925,
+      "rewards/margins": -0.007314275950193405,
+      "rewards/rejected": 0.0035476074554026127,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.9439821243286133,
-      "logits/rejected": -1.9459091424942017,
-      "logps/chosen": -32.50756072998047,
-      "logps/rejected": -33.15720748901367,
-      "loss": 0.489,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.06483479589223862,
-      "rewards/margins": 0.05491489917039871,
-      "rewards/rejected": 0.009919902309775352,
+      "logits/chosen": -1.9399009943008423,
+      "logits/rejected": -1.9418485164642334,
+      "logps/chosen": -32.5726203918457,
+      "logps/rejected": -33.18496322631836,
+      "loss": 0.5967,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.019294817000627518,
+      "rewards/margins": 0.028803948312997818,
+      "rewards/rejected": -0.009509134106338024,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.075462579727173,
-      "logits/rejected": -2.0804481506347656,
-      "logps/chosen": -33.86588668823242,
-      "logps/rejected": -36.501365661621094,
-      "loss": 0.491,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.08581922948360443,
-      "rewards/margins": 0.038563162088394165,
-      "rewards/rejected": 0.04725607484579086,
+      "logits/chosen": -2.0729470252990723,
+      "logits/rejected": -2.077918529510498,
+      "logps/chosen": -33.93268585205078,
+      "logps/rejected": -36.637229919433594,
+      "loss": 0.5297,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.039063483476638794,
+      "rewards/margins": 0.08691467344760895,
+      "rewards/rejected": -0.04785118252038956,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.9374949932098389,
-      "logits/rejected": -1.9405946731567383,
-      "logps/chosen": -34.200557708740234,
-      "logps/rejected": -34.543209075927734,
-      "loss": 0.4719,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.15907494723796844,
-      "rewards/margins": 0.1246359720826149,
-      "rewards/rejected": 0.03443896025419235,
+      "logits/chosen": -1.9366486072540283,
+      "logits/rejected": -1.939780592918396,
+      "logps/chosen": -34.30228805541992,
+      "logps/rejected": -34.652069091796875,
+      "loss": 0.4209,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.08786438405513763,
+      "rewards/margins": 0.12962770462036133,
+      "rewards/rejected": -0.041763320565223694,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.9453551769256592,
-      "logits/rejected": -1.9498746395111084,
-      "logps/chosen": -32.2825927734375,
-      "logps/rejected": -32.25676727294922,
-      "loss": 0.4858,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.12937995791435242,
-      "rewards/margins": 0.05788033455610275,
-      "rewards/rejected": 0.07149962335824966,
+      "logits/chosen": -1.946979284286499,
+      "logits/rejected": -1.951521635055542,
+      "logps/chosen": -32.4210205078125,
+      "logps/rejected": -32.337799072265625,
+      "loss": 0.6191,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.03248247504234314,
+      "rewards/margins": 0.017702888697385788,
+      "rewards/rejected": 0.014779585413634777,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.04329776763916,
-      "logits/rejected": -2.0412919521331787,
-      "logps/chosen": -32.015525817871094,
-      "logps/rejected": -31.169103622436523,
-      "loss": 0.4799,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.15771625936031342,
-      "rewards/margins": 0.08974156528711319,
-      "rewards/rejected": 0.06797470152378082,
+      "logits/chosen": -2.045017719268799,
+      "logits/rejected": -2.043015241622925,
+      "logps/chosen": -32.16361618041992,
+      "logps/rejected": -31.273128509521484,
+      "loss": 0.4711,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.05405203625559807,
+      "rewards/margins": 0.05889582633972168,
+      "rewards/rejected": -0.004843792412430048,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.238145351409912,
-      "eval_logits/rejected": -2.2333030700683594,
-      "eval_logps/chosen": -33.86069107055664,
-      "eval_logps/rejected": -37.356990814208984,
-      "eval_loss": 0.49784979224205017,
-      "eval_rewards/accuracies": 0.5340532064437866,
-      "eval_rewards/chosen": 0.12170296907424927,
-      "eval_rewards/margins": 0.009964452125132084,
-      "eval_rewards/rejected": 0.11173851788043976,
-      "eval_runtime": 145.9039,
-      "eval_samples_per_second": 2.351,
+      "eval_logits/chosen": -2.240097761154175,
+      "eval_logits/rejected": -2.2352383136749268,
+      "eval_logps/chosen": -34.01129150390625,
+      "eval_logps/rejected": -37.49793243408203,
+      "eval_loss": 0.5754798054695129,
+      "eval_rewards/accuracies": 0.5195183157920837,
+      "eval_rewards/chosen": 0.016279777511954308,
+      "eval_rewards/margins": 0.0031977586913853884,
+      "eval_rewards/rejected": 0.013082021847367287,
+      "eval_runtime": 145.9275,
+      "eval_samples_per_second": 2.35,
       "eval_steps_per_second": 0.295,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.9996016025543213,
-      "logits/rejected": -1.9972642660140991,
-      "logps/chosen": -32.95330810546875,
-      "logps/rejected": -33.88172912597656,
-      "loss": 0.4742,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.20335713028907776,
-      "rewards/margins": 0.10000836849212646,
-      "rewards/rejected": 0.10334876924753189,
+      "logits/chosen": -2.0017762184143066,
+      "logits/rejected": -1.9993689060211182,
+      "logps/chosen": -33.13564682006836,
+      "logps/rejected": -33.997642517089844,
+      "loss": 0.6759,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.07571890205144882,
+      "rewards/margins": 0.05350743606686592,
+      "rewards/rejected": 0.022211460396647453,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.0096044540405273,
-      "logits/rejected": -2.0012660026550293,
-      "logps/chosen": -32.153419494628906,
-      "logps/rejected": -31.961782455444336,
-      "loss": 0.4848,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.20403961837291718,
-      "rewards/margins": 0.06423385441303253,
-      "rewards/rejected": 0.13980577886104584,
+      "logits/chosen": -2.013158082962036,
+      "logits/rejected": -2.004814624786377,
+      "logps/chosen": -32.32666015625,
+      "logps/rejected": -32.12488555908203,
+      "loss": 0.5572,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.08277235180139542,
+      "rewards/margins": 0.05714136362075806,
+      "rewards/rejected": 0.02563098631799221,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.038412570953369,
-      "logits/rejected": -2.0304617881774902,
-      "logps/chosen": -30.165945053100586,
-      "logps/rejected": -31.880346298217773,
-      "loss": 0.4787,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.21683910489082336,
-      "rewards/margins": 0.0959777683019638,
-      "rewards/rejected": 0.12086131423711777,
+      "logits/chosen": -2.039836883544922,
+      "logits/rejected": -2.031825542449951,
+      "logps/chosen": -30.349987030029297,
+      "logps/rejected": -32.029136657714844,
+      "loss": 0.5109,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0880090743303299,
+      "rewards/margins": 0.07130132615566254,
+      "rewards/rejected": 0.016707751899957657,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.9683482646942139,
-      "logits/rejected": -1.9784959554672241,
-      "logps/chosen": -31.041671752929688,
-      "logps/rejected": -32.388893127441406,
-      "loss": 0.4621,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.264592707157135,
-      "rewards/margins": 0.16240297257900238,
-      "rewards/rejected": 0.10218973457813263,
+      "logits/chosen": -1.9694139957427979,
+      "logits/rejected": -1.9796711206436157,
+      "logps/chosen": -31.24441909790039,
+      "logps/rejected": -32.56440353393555,
+      "loss": 0.5037,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.12266886234283447,
+      "rewards/margins": 0.14333835244178772,
+      "rewards/rejected": -0.02066950313746929,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.8799880743026733,
-      "logits/rejected": -1.8811382055282593,
-      "logps/chosen": -33.686256408691406,
-      "logps/rejected": -34.564491271972656,
-      "loss": 0.4503,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.3545047640800476,
-      "rewards/margins": 0.21957096457481384,
-      "rewards/rejected": 0.134933739900589,
+      "logits/chosen": -1.8828538656234741,
+      "logits/rejected": -1.8840004205703735,
+      "logps/chosen": -34.006561279296875,
+      "logps/rejected": -34.81753921508789,
+      "loss": 0.4564,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.13028928637504578,
+      "rewards/margins": 0.17248663306236267,
+      "rewards/rejected": -0.04219735041260719,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.931627869606018,
-      "logits/rejected": -1.9282808303833008,
-      "logps/chosen": -35.77287292480469,
-      "logps/rejected": -32.49742889404297,
-      "loss": 0.4707,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.27916350960731506,
-      "rewards/margins": 0.12143567949533463,
-      "rewards/rejected": 0.15772780776023865,
+      "logits/chosen": -1.9349613189697266,
+      "logits/rejected": -1.9315464496612549,
+      "logps/chosen": -36.059181213378906,
+      "logps/rejected": -32.72267150878906,
+      "loss": 0.487,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.07874591648578644,
+      "rewards/margins": 0.07869003713130951,
+      "rewards/rejected": 5.587190389633179e-05,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.0332443714141846,
-      "logits/rejected": -2.0259475708007812,
-      "logps/chosen": -33.251670837402344,
-      "logps/rejected": -31.17826271057129,
-      "loss": 0.439,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.382335364818573,
-      "rewards/margins": 0.2608257234096527,
-      "rewards/rejected": 0.1215096265077591,
+      "logits/chosen": -2.036032199859619,
+      "logits/rejected": -2.028651714324951,
+      "logps/chosen": -33.573509216308594,
+      "logps/rejected": -31.36956214904785,
+      "loss": 0.4465,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.15704451501369476,
+      "rewards/margins": 0.16944527626037598,
+      "rewards/rejected": -0.012400749139487743,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.040196657180786,
-      "logits/rejected": -2.045389413833618,
-      "logps/chosen": -31.95781898498535,
-      "logps/rejected": -32.18687057495117,
-      "loss": 0.4516,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.4108788073062897,
-      "rewards/margins": 0.20208871364593506,
-      "rewards/rejected": 0.20879006385803223,
+      "logits/chosen": -2.042617082595825,
+      "logits/rejected": -2.0478556156158447,
+      "logps/chosen": -32.33074188232422,
+      "logps/rejected": -32.472496032714844,
+      "loss": 0.3727,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.14983226358890533,
+      "rewards/margins": 0.14098060131072998,
+      "rewards/rejected": 0.008851657621562481,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.040775775909424,
-      "logits/rejected": -2.038043737411499,
-      "logps/chosen": -31.056228637695312,
-      "logps/rejected": -31.10147476196289,
-      "loss": 0.4647,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.30290743708610535,
-      "rewards/margins": 0.1511872559785843,
-      "rewards/rejected": 0.15172019600868225,
+      "logits/chosen": -2.0445618629455566,
+      "logits/rejected": -2.0417864322662354,
+      "logps/chosen": -31.317855834960938,
+      "logps/rejected": -31.348400115966797,
+      "loss": 0.4609,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.11976919323205948,
+      "rewards/margins": 0.14090058207511902,
+      "rewards/rejected": -0.021131375804543495,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.9112863540649414,
-      "logits/rejected": -1.9159523248672485,
-      "logps/chosen": -31.08163070678711,
-      "logps/rejected": -32.598487854003906,
-      "loss": 0.4453,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.3672560155391693,
-      "rewards/margins": 0.2325470894575119,
-      "rewards/rejected": 0.1347089260816574,
+      "logits/chosen": -1.9139589071273804,
+      "logits/rejected": -1.9186115264892578,
+      "logps/chosen": -31.420101165771484,
+      "logps/rejected": -32.79216766357422,
+      "loss": 0.5061,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.13032536208629608,
+      "rewards/margins": 0.13119173049926758,
+      "rewards/rejected": -0.0008663811022415757,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.234708309173584,
-      "eval_logits/rejected": -2.229902505874634,
-      "eval_logps/chosen": -33.76850891113281,
-      "eval_logps/rejected": -37.29365921020508,
-      "eval_loss": 0.49275800585746765,
-      "eval_rewards/accuracies": 0.5369601845741272,
-      "eval_rewards/chosen": 0.18622951209545135,
-      "eval_rewards/margins": 0.030155813321471214,
-      "eval_rewards/rejected": 0.1560736745595932,
-      "eval_runtime": 145.5287,
-      "eval_samples_per_second": 2.357,
+      "eval_logits/chosen": -2.2385354042053223,
+      "eval_logits/rejected": -2.2336885929107666,
+      "eval_logps/chosen": -34.05002975463867,
+      "eval_logps/rejected": -37.54545974731445,
+      "eval_loss": 0.5877167582511902,
+      "eval_rewards/accuracies": 0.4991694688796997,
+      "eval_rewards/chosen": -0.010832725092768669,
+      "eval_rewards/margins": 0.009353035129606724,
+      "eval_rewards/rejected": -0.020185761153697968,
+      "eval_runtime": 145.6958,
+      "eval_samples_per_second": 2.354,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.022939443588257,
-      "logits/rejected": -2.033546209335327,
-      "logps/chosen": -31.5457706451416,
-      "logps/rejected": -33.67595672607422,
-      "loss": 0.4628,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.2978750169277191,
-      "rewards/margins": 0.16715146601200104,
-      "rewards/rejected": 0.13072356581687927,
+      "logits/chosen": -2.026780366897583,
+      "logits/rejected": -2.0374534130096436,
+      "logps/chosen": -31.786846160888672,
+      "logps/rejected": -33.922447204589844,
+      "loss": 0.4225,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.12911871075630188,
+      "rewards/margins": 0.17094141244888306,
+      "rewards/rejected": -0.041822709143161774,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.9158331155776978,
-      "logits/rejected": -1.9305419921875,
-      "logps/chosen": -29.57406997680664,
-      "logps/rejected": -31.42013168334961,
-      "loss": 0.4363,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.37622833251953125,
-      "rewards/margins": 0.2795966863632202,
-      "rewards/rejected": 0.09663165360689163,
+      "logits/chosen": -1.9182548522949219,
+      "logits/rejected": -1.933038353919983,
+      "logps/chosen": -29.936147689819336,
+      "logps/rejected": -31.559993743896484,
+      "loss": 0.4429,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.12277615070343018,
+      "rewards/margins": 0.12404797971248627,
+      "rewards/rejected": -0.0012718416983261704,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.9733517169952393,
-      "logits/rejected": -1.9773433208465576,
-      "logps/chosen": -32.83649444580078,
-      "logps/rejected": -31.40326499938965,
-      "loss": 0.4311,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.40668267011642456,
-      "rewards/margins": 0.304456502199173,
-      "rewards/rejected": 0.10222617536783218,
+      "logits/chosen": -1.9737575054168701,
+      "logits/rejected": -1.9777438640594482,
+      "logps/chosen": -33.20977783203125,
+      "logps/rejected": -31.5734920501709,
+      "loss": 0.3768,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.14538228511810303,
+      "rewards/margins": 0.16231416165828705,
+      "rewards/rejected": -0.016931891441345215,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.9713729619979858,
-      "logits/rejected": -1.9496545791625977,
-      "logps/chosen": -33.5602912902832,
-      "logps/rejected": -34.85026550292969,
-      "loss": 0.4227,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.41541916131973267,
-      "rewards/margins": 0.3450462222099304,
-      "rewards/rejected": 0.07037289440631866,
+      "logits/chosen": -1.9738047122955322,
+      "logits/rejected": -1.9518985748291016,
+      "logps/chosen": -33.94301223754883,
+      "logps/rejected": -35.023075103759766,
+      "loss": 0.4304,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.1475195735692978,
+      "rewards/margins": 0.19811663031578064,
+      "rewards/rejected": -0.05059707164764404,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.012058734893799,
-      "logits/rejected": -2.0087602138519287,
-      "logps/chosen": -32.46921157836914,
-      "logps/rejected": -36.012245178222656,
-      "loss": 0.4613,
+      "logits/chosen": -2.0155911445617676,
+      "logits/rejected": -2.0122878551483154,
+      "logps/chosen": -32.76276397705078,
+      "logps/rejected": -36.25225830078125,
+      "loss": 0.4155,
       "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.31910061836242676,
-      "rewards/margins": 0.17022350430488586,
-      "rewards/rejected": 0.1488770991563797,
+      "rewards/chosen": 0.11361332982778549,
+      "rewards/margins": 0.1327473670244217,
+      "rewards/rejected": -0.019134048372507095,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.878584861755371,
-      "logits/rejected": -1.876153588294983,
-      "logps/chosen": -33.71690368652344,
-      "logps/rejected": -35.26210403442383,
-      "loss": 0.4631,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.33004888892173767,
-      "rewards/margins": 0.1602855622768402,
-      "rewards/rejected": 0.16976332664489746,
+      "logits/chosen": -1.8830368518829346,
+      "logits/rejected": -1.880629539489746,
+      "logps/chosen": -34.01710891723633,
+      "logps/rejected": -35.53314971923828,
+      "loss": 0.4219,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.11990444362163544,
+      "rewards/margins": 0.1398761123418808,
+      "rewards/rejected": -0.019971664994955063,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.863705039024353,
-      "logits/rejected": -1.8611875772476196,
-      "logps/chosen": -33.94118881225586,
-      "logps/rejected": -31.579448699951172,
-      "loss": 0.4582,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.3194878101348877,
-      "rewards/margins": 0.18517015874385834,
-      "rewards/rejected": 0.13431766629219055,
+      "logits/chosen": -1.8685131072998047,
+      "logits/rejected": -1.8660367727279663,
+      "logps/chosen": -34.2363395690918,
+      "logps/rejected": -31.768321990966797,
+      "loss": 0.4918,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.11288031190633774,
+      "rewards/margins": 0.11077453196048737,
+      "rewards/rejected": 0.0021057710982859135,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.96773362159729,
-      "logits/rejected": -1.957313895225525,
-      "logps/chosen": -34.72475814819336,
-      "logps/rejected": -31.634963989257812,
-      "loss": 0.4297,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.4423200190067291,
-      "rewards/margins": 0.298100084066391,
-      "rewards/rejected": 0.14421990513801575,
+      "logits/chosen": -1.9722087383270264,
+      "logits/rejected": -1.961612343788147,
+      "logps/chosen": -35.04926300048828,
+      "logps/rejected": -31.826486587524414,
+      "loss": 0.3417,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.21516695618629456,
+      "rewards/margins": 0.20501188933849335,
+      "rewards/rejected": 0.010155050083994865,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.0632526874542236,
-      "logits/rejected": -2.0484511852264404,
-      "logps/chosen": -30.412399291992188,
-      "logps/rejected": -32.339237213134766,
-      "loss": 0.4682,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.3582096993923187,
-      "rewards/margins": 0.13939058780670166,
-      "rewards/rejected": 0.21881911158561707,
+      "logits/chosen": -2.068380832672119,
+      "logits/rejected": -2.05340313911438,
+      "logps/chosen": -30.748706817626953,
+      "logps/rejected": -32.63469314575195,
+      "loss": 0.4741,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.12279413640499115,
+      "rewards/margins": 0.11079368740320206,
+      "rewards/rejected": 0.012000440619885921,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9345197677612305,
-      "logits/rejected": -1.9320701360702515,
-      "logps/chosen": -32.11776351928711,
-      "logps/rejected": -30.692516326904297,
-      "loss": 0.3947,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.5665432810783386,
-      "rewards/margins": 0.4795381426811218,
-      "rewards/rejected": 0.08700509369373322,
+      "logits/chosen": -1.9397351741790771,
+      "logits/rejected": -1.9371974468231201,
+      "logps/chosen": -32.603660583496094,
+      "logps/rejected": -30.863941192626953,
+      "loss": 0.3371,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.2264125794172287,
+      "rewards/margins": 0.25940603017807007,
+      "rewards/rejected": -0.03299345448613167,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2321932315826416,
-      "eval_logits/rejected": -2.227381944656372,
-      "eval_logps/chosen": -33.75507736206055,
-      "eval_logps/rejected": -37.28935623168945,
-      "eval_loss": 0.49099090695381165,
-      "eval_rewards/accuracies": 0.5564784407615662,
-      "eval_rewards/chosen": 0.1956319361925125,
-      "eval_rewards/margins": 0.03654861077666283,
-      "eval_rewards/rejected": 0.15908333659172058,
-      "eval_runtime": 145.678,
-      "eval_samples_per_second": 2.355,
-      "eval_steps_per_second": 0.295,
+      "eval_logits/chosen": -2.2370686531066895,
+      "eval_logits/rejected": -2.2322022914886475,
+      "eval_logps/chosen": -34.03439712524414,
+      "eval_logps/rejected": -37.53529357910156,
+      "eval_loss": 0.5842657089233398,
+      "eval_rewards/accuracies": 0.5278239250183105,
+      "eval_rewards/chosen": 0.00010742468293756247,
+      "eval_rewards/margins": 0.013178782537579536,
+      "eval_rewards/rejected": -0.013071359135210514,
+      "eval_runtime": 145.5131,
+      "eval_samples_per_second": 2.357,
+      "eval_steps_per_second": 0.296,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "grad_norm": 5.65625,
-      "learning_rate": 4.84533120650964e-06,
-      "logits/chosen": -2.069279909133911,
-      "logits/rejected": -2.0565598011016846,
-      "logps/chosen": -31.766155242919922,
-      "logps/rejected": -32.60447311401367,
-      "loss": 0.3879,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.5335645079612732,
-      "rewards/margins": 0.5079580545425415,
-      "rewards/rejected": 0.025606412440538406,
+      "learning_rate": 5.576113578589035e-07,
+      "logits/chosen": -1.9219080209732056,
+      "logits/rejected": -1.9186557531356812,
+      "logps/chosen": -31.35263442993164,
+      "logps/rejected": -33.746925354003906,
+      "loss": 0.4182,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.17550265789031982,
+      "rewards/margins": 0.17884370684623718,
+      "rewards/rejected": -0.0033410280011594296,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "grad_norm": 6.9375,
-      "learning_rate": 4.825108134172131e-06,
-      "logits/chosen": -1.9792858362197876,
-      "logits/rejected": -1.9706569910049438,
-      "logps/chosen": -31.41892433166504,
-      "logps/rejected": -30.155033111572266,
-      "loss": 0.3743,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.6552442312240601,
-      "rewards/margins": 0.5947929620742798,
-      "rewards/rejected": 0.060451291501522064,
+      "learning_rate": 4.229036944380913e-07,
+      "logits/chosen": -1.9732515811920166,
+      "logits/rejected": -1.9609721899032593,
+      "logps/chosen": -34.435001373291016,
+      "logps/rejected": -33.593482971191406,
+      "loss": 0.3778,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.11274988949298859,
+      "rewards/margins": 0.15099774301052094,
+      "rewards/rejected": -0.03824785351753235,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "grad_norm": 8.1875,
-      "learning_rate": 4.80369052967602e-06,
-      "logits/chosen": -1.9177625179290771,
-      "logits/rejected": -1.9297670125961304,
-      "logps/chosen": -29.455638885498047,
-      "logps/rejected": -33.40266036987305,
-      "loss": 0.3501,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.7165408730506897,
-      "rewards/margins": 0.7168257832527161,
-      "rewards/rejected": -0.00028486884548328817,
+      "learning_rate": 3.053082288996112e-07,
+      "logits/chosen": -2.0085272789001465,
+      "logits/rejected": -2.0070912837982178,
+      "logps/chosen": -33.31962585449219,
+      "logps/rejected": -32.51466751098633,
+      "loss": 0.4061,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.1163158044219017,
+      "rewards/margins": 0.14087431132793427,
+      "rewards/rejected": -0.024558518081903458,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "grad_norm": 9.25,
-      "learning_rate": 4.781089396387968e-06,
-      "logits/chosen": -1.880617380142212,
-      "logits/rejected": -1.8715527057647705,
-      "logps/chosen": -33.60810089111328,
-      "logps/rejected": -35.90662384033203,
-      "loss": 0.3405,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.7635797262191772,
-      "rewards/margins": 0.7985055446624756,
-      "rewards/rejected": -0.03492575138807297,
+      "learning_rate": 2.0579377374915805e-07,
+      "logits/chosen": -2.0951385498046875,
+      "logits/rejected": -2.0793802738189697,
+      "logps/chosen": -33.88261032104492,
+      "logps/rejected": -33.10671615600586,
+      "loss": 0.4042,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.20774619281291962,
+      "rewards/margins": 0.19982238113880157,
+      "rewards/rejected": 0.00792380329221487,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "grad_norm": 5.1875,
-      "learning_rate": 4.757316345716554e-06,
-      "logits/chosen": -1.9337537288665771,
-      "logits/rejected": -1.9343976974487305,
-      "logps/chosen": -33.176937103271484,
-      "logps/rejected": -33.779075622558594,
-      "loss": 0.3349,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.8598405718803406,
-      "rewards/margins": 0.8109108805656433,
-      "rewards/rejected": 0.048929743468761444,
+      "learning_rate": 1.2518018074041684e-07,
+      "logits/chosen": -1.9677631855010986,
+      "logits/rejected": -1.9668546915054321,
+      "logps/chosen": -32.957977294921875,
+      "logps/rejected": -32.483890533447266,
+      "loss": 0.4874,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.2268829345703125,
+      "rewards/margins": 0.2400023490190506,
+      "rewards/rejected": -0.013119394890964031,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "grad_norm": 6.3125,
-      "learning_rate": 4.73238359114687e-06,
-      "logits/chosen": -2.060123920440674,
-      "logits/rejected": -2.066301107406616,
-      "logps/chosen": -30.665380477905273,
-      "logps/rejected": -32.581119537353516,
-      "loss": 0.3792,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.6355056762695312,
-      "rewards/margins": 0.5744917988777161,
-      "rewards/rejected": 0.061013948172330856,
+      "learning_rate": 6.41315865106129e-08,
+      "logits/chosen": -1.9235498905181885,
+      "logits/rejected": -1.933876395225525,
+      "logps/chosen": -32.016578674316406,
+      "logps/rejected": -35.32080841064453,
+      "loss": 0.4297,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.13167943060398102,
+      "rewards/margins": 0.15411263704299927,
+      "rewards/rejected": -0.02243318222463131,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "grad_norm": 7.78125,
-      "learning_rate": 4.706303941965804e-06,
-      "logits/chosen": -1.9897676706314087,
-      "logits/rejected": -1.989344835281372,
-      "logps/chosen": -32.362205505371094,
-      "logps/rejected": -35.91802215576172,
-      "loss": 0.3617,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.7579163908958435,
-      "rewards/margins": 0.6656854748725891,
-      "rewards/rejected": 0.09223100543022156,
+      "learning_rate": 2.3150941078050325e-08,
+      "logits/chosen": -2.062828540802002,
+      "logits/rejected": -2.056312322616577,
+      "logps/chosen": -33.43348693847656,
+      "logps/rejected": -29.213970184326172,
+      "loss": 0.3963,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.15293903648853302,
+      "rewards/margins": 0.14422088861465454,
+      "rewards/rejected": 0.008718125522136688,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "grad_norm": 5.71875,
-      "learning_rate": 4.679090796681225e-06,
-      "logits/chosen": -2.022163152694702,
-      "logits/rejected": -2.017575740814209,
-      "logps/chosen": -29.644603729248047,
-      "logps/rejected": -29.16855239868164,
-      "loss": 0.3514,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.746194064617157,
-      "rewards/margins": 0.714457094669342,
-      "rewards/rejected": 0.03173698112368584,
+      "learning_rate": 2.575864278703266e-09,
+      "logits/chosen": -1.9224169254302979,
+      "logits/rejected": -1.9245796203613281,
+      "logps/chosen": -33.9437255859375,
+      "logps/rejected": -30.860538482666016,
+      "loss": 0.4119,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.20672598481178284,
+      "rewards/margins": 0.19826285541057587,
+      "rewards/rejected": 0.008463130332529545,
       "step": 380
     },
     {
-      "epoch": 1.01,
-      "grad_norm": 8.0625,
-      "learning_rate": 4.650758136138454e-06,
-      "logits/chosen": -1.7921892404556274,
-      "logits/rejected": -1.7986253499984741,
-      "logps/chosen": -31.069808959960938,
-      "logps/rejected": -36.217376708984375,
-      "loss": 0.3052,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.9211344718933105,
-      "rewards/margins": 0.9904948472976685,
-      "rewards/rejected": -0.06936033815145493,
-      "step": 390
-    },
-    {
-      "epoch": 1.04,
-      "grad_norm": 5.84375,
-      "learning_rate": 4.621320516337559e-06,
-      "logits/chosen": -1.946427583694458,
-      "logits/rejected": -1.9403005838394165,
-      "logps/chosen": -32.449764251708984,
-      "logps/rejected": -32.27871322631836,
-      "loss": 0.3136,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.968052864074707,
-      "rewards/margins": 0.9469360113143921,
-      "rewards/rejected": 0.021116863936185837,
-      "step": 400
-    },
-    {
-      "epoch": 1.04,
-      "eval_logits/chosen": -2.2079687118530273,
-      "eval_logits/rejected": -2.2031569480895996,
-      "eval_logps/chosen": -33.62800598144531,
-      "eval_logps/rejected": -37.196067810058594,
-      "eval_loss": 0.48570817708969116,
-      "eval_rewards/accuracies": 0.579734206199646,
-      "eval_rewards/chosen": 0.2845819294452667,
-      "eval_rewards/margins": 0.06019504740834236,
-      "eval_rewards/rejected": 0.22438688576221466,
-      "eval_runtime": 145.4274,
-      "eval_samples_per_second": 2.359,
-      "eval_steps_per_second": 0.296,
-      "step": 400
-    },
-    {
-      "epoch": 1.06,
-      "grad_norm": 5.96875,
-      "learning_rate": 4.590793060955158e-06,
-      "logits/chosen": -1.9444068670272827,
-      "logits/rejected": -1.951677918434143,
-      "logps/chosen": -28.08418846130371,
-      "logps/rejected": -29.256505966186523,
-      "loss": 0.3391,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.7253325581550598,
-      "rewards/margins": 0.836132824420929,
-      "rewards/rejected": -0.11080024391412735,
-      "step": 410
-    },
-    {
-      "epoch": 1.09,
-      "grad_norm": 5.4375,
-      "learning_rate": 4.559191453574582e-06,
-      "logits/chosen": -1.966160774230957,
-      "logits/rejected": -1.965192437171936,
-      "logps/chosen": -32.974876403808594,
-      "logps/rejected": -30.77309226989746,
-      "loss": 0.3464,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.8930822610855103,
-      "rewards/margins": 0.774114727973938,
-      "rewards/rejected": 0.11896754801273346,
-      "step": 420
-    },
-    {
-      "epoch": 1.12,
-      "grad_norm": 6.09375,
-      "learning_rate": 4.52653192962838e-06,
-      "logits/chosen": -1.9592077732086182,
-      "logits/rejected": -1.9421688318252563,
-      "logps/chosen": -29.91006851196289,
-      "logps/rejected": -33.13286590576172,
-      "loss": 0.3285,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.7469960451126099,
-      "rewards/margins": 0.8916017413139343,
-      "rewards/rejected": -0.14460588991641998,
-      "step": 430
-    },
-    {
-      "epoch": 1.14,
-      "grad_norm": 5.3125,
-      "learning_rate": 4.492831268057307e-06,
-      "logits/chosen": -1.989438772201538,
-      "logits/rejected": -1.9913489818572998,
-      "logps/chosen": -35.05497741699219,
-      "logps/rejected": -34.920108795166016,
-      "loss": 0.2832,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 1.0901695489883423,
-      "rewards/margins": 1.1322667598724365,
-      "rewards/rejected": -0.0420973002910614,
-      "step": 440
-    },
-    {
-      "epoch": 1.17,
-      "grad_norm": 6.5,
-      "learning_rate": 4.458106782690094e-06,
-      "logits/chosen": -2.0655436515808105,
-      "logits/rejected": -2.0654349327087402,
-      "logps/chosen": -31.32794761657715,
-      "logps/rejected": -33.56629180908203,
-      "loss": 0.3287,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.9137799143791199,
-      "rewards/margins": 0.9181026220321655,
-      "rewards/rejected": -0.004322645254433155,
-      "step": 450
-    },
-    {
-      "epoch": 1.19,
-      "grad_norm": 5.28125,
-      "learning_rate": 4.422376313348405e-06,
-      "logits/chosen": -2.0107359886169434,
-      "logits/rejected": -2.0033280849456787,
-      "logps/chosen": -30.83009910583496,
-      "logps/rejected": -36.22407150268555,
-      "loss": 0.2765,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 1.1031007766723633,
-      "rewards/margins": 1.2221224308013916,
-      "rewards/rejected": -0.11902163922786713,
-      "step": 460
-    },
-    {
-      "epoch": 1.22,
-      "grad_norm": 6.28125,
-      "learning_rate": 4.3856582166815696e-06,
-      "logits/chosen": -1.9149672985076904,
-      "logits/rejected": -1.9115540981292725,
-      "logps/chosen": -32.367496490478516,
-      "logps/rejected": -32.911216735839844,
-      "loss": 0.2962,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 1.0953609943389893,
-      "rewards/margins": 1.0864982604980469,
-      "rewards/rejected": 0.008862635120749474,
-      "step": 470
-    },
-    {
-      "epoch": 1.25,
-      "grad_norm": 5.5,
-      "learning_rate": 4.347971356735789e-06,
-      "logits/chosen": -2.040391445159912,
-      "logits/rejected": -2.033470630645752,
-      "logps/chosen": -29.63046646118164,
-      "logps/rejected": -32.03357696533203,
-      "loss": 0.3336,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.8386493921279907,
-      "rewards/margins": 0.8667130470275879,
-      "rewards/rejected": -0.028063708916306496,
-      "step": 480
-    },
-    {
-      "epoch": 1.27,
-      "grad_norm": 5.59375,
-      "learning_rate": 4.309335095262675e-06,
-      "logits/chosen": -1.9858951568603516,
-      "logits/rejected": -1.987535834312439,
-      "logps/chosen": -33.76915740966797,
-      "logps/rejected": -33.62469482421875,
-      "loss": 0.2834,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 1.108822226524353,
-      "rewards/margins": 1.0970509052276611,
-      "rewards/rejected": 0.011771360412240028,
-      "step": 490
-    },
-    {
-      "epoch": 1.3,
-      "grad_norm": 6.375,
-      "learning_rate": 4.269769281772082e-06,
-      "logits/chosen": -1.8744853734970093,
-      "logits/rejected": -1.8722003698349,
-      "logps/chosen": -31.54471206665039,
-      "logps/rejected": -36.603973388671875,
-      "loss": 0.2784,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 1.1393038034439087,
-      "rewards/margins": 1.2370599508285522,
-      "rewards/rejected": -0.09775615483522415,
-      "step": 500
-    },
-    {
-      "epoch": 1.3,
-      "eval_logits/chosen": -2.2098300457000732,
-      "eval_logits/rejected": -2.205024480819702,
-      "eval_logps/chosen": -33.61191177368164,
-      "eval_logps/rejected": -37.15669250488281,
-      "eval_loss": 0.48908427357673645,
-      "eval_rewards/accuracies": 0.5220099687576294,
-      "eval_rewards/chosen": 0.29585084319114685,
-      "eval_rewards/margins": 0.04390101507306099,
-      "eval_rewards/rejected": 0.25194987654685974,
-      "eval_runtime": 145.3523,
-      "eval_samples_per_second": 2.36,
-      "eval_steps_per_second": 0.296,
-      "step": 500
-    },
-    {
-      "epoch": 1.32,
-      "grad_norm": 4.75,
-      "learning_rate": 4.22929424333435e-06,
-      "logits/chosen": -1.978362798690796,
-      "logits/rejected": -1.98308527469635,
-      "logps/chosen": -31.69674301147461,
-      "logps/rejected": -31.60154151916504,
-      "loss": 0.2762,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 1.2324289083480835,
-      "rewards/margins": 1.2012327909469604,
-      "rewards/rejected": 0.03119627758860588,
-      "step": 510
-    },
-    {
-      "epoch": 1.35,
-      "grad_norm": 5.5625,
-      "learning_rate": 4.1879307741372085e-06,
-      "logits/chosen": -2.0070841312408447,
-      "logits/rejected": -2.01804256439209,
-      "logps/chosen": -29.7496280670166,
-      "logps/rejected": -31.527536392211914,
-      "loss": 0.2821,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 1.148550271987915,
-      "rewards/margins": 1.16244375705719,
-      "rewards/rejected": -0.013893580064177513,
-      "step": 520
-    },
-    {
-      "epoch": 1.38,
-      "grad_norm": 4.09375,
-      "learning_rate": 4.145700124802693e-06,
-      "logits/chosen": -1.9333133697509766,
-      "logits/rejected": -1.9299663305282593,
-      "logps/chosen": -30.964946746826172,
-      "logps/rejected": -32.423377990722656,
-      "loss": 0.2962,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 1.0188950300216675,
-      "rewards/margins": 1.061314344406128,
-      "rewards/rejected": -0.0424191839993,
-      "step": 530
-    },
-    {
-      "epoch": 1.4,
-      "grad_norm": 6.15625,
-      "learning_rate": 4.102623991469562e-06,
-      "logits/chosen": -1.7982631921768188,
-      "logits/rejected": -1.8074582815170288,
-      "logps/chosen": -30.847604751586914,
-      "logps/rejected": -31.964679718017578,
-      "loss": 0.2773,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 1.205248475074768,
-      "rewards/margins": 1.2813327312469482,
-      "rewards/rejected": -0.07608427852392197,
-      "step": 540
-    },
-    {
-      "epoch": 1.43,
-      "grad_norm": 5.40625,
-      "learning_rate": 4.058724504646834e-06,
-      "logits/chosen": -1.8933544158935547,
-      "logits/rejected": -1.8870652914047241,
-      "logps/chosen": -31.835384368896484,
-      "logps/rejected": -30.871362686157227,
-      "loss": 0.2807,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 1.3110452890396118,
-      "rewards/margins": 1.2270557880401611,
-      "rewards/rejected": 0.08398932963609695,
-      "step": 550
-    },
-    {
-      "epoch": 1.45,
-      "grad_norm": 5.15625,
-      "learning_rate": 4.014024217844167e-06,
-      "logits/chosen": -1.9839521646499634,
-      "logits/rejected": -1.9819648265838623,
-      "logps/chosen": -32.755332946777344,
-      "logps/rejected": -31.447025299072266,
-      "loss": 0.2809,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 1.271106481552124,
-      "rewards/margins": 1.2358547449111938,
-      "rewards/rejected": 0.035251684486866,
-      "step": 560
-    },
-    {
-      "epoch": 1.48,
-      "grad_norm": 6.34375,
-      "learning_rate": 3.968546095984911e-06,
-      "logits/chosen": -1.8162040710449219,
-      "logits/rejected": -1.814014196395874,
-      "logps/chosen": -31.14450454711914,
-      "logps/rejected": -30.988576889038086,
-      "loss": 0.2898,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 1.1563327312469482,
-      "rewards/margins": 1.146722435951233,
-      "rewards/rejected": 0.009610283188521862,
-      "step": 570
-    },
-    {
-      "epoch": 1.51,
-      "grad_norm": 5.5625,
-      "learning_rate": 3.922313503607806e-06,
-      "logits/chosen": -1.95193612575531,
-      "logits/rejected": -1.948587417602539,
-      "logps/chosen": -29.390304565429688,
-      "logps/rejected": -34.86743927001953,
-      "loss": 0.2599,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 1.2000386714935303,
-      "rewards/margins": 1.3689311742782593,
-      "rewards/rejected": -0.16889244318008423,
-      "step": 580
-    },
-    {
-      "epoch": 1.53,
-      "grad_norm": 5.1875,
-      "learning_rate": 3.875350192863368e-06,
-      "logits/chosen": -1.8900426626205444,
-      "logits/rejected": -1.8936386108398438,
-      "logps/chosen": -28.26003646850586,
-      "logps/rejected": -30.714879989624023,
-      "loss": 0.3068,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.9484975934028625,
-      "rewards/margins": 0.9877050518989563,
-      "rewards/rejected": -0.03920745849609375,
-      "step": 590
-    },
-    {
-      "epoch": 1.56,
-      "grad_norm": 6.03125,
-      "learning_rate": 3.8276802913111436e-06,
-      "logits/chosen": -1.9284902811050415,
-      "logits/rejected": -1.9283053874969482,
-      "logps/chosen": -30.269977569580078,
-      "logps/rejected": -31.168771743774414,
-      "loss": 0.2593,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 1.3660885095596313,
-      "rewards/margins": 1.3491170406341553,
-      "rewards/rejected": 0.016971405595541,
-      "step": 600
-    },
-    {
-      "epoch": 1.56,
-      "eval_logits/chosen": -2.1913671493530273,
-      "eval_logits/rejected": -2.186584949493408,
-      "eval_logps/chosen": -33.55670166015625,
-      "eval_logps/rejected": -37.16816711425781,
-      "eval_loss": 0.4795128405094147,
-      "eval_rewards/accuracies": 0.574335515499115,
-      "eval_rewards/chosen": 0.3344981372356415,
-      "eval_rewards/margins": 0.09058075398206711,
-      "eval_rewards/rejected": 0.24391737580299377,
-      "eval_runtime": 145.1862,
-      "eval_samples_per_second": 2.362,
-      "eval_steps_per_second": 0.296,
-      "step": 600
-    },
-    {
-      "epoch": 1.58,
-      "grad_norm": 5.625,
-      "learning_rate": 3.7793282895240927e-06,
-      "logits/chosen": -1.9897689819335938,
-      "logits/rejected": -1.9904531240463257,
-      "logps/chosen": -32.8941650390625,
-      "logps/rejected": -33.11201477050781,
-      "loss": 0.2422,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 1.4951863288879395,
-      "rewards/margins": 1.5146669149398804,
-      "rewards/rejected": -0.019480543211102486,
-      "step": 610
-    },
-    {
-      "epoch": 1.61,
-      "grad_norm": 3.875,
-      "learning_rate": 3.730319028506478e-06,
-      "logits/chosen": -1.953918695449829,
-      "logits/rejected": -1.9515819549560547,
-      "logps/chosen": -31.27206039428711,
-      "logps/rejected": -32.174869537353516,
-      "loss": 0.2593,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.297335147857666,
-      "rewards/margins": 1.3708014488220215,
-      "rewards/rejected": -0.07346634566783905,
-      "step": 620
-    },
-    {
-      "epoch": 1.64,
-      "grad_norm": 6.9375,
-      "learning_rate": 3.6806776869317074e-06,
-      "logits/chosen": -1.967087745666504,
-      "logits/rejected": -1.9582030773162842,
-      "logps/chosen": -30.80006980895996,
-      "logps/rejected": -31.055767059326172,
-      "loss": 0.2573,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 1.3832311630249023,
-      "rewards/margins": 1.4675111770629883,
-      "rewards/rejected": -0.08427979052066803,
-      "step": 630
-    },
-    {
-      "epoch": 1.66,
-      "grad_norm": 6.21875,
-      "learning_rate": 3.6304297682067146e-06,
-      "logits/chosen": -1.9678598642349243,
-      "logits/rejected": -1.9646453857421875,
-      "logps/chosen": -30.486230850219727,
-      "logps/rejected": -32.526588439941406,
-      "loss": 0.2662,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.1985342502593994,
-      "rewards/margins": 1.2735540866851807,
-      "rewards/rejected": -0.07501962780952454,
-      "step": 640
-    },
-    {
-      "epoch": 1.69,
-      "grad_norm": 5.25,
-      "learning_rate": 3.579601087369492e-06,
-      "logits/chosen": -1.9753246307373047,
-      "logits/rejected": -1.9776771068572998,
-      "logps/chosen": -31.804119110107422,
-      "logps/rejected": -33.76448440551758,
-      "loss": 0.2474,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.2947121858596802,
-      "rewards/margins": 1.3514697551727295,
-      "rewards/rejected": -0.056757647544145584,
-      "step": 650
-    },
-    {
-      "epoch": 1.71,
-      "grad_norm": 7.84375,
-      "learning_rate": 3.5282177578265295e-06,
-      "logits/chosen": -1.8884271383285522,
-      "logits/rejected": -1.8889974355697632,
-      "logps/chosen": -31.999134063720703,
-      "logps/rejected": -31.639989852905273,
-      "loss": 0.2419,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.363389253616333,
-      "rewards/margins": 1.3863054513931274,
-      "rewards/rejected": -0.022916005924344063,
-      "step": 660
-    },
-    {
-      "epoch": 1.74,
-      "grad_norm": 6.15625,
-      "learning_rate": 3.476306177936961e-06,
-      "logits/chosen": -1.9295670986175537,
-      "logits/rejected": -1.920019507408142,
-      "logps/chosen": -32.041465759277344,
-      "logps/rejected": -32.48603057861328,
-      "loss": 0.2352,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.3718717098236084,
-      "rewards/margins": 1.4966375827789307,
-      "rewards/rejected": -0.1247657760977745,
-      "step": 670
-    },
-    {
-      "epoch": 1.77,
-      "grad_norm": 3.453125,
-      "learning_rate": 3.423893017450324e-06,
-      "logits/chosen": -1.827956199645996,
-      "logits/rejected": -1.8249555826187134,
-      "logps/chosen": -29.395336151123047,
-      "logps/rejected": -34.349639892578125,
-      "loss": 0.2442,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.445934534072876,
-      "rewards/margins": 1.4676737785339355,
-      "rewards/rejected": -0.021739140152931213,
-      "step": 680
-    },
-    {
-      "epoch": 1.79,
-      "grad_norm": 4.75,
-      "learning_rate": 3.3710052038048794e-06,
-      "logits/chosen": -1.8876937627792358,
-      "logits/rejected": -1.8869459629058838,
-      "logps/chosen": -32.98918151855469,
-      "logps/rejected": -35.476680755615234,
-      "loss": 0.2165,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.5967752933502197,
-      "rewards/margins": 1.6827762126922607,
-      "rewards/rejected": -0.08600117266178131,
-      "step": 690
-    },
-    {
-      "epoch": 1.82,
-      "grad_norm": 4.15625,
-      "learning_rate": 3.3176699082935546e-06,
-      "logits/chosen": -1.862781286239624,
-      "logits/rejected": -1.8658040761947632,
-      "logps/chosen": -30.54743003845215,
-      "logps/rejected": -35.61197280883789,
-      "loss": 0.2606,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 1.4387942552566528,
-      "rewards/margins": 1.418005108833313,
-      "rewards/rejected": 0.020789090543985367,
-      "step": 700
-    },
-    {
-      "epoch": 1.82,
-      "eval_logits/chosen": -2.1835741996765137,
-      "eval_logits/rejected": -2.1788299083709717,
-      "eval_logps/chosen": -33.57905578613281,
-      "eval_logps/rejected": -37.20835876464844,
-      "eval_loss": 0.47641801834106445,
-      "eval_rewards/accuracies": 0.6063122749328613,
-      "eval_rewards/chosen": 0.3188474476337433,
-      "eval_rewards/margins": 0.10306696593761444,
-      "eval_rewards/rejected": 0.21578048169612885,
-      "eval_runtime": 145.3074,
-      "eval_samples_per_second": 2.361,
-      "eval_steps_per_second": 0.296,
-      "step": 700
-    },
-    {
-      "epoch": 1.84,
-      "grad_norm": 5.8125,
-      "learning_rate": 3.2639145321045933e-06,
-      "logits/chosen": -1.970207929611206,
-      "logits/rejected": -1.972961187362671,
-      "logps/chosen": -32.8425407409668,
-      "logps/rejected": -34.31882858276367,
-      "loss": 0.2562,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 1.4371341466903687,
-      "rewards/margins": 1.471206784248352,
-      "rewards/rejected": -0.034072600305080414,
-      "step": 710
-    },
-    {
-      "epoch": 1.87,
-      "grad_norm": 6.8125,
-      "learning_rate": 3.2097666922441107e-06,
-      "logits/chosen": -1.8238359689712524,
-      "logits/rejected": -1.8179490566253662,
-      "logps/chosen": -32.64445877075195,
-      "logps/rejected": -32.34913635253906,
-      "loss": 0.2397,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 1.5321400165557861,
-      "rewards/margins": 1.530247449874878,
-      "rewards/rejected": 0.0018926591146737337,
-      "step": 720
-    },
-    {
-      "epoch": 1.9,
-      "grad_norm": 3.765625,
-      "learning_rate": 3.1552542073477554e-06,
-      "logits/chosen": -1.996241569519043,
-      "logits/rejected": -1.9933366775512695,
-      "logps/chosen": -28.67278480529785,
-      "logps/rejected": -31.668869018554688,
-      "loss": 0.2416,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.3861420154571533,
-      "rewards/margins": 1.5406121015548706,
-      "rewards/rejected": -0.1544700562953949,
-      "step": 730
-    },
-    {
-      "epoch": 1.92,
-      "grad_norm": 3.875,
-      "learning_rate": 3.100405083388799e-06,
-      "logits/chosen": -1.8369905948638916,
-      "logits/rejected": -1.8370345830917358,
-      "logps/chosen": -31.223901748657227,
-      "logps/rejected": -37.633975982666016,
-      "loss": 0.2245,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 1.5674915313720703,
-      "rewards/margins": 1.571808099746704,
-      "rewards/rejected": -0.004316648934036493,
-      "step": 740
-    },
-    {
-      "epoch": 1.95,
-      "grad_norm": 3.1875,
-      "learning_rate": 3.0452474992899645e-06,
-      "logits/chosen": -1.7186479568481445,
-      "logits/rejected": -1.7238624095916748,
-      "logps/chosen": -34.5988655090332,
-      "logps/rejected": -33.955909729003906,
-      "loss": 0.2323,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 1.6335166692733765,
-      "rewards/margins": 1.6220157146453857,
-      "rewards/rejected": 0.011500892229378223,
-      "step": 750
-    },
-    {
-      "epoch": 1.97,
-      "grad_norm": 5.625,
-      "learning_rate": 2.989809792446417e-06,
-      "logits/chosen": -1.9235740900039673,
-      "logits/rejected": -1.9250673055648804,
-      "logps/chosen": -30.627086639404297,
-      "logps/rejected": -32.82661437988281,
-      "loss": 0.2421,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 1.385018229484558,
-      "rewards/margins": 1.4181984663009644,
-      "rewards/rejected": -0.03318030759692192,
-      "step": 760
-    },
-    {
-      "epoch": 2.0,
-      "grad_norm": 5.34375,
-      "learning_rate": 2.9341204441673267e-06,
-      "logits/chosen": -1.894831895828247,
-      "logits/rejected": -1.8939815759658813,
-      "logps/chosen": -30.067148208618164,
-      "logps/rejected": -34.84111785888672,
-      "loss": 0.2551,
-      "rewards/accuracies": 0.85833340883255,
-      "rewards/chosen": 1.4759252071380615,
-      "rewards/margins": 1.3449987173080444,
-      "rewards/rejected": 0.13092657923698425,
-      "step": 770
-    },
-    {
-      "epoch": 2.03,
-      "grad_norm": 3.671875,
-      "learning_rate": 2.878208065043501e-06,
-      "logits/chosen": -1.895990014076233,
-      "logits/rejected": -1.8953578472137451,
-      "logps/chosen": -32.30732727050781,
-      "logps/rejected": -31.775619506835938,
-      "loss": 0.1909,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.6387954950332642,
-      "rewards/margins": 1.8880647420883179,
-      "rewards/rejected": -0.24926936626434326,
-      "step": 780
-    },
-    {
-      "epoch": 2.05,
-      "grad_norm": 4.90625,
-      "learning_rate": 2.8221013802485974e-06,
-      "logits/chosen": -1.9268324375152588,
-      "logits/rejected": -1.9253270626068115,
-      "logps/chosen": -27.4288387298584,
-      "logps/rejected": -33.15345764160156,
-      "loss": 0.1931,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.4940404891967773,
-      "rewards/margins": 1.8560407161712646,
-      "rewards/rejected": -0.36200040578842163,
-      "step": 790
-    },
-    {
-      "epoch": 2.08,
-      "grad_norm": 3.78125,
-      "learning_rate": 2.76582921478147e-06,
-      "logits/chosen": -1.9773414134979248,
-      "logits/rejected": -1.974020004272461,
-      "logps/chosen": -29.959461212158203,
-      "logps/rejected": -34.62144470214844,
-      "loss": 0.1758,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.7274243831634521,
-      "rewards/margins": 2.023111581802368,
-      "rewards/rejected": -0.2956870496273041,
-      "step": 800
-    },
-    {
-      "epoch": 2.08,
-      "eval_logits/chosen": -2.172740936279297,
-      "eval_logits/rejected": -2.1680266857147217,
-      "eval_logps/chosen": -33.62887191772461,
-      "eval_logps/rejected": -37.266761779785156,
-      "eval_loss": 0.4767366945743561,
-      "eval_rewards/accuracies": 0.5859634280204773,
-      "eval_rewards/chosen": 0.28397703170776367,
-      "eval_rewards/margins": 0.10907654464244843,
-      "eval_rewards/rejected": 0.17490047216415405,
-      "eval_runtime": 145.1754,
-      "eval_samples_per_second": 2.363,
-      "eval_steps_per_second": 0.296,
-      "step": 800
-    },
-    {
-      "epoch": 2.1,
-      "grad_norm": 2.46875,
-      "learning_rate": 2.7094204786572254e-06,
-      "logits/chosen": -1.8107726573944092,
-      "logits/rejected": -1.8030484914779663,
-      "logps/chosen": -31.506906509399414,
-      "logps/rejected": -35.04960250854492,
-      "loss": 0.1452,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 2.0645337104797363,
-      "rewards/margins": 2.407174587249756,
-      "rewards/rejected": -0.3426409363746643,
-      "step": 810
-    },
-    {
-      "epoch": 2.13,
-      "grad_norm": 4.40625,
-      "learning_rate": 2.6529041520546072e-06,
-      "logits/chosen": -1.887786865234375,
-      "logits/rejected": -1.8982326984405518,
-      "logps/chosen": -33.01645278930664,
-      "logps/rejected": -32.74128341674805,
-      "loss": 0.1899,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.8754699230194092,
-      "rewards/margins": 2.0923609733581543,
-      "rewards/rejected": -0.2168913334608078,
-      "step": 820
-    },
-    {
-      "epoch": 2.16,
-      "grad_norm": 3.796875,
-      "learning_rate": 2.5963092704273302e-06,
-      "logits/chosen": -1.9369271993637085,
-      "logits/rejected": -1.9413642883300781,
-      "logps/chosen": -32.51754379272461,
-      "logps/rejected": -29.556041717529297,
-      "loss": 0.1869,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.6696659326553345,
-      "rewards/margins": 1.8711254596710205,
-      "rewards/rejected": -0.20145969092845917,
-      "step": 830
-    },
-    {
-      "epoch": 2.18,
-      "grad_norm": 4.46875,
-      "learning_rate": 2.53966490958702e-06,
-      "logits/chosen": -1.9355465173721313,
-      "logits/rejected": -1.9433647394180298,
-      "logps/chosen": -32.05677795410156,
-      "logps/rejected": -30.385608673095703,
-      "loss": 0.1604,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.911877989768982,
-      "rewards/margins": 2.2884128093719482,
-      "rewards/rejected": -0.3765348792076111,
-      "step": 840
-    },
-    {
-      "epoch": 2.21,
-      "grad_norm": 4.625,
-      "learning_rate": 2.4830001707654135e-06,
-      "logits/chosen": -1.866355538368225,
-      "logits/rejected": -1.8570215702056885,
-      "logps/chosen": -29.2946720123291,
-      "logps/rejected": -32.352394104003906,
-      "loss": 0.1771,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.7489668130874634,
-      "rewards/margins": 1.9859797954559326,
-      "rewards/rejected": -0.23701317608356476,
-      "step": 850
-    },
-    {
-      "epoch": 2.23,
-      "grad_norm": 3.6875,
-      "learning_rate": 2.4263441656635054e-06,
-      "logits/chosen": -2.0053553581237793,
-      "logits/rejected": -1.9958158731460571,
-      "logps/chosen": -23.93155288696289,
-      "logps/rejected": -30.337451934814453,
-      "loss": 0.1938,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.604762077331543,
-      "rewards/margins": 1.9303429126739502,
-      "rewards/rejected": -0.3255811333656311,
-      "step": 860
-    },
-    {
-      "epoch": 2.26,
-      "grad_norm": 3.4375,
-      "learning_rate": 2.3697260014953107e-06,
-      "logits/chosen": -1.8573694229125977,
-      "logits/rejected": -1.8587795495986938,
-      "logps/chosen": -31.69219398498535,
-      "logps/rejected": -30.3853816986084,
-      "loss": 0.1718,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.8485662937164307,
-      "rewards/margins": 2.130580186843872,
-      "rewards/rejected": -0.28201407194137573,
-      "step": 870
-    },
-    {
-      "epoch": 2.29,
-      "grad_norm": 3.1875,
-      "learning_rate": 2.3131747660339396e-06,
-      "logits/chosen": -1.8693485260009766,
-      "logits/rejected": -1.8701515197753906,
-      "logps/chosen": -29.962982177734375,
-      "logps/rejected": -33.51511764526367,
-      "loss": 0.1879,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 1.6983861923217773,
-      "rewards/margins": 2.0233235359191895,
-      "rewards/rejected": -0.3249376118183136,
-      "step": 880
-    },
-    {
-      "epoch": 2.31,
-      "grad_norm": 3.25,
-      "learning_rate": 2.256719512667651e-06,
-      "logits/chosen": -1.7684412002563477,
-      "logits/rejected": -1.7669271230697632,
-      "logps/chosen": -32.97483825683594,
-      "logps/rejected": -36.623138427734375,
-      "loss": 0.1579,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.8830540180206299,
-      "rewards/margins": 2.405186176300049,
-      "rewards/rejected": -0.522132396697998,
-      "step": 890
-    },
-    {
-      "epoch": 2.34,
-      "grad_norm": 3.296875,
-      "learning_rate": 2.2003892454735786e-06,
-      "logits/chosen": -1.9180320501327515,
-      "logits/rejected": -1.9112794399261475,
-      "logps/chosen": -29.328502655029297,
-      "logps/rejected": -33.22010803222656,
-      "loss": 0.1687,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.8718944787979126,
-      "rewards/margins": 2.1953208446502686,
-      "rewards/rejected": -0.3234265446662903,
-      "step": 900
-    },
-    {
-      "epoch": 2.34,
-      "eval_logits/chosen": -2.1673672199249268,
-      "eval_logits/rejected": -2.1626341342926025,
-      "eval_logps/chosen": -33.620479583740234,
-      "eval_logps/rejected": -37.25470733642578,
-      "eval_loss": 0.4770047068595886,
-      "eval_rewards/accuracies": 0.5485880374908447,
-      "eval_rewards/chosen": 0.2898494601249695,
-      "eval_rewards/margins": 0.10651160031557083,
-      "eval_rewards/rejected": 0.18333788216114044,
-      "eval_runtime": 145.3634,
-      "eval_samples_per_second": 2.36,
-      "eval_steps_per_second": 0.296,
-      "step": 900
-    },
-    {
-      "epoch": 2.36,
-      "grad_norm": 4.0625,
-      "learning_rate": 2.1442129043167877e-06,
-      "logits/chosen": -1.9585449695587158,
-      "logits/rejected": -1.953966498374939,
-      "logps/chosen": -31.010263442993164,
-      "logps/rejected": -35.009552001953125,
-      "loss": 0.164,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.783191442489624,
-      "rewards/margins": 2.119842052459717,
-      "rewards/rejected": -0.3366504907608032,
-      "step": 910
-    },
-    {
-      "epoch": 2.39,
-      "grad_norm": 3.453125,
-      "learning_rate": 2.088219349982323e-06,
-      "logits/chosen": -1.8792098760604858,
-      "logits/rejected": -1.8842833042144775,
-      "logps/chosen": -32.50172805786133,
-      "logps/rejected": -32.84755325317383,
-      "loss": 0.1644,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.9927990436553955,
-      "rewards/margins": 2.236151933670044,
-      "rewards/rejected": -0.24335305392742157,
-      "step": 920
-    },
-    {
-      "epoch": 2.42,
-      "grad_norm": 4.15625,
-      "learning_rate": 2.0324373493478803e-06,
-      "logits/chosen": -1.9770915508270264,
-      "logits/rejected": -1.9679561853408813,
-      "logps/chosen": -29.701152801513672,
-      "logps/rejected": -34.7978630065918,
-      "loss": 0.1669,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.8086459636688232,
-      "rewards/margins": 2.0571932792663574,
-      "rewards/rejected": -0.2485475093126297,
-      "step": 930
-    },
-    {
-      "epoch": 2.44,
-      "grad_norm": 3.78125,
-      "learning_rate": 1.976895560604729e-06,
-      "logits/chosen": -1.903071641921997,
-      "logits/rejected": -1.8999197483062744,
-      "logps/chosen": -29.023662567138672,
-      "logps/rejected": -32.71151351928711,
-      "loss": 0.2019,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.585354208946228,
-      "rewards/margins": 1.757653832435608,
-      "rewards/rejected": -0.17229948937892914,
-      "step": 940
-    },
-    {
-      "epoch": 2.47,
-      "grad_norm": 3.4375,
-      "learning_rate": 1.921622518534466e-06,
-      "logits/chosen": -1.842889428138733,
-      "logits/rejected": -1.8503259420394897,
-      "logps/chosen": -30.36769676208496,
-      "logps/rejected": -35.7653923034668,
-      "loss": 0.2098,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.7523466348648071,
-      "rewards/margins": 1.903167963027954,
-      "rewards/rejected": -0.15082135796546936,
-      "step": 950
-    },
-    {
-      "epoch": 2.49,
-      "grad_norm": 4.15625,
-      "learning_rate": 1.8666466198491794e-06,
-      "logits/chosen": -1.8521137237548828,
-      "logits/rejected": -1.8454768657684326,
-      "logps/chosen": -31.145715713500977,
-      "logps/rejected": -37.1761589050293,
-      "loss": 0.1613,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.877960205078125,
-      "rewards/margins": 2.282109260559082,
-      "rewards/rejected": -0.4041489064693451,
-      "step": 960
-    },
-    {
-      "epoch": 2.52,
-      "grad_norm": 4.1875,
-      "learning_rate": 1.8119961086025376e-06,
-      "logits/chosen": -1.8534562587738037,
-      "logits/rejected": -1.8540384769439697,
-      "logps/chosen": -28.50092124938965,
-      "logps/rejected": -33.28984832763672,
-      "loss": 0.1593,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.9148372411727905,
-      "rewards/margins": 2.1308465003967285,
-      "rewards/rejected": -0.2160094678401947,
-      "step": 970
-    },
-    {
-      "epoch": 2.55,
-      "grad_norm": 4.375,
-      "learning_rate": 1.7576990616793139e-06,
-      "logits/chosen": -1.8584295511245728,
-      "logits/rejected": -1.8696168661117554,
-      "logps/chosen": -30.254512786865234,
-      "logps/rejected": -34.45421600341797,
-      "loss": 0.1653,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 1.97552490234375,
-      "rewards/margins": 2.201627731323242,
-      "rewards/rejected": -0.22610275447368622,
-      "step": 980
-    },
-    {
-      "epoch": 2.57,
-      "grad_norm": 2.828125,
-      "learning_rate": 1.7037833743707892e-06,
-      "logits/chosen": -1.926839828491211,
-      "logits/rejected": -1.924412488937378,
-      "logps/chosen": -33.426719665527344,
-      "logps/rejected": -32.59941101074219,
-      "loss": 0.1853,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 1.8860301971435547,
-      "rewards/margins": 2.0261569023132324,
-      "rewards/rejected": -0.14012634754180908,
-      "step": 990
-    },
-    {
-      "epoch": 2.6,
-      "grad_norm": 4.125,
-      "learning_rate": 1.6502767460434588e-06,
-      "logits/chosen": -1.8978465795516968,
-      "logits/rejected": -1.9018335342407227,
-      "logps/chosen": -31.86910629272461,
-      "logps/rejected": -34.74235153198242,
-      "loss": 0.1826,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.8159281015396118,
-      "rewards/margins": 1.9953454732894897,
-      "rewards/rejected": -0.17941740155220032,
-      "step": 1000
-    },
-    {
-      "epoch": 2.6,
-      "eval_logits/chosen": -2.162538528442383,
-      "eval_logits/rejected": -2.157820701599121,
-      "eval_logps/chosen": -33.64886474609375,
-      "eval_logps/rejected": -37.291744232177734,
-      "eval_loss": 0.4763668477535248,
-      "eval_rewards/accuracies": 0.5830564498901367,
-      "eval_rewards/chosen": 0.26998022198677063,
-      "eval_rewards/margins": 0.11256679147481918,
-      "eval_rewards/rejected": 0.15741342306137085,
-      "eval_runtime": 145.331,
-      "eval_samples_per_second": 2.36,
-      "eval_steps_per_second": 0.296,
-      "step": 1000
-    },
-    {
-      "epoch": 2.62,
-      "grad_norm": 5.9375,
-      "learning_rate": 1.5972066659083796e-06,
-      "logits/chosen": -1.9077873229980469,
-      "logits/rejected": -1.9126176834106445,
-      "logps/chosen": -30.2926082611084,
-      "logps/rejected": -32.68244171142578,
-      "loss": 0.2024,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 1.7554700374603271,
-      "rewards/margins": 1.8682985305786133,
-      "rewards/rejected": -0.11282841116189957,
-      "step": 1010
-    },
-    {
-      "epoch": 2.65,
-      "grad_norm": 3.734375,
-      "learning_rate": 1.5446003988985041e-06,
-      "logits/chosen": -1.8675119876861572,
-      "logits/rejected": -1.8702566623687744,
-      "logps/chosen": -27.929931640625,
-      "logps/rejected": -32.15419006347656,
-      "loss": 0.1718,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.7609975337982178,
-      "rewards/margins": 2.079514503479004,
-      "rewards/rejected": -0.31851673126220703,
-      "step": 1020
-    },
-    {
-      "epoch": 2.68,
-      "grad_norm": 2.671875,
-      "learning_rate": 1.4924849716612211e-06,
-      "logits/chosen": -1.9224603176116943,
-      "logits/rejected": -1.916998267173767,
-      "logps/chosen": -31.501073837280273,
-      "logps/rejected": -33.88199996948242,
-      "loss": 0.1823,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.7416280508041382,
-      "rewards/margins": 1.9964303970336914,
-      "rewards/rejected": -0.2548021972179413,
-      "step": 1030
-    },
-    {
-      "epoch": 2.7,
-      "grad_norm": 4.71875,
-      "learning_rate": 1.440887158673332e-06,
-      "logits/chosen": -1.8555800914764404,
-      "logits/rejected": -1.8593838214874268,
-      "logps/chosen": -33.17572021484375,
-      "logps/rejected": -34.55617141723633,
-      "loss": 0.19,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.8046057224273682,
-      "rewards/margins": 1.9002208709716797,
-      "rewards/rejected": -0.09561540186405182,
-      "step": 1040
-    },
-    {
-      "epoch": 2.73,
-      "grad_norm": 3.5,
-      "learning_rate": 1.3898334684855647e-06,
-      "logits/chosen": -1.8540366888046265,
-      "logits/rejected": -1.866837501525879,
-      "logps/chosen": -29.41763687133789,
-      "logps/rejected": -33.34986114501953,
-      "loss": 0.1906,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 1.711678147315979,
-      "rewards/margins": 1.9425528049468994,
-      "rewards/rejected": -0.23087480664253235,
-      "step": 1050
-    },
-    {
-      "epoch": 2.75,
-      "grad_norm": 4.0625,
-      "learning_rate": 1.3393501301037245e-06,
-      "logits/chosen": -1.8432916402816772,
-      "logits/rejected": -1.8369777202606201,
-      "logps/chosen": -29.536724090576172,
-      "logps/rejected": -33.66853713989258,
-      "loss": 0.1732,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.835822343826294,
-      "rewards/margins": 2.0812220573425293,
-      "rewards/rejected": -0.24539966881275177,
-      "step": 1060
-    },
-    {
-      "epoch": 2.78,
-      "grad_norm": 3.59375,
-      "learning_rate": 1.2894630795134454e-06,
-      "logits/chosen": -1.9685659408569336,
-      "logits/rejected": -1.9693739414215088,
-      "logps/chosen": -30.985530853271484,
-      "logps/rejected": -33.102413177490234,
-      "loss": 0.1709,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.7930816411972046,
-      "rewards/margins": 1.9734855890274048,
-      "rewards/rejected": -0.180403932929039,
-      "step": 1070
-    },
-    {
-      "epoch": 2.81,
-      "grad_norm": 5.28125,
-      "learning_rate": 1.2401979463554984e-06,
-      "logits/chosen": -1.9013326168060303,
-      "logits/rejected": -1.9002430438995361,
-      "logps/chosen": -31.509572982788086,
-      "logps/rejected": -33.25670623779297,
-      "loss": 0.2033,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 1.8061821460723877,
-      "rewards/margins": 1.8515968322753906,
-      "rewards/rejected": -0.045414675027132034,
-      "step": 1080
-    },
-    {
-      "epoch": 2.83,
-      "grad_norm": 3.234375,
-      "learning_rate": 1.1915800407584705e-06,
-      "logits/chosen": -1.9260213375091553,
-      "logits/rejected": -1.918617606163025,
-      "logps/chosen": -31.691287994384766,
-      "logps/rejected": -31.300344467163086,
-      "loss": 0.1849,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.774212121963501,
-      "rewards/margins": 1.955381155014038,
-      "rewards/rejected": -0.1811690330505371,
-      "step": 1090
-    },
-    {
-      "epoch": 2.86,
-      "grad_norm": 2.796875,
-      "learning_rate": 1.1436343403356019e-06,
-      "logits/chosen": -1.896405577659607,
-      "logits/rejected": -1.8956992626190186,
-      "logps/chosen": -32.56309509277344,
-      "logps/rejected": -36.86756134033203,
-      "loss": 0.1541,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.9987242221832275,
-      "rewards/margins": 2.359121322631836,
-      "rewards/rejected": -0.3603973388671875,
-      "step": 1100
-    },
-    {
-      "epoch": 2.86,
-      "eval_logits/chosen": -2.1608357429504395,
-      "eval_logits/rejected": -2.1561272144317627,
-      "eval_logps/chosen": -33.625431060791016,
-      "eval_logps/rejected": -37.27484893798828,
-      "eval_loss": 0.47510504722595215,
-      "eval_rewards/accuracies": 0.5776578187942505,
-      "eval_rewards/chosen": 0.28638461232185364,
-      "eval_rewards/margins": 0.11714447289705276,
-      "eval_rewards/rejected": 0.16924014687538147,
-      "eval_runtime": 145.2146,
-      "eval_samples_per_second": 2.362,
-      "eval_steps_per_second": 0.296,
-      "step": 1100
-    },
-    {
-      "epoch": 2.88,
-      "grad_norm": 3.734375,
-      "learning_rate": 1.0963854773524548e-06,
-      "logits/chosen": -1.9410547018051147,
-      "logits/rejected": -1.946345567703247,
-      "logps/chosen": -32.88351821899414,
-      "logps/rejected": -35.84196472167969,
-      "loss": 0.1916,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.8085190057754517,
-      "rewards/margins": 2.0173850059509277,
-      "rewards/rejected": -0.2088661640882492,
-      "step": 1110
-    },
-    {
-      "epoch": 2.91,
-      "grad_norm": 2.40625,
-      "learning_rate": 1.049857726072005e-06,
-      "logits/chosen": -1.914781928062439,
-      "logits/rejected": -1.912883996963501,
-      "logps/chosen": -30.12225341796875,
-      "logps/rejected": -33.4071044921875,
-      "loss": 0.1829,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.74576735496521,
-      "rewards/margins": 2.0428128242492676,
-      "rewards/rejected": -0.29704537987709045,
-      "step": 1120
-    },
-    {
-      "epoch": 2.94,
-      "grad_norm": 2.828125,
-      "learning_rate": 1.0040749902836508e-06,
-      "logits/chosen": -1.8179181814193726,
-      "logits/rejected": -1.8206218481063843,
-      "logps/chosen": -27.161823272705078,
-      "logps/rejected": -30.888957977294922,
-      "loss": 0.1529,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.8107696771621704,
-      "rewards/margins": 2.2615151405334473,
-      "rewards/rejected": -0.4507456421852112,
-      "step": 1130
-    },
-    {
-      "epoch": 2.96,
-      "grad_norm": 3.171875,
-      "learning_rate": 9.59060791022566e-07,
-      "logits/chosen": -1.916400671005249,
-      "logits/rejected": -1.9134107828140259,
-      "logps/chosen": -30.4929141998291,
-      "logps/rejected": -32.6131477355957,
-      "loss": 0.178,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.897585153579712,
-      "rewards/margins": 1.957972764968872,
-      "rewards/rejected": -0.060387589037418365,
-      "step": 1140
-    },
-    {
-      "epoch": 2.99,
-      "grad_norm": 4.5,
-      "learning_rate": 9.148382544856885e-07,
-      "logits/chosen": -1.8358739614486694,
-      "logits/rejected": -1.8296668529510498,
-      "logps/chosen": -25.861547470092773,
-      "logps/rejected": -32.6159553527832,
-      "loss": 0.2002,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.5935077667236328,
-      "rewards/margins": 1.9208663702011108,
-      "rewards/rejected": -0.32735878229141235,
-      "step": 1150
-    },
-    {
-      "epoch": 3.01,
-      "grad_norm": 2.78125,
-      "learning_rate": 8.714301001505568e-07,
-      "logits/chosen": -1.9491764307022095,
-      "logits/rejected": -1.9467414617538452,
-      "logps/chosen": -30.44895362854004,
-      "logps/rejected": -35.62247848510742,
-      "loss": 0.1676,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.852980375289917,
-      "rewards/margins": 2.224640369415283,
-      "rewards/rejected": -0.3716599941253662,
-      "step": 1160
-    },
-    {
-      "epoch": 3.04,
-      "grad_norm": 2.578125,
-      "learning_rate": 8.288586291031025e-07,
-      "logits/chosen": -1.932654619216919,
-      "logits/rejected": -1.930672287940979,
-      "logps/chosen": -28.86700439453125,
-      "logps/rejected": -32.90315246582031,
-      "loss": 0.1532,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.9231812953948975,
-      "rewards/margins": 2.300574779510498,
-      "rewards/rejected": -0.37739327549934387,
-      "step": 1170
-    },
-    {
-      "epoch": 3.06,
-      "grad_norm": 3.125,
-      "learning_rate": 7.871457125803897e-07,
-      "logits/chosen": -1.9274743795394897,
-      "logits/rejected": -1.9154332876205444,
-      "logps/chosen": -33.33545684814453,
-      "logps/rejected": -34.530860900878906,
-      "loss": 0.1634,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.8968908786773682,
-      "rewards/margins": 2.218459367752075,
-      "rewards/rejected": -0.3215683400630951,
-      "step": 1180
-    },
-    {
-      "epoch": 3.09,
-      "grad_norm": 4.5,
-      "learning_rate": 7.463127807341966e-07,
-      "logits/chosen": -1.819190263748169,
-      "logits/rejected": -1.8140865564346313,
-      "logps/chosen": -32.35272979736328,
-      "logps/rejected": -34.45198440551758,
-      "loss": 0.1541,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 2.097872734069824,
-      "rewards/margins": 2.247788906097412,
-      "rewards/rejected": -0.14991624653339386,
-      "step": 1190
-    },
-    {
-      "epoch": 3.12,
-      "grad_norm": 3.046875,
-      "learning_rate": 7.063808116212021e-07,
-      "logits/chosen": -1.8649425506591797,
-      "logits/rejected": -1.8679530620574951,
-      "logps/chosen": -30.119836807250977,
-      "logps/rejected": -31.720102310180664,
-      "loss": 0.194,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.746681809425354,
-      "rewards/margins": 1.91042959690094,
-      "rewards/rejected": -0.163747638463974,
-      "step": 1200
-    },
-    {
-      "epoch": 3.12,
-      "eval_logits/chosen": -2.1612026691436768,
-      "eval_logits/rejected": -2.156489610671997,
-      "eval_logps/chosen": -33.626529693603516,
-      "eval_logps/rejected": -37.28034210205078,
-      "eval_loss": 0.47478199005126953,
-      "eval_rewards/accuracies": 0.5801494717597961,
-      "eval_rewards/chosen": 0.28561660647392273,
-      "eval_rewards/margins": 0.12022235989570618,
-      "eval_rewards/rejected": 0.16539424657821655,
-      "eval_runtime": 145.3188,
-      "eval_samples_per_second": 2.36,
-      "eval_steps_per_second": 0.296,
-      "step": 1200
-    },
-    {
-      "epoch": 3.14,
-      "grad_norm": 3.8125,
-      "learning_rate": 6.673703204254348e-07,
-      "logits/chosen": -1.8913599252700806,
-      "logits/rejected": -1.8870538473129272,
-      "logps/chosen": -28.591426849365234,
-      "logps/rejected": -31.342187881469727,
-      "loss": 0.1557,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.8775701522827148,
-      "rewards/margins": 2.181436538696289,
-      "rewards/rejected": -0.30386656522750854,
-      "step": 1210
-    },
-    {
-      "epoch": 3.17,
-      "grad_norm": 3.296875,
-      "learning_rate": 6.293013489185315e-07,
-      "logits/chosen": -1.9030640125274658,
-      "logits/rejected": -1.897794485092163,
-      "logps/chosen": -32.43680953979492,
-      "logps/rejected": -35.05671691894531,
-      "loss": 0.1566,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.9859931468963623,
-      "rewards/margins": 2.3267762660980225,
-      "rewards/rejected": -0.3407830595970154,
-      "step": 1220
-    },
-    {
-      "epoch": 3.19,
-      "grad_norm": 2.828125,
-      "learning_rate": 5.921934551632086e-07,
-      "logits/chosen": -1.9055864810943604,
-      "logits/rejected": -1.892482042312622,
-      "logps/chosen": -30.573678970336914,
-      "logps/rejected": -34.583587646484375,
-      "loss": 0.1759,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 1.7642362117767334,
-      "rewards/margins": 2.0888543128967285,
-      "rewards/rejected": -0.32461827993392944,
-      "step": 1230
-    },
-    {
-      "epoch": 3.22,
-      "grad_norm": 2.5,
-      "learning_rate": 5.560657034652405e-07,
-      "logits/chosen": -1.9504817724227905,
-      "logits/rejected": -1.9479200839996338,
-      "logps/chosen": -32.358009338378906,
-      "logps/rejected": -32.21631622314453,
-      "loss": 0.168,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.8724260330200195,
-      "rewards/margins": 2.13594126701355,
-      "rewards/rejected": -0.2635149657726288,
-      "step": 1240
-    },
-    {
-      "epoch": 3.25,
-      "grad_norm": 2.421875,
-      "learning_rate": 5.2093665457911e-07,
-      "logits/chosen": -1.8791261911392212,
-      "logits/rejected": -1.8764369487762451,
-      "logps/chosen": -31.556161880493164,
-      "logps/rejected": -35.48642349243164,
-      "loss": 0.1379,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.987993597984314,
-      "rewards/margins": 2.4705002307891846,
-      "rewards/rejected": -0.48250651359558105,
-      "step": 1250
-    },
-    {
-      "epoch": 3.27,
-      "grad_norm": 2.640625,
-      "learning_rate": 4.868243561723535e-07,
-      "logits/chosen": -1.9446487426757812,
-      "logits/rejected": -1.9393678903579712,
-      "logps/chosen": -28.063180923461914,
-      "logps/rejected": -32.76237487792969,
-      "loss": 0.1586,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.8883317708969116,
-      "rewards/margins": 2.1542108058929443,
-      "rewards/rejected": -0.2658790349960327,
-      "step": 1260
-    },
-    {
-      "epoch": 3.3,
-      "grad_norm": 3.375,
-      "learning_rate": 4.537463335535161e-07,
-      "logits/chosen": -1.9708101749420166,
-      "logits/rejected": -1.975584626197815,
-      "logps/chosen": -29.93453025817871,
-      "logps/rejected": -32.06245422363281,
-      "loss": 0.1628,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.9435287714004517,
-      "rewards/margins": 2.210292339324951,
-      "rewards/rejected": -0.2667638957500458,
-      "step": 1270
-    },
-    {
-      "epoch": 3.32,
-      "grad_norm": 4.78125,
-      "learning_rate": 4.217195806684629e-07,
-      "logits/chosen": -1.8651002645492554,
-      "logits/rejected": -1.872433066368103,
-      "logps/chosen": -32.172630310058594,
-      "logps/rejected": -33.01711654663086,
-      "loss": 0.1714,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.914229154586792,
-      "rewards/margins": 2.2074227333068848,
-      "rewards/rejected": -0.29319363832473755,
-      "step": 1280
-    },
-    {
-      "epoch": 3.35,
-      "grad_norm": 3.84375,
-      "learning_rate": 3.907605513696808e-07,
-      "logits/chosen": -1.7700433731079102,
-      "logits/rejected": -1.7724153995513916,
-      "logps/chosen": -30.311264038085938,
-      "logps/rejected": -36.89826202392578,
-      "loss": 0.1748,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.9744415283203125,
-      "rewards/margins": 2.2152342796325684,
-      "rewards/rejected": -0.2407928705215454,
-      "step": 1290
-    },
-    {
-      "epoch": 3.38,
-      "grad_norm": 1.546875,
-      "learning_rate": 3.6088515096305675e-07,
-      "logits/chosen": -1.8248789310455322,
-      "logits/rejected": -1.82831609249115,
-      "logps/chosen": -30.20694923400879,
-      "logps/rejected": -33.00934600830078,
-      "loss": 0.1414,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 2.1068058013916016,
-      "rewards/margins": 2.488433361053467,
-      "rewards/rejected": -0.38162797689437866,
-      "step": 1300
-    },
-    {
-      "epoch": 3.38,
-      "eval_logits/chosen": -2.160539150238037,
-      "eval_logits/rejected": -2.1558327674865723,
-      "eval_logps/chosen": -33.62611389160156,
-      "eval_logps/rejected": -37.27513122558594,
-      "eval_loss": 0.47534918785095215,
-      "eval_rewards/accuracies": 0.5830564498901367,
-      "eval_rewards/chosen": 0.2859070897102356,
-      "eval_rewards/margins": 0.11686603724956512,
-      "eval_rewards/rejected": 0.16904108226299286,
-      "eval_runtime": 145.3117,
-      "eval_samples_per_second": 2.36,
-      "eval_steps_per_second": 0.296,
-      "step": 1300
-    },
-    {
-      "epoch": 3.4,
-      "grad_norm": 2.8125,
-      "learning_rate": 3.321087280364757e-07,
-      "logits/chosen": -1.9337997436523438,
-      "logits/rejected": -1.9138942956924438,
-      "logps/chosen": -28.2788143157959,
-      "logps/rejected": -35.673282623291016,
-      "loss": 0.1617,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.984209418296814,
-      "rewards/margins": 2.339022159576416,
-      "rewards/rejected": -0.3548130691051483,
-      "step": 1310
-    },
-    {
-      "epoch": 3.43,
-      "grad_norm": 2.890625,
-      "learning_rate": 3.044460665744284e-07,
-      "logits/chosen": -1.9120073318481445,
-      "logits/rejected": -1.917673110961914,
-      "logps/chosen": -29.9759578704834,
-      "logps/rejected": -33.02309036254883,
-      "loss": 0.1289,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 2.1404826641082764,
-      "rewards/margins": 2.5131189823150635,
-      "rewards/rejected": -0.3726363182067871,
-      "step": 1320
-    },
-    {
-      "epoch": 3.45,
-      "grad_norm": 3.6875,
-      "learning_rate": 2.779113783626916e-07,
-      "logits/chosen": -1.884010910987854,
-      "logits/rejected": -1.879150390625,
-      "logps/chosen": -30.3497257232666,
-      "logps/rejected": -35.16407012939453,
-      "loss": 0.154,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.8709485530853271,
-      "rewards/margins": 2.2851948738098145,
-      "rewards/rejected": -0.41424623131752014,
-      "step": 1330
-    },
-    {
-      "epoch": 3.48,
-      "grad_norm": 2.046875,
-      "learning_rate": 2.5251829568697204e-07,
-      "logits/chosen": -1.7457748651504517,
-      "logits/rejected": -1.755414366722107,
-      "logps/chosen": -30.9390811920166,
-      "logps/rejected": -31.315013885498047,
-      "loss": 0.1642,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 2.0023577213287354,
-      "rewards/margins": 2.198988676071167,
-      "rewards/rejected": -0.19663105905056,
-      "step": 1340
-    },
-    {
-      "epoch": 3.51,
-      "grad_norm": 3.21875,
-      "learning_rate": 2.2827986432927774e-07,
-      "logits/chosen": -1.8068130016326904,
-      "logits/rejected": -1.8115451335906982,
-      "logps/chosen": -30.966552734375,
-      "logps/rejected": -33.47523880004883,
-      "loss": 0.1355,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 2.0890681743621826,
-      "rewards/margins": 2.2783665657043457,
-      "rewards/rejected": -0.18929778039455414,
-      "step": 1350
-    },
-    {
-      "epoch": 3.53,
-      "grad_norm": 3.96875,
-      "learning_rate": 2.0520853686560177e-07,
-      "logits/chosen": -1.8347194194793701,
-      "logits/rejected": -1.840275764465332,
-      "logps/chosen": -31.000539779663086,
-      "logps/rejected": -35.11589813232422,
-      "loss": 0.1568,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.981787085533142,
-      "rewards/margins": 2.36877179145813,
-      "rewards/rejected": -0.38698458671569824,
-      "step": 1360
-    },
-    {
-      "epoch": 3.56,
-      "grad_norm": 2.125,
-      "learning_rate": 1.833161662683672e-07,
-      "logits/chosen": -1.9270436763763428,
-      "logits/rejected": -1.9225692749023438,
-      "logps/chosen": -29.990549087524414,
-      "logps/rejected": -31.350738525390625,
-      "loss": 0.169,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.8515956401824951,
-      "rewards/margins": 2.158782482147217,
-      "rewards/rejected": -0.30718696117401123,
-      "step": 1370
-    },
-    {
-      "epoch": 3.58,
-      "grad_norm": 3.859375,
-      "learning_rate": 1.626139998169246e-07,
-      "logits/chosen": -1.8426074981689453,
-      "logits/rejected": -1.8444961309432983,
-      "logps/chosen": -27.056955337524414,
-      "logps/rejected": -31.148794174194336,
-      "loss": 0.1754,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.8216432332992554,
-      "rewards/margins": 2.104875087738037,
-      "rewards/rejected": -0.2832319140434265,
-      "step": 1380
-    },
-    {
-      "epoch": 3.61,
-      "grad_norm": 4.9375,
-      "learning_rate": 1.4311267331922535e-07,
-      "logits/chosen": -1.836014986038208,
-      "logits/rejected": -1.8406941890716553,
-      "logps/chosen": -29.474157333374023,
-      "logps/rejected": -33.005123138427734,
-      "loss": 0.185,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 1.7817341089248657,
-      "rewards/margins": 1.7936760187149048,
-      "rewards/rejected": -0.011941825971007347,
-      "step": 1390
-    },
-    {
-      "epoch": 3.64,
-      "grad_norm": 2.03125,
-      "learning_rate": 1.2482220564763669e-07,
-      "logits/chosen": -1.9031331539154053,
-      "logits/rejected": -1.9063663482666016,
-      "logps/chosen": -32.08364486694336,
-      "logps/rejected": -33.883567810058594,
-      "loss": 0.1492,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.9928478002548218,
-      "rewards/margins": 2.3148512840270996,
-      "rewards/rejected": -0.32200372219085693,
-      "step": 1400
-    },
-    {
-      "epoch": 3.64,
-      "eval_logits/chosen": -2.1602745056152344,
-      "eval_logits/rejected": -2.155562400817871,
-      "eval_logps/chosen": -33.62794876098633,
-      "eval_logps/rejected": -37.28424072265625,
-      "eval_loss": 0.4743622839450836,
-      "eval_rewards/accuracies": 0.5917773842811584,
-      "eval_rewards/chosen": 0.2846195697784424,
-      "eval_rewards/margins": 0.12195253372192383,
-      "eval_rewards/rejected": 0.16266703605651855,
-      "eval_runtime": 145.0405,
-      "eval_samples_per_second": 2.365,
-      "eval_steps_per_second": 0.296,
-      "step": 1400
-    },
-    {
-      "epoch": 3.66,
-      "grad_norm": 3.1875,
-      "learning_rate": 1.0775199359171346e-07,
-      "logits/chosen": -1.9723188877105713,
-      "logits/rejected": -1.9656269550323486,
-      "logps/chosen": -30.971176147460938,
-      "logps/rejected": -34.55162811279297,
-      "loss": 0.1679,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.9819583892822266,
-      "rewards/margins": 2.196903705596924,
-      "rewards/rejected": -0.2149454653263092,
-      "step": 1410
-    },
-    {
-      "epoch": 3.69,
-      "grad_norm": 4.28125,
-      "learning_rate": 9.191080703056604e-08,
-      "logits/chosen": -1.8590764999389648,
-      "logits/rejected": -1.8698148727416992,
-      "logps/chosen": -31.296035766601562,
-      "logps/rejected": -34.09710693359375,
-      "loss": 0.1478,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.9972867965698242,
-      "rewards/margins": 2.3515076637268066,
-      "rewards/rejected": -0.35422074794769287,
-      "step": 1420
-    },
-    {
-      "epoch": 3.71,
-      "grad_norm": 3.78125,
-      "learning_rate": 7.730678442730539e-08,
-      "logits/chosen": -1.9422740936279297,
-      "logits/rejected": -1.9541749954223633,
-      "logps/chosen": -32.07694625854492,
-      "logps/rejected": -34.164581298828125,
-      "loss": 0.1459,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 2.0887014865875244,
-      "rewards/margins": 2.4570722579956055,
-      "rewards/rejected": -0.3683708906173706,
-      "step": 1430
-    },
-    {
-      "epoch": 3.74,
-      "grad_norm": 3.015625,
-      "learning_rate": 6.394742864787806e-08,
-      "logits/chosen": -1.9317998886108398,
-      "logits/rejected": -1.933950424194336,
-      "logps/chosen": -30.444339752197266,
-      "logps/rejected": -34.485233306884766,
-      "loss": 0.1473,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.9348328113555908,
-      "rewards/margins": 2.3725178241729736,
-      "rewards/rejected": -0.4376850128173828,
-      "step": 1440
-    },
-    {
-      "epoch": 3.77,
-      "grad_norm": 3.859375,
-      "learning_rate": 5.183960310644748e-08,
-      "logits/chosen": -1.9155280590057373,
-      "logits/rejected": -1.9085607528686523,
-      "logps/chosen": -32.7889289855957,
-      "logps/rejected": -34.09833526611328,
-      "loss": 0.172,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.9455804824829102,
-      "rewards/margins": 2.1751322746276855,
-      "rewards/rejected": -0.22955195605754852,
-      "step": 1450
-    },
-    {
-      "epoch": 3.79,
-      "grad_norm": 2.875,
-      "learning_rate": 4.098952823928693e-08,
-      "logits/chosen": -1.884874701499939,
-      "logits/rejected": -1.890824556350708,
-      "logps/chosen": -28.527862548828125,
-      "logps/rejected": -34.241851806640625,
-      "loss": 0.159,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.879720687866211,
-      "rewards/margins": 2.2800161838531494,
-      "rewards/rejected": -0.40029558539390564,
-      "step": 1460
-    },
-    {
-      "epoch": 3.82,
-      "grad_norm": 3.25,
-      "learning_rate": 3.1402778309014284e-08,
-      "logits/chosen": -1.8300120830535889,
-      "logits/rejected": -1.8280490636825562,
-      "logps/chosen": -28.143136978149414,
-      "logps/rejected": -30.871578216552734,
-      "loss": 0.1654,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.8916912078857422,
-      "rewards/margins": 2.0858335494995117,
-      "rewards/rejected": -0.1941421777009964,
-      "step": 1470
-    },
-    {
-      "epoch": 3.84,
-      "grad_norm": 3.6875,
-      "learning_rate": 2.3084278540791427e-08,
-      "logits/chosen": -2.004854440689087,
-      "logits/rejected": -1.999352216720581,
-      "logps/chosen": -32.60303497314453,
-      "logps/rejected": -32.43815994262695,
-      "loss": 0.16,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.8775575160980225,
-      "rewards/margins": 2.14457368850708,
-      "rewards/rejected": -0.26701638102531433,
-      "step": 1480
-    },
-    {
-      "epoch": 3.87,
-      "grad_norm": 3.140625,
-      "learning_rate": 1.6038302591975807e-08,
-      "logits/chosen": -1.8964626789093018,
-      "logits/rejected": -1.8985786437988281,
-      "logps/chosen": -26.187755584716797,
-      "logps/rejected": -28.33608627319336,
-      "loss": 0.1956,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 1.6020386219024658,
-      "rewards/margins": 1.821703314781189,
-      "rewards/rejected": -0.21966469287872314,
-      "step": 1490
-    },
-    {
-      "epoch": 3.9,
-      "grad_norm": 4.375,
-      "learning_rate": 1.0268470356514237e-08,
-      "logits/chosen": -1.905769944190979,
-      "logits/rejected": -1.9003044366836548,
-      "logps/chosen": -30.289718627929688,
-      "logps/rejected": -32.631587982177734,
-      "loss": 0.1694,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.7577499151229858,
-      "rewards/margins": 2.221593141555786,
-      "rewards/rejected": -0.46384334564208984,
-      "step": 1500
-    },
-    {
-      "epoch": 3.9,
-      "eval_logits/chosen": -2.1607134342193604,
-      "eval_logits/rejected": -2.1560044288635254,
-      "eval_logps/chosen": -33.631370544433594,
-      "eval_logps/rejected": -37.28603744506836,
-      "eval_loss": 0.47470974922180176,
-      "eval_rewards/accuracies": 0.5568937063217163,
-      "eval_rewards/chosen": 0.2822282910346985,
-      "eval_rewards/margins": 0.12082076817750931,
-      "eval_rewards/rejected": 0.16140751540660858,
-      "eval_runtime": 145.1244,
-      "eval_samples_per_second": 2.363,
-      "eval_steps_per_second": 0.296,
-      "step": 1500
-    },
-    {
-      "epoch": 3.92,
-      "grad_norm": 5.78125,
-      "learning_rate": 5.777746105209147e-09,
-      "logits/chosen": -1.8295776844024658,
-      "logits/rejected": -1.8337701559066772,
-      "logps/chosen": -31.576879501342773,
-      "logps/rejected": -34.6661262512207,
-      "loss": 0.1827,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.8158810138702393,
-      "rewards/margins": 1.994899034500122,
-      "rewards/rejected": -0.17901787161827087,
-      "step": 1510
-    },
-    {
-      "epoch": 3.95,
-      "grad_norm": 2.1875,
-      "learning_rate": 2.5684369628148352e-09,
-      "logits/chosen": -1.8878551721572876,
-      "logits/rejected": -1.8862745761871338,
-      "logps/chosen": -28.058696746826172,
-      "logps/rejected": -33.05814743041992,
-      "loss": 0.1623,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.872835397720337,
-      "rewards/margins": 2.200737953186035,
-      "rewards/rejected": -0.3279026448726654,
-      "step": 1520
-    },
-    {
-      "epoch": 3.97,
-      "grad_norm": 4.0625,
-      "learning_rate": 6.421917227455999e-10,
-      "logits/chosen": -1.9854892492294312,
-      "logits/rejected": -1.9778953790664673,
-      "logps/chosen": -25.75176429748535,
-      "logps/rejected": -29.129873275756836,
-      "loss": 0.1804,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.6178255081176758,
-      "rewards/margins": 1.938612699508667,
-      "rewards/rejected": -0.3207874298095703,
-      "step": 1530
-    },
-    {
-      "epoch": 4.0,
-      "grad_norm": 2.921875,
-      "learning_rate": 0.0,
-      "logits/chosen": -1.8875421285629272,
-      "logits/rejected": -1.877659797668457,
-      "logps/chosen": -30.428552627563477,
-      "logps/rejected": -36.047637939453125,
-      "loss": 0.1484,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.8463512659072876,
-      "rewards/margins": 2.291874408721924,
-      "rewards/rejected": -0.445523202419281,
-      "step": 1540
-    },
-    {
-      "epoch": 4.0,
-      "step": 1540,
+      "epoch": 1.0,
+      "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.17319794591371115,
-      "train_runtime": 10768.0107,
-      "train_samples_per_second": 1.144,
-      "train_steps_per_second": 0.143
+      "train_loss": 0.46899499707407766,
+      "train_runtime": 3249.86,
+      "train_samples_per_second": 0.947,
+      "train_steps_per_second": 0.118
     }
   ],
   "logging_steps": 10,
-  "max_steps": 1540,
+  "max_steps": 385,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 4,
+  "num_train_epochs": 1,
   "save_steps": 100,
   "total_flos": 0.0,
   "train_batch_size": 4,