diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -11,10 +11,10 @@
     {
       "epoch": 0.0,
       "learning_rate": 4e-08,
-      "logits/chosen": -2.5970396995544434,
-      "logits/rejected": -2.6002578735351562,
-      "logps/chosen": -391.7848205566406,
-      "logps/rejected": -379.88592529296875,
+      "logits/chosen": -2.5782995223999023,
+      "logits/rejected": -2.5818521976470947,
+      "logps/chosen": -405.7000427246094,
+      "logps/rejected": -395.45977783203125,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,1939 +25,1939 @@
     {
       "epoch": 0.01,
       "learning_rate": 4.0000000000000003e-07,
-      "logits/chosen": -2.6321280002593994,
-      "logits/rejected": -2.654905319213867,
-      "logps/chosen": -415.5525207519531,
-      "logps/rejected": -451.3063049316406,
-      "loss": 0.6934,
-      "rewards/accuracies": 0.4166666567325592,
-      "rewards/chosen": 3.593924702727236e-05,
-      "rewards/margins": -0.0004077497578691691,
-      "rewards/rejected": 0.00044368900125846267,
+      "logits/chosen": -2.6189024448394775,
+      "logits/rejected": -2.6425905227661133,
+      "logps/chosen": -430.23095703125,
+      "logps/rejected": -465.6946105957031,
+      "loss": 0.6937,
+      "rewards/accuracies": 0.3402777910232544,
+      "rewards/chosen": -0.0004277642001397908,
+      "rewards/margins": -0.0011669597588479519,
+      "rewards/rejected": 0.0007391955587081611,
       "step": 10
     },
     {
       "epoch": 0.02,
       "learning_rate": 8.000000000000001e-07,
-      "logits/chosen": -2.5803229808807373,
-      "logits/rejected": -2.6041855812072754,
-      "logps/chosen": -378.57830810546875,
-      "logps/rejected": -406.0473937988281,
-      "loss": 0.6938,
-      "rewards/accuracies": 0.41874998807907104,
-      "rewards/chosen": -0.0005379965296015143,
-      "rewards/margins": -0.0012370327021926641,
-      "rewards/rejected": 0.0006990364054217935,
+      "logits/chosen": -2.565629482269287,
+      "logits/rejected": -2.587047576904297,
+      "logps/chosen": -392.698974609375,
+      "logps/rejected": -420.3116149902344,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0005297298775985837,
+      "rewards/margins": -9.570456313667819e-05,
+      "rewards/rejected": 0.0006254343315958977,
       "step": 20
     },
     {
       "epoch": 0.02,
       "learning_rate": 1.2000000000000002e-06,
-      "logits/chosen": -2.5524168014526367,
-      "logits/rejected": -2.610682249069214,
-      "logps/chosen": -405.12225341796875,
-      "logps/rejected": -437.0355529785156,
-      "loss": 0.6931,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -2.3546338070445927e-06,
-      "rewards/margins": 4.741999146062881e-05,
-      "rewards/rejected": -4.97746623295825e-05,
+      "logits/chosen": -2.5378575325012207,
+      "logits/rejected": -2.594695568084717,
+      "logps/chosen": -418.8670349121094,
+      "logps/rejected": -452.1278381347656,
+      "loss": 0.6935,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.0007869214750826359,
+      "rewards/margins": -0.0006720771198160946,
+      "rewards/rejected": -0.00011484431888675317,
       "step": 30
     },
     {
       "epoch": 0.03,
       "learning_rate": 1.6000000000000001e-06,
-      "logits/chosen": -2.551060199737549,
-      "logits/rejected": -2.5709078311920166,
-      "logps/chosen": -374.8092346191406,
-      "logps/rejected": -387.48931884765625,
-      "loss": 0.6934,
-      "rewards/accuracies": 0.46875,
-      "rewards/chosen": -0.0006756076472811401,
-      "rewards/margins": -0.00045229861279949546,
-      "rewards/rejected": -0.00022330903448164463,
+      "logits/chosen": -2.53520131111145,
+      "logits/rejected": -2.5557379722595215,
+      "logps/chosen": -387.7684326171875,
+      "logps/rejected": -400.6465759277344,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.0008191015804186463,
+      "rewards/margins": -7.259581616381183e-05,
+      "rewards/rejected": -0.0007465056842193007,
       "step": 40
     },
     {
       "epoch": 0.04,
       "learning_rate": 2.0000000000000003e-06,
-      "logits/chosen": -2.5436787605285645,
-      "logits/rejected": -2.540389060974121,
-      "logps/chosen": -366.8270568847656,
-      "logps/rejected": -374.41497802734375,
-      "loss": 0.6934,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": -0.0006743489648215473,
-      "rewards/margins": -0.00049586221575737,
-      "rewards/rejected": -0.0001784868072718382,
+      "logits/chosen": -2.5300962924957275,
+      "logits/rejected": -2.5252957344055176,
+      "logps/chosen": -379.98785400390625,
+      "logps/rejected": -387.24755859375,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.001431723590940237,
+      "rewards/margins": -0.00021147821098566055,
+      "rewards/rejected": -0.0012202452635392547,
       "step": 50
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.4000000000000003e-06,
-      "logits/chosen": -2.5643861293792725,
-      "logits/rejected": -2.5933163166046143,
-      "logps/chosen": -389.86444091796875,
-      "logps/rejected": -415.7872619628906,
-      "loss": 0.6932,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.0006498451111838222,
-      "rewards/margins": -0.00017824865062721074,
-      "rewards/rejected": -0.0004715964605566114,
+      "logits/chosen": -2.5475428104400635,
+      "logits/rejected": -2.571965456008911,
+      "logps/chosen": -404.24560546875,
+      "logps/rejected": -431.3631896972656,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.0013539999490603805,
+      "rewards/margins": 0.0010300865396857262,
+      "rewards/rejected": -0.0023840866051614285,
       "step": 60
     },
     {
       "epoch": 0.06,
       "learning_rate": 2.8000000000000003e-06,
-      "logits/chosen": -2.608799457550049,
-      "logits/rejected": -2.6101136207580566,
-      "logps/chosen": -390.1094970703125,
-      "logps/rejected": -396.507568359375,
-      "loss": 0.693,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -0.0011631803354248405,
-      "rewards/margins": 0.00039177463622763753,
-      "rewards/rejected": -0.0015549547970294952,
+      "logits/chosen": -2.5969393253326416,
+      "logits/rejected": -2.601594924926758,
+      "logps/chosen": -404.23236083984375,
+      "logps/rejected": -410.1368713378906,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.003309990745037794,
+      "rewards/margins": 0.0009536609286442399,
+      "rewards/rejected": -0.004263651557266712,
       "step": 70
     },
     {
       "epoch": 0.06,
       "learning_rate": 3.2000000000000003e-06,
-      "logits/chosen": -2.6286911964416504,
-      "logits/rejected": -2.6467819213867188,
-      "logps/chosen": -383.10284423828125,
-      "logps/rejected": -406.47393798828125,
-      "loss": 0.6933,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": -0.0017519788816571236,
-      "rewards/margins": -0.0002883758279494941,
-      "rewards/rejected": -0.0014636029954999685,
+      "logits/chosen": -2.6136419773101807,
+      "logits/rejected": -2.63590669631958,
+      "logps/chosen": -396.66680908203125,
+      "logps/rejected": -420.2757263183594,
+      "loss": 0.693,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.0044545335695147514,
+      "rewards/margins": 0.0003312411135993898,
+      "rewards/rejected": -0.004785774741321802,
       "step": 80
     },
     {
       "epoch": 0.07,
       "learning_rate": 3.6000000000000003e-06,
-      "logits/chosen": -2.6428580284118652,
-      "logits/rejected": -2.6388907432556152,
-      "logps/chosen": -431.7904357910156,
-      "logps/rejected": -446.31732177734375,
-      "loss": 0.6927,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.0028546121902763844,
-      "rewards/margins": 0.0008213006658479571,
-      "rewards/rejected": -0.0036759127397090197,
+      "logits/chosen": -2.6318578720092773,
+      "logits/rejected": -2.627044677734375,
+      "logps/chosen": -447.1123962402344,
+      "logps/rejected": -461.43994140625,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.005481979344040155,
+      "rewards/margins": 0.0011420813389122486,
+      "rewards/rejected": -0.006624061614274979,
       "step": 90
     },
     {
       "epoch": 0.08,
       "learning_rate": 4.000000000000001e-06,
-      "logits/chosen": -2.6136410236358643,
-      "logits/rejected": -2.6054491996765137,
-      "logps/chosen": -382.89886474609375,
-      "logps/rejected": -399.26068115234375,
-      "loss": 0.6932,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -0.003379733068868518,
-      "rewards/margins": -0.00018416713282931596,
-      "rewards/rejected": -0.0031955656595528126,
+      "logits/chosen": -2.6037490367889404,
+      "logits/rejected": -2.600301504135132,
+      "logps/chosen": -396.48162841796875,
+      "logps/rejected": -414.06121826171875,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.007383377756923437,
+      "rewards/margins": -0.0003390449855942279,
+      "rewards/rejected": -0.007044332567602396,
       "step": 100
     },
     {
       "epoch": 0.08,
-      "eval_logits/chosen": -2.490821361541748,
-      "eval_logits/rejected": -2.497763156890869,
-      "eval_logps/chosen": -322.7948913574219,
-      "eval_logps/rejected": -332.520751953125,
-      "eval_loss": 0.6930087208747864,
-      "eval_rewards/accuracies": 0.5220000147819519,
-      "eval_rewards/chosen": -0.002995692892000079,
-      "eval_rewards/margins": 0.0002869492454919964,
-      "eval_rewards/rejected": -0.0032826424576342106,
-      "eval_runtime": 1592.8954,
-      "eval_samples_per_second": 1.88,
-      "eval_steps_per_second": 0.235,
+      "eval_logits/chosen": -2.4768383502960205,
+      "eval_logits/rejected": -2.483818292617798,
+      "eval_logps/chosen": -335.0983581542969,
+      "eval_logps/rejected": -344.84783935546875,
+      "eval_loss": 0.6929709315299988,
+      "eval_rewards/accuracies": 0.5176666378974915,
+      "eval_rewards/chosen": -0.007662294432520866,
+      "eval_rewards/margins": 0.0003858131531160325,
+      "eval_rewards/rejected": -0.008048107847571373,
+      "eval_runtime": 1616.6225,
+      "eval_samples_per_second": 1.852,
+      "eval_steps_per_second": 0.232,
       "step": 100
     },
     {
       "epoch": 0.09,
       "learning_rate": 4.4e-06,
-      "logits/chosen": -2.6191463470458984,
-      "logits/rejected": -2.626410722732544,
-      "logps/chosen": -418.57061767578125,
-      "logps/rejected": -431.57568359375,
-      "loss": 0.6929,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.004728841129690409,
-      "rewards/margins": 0.00041932202293537557,
-      "rewards/rejected": -0.005148162599653006,
+      "logits/chosen": -2.6104788780212402,
+      "logits/rejected": -2.6167144775390625,
+      "logps/chosen": -433.07000732421875,
+      "logps/rejected": -446.54840087890625,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.009052934125065804,
+      "rewards/margins": 0.0011820209911093116,
+      "rewards/rejected": -0.01023495476692915,
       "step": 110
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.800000000000001e-06,
-      "logits/chosen": -2.616450309753418,
-      "logits/rejected": -2.6316654682159424,
-      "logps/chosen": -387.65936279296875,
-      "logps/rejected": -406.0279235839844,
-      "loss": 0.6924,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": -0.006956800818443298,
-      "rewards/margins": 0.00143801374360919,
-      "rewards/rejected": -0.008394813165068626,
+      "logits/chosen": -2.603712797164917,
+      "logits/rejected": -2.6226086616516113,
+      "logps/chosen": -402.2064208984375,
+      "logps/rejected": -420.73345947265625,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.014477076940238476,
+      "rewards/margins": 0.0004566563293337822,
+      "rewards/rejected": -0.014933735132217407,
       "step": 120
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999755876225375e-06,
-      "logits/chosen": -2.5832996368408203,
-      "logits/rejected": -2.5880000591278076,
-      "logps/chosen": -411.5313415527344,
-      "logps/rejected": -434.4432678222656,
-      "loss": 0.6925,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.009584048762917519,
-      "rewards/margins": 0.0013888883404433727,
-      "rewards/rejected": -0.010972936637699604,
+      "logits/chosen": -2.5681347846984863,
+      "logits/rejected": -2.575467348098755,
+      "logps/chosen": -426.24334716796875,
+      "logps/rejected": -450.0921936035156,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.014384260401129723,
+      "rewards/margins": 0.0017711615655571222,
+      "rewards/rejected": -0.0161554254591465,
       "step": 130
     },
     {
       "epoch": 0.11,
       "learning_rate": 4.997803172081864e-06,
-      "logits/chosen": -2.5645198822021484,
-      "logits/rejected": -2.5772173404693604,
-      "logps/chosen": -400.5196533203125,
-      "logps/rejected": -419.6172790527344,
+      "logits/chosen": -2.549048900604248,
+      "logits/rejected": -2.562979221343994,
+      "logps/chosen": -415.5914001464844,
+      "logps/rejected": -434.4419860839844,
       "loss": 0.6928,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -0.013501152396202087,
-      "rewards/margins": 0.0006207471014931798,
-      "rewards/rejected": -0.01412190031260252,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.0187313761562109,
+      "rewards/margins": 0.0007000064942985773,
+      "rewards/rejected": -0.019431382417678833,
       "step": 140
     },
     {
       "epoch": 0.12,
       "learning_rate": 4.9938992891651825e-06,
-      "logits/chosen": -2.581132411956787,
-      "logits/rejected": -2.5872106552124023,
-      "logps/chosen": -382.4533386230469,
-      "logps/rejected": -414.8580627441406,
-      "loss": 0.6922,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -0.016398200765252113,
-      "rewards/margins": 0.0018568082014098763,
-      "rewards/rejected": -0.01825500652194023,
+      "logits/chosen": -2.571101665496826,
+      "logits/rejected": -2.579190254211426,
+      "logps/chosen": -396.6254577636719,
+      "logps/rejected": -428.6380310058594,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.014721980318427086,
+      "rewards/margins": 0.0025038637686520815,
+      "rewards/rejected": -0.0172258447855711,
       "step": 150
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.988047277024456e-06,
-      "logits/chosen": -2.6221516132354736,
-      "logits/rejected": -2.6419806480407715,
-      "logps/chosen": -396.94757080078125,
-      "logps/rejected": -416.6105041503906,
+      "logits/chosen": -2.6119332313537598,
+      "logits/rejected": -2.6318445205688477,
+      "logps/chosen": -410.3868713378906,
+      "logps/rejected": -429.6300354003906,
       "loss": 0.6922,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -0.019227243959903717,
-      "rewards/margins": 0.0020019258372485638,
-      "rewards/rejected": -0.021229172125458717,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.01405310444533825,
+      "rewards/margins": 0.001986552495509386,
+      "rewards/rejected": -0.016039658337831497,
       "step": 160
     },
     {
       "epoch": 0.14,
       "learning_rate": 4.980251707005417e-06,
-      "logits/chosen": -2.618068218231201,
-      "logits/rejected": -2.6144068241119385,
-      "logps/chosen": -425.07244873046875,
-      "logps/rejected": -435.1886291503906,
-      "loss": 0.6935,
+      "logits/chosen": -2.6066763401031494,
+      "logits/rejected": -2.602252721786499,
+      "logps/chosen": -439.21417236328125,
+      "logps/rejected": -450.12139892578125,
+      "loss": 0.6931,
       "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.02663121186196804,
-      "rewards/margins": -0.0007048315601423383,
-      "rewards/rejected": -0.0259263813495636,
+      "rewards/chosen": -0.016262266784906387,
+      "rewards/margins": 0.0002180577430408448,
+      "rewards/rejected": -0.016480324789881706,
       "step": 170
     },
     {
       "epoch": 0.14,
       "learning_rate": 4.970518668679459e-06,
-      "logits/chosen": -2.5808920860290527,
-      "logits/rejected": -2.5735676288604736,
-      "logps/chosen": -417.1087341308594,
-      "logps/rejected": -421.8807678222656,
-      "loss": 0.6932,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": -0.026894057169556618,
-      "rewards/margins": -6.247097917366773e-05,
-      "rewards/rejected": -0.02683158591389656,
+      "logits/chosen": -2.5717625617980957,
+      "logits/rejected": -2.564636468887329,
+      "logps/chosen": -430.0758361816406,
+      "logps/rejected": -434.64593505859375,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.011936081573367119,
+      "rewards/margins": 0.0035087396390736103,
+      "rewards/rejected": -0.015444819815456867,
       "step": 180
     },
     {
       "epoch": 0.15,
       "learning_rate": 4.958855765086722e-06,
-      "logits/chosen": -2.606273651123047,
-      "logits/rejected": -2.608921527862549,
-      "logps/chosen": -391.02362060546875,
-      "logps/rejected": -399.1781311035156,
-      "loss": 0.6927,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -0.02517121471464634,
-      "rewards/margins": 0.0010087827686220407,
-      "rewards/rejected": -0.026179995387792587,
+      "logits/chosen": -2.589442491531372,
+      "logits/rejected": -2.5929834842681885,
+      "logps/chosen": -402.8868408203125,
+      "logps/rejected": -411.501708984375,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.01052446011453867,
+      "rewards/margins": 0.0019376948475837708,
+      "rewards/rejected": -0.01246215496212244,
       "step": 190
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.945272106796919e-06,
-      "logits/chosen": -2.578331708908081,
-      "logits/rejected": -2.594592332839966,
-      "logps/chosen": -388.1728515625,
-      "logps/rejected": -412.2106018066406,
-      "loss": 0.6921,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.022485502064228058,
-      "rewards/margins": 0.0022102731745690107,
-      "rewards/rejected": -0.024695774540305138,
+      "logits/chosen": -2.5665698051452637,
+      "logits/rejected": -2.5826125144958496,
+      "logps/chosen": -399.679443359375,
+      "logps/rejected": -424.7906188964844,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.01273694820702076,
+      "rewards/margins": 0.0011030982714146376,
+      "rewards/rejected": -0.01384004671126604,
       "step": 200
     },
     {
       "epoch": 0.16,
-      "eval_logits/chosen": -2.4899628162384033,
-      "eval_logits/rejected": -2.497047185897827,
-      "eval_logps/chosen": -324.8166809082031,
-      "eval_logps/rejected": -334.6196594238281,
-      "eval_loss": 0.6926525831222534,
-      "eval_rewards/accuracies": 0.5183333158493042,
-      "eval_rewards/chosen": -0.023213516920804977,
-      "eval_rewards/margins": 0.0010577173670753837,
-      "eval_rewards/rejected": -0.02427123300731182,
-      "eval_runtime": 1586.0043,
-      "eval_samples_per_second": 1.888,
-      "eval_steps_per_second": 0.236,
+      "eval_logits/chosen": -2.476593017578125,
+      "eval_logits/rejected": -2.4835994243621826,
+      "eval_logps/chosen": -335.71136474609375,
+      "eval_logps/rejected": -345.59197998046875,
+      "eval_loss": 0.6923276782035828,
+      "eval_rewards/accuracies": 0.5426666736602783,
+      "eval_rewards/chosen": -0.013792484067380428,
+      "eval_rewards/margins": 0.0016968329437077045,
+      "eval_rewards/rejected": -0.015489316545426846,
+      "eval_runtime": 1616.1511,
+      "eval_samples_per_second": 1.853,
+      "eval_steps_per_second": 0.232,
       "step": 200
     },
     {
       "epoch": 0.17,
       "learning_rate": 4.929778304792537e-06,
-      "logits/chosen": -2.5928235054016113,
-      "logits/rejected": -2.601378917694092,
-      "logps/chosen": -424.74066162109375,
-      "logps/rejected": -424.7421875,
-      "loss": 0.6914,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.029381360858678818,
-      "rewards/margins": 0.003531560767441988,
-      "rewards/rejected": -0.03291292116045952,
+      "logits/chosen": -2.58109974861145,
+      "logits/rejected": -2.592872381210327,
+      "logps/chosen": -438.9920959472656,
+      "logps/rejected": -439.1341247558594,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.017467942088842392,
+      "rewards/margins": 0.005076024681329727,
+      "rewards/rejected": -0.02254396490752697,
       "step": 210
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.912386462179987e-06,
-      "logits/chosen": -2.630770206451416,
-      "logits/rejected": -2.637812376022339,
-      "logps/chosen": -404.63275146484375,
-      "logps/rejected": -440.338134765625,
-      "loss": 0.6902,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -0.035039953887462616,
-      "rewards/margins": 0.005983405746519566,
-      "rewards/rejected": -0.041023362427949905,
+      "logits/chosen": -2.6212189197540283,
+      "logits/rejected": -2.6276934146881104,
+      "logps/chosen": -417.43359375,
+      "logps/rejected": -453.41119384765625,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.022168174386024475,
+      "rewards/margins": 0.00578011991456151,
+      "rewards/rejected": -0.027948295697569847,
       "step": 220
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.893110164735167e-06,
-      "logits/chosen": -2.6500868797302246,
-      "logits/rejected": -2.6668245792388916,
-      "logps/chosen": -419.52508544921875,
-      "logps/rejected": -435.5750427246094,
-      "loss": 0.6928,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": -0.04090961441397667,
-      "rewards/margins": 0.000937508069910109,
-      "rewards/rejected": -0.04184712469577789,
+      "logits/chosen": -2.637918710708618,
+      "logits/rejected": -2.6529829502105713,
+      "logps/chosen": -433.37969970703125,
+      "logps/rejected": -449.2330627441406,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.02808026596903801,
+      "rewards/margins": 0.003863722085952759,
+      "rewards/rejected": -0.03194398805499077,
       "step": 230
     },
     {
       "epoch": 0.19,
       "learning_rate": 4.871964470290823e-06,
-      "logits/chosen": -2.5765552520751953,
-      "logits/rejected": -2.603816509246826,
-      "logps/chosen": -419.22332763671875,
-      "logps/rejected": -441.0245666503906,
-      "loss": 0.692,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": -0.03845367208123207,
-      "rewards/margins": 0.0023850365541875362,
-      "rewards/rejected": -0.040838707238435745,
+      "logits/chosen": -2.5649704933166504,
+      "logits/rejected": -2.590609073638916,
+      "logps/chosen": -433.03271484375,
+      "logps/rejected": -454.35528564453125,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.02802122011780739,
+      "rewards/margins": 0.005218953359872103,
+      "rewards/rejected": -0.0332401767373085,
       "step": 240
     },
     {
       "epoch": 0.2,
       "learning_rate": 4.848965896974006e-06,
-      "logits/chosen": -2.62872052192688,
-      "logits/rejected": -2.652649402618408,
-      "logps/chosen": -402.7193603515625,
-      "logps/rejected": -426.7840270996094,
-      "loss": 0.6924,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": -0.038372453302145004,
-      "rewards/margins": 0.0015891485381871462,
-      "rewards/rejected": -0.03996160253882408,
+      "logits/chosen": -2.61820650100708,
+      "logits/rejected": -2.6432509422302246,
+      "logps/chosen": -416.4977111816406,
+      "logps/rejected": -439.9608459472656,
+      "loss": 0.693,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.03262058645486832,
+      "rewards/margins": 0.0004899540217593312,
+      "rewards/rejected": -0.033110540360212326,
       "step": 250
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.8241324103028055e-06,
-      "logits/chosen": -2.6180949211120605,
-      "logits/rejected": -2.5828635692596436,
-      "logps/chosen": -408.49200439453125,
-      "logps/rejected": -419.191650390625,
-      "loss": 0.6918,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.037816666066646576,
-      "rewards/margins": 0.002941995393484831,
-      "rewards/rejected": -0.040758661925792694,
+      "logits/chosen": -2.6041173934936523,
+      "logits/rejected": -2.574268341064453,
+      "logps/chosen": -422.29437255859375,
+      "logps/rejected": -432.9288635253906,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.03160303831100464,
+      "rewards/margins": 0.00435088574886322,
+      "rewards/rejected": -0.03595392405986786,
       "step": 260
     },
     {
       "epoch": 0.22,
       "learning_rate": 4.797483409152438e-06,
-      "logits/chosen": -2.617037296295166,
-      "logits/rejected": -2.636782169342041,
-      "logps/chosen": -391.20391845703125,
-      "logps/rejected": -412.9513244628906,
-      "loss": 0.691,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": -0.0356525257229805,
-      "rewards/margins": 0.004444011487066746,
-      "rewards/rejected": -0.04009654000401497,
+      "logits/chosen": -2.6138675212860107,
+      "logits/rejected": -2.632070541381836,
+      "logps/chosen": -403.8775939941406,
+      "logps/rejected": -425.6439514160156,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.024003220722079277,
+      "rewards/margins": 0.0069326674565672874,
+      "rewards/rejected": -0.03093588910996914,
       "step": 270
     },
     {
       "epoch": 0.22,
       "learning_rate": 4.769039710601669e-06,
-      "logits/chosen": -2.6231722831726074,
-      "logits/rejected": -2.6144392490386963,
-      "logps/chosen": -387.4334411621094,
-      "logps/rejected": -403.40057373046875,
-      "loss": 0.6898,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.03822268173098564,
-      "rewards/margins": 0.00696157943457365,
-      "rewards/rejected": -0.045184262096881866,
+      "logits/chosen": -2.617337226867676,
+      "logits/rejected": -2.6075143814086914,
+      "logps/chosen": -399.85308837890625,
+      "logps/rejected": -415.65460205078125,
+      "loss": 0.69,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.022890212014317513,
+      "rewards/margins": 0.006663457490503788,
+      "rewards/rejected": -0.029553670436143875,
       "step": 280
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.738823533671383e-06,
-      "logits/chosen": -2.614532709121704,
-      "logits/rejected": -2.6345345973968506,
-      "logps/chosen": -421.5420837402344,
-      "logps/rejected": -442.87994384765625,
-      "loss": 0.6916,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.04416601359844208,
-      "rewards/margins": 0.003424243303015828,
-      "rewards/rejected": -0.047590263187885284,
+      "logits/chosen": -2.607677936553955,
+      "logits/rejected": -2.626984119415283,
+      "logps/chosen": -433.34503173828125,
+      "logps/rejected": -455.47113037109375,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.016143614426255226,
+      "rewards/margins": 0.0021604117937386036,
+      "rewards/rejected": -0.018304025754332542,
       "step": 290
     },
     {
       "epoch": 0.24,
       "learning_rate": 4.706858481968017e-06,
-      "logits/chosen": -2.5896575450897217,
-      "logits/rejected": -2.5868849754333496,
-      "logps/chosen": -402.4905700683594,
-      "logps/rejected": -409.7896423339844,
-      "loss": 0.6913,
+      "logits/chosen": -2.5796170234680176,
+      "logits/rejected": -2.5774731636047363,
+      "logps/chosen": -412.5723571777344,
+      "logps/rejected": -420.7684631347656,
+      "loss": 0.6906,
       "rewards/accuracies": 0.53125,
-      "rewards/chosen": -0.04842864349484444,
-      "rewards/margins": 0.003919907845556736,
-      "rewards/rejected": -0.052348554134368896,
+      "rewards/chosen": -0.015171040780842304,
+      "rewards/margins": 0.005369971971958876,
+      "rewards/rejected": -0.020541010424494743,
       "step": 300
     },
     {
       "epoch": 0.24,
-      "eval_logits/chosen": -2.4894840717315674,
-      "eval_logits/rejected": -2.4966704845428467,
-      "eval_logps/chosen": -326.6393127441406,
-      "eval_logps/rejected": -336.60589599609375,
-      "eval_loss": 0.6919258236885071,
-      "eval_rewards/accuracies": 0.5339999794960022,
-      "eval_rewards/chosen": -0.0414397232234478,
-      "eval_rewards/margins": 0.0026943066623061895,
-      "eval_rewards/rejected": -0.04413403198122978,
-      "eval_runtime": 1585.7818,
-      "eval_samples_per_second": 1.888,
-      "eval_steps_per_second": 0.236,
+      "eval_logits/chosen": -2.4809017181396484,
+      "eval_logits/rejected": -2.487875461578369,
+      "eval_logps/chosen": -335.6323547363281,
+      "eval_logps/rejected": -345.656005859375,
+      "eval_loss": 0.691673219203949,
+      "eval_rewards/accuracies": 0.5523333549499512,
+      "eval_rewards/chosen": -0.01300230622291565,
+      "eval_rewards/margins": 0.0031275488436222076,
+      "eval_rewards/rejected": -0.016129855066537857,
+      "eval_runtime": 1615.9746,
+      "eval_samples_per_second": 1.853,
+      "eval_steps_per_second": 0.232,
       "step": 300
     },
     {
       "epoch": 0.25,
       "learning_rate": 4.673169525245416e-06,
-      "logits/chosen": -2.5968692302703857,
-      "logits/rejected": -2.585181474685669,
-      "logps/chosen": -404.8775939941406,
-      "logps/rejected": -433.09954833984375,
+      "logits/chosen": -2.5858876705169678,
+      "logits/rejected": -2.5731964111328125,
+      "logps/chosen": -414.9142150878906,
+      "logps/rejected": -443.6748962402344,
       "loss": 0.6902,
       "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.04641267657279968,
-      "rewards/margins": 0.00627540796995163,
-      "rewards/rejected": -0.052688080817461014,
+      "rewards/chosen": -0.010048646479845047,
+      "rewards/margins": 0.006063834764063358,
+      "rewards/rejected": -0.01611248217523098,
       "step": 310
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.63778297989952e-06,
-      "logits/chosen": -2.604461193084717,
-      "logits/rejected": -2.610708475112915,
-      "logps/chosen": -403.58184814453125,
-      "logps/rejected": -424.60772705078125,
-      "loss": 0.691,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.05133752152323723,
-      "rewards/margins": 0.004545481409877539,
-      "rewards/rejected": -0.0558830089867115,
+      "logits/chosen": -2.5999462604522705,
+      "logits/rejected": -2.6060266494750977,
+      "logps/chosen": -412.861328125,
+      "logps/rejected": -434.5308532714844,
+      "loss": 0.689,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.006168536841869354,
+      "rewards/margins": 0.00842990167438984,
+      "rewards/rejected": -0.014598439447581768,
       "step": 320
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.60072648841109e-06,
-      "logits/chosen": -2.589761734008789,
-      "logits/rejected": -2.5855839252471924,
-      "logps/chosen": -411.21026611328125,
-      "logps/rejected": -430.5553283691406,
-      "loss": 0.6872,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.0565360002219677,
-      "rewards/margins": 0.012383319437503815,
-      "rewards/rejected": -0.06891932338476181,
+      "logits/chosen": -2.58223295211792,
+      "logits/rejected": -2.5768980979919434,
+      "logps/chosen": -420.7120666503906,
+      "logps/rejected": -439.38421630859375,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.00741790235042572,
+      "rewards/margins": 0.01021712925285101,
+      "rewards/rejected": -0.017635030671954155,
       "step": 330
     },
     {
       "epoch": 0.27,
       "learning_rate": 4.562028997752574e-06,
-      "logits/chosen": -2.5468997955322266,
-      "logits/rejected": -2.5858261585235596,
-      "logps/chosen": -409.3692321777344,
-      "logps/rejected": -441.8675842285156,
-      "loss": 0.6876,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -0.04863084852695465,
-      "rewards/margins": 0.01160926092416048,
-      "rewards/rejected": -0.06024010106921196,
+      "logits/chosen": -2.5425148010253906,
+      "logits/rejected": -2.5803589820861816,
+      "logps/chosen": -418.337890625,
+      "logps/rejected": -450.64459228515625,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0036701434291899204,
+      "rewards/margins": 0.009488197974860668,
+      "rewards/rejected": -0.0058180540800094604,
       "step": 340
     },
     {
       "epoch": 0.28,
       "learning_rate": 4.521720736775947e-06,
-      "logits/chosen": -2.604952335357666,
-      "logits/rejected": -2.596958637237549,
-      "logps/chosen": -426.7057189941406,
-      "logps/rejected": -427.89971923828125,
-      "loss": 0.6907,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.06284764409065247,
-      "rewards/margins": 0.005581230856478214,
-      "rewards/rejected": -0.0684288740158081,
+      "logits/chosen": -2.5993804931640625,
+      "logits/rejected": -2.5906174182891846,
+      "logps/chosen": -435.66845703125,
+      "logps/rejected": -436.15594482421875,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.003759814891964197,
+      "rewards/margins": 0.006070856936275959,
+      "rewards/rejected": -0.009830671362578869,
       "step": 350
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.479833192599198e-06,
-      "logits/chosen": -2.6308584213256836,
-      "logits/rejected": -2.627711057662964,
-      "logps/chosen": -435.3694763183594,
-      "logps/rejected": -454.00775146484375,
-      "loss": 0.6867,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.06149282306432724,
-      "rewards/margins": 0.013573038391768932,
-      "rewards/rejected": -0.07506586611270905,
+      "logits/chosen": -2.6347668170928955,
+      "logits/rejected": -2.628312110900879,
+      "logps/chosen": -444.01788330078125,
+      "logps/rejected": -462.4187927246094,
+      "loss": 0.688,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": 0.007655065506696701,
+      "rewards/margins": 0.010737383738160133,
+      "rewards/rejected": -0.003082319162786007,
       "step": 360
     },
     {
       "epoch": 0.3,
       "learning_rate": 4.436399086009928e-06,
-      "logits/chosen": -2.6563782691955566,
-      "logits/rejected": -2.6729133129119873,
-      "logps/chosen": -409.5260925292969,
-      "logps/rejected": -424.39208984375,
-      "loss": 0.6874,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.07003921270370483,
-      "rewards/margins": 0.012296813540160656,
-      "rewards/rejected": -0.08233602344989777,
+      "logits/chosen": -2.65555477142334,
+      "logits/rejected": -2.6738853454589844,
+      "logps/chosen": -417.18560791015625,
+      "logps/rejected": -433.4161682128906,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.007278016768395901,
+      "rewards/margins": 0.009610911831259727,
+      "rewards/rejected": -0.016888927668333054,
       "step": 370
     },
     {
       "epoch": 0.3,
       "learning_rate": 4.391452345905239e-06,
-      "logits/chosen": -2.5911147594451904,
-      "logits/rejected": -2.6013450622558594,
-      "logps/chosen": -420.21124267578125,
-      "logps/rejected": -429.5535583496094,
-      "loss": 0.6907,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.10565753281116486,
-      "rewards/margins": 0.006660384126007557,
-      "rewards/rejected": -0.11231790482997894,
+      "logits/chosen": -2.5911710262298584,
+      "logits/rejected": -2.6055374145507812,
+      "logps/chosen": -426.5325622558594,
+      "logps/rejected": -434.95758056640625,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.020730096846818924,
+      "rewards/margins": 0.006341870874166489,
+      "rewards/rejected": -0.027071967720985413,
       "step": 380
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.3450280827879125e-06,
-      "logits/chosen": -2.6037206649780273,
-      "logits/rejected": -2.6106367111206055,
-      "logps/chosen": -413.44122314453125,
-      "logps/rejected": -427.99951171875,
-      "loss": 0.6882,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.10301616042852402,
-      "rewards/margins": 0.011568797752261162,
-      "rewards/rejected": -0.11458496004343033,
+      "logits/chosen": -2.6074564456939697,
+      "logits/rejected": -2.614403486251831,
+      "logps/chosen": -418.90106201171875,
+      "logps/rejected": -433.29046630859375,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.016112228855490685,
+      "rewards/margins": 0.01000655721873045,
+      "rewards/rejected": -0.02611878514289856,
       "step": 390
     },
     {
       "epoch": 0.32,
       "learning_rate": 4.297162561339554e-06,
-      "logits/chosen": -2.522566556930542,
-      "logits/rejected": -2.563248634338379,
-      "logps/chosen": -421.3636169433594,
-      "logps/rejected": -440.02276611328125,
-      "loss": 0.6893,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": -0.09894236922264099,
-      "rewards/margins": 0.009308888576924801,
-      "rewards/rejected": -0.10825125873088837,
+      "logits/chosen": -2.5204920768737793,
+      "logits/rejected": -2.5613515377044678,
+      "logps/chosen": -425.9031677246094,
+      "logps/rejected": -446.0582580566406,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.00986536405980587,
+      "rewards/margins": 0.010156005620956421,
+      "rewards/rejected": -0.02002136781811714,
       "step": 400
     },
     {
       "epoch": 0.32,
-      "eval_logits/chosen": -2.4952824115753174,
-      "eval_logits/rejected": -2.5022895336151123,
-      "eval_logps/chosen": -330.40167236328125,
-      "eval_logps/rejected": -341.0243835449219,
-      "eval_loss": 0.6891305446624756,
-      "eval_rewards/accuracies": 0.5546666383743286,
-      "eval_rewards/chosen": -0.0790637731552124,
-      "eval_rewards/margins": 0.009254684671759605,
-      "eval_rewards/rejected": -0.08831847459077835,
-      "eval_runtime": 1591.081,
-      "eval_samples_per_second": 1.882,
-      "eval_steps_per_second": 0.236,
+      "eval_logits/chosen": -2.490081787109375,
+      "eval_logits/rejected": -2.497150182723999,
+      "eval_logps/chosen": -335.07940673828125,
+      "eval_logps/rejected": -345.4989929199219,
+      "eval_loss": 0.6898096203804016,
+      "eval_rewards/accuracies": 0.5806666612625122,
+      "eval_rewards/chosen": -0.007472673896700144,
+      "eval_rewards/margins": 0.007086611352860928,
+      "eval_rewards/rejected": -0.014559285715222359,
+      "eval_runtime": 1615.277,
+      "eval_samples_per_second": 1.854,
+      "eval_steps_per_second": 0.232,
       "step": 400
     },
     {
       "epoch": 0.33,
       "learning_rate": 4.247893172092157e-06,
-      "logits/chosen": -2.5598597526550293,
-      "logits/rejected": -2.5833513736724854,
-      "logps/chosen": -404.2002868652344,
-      "logps/rejected": -435.16357421875,
-      "loss": 0.6857,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.10219142585992813,
-      "rewards/margins": 0.016902286559343338,
-      "rewards/rejected": -0.11909371614456177,
+      "logits/chosen": -2.556941509246826,
+      "logits/rejected": -2.578604221343994,
+      "logps/chosen": -409.3045959472656,
+      "logps/rejected": -440.68914794921875,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.01420166902244091,
+      "rewards/margins": 0.013790594413876534,
+      "rewards/rejected": -0.027992259711027145,
       "step": 410
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.197258402220187e-06,
-      "logits/chosen": -2.5842721462249756,
-      "logits/rejected": -2.5888819694519043,
-      "logps/chosen": -414.70166015625,
-      "logps/rejected": -425.90667724609375,
-      "loss": 0.6896,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": -0.12604860961437225,
-      "rewards/margins": 0.009105135686695576,
-      "rewards/rejected": -0.13515374064445496,
+      "logits/chosen": -2.5752522945404053,
+      "logits/rejected": -2.5826656818389893,
+      "logps/chosen": -419.6229553222656,
+      "logps/rejected": -430.58648681640625,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.03458824008703232,
+      "rewards/margins": 0.005263219587504864,
+      "rewards/rejected": -0.03985145688056946,
       "step": 420
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.145297805476023e-06,
-      "logits/chosen": -2.639338493347168,
-      "logits/rejected": -2.6249961853027344,
-      "logps/chosen": -412.44232177734375,
-      "logps/rejected": -427.6927185058594,
-      "loss": 0.6859,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.11504076421260834,
-      "rewards/margins": 0.016550499945878983,
-      "rewards/rejected": -0.13159126043319702,
+      "logits/chosen": -2.6343460083007812,
+      "logits/rejected": -2.621222972869873,
+      "logps/chosen": -417.43017578125,
+      "logps/rejected": -432.63818359375,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.026808559894561768,
+      "rewards/margins": 0.013837064616382122,
+      "rewards/rejected": -0.040645625442266464,
       "step": 430
     },
     {
       "epoch": 0.35,
       "learning_rate": 4.092051971292228e-06,
-      "logits/chosen": -2.598680019378662,
-      "logits/rejected": -2.617053508758545,
-      "logps/chosen": -404.762939453125,
-      "logps/rejected": -424.2640686035156,
-      "loss": 0.6839,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.1352802962064743,
-      "rewards/margins": 0.02169608511030674,
-      "rewards/rejected": -0.1569763869047165,
+      "logits/chosen": -2.593862771987915,
+      "logits/rejected": -2.611497163772583,
+      "logps/chosen": -407.7058410644531,
+      "logps/rejected": -427.201171875,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.03197476640343666,
+      "rewards/margins": 0.015016555786132812,
+      "rewards/rejected": -0.04699132218956947,
       "step": 440
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.037562493074792e-06,
-      "logits/chosen": -2.5920491218566895,
-      "logits/rejected": -2.59037446975708,
-      "logps/chosen": -434.03118896484375,
-      "logps/rejected": -439.4048767089844,
-      "loss": 0.6832,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -0.1662554293870926,
-      "rewards/margins": 0.024613002315163612,
-      "rewards/rejected": -0.19086843729019165,
+      "logits/chosen": -2.5881259441375732,
+      "logits/rejected": -2.584801435470581,
+      "logps/chosen": -435.7823181152344,
+      "logps/rejected": -439.61297607421875,
+      "loss": 0.6835,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.042981140315532684,
+      "rewards/margins": 0.020602630451321602,
+      "rewards/rejected": -0.06358377635478973,
       "step": 450
     },
     {
       "epoch": 0.37,
       "learning_rate": 3.981871935712112e-06,
-      "logits/chosen": -2.6342484951019287,
-      "logits/rejected": -2.6563720703125,
-      "logps/chosen": -411.9359436035156,
-      "logps/rejected": -435.57586669921875,
-      "loss": 0.6833,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -0.16083207726478577,
-      "rewards/margins": 0.023298023268580437,
-      "rewards/rejected": -0.18413010239601135,
+      "logits/chosen": -2.6311769485473633,
+      "logits/rejected": -2.6546669006347656,
+      "logps/chosen": -413.837646484375,
+      "logps/rejected": -437.16986083984375,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.03683266043663025,
+      "rewards/margins": 0.019854258745908737,
+      "rewards/rejected": -0.056686919182538986,
       "step": 460
     },
     {
       "epoch": 0.38,
       "learning_rate": 3.925023802325094e-06,
-      "logits/chosen": -2.664499044418335,
-      "logits/rejected": -2.6798224449157715,
-      "logps/chosen": -434.82354736328125,
-      "logps/rejected": -453.67938232421875,
-      "loss": 0.6776,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.18204860389232635,
-      "rewards/margins": 0.03722093254327774,
-      "rewards/rejected": -0.2192695438861847,
+      "logits/chosen": -2.6563210487365723,
+      "logits/rejected": -2.6776556968688965,
+      "logps/chosen": -437.06988525390625,
+      "logps/rejected": -455.8389587402344,
+      "loss": 0.6788,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.052065037190914154,
+      "rewards/margins": 0.03126855939626694,
+      "rewards/rejected": -0.08333360403776169,
       "step": 470
     },
     {
       "epoch": 0.38,
       "learning_rate": 3.867062500284342e-06,
-      "logits/chosen": -2.6116766929626465,
-      "logits/rejected": -2.627462863922119,
-      "logps/chosen": -422.6748962402344,
-      "logps/rejected": -454.072509765625,
-      "loss": 0.6764,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.23436252772808075,
-      "rewards/margins": 0.041565194725990295,
-      "rewards/rejected": -0.27592775225639343,
+      "logits/chosen": -2.6086373329162598,
+      "logits/rejected": -2.623145818710327,
+      "logps/chosen": -424.25,
+      "logps/rejected": -455.4474182128906,
+      "loss": 0.6785,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.10639921575784683,
+      "rewards/margins": 0.033148206770420074,
+      "rewards/rejected": -0.1395474076271057,
       "step": 480
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8080333065209885e-06,
-      "logits/chosen": -2.6146063804626465,
-      "logits/rejected": -2.6115143299102783,
-      "logps/chosen": -428.1953125,
-      "logps/rejected": -423.606201171875,
-      "loss": 0.7007,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.31299567222595215,
-      "rewards/margins": -0.0009805113077163696,
-      "rewards/rejected": -0.3120151460170746,
+      "logits/chosen": -2.6073594093322754,
+      "logits/rejected": -2.6054632663726807,
+      "logps/chosen": -428.15185546875,
+      "logps/rejected": -422.6666564941406,
+      "loss": 0.6961,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.1707339584827423,
+      "rewards/margins": 0.0015567988157272339,
+      "rewards/rejected": -0.17229075729846954,
       "step": 490
     },
     {
       "epoch": 0.4,
       "learning_rate": 3.7479823321582624e-06,
-      "logits/chosen": -2.6379127502441406,
-      "logits/rejected": -2.66550874710083,
-      "logps/chosen": -430.17572021484375,
-      "logps/rejected": -461.09942626953125,
-      "loss": 0.6724,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.24496836960315704,
-      "rewards/margins": 0.04783773794770241,
-      "rewards/rejected": -0.29280608892440796,
+      "logits/chosen": -2.6313910484313965,
+      "logits/rejected": -2.6622931957244873,
+      "logps/chosen": -435.267578125,
+      "logps/rejected": -466.4607849121094,
+      "loss": 0.6753,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.16189143061637878,
+      "rewards/margins": 0.04039152339100838,
+      "rewards/rejected": -0.20228295028209686,
       "step": 500
     },
     {
       "epoch": 0.4,
-      "eval_logits/chosen": -2.5028529167175293,
-      "eval_logits/rejected": -2.5099613666534424,
-      "eval_logps/chosen": -342.6784973144531,
-      "eval_logps/rejected": -354.7255859375,
-      "eval_loss": 0.684412956237793,
-      "eval_rewards/accuracies": 0.5529999732971191,
-      "eval_rewards/chosen": -0.20183168351650238,
-      "eval_rewards/margins": 0.023498954251408577,
-      "eval_rewards/rejected": -0.2253306657075882,
-      "eval_runtime": 1585.7151,
-      "eval_samples_per_second": 1.888,
-      "eval_steps_per_second": 0.236,
+      "eval_logits/chosen": -2.491628885269165,
+      "eval_logits/rejected": -2.4985597133636475,
+      "eval_logps/chosen": -348.1782531738281,
+      "eval_logps/rejected": -359.8316650390625,
+      "eval_loss": 0.6855587959289551,
+      "eval_rewards/accuracies": 0.5630000233650208,
+      "eval_rewards/chosen": -0.13846127688884735,
+      "eval_rewards/margins": 0.019424747675657272,
+      "eval_rewards/rejected": -0.15788602828979492,
+      "eval_runtime": 1618.1001,
+      "eval_samples_per_second": 1.85,
+      "eval_steps_per_second": 0.232,
       "step": 500
     },
     {
       "epoch": 0.41,
       "learning_rate": 3.686956486491419e-06,
-      "logits/chosen": -2.59777569770813,
-      "logits/rejected": -2.600053071975708,
-      "logps/chosen": -440.66217041015625,
-      "logps/rejected": -454.6631774902344,
-      "loss": 0.6921,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.2590482234954834,
-      "rewards/margins": 0.0103686498478055,
-      "rewards/rejected": -0.269416868686676,
+      "logits/chosen": -2.585524082183838,
+      "logits/rejected": -2.593437671661377,
+      "logps/chosen": -448.7865295410156,
+      "logps/rejected": -462.56890869140625,
+      "loss": 0.6939,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.1920996755361557,
+      "rewards/margins": 0.004705216269940138,
+      "rewards/rejected": -0.19680488109588623,
       "step": 510
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.625003440344166e-06,
-      "logits/chosen": -2.6117706298828125,
-      "logits/rejected": -2.625525712966919,
-      "logps/chosen": -416.2491760253906,
-      "logps/rejected": -427.0350646972656,
-      "loss": 0.6781,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -0.228049635887146,
-      "rewards/margins": 0.037227652966976166,
-      "rewards/rejected": -0.26527729630470276,
+      "logits/chosen": -2.603358268737793,
+      "logits/rejected": -2.620025157928467,
+      "logps/chosen": -422.2713317871094,
+      "logps/rejected": -432.52557373046875,
+      "loss": 0.678,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.14511428773403168,
+      "rewards/margins": 0.03498765081167221,
+      "rewards/rejected": -0.18010194599628448,
       "step": 520
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.562171588830231e-06,
-      "logits/chosen": -2.670912265777588,
-      "logits/rejected": -2.6928374767303467,
-      "logps/chosen": -428.692626953125,
-      "logps/rejected": -462.62274169921875,
-      "loss": 0.677,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.22094754874706268,
-      "rewards/margins": 0.041822899132966995,
-      "rewards/rejected": -0.26277047395706177,
+      "logits/chosen": -2.666226625442505,
+      "logits/rejected": -2.6871166229248047,
+      "logps/chosen": -434.64678955078125,
+      "logps/rejected": -468.7852478027344,
+      "loss": 0.6804,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.13358792662620544,
+      "rewards/margins": 0.031595379114151,
+      "rewards/rejected": -0.16518330574035645,
       "step": 530
     },
     {
       "epoch": 0.43,
       "learning_rate": 3.4985100135491245e-06,
-      "logits/chosen": -2.6906635761260986,
-      "logits/rejected": -2.708909511566162,
-      "logps/chosen": -444.14483642578125,
-      "logps/rejected": -480.6028747558594,
-      "loss": 0.6691,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -0.2567470967769623,
-      "rewards/margins": 0.058775972574949265,
-      "rewards/rejected": -0.31552308797836304,
+      "logits/chosen": -2.678806781768799,
+      "logits/rejected": -2.698000907897949,
+      "logps/chosen": -453.0816955566406,
+      "logps/rejected": -490.1700134277344,
+      "loss": 0.6683,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.19373668730258942,
+      "rewards/margins": 0.05805329233407974,
+      "rewards/rejected": -0.25178998708724976,
       "step": 540
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4340684442456673e-06,
-      "logits/chosen": -2.5944106578826904,
-      "logits/rejected": -2.6143410205841064,
-      "logps/chosen": -438.25738525390625,
-      "logps/rejected": -459.01763916015625,
-      "loss": 0.6759,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.3285837173461914,
-      "rewards/margins": 0.04724721983075142,
-      "rewards/rejected": -0.37583091855049133,
+      "logits/chosen": -2.5824878215789795,
+      "logits/rejected": -2.600515842437744,
+      "logps/chosen": -450.08758544921875,
+      "logps/rejected": -470.90838623046875,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.30373430252075195,
+      "rewards/margins": 0.03900415450334549,
+      "rewards/rejected": -0.34273844957351685,
       "step": 550
     },
     {
       "epoch": 0.45,
       "learning_rate": 3.3688972199631974e-06,
-      "logits/chosen": -2.6361145973205566,
-      "logits/rejected": -2.613219976425171,
-      "logps/chosen": -441.1446838378906,
-      "logps/rejected": -451.8172912597656,
-      "loss": 0.6769,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.3550431728363037,
-      "rewards/margins": 0.05103546380996704,
-      "rewards/rejected": -0.40607863664627075,
+      "logits/chosen": -2.632826328277588,
+      "logits/rejected": -2.6091580390930176,
+      "logps/chosen": -455.7169494628906,
+      "logps/rejected": -464.70684814453125,
+      "loss": 0.6806,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.35484108328819275,
+      "rewards/margins": 0.0392206534743309,
+      "rewards/rejected": -0.39406174421310425,
       "step": 560
     },
     {
       "epoch": 0.46,
       "learning_rate": 3.3030472497208354e-06,
-      "logits/chosen": -2.6290247440338135,
-      "logits/rejected": -2.652219295501709,
-      "logps/chosen": -426.34234619140625,
-      "logps/rejected": -484.0997009277344,
-      "loss": 0.6598,
+      "logits/chosen": -2.62416672706604,
+      "logits/rejected": -2.6495113372802734,
+      "logps/chosen": -439.15106201171875,
+      "logps/rejected": -497.6800842285156,
+      "loss": 0.6604,
       "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.3327481150627136,
-      "rewards/margins": 0.08233290165662766,
-      "rewards/rejected": -0.4150810241699219,
+      "rewards/chosen": -0.32003355026245117,
+      "rewards/margins": 0.07973183691501617,
+      "rewards/rejected": -0.39976540207862854,
       "step": 570
     },
     {
       "epoch": 0.46,
       "learning_rate": 3.236569972745492e-06,
-      "logits/chosen": -2.640798807144165,
-      "logits/rejected": -2.6551499366760254,
-      "logps/chosen": -419.3091735839844,
-      "logps/rejected": -434.71527099609375,
-      "loss": 0.6828,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.31533199548721313,
-      "rewards/margins": 0.03718234971165657,
-      "rewards/rejected": -0.3525143265724182,
+      "logits/chosen": -2.6336703300476074,
+      "logits/rejected": -2.649231195449829,
+      "logps/chosen": -431.4183044433594,
+      "logps/rejected": -447.3372497558594,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.30185872316360474,
+      "rewards/margins": 0.040949635207653046,
+      "rewards/rejected": -0.342808336019516,
       "step": 580
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.1695173182897126e-06,
-      "logits/chosen": -2.5991015434265137,
-      "logits/rejected": -2.6085472106933594,
-      "logps/chosen": -435.3345642089844,
-      "logps/rejected": -467.74078369140625,
-      "loss": 0.6698,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.3784990906715393,
-      "rewards/margins": 0.060858141630887985,
-      "rewards/rejected": -0.4393572211265564,
+      "logits/chosen": -2.5993151664733887,
+      "logits/rejected": -2.607579469680786,
+      "logps/chosen": -448.4823303222656,
+      "logps/rejected": -482.7305603027344,
+      "loss": 0.6645,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.3687121272087097,
+      "rewards/margins": 0.07179446518421173,
+      "rewards/rejected": -0.44050654768943787,
       "step": 590
     },
     {
       "epoch": 0.48,
       "learning_rate": 3.10194166506673e-06,
-      "logits/chosen": -2.624173164367676,
-      "logits/rejected": -2.649747848510742,
-      "logps/chosen": -443.6612243652344,
-      "logps/rejected": -470.52447509765625,
-      "loss": 0.6849,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.3967738747596741,
-      "rewards/margins": 0.03684181347489357,
-      "rewards/rejected": -0.43361568450927734,
+      "logits/chosen": -2.6293110847473145,
+      "logits/rejected": -2.649170398712158,
+      "logps/chosen": -456.01605224609375,
+      "logps/rejected": -483.74298095703125,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.3835269808769226,
+      "rewards/margins": 0.037176936864852905,
+      "rewards/rejected": -0.4207039475440979,
       "step": 600
     },
     {
       "epoch": 0.48,
-      "eval_logits/chosen": -2.534677505493164,
-      "eval_logits/rejected": -2.5411627292633057,
-      "eval_logps/chosen": -356.15911865234375,
-      "eval_logps/rejected": -369.895751953125,
-      "eval_loss": 0.6804664731025696,
-      "eval_rewards/accuracies": 0.5596666932106018,
-      "eval_rewards/chosen": -0.33663830161094666,
-      "eval_rewards/margins": 0.04039384424686432,
-      "eval_rewards/rejected": -0.3770321309566498,
-      "eval_runtime": 1591.4153,
-      "eval_samples_per_second": 1.881,
-      "eval_steps_per_second": 0.236,
+      "eval_logits/chosen": -2.533261299133301,
+      "eval_logits/rejected": -2.5394067764282227,
+      "eval_logps/chosen": -366.2155456542969,
+      "eval_logps/rejected": -379.6049499511719,
+      "eval_loss": 0.6814756393432617,
+      "eval_rewards/accuracies": 0.5666666626930237,
+      "eval_rewards/chosen": -0.3188343644142151,
+      "eval_rewards/margins": 0.03678455203771591,
+      "eval_rewards/rejected": -0.3556188941001892,
+      "eval_runtime": 1615.2404,
+      "eval_samples_per_second": 1.854,
+      "eval_steps_per_second": 0.232,
       "step": 600
     },
     {
       "epoch": 0.49,
       "learning_rate": 3.0338958003344115e-06,
-      "logits/chosen": -2.628021717071533,
-      "logits/rejected": -2.654170036315918,
-      "logps/chosen": -423.56768798828125,
-      "logps/rejected": -451.36041259765625,
-      "loss": 0.6603,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.39979976415634155,
-      "rewards/margins": 0.08663686364889145,
-      "rewards/rejected": -0.486436665058136,
+      "logits/chosen": -2.627558946609497,
+      "logits/rejected": -2.6516823768615723,
+      "logps/chosen": -437.21160888671875,
+      "logps/rejected": -464.7273864746094,
+      "loss": 0.6661,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.4052312970161438,
+      "rewards/margins": 0.0762629583477974,
+      "rewards/rejected": -0.4814942479133606,
       "step": 610
     },
     {
       "epoch": 0.5,
       "learning_rate": 2.9654328786600823e-06,
-      "logits/chosen": -2.6157093048095703,
-      "logits/rejected": -2.6425511837005615,
-      "logps/chosen": -437.9432678222656,
-      "logps/rejected": -482.203125,
-      "loss": 0.6558,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.4792413115501404,
-      "rewards/margins": 0.10463404655456543,
-      "rewards/rejected": -0.5838753581047058,
+      "logits/chosen": -2.619433879852295,
+      "logits/rejected": -2.6456384658813477,
+      "logps/chosen": -452.12762451171875,
+      "logps/rejected": -496.452392578125,
+      "loss": 0.6522,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.47861284017562866,
+      "rewards/margins": 0.10839803516864777,
+      "rewards/rejected": -0.5870109796524048,
       "step": 620
     },
     {
       "epoch": 0.5,
       "learning_rate": 2.896606380398402e-06,
-      "logits/chosen": -2.6413679122924805,
-      "logits/rejected": -2.6552627086639404,
-      "logps/chosen": -455.11260986328125,
-      "logps/rejected": -468.00885009765625,
-      "loss": 0.6878,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.5618987083435059,
-      "rewards/margins": 0.03895387798547745,
-      "rewards/rejected": -0.6008526086807251,
+      "logits/chosen": -2.639660120010376,
+      "logits/rejected": -2.652381181716919,
+      "logps/chosen": -472.73089599609375,
+      "logps/rejected": -484.69561767578125,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.5940805673599243,
+      "rewards/margins": 0.029440635815262794,
+      "rewards/rejected": -0.6235212087631226,
       "step": 630
     },
     {
       "epoch": 0.51,
       "learning_rate": 2.827470069914772e-06,
-      "logits/chosen": -2.681443214416504,
-      "logits/rejected": -2.6886324882507324,
-      "logps/chosen": -459.51953125,
-      "logps/rejected": -491.306396484375,
-      "loss": 0.6753,
+      "logits/chosen": -2.6815245151519775,
+      "logits/rejected": -2.687138080596924,
+      "logps/chosen": -475.3055114746094,
+      "logps/rejected": -506.28643798828125,
+      "loss": 0.6782,
       "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.5284784436225891,
-      "rewards/margins": 0.06967002153396606,
-      "rewards/rejected": -0.5981484651565552,
+      "rewards/chosen": -0.5460494160652161,
+      "rewards/margins": 0.060324352234601974,
+      "rewards/rejected": -0.6063737869262695,
       "step": 640
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7580779535868675e-06,
-      "logits/chosen": -2.630887508392334,
-      "logits/rejected": -2.645677328109741,
-      "logps/chosen": -457.000732421875,
-      "logps/rejected": -472.52545166015625,
-      "loss": 0.6748,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.47293543815612793,
-      "rewards/margins": 0.06035799905657768,
-      "rewards/rejected": -0.5332934260368347,
+      "logits/chosen": -2.6303911209106445,
+      "logits/rejected": -2.646120071411133,
+      "logps/chosen": -471.96533203125,
+      "logps/rejected": -487.72271728515625,
+      "loss": 0.675,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.489617258310318,
+      "rewards/margins": 0.05899134278297424,
+      "rewards/rejected": -0.548608660697937,
       "step": 650
     },
     {
       "epoch": 0.53,
       "learning_rate": 2.688484237617129e-06,
-      "logits/chosen": -2.6493170261383057,
-      "logits/rejected": -2.68170166015625,
-      "logps/chosen": -453.5931091308594,
-      "logps/rejected": -479.94818115234375,
-      "loss": 0.6716,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.4414174556732178,
-      "rewards/margins": 0.06612064689397812,
-      "rewards/rejected": -0.5075381398200989,
+      "logits/chosen": -2.647108316421509,
+      "logits/rejected": -2.6790130138397217,
+      "logps/chosen": -467.047607421875,
+      "logps/rejected": -493.06524658203125,
+      "loss": 0.6725,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.4314625859260559,
+      "rewards/margins": 0.063529834151268,
+      "rewards/rejected": -0.4949924349784851,
       "step": 660
     },
     {
       "epoch": 0.54,
       "learning_rate": 2.6187432856891585e-06,
-      "logits/chosen": -2.6594104766845703,
-      "logits/rejected": -2.6713552474975586,
-      "logps/chosen": -456.5660095214844,
-      "logps/rejected": -488.9071350097656,
-      "loss": 0.6665,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.44480568170547485,
-      "rewards/margins": 0.07257355749607086,
-      "rewards/rejected": -0.5173791646957397,
+      "logits/chosen": -2.658151626586914,
+      "logits/rejected": -2.6678333282470703,
+      "logps/chosen": -471.19842529296875,
+      "logps/rejected": -504.4642639160156,
+      "loss": 0.6669,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.44427603483200073,
+      "rewards/margins": 0.07264076918363571,
+      "rewards/rejected": -0.516916811466217,
       "step": 670
     },
     {
       "epoch": 0.54,
       "learning_rate": 2.548909576501096e-06,
-      "logits/chosen": -2.7302539348602295,
-      "logits/rejected": -2.7218852043151855,
-      "logps/chosen": -458.1416931152344,
-      "logps/rejected": -482.7391662597656,
-      "loss": 0.6629,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.4737418591976166,
-      "rewards/margins": 0.08715833723545074,
-      "rewards/rejected": -0.5609001517295837,
+      "logits/chosen": -2.7318921089172363,
+      "logits/rejected": -2.7216339111328125,
+      "logps/chosen": -474.5341796875,
+      "logps/rejected": -498.4226989746094,
+      "loss": 0.6642,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.4827556014060974,
+      "rewards/margins": 0.08245094865560532,
+      "rewards/rejected": -0.5652065873146057,
       "step": 680
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.4790376612091503e-06,
-      "logits/chosen": -2.7039058208465576,
-      "logits/rejected": -2.7273173332214355,
-      "logps/chosen": -473.97064208984375,
-      "logps/rejected": -498.3961486816406,
-      "loss": 0.6715,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.5430811643600464,
-      "rewards/margins": 0.08151289075613022,
-      "rewards/rejected": -0.6245940327644348,
+      "logits/chosen": -2.70528507232666,
+      "logits/rejected": -2.7278404235839844,
+      "logps/chosen": -492.2674255371094,
+      "logps/rejected": -518.07177734375,
+      "loss": 0.6686,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.579769492149353,
+      "rewards/margins": 0.0886869877576828,
+      "rewards/rejected": -0.6684564352035522,
       "step": 690
     },
     {
       "epoch": 0.56,
       "learning_rate": 2.40918212081453e-06,
-      "logits/chosen": -2.6450188159942627,
-      "logits/rejected": -2.667860507965088,
-      "logps/chosen": -424.064208984375,
-      "logps/rejected": -463.056640625,
-      "loss": 0.6503,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.4541637897491455,
-      "rewards/margins": 0.10934968292713165,
-      "rewards/rejected": -0.5635135173797607,
+      "logits/chosen": -2.643637180328369,
+      "logits/rejected": -2.6695291996002197,
+      "logps/chosen": -438.7723083496094,
+      "logps/rejected": -477.81451416015625,
+      "loss": 0.6535,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.4713774621486664,
+      "rewards/margins": 0.10217118263244629,
+      "rewards/rejected": -0.5735486745834351,
       "step": 700
     },
     {
       "epoch": 0.56,
-      "eval_logits/chosen": -2.543064832687378,
-      "eval_logits/rejected": -2.5492172241210938,
-      "eval_logps/chosen": -366.25225830078125,
-      "eval_logps/rejected": -381.38427734375,
-      "eval_loss": 0.677395224571228,
-      "eval_rewards/accuracies": 0.5630000233650208,
-      "eval_rewards/chosen": -0.4375694692134857,
-      "eval_rewards/margins": 0.054348159581422806,
-      "eval_rewards/rejected": -0.4919176399707794,
-      "eval_runtime": 1584.8471,
-      "eval_samples_per_second": 1.889,
-      "eval_steps_per_second": 0.237,
+      "eval_logits/chosen": -2.5424647331237793,
+      "eval_logits/rejected": -2.5483226776123047,
+      "eval_logps/chosen": -376.3719177246094,
+      "eval_logps/rejected": -391.4495544433594,
+      "eval_loss": 0.6770339012145996,
+      "eval_rewards/accuracies": 0.5763333439826965,
+      "eval_rewards/chosen": -0.42039749026298523,
+      "eval_rewards/margins": 0.05366775020956993,
+      "eval_rewards/rejected": -0.47406527400016785,
+      "eval_runtime": 1614.5505,
+      "eval_samples_per_second": 1.854,
+      "eval_steps_per_second": 0.232,
       "step": 700
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3393975235270654e-06,
-      "logits/chosen": -2.6429007053375244,
-      "logits/rejected": -2.635617256164551,
-      "logps/chosen": -460.6632385253906,
-      "logps/rejected": -495.57098388671875,
-      "loss": 0.6491,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.5247341990470886,
-      "rewards/margins": 0.12096454948186874,
-      "rewards/rejected": -0.6456986665725708,
+      "logits/chosen": -2.6486973762512207,
+      "logits/rejected": -2.6384849548339844,
+      "logps/chosen": -470.92498779296875,
+      "logps/rejected": -504.40740966796875,
+      "loss": 0.6546,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.49029403924942017,
+      "rewards/margins": 0.10647070407867432,
+      "rewards/rejected": -0.5967647433280945,
       "step": 710
     },
     {
       "epoch": 0.58,
       "learning_rate": 2.2697383821388153e-06,
-      "logits/chosen": -2.665684700012207,
-      "logits/rejected": -2.63784122467041,
-      "logps/chosen": -454.69110107421875,
-      "logps/rejected": -466.15020751953125,
-      "loss": 0.6856,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -0.49990567564964294,
-      "rewards/margins": 0.04791254550218582,
-      "rewards/rejected": -0.547818124294281,
+      "logits/chosen": -2.6679904460906982,
+      "logits/rejected": -2.6391725540161133,
+      "logps/chosen": -463.16632080078125,
+      "logps/rejected": -473.4232482910156,
+      "loss": 0.6867,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.4399467408657074,
+      "rewards/margins": 0.04117538779973984,
+      "rewards/rejected": -0.48112210631370544,
       "step": 720
     },
     {
       "epoch": 0.58,
       "learning_rate": 2.2002591114409657e-06,
-      "logits/chosen": -2.672154426574707,
-      "logits/rejected": -2.6631722450256348,
-      "logps/chosen": -453.4434509277344,
-      "logps/rejected": -476.5977478027344,
-      "loss": 0.6716,
+      "logits/chosen": -2.677816867828369,
+      "logits/rejected": -2.6684961318969727,
+      "logps/chosen": -457.8728942871094,
+      "logps/rejected": -479.80609130859375,
+      "loss": 0.6756,
       "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": -0.4569157660007477,
-      "rewards/margins": 0.07301469147205353,
-      "rewards/rejected": -0.5299304723739624,
+      "rewards/chosen": -0.3563249111175537,
+      "rewards/margins": 0.05807635933160782,
+      "rewards/rejected": -0.41440123319625854,
       "step": 730
     },
     {
       "epoch": 0.59,
       "learning_rate": 2.131013985717285e-06,
-      "logits/chosen": -2.6686697006225586,
-      "logits/rejected": -2.662213087081909,
-      "logps/chosen": -456.8734436035156,
-      "logps/rejected": -498.776123046875,
-      "loss": 0.6499,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.38933560252189636,
-      "rewards/margins": 0.1104939803481102,
-      "rewards/rejected": -0.49982959032058716,
+      "logits/chosen": -2.678981065750122,
+      "logits/rejected": -2.669673204421997,
+      "logps/chosen": -464.6796875,
+      "logps/rejected": -506.2353515625,
+      "loss": 0.6495,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.320025771856308,
+      "rewards/margins": 0.10963205248117447,
+      "rewards/rejected": -0.42965784668922424,
       "step": 740
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.062057096347338e-06,
-      "logits/chosen": -2.6757967472076416,
-      "logits/rejected": -2.683659553527832,
-      "logps/chosen": -449.1112365722656,
-      "logps/rejected": -460.0345764160156,
-      "loss": 0.6687,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -0.40121787786483765,
-      "rewards/margins": 0.07739736139774323,
-      "rewards/rejected": -0.47861528396606445,
+      "logits/chosen": -2.6860058307647705,
+      "logits/rejected": -2.694854736328125,
+      "logps/chosen": -461.962890625,
+      "logps/rejected": -472.4624938964844,
+      "loss": 0.6667,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.38755279779434204,
+      "rewards/margins": 0.08047433197498322,
+      "rewards/rejected": -0.46802711486816406,
       "step": 750
     },
     {
       "epoch": 0.61,
       "learning_rate": 1.9934423095525733e-06,
-      "logits/chosen": -2.660757541656494,
-      "logits/rejected": -2.641571521759033,
-      "logps/chosen": -453.0924377441406,
-      "logps/rejected": -462.0196228027344,
-      "loss": 0.6656,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.3819692134857178,
-      "rewards/margins": 0.08219519257545471,
-      "rewards/rejected": -0.4641643464565277,
+      "logits/chosen": -2.6688058376312256,
+      "logits/rejected": -2.6453909873962402,
+      "logps/chosen": -467.1205139160156,
+      "logps/rejected": -475.4339904785156,
+      "loss": 0.6708,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.385872483253479,
+      "rewards/margins": 0.07536690682172775,
+      "rewards/rejected": -0.46123939752578735,
       "step": 760
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.9252232243182986e-06,
-      "logits/chosen": -2.6494193077087402,
-      "logits/rejected": -2.6644535064697266,
-      "logps/chosen": -420.4537658691406,
-      "logps/rejected": -452.49700927734375,
-      "loss": 0.6659,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.4124923348426819,
-      "rewards/margins": 0.08118386566638947,
-      "rewards/rejected": -0.49367618560791016,
+      "logits/chosen": -2.656658172607422,
+      "logits/rejected": -2.6702752113342285,
+      "logps/chosen": -434.9510192871094,
+      "logps/rejected": -466.2872009277344,
+      "loss": 0.6669,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.4157305657863617,
+      "rewards/margins": 0.0795067697763443,
+      "rewards/rejected": -0.4952372908592224,
       "step": 770
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8574531305244043e-06,
-      "logits/chosen": -2.674877882003784,
-      "logits/rejected": -2.698631763458252,
-      "logps/chosen": -459.5841369628906,
-      "logps/rejected": -493.99969482421875,
-      "loss": 0.6453,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.4147106111049652,
-      "rewards/margins": 0.12960824370384216,
-      "rewards/rejected": -0.5443187952041626,
+      "logits/chosen": -2.683166980743408,
+      "logits/rejected": -2.705665111541748,
+      "logps/chosen": -473.3404846191406,
+      "logps/rejected": -507.530029296875,
+      "loss": 0.6445,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.39788818359375,
+      "rewards/margins": 0.13318848609924316,
+      "rewards/rejected": -0.5310766696929932,
       "step": 780
     },
     {
       "epoch": 0.63,
       "learning_rate": 1.7901849673175559e-06,
-      "logits/chosen": -2.69256329536438,
-      "logits/rejected": -2.7218129634857178,
-      "logps/chosen": -455.54736328125,
-      "logps/rejected": -486.2936096191406,
-      "loss": 0.66,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.38235577940940857,
-      "rewards/margins": 0.09029395878314972,
-      "rewards/rejected": -0.4726497530937195,
+      "logits/chosen": -2.7003445625305176,
+      "logits/rejected": -2.7315449714660645,
+      "logps/chosen": -467.36383056640625,
+      "logps/rejected": -495.95709228515625,
+      "loss": 0.6692,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.3450160622596741,
+      "rewards/margins": 0.07192648947238922,
+      "rewards/rejected": -0.4169425368309021,
       "step": 790
     },
     {
       "epoch": 0.64,
       "learning_rate": 1.7234712817573555e-06,
-      "logits/chosen": -2.670511484146118,
-      "logits/rejected": -2.6233620643615723,
-      "logps/chosen": -479.3063049316406,
-      "logps/rejected": -484.30718994140625,
-      "loss": 0.6841,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.45734840631484985,
-      "rewards/margins": 0.05288088321685791,
-      "rewards/rejected": -0.510229229927063,
+      "logits/chosen": -2.680144786834717,
+      "logits/rejected": -2.6287026405334473,
+      "logps/chosen": -486.04278564453125,
+      "logps/rejected": -492.35162353515625,
+      "loss": 0.6764,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.37955719232559204,
+      "rewards/margins": 0.06455205380916595,
+      "rewards/rejected": -0.4441092014312744,
       "step": 800
     },
     {
       "epoch": 0.64,
-      "eval_logits/chosen": -2.559168577194214,
-      "eval_logits/rejected": -2.56618070602417,
-      "eval_logps/chosen": -354.3205871582031,
-      "eval_logps/rejected": -370.067626953125,
-      "eval_loss": 0.6735012531280518,
-      "eval_rewards/accuracies": 0.5913333296775818,
-      "eval_rewards/chosen": -0.31825292110443115,
-      "eval_rewards/margins": 0.06049799174070358,
-      "eval_rewards/rejected": -0.37875089049339294,
-      "eval_runtime": 1585.6649,
-      "eval_samples_per_second": 1.888,
-      "eval_steps_per_second": 0.236,
+      "eval_logits/chosen": -2.5651443004608154,
+      "eval_logits/rejected": -2.571413516998291,
+      "eval_logps/chosen": -359.1413269042969,
+      "eval_logps/rejected": -374.912841796875,
+      "eval_loss": 0.672364354133606,
+      "eval_rewards/accuracies": 0.5989999771118164,
+      "eval_rewards/chosen": -0.24809196591377258,
+      "eval_rewards/margins": 0.06060539186000824,
+      "eval_rewards/rejected": -0.308697372674942,
+      "eval_runtime": 1620.9312,
+      "eval_samples_per_second": 1.847,
+      "eval_steps_per_second": 0.231,
       "step": 800
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6573641877687936e-06,
-      "logits/chosen": -2.688758373260498,
-      "logits/rejected": -2.7048513889312744,
-      "logps/chosen": -433.21484375,
-      "logps/rejected": -472.264892578125,
-      "loss": 0.6559,
+      "logits/chosen": -2.696709394454956,
+      "logits/rejected": -2.713754653930664,
+      "logps/chosen": -439.42864990234375,
+      "logps/rejected": -479.58612060546875,
+      "loss": 0.6545,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.3647409975528717,
-      "rewards/margins": 0.10189852863550186,
-      "rewards/rejected": -0.46663951873779297,
+      "rewards/chosen": -0.28763246536254883,
+      "rewards/margins": 0.10148018598556519,
+      "rewards/rejected": -0.389112651348114,
       "step": 810
     },
     {
       "epoch": 0.66,
       "learning_rate": 1.591915325433034e-06,
-      "logits/chosen": -2.705111265182495,
-      "logits/rejected": -2.690777540206909,
-      "logps/chosen": -436.6507263183594,
-      "logps/rejected": -449.94891357421875,
-      "loss": 0.6732,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.39074164628982544,
-      "rewards/margins": 0.06702588498592377,
-      "rewards/rejected": -0.4577675461769104,
+      "logits/chosen": -2.7134814262390137,
+      "logits/rejected": -2.6989309787750244,
+      "logps/chosen": -442.7323303222656,
+      "logps/rejected": -455.2479553222656,
+      "loss": 0.6775,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.31017300486564636,
+      "rewards/margins": 0.056103698909282684,
+      "rewards/rejected": -0.36627671122550964,
       "step": 820
     },
     {
       "epoch": 0.66,
       "learning_rate": 1.5271758206483664e-06,
-      "logits/chosen": -2.7012417316436768,
-      "logits/rejected": -2.692389726638794,
-      "logps/chosen": -457.00238037109375,
-      "logps/rejected": -475.9202575683594,
+      "logits/chosen": -2.7078943252563477,
+      "logits/rejected": -2.699531078338623,
+      "logps/chosen": -458.97772216796875,
+      "logps/rejected": -478.4125061035156,
       "loss": 0.6669,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.4308633804321289,
-      "rewards/margins": 0.09242836385965347,
-      "rewards/rejected": -0.5232917666435242,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.31177014112472534,
+      "rewards/margins": 0.0876721516251564,
+      "rewards/rejected": -0.39944231510162354,
       "step": 830
     },
     {
       "epoch": 0.67,
       "learning_rate": 1.4631962451927966e-06,
-      "logits/chosen": -2.675565719604492,
-      "logits/rejected": -2.708937168121338,
-      "logps/chosen": -453.7381286621094,
-      "logps/rejected": -485.38360595703125,
-      "loss": 0.6646,
+      "logits/chosen": -2.689883232116699,
+      "logits/rejected": -2.7185115814208984,
+      "logps/chosen": -453.9341735839844,
+      "logps/rejected": -486.0030822753906,
+      "loss": 0.6594,
       "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.42438727617263794,
-      "rewards/margins": 0.09076879173517227,
-      "rewards/rejected": -0.515156090259552,
+      "rewards/chosen": -0.2792655825614929,
+      "rewards/margins": 0.0935577005147934,
+      "rewards/rejected": -0.3728232979774475,
       "step": 840
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4000265772195032e-06,
-      "logits/chosen": -2.706509590148926,
-      "logits/rejected": -2.7089927196502686,
-      "logps/chosen": -460.16326904296875,
-      "logps/rejected": -492.61163330078125,
-      "loss": 0.6616,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.4706672132015228,
-      "rewards/margins": 0.09530263394117355,
-      "rewards/rejected": -0.5659698247909546,
+      "logits/chosen": -2.7169957160949707,
+      "logits/rejected": -2.7198026180267334,
+      "logps/chosen": -462.4480895996094,
+      "logps/rejected": -495.20330810546875,
+      "loss": 0.6628,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.3546587824821472,
+      "rewards/margins": 0.09027893841266632,
+      "rewards/rejected": -0.44493770599365234,
       "step": 850
     },
     {
       "epoch": 0.69,
       "learning_rate": 1.3377161622160077e-06,
-      "logits/chosen": -2.688765048980713,
-      "logits/rejected": -2.686649799346924,
-      "logps/chosen": -452.43035888671875,
-      "logps/rejected": -478.95855712890625,
-      "loss": 0.6575,
+      "logits/chosen": -2.7009379863739014,
+      "logits/rejected": -2.696585178375244,
+      "logps/chosen": -458.6473083496094,
+      "logps/rejected": -485.0048828125,
+      "loss": 0.6556,
       "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.4564797878265381,
-      "rewards/margins": 0.09994712471961975,
-      "rewards/rejected": -0.5564268827438354,
+      "rewards/chosen": -0.3736417591571808,
+      "rewards/margins": 0.10007522255182266,
+      "rewards/rejected": -0.4737170338630676,
       "step": 860
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.276313674457553e-06,
-      "logits/chosen": -2.687105417251587,
-      "logits/rejected": -2.6988847255706787,
-      "logps/chosen": -431.041259765625,
-      "logps/rejected": -468.3299865722656,
-      "loss": 0.644,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.4324752390384674,
-      "rewards/margins": 0.13809339702129364,
-      "rewards/rejected": -0.5705686211585999,
+      "logits/chosen": -2.700366497039795,
+      "logits/rejected": -2.7102417945861816,
+      "logps/chosen": -439.9691467285156,
+      "logps/rejected": -479.43231201171875,
+      "loss": 0.6373,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.3865216374397278,
+      "rewards/margins": 0.15378279983997345,
+      "rewards/rejected": -0.5403044819831848,
       "step": 870
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2158670789848095e-06,
-      "logits/chosen": -2.6659626960754395,
-      "logits/rejected": -2.6399731636047363,
-      "logps/chosen": -455.6881408691406,
-      "logps/rejected": -471.60479736328125,
-      "loss": 0.6906,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.4899720549583435,
-      "rewards/margins": 0.03855130821466446,
-      "rewards/rejected": -0.528523325920105,
+      "logits/chosen": -2.675424098968506,
+      "logits/rejected": -2.6478264331817627,
+      "logps/chosen": -472.1514587402344,
+      "logps/rejected": -488.82000732421875,
+      "loss": 0.6835,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.5139307379722595,
+      "rewards/margins": 0.05513748526573181,
+      "rewards/rejected": -0.5690682530403137,
       "step": 880
     },
     {
       "epoch": 0.71,
       "learning_rate": 1.1564235941356016e-06,
-      "logits/chosen": -2.636784791946411,
-      "logits/rejected": -2.665184497833252,
-      "logps/chosen": -443.228759765625,
-      "logps/rejected": -486.3818359375,
-      "loss": 0.6375,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.49858155846595764,
-      "rewards/margins": 0.15248027443885803,
-      "rewards/rejected": -0.6510618329048157,
+      "logits/chosen": -2.647432804107666,
+      "logits/rejected": -2.6728672981262207,
+      "logps/chosen": -459.1905212402344,
+      "logps/rejected": -504.0580139160156,
+      "loss": 0.6369,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.5291575789451599,
+      "rewards/margins": 0.15966546535491943,
+      "rewards/rejected": -0.6888231039047241,
       "step": 890
     },
     {
       "epoch": 0.72,
       "learning_rate": 1.0980296546599254e-06,
-      "logits/chosen": -2.6810688972473145,
-      "logits/rejected": -2.66475248336792,
-      "logps/chosen": -478.31512451171875,
-      "logps/rejected": -483.4214782714844,
-      "loss": 0.6773,
+      "logits/chosen": -2.6834964752197266,
+      "logits/rejected": -2.671112537384033,
+      "logps/chosen": -498.71478271484375,
+      "logps/rejected": -504.4794006347656,
+      "loss": 0.6753,
       "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.5605891942977905,
-      "rewards/margins": 0.07103095203638077,
-      "rewards/rejected": -0.6316200494766235,
+      "rewards/chosen": -0.6096609830856323,
+      "rewards/margins": 0.08308340609073639,
+      "rewards/rejected": -0.6927444934844971,
       "step": 900
     },
     {
       "epoch": 0.72,
-      "eval_logits/chosen": -2.5706326961517334,
-      "eval_logits/rejected": -2.577380418777466,
-      "eval_logps/chosen": -362.3546142578125,
-      "eval_logps/rejected": -378.96929931640625,
-      "eval_loss": 0.6724240183830261,
-      "eval_rewards/accuracies": 0.5886666774749756,
-      "eval_rewards/chosen": -0.3985931873321533,
-      "eval_rewards/margins": 0.06917466968297958,
-      "eval_rewards/rejected": -0.4677678346633911,
-      "eval_runtime": 1591.0139,
-      "eval_samples_per_second": 1.882,
-      "eval_steps_per_second": 0.236,
+      "eval_logits/chosen": -2.5749759674072266,
+      "eval_logits/rejected": -2.5806519985198975,
+      "eval_logps/chosen": -377.15924072265625,
+      "eval_logps/rejected": -394.6670837402344,
+      "eval_loss": 0.6703553795814514,
+      "eval_rewards/accuracies": 0.5983333587646484,
+      "eval_rewards/chosen": -0.4282708764076233,
+      "eval_rewards/margins": 0.0779690071940422,
+      "eval_rewards/rejected": -0.5062398910522461,
+      "eval_runtime": 1623.0619,
+      "eval_samples_per_second": 1.845,
+      "eval_steps_per_second": 0.231,
       "step": 900
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.040730875447083e-06,
-      "logits/chosen": -2.6869008541107178,
-      "logits/rejected": -2.6931540966033936,
-      "logps/chosen": -458.8529357910156,
-      "logps/rejected": -475.8218688964844,
-      "loss": 0.6623,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.47533708810806274,
-      "rewards/margins": 0.09188753366470337,
-      "rewards/rejected": -0.5672245621681213,
+      "logits/chosen": -2.694941997528076,
+      "logits/rejected": -2.698484182357788,
+      "logps/chosen": -475.06201171875,
+      "logps/rejected": -494.41033935546875,
+      "loss": 0.656,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.5018836259841919,
+      "rewards/margins": 0.11171841621398926,
+      "rewards/rejected": -0.6136020421981812,
       "step": 910
     },
     {
       "epoch": 0.74,
       "learning_rate": 9.845720158932414e-07,
-      "logits/chosen": -2.655848741531372,
-      "logits/rejected": -2.650817632675171,
-      "logps/chosen": -427.18896484375,
-      "logps/rejected": -438.2098693847656,
-      "loss": 0.6659,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -0.4379213750362396,
-      "rewards/margins": 0.079404316842556,
-      "rewards/rejected": -0.5173256993293762,
+      "logits/chosen": -2.6612589359283447,
+      "logits/rejected": -2.6566224098205566,
+      "logps/chosen": -442.80841064453125,
+      "logps/rejected": -453.392578125,
+      "loss": 0.6669,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.4533756375312805,
+      "rewards/margins": 0.07945097982883453,
+      "rewards/rejected": -0.5328266024589539,
       "step": 920
     },
     {
       "epoch": 0.74,
       "learning_rate": 9.295969449372796e-07,
-      "logits/chosen": -2.682236433029175,
-      "logits/rejected": -2.706988573074341,
-      "logps/chosen": -434.9564514160156,
-      "logps/rejected": -458.70831298828125,
-      "loss": 0.6677,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.38662606477737427,
-      "rewards/margins": 0.08299224078655243,
-      "rewards/rejected": -0.4696182608604431,
+      "logits/chosen": -2.6861672401428223,
+      "logits/rejected": -2.712158441543579,
+      "logps/chosen": -449.2447204589844,
+      "logps/rejected": -474.03314208984375,
+      "loss": 0.6672,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.39123302698135376,
+      "rewards/margins": 0.08886702358722687,
+      "rewards/rejected": -0.4801000654697418,
       "step": 930
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.758486067922176e-07,
-      "logits/chosen": -2.68632435798645,
-      "logits/rejected": -2.700981616973877,
-      "logps/chosen": -425.8504943847656,
-      "logps/rejected": -465.1412048339844,
-      "loss": 0.651,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.372600257396698,
-      "rewards/margins": 0.1192459687590599,
-      "rewards/rejected": -0.4918461740016937,
+      "logits/chosen": -2.6912190914154053,
+      "logits/rejected": -2.708655834197998,
+      "logps/chosen": -439.08233642578125,
+      "logps/rejected": -478.17950439453125,
+      "loss": 0.6498,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.35874584317207336,
+      "rewards/margins": 0.1233411431312561,
+      "rewards/rejected": -0.48208698630332947,
       "step": 940
     },
     {
       "epoch": 0.76,
       "learning_rate": 8.233689873990006e-07,
-      "logits/chosen": -2.661367654800415,
-      "logits/rejected": -2.663963794708252,
-      "logps/chosen": -430.2080993652344,
-      "logps/rejected": -466.76092529296875,
-      "loss": 0.6451,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.3554816246032715,
-      "rewards/margins": 0.12667140364646912,
-      "rewards/rejected": -0.4821530282497406,
+      "logits/chosen": -2.672585964202881,
+      "logits/rejected": -2.679046392440796,
+      "logps/chosen": -440.7559509277344,
+      "logps/rejected": -479.00164794921875,
+      "loss": 0.6422,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.32873159646987915,
+      "rewards/margins": 0.13754822313785553,
+      "rewards/rejected": -0.4662798345088959,
       "step": 950
     },
     {
       "epoch": 0.77,
       "learning_rate": 7.721990816288555e-07,
-      "logits/chosen": -2.7383949756622314,
-      "logits/rejected": -2.7589166164398193,
-      "logps/chosen": -413.57135009765625,
-      "logps/rejected": -441.5545349121094,
-      "loss": 0.6724,
+      "logits/chosen": -2.7491555213928223,
+      "logits/rejected": -2.769662380218506,
+      "logps/chosen": -422.5555114746094,
+      "logps/rejected": -451.8642578125,
+      "loss": 0.6705,
       "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.371616005897522,
-      "rewards/margins": 0.06721927225589752,
-      "rewards/rejected": -0.4388352930545807,
+      "rewards/chosen": -0.32741206884384155,
+      "rewards/margins": 0.07545635104179382,
+      "rewards/rejected": -0.4028684198856354,
       "step": 960
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.223788612598148e-07,
-      "logits/chosen": -2.7295823097229004,
-      "logits/rejected": -2.740691661834717,
-      "logps/chosen": -443.89776611328125,
-      "logps/rejected": -474.42254638671875,
-      "loss": 0.6495,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.332157701253891,
-      "rewards/margins": 0.1200638860464096,
-      "rewards/rejected": -0.4522215723991394,
+      "logits/chosen": -2.7415757179260254,
+      "logits/rejected": -2.750936269760132,
+      "logps/chosen": -452.7373962402344,
+      "logps/rejected": -484.138671875,
+      "loss": 0.6439,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.27024340629577637,
+      "rewards/margins": 0.12849459052085876,
+      "rewards/rejected": -0.39873799681663513,
       "step": 970
     },
     {
       "epoch": 0.78,
       "learning_rate": 6.73947243752448e-07,
-      "logits/chosen": -2.70650053024292,
-      "logits/rejected": -2.6953940391540527,
-      "logps/chosen": -460.09918212890625,
-      "logps/rejected": -455.58489990234375,
-      "loss": 0.6806,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.4144526422023773,
-      "rewards/margins": 0.05592719465494156,
-      "rewards/rejected": -0.4703798294067383,
+      "logits/chosen": -2.718167304992676,
+      "logits/rejected": -2.7125675678253174,
+      "logps/chosen": -470.5419921875,
+      "logps/rejected": -465.5101013183594,
+      "loss": 0.6762,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.36285415291786194,
+      "rewards/margins": 0.06569617241621017,
+      "rewards/rejected": -0.4285503029823303,
       "step": 980
     },
     {
       "epoch": 0.79,
       "learning_rate": 6.269420618491759e-07,
-      "logits/chosen": -2.7356605529785156,
-      "logits/rejected": -2.737133026123047,
-      "logps/chosen": -423.57537841796875,
-      "logps/rejected": -452.10260009765625,
-      "loss": 0.6665,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.3517349660396576,
-      "rewards/margins": 0.08071886003017426,
-      "rewards/rejected": -0.43245387077331543,
+      "logits/chosen": -2.75072979927063,
+      "logits/rejected": -2.751321315765381,
+      "logps/chosen": -432.2750549316406,
+      "logps/rejected": -462.3431701660156,
+      "loss": 0.6574,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2954108715057373,
+      "rewards/margins": 0.09935091435909271,
+      "rewards/rejected": -0.3947618007659912,
       "step": 990
     },
     {
       "epoch": 0.8,
       "learning_rate": 5.814000340209267e-07,
-      "logits/chosen": -2.7007267475128174,
-      "logits/rejected": -2.700061798095703,
-      "logps/chosen": -442.41705322265625,
-      "logps/rejected": -477.9020080566406,
-      "loss": 0.657,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.34119993448257446,
-      "rewards/margins": 0.10114644467830658,
-      "rewards/rejected": -0.44234639406204224,
+      "logits/chosen": -2.7202508449554443,
+      "logits/rejected": -2.721200466156006,
+      "logps/chosen": -453.22100830078125,
+      "logps/rejected": -491.32080078125,
+      "loss": 0.6459,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.30317172408103943,
+      "rewards/margins": 0.12803982198238373,
+      "rewards/rejected": -0.43121156096458435,
       "step": 1000
     },
     {
       "epoch": 0.8,
-      "eval_logits/chosen": -2.570801258087158,
-      "eval_logits/rejected": -2.578376293182373,
-      "eval_logps/chosen": -350.23724365234375,
-      "eval_logps/rejected": -366.5909118652344,
-      "eval_loss": 0.6711118221282959,
-      "eval_rewards/accuracies": 0.5996666550636292,
-      "eval_rewards/chosen": -0.277418851852417,
-      "eval_rewards/margins": 0.06656476855278015,
-      "eval_rewards/rejected": -0.34398359060287476,
-      "eval_runtime": 1581.2525,
-      "eval_samples_per_second": 1.893,
-      "eval_steps_per_second": 0.237,
+      "eval_logits/chosen": -2.5858356952667236,
+      "eval_logits/rejected": -2.59238338470459,
+      "eval_logps/chosen": -358.3893737792969,
+      "eval_logps/rejected": -375.67333984375,
+      "eval_loss": 0.6679545640945435,
+      "eval_rewards/accuracies": 0.612666666507721,
+      "eval_rewards/chosen": -0.24057278037071228,
+      "eval_rewards/margins": 0.07572992891073227,
+      "eval_rewards/rejected": -0.31630268692970276,
+      "eval_runtime": 1618.0765,
+      "eval_samples_per_second": 1.85,
+      "eval_steps_per_second": 0.232,
       "step": 1000
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.373567357842111e-07,
-      "logits/chosen": -2.6614608764648438,
-      "logits/rejected": -2.694235324859619,
-      "logps/chosen": -432.1556091308594,
-      "logps/rejected": -458.6222229003906,
-      "loss": 0.6625,
+      "logits/chosen": -2.6775119304656982,
+      "logits/rejected": -2.707977771759033,
+      "logps/chosen": -443.8631896972656,
+      "logps/rejected": -472.0171813964844,
+      "loss": 0.6591,
       "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.2784090042114258,
-      "rewards/margins": 0.0895313173532486,
-      "rewards/rejected": -0.36794033646583557,
+      "rewards/chosen": -0.25422340631484985,
+      "rewards/margins": 0.09956096857786179,
+      "rewards/rejected": -0.35378438234329224,
       "step": 1010
     },
     {
       "epoch": 0.82,
       "learning_rate": 4.948465719110226e-07,
-      "logits/chosen": -2.6836206912994385,
-      "logits/rejected": -2.697563409805298,
-      "logps/chosen": -415.11041259765625,
-      "logps/rejected": -438.9033203125,
-      "loss": 0.6496,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.31501251459121704,
-      "rewards/margins": 0.11396019160747528,
-      "rewards/rejected": -0.4289727210998535,
+      "logits/chosen": -2.69923996925354,
+      "logits/rejected": -2.714949607849121,
+      "logps/chosen": -425.83245849609375,
+      "logps/rejected": -450.9254455566406,
+      "loss": 0.6449,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2896645665168762,
+      "rewards/margins": 0.12861916422843933,
+      "rewards/rejected": -0.41828376054763794,
       "step": 1020
     },
     {
       "epoch": 0.82,
       "learning_rate": 4.539027495532766e-07,
-      "logits/chosen": -2.6660382747650146,
-      "logits/rejected": -2.6685073375701904,
-      "logps/chosen": -425.77490234375,
-      "logps/rejected": -446.0552673339844,
-      "loss": 0.6661,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.35333576798439026,
-      "rewards/margins": 0.08899492025375366,
-      "rewards/rejected": -0.4423306882381439,
+      "logits/chosen": -2.6870169639587402,
+      "logits/rejected": -2.6863129138946533,
+      "logps/chosen": -437.5118713378906,
+      "logps/rejected": -460.01373291015625,
+      "loss": 0.6558,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.3346652388572693,
+      "rewards/margins": 0.11685864627361298,
+      "rewards/rejected": -0.45152387022972107,
       "step": 1030
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.14557252302783e-07,
-      "logits/chosen": -2.6789193153381348,
-      "logits/rejected": -2.6913952827453613,
-      "logps/chosen": -429.2127990722656,
-      "logps/rejected": -461.7294006347656,
-      "loss": 0.6592,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.3878576159477234,
-      "rewards/margins": 0.0975080356001854,
-      "rewards/rejected": -0.48536568880081177,
+      "logits/chosen": -2.691601276397705,
+      "logits/rejected": -2.7064461708068848,
+      "logps/chosen": -443.50445556640625,
+      "logps/rejected": -478.5107421875,
+      "loss": 0.6522,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3916120231151581,
+      "rewards/margins": 0.11733059585094452,
+      "rewards/rejected": -0.5089426636695862,
       "step": 1040
     },
     {
       "epoch": 0.84,
       "learning_rate": 3.7684081520700884e-07,
-      "logits/chosen": -2.6964402198791504,
-      "logits/rejected": -2.70263934135437,
-      "logps/chosen": -462.135986328125,
-      "logps/rejected": -464.80035400390625,
-      "loss": 0.6855,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.437302827835083,
-      "rewards/margins": 0.041490085422992706,
-      "rewards/rejected": -0.4787929654121399,
+      "logits/chosen": -2.716027021408081,
+      "logits/rejected": -2.721525192260742,
+      "logps/chosen": -477.7105407714844,
+      "logps/rejected": -481.0179748535156,
+      "loss": 0.6819,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.4463655352592468,
+      "rewards/margins": 0.05458784103393555,
+      "rewards/rejected": -0.5009533166885376,
       "step": 1050
     },
     {
       "epoch": 0.85,
       "learning_rate": 3.407829007601507e-07,
-      "logits/chosen": -2.6650023460388184,
-      "logits/rejected": -2.6826632022857666,
-      "logps/chosen": -443.31982421875,
-      "logps/rejected": -482.0733947753906,
-      "loss": 0.6466,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.37219929695129395,
-      "rewards/margins": 0.13468633592128754,
-      "rewards/rejected": -0.5068856477737427,
+      "logits/chosen": -2.6834683418273926,
+      "logits/rejected": -2.6970880031585693,
+      "logps/chosen": -457.0318298339844,
+      "logps/rejected": -498.78485107421875,
+      "loss": 0.6357,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.36882373690605164,
+      "rewards/margins": 0.1661624312400818,
+      "rewards/rejected": -0.5349861979484558,
       "step": 1060
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.064116758882724e-07,
-      "logits/chosen": -2.704041004180908,
-      "logits/rejected": -2.712378978729248,
-      "logps/chosen": -449.221435546875,
-      "logps/rejected": -488.82733154296875,
-      "loss": 0.6569,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.38919639587402344,
-      "rewards/margins": 0.10973678529262543,
-      "rewards/rejected": -0.49893322587013245,
+      "logits/chosen": -2.7176353931427,
+      "logits/rejected": -2.7239794731140137,
+      "logps/chosen": -464.91534423828125,
+      "logps/rejected": -508.00927734375,
+      "loss": 0.6459,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.38777628540992737,
+      "rewards/margins": 0.14257176220417023,
+      "rewards/rejected": -0.5303480625152588,
       "step": 1070
     },
     {
       "epoch": 0.86,
       "learning_rate": 2.737539899464908e-07,
-      "logits/chosen": -2.66728138923645,
-      "logits/rejected": -2.6824071407318115,
-      "logps/chosen": -408.9554138183594,
-      "logps/rejected": -435.3258361816406,
-      "loss": 0.6621,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.34276992082595825,
-      "rewards/margins": 0.08587387949228287,
-      "rewards/rejected": -0.4286437928676605,
+      "logits/chosen": -2.6851329803466797,
+      "logits/rejected": -2.698014736175537,
+      "logps/chosen": -423.3038024902344,
+      "logps/rejected": -450.4856872558594,
+      "loss": 0.6599,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.3559909462928772,
+      "rewards/margins": 0.09332160651683807,
+      "rewards/rejected": -0.44931259751319885,
       "step": 1080
     },
     {
       "epoch": 0.87,
       "learning_rate": 2.4283535374538645e-07,
-      "logits/chosen": -2.731504440307617,
-      "logits/rejected": -2.733990430831909,
-      "logps/chosen": -450.702392578125,
-      "logps/rejected": -473.677490234375,
-      "loss": 0.6486,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.3203676640987396,
-      "rewards/margins": 0.12553098797798157,
-      "rewards/rejected": -0.4458986818790436,
+      "logits/chosen": -2.744837999343872,
+      "logits/rejected": -2.747127056121826,
+      "logps/chosen": -465.623046875,
+      "logps/rejected": -489.3487243652344,
+      "loss": 0.6522,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.3236229419708252,
+      "rewards/margins": 0.12912145256996155,
+      "rewards/rejected": -0.45274439454078674,
       "step": 1090
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.1367991962303298e-07,
-      "logits/chosen": -2.6853950023651123,
-      "logits/rejected": -2.702267646789551,
-      "logps/chosen": -415.25048828125,
-      "logps/rejected": -443.6421813964844,
-      "loss": 0.6577,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.361422061920166,
-      "rewards/margins": 0.10248668491840363,
-      "rewards/rejected": -0.46390876173973083,
+      "logits/chosen": -2.7007431983947754,
+      "logits/rejected": -2.71804141998291,
+      "logps/chosen": -428.20147705078125,
+      "logps/rejected": -458.3008728027344,
+      "loss": 0.6541,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.3504807949066162,
+      "rewards/margins": 0.11653472483158112,
+      "rewards/rejected": -0.46701550483703613,
       "step": 1100
     },
     {
       "epoch": 0.88,
-      "eval_logits/chosen": -2.5729336738586426,
-      "eval_logits/rejected": -2.580476999282837,
-      "eval_logps/chosen": -351.83758544921875,
-      "eval_logps/rejected": -368.46795654296875,
-      "eval_loss": 0.6706418395042419,
-      "eval_rewards/accuracies": 0.5993333458900452,
-      "eval_rewards/chosen": -0.29342252016067505,
-      "eval_rewards/margins": 0.06933192163705826,
-      "eval_rewards/rejected": -0.3627544641494751,
-      "eval_runtime": 1589.982,
-      "eval_samples_per_second": 1.883,
-      "eval_steps_per_second": 0.236,
+      "eval_logits/chosen": -2.587815046310425,
+      "eval_logits/rejected": -2.594228744506836,
+      "eval_logps/chosen": -362.3882141113281,
+      "eval_logps/rejected": -380.2968444824219,
+      "eval_loss": 0.6669934988021851,
+      "eval_rewards/accuracies": 0.6156666874885559,
+      "eval_rewards/chosen": -0.28056082129478455,
+      "eval_rewards/margins": 0.08197694271802902,
+      "eval_rewards/rejected": -0.36253777146339417,
+      "eval_runtime": 1611.4517,
+      "eval_samples_per_second": 1.858,
+      "eval_steps_per_second": 0.233,
       "step": 1100
     },
     {
       "epoch": 0.89,
       "learning_rate": 1.8631046257820278e-07,
-      "logits/chosen": -2.713442802429199,
-      "logits/rejected": -2.7153563499450684,
-      "logps/chosen": -448.9380798339844,
-      "logps/rejected": -459.6236267089844,
-      "loss": 0.6822,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.40070003271102905,
-      "rewards/margins": 0.04853721335530281,
-      "rewards/rejected": -0.44923728704452515,
+      "logits/chosen": -2.728276014328003,
+      "logits/rejected": -2.7310478687286377,
+      "logps/chosen": -461.82879638671875,
+      "logps/rejected": -471.50775146484375,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.3837016522884369,
+      "rewards/margins": 0.04261630028486252,
+      "rewards/rejected": -0.4263179898262024,
       "step": 1110
     },
     {
       "epoch": 0.9,
       "learning_rate": 1.6074836247950143e-07,
-      "logits/chosen": -2.7305519580841064,
-      "logits/rejected": -2.727928638458252,
-      "logps/chosen": -458.64208984375,
-      "logps/rejected": -479.4064025878906,
-      "loss": 0.6573,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.35037961602211,
-      "rewards/margins": 0.09948080033063889,
-      "rewards/rejected": -0.4498603940010071,
+      "logits/chosen": -2.7541463375091553,
+      "logits/rejected": -2.7452800273895264,
+      "logps/chosen": -471.38446044921875,
+      "logps/rejected": -493.8583068847656,
+      "loss": 0.656,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.32993000745773315,
+      "rewards/margins": 0.11021927744150162,
+      "rewards/rejected": -0.44014930725097656,
       "step": 1120
     },
     {
       "epoch": 0.9,
       "learning_rate": 1.370135873643097e-07,
-      "logits/chosen": -2.6915557384490967,
-      "logits/rejected": -2.675395965576172,
-      "logps/chosen": -444.5713806152344,
-      "logps/rejected": -442.8504943847656,
-      "loss": 0.6738,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.3383481502532959,
-      "rewards/margins": 0.06244407966732979,
-      "rewards/rejected": -0.4007922112941742,
+      "logits/chosen": -2.711472749710083,
+      "logits/rejected": -2.6935694217681885,
+      "logps/chosen": -458.6504821777344,
+      "logps/rejected": -458.78839111328125,
+      "loss": 0.6649,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.33027976751327515,
+      "rewards/margins": 0.08629810065031052,
+      "rewards/rejected": -0.41657787561416626,
       "step": 1130
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.1512467784059372e-07,
-      "logits/chosen": -2.697908639907837,
-      "logits/rejected": -2.7043871879577637,
-      "logps/chosen": -409.3360595703125,
-      "logps/rejected": -430.56939697265625,
-      "loss": 0.6764,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -0.353982150554657,
-      "rewards/margins": 0.07203380763530731,
-      "rewards/rejected": -0.4260159432888031,
+      "logits/chosen": -2.7174072265625,
+      "logits/rejected": -2.726274013519287,
+      "logps/chosen": -421.85833740234375,
+      "logps/rejected": -443.1368713378906,
+      "loss": 0.6782,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.3433849811553955,
+      "rewards/margins": 0.07356986403465271,
+      "rewards/rejected": -0.4169548451900482,
       "step": 1140
     },
     {
       "epoch": 0.92,
       "learning_rate": 9.509873260376251e-08,
-      "logits/chosen": -2.6704964637756348,
-      "logits/rejected": -2.680117130279541,
-      "logps/chosen": -426.01568603515625,
-      "logps/rejected": -481.14056396484375,
-      "loss": 0.6301,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.292033851146698,
-      "rewards/margins": 0.16145673394203186,
-      "rewards/rejected": -0.45349058508872986,
+      "logits/chosen": -2.6908252239227295,
+      "logits/rejected": -2.697906017303467,
+      "logps/chosen": -438.10089111328125,
+      "logps/rejected": -496.07269287109375,
+      "loss": 0.6257,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2791033387184143,
+      "rewards/margins": 0.17611822485923767,
+      "rewards/rejected": -0.455221563577652,
       "step": 1150
     },
     {
       "epoch": 0.93,
       "learning_rate": 7.695139507988559e-08,
-      "logits/chosen": -2.690162181854248,
-      "logits/rejected": -2.6840763092041016,
-      "logps/chosen": -455.21429443359375,
-      "logps/rejected": -471.9693908691406,
-      "loss": 0.6911,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.39844316244125366,
-      "rewards/margins": 0.04045991227030754,
-      "rewards/rejected": -0.4389030337333679,
+      "logits/chosen": -2.7109904289245605,
+      "logits/rejected": -2.704934597015381,
+      "logps/chosen": -467.7078552246094,
+      "logps/rejected": -485.38580322265625,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.37871813774108887,
+      "rewards/margins": 0.04057822376489639,
+      "rewards/rejected": -0.41929635405540466,
       "step": 1160
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.069684120570684e-08,
-      "logits/chosen": -2.6355466842651367,
-      "logits/rejected": -2.6722214221954346,
-      "logps/chosen": -428.15887451171875,
-      "logps/rejected": -469.9241638183594,
-      "loss": 0.6556,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.31940728425979614,
-      "rewards/margins": 0.10891290009021759,
-      "rewards/rejected": -0.42832016944885254,
+      "logits/chosen": -2.6606979370117188,
+      "logits/rejected": -2.6916613578796387,
+      "logps/chosen": -442.80462646484375,
+      "logps/rejected": -487.12701416015625,
+      "loss": 0.6493,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.3210916817188263,
+      "rewards/margins": 0.13079658150672913,
+      "rewards/rejected": -0.45188823342323303,
       "step": 1170
     },
     {
       "epoch": 0.94,
       "learning_rate": 4.634776835499871e-08,
-      "logits/chosen": -2.648555278778076,
-      "logits/rejected": -2.648024082183838,
-      "logps/chosen": -414.77166748046875,
-      "logps/rejected": -438.7622985839844,
-      "loss": 0.6786,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.33630624413490295,
-      "rewards/margins": 0.059269875288009644,
-      "rewards/rejected": -0.3955760896205902,
+      "logits/chosen": -2.669241428375244,
+      "logits/rejected": -2.6664905548095703,
+      "logps/chosen": -426.0979919433594,
+      "logps/rejected": -453.2581481933594,
+      "loss": 0.6681,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.31668686866760254,
+      "rewards/margins": 0.09003134816884995,
+      "rewards/rejected": -0.4067181944847107,
       "step": 1180
     },
     {
       "epoch": 0.95,
       "learning_rate": 3.3915385419908964e-08,
-      "logits/chosen": -2.724853992462158,
-      "logits/rejected": -2.7020461559295654,
-      "logps/chosen": -443.90399169921875,
-      "logps/rejected": -462.37384033203125,
-      "loss": 0.667,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.35228732228279114,
-      "rewards/margins": 0.08694271743297577,
-      "rewards/rejected": -0.4392300546169281,
+      "logits/chosen": -2.7456703186035156,
+      "logits/rejected": -2.72196626663208,
+      "logps/chosen": -457.28515625,
+      "logps/rejected": -476.6426696777344,
+      "loss": 0.6645,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.33392366766929626,
+      "rewards/margins": 0.10097813606262207,
+      "rewards/rejected": -0.43490180373191833,
       "step": 1190
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3409404055043938e-08,
-      "logits/chosen": -2.6520884037017822,
-      "logits/rejected": -2.677919387817383,
-      "logps/chosen": -431.74053955078125,
-      "logps/rejected": -462.65179443359375,
-      "loss": 0.6444,
+      "logits/chosen": -2.6708192825317383,
+      "logits/rejected": -2.6985325813293457,
+      "logps/chosen": -443.9369201660156,
+      "logps/rejected": -475.53955078125,
+      "loss": 0.6422,
       "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.3235710561275482,
-      "rewards/margins": 0.12747570872306824,
-      "rewards/rejected": -0.45104676485061646,
+      "rewards/chosen": -0.3015444874763489,
+      "rewards/margins": 0.1372503787279129,
+      "rewards/rejected": -0.43879491090774536,
       "step": 1200
     },
     {
       "epoch": 0.96,
-      "eval_logits/chosen": -2.5725319385528564,
-      "eval_logits/rejected": -2.580101490020752,
-      "eval_logps/chosen": -351.09490966796875,
-      "eval_logps/rejected": -367.6591796875,
-      "eval_loss": 0.670768678188324,
-      "eval_rewards/accuracies": 0.5993333458900452,
-      "eval_rewards/chosen": -0.2859960198402405,
-      "eval_rewards/margins": 0.06867069751024246,
-      "eval_rewards/rejected": -0.35466670989990234,
-      "eval_runtime": 1585.5273,
-      "eval_samples_per_second": 1.888,
-      "eval_steps_per_second": 0.237,
+      "eval_logits/chosen": -2.5898184776306152,
+      "eval_logits/rejected": -2.596299409866333,
+      "eval_logps/chosen": -360.897216796875,
+      "eval_logps/rejected": -378.7738342285156,
+      "eval_loss": 0.6668795943260193,
+      "eval_rewards/accuracies": 0.6156666874885559,
+      "eval_rewards/chosen": -0.26565054059028625,
+      "eval_rewards/margins": 0.08165695518255234,
+      "eval_rewards/rejected": -0.3473075032234192,
+      "eval_runtime": 1611.7613,
+      "eval_samples_per_second": 1.858,
+      "eval_steps_per_second": 0.233,
       "step": 1200
     },
     {
       "epoch": 0.97,
       "learning_rate": 1.4838031091134186e-08,
-      "logits/chosen": -2.6522107124328613,
-      "logits/rejected": -2.6851000785827637,
-      "logps/chosen": -404.271484375,
-      "logps/rejected": -455.322998046875,
-      "loss": 0.6397,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.3024134933948517,
-      "rewards/margins": 0.13375119864940643,
-      "rewards/rejected": -0.4361646771430969,
+      "logits/chosen": -2.6734347343444824,
+      "logits/rejected": -2.7052102088928223,
+      "logps/chosen": -414.99566650390625,
+      "logps/rejected": -469.2049255371094,
+      "loss": 0.6318,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.2892751097679138,
+      "rewards/margins": 0.15364858508110046,
+      "rewards/rejected": -0.4429236948490143,
       "step": 1210
     },
     {
       "epoch": 0.98,
       "learning_rate": 8.207962124201774e-09,
-      "logits/chosen": -2.687406063079834,
-      "logits/rejected": -2.6915080547332764,
-      "logps/chosen": -446.8517150878906,
-      "logps/rejected": -463.1326599121094,
-      "loss": 0.6692,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.33363211154937744,
-      "rewards/margins": 0.08175511658191681,
-      "rewards/rejected": -0.41538724303245544,
+      "logits/chosen": -2.7094740867614746,
+      "logits/rejected": -2.7131965160369873,
+      "logps/chosen": -459.8565979003906,
+      "logps/rejected": -477.9366149902344,
+      "loss": 0.6578,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.3082069456577301,
+      "rewards/margins": 0.10618587583303452,
+      "rewards/rejected": -0.4143927991390228,
       "step": 1220
     },
     {
       "epoch": 0.98,
       "learning_rate": 3.5243762852441023e-09,
-      "logits/chosen": -2.68792462348938,
-      "logits/rejected": -2.705341339111328,
-      "logps/chosen": -433.271484375,
-      "logps/rejected": -466.77362060546875,
-      "loss": 0.6609,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.34372276067733765,
-      "rewards/margins": 0.1005091518163681,
-      "rewards/rejected": -0.44423192739486694,
+      "logits/chosen": -2.7074780464172363,
+      "logits/rejected": -2.7236151695251465,
+      "logps/chosen": -444.84161376953125,
+      "logps/rejected": -480.5126037597656,
+      "loss": 0.6562,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.31734251976013184,
+      "rewards/margins": 0.11356744915246964,
+      "rewards/rejected": -0.4309099614620209,
       "step": 1230
     },
     {
       "epoch": 0.99,
       "learning_rate": 7.909321945129278e-10,
-      "logits/chosen": -2.704862117767334,
-      "logits/rejected": -2.7122840881347656,
-      "logps/chosen": -461.7477111816406,
-      "logps/rejected": -496.6314392089844,
-      "loss": 0.6514,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.3258623480796814,
-      "rewards/margins": 0.12274013459682465,
-      "rewards/rejected": -0.4486025273799896,
+      "logits/chosen": -2.7246460914611816,
+      "logits/rejected": -2.733558177947998,
+      "logps/chosen": -476.4629821777344,
+      "logps/rejected": -513.1619873046875,
+      "loss": 0.6426,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.3152047097682953,
+      "rewards/margins": 0.15284833312034607,
+      "rewards/rejected": -0.46805301308631897,
       "step": 1240
     },
     {
       "epoch": 1.0,
       "step": 1249,
       "total_flos": 0.0,
-      "train_loss": 0.67484232365751,
-      "train_runtime": 40604.4701,
-      "train_samples_per_second": 0.492,
-      "train_steps_per_second": 0.031
+      "train_loss": 0.6736802890646257,
+      "train_runtime": 42164.1735,
+      "train_samples_per_second": 0.474,
+      "train_steps_per_second": 0.03
     }
   ],
   "logging_steps": 10,