diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -1,17 +1,17 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 3.0,
   "eval_steps": 100,
-  "global_step": 478,
+  "global_step": 1434,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 44.587841815889654,
-      "learning_rate": 1.0416666666666666e-08,
+      "grad_norm": 46.495066480188626,
+      "learning_rate": 3.4722222222222217e-09,
       "logits/chosen": -2.7660439014434814,
       "logits/rejected": -2.717564582824707,
       "logps/chosen": -269.8568420410156,
@@ -25,787 +25,2387 @@
     },
     {
       "epoch": 0.02,
-      "grad_norm": 44.96187115792532,
-      "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -2.5923614501953125,
-      "logits/rejected": -2.5627024173736572,
-      "logps/chosen": -264.8323974609375,
-      "logps/rejected": -251.58889770507812,
-      "loss": 0.6925,
-      "rewards/accuracies": 0.4583333432674408,
-      "rewards/chosen": -0.0015533770201727748,
-      "rewards/margins": 0.0002434460911899805,
-      "rewards/rejected": -0.0017968227621167898,
+      "grad_norm": 45.48250079754631,
+      "learning_rate": 3.472222222222222e-08,
+      "logits/chosen": -2.592761516571045,
+      "logits/rejected": -2.5630030632019043,
+      "logps/chosen": -264.7732238769531,
+      "logps/rejected": -251.50889587402344,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.3958333432674408,
+      "rewards/chosen": 0.0012306140270084143,
+      "rewards/margins": -0.0009704786934889853,
+      "rewards/rejected": 0.0022010933607816696,
       "step": 10
     },
     {
       "epoch": 0.04,
-      "grad_norm": 40.03767684952181,
-      "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -2.6519687175750732,
-      "logits/rejected": -2.603839635848999,
-      "logps/chosen": -282.8563537597656,
-      "logps/rejected": -298.2405700683594,
-      "loss": 0.6854,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.05863628536462784,
-      "rewards/margins": 0.013047178275883198,
-      "rewards/rejected": -0.07168345898389816,
+      "grad_norm": 41.93789081750966,
+      "learning_rate": 6.944444444444444e-08,
+      "logits/chosen": -2.6552841663360596,
+      "logits/rejected": -2.6074695587158203,
+      "logps/chosen": -281.4732360839844,
+      "logps/rejected": -296.79010009765625,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.004006093833595514,
+      "rewards/margins": 0.003165980102494359,
+      "rewards/rejected": 0.0008401140803471208,
       "step": 20
     },
     {
       "epoch": 0.06,
-      "grad_norm": 37.02758622940259,
-      "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -2.6573214530944824,
-      "logits/rejected": -2.5821313858032227,
-      "logps/chosen": -308.3706359863281,
-      "logps/rejected": -272.5185546875,
-      "loss": 0.6565,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.3464459776878357,
-      "rewards/margins": 0.08197540044784546,
-      "rewards/rejected": -0.42842140793800354,
+      "grad_norm": 41.15398691411152,
+      "learning_rate": 1.0416666666666667e-07,
+      "logits/chosen": -2.67468523979187,
+      "logits/rejected": -2.6028592586517334,
+      "logps/chosen": -300.5101013183594,
+      "logps/rejected": -263.88922119140625,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.008085623383522034,
+      "rewards/margins": 0.00503805186599493,
+      "rewards/rejected": 0.003047570353373885,
       "step": 30
     },
     {
       "epoch": 0.08,
-      "grad_norm": 36.1723201025049,
-      "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.5393643379211426,
-      "logits/rejected": -2.500762939453125,
-      "logps/chosen": -284.42169189453125,
-      "logps/rejected": -265.4296875,
-      "loss": 0.6088,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": -0.7783339619636536,
-      "rewards/margins": 0.27355003356933594,
-      "rewards/rejected": -1.0518839359283447,
+      "grad_norm": 39.57195743724939,
+      "learning_rate": 1.3888888888888888e-07,
+      "logits/chosen": -2.5917325019836426,
+      "logits/rejected": -2.559333562850952,
+      "logps/chosen": -266.6319274902344,
+      "logps/rejected": -244.5043487548828,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.024673160165548325,
+      "rewards/margins": 0.030291978269815445,
+      "rewards/rejected": -0.005618819035589695,
       "step": 40
     },
     {
       "epoch": 0.1,
-      "grad_norm": 33.00315442374998,
-      "learning_rate": 4.999733114418725e-07,
-      "logits/chosen": -2.4546306133270264,
-      "logits/rejected": -2.411423683166504,
-      "logps/chosen": -310.53973388671875,
-      "logps/rejected": -328.13543701171875,
-      "loss": 0.5675,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -1.386354684829712,
-      "rewards/margins": 0.38121098279953003,
-      "rewards/rejected": -1.7675657272338867,
+      "grad_norm": 38.787652763321745,
+      "learning_rate": 1.736111111111111e-07,
+      "logits/chosen": -2.5633091926574707,
+      "logits/rejected": -2.527255058288574,
+      "logps/chosen": -278.7782287597656,
+      "logps/rejected": -292.60064697265625,
+      "loss": 0.6686,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0476832240819931,
+      "rewards/margins": 0.03851194307208061,
+      "rewards/rejected": 0.009171287529170513,
       "step": 50
     },
     {
       "epoch": 0.13,
-      "grad_norm": 30.585794676467714,
-      "learning_rate": 4.990398100856366e-07,
-      "logits/chosen": -2.43892240524292,
-      "logits/rejected": -2.364809036254883,
-      "logps/chosen": -314.2215881347656,
-      "logps/rejected": -348.70916748046875,
-      "loss": 0.5305,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -1.9176769256591797,
-      "rewards/margins": 0.6137122511863708,
-      "rewards/rejected": -2.5313892364501953,
+      "grad_norm": 42.23287032668302,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -2.6264617443084717,
+      "logits/rejected": -2.5678651332855225,
+      "logps/chosen": -268.39715576171875,
+      "logps/rejected": -296.73492431640625,
+      "loss": 0.6517,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.1604687124490738,
+      "rewards/margins": 0.09314907342195511,
+      "rewards/rejected": 0.06731964647769928,
       "step": 60
     },
     {
       "epoch": 0.15,
-      "grad_norm": 34.59170831959664,
-      "learning_rate": 4.967775735898179e-07,
-      "logits/chosen": -2.2804741859436035,
-      "logits/rejected": -2.2772462368011475,
-      "logps/chosen": -316.0315246582031,
-      "logps/rejected": -317.4702453613281,
-      "loss": 0.4823,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -2.2686767578125,
-      "rewards/margins": 0.768974244594574,
-      "rewards/rejected": -3.0376508235931396,
+      "grad_norm": 37.074057156615645,
+      "learning_rate": 2.4305555555555555e-07,
+      "logits/chosen": -2.529860734939575,
+      "logits/rejected": -2.5335025787353516,
+      "logps/chosen": -261.59197998046875,
+      "logps/rejected": -255.90951538085938,
+      "loss": 0.6244,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.20122408866882324,
+      "rewards/margins": 0.160838782787323,
+      "rewards/rejected": 0.04038532078266144,
       "step": 70
     },
     {
       "epoch": 0.17,
-      "grad_norm": 45.81954671999,
-      "learning_rate": 4.931986719649298e-07,
-      "logits/chosen": -2.3903250694274902,
-      "logits/rejected": -2.3041982650756836,
-      "logps/chosen": -400.82733154296875,
-      "logps/rejected": -353.05218505859375,
-      "loss": 0.4507,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -3.5104241371154785,
-      "rewards/margins": 0.9392538070678711,
-      "rewards/rejected": -4.449677467346191,
+      "grad_norm": 53.60948399217281,
+      "learning_rate": 2.7777777777777776e-07,
+      "logits/chosen": -2.6630194187164307,
+      "logits/rejected": -2.5986924171447754,
+      "logps/chosen": -319.0962829589844,
+      "logps/rejected": -265.9037170410156,
+      "loss": 0.6056,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.1860823631286621,
+      "rewards/margins": 0.2783365547657013,
+      "rewards/rejected": -0.09225417673587799,
       "step": 80
     },
     {
       "epoch": 0.19,
-      "grad_norm": 42.77410751176114,
-      "learning_rate": 4.883222001996351e-07,
-      "logits/chosen": -2.164585590362549,
-      "logits/rejected": -2.10387921333313,
-      "logps/chosen": -390.2063903808594,
-      "logps/rejected": -397.9123840332031,
-      "loss": 0.3697,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -5.899279594421387,
-      "rewards/margins": 1.3764346837997437,
-      "rewards/rejected": -7.275714874267578,
+      "grad_norm": 39.28479543071564,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -2.5524282455444336,
+      "logits/rejected": -2.505441188812256,
+      "logps/chosen": -258.28790283203125,
+      "logps/rejected": -260.45709228515625,
+      "loss": 0.5784,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.04117143899202347,
+      "rewards/margins": 0.4441215991973877,
+      "rewards/rejected": -0.40295013785362244,
       "step": 90
     },
     {
       "epoch": 0.21,
-      "grad_norm": 16.846380740233837,
-      "learning_rate": 4.821741763807186e-07,
-      "logits/chosen": -1.4147034883499146,
-      "logits/rejected": -1.316133975982666,
-      "logps/chosen": -593.4131469726562,
-      "logps/rejected": -607.5406494140625,
-      "loss": 0.2032,
-      "rewards/accuracies": 0.981249988079071,
-      "rewards/chosen": -14.987457275390625,
-      "rewards/margins": 2.5381531715393066,
-      "rewards/rejected": -17.525609970092773,
+      "grad_norm": 37.33085523160704,
+      "learning_rate": 3.472222222222222e-07,
+      "logits/chosen": -2.5317625999450684,
+      "logits/rejected": -2.5069174766540527,
+      "logps/chosen": -255.98025512695312,
+      "logps/rejected": -263.0767517089844,
+      "loss": 0.582,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.21891236305236816,
+      "rewards/margins": 0.5213271975517273,
+      "rewards/rejected": -0.3024148643016815,
       "step": 100
     },
     {
       "epoch": 0.21,
-      "eval_logits/chosen": -1.0944379568099976,
-      "eval_logits/rejected": -0.9769749641418457,
-      "eval_logps/chosen": -683.9697265625,
-      "eval_logps/rejected": -698.75390625,
-      "eval_loss": 0.15141081809997559,
-      "eval_rewards/accuracies": 0.984375,
-      "eval_rewards/chosen": -18.961933135986328,
-      "eval_rewards/margins": 2.842402696609497,
-      "eval_rewards/rejected": -21.804336547851562,
-      "eval_runtime": 96.9098,
-      "eval_samples_per_second": 20.638,
-      "eval_steps_per_second": 0.33,
+      "eval_logits/chosen": -2.581188917160034,
+      "eval_logits/rejected": -2.5431487560272217,
+      "eval_logps/chosen": -254.23861694335938,
+      "eval_logps/rejected": -263.28759765625,
+      "eval_loss": 0.5878116488456726,
+      "eval_rewards/accuracies": 0.71875,
+      "eval_rewards/chosen": 0.41774246096611023,
+      "eval_rewards/margins": 0.44876235723495483,
+      "eval_rewards/rejected": -0.031019899994134903,
+      "eval_runtime": 97.2965,
+      "eval_samples_per_second": 20.556,
+      "eval_steps_per_second": 0.329,
       "step": 100
     },
     {
       "epoch": 0.23,
-      "grad_norm": 31.436051974346935,
-      "learning_rate": 4.747874028753375e-07,
-      "logits/chosen": -0.8883759379386902,
-      "logits/rejected": -0.5718872547149658,
-      "logps/chosen": -788.5586547851562,
-      "logps/rejected": -756.9668579101562,
-      "loss": 0.1262,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": -21.779033660888672,
-      "rewards/margins": 3.177509307861328,
-      "rewards/rejected": -24.956544876098633,
+      "grad_norm": 38.91387415080278,
+      "learning_rate": 3.819444444444444e-07,
+      "logits/chosen": -2.580745220184326,
+      "logits/rejected": -2.495126724243164,
+      "logps/chosen": -298.351806640625,
+      "logps/rejected": -262.37762451171875,
+      "loss": 0.5835,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.31141430139541626,
+      "rewards/margins": 0.5385011434555054,
+      "rewards/rejected": -0.2270868569612503,
       "step": 110
     },
     {
       "epoch": 0.25,
-      "grad_norm": 26.814434080959497,
-      "learning_rate": 4.662012913161997e-07,
-      "logits/chosen": -0.8410149812698364,
-      "logits/rejected": -0.72642582654953,
-      "logps/chosen": -807.6917724609375,
-      "logps/rejected": -798.2545776367188,
-      "loss": 0.1255,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": -23.6936092376709,
-      "rewards/margins": 3.226290464401245,
-      "rewards/rejected": -26.919897079467773,
+      "grad_norm": 38.5197424217738,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -2.473219633102417,
+      "logits/rejected": -2.4464592933654785,
+      "logps/chosen": -286.4003601074219,
+      "logps/rejected": -275.6720886230469,
+      "loss": 0.5583,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.2616604268550873,
+      "rewards/margins": 0.5291147828102112,
+      "rewards/rejected": -0.7907751798629761,
       "step": 120
     },
     {
       "epoch": 0.27,
-      "grad_norm": 17.577272722606644,
-      "learning_rate": 4.5646165232345103e-07,
-      "logits/chosen": -1.5011603832244873,
-      "logits/rejected": -1.4155353307724,
-      "logps/chosen": -792.2635498046875,
-      "logps/rejected": -789.98388671875,
-      "loss": 0.0779,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": -22.961719512939453,
-      "rewards/margins": 3.1156857013702393,
-      "rewards/rejected": -26.077404022216797,
+      "grad_norm": 37.759243136528056,
+      "learning_rate": 4.513888888888889e-07,
+      "logits/chosen": -2.463078498840332,
+      "logits/rejected": -2.4377052783966064,
+      "logps/chosen": -277.6775817871094,
+      "logps/rejected": -277.7251281738281,
+      "loss": 0.53,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.2162761688232422,
+      "rewards/margins": 0.6807397603988647,
+      "rewards/rejected": -0.46446362137794495,
       "step": 130
     },
     {
       "epoch": 0.29,
-      "grad_norm": 10.43094883526148,
-      "learning_rate": 4.456204510851956e-07,
-      "logits/chosen": -1.7753677368164062,
-      "logits/rejected": -1.7173267602920532,
-      "logps/chosen": -856.22509765625,
-      "logps/rejected": -853.39599609375,
-      "loss": 0.0783,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -24.311687469482422,
-      "rewards/margins": 3.3242690563201904,
-      "rewards/rejected": -27.635955810546875,
+      "grad_norm": 43.5485570360565,
+      "learning_rate": 4.861111111111111e-07,
+      "logits/chosen": -2.488098621368408,
+      "logits/rejected": -2.4522647857666016,
+      "logps/chosen": -313.66253662109375,
+      "logps/rejected": -313.8337707519531,
+      "loss": 0.5305,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.1151454821228981,
+      "rewards/margins": 0.7729904651641846,
+      "rewards/rejected": -0.6578450202941895,
       "step": 140
     },
     {
       "epoch": 0.31,
-      "grad_norm": 15.272453499297708,
-      "learning_rate": 4.337355301007335e-07,
-      "logits/chosen": -2.0142312049865723,
-      "logits/rejected": -1.9473145008087158,
-      "logps/chosen": -802.6983032226562,
-      "logps/rejected": -804.2092895507812,
-      "loss": 0.073,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -23.3763370513916,
-      "rewards/margins": 3.023676633834839,
-      "rewards/rejected": -26.400014877319336,
+      "grad_norm": 42.70051446601528,
+      "learning_rate": 4.999733114418725e-07,
+      "logits/chosen": -2.4509975910186768,
+      "logits/rejected": -2.384775161743164,
+      "logps/chosen": -286.3575744628906,
+      "logps/rejected": -292.4871520996094,
+      "loss": 0.5401,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.15666823089122772,
+      "rewards/margins": 0.6572391390800476,
+      "rewards/rejected": -0.8139073252677917,
       "step": 150
     },
     {
       "epoch": 0.33,
-      "grad_norm": 29.484186201457224,
-      "learning_rate": 4.2087030056579986e-07,
-      "logits/chosen": -1.9331315755844116,
-      "logits/rejected": -1.8146892786026,
-      "logps/chosen": -833.6810302734375,
-      "logps/rejected": -831.3583984375,
-      "loss": 0.0701,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -25.227581024169922,
-      "rewards/margins": 3.4691452980041504,
-      "rewards/rejected": -28.696725845336914,
+      "grad_norm": 53.334691841946395,
+      "learning_rate": 4.998102353328799e-07,
+      "logits/chosen": -2.4482672214508057,
+      "logits/rejected": -2.358886957168579,
+      "logps/chosen": -269.8271789550781,
+      "logps/rejected": -270.34710693359375,
+      "loss": 0.57,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.16204610466957092,
+      "rewards/margins": 0.808205246925354,
+      "rewards/rejected": -0.6461590528488159,
       "step": 160
     },
     {
       "epoch": 0.36,
-      "grad_norm": 11.98278360475796,
-      "learning_rate": 4.070934040463998e-07,
-      "logits/chosen": -1.782798409461975,
-      "logits/rejected": -1.7017097473144531,
-      "logps/chosen": -835.8298950195312,
-      "logps/rejected": -827.8191528320312,
-      "loss": 0.0575,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -26.392257690429688,
-      "rewards/margins": 3.4424545764923096,
-      "rewards/rejected": -29.834712982177734,
+      "grad_norm": 44.14727072303912,
+      "learning_rate": 4.994990066883491e-07,
+      "logits/chosen": -2.3647940158843994,
+      "logits/rejected": -2.2960338592529297,
+      "logps/chosen": -259.77276611328125,
+      "logps/rejected": -255.1217498779297,
+      "loss": 0.5382,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.5218764543533325,
+      "rewards/margins": 0.6779727339744568,
+      "rewards/rejected": -1.1998491287231445,
       "step": 170
     },
     {
       "epoch": 0.38,
-      "grad_norm": 106.03015109713562,
-      "learning_rate": 3.9247834624635404e-07,
-      "logits/chosen": -1.6001176834106445,
-      "logits/rejected": -1.5247141122817993,
-      "logps/chosen": -878.6383056640625,
-      "logps/rejected": -858.1409301757812,
-      "loss": 0.0515,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -28.002573013305664,
-      "rewards/margins": 3.525923490524292,
-      "rewards/rejected": -31.528493881225586,
+      "grad_norm": 86.93750039599435,
+      "learning_rate": 4.990398100856366e-07,
+      "logits/chosen": -2.234858989715576,
+      "logits/rejected": -2.1829886436462402,
+      "logps/chosen": -262.09600830078125,
+      "logps/rejected": -243.8761444091797,
+      "loss": 0.529,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.2868492603302002,
+      "rewards/margins": 0.5284022092819214,
+      "rewards/rejected": -0.8152514696121216,
       "step": 180
     },
     {
       "epoch": 0.4,
-      "grad_norm": 12.787000415115049,
-      "learning_rate": 3.7710310482256523e-07,
-      "logits/chosen": -1.763702154159546,
-      "logits/rejected": -1.7078170776367188,
-      "logps/chosen": -871.7747802734375,
-      "logps/rejected": -867.6023559570312,
-      "loss": 0.0546,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": -27.023944854736328,
-      "rewards/margins": 3.3684401512145996,
-      "rewards/rejected": -30.392383575439453,
+      "grad_norm": 42.637015477998524,
+      "learning_rate": 4.984329178560219e-07,
+      "logits/chosen": -2.275543689727783,
+      "logits/rejected": -2.2290995121002197,
+      "logps/chosen": -273.13580322265625,
+      "logps/rejected": -277.8387756347656,
+      "loss": 0.5193,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.09464643895626068,
+      "rewards/margins": 0.8095524907112122,
+      "rewards/rejected": -0.9041990041732788,
       "step": 190
     },
     {
       "epoch": 0.42,
-      "grad_norm": 33.0817046917894,
-      "learning_rate": 3.610497133404795e-07,
-      "logits/chosen": -1.5409306287765503,
-      "logits/rejected": -1.5207167863845825,
-      "logps/chosen": -881.9549560546875,
-      "logps/rejected": -885.0955810546875,
-      "loss": 0.05,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -27.992626190185547,
-      "rewards/margins": 3.659705638885498,
-      "rewards/rejected": -31.652332305908203,
+      "grad_norm": 224.49804573493958,
+      "learning_rate": 4.976786899231985e-07,
+      "logits/chosen": -2.2373015880584717,
+      "logits/rejected": -2.2193970680236816,
+      "logps/chosen": -264.38330078125,
+      "logps/rejected": -271.14654541015625,
+      "loss": 0.558,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.22433169186115265,
+      "rewards/margins": 0.7305465936660767,
+      "rewards/rejected": -0.9548781514167786,
       "step": 200
     },
     {
       "epoch": 0.42,
-      "eval_logits/chosen": -1.673904299736023,
-      "eval_logits/rejected": -1.6111565828323364,
-      "eval_logps/chosen": -882.258544921875,
-      "eval_logps/rejected": -894.3857421875,
-      "eval_loss": 0.04648973047733307,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -27.88492774963379,
-      "eval_rewards/margins": 3.7010064125061035,
-      "eval_rewards/rejected": -31.585933685302734,
-      "eval_runtime": 96.4309,
-      "eval_samples_per_second": 20.74,
-      "eval_steps_per_second": 0.332,
+      "eval_logits/chosen": -2.38926362991333,
+      "eval_logits/rejected": -2.3397982120513916,
+      "eval_logps/chosen": -261.4733581542969,
+      "eval_logps/rejected": -280.4190673828125,
+      "eval_loss": 0.5195851922035217,
+      "eval_rewards/accuracies": 0.77734375,
+      "eval_rewards/chosen": 0.05600578337907791,
+      "eval_rewards/margins": 0.9436004757881165,
+      "eval_rewards/rejected": -0.8875946998596191,
+      "eval_runtime": 96.7112,
+      "eval_samples_per_second": 20.68,
+      "eval_steps_per_second": 0.331,
       "step": 200
     },
     {
       "epoch": 0.44,
-      "grad_norm": 16.166495660873913,
-      "learning_rate": 3.4440382358952115e-07,
-      "logits/chosen": -1.6085878610610962,
-      "logits/rejected": -1.5104396343231201,
-      "logps/chosen": -908.0562744140625,
-      "logps/rejected": -881.0787353515625,
-      "loss": 0.0548,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -27.82638168334961,
-      "rewards/margins": 3.6651740074157715,
-      "rewards/rejected": -31.491561889648438,
+      "grad_norm": 57.978840386902434,
+      "learning_rate": 4.967775735898179e-07,
+      "logits/chosen": -2.3195013999938965,
+      "logits/rejected": -2.2525343894958496,
+      "logps/chosen": -293.5995178222656,
+      "logps/rejected": -273.7899475097656,
+      "loss": 0.5321,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.1953679621219635,
+      "rewards/margins": 0.9317470788955688,
+      "rewards/rejected": -1.12711501121521,
       "step": 210
     },
     {
       "epoch": 0.46,
-      "grad_norm": 17.20838092026964,
-      "learning_rate": 3.272542485937368e-07,
-      "logits/chosen": -1.5196223258972168,
-      "logits/rejected": -1.4271810054779053,
-      "logps/chosen": -905.3258666992188,
-      "logps/rejected": -896.2493286132812,
-      "loss": 0.0494,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": -28.79769515991211,
-      "rewards/margins": 3.7315757274627686,
-      "rewards/rejected": -32.529273986816406,
+      "grad_norm": 45.042317188060295,
+      "learning_rate": 4.957301032722118e-07,
+      "logits/chosen": -2.320981502532959,
+      "logits/rejected": -2.2780816555023193,
+      "logps/chosen": -275.7085266113281,
+      "logps/rejected": -272.6601257324219,
+      "loss": 0.5272,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5165706872940063,
+      "rewards/margins": 0.8332468271255493,
+      "rewards/rejected": -1.3498175144195557,
       "step": 220
     },
     {
       "epoch": 0.48,
-      "grad_norm": 7.8033732261846405,
-      "learning_rate": 3.096924887558854e-07,
-      "logits/chosen": -1.580828070640564,
-      "logits/rejected": -1.502069115638733,
-      "logps/chosen": -892.1995239257812,
-      "logps/rejected": -904.1240234375,
-      "loss": 0.0417,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -28.950037002563477,
-      "rewards/margins": 3.8692786693573,
-      "rewards/rejected": -32.819313049316406,
+      "grad_norm": 42.64386380595179,
+      "learning_rate": 4.945369001834514e-07,
+      "logits/chosen": -2.335122585296631,
+      "logits/rejected": -2.2802178859710693,
+      "logps/chosen": -251.1619415283203,
+      "logps/rejected": -268.8333740234375,
+      "loss": 0.5242,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.1148248165845871,
+      "rewards/margins": 0.939952552318573,
+      "rewards/rejected": -1.0547773838043213,
       "step": 230
     },
     {
       "epoch": 0.5,
-      "grad_norm": 6.378071285771052,
-      "learning_rate": 2.9181224366319943e-07,
-      "logits/chosen": -1.6854736804962158,
-      "logits/rejected": -1.6346015930175781,
-      "logps/chosen": -901.0328369140625,
-      "logps/rejected": -894.5446166992188,
-      "loss": 0.0422,
-      "rewards/accuracies": 0.981249988079071,
-      "rewards/chosen": -28.67997169494629,
-      "rewards/margins": 3.593839645385742,
-      "rewards/rejected": -32.27381134033203,
+      "grad_norm": 43.7138056070312,
+      "learning_rate": 4.931986719649298e-07,
+      "logits/chosen": -2.372040271759033,
+      "logits/rejected": -2.3250112533569336,
+      "logps/chosen": -267.7872619628906,
+      "logps/rejected": -270.08612060546875,
+      "loss": 0.4886,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.20435211062431335,
+      "rewards/margins": 0.8465341329574585,
+      "rewards/rejected": -1.0508863925933838,
       "step": 240
     },
     {
       "epoch": 0.52,
-      "grad_norm": 10.334975517501274,
-      "learning_rate": 2.7370891215954565e-07,
-      "logits/chosen": -1.6026891469955444,
-      "logits/rejected": -1.4646245241165161,
-      "logps/chosen": -961.8062744140625,
-      "logps/rejected": -934.1185302734375,
-      "loss": 0.0404,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -29.86271095275879,
-      "rewards/margins": 3.8670907020568848,
-      "rewards/rejected": -33.72980499267578,
+      "grad_norm": 46.84966459893474,
+      "learning_rate": 4.91716212266689e-07,
+      "logits/chosen": -2.2988550662994385,
+      "logits/rejected": -2.2003023624420166,
+      "logps/chosen": -298.1326904296875,
+      "logps/rejected": -279.72857666015625,
+      "loss": 0.4983,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.0028904906939715147,
+      "rewards/margins": 1.0131902694702148,
+      "rewards/rejected": -1.0102999210357666,
       "step": 250
     },
     {
       "epoch": 0.54,
-      "grad_norm": 18.111953364200915,
-      "learning_rate": 2.55479083351317e-07,
-      "logits/chosen": -1.6801973581314087,
-      "logits/rejected": -1.6076374053955078,
-      "logps/chosen": -956.5700073242188,
-      "logps/rejected": -927.5736083984375,
-      "loss": 0.0376,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -29.801631927490234,
-      "rewards/margins": 3.8780322074890137,
-      "rewards/rejected": -33.679664611816406,
+      "grad_norm": 45.60631693718858,
+      "learning_rate": 4.900904002767367e-07,
+      "logits/chosen": -2.359170913696289,
+      "logits/rejected": -2.298555850982666,
+      "logps/chosen": -298.26898193359375,
+      "logps/rejected": -277.553955078125,
+      "loss": 0.4979,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.1978708803653717,
+      "rewards/margins": 0.9808057546615601,
+      "rewards/rejected": -1.178676724433899,
       "step": 260
     },
     {
       "epoch": 0.56,
-      "grad_norm": 6.968132884377489,
-      "learning_rate": 2.3722002126275822e-07,
-      "logits/chosen": -1.8389537334442139,
-      "logits/rejected": -1.7772973775863647,
-      "logps/chosen": -918.482421875,
-      "logps/rejected": -908.02197265625,
-      "loss": 0.0524,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -28.866891860961914,
-      "rewards/margins": 3.6519908905029297,
-      "rewards/rejected": -32.518882751464844,
+      "grad_norm": 43.06547616205831,
+      "learning_rate": 4.883222001996351e-07,
+      "logits/chosen": -2.3876845836639404,
+      "logits/rejected": -2.324982166290283,
+      "logps/chosen": -285.72674560546875,
+      "logps/rejected": -283.54534912109375,
+      "loss": 0.529,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.43653813004493713,
+      "rewards/margins": 0.8585146069526672,
+      "rewards/rejected": -1.2950527667999268,
       "step": 270
     },
     {
       "epoch": 0.59,
-      "grad_norm": 5.666955347570076,
-      "learning_rate": 2.19029145890313e-07,
-      "logits/chosen": -1.7398706674575806,
-      "logits/rejected": -1.6362075805664062,
-      "logps/chosen": -919.9739990234375,
-      "logps/rejected": -911.2005615234375,
-      "loss": 0.0406,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -29.623361587524414,
-      "rewards/margins": 3.8273494243621826,
-      "rewards/rejected": -33.450706481933594,
+      "grad_norm": 47.726430451771456,
+      "learning_rate": 4.864126606846696e-07,
+      "logits/chosen": -2.3547611236572266,
+      "logits/rejected": -2.261406898498535,
+      "logps/chosen": -271.0657043457031,
+      "logps/rejected": -269.5400390625,
+      "loss": 0.5339,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.4694308340549469,
+      "rewards/margins": 0.8982539176940918,
+      "rewards/rejected": -1.3676847219467163,
       "step": 280
     },
     {
       "epoch": 0.61,
-      "grad_norm": 6.4155337396544745,
-      "learning_rate": 2.0100351342479216e-07,
-      "logits/chosen": -1.8250000476837158,
-      "logits/rejected": -1.7619469165802002,
-      "logps/chosen": -911.2184448242188,
-      "logps/rejected": -913.6142578125,
-      "loss": 0.039,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -29.705902099609375,
-      "rewards/margins": 3.8770148754119873,
-      "rewards/rejected": -33.58292007446289,
+      "grad_norm": 43.598738742809324,
+      "learning_rate": 4.843629142039366e-07,
+      "logits/chosen": -2.398907423019409,
+      "logits/rejected": -2.3398590087890625,
+      "logps/chosen": -260.32208251953125,
+      "logps/rejected": -267.5470886230469,
+      "loss": 0.5116,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.4617377817630768,
+      "rewards/margins": 0.8178228139877319,
+      "rewards/rejected": -1.2795606851577759,
       "step": 290
     },
     {
       "epoch": 0.63,
-      "grad_norm": 37.843596978009316,
-      "learning_rate": 1.8323929841460178e-07,
-      "logits/chosen": -1.6457163095474243,
-      "logits/rejected": -1.5310847759246826,
-      "logps/chosen": -981.1585083007812,
-      "logps/rejected": -952.3566284179688,
-      "loss": 0.0325,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -30.607309341430664,
-      "rewards/margins": 4.005252838134766,
-      "rewards/rejected": -34.6125602722168,
+      "grad_norm": 53.461838586248135,
+      "learning_rate": 4.821741763807186e-07,
+      "logits/chosen": -2.358552932739258,
+      "logits/rejected": -2.2632975578308105,
+      "logps/chosen": -306.07708740234375,
+      "logps/rejected": -284.6916198730469,
+      "loss": 0.4914,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.254052072763443,
+      "rewards/margins": 0.9752548336982727,
+      "rewards/rejected": -1.2293068170547485,
       "step": 300
     },
     {
       "epoch": 0.63,
-      "eval_logits/chosen": -1.5405726432800293,
-      "eval_logits/rejected": -1.475441575050354,
-      "eval_logps/chosen": -959.3289794921875,
-      "eval_logps/rejected": -971.60546875,
-      "eval_loss": 0.043682657182216644,
-      "eval_rewards/accuracies": 0.9921875,
-      "eval_rewards/chosen": -31.353103637695312,
-      "eval_rewards/margins": 4.093813896179199,
-      "eval_rewards/rejected": -35.44691467285156,
-      "eval_runtime": 96.4789,
-      "eval_samples_per_second": 20.73,
-      "eval_steps_per_second": 0.332,
+      "eval_logits/chosen": -2.3652639389038086,
+      "eval_logits/rejected": -2.3039121627807617,
+      "eval_logps/chosen": -264.29364013671875,
+      "eval_logps/rejected": -286.6200866699219,
+      "eval_loss": 0.5109529495239258,
+      "eval_rewards/accuracies": 0.765625,
+      "eval_rewards/chosen": -0.08500880002975464,
+      "eval_rewards/margins": 1.1126374006271362,
+      "eval_rewards/rejected": -1.197646141052246,
+      "eval_runtime": 96.7466,
+      "eval_samples_per_second": 20.673,
+      "eval_steps_per_second": 0.331,
       "step": 300
     },
     {
       "epoch": 0.65,
-      "grad_norm": 6.233510149859438,
-      "learning_rate": 1.6583128063291573e-07,
-      "logits/chosen": -1.4836466312408447,
-      "logits/rejected": -1.4385102987289429,
-      "logps/chosen": -986.4183349609375,
-      "logps/rejected": -960.0974731445312,
-      "loss": 0.035,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -31.098709106445312,
-      "rewards/margins": 3.980337619781494,
-      "rewards/rejected": -35.07904815673828,
+      "grad_norm": 39.546683435658494,
+      "learning_rate": 4.798477452685468e-07,
+      "logits/chosen": -2.268113613128662,
+      "logits/rejected": -2.2365353107452393,
+      "logps/chosen": -298.82098388671875,
+      "logps/rejected": -280.2306823730469,
+      "loss": 0.4878,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.17425382137298584,
+      "rewards/margins": 0.9114526510238647,
+      "rewards/rejected": -1.0857064723968506,
       "step": 310
     },
     {
       "epoch": 0.67,
-      "grad_norm": 13.607372248281527,
-      "learning_rate": 1.488723393865766e-07,
-      "logits/chosen": -1.6368532180786133,
-      "logits/rejected": -1.5847358703613281,
-      "logps/chosen": -968.7678833007812,
-      "logps/rejected": -929.6497192382812,
-      "loss": 0.0337,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -30.379409790039062,
-      "rewards/margins": 3.8619182109832764,
-      "rewards/rejected": -34.24132537841797,
+      "grad_norm": 46.66091222824393,
+      "learning_rate": 4.773850005813776e-07,
+      "logits/chosen": -2.2866523265838623,
+      "logits/rejected": -2.221771240234375,
+      "logps/chosen": -301.2940368652344,
+      "logps/rejected": -271.4169921875,
+      "loss": 0.4739,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3811998963356018,
+      "rewards/margins": 0.9484872817993164,
+      "rewards/rejected": -1.329687237739563,
       "step": 320
     },
     {
       "epoch": 0.69,
-      "grad_norm": 5.6023349740216055,
-      "learning_rate": 1.3245295796480788e-07,
-      "logits/chosen": -1.7128045558929443,
-      "logits/rejected": -1.6170637607574463,
-      "logps/chosen": -938.986328125,
-      "logps/rejected": -945.0597534179688,
-      "loss": 0.0328,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -30.454931259155273,
-      "rewards/margins": 3.823124647140503,
-      "rewards/rejected": -34.278053283691406,
+      "grad_norm": 38.86653009667017,
+      "learning_rate": 4.747874028753375e-07,
+      "logits/chosen": -2.278787136077881,
+      "logits/rejected": -2.1933655738830566,
+      "logps/chosen": -265.38922119140625,
+      "logps/rejected": -282.16143798828125,
+      "loss": 0.4954,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.1589515209197998,
+      "rewards/margins": 0.9741891026496887,
+      "rewards/rejected": -1.1331405639648438,
       "step": 330
     },
     {
       "epoch": 0.71,
-      "grad_norm": 10.620179504721115,
-      "learning_rate": 1.1666074087171627e-07,
-      "logits/chosen": -1.6625759601593018,
-      "logits/rejected": -1.5825822353363037,
-      "logps/chosen": -979.2620239257812,
-      "logps/rejected": -979.6686401367188,
-      "loss": 0.0411,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -31.017040252685547,
-      "rewards/margins": 3.964301586151123,
-      "rewards/rejected": -34.981346130371094,
+      "grad_norm": 52.563625371137256,
+      "learning_rate": 4.720564926825267e-07,
+      "logits/chosen": -2.261157751083374,
+      "logits/rejected": -2.1779565811157227,
+      "logps/chosen": -300.0699462890625,
+      "logps/rejected": -313.32012939453125,
+      "loss": 0.4961,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.5037744641304016,
+      "rewards/margins": 1.1601417064666748,
+      "rewards/rejected": -1.663915991783142,
       "step": 340
     },
     {
       "epoch": 0.73,
-      "grad_norm": 5.827741295121058,
-      "learning_rate": 1.0157994641835734e-07,
-      "logits/chosen": -1.7182893753051758,
-      "logits/rejected": -1.638744592666626,
-      "logps/chosen": -931.17919921875,
-      "logps/rejected": -926.6099853515625,
-      "loss": 0.0325,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -30.283557891845703,
-      "rewards/margins": 4.01150369644165,
-      "rewards/rejected": -34.29505157470703,
+      "grad_norm": 44.99037304721281,
+      "learning_rate": 4.6919388959739e-07,
+      "logits/chosen": -2.2558467388153076,
+      "logits/rejected": -2.163471221923828,
+      "logps/chosen": -268.26397705078125,
+      "logps/rejected": -272.03729248046875,
+      "loss": 0.465,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.5026377439498901,
+      "rewards/margins": 1.063787579536438,
+      "rewards/rejected": -1.566425085067749,
       "step": 350
     },
     {
       "epoch": 0.75,
-      "grad_norm": 22.94292317488105,
-      "learning_rate": 8.729103716819111e-08,
-      "logits/chosen": -1.781154990196228,
-      "logits/rejected": -1.6256592273712158,
-      "logps/chosen": -978.6188354492188,
-      "logps/rejected": -948.31689453125,
-      "loss": 0.035,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -30.416706085205078,
-      "rewards/margins": 3.933894395828247,
-      "rewards/rejected": -34.3505973815918,
+      "grad_norm": 62.16268030811922,
+      "learning_rate": 4.662012913161997e-07,
+      "logits/chosen": -2.2976787090301514,
+      "logits/rejected": -2.1752614974975586,
+      "logps/chosen": -306.94708251953125,
+      "logps/rejected": -288.148193359375,
+      "loss": 0.52,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.212753027677536,
+      "rewards/margins": 1.1294101476669312,
+      "rewards/rejected": -1.3421632051467896,
       "step": 360
     },
     {
       "epoch": 0.77,
-      "grad_norm": 8.723670578307665,
-      "learning_rate": 7.387025063449081e-08,
-      "logits/chosen": -1.5016860961914062,
-      "logits/rejected": -1.426626205444336,
-      "logps/chosen": -981.3603515625,
-      "logps/rejected": -954.1197509765625,
-      "loss": 0.036,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -31.7712345123291,
-      "rewards/margins": 4.02304744720459,
-      "rewards/rejected": -35.794281005859375,
+      "grad_norm": 48.68068839220641,
+      "learning_rate": 4.6308047263021925e-07,
+      "logits/chosen": -2.221646785736084,
+      "logits/rejected": -2.1589419841766357,
+      "logps/chosen": -286.72344970703125,
+      "logps/rejected": -267.8009033203125,
+      "loss": 0.5138,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.5695220232009888,
+      "rewards/margins": 0.9088203310966492,
+      "rewards/rejected": -1.4783422946929932,
       "step": 370
     },
     {
       "epoch": 0.79,
-      "grad_norm": 5.736646478676148,
-      "learning_rate": 6.138919252022435e-08,
-      "logits/chosen": -1.4259135723114014,
-      "logits/rejected": -1.416859745979309,
-      "logps/chosen": -947.4265747070312,
-      "logps/rejected": -977.1536254882812,
-      "loss": 0.0345,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": -31.857385635375977,
-      "rewards/margins": 4.064465045928955,
-      "rewards/rejected": -35.921852111816406,
+      "grad_norm": 40.16847273705918,
+      "learning_rate": 4.5983328437314523e-07,
+      "logits/chosen": -2.1364545822143555,
+      "logits/rejected": -2.1142001152038574,
+      "logps/chosen": -257.01513671875,
+      "logps/rejected": -296.47222900390625,
+      "loss": 0.4949,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.8765251040458679,
+      "rewards/margins": 1.0112650394439697,
+      "rewards/rejected": -1.8877900838851929,
       "step": 380
     },
     {
       "epoch": 0.82,
-      "grad_norm": 7.582969385527688,
-      "learning_rate": 4.991445467064689e-08,
-      "logits/chosen": -1.6060682535171509,
-      "logits/rejected": -1.5537000894546509,
-      "logps/chosen": -1001.6559448242188,
-      "logps/rejected": -990.7487182617188,
-      "loss": 0.0321,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -31.320858001708984,
-      "rewards/margins": 4.018052101135254,
-      "rewards/rejected": -35.33891296386719,
+      "grad_norm": 51.78049342719275,
+      "learning_rate": 4.5646165232345103e-07,
+      "logits/chosen": -2.223437547683716,
+      "logits/rejected": -2.1807045936584473,
+      "logps/chosen": -312.63714599609375,
+      "logps/rejected": -308.8570861816406,
+      "loss": 0.4921,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.3500111699104309,
+      "rewards/margins": 0.8943204879760742,
+      "rewards/rejected": -1.2443315982818604,
       "step": 390
     },
     {
       "epoch": 0.84,
-      "grad_norm": 6.922783557699858,
-      "learning_rate": 3.9507259776993954e-08,
-      "logits/chosen": -1.6298010349273682,
-      "logits/rejected": -1.5356940031051636,
-      "logps/chosen": -959.9840698242188,
-      "logps/rejected": -967.3712158203125,
-      "loss": 0.0337,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -31.025197982788086,
-      "rewards/margins": 3.9927406311035156,
-      "rewards/rejected": -35.01793670654297,
+      "grad_norm": 42.79568140865266,
+      "learning_rate": 4.529675760622843e-07,
+      "logits/chosen": -2.2350916862487793,
+      "logits/rejected": -2.148932695388794,
+      "logps/chosen": -276.3843078613281,
+      "logps/rejected": -295.5137939453125,
+      "loss": 0.4922,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.29245343804359436,
+      "rewards/margins": 1.1326130628585815,
+      "rewards/rejected": -1.425066590309143,
       "step": 400
     },
     {
       "epoch": 0.84,
-      "eval_logits/chosen": -1.7575984001159668,
-      "eval_logits/rejected": -1.7038744688034058,
-      "eval_logps/chosen": -942.9697265625,
-      "eval_logps/rejected": -955.4846801757812,
-      "eval_loss": 0.039761096239089966,
-      "eval_rewards/accuracies": 1.0,
-      "eval_rewards/chosen": -30.616931915283203,
-      "eval_rewards/margins": 4.02393913269043,
-      "eval_rewards/rejected": -34.640872955322266,
-      "eval_runtime": 96.5164,
-      "eval_samples_per_second": 20.722,
-      "eval_steps_per_second": 0.332,
+      "eval_logits/chosen": -2.3144989013671875,
+      "eval_logits/rejected": -2.2569639682769775,
+      "eval_logps/chosen": -263.2853698730469,
+      "eval_logps/rejected": -285.4248352050781,
+      "eval_loss": 0.5095034837722778,
+      "eval_rewards/accuracies": 0.78515625,
+      "eval_rewards/chosen": -0.0345957949757576,
+      "eval_rewards/margins": 1.1032863855361938,
+      "eval_rewards/rejected": -1.137882113456726,
+      "eval_runtime": 96.7428,
+      "eval_samples_per_second": 20.673,
+      "eval_steps_per_second": 0.331,
       "step": 400
     },
     {
       "epoch": 0.86,
-      "grad_norm": 5.463636168348194,
-      "learning_rate": 3.022313472693447e-08,
-      "logits/chosen": -1.7122160196304321,
-      "logits/rejected": -1.5953336954116821,
-      "logps/chosen": -985.7266845703125,
-      "logps/rejected": -965.8865356445312,
-      "loss": 0.0352,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -30.906414031982422,
-      "rewards/margins": 3.947056531906128,
-      "rewards/rejected": -34.85347366333008,
+      "grad_norm": 41.17913551343322,
+      "learning_rate": 4.493531277875948e-07,
+      "logits/chosen": -2.300753116607666,
+      "logits/rejected": -2.2138209342956543,
+      "logps/chosen": -300.85986328125,
+      "logps/rejected": -293.10089111328125,
+      "loss": 0.4939,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.09711817651987076,
+      "rewards/margins": 1.1170685291290283,
+      "rewards/rejected": -1.214186668395996,
       "step": 410
     },
     {
       "epoch": 0.88,
-      "grad_norm": 6.657502927820359,
-      "learning_rate": 2.2111614344599684e-08,
-      "logits/chosen": -1.7468277215957642,
-      "logits/rejected": -1.7015721797943115,
-      "logps/chosen": -971.2100830078125,
-      "logps/rejected": -959.1683349609375,
-      "loss": 0.0355,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -30.336135864257812,
-      "rewards/margins": 3.8991711139678955,
-      "rewards/rejected": -34.23530578613281,
+      "grad_norm": 44.7817002923167,
+      "learning_rate": 4.456204510851956e-07,
+      "logits/chosen": -2.296912670135498,
+      "logits/rejected": -2.2389960289001465,
+      "logps/chosen": -304.0657043457031,
+      "logps/rejected": -303.19512939453125,
+      "loss": 0.4853,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.34959596395492554,
+      "rewards/margins": 1.0870481729507446,
+      "rewards/rejected": -1.4366440773010254,
       "step": 420
     },
     {
       "epoch": 0.9,
-      "grad_norm": 6.072288161895055,
-      "learning_rate": 1.521597710086439e-08,
-      "logits/chosen": -1.6179310083389282,
-      "logits/rejected": -1.541458249092102,
-      "logps/chosen": -961.2311401367188,
-      "logps/rejected": -949.5515747070312,
-      "loss": 0.0303,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -30.698688507080078,
-      "rewards/margins": 3.8898932933807373,
-      "rewards/rejected": -34.58858108520508,
+      "grad_norm": 42.837722286044084,
+      "learning_rate": 4.4177175965748804e-07,
+      "logits/chosen": -2.190886974334717,
+      "logits/rejected": -2.1311051845550537,
+      "logps/chosen": -289.19403076171875,
+      "logps/rejected": -291.84967041015625,
+      "loss": 0.4704,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.5077947378158569,
+      "rewards/margins": 1.1956888437271118,
+      "rewards/rejected": -1.7034835815429688,
       "step": 430
     },
     {
       "epoch": 0.92,
-      "grad_norm": 4.776332402627242,
-      "learning_rate": 9.57301420397924e-09,
-      "logits/chosen": -1.6823447942733765,
-      "logits/rejected": -1.6038888692855835,
-      "logps/chosen": -972.6544799804688,
-      "logps/rejected": -967.3916015625,
-      "loss": 0.032,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -30.97427749633789,
-      "rewards/margins": 3.939715623855591,
-      "rewards/rejected": -34.91399383544922,
+      "grad_norm": 50.46876260755674,
+      "learning_rate": 4.378093360106022e-07,
+      "logits/chosen": -2.275932788848877,
+      "logits/rejected": -2.206188678741455,
+      "logps/chosen": -291.9374694824219,
+      "logps/rejected": -296.0220947265625,
+      "loss": 0.4947,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3800189793109894,
+      "rewards/margins": 0.9654985666275024,
+      "rewards/rejected": -1.3455175161361694,
       "step": 440
     },
     {
       "epoch": 0.94,
-      "grad_norm": 6.710451381468546,
-      "learning_rate": 5.212833302556258e-09,
-      "logits/chosen": -1.6518666744232178,
-      "logits/rejected": -1.6050903797149658,
-      "logps/chosen": -977.6473388671875,
-      "logps/rejected": -1001.37646484375,
-      "loss": 0.0331,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -30.778345108032227,
-      "rewards/margins": 3.9367527961730957,
-      "rewards/rejected": -34.71509552001953,
+      "grad_norm": 49.16582597289157,
+      "learning_rate": 4.337355301007335e-07,
+      "logits/chosen": -2.2473514080047607,
+      "logits/rejected": -2.212566375732422,
+      "logps/chosen": -304.7669372558594,
+      "logps/rejected": -337.439453125,
+      "loss": 0.4809,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.5541377663612366,
+      "rewards/margins": 0.9641104936599731,
+      "rewards/rejected": -1.5182483196258545,
       "step": 450
     },
     {
       "epoch": 0.96,
-      "grad_norm": 7.613326855771005,
-      "learning_rate": 2.158697848236607e-09,
-      "logits/chosen": -1.6514955759048462,
-      "logits/rejected": -1.5610146522521973,
-      "logps/chosen": -956.8262939453125,
-      "logps/rejected": -938.9820556640625,
-      "loss": 0.0315,
-      "rewards/accuracies": 1.0,
-      "rewards/chosen": -30.818777084350586,
-      "rewards/margins": 3.9959335327148438,
-      "rewards/rejected": -34.8147087097168,
+      "grad_norm": 45.063096434401295,
+      "learning_rate": 4.2955275794047627e-07,
+      "logits/chosen": -2.255167007446289,
+      "logits/rejected": -2.2048747539520264,
+      "logps/chosen": -277.2271423339844,
+      "logps/rejected": -268.5289001464844,
+      "loss": 0.4941,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.26313039660453796,
+      "rewards/margins": 1.0289232730865479,
+      "rewards/rejected": -1.2920535802841187,
       "step": 460
     },
     {
       "epoch": 0.98,
-      "grad_norm": 8.360004131403477,
-      "learning_rate": 4.269029751107489e-10,
-      "logits/chosen": -1.6803505420684814,
-      "logits/rejected": -1.592365026473999,
-      "logps/chosen": -961.8359375,
-      "logps/rejected": -975.6350708007812,
-      "loss": 0.0311,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": -30.741174697875977,
-      "rewards/margins": 3.923020839691162,
-      "rewards/rejected": -34.6641960144043,
+      "grad_norm": 44.06520192137465,
+      "learning_rate": 4.252635001659837e-07,
+      "logits/chosen": -2.2488787174224854,
+      "logits/rejected": -2.1796398162841797,
+      "logps/chosen": -284.37713623046875,
+      "logps/rejected": -309.53546142578125,
+      "loss": 0.4733,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.28392332792282104,
+      "rewards/margins": 1.0752943754196167,
+      "rewards/rejected": -1.3592177629470825,
       "step": 470
     },
     {
       "epoch": 1.0,
-      "step": 478,
+      "grad_norm": 20.757199114043438,
+      "learning_rate": 4.2087030056579986e-07,
+      "logits/chosen": -2.2137668132781982,
+      "logits/rejected": -2.1583967208862305,
+      "logps/chosen": -291.67376708984375,
+      "logps/rejected": -288.5511169433594,
+      "loss": 0.417,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.14164048433303833,
+      "rewards/margins": 1.205784559249878,
+      "rewards/rejected": -1.347425103187561,
+      "step": 480
+    },
+    {
+      "epoch": 1.03,
+      "grad_norm": 30.758787856873905,
+      "learning_rate": 4.163757645722403e-07,
+      "logits/chosen": -2.273852825164795,
+      "logits/rejected": -2.1564927101135254,
+      "logps/chosen": -292.22467041015625,
+      "logps/rejected": -292.77349853515625,
+      "loss": 0.1895,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 0.2319210022687912,
+      "rewards/margins": 2.500307559967041,
+      "rewards/rejected": -2.2683863639831543,
+      "step": 490
+    },
+    {
+      "epoch": 1.05,
+      "grad_norm": 30.542382675770316,
+      "learning_rate": 4.117825577162134e-07,
+      "logits/chosen": -2.2761318683624268,
+      "logits/rejected": -2.1567749977111816,
+      "logps/chosen": -279.54486083984375,
+      "logps/rejected": -309.9883728027344,
+      "loss": 0.1908,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.061248529702425,
+      "rewards/margins": 2.6951680183410645,
+      "rewards/rejected": -2.6339192390441895,
+      "step": 500
+    },
+    {
+      "epoch": 1.05,
+      "eval_logits/chosen": -2.2441868782043457,
+      "eval_logits/rejected": -2.165961980819702,
+      "eval_logps/chosen": -269.8425598144531,
+      "eval_logps/rejected": -300.6473693847656,
+      "eval_loss": 0.5178976058959961,
+      "eval_rewards/accuracies": 0.78515625,
+      "eval_rewards/chosen": -0.36245518922805786,
+      "eval_rewards/margins": 1.5365545749664307,
+      "eval_rewards/rejected": -1.8990097045898438,
+      "eval_runtime": 96.6241,
+      "eval_samples_per_second": 20.699,
+      "eval_steps_per_second": 0.331,
+      "step": 500
+    },
+    {
+      "epoch": 1.07,
+      "grad_norm": 18.232913631558603,
+      "learning_rate": 4.070934040463998e-07,
+      "logits/chosen": -2.1585049629211426,
+      "logits/rejected": -2.0768818855285645,
+      "logps/chosen": -279.84527587890625,
+      "logps/rejected": -321.41375732421875,
+      "loss": 0.1751,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1159655898809433,
+      "rewards/margins": 2.8791768550872803,
+      "rewards/rejected": -2.763211250305176,
+      "step": 510
+    },
+    {
+      "epoch": 1.09,
+      "grad_norm": 28.960170029042352,
+      "learning_rate": 4.023110845137273e-07,
+      "logits/chosen": -2.215406894683838,
+      "logits/rejected": -2.0539145469665527,
+      "logps/chosen": -276.207763671875,
+      "logps/rejected": -310.82513427734375,
+      "loss": 0.1755,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.07207924127578735,
+      "rewards/margins": 2.9146130084991455,
+      "rewards/rejected": -2.986691951751709,
+      "step": 520
+    },
+    {
+      "epoch": 1.11,
+      "grad_norm": 35.158803563669196,
+      "learning_rate": 3.974384353220992e-07,
+      "logits/chosen": -2.187908172607422,
+      "logits/rejected": -2.049764394760132,
+      "logps/chosen": -256.0184326171875,
+      "logps/rejected": -290.10821533203125,
+      "loss": 0.167,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -0.13029596209526062,
+      "rewards/margins": 2.844452381134033,
+      "rewards/rejected": -2.974748134613037,
+      "step": 530
+    },
+    {
+      "epoch": 1.13,
+      "grad_norm": 38.75397385029711,
+      "learning_rate": 3.9247834624635404e-07,
+      "logits/chosen": -2.1720824241638184,
+      "logits/rejected": -2.047842502593994,
+      "logps/chosen": -267.35888671875,
+      "logps/rejected": -331.7341003417969,
+      "loss": 0.1939,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -0.3224119246006012,
+      "rewards/margins": 3.103816509246826,
+      "rewards/rejected": -3.4262282848358154,
+      "step": 540
+    },
+    {
+      "epoch": 1.15,
+      "grad_norm": 21.6571458368692,
+      "learning_rate": 3.8743375891845556e-07,
+      "logits/chosen": -2.2194952964782715,
+      "logits/rejected": -2.088972806930542,
+      "logps/chosen": -301.56414794921875,
+      "logps/rejected": -329.8552551269531,
+      "loss": 0.1541,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -0.44850125908851624,
+      "rewards/margins": 2.9901504516601562,
+      "rewards/rejected": -3.4386515617370605,
+      "step": 550
+    },
+    {
+      "epoch": 1.17,
+      "grad_norm": 41.5671178301598,
+      "learning_rate": 3.823076650829267e-07,
+      "logits/chosen": -2.198901891708374,
+      "logits/rejected": -2.0278093814849854,
+      "logps/chosen": -261.13189697265625,
+      "logps/rejected": -295.24127197265625,
+      "loss": 0.1609,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.0586366169154644,
+      "rewards/margins": 2.883596897125244,
+      "rewards/rejected": -2.9422335624694824,
+      "step": 560
+    },
+    {
+      "epoch": 1.19,
+      "grad_norm": 22.403810834900185,
+      "learning_rate": 3.7710310482256523e-07,
+      "logits/chosen": -2.10243558883667,
+      "logits/rejected": -1.9939591884613037,
+      "logps/chosen": -281.2176208496094,
+      "logps/rejected": -338.7304382324219,
+      "loss": 0.1581,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.1340499371290207,
+      "rewards/margins": 3.009289503097534,
+      "rewards/rejected": -3.1433396339416504,
+      "step": 570
+    },
+    {
+      "epoch": 1.21,
+      "grad_norm": 41.02190574631486,
+      "learning_rate": 3.718231647554911e-07,
+      "logits/chosen": -2.1406311988830566,
+      "logits/rejected": -1.9555755853652954,
+      "logps/chosen": -292.01739501953125,
+      "logps/rejected": -329.15045166015625,
+      "loss": 0.1561,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -0.6274134516716003,
+      "rewards/margins": 3.1701507568359375,
+      "rewards/rejected": -3.7975640296936035,
+      "step": 580
+    },
+    {
+      "epoch": 1.23,
+      "grad_norm": 24.855938354465067,
+      "learning_rate": 3.664709762045961e-07,
+      "logits/chosen": -2.1577706336975098,
+      "logits/rejected": -2.070456027984619,
+      "logps/chosen": -284.9651794433594,
+      "logps/rejected": -321.62994384765625,
+      "loss": 0.1622,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -0.7565070390701294,
+      "rewards/margins": 2.949742078781128,
+      "rewards/rejected": -3.706249237060547,
+      "step": 590
+    },
+    {
+      "epoch": 1.26,
+      "grad_norm": 34.33138656502216,
+      "learning_rate": 3.610497133404795e-07,
+      "logits/chosen": -2.268091917037964,
+      "logits/rejected": -2.1013598442077637,
+      "logps/chosen": -285.04327392578125,
+      "logps/rejected": -315.2426452636719,
+      "loss": 0.1675,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -0.31151503324508667,
+      "rewards/margins": 3.0706770420074463,
+      "rewards/rejected": -3.3821918964385986,
+      "step": 600
+    },
+    {
+      "epoch": 1.26,
+      "eval_logits/chosen": -2.2219574451446533,
+      "eval_logits/rejected": -2.1248531341552734,
+      "eval_logps/chosen": -287.2300109863281,
+      "eval_logps/rejected": -324.08123779296875,
+      "eval_loss": 0.5377076268196106,
+      "eval_rewards/accuracies": 0.80078125,
+      "eval_rewards/chosen": -1.2318270206451416,
+      "eval_rewards/margins": 1.8388763666152954,
+      "eval_rewards/rejected": -3.0707035064697266,
+      "eval_runtime": 96.6082,
+      "eval_samples_per_second": 20.702,
+      "eval_steps_per_second": 0.331,
+      "step": 600
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 24.42926394663865,
+      "learning_rate": 3.555625912989747e-07,
+      "logits/chosen": -2.2549402713775635,
+      "logits/rejected": -2.0435824394226074,
+      "logps/chosen": -345.69683837890625,
+      "logps/rejected": -355.43841552734375,
+      "loss": 0.1641,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.57981938123703,
+      "rewards/margins": 3.4938979148864746,
+      "rewards/rejected": -4.0737175941467285,
+      "step": 610
+    },
+    {
+      "epoch": 1.3,
+      "grad_norm": 40.76003061063018,
+      "learning_rate": 3.500128642743793e-07,
+      "logits/chosen": -1.9793628454208374,
+      "logits/rejected": -1.8766018152236938,
+      "logps/chosen": -275.6058654785156,
+      "logps/rejected": -325.64532470703125,
+      "loss": 0.1676,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -0.4727633595466614,
+      "rewards/margins": 3.388253688812256,
+      "rewards/rejected": -3.8610172271728516,
+      "step": 620
+    },
+    {
+      "epoch": 1.32,
+      "grad_norm": 27.005984360116628,
+      "learning_rate": 3.4440382358952115e-07,
+      "logits/chosen": -2.0091536045074463,
+      "logits/rejected": -1.8107984066009521,
+      "logps/chosen": -286.50762939453125,
+      "logps/rejected": -327.90350341796875,
+      "loss": 0.1604,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -0.3183102309703827,
+      "rewards/margins": 3.1906726360321045,
+      "rewards/rejected": -3.5089828968048096,
+      "step": 630
+    },
+    {
+      "epoch": 1.34,
+      "grad_norm": 23.606941374377882,
+      "learning_rate": 3.387387957438061e-07,
+      "logits/chosen": -1.9466636180877686,
+      "logits/rejected": -1.7844781875610352,
+      "logps/chosen": -283.4999084472656,
+      "logps/rejected": -322.8622131347656,
+      "loss": 0.1551,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -0.5999014973640442,
+      "rewards/margins": 3.3343536853790283,
+      "rewards/rejected": -3.9342548847198486,
+      "step": 640
+    },
+    {
+      "epoch": 1.36,
+      "grad_norm": 35.62460431561092,
+      "learning_rate": 3.33021140440403e-07,
+      "logits/chosen": -2.041600465774536,
+      "logits/rejected": -1.8977054357528687,
+      "logps/chosen": -282.59033203125,
+      "logps/rejected": -359.6671447753906,
+      "loss": 0.1662,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.652147650718689,
+      "rewards/margins": 3.4888508319854736,
+      "rewards/rejected": -4.140998363494873,
+      "step": 650
+    },
+    {
+      "epoch": 1.38,
+      "grad_norm": 40.45521004861816,
+      "learning_rate": 3.272542485937368e-07,
+      "logits/chosen": -2.2044432163238525,
+      "logits/rejected": -1.995490312576294,
+      "logps/chosen": -281.81591796875,
+      "logps/rejected": -305.563720703125,
+      "loss": 0.1562,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -0.9047712087631226,
+      "rewards/margins": 3.0545411109924316,
+      "rewards/rejected": -3.9593119621276855,
+      "step": 660
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 25.83353768539722,
+      "learning_rate": 3.214415403184725e-07,
+      "logits/chosen": -2.136054515838623,
+      "logits/rejected": -2.0022921562194824,
+      "logps/chosen": -287.70721435546875,
+      "logps/rejected": -347.98626708984375,
+      "loss": 0.1527,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -0.8447133898735046,
+      "rewards/margins": 3.195286512374878,
+      "rewards/rejected": -4.039999961853027,
+      "step": 670
+    },
+    {
+      "epoch": 1.42,
+      "grad_norm": 33.98889011519535,
+      "learning_rate": 3.155864629011798e-07,
+      "logits/chosen": -2.006204605102539,
+      "logits/rejected": -1.8692758083343506,
+      "logps/chosen": -266.548583984375,
+      "logps/rejected": -349.1086730957031,
+      "loss": 0.1618,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -0.7886885404586792,
+      "rewards/margins": 3.2961840629577637,
+      "rewards/rejected": -4.084872722625732,
+      "step": 680
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 23.86841822029929,
+      "learning_rate": 3.096924887558854e-07,
+      "logits/chosen": -2.0170860290527344,
+      "logits/rejected": -1.8624064922332764,
+      "logps/chosen": -289.9138488769531,
+      "logps/rejected": -340.1861877441406,
+      "loss": 0.1705,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.879202663898468,
+      "rewards/margins": 3.349952220916748,
+      "rewards/rejected": -4.22915506362915,
+      "step": 690
+    },
+    {
+      "epoch": 1.46,
+      "grad_norm": 30.325436164549153,
+      "learning_rate": 3.0376311336472157e-07,
+      "logits/chosen": -1.9222033023834229,
+      "logits/rejected": -1.7393171787261963,
+      "logps/chosen": -308.6827087402344,
+      "logps/rejected": -357.66937255859375,
+      "loss": 0.1567,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -1.2114474773406982,
+      "rewards/margins": 3.2933521270751953,
+      "rewards/rejected": -4.504799842834473,
+      "step": 700
+    },
+    {
+      "epoch": 1.46,
+      "eval_logits/chosen": -2.045348882675171,
+      "eval_logits/rejected": -1.9284720420837402,
+      "eval_logps/chosen": -298.781982421875,
+      "eval_logps/rejected": -333.33538818359375,
+      "eval_loss": 0.5347517132759094,
+      "eval_rewards/accuracies": 0.7890625,
+      "eval_rewards/chosen": -1.8094260692596436,
+      "eval_rewards/margins": 1.7239831686019897,
+      "eval_rewards/rejected": -3.5334088802337646,
+      "eval_runtime": 96.8349,
+      "eval_samples_per_second": 20.654,
+      "eval_steps_per_second": 0.33,
+      "step": 700
+    },
+    {
+      "epoch": 1.49,
+      "grad_norm": 44.30207392759187,
+      "learning_rate": 2.9780185320489397e-07,
+      "logits/chosen": -1.9471759796142578,
+      "logits/rejected": -1.777573585510254,
+      "logps/chosen": -307.9242858886719,
+      "logps/rejected": -352.77618408203125,
+      "loss": 0.1417,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -0.9882961511611938,
+      "rewards/margins": 3.599376678466797,
+      "rewards/rejected": -4.587672233581543,
+      "step": 710
+    },
+    {
+      "epoch": 1.51,
+      "grad_norm": 37.177307769645225,
+      "learning_rate": 2.9181224366319943e-07,
+      "logits/chosen": -1.9124629497528076,
+      "logits/rejected": -1.7028827667236328,
+      "logps/chosen": -315.4266662597656,
+      "logps/rejected": -355.34857177734375,
+      "loss": 0.166,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.7210714817047119,
+      "rewards/margins": 3.6800003051757812,
+      "rewards/rejected": -4.401071071624756,
+      "step": 720
+    },
+    {
+      "epoch": 1.53,
+      "grad_norm": 39.29672592174553,
+      "learning_rate": 2.857978369393279e-07,
+      "logits/chosen": -2.243919849395752,
+      "logits/rejected": -2.1087582111358643,
+      "logps/chosen": -308.04412841796875,
+      "logps/rejected": -345.15594482421875,
+      "loss": 0.152,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.5387176871299744,
+      "rewards/margins": 3.3322176933288574,
+      "rewards/rejected": -3.8709354400634766,
+      "step": 730
+    },
+    {
+      "epoch": 1.55,
+      "grad_norm": 24.15001856800991,
+      "learning_rate": 2.797621999391938e-07,
+      "logits/chosen": -2.3955087661743164,
+      "logits/rejected": -2.282515048980713,
+      "logps/chosen": -296.63604736328125,
+      "logps/rejected": -331.5033264160156,
+      "loss": 0.1584,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -1.0399105548858643,
+      "rewards/margins": 3.2611820697784424,
+      "rewards/rejected": -4.301093101501465,
+      "step": 740
+    },
+    {
+      "epoch": 1.57,
+      "grad_norm": 28.800655620715126,
+      "learning_rate": 2.7370891215954565e-07,
+      "logits/chosen": -2.291198253631592,
+      "logits/rejected": -2.1874325275421143,
+      "logps/chosen": -294.86126708984375,
+      "logps/rejected": -342.1693420410156,
+      "loss": 0.1327,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -0.5225303769111633,
+      "rewards/margins": 3.4790916442871094,
+      "rewards/rejected": -4.001622200012207,
+      "step": 750
+    },
+    {
+      "epoch": 1.59,
+      "grad_norm": 36.52736639617503,
+      "learning_rate": 2.676415635651091e-07,
+      "logits/chosen": -2.2511188983917236,
+      "logits/rejected": -2.1171295642852783,
+      "logps/chosen": -314.3142395019531,
+      "logps/rejected": -401.13018798828125,
+      "loss": 0.1373,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.4879869818687439,
+      "rewards/margins": 4.032652854919434,
+      "rewards/rejected": -4.520639896392822,
+      "step": 760
+    },
+    {
+      "epoch": 1.61,
+      "grad_norm": 22.81336311327227,
+      "learning_rate": 2.615637524595207e-07,
+      "logits/chosen": -2.2899999618530273,
+      "logits/rejected": -2.117185115814209,
+      "logps/chosen": -290.29815673828125,
+      "logps/rejected": -304.9264831542969,
+      "loss": 0.137,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.9585368037223816,
+      "rewards/margins": 3.114755153656006,
+      "rewards/rejected": -4.0732927322387695,
+      "step": 770
+    },
+    {
+      "epoch": 1.63,
+      "grad_norm": 25.499517087179957,
+      "learning_rate": 2.55479083351317e-07,
+      "logits/chosen": -2.2676925659179688,
+      "logits/rejected": -2.083047866821289,
+      "logps/chosen": -301.6076965332031,
+      "logps/rejected": -364.6853942871094,
+      "loss": 0.1536,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -0.8435381054878235,
+      "rewards/margins": 3.4508233070373535,
+      "rewards/rejected": -4.294361114501953,
+      "step": 780
+    },
+    {
+      "epoch": 1.65,
+      "grad_norm": 27.418147190397534,
+      "learning_rate": 2.4939116481624407e-07,
+      "logits/chosen": -2.220470666885376,
+      "logits/rejected": -1.959284782409668,
+      "logps/chosen": -299.3353576660156,
+      "logps/rejected": -319.30914306640625,
+      "loss": 0.1568,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.8999794125556946,
+      "rewards/margins": 3.3365378379821777,
+      "rewards/rejected": -4.236516952514648,
+      "step": 790
+    },
+    {
+      "epoch": 1.67,
+      "grad_norm": 22.42449451210148,
+      "learning_rate": 2.4330360735715374e-07,
+      "logits/chosen": -2.2539217472076416,
+      "logits/rejected": -2.0446996688842773,
+      "logps/chosen": -333.44366455078125,
+      "logps/rejected": -357.18841552734375,
+      "loss": 0.1475,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -1.156060814857483,
+      "rewards/margins": 3.485567092895508,
+      "rewards/rejected": -4.641627311706543,
+      "step": 800
+    },
+    {
+      "epoch": 1.67,
+      "eval_logits/chosen": -2.2409238815307617,
+      "eval_logits/rejected": -2.1201870441436768,
+      "eval_logps/chosen": -296.3533020019531,
+      "eval_logps/rejected": -332.4950866699219,
+      "eval_loss": 0.538158118724823,
+      "eval_rewards/accuracies": 0.80078125,
+      "eval_rewards/chosen": -1.6879926919937134,
+      "eval_rewards/margins": 1.8034026622772217,
+      "eval_rewards/rejected": -3.4913952350616455,
+      "eval_runtime": 96.753,
+      "eval_samples_per_second": 20.671,
+      "eval_steps_per_second": 0.331,
+      "step": 800
+    },
+    {
+      "epoch": 1.69,
+      "grad_norm": 27.648844978471217,
+      "learning_rate": 2.3722002126275822e-07,
+      "logits/chosen": -2.118046998977661,
+      "logits/rejected": -1.9478477239608765,
+      "logps/chosen": -295.06268310546875,
+      "logps/rejected": -350.56829833984375,
+      "loss": 0.1534,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.974108099937439,
+      "rewards/margins": 3.4369990825653076,
+      "rewards/rejected": -4.411107540130615,
+      "step": 810
+    },
+    {
+      "epoch": 1.72,
+      "grad_norm": 33.288825778542474,
+      "learning_rate": 2.311440144665108e-07,
+      "logits/chosen": -2.166576623916626,
+      "logits/rejected": -2.005044460296631,
+      "logps/chosen": -285.5037841796875,
+      "logps/rejected": -354.74578857421875,
+      "loss": 0.1616,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": -0.7247167229652405,
+      "rewards/margins": 3.4520015716552734,
+      "rewards/rejected": -4.176717758178711,
+      "step": 820
+    },
+    {
+      "epoch": 1.74,
+      "grad_norm": 28.54134777409646,
+      "learning_rate": 2.2507919040688398e-07,
+      "logits/chosen": -2.035219669342041,
+      "logits/rejected": -1.9823194742202759,
+      "logps/chosen": -302.2669677734375,
+      "logps/rejected": -337.7466125488281,
+      "loss": 0.1361,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": -0.7065524458885193,
+      "rewards/margins": 3.6438660621643066,
+      "rewards/rejected": -4.350418567657471,
+      "step": 830
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 29.525737237536532,
+      "learning_rate": 2.19029145890313e-07,
+      "logits/chosen": -2.0496773719787598,
+      "logits/rejected": -1.8104021549224854,
+      "logps/chosen": -295.2398376464844,
+      "logps/rejected": -348.8295593261719,
+      "loss": 0.137,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -1.180415391921997,
+      "rewards/margins": 4.056873798370361,
+      "rewards/rejected": -5.237288951873779,
+      "step": 840
+    },
+    {
+      "epoch": 1.78,
+      "grad_norm": 29.418652242465264,
+      "learning_rate": 2.1299746895807268e-07,
+      "logits/chosen": -2.0320029258728027,
+      "logits/rejected": -1.8159306049346924,
+      "logps/chosen": -298.8084716796875,
+      "logps/rejected": -351.4793395996094,
+      "loss": 0.1426,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.8422020077705383,
+      "rewards/margins": 3.6892521381378174,
+      "rewards/rejected": -4.531454563140869,
+      "step": 850
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 28.75219392191004,
+      "learning_rate": 2.0698773675835246e-07,
+      "logits/chosen": -1.9433095455169678,
+      "logits/rejected": -1.7687761783599854,
+      "logps/chosen": -307.7235107421875,
+      "logps/rejected": -342.0555114746094,
+      "loss": 0.1486,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.4170432090759277,
+      "rewards/margins": 3.423736095428467,
+      "rewards/rejected": -4.8407793045043945,
+      "step": 860
+    },
+    {
+      "epoch": 1.82,
+      "grad_norm": 36.42272712771708,
+      "learning_rate": 2.0100351342479216e-07,
+      "logits/chosen": -1.9862619638442993,
+      "logits/rejected": -1.750312089920044,
+      "logps/chosen": -312.4382019042969,
+      "logps/rejected": -355.03045654296875,
+      "loss": 0.1468,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -1.4696812629699707,
+      "rewards/margins": 3.662865400314331,
+      "rewards/rejected": -5.132546901702881,
+      "step": 870
+    },
+    {
+      "epoch": 1.84,
+      "grad_norm": 29.128706338046875,
+      "learning_rate": 1.9504834796273545e-07,
+      "logits/chosen": -2.0059654712677,
+      "logits/rejected": -1.7494417428970337,
+      "logps/chosen": -284.04669189453125,
+      "logps/rejected": -307.86334228515625,
+      "loss": 0.1514,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -1.4539439678192139,
+      "rewards/margins": 3.2319369316101074,
+      "rewards/rejected": -4.6858811378479,
+      "step": 880
+    },
+    {
+      "epoch": 1.86,
+      "grad_norm": 54.834861913718775,
+      "learning_rate": 1.8912577214445558e-07,
+      "logits/chosen": -2.125431537628174,
+      "logits/rejected": -1.975856065750122,
+      "logps/chosen": -302.9638671875,
+      "logps/rejected": -345.03851318359375,
+      "loss": 0.1219,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -0.8938841819763184,
+      "rewards/margins": 3.579861879348755,
+      "rewards/rejected": -4.473746299743652,
+      "step": 890
+    },
+    {
+      "epoch": 1.88,
+      "grad_norm": 36.4163607482829,
+      "learning_rate": 1.8323929841460178e-07,
+      "logits/chosen": -2.0952582359313965,
+      "logits/rejected": -1.9091037511825562,
+      "logps/chosen": -300.1233215332031,
+      "logps/rejected": -345.5321960449219,
+      "loss": 0.1422,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -1.1558645963668823,
+      "rewards/margins": 3.8015289306640625,
+      "rewards/rejected": -4.957393169403076,
+      "step": 900
+    },
+    {
+      "epoch": 1.88,
+      "eval_logits/chosen": -2.198045492172241,
+      "eval_logits/rejected": -2.0630321502685547,
+      "eval_logps/chosen": -296.03240966796875,
+      "eval_logps/rejected": -335.6015625,
+      "eval_loss": 0.5517675280570984,
+      "eval_rewards/accuracies": 0.78515625,
+      "eval_rewards/chosen": -1.67194664478302,
+      "eval_rewards/margins": 1.9747719764709473,
+      "eval_rewards/rejected": -3.6467185020446777,
+      "eval_runtime": 96.8149,
+      "eval_samples_per_second": 20.658,
+      "eval_steps_per_second": 0.331,
+      "step": 900
+    },
+    {
+      "epoch": 1.9,
+      "grad_norm": 32.93263141944983,
+      "learning_rate": 1.7739241780710745e-07,
+      "logits/chosen": -2.067739963531494,
+      "logits/rejected": -1.879922866821289,
+      "logps/chosen": -285.4407958984375,
+      "logps/rejected": -354.6173400878906,
+      "loss": 0.1572,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": -1.3456696271896362,
+      "rewards/margins": 3.62982439994812,
+      "rewards/rejected": -4.975494384765625,
+      "step": 910
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 30.163608722340616,
+      "learning_rate": 1.7158859787479653e-07,
+      "logits/chosen": -1.945054054260254,
+      "logits/rejected": -1.7525079250335693,
+      "logps/chosen": -271.43670654296875,
+      "logps/rejected": -355.9076232910156,
+      "loss": 0.1457,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -1.121407151222229,
+      "rewards/margins": 3.6710968017578125,
+      "rewards/rejected": -4.79250431060791,
+      "step": 920
+    },
+    {
+      "epoch": 1.95,
+      "grad_norm": 38.64417902485811,
+      "learning_rate": 1.6583128063291573e-07,
+      "logits/chosen": -2.06476092338562,
+      "logits/rejected": -1.8927568197250366,
+      "logps/chosen": -288.0918884277344,
+      "logps/rejected": -331.8973388671875,
+      "loss": 0.1544,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -1.375701904296875,
+      "rewards/margins": 3.178281784057617,
+      "rewards/rejected": -4.55398416519165,
+      "step": 930
+    },
+    {
+      "epoch": 1.97,
+      "grad_norm": 33.76607136138911,
+      "learning_rate": 1.6012388051781152e-07,
+      "logits/chosen": -2.157871961593628,
+      "logits/rejected": -1.855956792831421,
+      "logps/chosen": -337.51568603515625,
+      "logps/rejected": -342.9998779296875,
+      "loss": 0.1424,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -1.327646255493164,
+      "rewards/margins": 3.696226119995117,
+      "rewards/rejected": -5.0238728523254395,
+      "step": 940
+    },
+    {
+      "epoch": 1.99,
+      "grad_norm": 34.755846610400575,
+      "learning_rate": 1.54469782361964e-07,
+      "logits/chosen": -1.9773037433624268,
+      "logits/rejected": -1.8121259212493896,
+      "logps/chosen": -310.9110107421875,
+      "logps/rejected": -364.62359619140625,
+      "loss": 0.145,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -1.399800181388855,
+      "rewards/margins": 3.679616928100586,
+      "rewards/rejected": -5.0794172286987305,
+      "step": 950
+    },
+    {
+      "epoch": 2.01,
+      "grad_norm": 9.487017101643959,
+      "learning_rate": 1.488723393865766e-07,
+      "logits/chosen": -1.9744060039520264,
+      "logits/rejected": -1.662217140197754,
+      "logps/chosen": -328.89556884765625,
+      "logps/rejected": -356.9366149902344,
+      "loss": 0.0972,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -1.0852992534637451,
+      "rewards/margins": 4.1752800941467285,
+      "rewards/rejected": -5.260579586029053,
+      "step": 960
+    },
+    {
+      "epoch": 2.03,
+      "grad_norm": 6.928449852103405,
+      "learning_rate": 1.4333487121291395e-07,
+      "logits/chosen": -1.9290939569473267,
+      "logits/rejected": -1.646998643875122,
+      "logps/chosen": -290.2763977050781,
+      "logps/rejected": -359.6099548339844,
+      "loss": 0.0377,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -0.9613211750984192,
+      "rewards/margins": 4.470429420471191,
+      "rewards/rejected": -5.431751251220703,
+      "step": 970
+    },
+    {
+      "epoch": 2.05,
+      "grad_norm": 12.357238308252128,
+      "learning_rate": 1.3786066189356627e-07,
+      "logits/chosen": -1.840659499168396,
+      "logits/rejected": -1.3801103830337524,
+      "logps/chosen": -323.639892578125,
+      "logps/rejected": -382.70281982421875,
+      "loss": 0.0412,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1784292459487915,
+      "rewards/margins": 4.754664897918701,
+      "rewards/rejected": -5.933094501495361,
+      "step": 980
+    },
+    {
+      "epoch": 2.07,
+      "grad_norm": 12.765885065958322,
+      "learning_rate": 1.3245295796480788e-07,
+      "logits/chosen": -1.6344573497772217,
+      "logits/rejected": -1.3037782907485962,
+      "logps/chosen": -280.02520751953125,
+      "logps/rejected": -373.9437561035156,
+      "loss": 0.0483,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -1.5978093147277832,
+      "rewards/margins": 4.701867580413818,
+      "rewards/rejected": -6.299676895141602,
+      "step": 990
+    },
+    {
+      "epoch": 2.09,
+      "grad_norm": 10.639657237631194,
+      "learning_rate": 1.2711496652120578e-07,
+      "logits/chosen": -1.6058915853500366,
+      "logits/rejected": -1.2118239402770996,
+      "logps/chosen": -295.133056640625,
+      "logps/rejected": -380.3929748535156,
+      "loss": 0.044,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -1.539745807647705,
+      "rewards/margins": 5.079216957092285,
+      "rewards/rejected": -6.61896276473999,
+      "step": 1000
+    },
+    {
+      "epoch": 2.09,
+      "eval_logits/chosen": -1.7406296730041504,
+      "eval_logits/rejected": -1.4629344940185547,
+      "eval_logps/chosen": -316.4520263671875,
+      "eval_logps/rejected": -365.49591064453125,
+      "eval_loss": 0.6058282852172852,
+      "eval_rewards/accuracies": 0.7890625,
+      "eval_rewards/chosen": -2.6929280757904053,
+      "eval_rewards/margins": 2.4485080242156982,
+      "eval_rewards/rejected": -5.1414361000061035,
+      "eval_runtime": 96.8107,
+      "eval_samples_per_second": 20.659,
+      "eval_steps_per_second": 0.331,
+      "step": 1000
+    },
+    {
+      "epoch": 2.11,
+      "grad_norm": 8.532531818207984,
+      "learning_rate": 1.2184985331361878e-07,
+      "logits/chosen": -1.5615358352661133,
+      "logits/rejected": -1.3172276020050049,
+      "logps/chosen": -293.0592041015625,
+      "logps/rejected": -389.7725830078125,
+      "loss": 0.0365,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -1.6128288507461548,
+      "rewards/margins": 4.956469535827637,
+      "rewards/rejected": -6.56929874420166,
+      "step": 1010
+    },
+    {
+      "epoch": 2.13,
+      "grad_norm": 8.404017113839789,
+      "learning_rate": 1.1666074087171627e-07,
+      "logits/chosen": -1.585607886314392,
+      "logits/rejected": -1.2554726600646973,
+      "logps/chosen": -287.69989013671875,
+      "logps/rejected": -414.93267822265625,
+      "loss": 0.0337,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -1.7983949184417725,
+      "rewards/margins": 5.210457801818848,
+      "rewards/rejected": -7.008852481842041,
+      "step": 1020
+    },
+    {
+      "epoch": 2.15,
+      "grad_norm": 10.846070292764287,
+      "learning_rate": 1.115507066521304e-07,
+      "logits/chosen": -1.5471771955490112,
+      "logits/rejected": -1.1083250045776367,
+      "logps/chosen": -304.71533203125,
+      "logps/rejected": -398.4665222167969,
+      "loss": 0.0418,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -1.608544111251831,
+      "rewards/margins": 5.360156536102295,
+      "rewards/rejected": -6.968701362609863,
+      "step": 1030
+    },
+    {
+      "epoch": 2.18,
+      "grad_norm": 11.175167320601789,
+      "learning_rate": 1.065227812133381e-07,
+      "logits/chosen": -1.481093168258667,
+      "logits/rejected": -0.8541752696037292,
+      "logps/chosen": -340.7627868652344,
+      "logps/rejected": -378.1071472167969,
+      "loss": 0.0369,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -2.031552791595459,
+      "rewards/margins": 5.121342658996582,
+      "rewards/rejected": -7.152895927429199,
+      "step": 1040
+    },
+    {
+      "epoch": 2.2,
+      "grad_norm": 9.191567382424724,
+      "learning_rate": 1.0157994641835734e-07,
+      "logits/chosen": -1.3451311588287354,
+      "logits/rejected": -0.851954460144043,
+      "logps/chosen": -312.46246337890625,
+      "logps/rejected": -443.61480712890625,
+      "loss": 0.0366,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.4045233726501465,
+      "rewards/margins": 5.692896842956543,
+      "rewards/rejected": -8.097419738769531,
+      "step": 1050
+    },
+    {
+      "epoch": 2.22,
+      "grad_norm": 10.368281313443552,
+      "learning_rate": 9.672513366632259e-08,
+      "logits/chosen": -1.4387218952178955,
+      "logits/rejected": -0.9104587435722351,
+      "logps/chosen": -313.5397644042969,
+      "logps/rejected": -392.3639221191406,
+      "loss": 0.0335,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -2.1655592918395996,
+      "rewards/margins": 5.339682579040527,
+      "rewards/rejected": -7.505242347717285,
+      "step": 1060
+    },
+    {
+      "epoch": 2.24,
+      "grad_norm": 9.399192097048465,
+      "learning_rate": 9.196122215398824e-08,
+      "logits/chosen": -1.5347858667373657,
+      "logits/rejected": -1.1239128112792969,
+      "logps/chosen": -345.493408203125,
+      "logps/rejected": -442.33880615234375,
+      "loss": 0.0335,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -2.0253329277038574,
+      "rewards/margins": 5.696555137634277,
+      "rewards/rejected": -7.721887111663818,
+      "step": 1070
+    },
+    {
+      "epoch": 2.26,
+      "grad_norm": 13.854211150002678,
+      "learning_rate": 8.729103716819111e-08,
+      "logits/chosen": -1.4636061191558838,
+      "logits/rejected": -0.9430959820747375,
+      "logps/chosen": -348.165771484375,
+      "logps/rejected": -437.73150634765625,
+      "loss": 0.0324,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -1.851531982421875,
+      "rewards/margins": 5.859683990478516,
+      "rewards/rejected": -7.711215972900391,
+      "step": 1080
+    },
+    {
+      "epoch": 2.28,
+      "grad_norm": 13.012519621486515,
+      "learning_rate": 8.271734841028552e-08,
+      "logits/chosen": -1.4426562786102295,
+      "logits/rejected": -0.9428352117538452,
+      "logps/chosen": -357.9177551269531,
+      "logps/rejected": -440.7193298339844,
+      "loss": 0.0276,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.550102710723877,
+      "rewards/margins": 5.568441390991211,
+      "rewards/rejected": -8.11854362487793,
+      "step": 1090
+    },
+    {
+      "epoch": 2.3,
+      "grad_norm": 7.343017253233073,
+      "learning_rate": 7.824286835354262e-08,
+      "logits/chosen": -1.2438175678253174,
+      "logits/rejected": -0.7905367612838745,
+      "logps/chosen": -294.59564208984375,
+      "logps/rejected": -394.0030517578125,
+      "loss": 0.0307,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -1.902488350868225,
+      "rewards/margins": 5.68535041809082,
+      "rewards/rejected": -7.587839603424072,
+      "step": 1100
+    },
+    {
+      "epoch": 2.3,
+      "eval_logits/chosen": -1.330996036529541,
+      "eval_logits/rejected": -0.9162449240684509,
+      "eval_logps/chosen": -337.038330078125,
+      "eval_logps/rejected": -397.1616516113281,
+      "eval_loss": 0.6699962019920349,
+      "eval_rewards/accuracies": 0.76953125,
+      "eval_rewards/chosen": -3.722243070602417,
+      "eval_rewards/margins": 3.002480983734131,
+      "eval_rewards/rejected": -6.724723815917969,
+      "eval_runtime": 96.8573,
+      "eval_samples_per_second": 20.649,
+      "eval_steps_per_second": 0.33,
+      "step": 1100
+    },
+    {
+      "epoch": 2.32,
+      "grad_norm": 22.14619972377515,
+      "learning_rate": 7.387025063449081e-08,
+      "logits/chosen": -1.2601680755615234,
+      "logits/rejected": -0.4739012122154236,
+      "logps/chosen": -334.8611755371094,
+      "logps/rejected": -403.3932800292969,
+      "loss": 0.0294,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.9858272075653076,
+      "rewards/margins": 5.638393878936768,
+      "rewards/rejected": -8.624221801757812,
+      "step": 1110
+    },
+    {
+      "epoch": 2.34,
+      "grad_norm": 22.797319211696394,
+      "learning_rate": 6.960208847914884e-08,
+      "logits/chosen": -1.1370799541473389,
+      "logits/rejected": -0.7852309942245483,
+      "logps/chosen": -302.19561767578125,
+      "logps/rejected": -427.47283935546875,
+      "loss": 0.0337,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -2.330061912536621,
+      "rewards/margins": 6.187905311584473,
+      "rewards/rejected": -8.517967224121094,
+      "step": 1120
+    },
+    {
+      "epoch": 2.36,
+      "grad_norm": 12.447087206832133,
+      "learning_rate": 6.544091316508646e-08,
+      "logits/chosen": -1.1922690868377686,
+      "logits/rejected": -0.7016154527664185,
+      "logps/chosen": -337.29254150390625,
+      "logps/rejected": -419.9600524902344,
+      "loss": 0.0309,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -2.260216474533081,
+      "rewards/margins": 6.044236660003662,
+      "rewards/rejected": -8.30445384979248,
+      "step": 1130
+    },
+    {
+      "epoch": 2.38,
+      "grad_norm": 6.670071848884272,
+      "learning_rate": 6.138919252022435e-08,
+      "logits/chosen": -1.2661590576171875,
+      "logits/rejected": -0.726656973361969,
+      "logps/chosen": -337.79095458984375,
+      "logps/rejected": -444.1019592285156,
+      "loss": 0.0321,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.347249746322632,
+      "rewards/margins": 5.921750068664551,
+      "rewards/rejected": -8.268999099731445,
+      "step": 1140
+    },
+    {
+      "epoch": 2.41,
+      "grad_norm": 20.202187916611614,
+      "learning_rate": 5.7449329459262895e-08,
+      "logits/chosen": -1.3327900171279907,
+      "logits/rejected": -0.7205911874771118,
+      "logps/chosen": -327.96160888671875,
+      "logps/rejected": -431.84246826171875,
+      "loss": 0.0315,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -2.5763912200927734,
+      "rewards/margins": 6.172744274139404,
+      "rewards/rejected": -8.749135971069336,
+      "step": 1150
+    },
+    {
+      "epoch": 2.43,
+      "grad_norm": 12.705067042675866,
+      "learning_rate": 5.362366055860934e-08,
+      "logits/chosen": -1.3245675563812256,
+      "logits/rejected": -0.8294090032577515,
+      "logps/chosen": -339.1673278808594,
+      "logps/rejected": -427.9764709472656,
+      "loss": 0.0264,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.199721336364746,
+      "rewards/margins": 5.974953651428223,
+      "rewards/rejected": -8.174674034118652,
+      "step": 1160
+    },
+    {
+      "epoch": 2.45,
+      "grad_norm": 10.500886681852085,
+      "learning_rate": 4.991445467064689e-08,
+      "logits/chosen": -1.205335259437561,
+      "logits/rejected": -0.8114490509033203,
+      "logps/chosen": -309.63421630859375,
+      "logps/rejected": -438.3241271972656,
+      "loss": 0.0317,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -2.618736982345581,
+      "rewards/margins": 6.0979084968566895,
+      "rewards/rejected": -8.716645240783691,
+      "step": 1170
+    },
+    {
+      "epoch": 2.47,
+      "grad_norm": 12.519852374223428,
+      "learning_rate": 4.6323911578168146e-08,
+      "logits/chosen": -1.1557085514068604,
+      "logits/rejected": -0.6752947568893433,
+      "logps/chosen": -306.9227294921875,
+      "logps/rejected": -395.4352722167969,
+      "loss": 0.0323,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.859976053237915,
+      "rewards/margins": 5.740398406982422,
+      "rewards/rejected": -8.600374221801758,
+      "step": 1180
+    },
+    {
+      "epoch": 2.49,
+      "grad_norm": 14.10405859024082,
+      "learning_rate": 4.285416068977166e-08,
+      "logits/chosen": -1.2870628833770752,
+      "logits/rejected": -0.8281751871109009,
+      "logps/chosen": -295.5076904296875,
+      "logps/rejected": -401.1368713378906,
+      "loss": 0.0387,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.580979585647583,
+      "rewards/margins": 6.097764015197754,
+      "rewards/rejected": -8.678743362426758,
+      "step": 1190
+    },
+    {
+      "epoch": 2.51,
+      "grad_norm": 10.124969755651362,
+      "learning_rate": 3.9507259776993954e-08,
+      "logits/chosen": -1.2816931009292603,
+      "logits/rejected": -0.8490222096443176,
+      "logps/chosen": -333.55950927734375,
+      "logps/rejected": -454.38433837890625,
+      "loss": 0.0317,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.4796204566955566,
+      "rewards/margins": 6.314810752868652,
+      "rewards/rejected": -8.794431686401367,
+      "step": 1200
+    },
+    {
+      "epoch": 2.51,
+      "eval_logits/chosen": -1.2927337884902954,
+      "eval_logits/rejected": -0.9226770401000977,
+      "eval_logps/chosen": -341.82611083984375,
+      "eval_logps/rejected": -401.94482421875,
+      "eval_loss": 0.6710954308509827,
+      "eval_rewards/accuracies": 0.77734375,
+      "eval_rewards/chosen": -3.9616329669952393,
+      "eval_rewards/margins": 3.002251148223877,
+      "eval_rewards/rejected": -6.963884353637695,
+      "eval_runtime": 96.6177,
+      "eval_samples_per_second": 20.7,
+      "eval_steps_per_second": 0.331,
+      "step": 1200
+    },
+    {
+      "epoch": 2.53,
+      "grad_norm": 9.343788604433174,
+      "learning_rate": 3.6285193753926995e-08,
+      "logits/chosen": -1.2478582859039307,
+      "logits/rejected": -0.6749597787857056,
+      "logps/chosen": -335.22381591796875,
+      "logps/rejected": -448.06427001953125,
+      "loss": 0.0372,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -3.044740915298462,
+      "rewards/margins": 5.88831090927124,
+      "rewards/rejected": -8.933051109313965,
+      "step": 1210
+    },
+    {
+      "epoch": 2.55,
+      "grad_norm": 15.370939596872683,
+      "learning_rate": 3.3189873500044376e-08,
+      "logits/chosen": -1.2345765829086304,
+      "logits/rejected": -0.7880679368972778,
+      "logps/chosen": -316.099853515625,
+      "logps/rejected": -424.1470642089844,
+      "loss": 0.0304,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -2.545562267303467,
+      "rewards/margins": 6.050265312194824,
+      "rewards/rejected": -8.595827102661133,
+      "step": 1220
+    },
+    {
+      "epoch": 2.57,
+      "grad_norm": 9.256523996293426,
+      "learning_rate": 3.022313472693447e-08,
+      "logits/chosen": -1.2926331758499146,
+      "logits/rejected": -0.7623311877250671,
+      "logps/chosen": -323.86981201171875,
+      "logps/rejected": -432.7613220214844,
+      "loss": 0.0326,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -3.008089065551758,
+      "rewards/margins": 5.656081676483154,
+      "rewards/rejected": -8.66417121887207,
+      "step": 1230
+    },
+    {
+      "epoch": 2.59,
+      "grad_norm": 13.108228798737402,
+      "learning_rate": 2.738673688961296e-08,
+      "logits/chosen": -1.4012349843978882,
+      "logits/rejected": -0.9218941926956177,
+      "logps/chosen": -351.8048400878906,
+      "logps/rejected": -434.9781799316406,
+      "loss": 0.0339,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -2.5394856929779053,
+      "rewards/margins": 5.643149375915527,
+      "rewards/rejected": -8.182635307312012,
+      "step": 1240
+    },
+    {
+      "epoch": 2.62,
+      "grad_norm": 7.525159999057864,
+      "learning_rate": 2.4682362143059797e-08,
+      "logits/chosen": -1.2464749813079834,
+      "logits/rejected": -0.6382617354393005,
+      "logps/chosen": -326.1867370605469,
+      "logps/rejected": -427.4964294433594,
+      "loss": 0.0245,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -2.6778645515441895,
+      "rewards/margins": 6.1839213371276855,
+      "rewards/rejected": -8.861784934997559,
+      "step": 1250
+    },
+    {
+      "epoch": 2.64,
+      "grad_norm": 20.76488422355617,
+      "learning_rate": 2.2111614344599684e-08,
+      "logits/chosen": -1.1929394006729126,
+      "logits/rejected": -0.6367761492729187,
+      "logps/chosen": -346.1959533691406,
+      "logps/rejected": -437.5431213378906,
+      "loss": 0.0344,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -2.5477092266082764,
+      "rewards/margins": 5.813665390014648,
+      "rewards/rejected": -8.361373901367188,
+      "step": 1260
+    },
+    {
+      "epoch": 2.66,
+      "grad_norm": 8.669045699213816,
+      "learning_rate": 1.9676018102718213e-08,
+      "logits/chosen": -1.247870922088623,
+      "logits/rejected": -0.6409175395965576,
+      "logps/chosen": -365.83392333984375,
+      "logps/rejected": -399.61962890625,
+      "loss": 0.0338,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.6575913429260254,
+      "rewards/margins": 5.685526371002197,
+      "rewards/rejected": -8.343117713928223,
+      "step": 1270
+    },
+    {
+      "epoch": 2.68,
+      "grad_norm": 22.62219866138702,
+      "learning_rate": 1.7377017872876987e-08,
+      "logits/chosen": -1.208968162536621,
+      "logits/rejected": -0.6456762552261353,
+      "logps/chosen": -317.1292419433594,
+      "logps/rejected": -430.59613037109375,
+      "loss": 0.0336,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.8459768295288086,
+      "rewards/margins": 5.9941020011901855,
+      "rewards/rejected": -8.840079307556152,
+      "step": 1280
+    },
+    {
+      "epoch": 2.7,
+      "grad_norm": 20.33027332534465,
+      "learning_rate": 1.521597710086439e-08,
+      "logits/chosen": -1.1466628313064575,
+      "logits/rejected": -0.7244657278060913,
+      "logps/chosen": -356.29986572265625,
+      "logps/rejected": -458.36279296875,
+      "loss": 0.0348,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -2.7150323390960693,
+      "rewards/margins": 6.071440696716309,
+      "rewards/rejected": -8.786473274230957,
+      "step": 1290
+    },
+    {
+      "epoch": 2.72,
+      "grad_norm": 11.138423191199344,
+      "learning_rate": 1.3194177414189905e-08,
+      "logits/chosen": -1.0380289554595947,
+      "logits/rejected": -0.5761706233024597,
+      "logps/chosen": -299.1542663574219,
+      "logps/rejected": -413.3262634277344,
+      "loss": 0.0264,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -2.816028118133545,
+      "rewards/margins": 5.865799903869629,
+      "rewards/rejected": -8.681828498840332,
+      "step": 1300
+    },
+    {
+      "epoch": 2.72,
+      "eval_logits/chosen": -1.2189825773239136,
+      "eval_logits/rejected": -0.8369507789611816,
+      "eval_logps/chosen": -347.2215576171875,
+      "eval_logps/rejected": -407.835205078125,
+      "eval_loss": 0.6777693033218384,
+      "eval_rewards/accuracies": 0.77734375,
+      "eval_rewards/chosen": -4.231404781341553,
+      "eval_rewards/margins": 3.0269954204559326,
+      "eval_rewards/rejected": -7.258399486541748,
+      "eval_runtime": 96.5907,
+      "eval_samples_per_second": 20.706,
+      "eval_steps_per_second": 0.331,
+      "step": 1300
+    },
+    {
+      "epoch": 2.74,
+      "grad_norm": 18.646420478229594,
+      "learning_rate": 1.1312817862001945e-08,
+      "logits/chosen": -1.174865961074829,
+      "logits/rejected": -0.5988924503326416,
+      "logps/chosen": -335.7987060546875,
+      "logps/rejected": -438.8302307128906,
+      "loss": 0.0323,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -3.0069799423217773,
+      "rewards/margins": 6.027297496795654,
+      "rewards/rejected": -9.034276962280273,
+      "step": 1310
+    },
+    {
+      "epoch": 2.76,
+      "grad_norm": 9.37653587990746,
+      "learning_rate": 9.57301420397924e-09,
+      "logits/chosen": -1.2077335119247437,
+      "logits/rejected": -0.6788758039474487,
+      "logps/chosen": -344.1173400878906,
+      "logps/rejected": -449.73822021484375,
+      "loss": 0.0296,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -2.7568278312683105,
+      "rewards/margins": 6.043575763702393,
+      "rewards/rejected": -8.800403594970703,
+      "step": 1320
+    },
+    {
+      "epoch": 2.78,
+      "grad_norm": 19.319012047791922,
+      "learning_rate": 7.975798248618076e-09,
+      "logits/chosen": -1.2139251232147217,
+      "logits/rejected": -0.5430259108543396,
+      "logps/chosen": -304.11981201171875,
+      "logps/rejected": -399.5338134765625,
+      "loss": 0.0403,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -2.8824665546417236,
+      "rewards/margins": 5.712244510650635,
+      "rewards/rejected": -8.594710350036621,
+      "step": 1330
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 15.988430571781114,
+      "learning_rate": 6.522117241307606e-09,
+      "logits/chosen": -1.257893681526184,
+      "logits/rejected": -0.7202562093734741,
+      "logps/chosen": -358.0582275390625,
+      "logps/rejected": -453.0985412597656,
+      "loss": 0.0249,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.752913475036621,
+      "rewards/margins": 5.904249668121338,
+      "rewards/rejected": -8.6571626663208,
+      "step": 1340
+    },
+    {
+      "epoch": 2.82,
+      "grad_norm": 18.522259659368,
+      "learning_rate": 5.212833302556258e-09,
+      "logits/chosen": -1.0940848588943481,
+      "logits/rejected": -0.642852783203125,
+      "logps/chosen": -330.330810546875,
+      "logps/rejected": -439.7862243652344,
+      "loss": 0.0316,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -3.221640110015869,
+      "rewards/margins": 5.774725914001465,
+      "rewards/rejected": -8.996365547180176,
+      "step": 1350
+    },
+    {
+      "epoch": 2.85,
+      "grad_norm": 14.304064916507468,
+      "learning_rate": 4.048722916702302e-09,
+      "logits/chosen": -1.2064802646636963,
+      "logits/rejected": -0.7113040685653687,
+      "logps/chosen": -350.12359619140625,
+      "logps/rejected": -433.10784912109375,
+      "loss": 0.0292,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -3.004544734954834,
+      "rewards/margins": 5.731839179992676,
+      "rewards/rejected": -8.736384391784668,
+      "step": 1360
+    },
+    {
+      "epoch": 2.87,
+      "grad_norm": 9.460405793660744,
+      "learning_rate": 3.030476471411664e-09,
+      "logits/chosen": -1.2229253053665161,
+      "logits/rejected": -0.5610889196395874,
+      "logps/chosen": -358.481689453125,
+      "logps/rejected": -432.617919921875,
+      "loss": 0.0255,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -3.1794846057891846,
+      "rewards/margins": 6.04774284362793,
+      "rewards/rejected": -9.227226257324219,
+      "step": 1370
+    },
+    {
+      "epoch": 2.89,
+      "grad_norm": 12.222248359736108,
+      "learning_rate": 2.158697848236607e-09,
+      "logits/chosen": -1.3239306211471558,
+      "logits/rejected": -0.696345865726471,
+      "logps/chosen": -345.43890380859375,
+      "logps/rejected": -420.57415771484375,
+      "loss": 0.0324,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.8451218605041504,
+      "rewards/margins": 5.87528657913208,
+      "rewards/rejected": -8.72040843963623,
+      "step": 1380
+    },
+    {
+      "epoch": 2.91,
+      "grad_norm": 12.712161033692837,
+      "learning_rate": 1.4339040644774092e-09,
+      "logits/chosen": -1.1367696523666382,
+      "logits/rejected": -0.6753785014152527,
+      "logps/chosen": -300.333984375,
+      "logps/rejected": -410.33966064453125,
+      "loss": 0.0346,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -2.8657116889953613,
+      "rewards/margins": 5.644627571105957,
+      "rewards/rejected": -8.510337829589844,
+      "step": 1390
+    },
+    {
+      "epoch": 2.93,
+      "grad_norm": 7.479511958985555,
+      "learning_rate": 8.56524966559885e-10,
+      "logits/chosen": -1.297228455543518,
+      "logits/rejected": -0.9393932223320007,
+      "logps/chosen": -337.4761962890625,
+      "logps/rejected": -469.80499267578125,
+      "loss": 0.0343,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.661848306655884,
+      "rewards/margins": 5.969611644744873,
+      "rewards/rejected": -8.631460189819336,
+      "step": 1400
+    },
+    {
+      "epoch": 2.93,
+      "eval_logits/chosen": -1.2134709358215332,
+      "eval_logits/rejected": -0.8310315608978271,
+      "eval_logps/chosen": -347.1476135253906,
+      "eval_logps/rejected": -408.3960876464844,
+      "eval_loss": 0.6824045181274414,
+      "eval_rewards/accuracies": 0.77734375,
+      "eval_rewards/chosen": -4.227706432342529,
+      "eval_rewards/margins": 3.058739423751831,
+      "eval_rewards/rejected": -7.286445617675781,
+      "eval_runtime": 96.8438,
+      "eval_samples_per_second": 20.652,
+      "eval_steps_per_second": 0.33,
+      "step": 1400
+    },
+    {
+      "epoch": 2.95,
+      "grad_norm": 25.645964088265643,
+      "learning_rate": 4.269029751107489e-10,
+      "logits/chosen": -1.169325590133667,
+      "logits/rejected": -0.646837055683136,
+      "logps/chosen": -327.53558349609375,
+      "logps/rejected": -438.29022216796875,
+      "loss": 0.0315,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": -2.6820549964904785,
+      "rewards/margins": 6.348910331726074,
+      "rewards/rejected": -9.030964851379395,
+      "step": 1410
+    },
+    {
+      "epoch": 2.97,
+      "grad_norm": 10.010864148743238,
+      "learning_rate": 1.4529288188125377e-10,
+      "logits/chosen": -1.1855485439300537,
+      "logits/rejected": -0.6920149326324463,
+      "logps/chosen": -337.4898376464844,
+      "logps/rejected": -440.05987548828125,
+      "loss": 0.0257,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.923854351043701,
+      "rewards/margins": 6.196907997131348,
+      "rewards/rejected": -9.120763778686523,
+      "step": 1420
+    },
+    {
+      "epoch": 2.99,
+      "grad_norm": 37.578079095213596,
+      "learning_rate": 1.1861698640563966e-11,
+      "logits/chosen": -1.1965336799621582,
+      "logits/rejected": -0.6326996684074402,
+      "logps/chosen": -348.0246276855469,
+      "logps/rejected": -445.17431640625,
+      "loss": 0.0318,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.8414111137390137,
+      "rewards/margins": 6.086348533630371,
+      "rewards/rejected": -8.927759170532227,
+      "step": 1430
+    },
+    {
+      "epoch": 3.0,
+      "step": 1434,
       "total_flos": 0.0,
-      "train_loss": 0.14747725651603363,
-      "train_runtime": 7551.087,
-      "train_samples_per_second": 8.096,
-      "train_steps_per_second": 0.063
+      "train_loss": 0.007373018379906397,
+      "train_runtime": 5373.2533,
+      "train_samples_per_second": 34.132,
+      "train_steps_per_second": 0.267
     }
   ],
   "logging_steps": 10,
-  "max_steps": 478,
+  "max_steps": 1434,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 3,
   "save_steps": 100,
   "total_flos": 0.0,
   "train_batch_size": 8,