diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9990186457311089,
+  "epoch": 9.99018645731109,
   "eval_steps": 100,
-  "global_step": 509,
+  "global_step": 5090,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1078,19 +1078,9640 @@
       "step": 500
     },
     {
-      "epoch": 0.9990186457311089,
-      "step": 509,
+      "epoch": 1.000981354268891,
+      "grad_norm": 3.606361815253299,
+      "learning_rate": 4.99999941211936e-07,
+      "logits/chosen": -1.1953125,
+      "logits/rejected": -1.234375,
+      "logps/chosen": -486.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.000675201416015625,
+      "logps/rejected": -450.0,
+      "logps/rejected_bottom_tokens": -14.3125,
+      "logps/rejected_top_tokens": -0.000675201416015625,
+      "loss": 0.6145,
+      "rewards/accuracies": 0.6649999618530273,
+      "rewards/chosen": -0.51953125,
+      "rewards/margins": 0.2138671875,
+      "rewards/rejected": -0.734375,
+      "step": 510
+    },
+    {
+      "epoch": 1.0206084396467126,
+      "grad_norm": 3.5675558410073815,
+      "learning_rate": 4.999928866777183e-07,
+      "logits/chosen": -1.15625,
+      "logits/rejected": -1.203125,
+      "logps/chosen": -440.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.00072479248046875,
+      "logps/rejected": -386.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000720977783203125,
+      "loss": 0.6316,
+      "rewards/accuracies": 0.6800000071525574,
+      "rewards/chosen": -0.546875,
+      "rewards/margins": 0.26953125,
+      "rewards/rejected": -0.8203125,
+      "step": 520
+    },
+    {
+      "epoch": 1.0402355250245339,
+      "grad_norm": 3.7459335473757993,
+      "learning_rate": 4.999740749108743e-07,
+      "logits/chosen": -1.1796875,
+      "logits/rejected": -1.2265625,
+      "logps/chosen": -466.0,
+      "logps/chosen_bottom_tokens": -14.3125,
+      "logps/chosen_top_tokens": -0.000682830810546875,
+      "logps/rejected": -428.0,
+      "logps/rejected_bottom_tokens": -14.25,
+      "logps/rejected_top_tokens": -0.000652313232421875,
+      "loss": 0.6021,
+      "rewards/accuracies": 0.6950000524520874,
+      "rewards/chosen": -0.546875,
+      "rewards/margins": 0.29296875,
+      "rewards/rejected": -0.83984375,
+      "step": 530
+    },
+    {
+      "epoch": 1.0598626104023552,
+      "grad_norm": 3.722276672156144,
+      "learning_rate": 4.999435067961269e-07,
+      "logits/chosen": -1.1640625,
+      "logits/rejected": -1.203125,
+      "logps/chosen": -426.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.00066375732421875,
+      "logps/rejected": -418.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.0006866455078125,
+      "loss": 0.6311,
+      "rewards/accuracies": 0.6299999952316284,
+      "rewards/chosen": -0.578125,
+      "rewards/margins": 0.2275390625,
+      "rewards/rejected": -0.8046875,
+      "step": 540
+    },
+    {
+      "epoch": 1.0794896957801767,
+      "grad_norm": 3.8251776486836486,
+      "learning_rate": 4.999011837711027e-07,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -1.2265625,
+      "logps/chosen": -428.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.00058746337890625,
+      "logps/rejected": -400.0,
+      "logps/rejected_bottom_tokens": -14.1875,
+      "logps/rejected_top_tokens": -0.000579833984375,
+      "loss": 0.6112,
+      "rewards/accuracies": 0.6100000143051147,
+      "rewards/chosen": -0.57421875,
+      "rewards/margins": 0.2353515625,
+      "rewards/rejected": -0.80859375,
+      "step": 550
+    },
+    {
+      "epoch": 1.099116781157998,
+      "grad_norm": 3.7487868364304684,
+      "learning_rate": 4.998471078262648e-07,
+      "logits/chosen": -1.15625,
+      "logits/rejected": -1.171875,
+      "logps/chosen": -430.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000652313232421875,
+      "logps/rejected": -414.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000652313232421875,
+      "loss": 0.6098,
+      "rewards/accuracies": 0.6899999976158142,
+      "rewards/chosen": -0.546875,
+      "rewards/margins": 0.265625,
+      "rewards/rejected": -0.8125,
+      "step": 560
+    },
+    {
+      "epoch": 1.1187438665358194,
+      "grad_norm": 5.204331585494727,
+      "learning_rate": 4.997812815048196e-07,
+      "logits/chosen": -1.25,
+      "logits/rejected": -1.2734375,
+      "logps/chosen": -408.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.00057220458984375,
+      "logps/rejected": -412.0,
+      "logps/rejected_bottom_tokens": -14.1875,
+      "logps/rejected_top_tokens": -0.0005645751953125,
+      "loss": 0.6146,
+      "rewards/accuracies": 0.6899999976158142,
+      "rewards/chosen": -0.515625,
+      "rewards/margins": 0.341796875,
+      "rewards/rejected": -0.859375,
+      "step": 570
+    },
+    {
+      "epoch": 1.138370951913641,
+      "grad_norm": 3.6081766106504736,
+      "learning_rate": 4.997037079025965e-07,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -1.2265625,
+      "logps/chosen": -444.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.00054168701171875,
+      "logps/rejected": -426.0,
+      "logps/rejected_bottom_tokens": -14.1875,
+      "logps/rejected_top_tokens": -0.000553131103515625,
+      "loss": 0.6048,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.59375,
+      "rewards/margins": 0.2734375,
+      "rewards/rejected": -0.8671875,
+      "step": 580
+    },
+    {
+      "epoch": 1.1579980372914622,
+      "grad_norm": 4.870398963575735,
+      "learning_rate": 4.996143906679027e-07,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -1.2421875,
+      "logps/chosen": -432.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.000499725341796875,
+      "logps/rejected": -408.0,
+      "logps/rejected_bottom_tokens": -14.1875,
+      "logps/rejected_top_tokens": -0.00052642822265625,
+      "loss": 0.6143,
+      "rewards/accuracies": 0.6299999952316284,
+      "rewards/chosen": -0.57421875,
+      "rewards/margins": 0.22265625,
+      "rewards/rejected": -0.796875,
+      "step": 590
+    },
+    {
+      "epoch": 1.1776251226692835,
+      "grad_norm": 3.826862126663985,
+      "learning_rate": 4.995133340013521e-07,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -1.25,
+      "logps/chosen": -442.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000606536865234375,
+      "logps/rejected": -424.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000545501708984375,
+      "loss": 0.6115,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.58203125,
+      "rewards/margins": 0.267578125,
+      "rewards/rejected": -0.8515625,
+      "step": 600
+    },
+    {
+      "epoch": 1.1776251226692835,
+      "eval_logits/chosen": -1.2109375,
+      "eval_logits/rejected": -1.2578125,
+      "eval_logps/chosen": -450.0,
+      "eval_logps/chosen_bottom_tokens": -14.25,
+      "eval_logps/chosen_top_tokens": -0.00060272216796875,
+      "eval_logps/rejected": -426.0,
+      "eval_logps/rejected_bottom_tokens": -14.125,
+      "eval_logps/rejected_top_tokens": -0.00058746337890625,
+      "eval_loss": 0.6120752096176147,
+      "eval_rewards/accuracies": 0.6805970072746277,
+      "eval_rewards/chosen": -0.5546875,
+      "eval_rewards/margins": 0.32421875,
+      "eval_rewards/rejected": -0.87890625,
+      "eval_runtime": 107.8869,
+      "eval_samples_per_second": 18.538,
+      "eval_steps_per_second": 0.621,
+      "step": 600
+    },
+    {
+      "epoch": 1.197252208047105,
+      "grad_norm": 3.870372977700712,
+      "learning_rate": 4.994005426556668e-07,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -1.21875,
+      "logps/chosen": -406.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.000598907470703125,
+      "logps/rejected": -412.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.000621795654296875,
+      "loss": 0.6124,
+      "rewards/accuracies": 0.6550000309944153,
+      "rewards/chosen": -0.5390625,
+      "rewards/margins": 0.28125,
+      "rewards/rejected": -0.8203125,
+      "step": 610
+    },
+    {
+      "epoch": 1.2168792934249264,
+      "grad_norm": 4.186566429764424,
+      "learning_rate": 4.99276021935454e-07,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -1.265625,
+      "logps/chosen": -430.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.00051116943359375,
+      "logps/rejected": -400.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.0005035400390625,
+      "loss": 0.6022,
+      "rewards/accuracies": 0.7150000333786011,
+      "rewards/chosen": -0.40625,
+      "rewards/margins": 0.34375,
+      "rewards/rejected": -0.75,
+      "step": 620
+    },
+    {
+      "epoch": 1.2365063788027477,
+      "grad_norm": 3.609372953026834,
+      "learning_rate": 4.991397776969566e-07,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -1.25,
+      "logps/chosen": -448.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000659942626953125,
+      "logps/rejected": -398.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000629425048828125,
+      "loss": 0.6018,
+      "rewards/accuracies": 0.6950000524520874,
+      "rewards/chosen": -0.6015625,
+      "rewards/margins": 0.30078125,
+      "rewards/rejected": -0.90234375,
+      "step": 630
+    },
+    {
+      "epoch": 1.2561334641805693,
+      "grad_norm": 4.171667413733576,
+      "learning_rate": 4.989918163477777e-07,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -1.2734375,
+      "logps/chosen": -424.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.00052642822265625,
+      "logps/rejected": -394.0,
+      "logps/rejected_bottom_tokens": -14.1875,
+      "logps/rejected_top_tokens": -0.00052642822265625,
+      "loss": 0.5944,
+      "rewards/accuracies": 0.6700000166893005,
+      "rewards/chosen": -0.58203125,
+      "rewards/margins": 0.333984375,
+      "rewards/rejected": -0.9140625,
+      "step": 640
+    },
+    {
+      "epoch": 1.2757605495583906,
+      "grad_norm": 4.098077395806722,
+      "learning_rate": 4.988321448465795e-07,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -1.28125,
+      "logps/chosen": -394.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.0005645751953125,
+      "logps/rejected": -414.0,
+      "logps/rejected_bottom_tokens": -14.1875,
+      "logps/rejected_top_tokens": -0.000576019287109375,
+      "loss": 0.6045,
+      "rewards/accuracies": 0.6650000810623169,
+      "rewards/chosen": -0.57421875,
+      "rewards/margins": 0.322265625,
+      "rewards/rejected": -0.8984375,
+      "step": 650
+    },
+    {
+      "epoch": 1.295387634936212,
+      "grad_norm": 3.766274950371864,
+      "learning_rate": 4.986607707027556e-07,
+      "logits/chosen": -1.203125,
+      "logits/rejected": -1.2421875,
+      "logps/chosen": -406.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.000614166259765625,
+      "logps/rejected": -402.0,
+      "logps/rejected_bottom_tokens": -14.1875,
+      "logps/rejected_top_tokens": -0.0006103515625,
+      "loss": 0.5906,
+      "rewards/accuracies": 0.6949999928474426,
+      "rewards/chosen": -0.625,
+      "rewards/margins": 0.3046875,
+      "rewards/rejected": -0.9296875,
+      "step": 660
+    },
+    {
+      "epoch": 1.3150147203140334,
+      "grad_norm": 4.3244931656081755,
+      "learning_rate": 4.984777019760778e-07,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -1.2265625,
+      "logps/chosen": -444.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.00083160400390625,
+      "logps/rejected": -428.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000812530517578125,
+      "loss": 0.5862,
+      "rewards/accuracies": 0.6399999856948853,
+      "rewards/chosen": -0.63671875,
+      "rewards/margins": 0.322265625,
+      "rewards/rejected": -0.9609375,
+      "step": 670
+    },
+    {
+      "epoch": 1.3346418056918548,
+      "grad_norm": 5.114477587957832,
+      "learning_rate": 4.982829472763176e-07,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -1.2734375,
+      "logps/chosen": -446.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.000568389892578125,
+      "logps/rejected": -440.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.000553131103515625,
+      "loss": 0.6053,
+      "rewards/accuracies": 0.6550000309944153,
+      "rewards/chosen": -0.59375,
+      "rewards/margins": 0.376953125,
+      "rewards/rejected": -0.97265625,
+      "step": 680
+    },
+    {
+      "epoch": 1.354268891069676,
+      "grad_norm": 4.909788993759657,
+      "learning_rate": 4.98076515762841e-07,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -1.2734375,
+      "logps/chosen": -452.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000682830810546875,
+      "logps/rejected": -430.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.00063323974609375,
+      "loss": 0.604,
+      "rewards/accuracies": 0.6299999952316284,
+      "rewards/chosen": -0.78125,
+      "rewards/margins": 0.28125,
+      "rewards/rejected": -1.0625,
+      "step": 690
+    },
+    {
+      "epoch": 1.3738959764474976,
+      "grad_norm": 4.433097765275912,
+      "learning_rate": 4.978584171441774e-07,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -1.2734375,
+      "logps/chosen": -448.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.00063323974609375,
+      "logps/rejected": -432.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.0006256103515625,
+      "loss": 0.607,
+      "rewards/accuracies": 0.6399999856948853,
+      "rewards/chosen": -0.81640625,
+      "rewards/margins": 0.298828125,
+      "rewards/rejected": -1.1171875,
+      "step": 700
+    },
+    {
+      "epoch": 1.3738959764474976,
+      "eval_logits/chosen": -1.234375,
+      "eval_logits/rejected": -1.28125,
+      "eval_logps/chosen": -460.0,
+      "eval_logps/chosen_bottom_tokens": -14.1875,
+      "eval_logps/chosen_top_tokens": -0.0006103515625,
+      "eval_logps/rejected": -438.0,
+      "eval_logps/rejected_bottom_tokens": -14.125,
+      "eval_logps/rejected_top_tokens": -0.000579833984375,
+      "eval_loss": 0.6068310737609863,
+      "eval_rewards/accuracies": 0.6985074877738953,
+      "eval_rewards/chosen": -0.6640625,
+      "eval_rewards/margins": 0.341796875,
+      "eval_rewards/rejected": -1.0078125,
+      "eval_runtime": 107.7706,
+      "eval_samples_per_second": 18.558,
+      "eval_steps_per_second": 0.622,
+      "step": 700
+    },
+    {
+      "epoch": 1.393523061825319,
+      "grad_norm": 3.765327199305878,
+      "learning_rate": 4.976286616775634e-07,
+      "logits/chosen": -1.234375,
+      "logits/rejected": -1.265625,
+      "logps/chosen": -418.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.000553131103515625,
+      "logps/rejected": -396.0,
+      "logps/rejected_bottom_tokens": -14.3125,
+      "logps/rejected_top_tokens": -0.00054168701171875,
+      "loss": 0.6011,
+      "rewards/accuracies": 0.6450001001358032,
+      "rewards/chosen": -0.61328125,
+      "rewards/margins": 0.279296875,
+      "rewards/rejected": -0.890625,
+      "step": 710
+    },
+    {
+      "epoch": 1.4131501472031402,
+      "grad_norm": 3.805472086883035,
+      "learning_rate": 4.973872601684603e-07,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -1.28125,
+      "logps/chosen": -480.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.00057220458984375,
+      "logps/rejected": -448.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.0005340576171875,
+      "loss": 0.5884,
+      "rewards/accuracies": 0.7050000429153442,
+      "rewards/chosen": -0.52734375,
+      "rewards/margins": 0.3046875,
+      "rewards/rejected": -0.83203125,
+      "step": 720
+    },
+    {
+      "epoch": 1.4327772325809618,
+      "grad_norm": 4.230218832508928,
+      "learning_rate": 4.971342239700461e-07,
+      "logits/chosen": -1.2109375,
+      "logits/rejected": -1.25,
+      "logps/chosen": -442.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000644683837890625,
+      "logps/rejected": -410.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000640869140625,
+      "loss": 0.5926,
+      "rewards/accuracies": 0.7100000381469727,
+      "rewards/chosen": -0.6328125,
+      "rewards/margins": 0.3828125,
+      "rewards/rejected": -1.015625,
+      "step": 730
+    },
+    {
+      "epoch": 1.452404317958783,
+      "grad_norm": 4.491255520471624,
+      "learning_rate": 4.96869564982681e-07,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -1.28125,
+      "logps/chosen": -414.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.00054931640625,
+      "logps/rejected": -434.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000522613525390625,
+      "loss": 0.6027,
+      "rewards/accuracies": 0.6750000715255737,
+      "rewards/chosen": -0.71484375,
+      "rewards/margins": 0.404296875,
+      "rewards/rejected": -1.1171875,
+      "step": 740
+    },
+    {
+      "epoch": 1.4720314033366044,
+      "grad_norm": 4.442897832667683,
+      "learning_rate": 4.965932956533486e-07,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -1.28125,
+      "logps/chosen": -464.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000591278076171875,
+      "logps/rejected": -450.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.00055694580078125,
+      "loss": 0.5909,
+      "rewards/accuracies": 0.6950000524520874,
+      "rewards/chosen": -0.7734375,
+      "rewards/margins": 0.40625,
+      "rewards/rejected": -1.1796875,
+      "step": 750
+    },
+    {
+      "epoch": 1.491658488714426,
+      "grad_norm": 4.749468874704089,
+      "learning_rate": 4.963054289750692e-07,
+      "logits/chosen": -1.1875,
+      "logits/rejected": -1.2265625,
+      "logps/chosen": -454.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000568389892578125,
+      "logps/rejected": -454.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.00052642822265625,
+      "loss": 0.5867,
+      "rewards/accuracies": 0.6949999928474426,
+      "rewards/chosen": -0.79296875,
+      "rewards/margins": 0.359375,
+      "rewards/rejected": -1.15625,
+      "step": 760
+    },
+    {
+      "epoch": 1.5112855740922473,
+      "grad_norm": 5.388562265861462,
+      "learning_rate": 4.960059784862905e-07,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -1.28125,
+      "logps/chosen": -500.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000553131103515625,
+      "logps/rejected": -446.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.00054931640625,
+      "loss": 0.5894,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.71875,
+      "rewards/margins": 0.32421875,
+      "rewards/rejected": -1.046875,
+      "step": 770
+    },
+    {
+      "epoch": 1.5309126594700686,
+      "grad_norm": 5.21775194242665,
+      "learning_rate": 4.956949582702491e-07,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -1.296875,
+      "logps/chosen": -448.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.00067138671875,
+      "logps/rejected": -418.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000621795654296875,
+      "loss": 0.5858,
+      "rewards/accuracies": 0.7399999499320984,
+      "rewards/chosen": -0.69140625,
+      "rewards/margins": 0.419921875,
+      "rewards/rejected": -1.109375,
+      "step": 780
+    },
+    {
+      "epoch": 1.5505397448478901,
+      "grad_norm": 4.798321151943007,
+      "learning_rate": 4.953723829543095e-07,
+      "logits/chosen": -1.25,
+      "logits/rejected": -1.28125,
+      "logps/chosen": -460.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.00060272216796875,
+      "logps/rejected": -458.0,
+      "logps/rejected_bottom_tokens": -14.1875,
+      "logps/rejected_top_tokens": -0.000591278076171875,
+      "loss": 0.6019,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.75,
+      "rewards/margins": 0.29296875,
+      "rewards/rejected": -1.0390625,
+      "step": 790
+    },
+    {
+      "epoch": 1.5701668302257115,
+      "grad_norm": 4.565201285919172,
+      "learning_rate": 4.950382677092754e-07,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -1.296875,
+      "logps/chosen": -470.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.000637054443359375,
+      "logps/rejected": -468.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.000576019287109375,
+      "loss": 0.5764,
+      "rewards/accuracies": 0.6399999856948853,
+      "rewards/chosen": -0.76171875,
+      "rewards/margins": 0.310546875,
+      "rewards/rejected": -1.0703125,
+      "step": 800
+    },
+    {
+      "epoch": 1.5701668302257115,
+      "eval_logits/chosen": -1.265625,
+      "eval_logits/rejected": -1.3125,
+      "eval_logps/chosen": -468.0,
+      "eval_logps/chosen_bottom_tokens": -14.25,
+      "eval_logps/chosen_top_tokens": -0.00067901611328125,
+      "eval_logps/rejected": -452.0,
+      "eval_logps/rejected_bottom_tokens": -14.125,
+      "eval_logps/rejected_top_tokens": -0.00067138671875,
+      "eval_loss": 0.5995947122573853,
+      "eval_rewards/accuracies": 0.6865671873092651,
+      "eval_rewards/chosen": -0.75,
+      "eval_rewards/margins": 0.388671875,
+      "eval_rewards/rejected": -1.140625,
+      "eval_runtime": 107.7895,
+      "eval_samples_per_second": 18.555,
+      "eval_steps_per_second": 0.622,
+      "step": 800
+    },
+    {
+      "epoch": 1.5897939156035328,
+      "grad_norm": 4.401209533392183,
+      "learning_rate": 4.946926282486765e-07,
+      "logits/chosen": -1.25,
+      "logits/rejected": -1.28125,
+      "logps/chosen": -468.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.00064849853515625,
+      "logps/rejected": -458.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.00061798095703125,
+      "loss": 0.5877,
+      "rewards/accuracies": 0.6799999475479126,
+      "rewards/chosen": -0.75,
+      "rewards/margins": 0.39453125,
+      "rewards/rejected": -1.140625,
+      "step": 810
+    },
+    {
+      "epoch": 1.6094210009813543,
+      "grad_norm": 4.172999078191103,
+      "learning_rate": 4.943354808280297e-07,
+      "logits/chosen": -1.2265625,
+      "logits/rejected": -1.2421875,
+      "logps/chosen": -424.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000598907470703125,
+      "logps/rejected": -428.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.000637054443359375,
+      "loss": 0.6005,
+      "rewards/accuracies": 0.6600000262260437,
+      "rewards/chosen": -0.671875,
+      "rewards/margins": 0.369140625,
+      "rewards/rejected": -1.0390625,
+      "step": 820
+    },
+    {
+      "epoch": 1.6290480863591756,
+      "grad_norm": 4.682327663420403,
+      "learning_rate": 4.93966842244074e-07,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -1.2890625,
+      "logps/chosen": -458.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.00058746337890625,
+      "logps/rejected": -416.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.00054168701171875,
+      "loss": 0.5873,
+      "rewards/accuracies": 0.7150000333786011,
+      "rewards/chosen": -0.65625,
+      "rewards/margins": 0.3515625,
+      "rewards/rejected": -1.0078125,
+      "step": 830
+    },
+    {
+      "epoch": 1.648675171736997,
+      "grad_norm": 3.971337026667362,
+      "learning_rate": 4.93586729833981e-07,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -1.3125,
+      "logps/chosen": -458.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000530242919921875,
+      "logps/rejected": -426.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000518798828125,
+      "loss": 0.5706,
+      "rewards/accuracies": 0.7149999737739563,
+      "rewards/chosen": -0.65234375,
+      "rewards/margins": 0.443359375,
+      "rewards/rejected": -1.09375,
+      "step": 840
+    },
+    {
+      "epoch": 1.6683022571148185,
+      "grad_norm": 4.9613043407153885,
+      "learning_rate": 4.931951614745395e-07,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -1.296875,
+      "logps/chosen": -474.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.000499725341796875,
+      "logps/rejected": -462.0,
+      "logps/rejected_bottom_tokens": -14.1875,
+      "logps/rejected_top_tokens": -0.000492095947265625,
+      "loss": 0.5698,
+      "rewards/accuracies": 0.6800000071525574,
+      "rewards/chosen": -0.640625,
+      "rewards/margins": 0.4140625,
+      "rewards/rejected": -1.0546875,
+      "step": 850
+    },
+    {
+      "epoch": 1.6879293424926398,
+      "grad_norm": 4.37364089862016,
+      "learning_rate": 4.927921555813147e-07,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -1.3203125,
+      "logps/chosen": -452.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.0005950927734375,
+      "logps/rejected": -430.0,
+      "logps/rejected_bottom_tokens": -14.1875,
+      "logps/rejected_top_tokens": -0.000568389892578125,
+      "loss": 0.5829,
+      "rewards/accuracies": 0.7049999237060547,
+      "rewards/chosen": -0.6015625,
+      "rewards/margins": 0.369140625,
+      "rewards/rejected": -0.96875,
+      "step": 860
+    },
+    {
+      "epoch": 1.7075564278704611,
+      "grad_norm": 4.934313256041272,
+      "learning_rate": 4.923777311077819e-07,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -1.296875,
+      "logps/chosen": -450.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.000640869140625,
+      "logps/rejected": -448.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.0005950927734375,
+      "loss": 0.5745,
+      "rewards/accuracies": 0.6750000715255737,
+      "rewards/chosen": -0.66796875,
+      "rewards/margins": 0.41015625,
+      "rewards/rejected": -1.078125,
+      "step": 870
+    },
+    {
+      "epoch": 1.7271835132482827,
+      "grad_norm": 4.398593383317482,
+      "learning_rate": 4.919519075444358e-07,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -1.328125,
+      "logps/chosen": -458.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.000644683837890625,
+      "logps/rejected": -432.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.00061798095703125,
+      "loss": 0.5903,
+      "rewards/accuracies": 0.7149999737739563,
+      "rewards/chosen": -0.9296875,
+      "rewards/margins": 0.427734375,
+      "rewards/rejected": -1.359375,
+      "step": 880
+    },
+    {
+      "epoch": 1.746810598626104,
+      "grad_norm": 4.552566240750762,
+      "learning_rate": 4.915147049178725e-07,
+      "logits/chosen": -1.2421875,
+      "logits/rejected": -1.3046875,
+      "logps/chosen": -468.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.000583648681640625,
+      "logps/rejected": -448.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.000583648681640625,
+      "loss": 0.5833,
+      "rewards/accuracies": 0.7500000596046448,
+      "rewards/chosen": -0.6875,
+      "rewards/margins": 0.46484375,
+      "rewards/rejected": -1.15625,
+      "step": 890
+    },
+    {
+      "epoch": 1.7664376840039253,
+      "grad_norm": 3.980827367632633,
+      "learning_rate": 4.910661437898493e-07,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -1.3046875,
+      "logps/chosen": -464.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.0005340576171875,
+      "logps/rejected": -472.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.00055694580078125,
+      "loss": 0.5903,
+      "rewards/accuracies": 0.6600000262260437,
+      "rewards/chosen": -0.67578125,
+      "rewards/margins": 0.375,
+      "rewards/rejected": -1.046875,
+      "step": 900
+    },
+    {
+      "epoch": 1.7664376840039253,
+      "eval_logits/chosen": -1.265625,
+      "eval_logits/rejected": -1.3125,
+      "eval_logps/chosen": -452.0,
+      "eval_logps/chosen_bottom_tokens": -14.25,
+      "eval_logps/chosen_top_tokens": -0.0005950927734375,
+      "eval_logps/rejected": -434.0,
+      "eval_logps/rejected_bottom_tokens": -14.125,
+      "eval_logps/rejected_top_tokens": -0.00057220458984375,
+      "eval_loss": 0.5984472632408142,
+      "eval_rewards/accuracies": 0.7044776082038879,
+      "eval_rewards/chosen": -0.58984375,
+      "eval_rewards/margins": 0.376953125,
+      "eval_rewards/rejected": -0.96484375,
+      "eval_runtime": 107.7395,
+      "eval_samples_per_second": 18.563,
+      "eval_steps_per_second": 0.622,
+      "step": 900
+    },
+    {
+      "epoch": 1.7860647693817469,
+      "grad_norm": 4.250190572849706,
+      "learning_rate": 4.906062452563164e-07,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -1.28125,
+      "logps/chosen": -424.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.00064849853515625,
+      "logps/rejected": -430.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000614166259765625,
+      "loss": 0.5943,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.67578125,
+      "rewards/margins": 0.26171875,
+      "rewards/rejected": -0.9375,
+      "step": 910
+    },
+    {
+      "epoch": 1.8056918547595682,
+      "grad_norm": 4.240685239707214,
+      "learning_rate": 4.901350309464256e-07,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -1.296875,
+      "logps/chosen": -458.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.000522613525390625,
+      "logps/rejected": -444.0,
+      "logps/rejected_bottom_tokens": -14.25,
+      "logps/rejected_top_tokens": -0.00052642822265625,
+      "loss": 0.5827,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -0.64453125,
+      "rewards/margins": 0.380859375,
+      "rewards/rejected": -1.0234375,
+      "step": 920
+    },
+    {
+      "epoch": 1.8253189401373895,
+      "grad_norm": 4.024817983900379,
+      "learning_rate": 4.896525230215123e-07,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -1.3046875,
+      "logps/chosen": -450.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000507354736328125,
+      "logps/rejected": -450.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000530242919921875,
+      "loss": 0.5686,
+      "rewards/accuracies": 0.7150000333786011,
+      "rewards/chosen": -0.640625,
+      "rewards/margins": 0.490234375,
+      "rewards/rejected": -1.1328125,
+      "step": 930
+    },
+    {
+      "epoch": 1.844946025515211,
+      "grad_norm": 4.38980118561382,
+      "learning_rate": 4.891587441740539e-07,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -1.328125,
+      "logps/chosen": -478.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.000518798828125,
+      "logps/rejected": -460.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.0005035400390625,
+      "loss": 0.5839,
+      "rewards/accuracies": 0.6450000405311584,
+      "rewards/chosen": -0.76953125,
+      "rewards/margins": 0.353515625,
+      "rewards/rejected": -1.125,
+      "step": 940
+    },
+    {
+      "epoch": 1.8645731108930323,
+      "grad_norm": 4.759599103438232,
+      "learning_rate": 4.886537176266024e-07,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -1.3203125,
+      "logps/chosen": -438.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.000629425048828125,
+      "logps/rejected": -414.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000621795654296875,
+      "loss": 0.5779,
+      "rewards/accuracies": 0.7150000333786011,
+      "rewards/chosen": -0.78515625,
+      "rewards/margins": 0.373046875,
+      "rewards/rejected": -1.15625,
+      "step": 950
+    },
+    {
+      "epoch": 1.8842001962708537,
+      "grad_norm": 4.786256988626395,
+      "learning_rate": 4.881374671306917e-07,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -1.3125,
+      "logps/chosen": -486.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.000576019287109375,
+      "logps/rejected": -480.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.00054931640625,
+      "loss": 0.5842,
+      "rewards/accuracies": 0.6799999475479126,
+      "rewards/chosen": -0.79296875,
+      "rewards/margins": 0.4375,
+      "rewards/rejected": -1.234375,
+      "step": 960
+    },
+    {
+      "epoch": 1.9038272816486752,
+      "grad_norm": 4.9859381733928965,
+      "learning_rate": 4.876100169657217e-07,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -1.3125,
+      "logps/chosen": -444.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000591278076171875,
+      "logps/rejected": -430.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000583648681640625,
+      "loss": 0.5824,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.76953125,
+      "rewards/margins": 0.455078125,
+      "rewards/rejected": -1.2265625,
+      "step": 970
+    },
+    {
+      "epoch": 1.9234543670264965,
+      "grad_norm": 5.322106257872434,
+      "learning_rate": 4.87071391937815e-07,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -1.3046875,
+      "logps/chosen": -420.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00075531005859375,
+      "logps/rejected": -418.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000732421875,
+      "loss": 0.5726,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.81640625,
+      "rewards/margins": 0.43359375,
+      "rewards/rejected": -1.25,
+      "step": 980
+    },
+    {
+      "epoch": 1.9430814524043178,
+      "grad_norm": 5.610031785982417,
+      "learning_rate": 4.865216173786516e-07,
+      "logits/chosen": -1.21875,
+      "logits/rejected": -1.2734375,
+      "logps/chosen": -464.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.000698089599609375,
+      "logps/rejected": -444.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.00067901611328125,
+      "loss": 0.5806,
+      "rewards/accuracies": 0.6800000071525574,
+      "rewards/chosen": -0.9296875,
+      "rewards/margins": 0.33984375,
+      "rewards/rejected": -1.2734375,
+      "step": 990
+    },
+    {
+      "epoch": 1.9627085377821394,
+      "grad_norm": 4.756477386020176,
+      "learning_rate": 4.859607191442768e-07,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -1.3203125,
+      "logps/chosen": -466.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.0006561279296875,
+      "logps/rejected": -444.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000667572021484375,
+      "loss": 0.5697,
+      "rewards/accuracies": 0.6850000023841858,
+      "rewards/chosen": -0.796875,
+      "rewards/margins": 0.42578125,
+      "rewards/rejected": -1.2265625,
+      "step": 1000
+    },
+    {
+      "epoch": 1.9627085377821394,
+      "eval_logits/chosen": -1.2734375,
+      "eval_logits/rejected": -1.3125,
+      "eval_logps/chosen": -468.0,
+      "eval_logps/chosen_bottom_tokens": -14.0625,
+      "eval_logps/chosen_top_tokens": -0.000667572021484375,
+      "eval_logps/rejected": -454.0,
+      "eval_logps/rejected_bottom_tokens": -14.0,
+      "eval_logps/rejected_top_tokens": -0.00061798095703125,
+      "eval_loss": 0.5921972393989563,
+      "eval_rewards/accuracies": 0.6865671873092651,
+      "eval_rewards/chosen": -0.73828125,
+      "eval_rewards/margins": 0.416015625,
+      "eval_rewards/rejected": -1.15625,
+      "eval_runtime": 107.7883,
+      "eval_samples_per_second": 18.555,
+      "eval_steps_per_second": 0.622,
+      "step": 1000
+    },
+    {
+      "epoch": 1.9823356231599607,
+      "grad_norm": 4.9024153356053946,
+      "learning_rate": 4.85388723613885e-07,
+      "logits/chosen": -1.25,
+      "logits/rejected": -1.3125,
+      "logps/chosen": -482.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000644683837890625,
+      "logps/rejected": -458.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000598907470703125,
+      "loss": 0.5777,
+      "rewards/accuracies": 0.7100000381469727,
+      "rewards/chosen": -0.8046875,
+      "rewards/margins": 0.423828125,
+      "rewards/rejected": -1.2265625,
+      "step": 1010
+    },
+    {
+      "epoch": 2.001962708537782,
+      "grad_norm": 4.429395780890561,
+      "learning_rate": 4.848056576885799e-07,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -1.3125,
+      "logps/chosen": -492.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.000560760498046875,
+      "logps/rejected": -498.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.0005340576171875,
+      "loss": 0.5638,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.828125,
+      "rewards/margins": 0.48046875,
+      "rewards/rejected": -1.3046875,
+      "step": 1020
+    },
+    {
+      "epoch": 2.0215897939156036,
+      "grad_norm": 4.728557301310735,
+      "learning_rate": 4.842115487901085e-07,
+      "logits/chosen": -1.25,
+      "logits/rejected": -1.2890625,
+      "logps/chosen": -456.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000736236572265625,
+      "logps/rejected": -462.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000690460205078125,
+      "loss": 0.5488,
+      "rewards/accuracies": 0.7199999690055847,
+      "rewards/chosen": -0.84375,
+      "rewards/margins": 0.46875,
+      "rewards/rejected": -1.3125,
+      "step": 1030
+    },
+    {
+      "epoch": 2.041216879293425,
+      "grad_norm": 4.919488885103522,
+      "learning_rate": 4.836064248595719e-07,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -1.3125,
+      "logps/chosen": -456.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.00061798095703125,
+      "logps/rejected": -454.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000598907470703125,
+      "loss": 0.5544,
+      "rewards/accuracies": 0.7199999690055847,
+      "rewards/chosen": -0.68359375,
+      "rewards/margins": 0.421875,
+      "rewards/rejected": -1.109375,
+      "step": 1040
+    },
+    {
+      "epoch": 2.060843964671246,
+      "grad_norm": 4.398420507697798,
+      "learning_rate": 4.829903143561113e-07,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -1.28125,
+      "logps/chosen": -476.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000701904296875,
+      "logps/rejected": -484.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000701904296875,
+      "loss": 0.5509,
+      "rewards/accuracies": 0.7400000691413879,
+      "rewards/chosen": -0.74609375,
+      "rewards/margins": 0.546875,
+      "rewards/rejected": -1.2890625,
+      "step": 1050
+    },
+    {
+      "epoch": 2.0804710500490677,
+      "grad_norm": 5.763513449763713,
+      "learning_rate": 4.82363246255569e-07,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -1.296875,
+      "logps/chosen": -466.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000644683837890625,
+      "logps/rejected": -474.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000690460205078125,
+      "loss": 0.5693,
+      "rewards/accuracies": 0.7000000476837158,
+      "rewards/chosen": -0.90234375,
+      "rewards/margins": 0.453125,
+      "rewards/rejected": -1.3515625,
+      "step": 1060
+    },
+    {
+      "epoch": 2.1000981354268893,
+      "grad_norm": 4.588276955004494,
+      "learning_rate": 4.817252500491263e-07,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -1.359375,
+      "logps/chosen": -438.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000576019287109375,
+      "logps/rejected": -428.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.00061798095703125,
+      "loss": 0.5475,
+      "rewards/accuracies": 0.7499999403953552,
+      "rewards/chosen": -0.703125,
+      "rewards/margins": 0.52734375,
+      "rewards/rejected": -1.2265625,
+      "step": 1070
+    },
+    {
+      "epoch": 2.1197252208047104,
+      "grad_norm": 4.954842485688175,
+      "learning_rate": 4.810763557419163e-07,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -1.2890625,
+      "logps/chosen": -504.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.000583648681640625,
+      "logps/rejected": -492.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.000576019287109375,
+      "loss": 0.565,
+      "rewards/accuracies": 0.6699999570846558,
+      "rewards/chosen": -0.828125,
+      "rewards/margins": 0.4453125,
+      "rewards/rejected": -1.2734375,
+      "step": 1080
+    },
+    {
+      "epoch": 2.139352306182532,
+      "grad_norm": 4.719628678027872,
+      "learning_rate": 4.804165938516125e-07,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -1.328125,
+      "logps/chosen": -430.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000629425048828125,
+      "logps/rejected": -436.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000606536865234375,
+      "loss": 0.5601,
+      "rewards/accuracies": 0.6599999666213989,
+      "rewards/chosen": -0.83203125,
+      "rewards/margins": 0.361328125,
+      "rewards/rejected": -1.1953125,
+      "step": 1090
+    },
+    {
+      "epoch": 2.1589793915603535,
+      "grad_norm": 5.455818727699527,
+      "learning_rate": 4.797459954069938e-07,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -1.3046875,
+      "logps/chosen": -436.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00066375732421875,
+      "logps/rejected": -442.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000644683837890625,
+      "loss": 0.5573,
+      "rewards/accuracies": 0.7099999785423279,
+      "rewards/chosen": -0.7734375,
+      "rewards/margins": 0.412109375,
+      "rewards/rejected": -1.1875,
+      "step": 1100
+    },
+    {
+      "epoch": 2.1589793915603535,
+      "eval_logits/chosen": -1.2890625,
+      "eval_logits/rejected": -1.328125,
+      "eval_logps/chosen": -476.0,
+      "eval_logps/chosen_bottom_tokens": -14.125,
+      "eval_logps/chosen_top_tokens": -0.000598907470703125,
+      "eval_logps/rejected": -466.0,
+      "eval_logps/rejected_bottom_tokens": -14.0,
+      "eval_logps/rejected_top_tokens": -0.000560760498046875,
+      "eval_loss": 0.5853906273841858,
+      "eval_rewards/accuracies": 0.6985074877738953,
+      "eval_rewards/chosen": -0.8203125,
+      "eval_rewards/margins": 0.45703125,
+      "eval_rewards/rejected": -1.28125,
+      "eval_runtime": 107.7163,
+      "eval_samples_per_second": 18.567,
+      "eval_steps_per_second": 0.622,
+      "step": 1100
+    },
+    {
+      "epoch": 2.1786064769381746,
+      "grad_norm": 4.965235050814131,
+      "learning_rate": 4.790645919464854e-07,
+      "logits/chosen": -1.265625,
+      "logits/rejected": -1.3203125,
+      "logps/chosen": -458.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.000621795654296875,
+      "logps/rejected": -430.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.0005950927734375,
+      "loss": 0.5628,
+      "rewards/accuracies": 0.7350000143051147,
+      "rewards/chosen": -0.87890625,
+      "rewards/margins": 0.435546875,
+      "rewards/rejected": -1.3125,
+      "step": 1110
+    },
+    {
+      "epoch": 2.198233562315996,
+      "grad_norm": 5.352102174842767,
+      "learning_rate": 4.783724155166751e-07,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -1.3203125,
+      "logps/chosen": -438.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000652313232421875,
+      "logps/rejected": -436.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.0006103515625,
+      "loss": 0.5384,
+      "rewards/accuracies": 0.6850000619888306,
+      "rewards/chosen": -0.86328125,
+      "rewards/margins": 0.4296875,
+      "rewards/rejected": -1.296875,
+      "step": 1120
+    },
+    {
+      "epoch": 2.2178606476938176,
+      "grad_norm": 5.559211111088488,
+      "learning_rate": 4.776694986708061e-07,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -1.3046875,
+      "logps/chosen": -468.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000759124755859375,
+      "logps/rejected": -450.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000751495361328125,
+      "loss": 0.5466,
+      "rewards/accuracies": 0.7249999642372131,
+      "rewards/chosen": -0.890625,
+      "rewards/margins": 0.458984375,
+      "rewards/rejected": -1.3515625,
+      "step": 1130
+    },
+    {
+      "epoch": 2.2374877330716387,
+      "grad_norm": 5.275098036354786,
+      "learning_rate": 4.76955874467247e-07,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -1.3515625,
+      "logps/chosen": -468.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.0007171630859375,
+      "logps/rejected": -490.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.0006866455078125,
+      "loss": 0.5552,
+      "rewards/accuracies": 0.6949999928474426,
+      "rewards/chosen": -0.78515625,
+      "rewards/margins": 0.5,
+      "rewards/rejected": -1.2890625,
+      "step": 1140
+    },
+    {
+      "epoch": 2.2571148184494603,
+      "grad_norm": 4.846395975422152,
+      "learning_rate": 4.762315764679353e-07,
+      "logits/chosen": -1.2890625,
+      "logits/rejected": -1.3515625,
+      "logps/chosen": -500.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.00064849853515625,
+      "logps/rejected": -472.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.0006561279296875,
+      "loss": 0.5436,
+      "rewards/accuracies": 0.7350000143051147,
+      "rewards/chosen": -0.71875,
+      "rewards/margins": 0.5234375,
+      "rewards/rejected": -1.2421875,
+      "step": 1150
+    },
+    {
+      "epoch": 2.276741903827282,
+      "grad_norm": 5.985956261716143,
+      "learning_rate": 4.7549663873680074e-07,
+      "logits/chosen": -1.2734375,
+      "logits/rejected": -1.2890625,
+      "logps/chosen": -462.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000762939453125,
+      "logps/rejected": -476.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000804901123046875,
+      "loss": 0.551,
+      "rewards/accuracies": 0.7150000333786011,
+      "rewards/chosen": -0.8046875,
+      "rewards/margins": 0.458984375,
+      "rewards/rejected": -1.265625,
+      "step": 1160
+    },
+    {
+      "epoch": 2.296368989205103,
+      "grad_norm": 6.348450013593529,
+      "learning_rate": 4.7475109583816233e-07,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.375,
+      "logps/chosen": -466.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000690460205078125,
+      "logps/rejected": -468.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000732421875,
+      "loss": 0.55,
+      "rewards/accuracies": 0.7300000190734863,
+      "rewards/chosen": -0.9609375,
+      "rewards/margins": 0.55859375,
+      "rewards/rejected": -1.5234375,
+      "step": 1170
+    },
+    {
+      "epoch": 2.3159960745829244,
+      "grad_norm": 4.561023721100259,
+      "learning_rate": 4.739949828351028e-07,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -1.328125,
+      "logps/chosen": -486.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000682830810546875,
+      "logps/rejected": -476.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000667572021484375,
+      "loss": 0.5583,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.9453125,
+      "rewards/margins": 0.494140625,
+      "rewards/rejected": -1.4375,
+      "step": 1180
+    },
+    {
+      "epoch": 2.335623159960746,
+      "grad_norm": 5.506208079321664,
+      "learning_rate": 4.732283352878199e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -1.3203125,
+      "logps/chosen": -474.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000537872314453125,
+      "logps/rejected": -532.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.00057220458984375,
+      "loss": 0.5368,
+      "rewards/accuracies": 0.7350000143051147,
+      "rewards/chosen": -0.8984375,
+      "rewards/margins": 0.62109375,
+      "rewards/rejected": -1.5234375,
+      "step": 1190
+    },
+    {
+      "epoch": 2.355250245338567,
+      "grad_norm": 5.428963571935742,
+      "learning_rate": 4.724511892519537e-07,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -1.3203125,
+      "logps/chosen": -490.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00067901611328125,
+      "logps/rejected": -484.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000621795654296875,
+      "loss": 0.5439,
+      "rewards/accuracies": 0.7350000143051147,
+      "rewards/chosen": -1.0390625,
+      "rewards/margins": 0.51171875,
+      "rewards/rejected": -1.5546875,
+      "step": 1200
+    },
+    {
+      "epoch": 2.355250245338567,
+      "eval_logits/chosen": -1.328125,
+      "eval_logits/rejected": -1.3671875,
+      "eval_logps/chosen": -504.0,
+      "eval_logps/chosen_bottom_tokens": -14.0625,
+      "eval_logps/chosen_top_tokens": -0.000652313232421875,
+      "eval_logps/rejected": -498.0,
+      "eval_logps/rejected_bottom_tokens": -13.9375,
+      "eval_logps/rejected_top_tokens": -0.00061798095703125,
+      "eval_loss": 0.584460437297821,
+      "eval_rewards/accuracies": 0.6865671873092651,
+      "eval_rewards/chosen": -1.1015625,
+      "eval_rewards/margins": 0.5078125,
+      "eval_rewards/rejected": -1.6171875,
+      "eval_runtime": 107.7958,
+      "eval_samples_per_second": 18.554,
+      "eval_steps_per_second": 0.622,
+      "step": 1200
+    },
+    {
+      "epoch": 2.3748773307163886,
+      "grad_norm": 5.209243987769614,
+      "learning_rate": 4.7166358127689104e-07,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -1.359375,
+      "logps/chosen": -506.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00058746337890625,
+      "logps/rejected": -510.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000614166259765625,
+      "loss": 0.5627,
+      "rewards/accuracies": 0.7600000500679016,
+      "rewards/chosen": -0.9921875,
+      "rewards/margins": 0.5390625,
+      "rewards/rejected": -1.53125,
+      "step": 1210
+    },
+    {
+      "epoch": 2.39450441609421,
+      "grad_norm": 4.794063744707617,
+      "learning_rate": 4.708655484040467e-07,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -1.34375,
+      "logps/chosen": -452.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00058746337890625,
+      "logps/rejected": -472.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.00061798095703125,
+      "loss": 0.5477,
+      "rewards/accuracies": 0.7150000333786011,
+      "rewards/chosen": -0.87890625,
+      "rewards/margins": 0.5390625,
+      "rewards/rejected": -1.421875,
+      "step": 1220
+    },
+    {
+      "epoch": 2.4141315014720313,
+      "grad_norm": 5.531608721469285,
+      "learning_rate": 4.7005712816512095e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -1.3515625,
+      "logps/chosen": -438.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000591278076171875,
+      "logps/rejected": -436.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.0005645751953125,
+      "loss": 0.5745,
+      "rewards/accuracies": 0.7199999690055847,
+      "rewards/chosen": -0.78515625,
+      "rewards/margins": 0.51171875,
+      "rewards/rejected": -1.296875,
+      "step": 1230
+    },
+    {
+      "epoch": 2.433758586849853,
+      "grad_norm": 5.267301228863608,
+      "learning_rate": 4.692383585803349e-07,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -1.34375,
+      "logps/chosen": -484.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000598907470703125,
+      "logps/rejected": -500.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.00061798095703125,
+      "loss": 0.5397,
+      "rewards/accuracies": 0.7150000333786011,
+      "rewards/chosen": -0.78125,
+      "rewards/margins": 0.52734375,
+      "rewards/rejected": -1.3046875,
+      "step": 1240
+    },
+    {
+      "epoch": 2.4533856722276743,
+      "grad_norm": 5.049573801663012,
+      "learning_rate": 4.6840927815664215e-07,
+      "logits/chosen": -1.2578125,
+      "logits/rejected": -1.3359375,
+      "logps/chosen": -552.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000736236572265625,
+      "logps/rejected": -510.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000701904296875,
+      "loss": 0.5352,
+      "rewards/accuracies": 0.7350000143051147,
+      "rewards/chosen": -1.0078125,
+      "rewards/margins": 0.52734375,
+      "rewards/rejected": -1.53125,
+      "step": 1250
+    },
+    {
+      "epoch": 2.4730127576054954,
+      "grad_norm": 4.988013454206055,
+      "learning_rate": 4.675699258859177e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -1.3515625,
+      "logps/chosen": -474.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000652313232421875,
+      "logps/rejected": -488.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000701904296875,
+      "loss": 0.5604,
+      "rewards/accuracies": 0.690000057220459,
+      "rewards/chosen": -1.046875,
+      "rewards/margins": 0.466796875,
+      "rewards/rejected": -1.515625,
+      "step": 1260
+    },
+    {
+      "epoch": 2.492639842983317,
+      "grad_norm": 4.599188572679597,
+      "learning_rate": 4.667203412431244e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -1.3671875,
+      "logps/chosen": -466.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.0006103515625,
+      "logps/rejected": -450.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000621795654296875,
+      "loss": 0.5489,
+      "rewards/accuracies": 0.7150000333786011,
+      "rewards/chosen": -0.8203125,
+      "rewards/margins": 0.490234375,
+      "rewards/rejected": -1.3125,
+      "step": 1270
+    },
+    {
+      "epoch": 2.5122669283611385,
+      "grad_norm": 4.9531276475109065,
+      "learning_rate": 4.658605641844563e-07,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -1.3671875,
+      "logps/chosen": -480.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.00063323974609375,
+      "logps/rejected": -466.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.0006561279296875,
+      "loss": 0.5545,
+      "rewards/accuracies": 0.6850000619888306,
+      "rewards/chosen": -1.0234375,
+      "rewards/margins": 0.404296875,
+      "rewards/rejected": -1.4296875,
+      "step": 1280
+    },
+    {
+      "epoch": 2.5318940137389596,
+      "grad_norm": 6.3499112776801345,
+      "learning_rate": 4.6499063514545934e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000701904296875,
+      "logps/rejected": -468.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000652313232421875,
+      "loss": 0.559,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.94140625,
+      "rewards/margins": 0.470703125,
+      "rewards/rejected": -1.4140625,
+      "step": 1290
+    },
+    {
+      "epoch": 2.551521099116781,
+      "grad_norm": 5.554588430830751,
+      "learning_rate": 4.6411059503913e-07,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -1.3125,
+      "logps/chosen": -436.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.0006256103515625,
+      "logps/rejected": -456.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000644683837890625,
+      "loss": 0.5487,
+      "rewards/accuracies": 0.6800000667572021,
+      "rewards/chosen": -0.9453125,
+      "rewards/margins": 0.474609375,
+      "rewards/rejected": -1.421875,
+      "step": 1300
+    },
+    {
+      "epoch": 2.551521099116781,
+      "eval_logits/chosen": -1.34375,
+      "eval_logits/rejected": -1.3828125,
+      "eval_logps/chosen": -482.0,
+      "eval_logps/chosen_bottom_tokens": -14.0625,
+      "eval_logps/chosen_top_tokens": -0.0006561279296875,
+      "eval_logps/rejected": -476.0,
+      "eval_logps/rejected_bottom_tokens": -14.0,
+      "eval_logps/rejected_top_tokens": -0.0006103515625,
+      "eval_loss": 0.5800756812095642,
+      "eval_rewards/accuracies": 0.6925373077392578,
+      "eval_rewards/chosen": -0.890625,
+      "eval_rewards/margins": 0.498046875,
+      "eval_rewards/rejected": -1.3828125,
+      "eval_runtime": 107.6532,
+      "eval_samples_per_second": 18.578,
+      "eval_steps_per_second": 0.622,
+      "step": 1300
+    },
+    {
+      "epoch": 2.5711481844946027,
+      "grad_norm": 4.998797445392259,
+      "learning_rate": 4.6322048525399093e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -1.359375,
+      "logps/chosen": -446.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000713348388671875,
+      "logps/rejected": -470.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000736236572265625,
+      "loss": 0.5418,
+      "rewards/accuracies": 0.7200000286102295,
+      "rewards/chosen": -0.84375,
+      "rewards/margins": 0.5703125,
+      "rewards/rejected": -1.4140625,
+      "step": 1310
+    },
+    {
+      "epoch": 2.590775269872424,
+      "grad_norm": 6.338454672487353,
+      "learning_rate": 4.6232034765214444e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -1.3359375,
+      "logps/chosen": -418.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000690460205078125,
+      "logps/rejected": -452.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000690460205078125,
+      "loss": 0.543,
+      "rewards/accuracies": 0.7099999785423279,
+      "rewards/chosen": -0.94140625,
+      "rewards/margins": 0.48828125,
+      "rewards/rejected": -1.4296875,
+      "step": 1320
+    },
+    {
+      "epoch": 2.6104023552502453,
+      "grad_norm": 6.31080886957666,
+      "learning_rate": 4.614102245673039e-07,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.359375,
+      "logps/chosen": -478.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000713348388671875,
+      "logps/rejected": -496.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000698089599609375,
+      "loss": 0.5482,
+      "rewards/accuracies": 0.6950000524520874,
+      "rewards/chosen": -1.09375,
+      "rewards/margins": 0.41015625,
+      "rewards/rejected": -1.5,
+      "step": 1330
+    },
+    {
+      "epoch": 2.630029440628067,
+      "grad_norm": 6.076214388108869,
+      "learning_rate": 4.604901588028023e-07,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -1.3515625,
+      "logps/chosen": -452.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.0007476806640625,
+      "logps/rejected": -476.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000732421875,
+      "loss": 0.521,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.0078125,
+      "rewards/margins": 0.7109375,
+      "rewards/rejected": -1.7109375,
+      "step": 1340
+    },
+    {
+      "epoch": 2.649656526005888,
+      "grad_norm": 6.946264433751439,
+      "learning_rate": 4.5956019362958e-07,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -1.359375,
+      "logps/chosen": -500.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00074005126953125,
+      "logps/rejected": -490.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000720977783203125,
+      "loss": 0.5622,
+      "rewards/accuracies": 0.6699999570846558,
+      "rewards/chosen": -1.046875,
+      "rewards/margins": 0.458984375,
+      "rewards/rejected": -1.5078125,
+      "step": 1350
+    },
+    {
+      "epoch": 2.6692836113837095,
+      "grad_norm": 5.855183962780445,
+      "learning_rate": 4.586203727841488e-07,
+      "logits/chosen": -1.28125,
+      "logits/rejected": -1.296875,
+      "logps/chosen": -424.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00064849853515625,
+      "logps/rejected": -466.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000675201416015625,
+      "loss": 0.5507,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.8671875,
+      "rewards/margins": 0.5703125,
+      "rewards/rejected": -1.4375,
+      "step": 1360
+    },
+    {
+      "epoch": 2.688910696761531,
+      "grad_norm": 4.854275700163788,
+      "learning_rate": 4.576707404665355e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -1.3671875,
+      "logps/chosen": -478.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.0005950927734375,
+      "logps/rejected": -498.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.0006103515625,
+      "loss": 0.5518,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.87890625,
+      "rewards/margins": 0.55859375,
+      "rewards/rejected": -1.4375,
+      "step": 1370
+    },
+    {
+      "epoch": 2.708537782139352,
+      "grad_norm": 4.985030119033812,
+      "learning_rate": 4.5671134133820333e-07,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -1.359375,
+      "logps/chosen": -470.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000614166259765625,
+      "logps/rejected": -460.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.0006103515625,
+      "loss": 0.5583,
+      "rewards/accuracies": 0.6899999976158142,
+      "rewards/chosen": -1.0546875,
+      "rewards/margins": 0.45703125,
+      "rewards/rejected": -1.5078125,
+      "step": 1380
+    },
+    {
+      "epoch": 2.7281648675171737,
+      "grad_norm": 4.702085035851529,
+      "learning_rate": 4.5574222051995084e-07,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -1.34375,
+      "logps/chosen": -432.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000698089599609375,
+      "logps/rejected": -458.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.00074005126953125,
+      "loss": 0.5567,
+      "rewards/accuracies": 0.6800000071525574,
+      "rewards/chosen": -0.90234375,
+      "rewards/margins": 0.4765625,
+      "rewards/rejected": -1.375,
+      "step": 1390
+    },
+    {
+      "epoch": 2.7477919528949952,
+      "grad_norm": 4.845696455395146,
+      "learning_rate": 4.547634235897906e-07,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -1.3203125,
+      "logps/chosen": -432.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00058746337890625,
+      "logps/rejected": -442.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000591278076171875,
+      "loss": 0.543,
+      "rewards/accuracies": 0.6850000023841858,
+      "rewards/chosen": -0.8125,
+      "rewards/margins": 0.46484375,
+      "rewards/rejected": -1.2734375,
+      "step": 1400
+    },
+    {
+      "epoch": 2.7477919528949952,
+      "eval_logits/chosen": -1.3359375,
+      "eval_logits/rejected": -1.375,
+      "eval_logps/chosen": -480.0,
+      "eval_logps/chosen_bottom_tokens": -14.0625,
+      "eval_logps/chosen_top_tokens": -0.000675201416015625,
+      "eval_logps/rejected": -474.0,
+      "eval_logps/rejected_bottom_tokens": -13.9375,
+      "eval_logps/rejected_top_tokens": -0.000606536865234375,
+      "eval_loss": 0.5784887671470642,
+      "eval_rewards/accuracies": 0.7134329080581665,
+      "eval_rewards/chosen": -0.8671875,
+      "eval_rewards/margins": 0.486328125,
+      "eval_rewards/rejected": -1.3515625,
+      "eval_runtime": 107.6618,
+      "eval_samples_per_second": 18.577,
+      "eval_steps_per_second": 0.622,
+      "step": 1400
+    },
+    {
+      "epoch": 2.7674190382728163,
+      "grad_norm": 4.4890429623349855,
+      "learning_rate": 4.537749965808052e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -1.375,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.0005645751953125,
+      "logps/rejected": -512.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.00054168701171875,
+      "loss": 0.5466,
+      "rewards/accuracies": 0.6800000667572021,
+      "rewards/chosen": -0.9765625,
+      "rewards/margins": 0.478515625,
+      "rewards/rejected": -1.453125,
+      "step": 1410
+    },
+    {
+      "epoch": 2.787046123650638,
+      "grad_norm": 6.95457084928623,
+      "learning_rate": 4.527769859789825e-07,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -1.359375,
+      "logps/chosen": -464.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.0006561279296875,
+      "logps/rejected": -444.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000621795654296875,
+      "loss": 0.5333,
+      "rewards/accuracies": 0.7099999785423279,
+      "rewards/chosen": -1.03125,
+      "rewards/margins": 0.5078125,
+      "rewards/rejected": -1.5390625,
+      "step": 1420
+    },
+    {
+      "epoch": 2.8066732090284594,
+      "grad_norm": 5.537816441709266,
+      "learning_rate": 4.517694387210291e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -1.375,
+      "logps/chosen": -484.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.00069427490234375,
+      "logps/rejected": -476.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.00067138671875,
+      "loss": 0.5315,
+      "rewards/accuracies": 0.7350000739097595,
+      "rewards/chosen": -1.0234375,
+      "rewards/margins": 0.51171875,
+      "rewards/rejected": -1.5390625,
+      "step": 1430
+    },
+    {
+      "epoch": 2.8263002944062805,
+      "grad_norm": 4.999382915134583,
+      "learning_rate": 4.507524021921633e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.3828125,
+      "logps/chosen": -478.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.00057220458984375,
+      "logps/rejected": -492.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000576019287109375,
+      "loss": 0.5444,
+      "rewards/accuracies": 0.7049999833106995,
+      "rewards/chosen": -0.9609375,
+      "rewards/margins": 0.55078125,
+      "rewards/rejected": -1.5078125,
+      "step": 1440
+    },
+    {
+      "epoch": 2.845927379784102,
+      "grad_norm": 4.978033889471835,
+      "learning_rate": 4.4972592422388635e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -1.3671875,
+      "logps/chosen": -488.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000576019287109375,
+      "logps/rejected": -484.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000560760498046875,
+      "loss": 0.5154,
+      "rewards/accuracies": 0.7799999713897705,
+      "rewards/chosen": -0.98046875,
+      "rewards/margins": 0.62890625,
+      "rewards/rejected": -1.609375,
+      "step": 1450
+    },
+    {
+      "epoch": 2.8655544651619236,
+      "grad_norm": 5.141801556909101,
+      "learning_rate": 4.486900530917328e-07,
+      "logits/chosen": -1.3125,
+      "logits/rejected": -1.3046875,
+      "logps/chosen": -478.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.00067138671875,
+      "logps/rejected": -524.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.00067138671875,
+      "loss": 0.5497,
+      "rewards/accuracies": 0.7400000095367432,
+      "rewards/chosen": -1.03125,
+      "rewards/margins": 0.609375,
+      "rewards/rejected": -1.640625,
+      "step": 1460
+    },
+    {
+      "epoch": 2.8851815505397447,
+      "grad_norm": 5.912963606968759,
+      "learning_rate": 4.4764483751300034e-07,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -488.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.000682830810546875,
+      "logps/rejected": -478.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.00064849853515625,
+      "loss": 0.5415,
+      "rewards/accuracies": 0.7000000476837158,
+      "rewards/chosen": -0.9765625,
+      "rewards/margins": 0.578125,
+      "rewards/rejected": -1.5546875,
+      "step": 1470
+    },
+    {
+      "epoch": 2.904808635917566,
+      "grad_norm": 6.0601731183438625,
+      "learning_rate": 4.465903266444585e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -536.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.00066375732421875,
+      "logps/rejected": -490.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.000621795654296875,
+      "loss": 0.5422,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.0390625,
+      "rewards/margins": 0.59375,
+      "rewards/rejected": -1.6328125,
+      "step": 1480
+    },
+    {
+      "epoch": 2.9244357212953878,
+      "grad_norm": 6.188981701284928,
+      "learning_rate": 4.4552657008003666e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.3671875,
+      "logps/chosen": -484.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000659942626953125,
+      "logps/rejected": -524.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000667572021484375,
+      "loss": 0.5532,
+      "rewards/accuracies": 0.7050000429153442,
+      "rewards/chosen": -1.140625,
+      "rewards/margins": 0.498046875,
+      "rewards/rejected": -1.6328125,
+      "step": 1490
+    },
+    {
+      "epoch": 2.944062806673209,
+      "grad_norm": 5.440504391323201,
+      "learning_rate": 4.444536178484919e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -1.3515625,
+      "logps/chosen": -486.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.00051116943359375,
+      "logps/rejected": -502.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000522613525390625,
+      "loss": 0.5382,
+      "rewards/accuracies": 0.7049999833106995,
+      "rewards/chosen": -1.09375,
+      "rewards/margins": 0.5078125,
+      "rewards/rejected": -1.6015625,
+      "step": 1500
+    },
+    {
+      "epoch": 2.944062806673209,
+      "eval_logits/chosen": -1.3515625,
+      "eval_logits/rejected": -1.390625,
+      "eval_logps/chosen": -506.0,
+      "eval_logps/chosen_bottom_tokens": -14.125,
+      "eval_logps/chosen_top_tokens": -0.000667572021484375,
+      "eval_logps/rejected": -504.0,
+      "eval_logps/rejected_bottom_tokens": -14.0,
+      "eval_logps/rejected_top_tokens": -0.000606536865234375,
+      "eval_loss": 0.5711303949356079,
+      "eval_rewards/accuracies": 0.6955224275588989,
+      "eval_rewards/chosen": -1.1171875,
+      "eval_rewards/margins": 0.55078125,
+      "eval_rewards/rejected": -1.6640625,
+      "eval_runtime": 107.6549,
+      "eval_samples_per_second": 18.578,
+      "eval_steps_per_second": 0.622,
+      "step": 1500
+    },
+    {
+      "epoch": 2.9636898920510304,
+      "grad_norm": 5.413703139778055,
+      "learning_rate": 4.4337152041105597e-07,
+      "logits/chosen": -1.3046875,
+      "logits/rejected": -1.359375,
+      "logps/chosen": -490.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.000579833984375,
+      "logps/rejected": -492.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.000598907470703125,
+      "loss": 0.5143,
+      "rewards/accuracies": 0.7699999809265137,
+      "rewards/chosen": -0.98828125,
+      "rewards/margins": 0.72265625,
+      "rewards/rejected": -1.7109375,
+      "step": 1510
+    },
+    {
+      "epoch": 2.983316977428852,
+      "grad_norm": 6.2588549813806615,
+      "learning_rate": 4.42280328659062e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.375,
+      "logps/chosen": -520.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000537872314453125,
+      "logps/rejected": -544.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000576019287109375,
+      "loss": 0.5463,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.078125,
+      "rewards/margins": 0.52734375,
+      "rewards/rejected": -1.609375,
+      "step": 1520
+    },
+    {
+      "epoch": 3.002944062806673,
+      "grad_norm": 5.850837619696955,
+      "learning_rate": 4.411800939115512e-07,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -508.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.0006256103515625,
+      "logps/rejected": -476.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.000598907470703125,
+      "loss": 0.5182,
+      "rewards/accuracies": 0.7799999713897705,
+      "rewards/chosen": -0.92578125,
+      "rewards/margins": 0.60546875,
+      "rewards/rejected": -1.53125,
+      "step": 1530
+    },
+    {
+      "epoch": 3.0225711481844946,
+      "grad_norm": 5.1157355583646105,
+      "learning_rate": 4.4007086791285955e-07,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -1.375,
+      "logps/chosen": -506.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000762939453125,
+      "logps/rejected": -494.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000728607177734375,
+      "loss": 0.5018,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.0234375,
+      "rewards/margins": 0.77734375,
+      "rewards/rejected": -1.8046875,
+      "step": 1540
+    },
+    {
+      "epoch": 3.042198233562316,
+      "grad_norm": 5.750764600991557,
+      "learning_rate": 4.3895270283018356e-07,
+      "logits/chosen": -1.296875,
+      "logits/rejected": -1.34375,
+      "logps/chosen": -470.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.00067901611328125,
+      "logps/rejected": -500.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.0006866455078125,
+      "loss": 0.5185,
+      "rewards/accuracies": 0.7900000810623169,
+      "rewards/chosen": -0.98828125,
+      "rewards/margins": 0.73046875,
+      "rewards/rejected": -1.71875,
+      "step": 1550
+    },
+    {
+      "epoch": 3.061825318940137,
+      "grad_norm": 6.883922300768385,
+      "learning_rate": 4.3782565125112764e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -520.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000606536865234375,
+      "logps/rejected": -500.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.00061798095703125,
+      "loss": 0.5135,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0390625,
+      "rewards/margins": 0.57421875,
+      "rewards/rejected": -1.6171875,
+      "step": 1560
+    },
+    {
+      "epoch": 3.0814524043179587,
+      "grad_norm": 6.429331009978246,
+      "learning_rate": 4.3668976618123035e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.3671875,
+      "logps/chosen": -488.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.000583648681640625,
+      "logps/rejected": -524.0,
+      "logps/rejected_bottom_tokens": -14.1875,
+      "logps/rejected_top_tokens": -0.00060272216796875,
+      "loss": 0.5316,
+      "rewards/accuracies": 0.7099999785423279,
+      "rewards/chosen": -1.1875,
+      "rewards/margins": 0.52734375,
+      "rewards/rejected": -1.71875,
+      "step": 1570
+    },
+    {
+      "epoch": 3.1010794896957803,
+      "grad_norm": 5.829585261996114,
+      "learning_rate": 4.3554510104147155e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -476.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000705718994140625,
+      "logps/rejected": -502.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.0006866455078125,
+      "loss": 0.5077,
+      "rewards/accuracies": 0.7450000047683716,
+      "rewards/chosen": -1.0390625,
+      "rewards/margins": 0.609375,
+      "rewards/rejected": -1.6484375,
+      "step": 1580
+    },
+    {
+      "epoch": 3.1207065750736014,
+      "grad_norm": 7.072235636297424,
+      "learning_rate": 4.3439170966576056e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -500.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.000606536865234375,
+      "logps/rejected": -532.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000576019287109375,
+      "loss": 0.5134,
+      "rewards/accuracies": 0.7850000262260437,
+      "rewards/chosen": -1.125,
+      "rewards/margins": 0.640625,
+      "rewards/rejected": -1.765625,
+      "step": 1590
+    },
+    {
+      "epoch": 3.140333660451423,
+      "grad_norm": 6.465350023606307,
+      "learning_rate": 4.332296462984034e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.3828125,
+      "logps/chosen": -466.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000606536865234375,
+      "logps/rejected": -494.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000629425048828125,
+      "loss": 0.5117,
+      "rewards/accuracies": 0.7450000047683716,
+      "rewards/chosen": -1.2265625,
+      "rewards/margins": 0.58984375,
+      "rewards/rejected": -1.8125,
+      "step": 1600
+    },
+    {
+      "epoch": 3.140333660451423,
+      "eval_logits/chosen": -1.359375,
+      "eval_logits/rejected": -1.3984375,
+      "eval_logps/chosen": -520.0,
+      "eval_logps/chosen_bottom_tokens": -14.125,
+      "eval_logps/chosen_top_tokens": -0.00067901611328125,
+      "eval_logps/rejected": -520.0,
+      "eval_logps/rejected_bottom_tokens": -14.0,
+      "eval_logps/rejected_top_tokens": -0.0006256103515625,
+      "eval_loss": 0.5712475776672363,
+      "eval_rewards/accuracies": 0.7044776082038879,
+      "eval_rewards/chosen": -1.25,
+      "eval_rewards/margins": 0.57421875,
+      "eval_rewards/rejected": -1.828125,
+      "eval_runtime": 107.7573,
+      "eval_samples_per_second": 18.56,
+      "eval_steps_per_second": 0.622,
+      "step": 1600
+    },
+    {
+      "epoch": 3.1599607458292445,
+      "grad_norm": 6.287359311382746,
+      "learning_rate": 4.3205896559155264e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000652313232421875,
+      "logps/rejected": -490.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000652313232421875,
+      "loss": 0.518,
+      "rewards/accuracies": 0.7849999666213989,
+      "rewards/chosen": -0.94140625,
+      "rewards/margins": 0.71484375,
+      "rewards/rejected": -1.65625,
+      "step": 1610
+    },
+    {
+      "epoch": 3.1795878312070656,
+      "grad_norm": 6.35600731483251,
+      "learning_rate": 4.3087972260263636e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -476.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00066375732421875,
+      "logps/rejected": -482.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000675201416015625,
+      "loss": 0.5027,
+      "rewards/accuracies": 0.7649999856948853,
+      "rewards/chosen": -0.9296875,
+      "rewards/margins": 0.609375,
+      "rewards/rejected": -1.5390625,
+      "step": 1620
+    },
+    {
+      "epoch": 3.199214916584887,
+      "grad_norm": 6.047993507610401,
+      "learning_rate": 4.2969197279176917e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -444.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.0005950927734375,
+      "logps/rejected": -472.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.00063323974609375,
+      "loss": 0.515,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.91015625,
+      "rewards/margins": 0.671875,
+      "rewards/rejected": -1.578125,
+      "step": 1630
+    },
+    {
+      "epoch": 3.2188420019627086,
+      "grad_norm": 6.629084333953029,
+      "learning_rate": 4.2849577201914376e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -1.3671875,
+      "logps/chosen": -488.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.00074005126953125,
+      "logps/rejected": -482.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000736236572265625,
+      "loss": 0.5021,
+      "rewards/accuracies": 0.7200000286102295,
+      "rewards/chosen": -1.0703125,
+      "rewards/margins": 0.61328125,
+      "rewards/rejected": -1.6875,
+      "step": 1640
+    },
+    {
+      "epoch": 3.2384690873405297,
+      "grad_norm": 6.476913702795223,
+      "learning_rate": 4.272911765424039e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -532.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.0008392333984375,
+      "logps/rejected": -548.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000942230224609375,
+      "loss": 0.5168,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.234375,
+      "rewards/margins": 0.69140625,
+      "rewards/rejected": -1.921875,
+      "step": 1650
+    },
+    {
+      "epoch": 3.2580961727183513,
+      "grad_norm": 5.8847665430956075,
+      "learning_rate": 4.2607824301399843e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -496.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.00070953369140625,
+      "logps/rejected": -524.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.00070953369140625,
+      "loss": 0.5208,
+      "rewards/accuracies": 0.7800000309944153,
+      "rewards/chosen": -1.078125,
+      "rewards/margins": 0.65625,
+      "rewards/rejected": -1.734375,
+      "step": 1660
+    },
+    {
+      "epoch": 3.277723258096173,
+      "grad_norm": 6.421810533396227,
+      "learning_rate": 4.2485702847851715e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -500.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.00064849853515625,
+      "logps/rejected": -552.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.00070953369140625,
+      "loss": 0.5022,
+      "rewards/accuracies": 0.7750000357627869,
+      "rewards/chosen": -1.0546875,
+      "rewards/margins": 0.68359375,
+      "rewards/rejected": -1.7421875,
+      "step": 1670
+    },
+    {
+      "epoch": 3.297350343473994,
+      "grad_norm": 6.6608511509443264,
+      "learning_rate": 4.2362759037000774e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.3828125,
+      "logps/chosen": -484.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.00067138671875,
+      "logps/rejected": -510.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.00067138671875,
+      "loss": 0.5126,
+      "rewards/accuracies": 0.7399999499320984,
+      "rewards/chosen": -1.109375,
+      "rewards/margins": 0.6484375,
+      "rewards/rejected": -1.75,
+      "step": 1680
+    },
+    {
+      "epoch": 3.3169774288518155,
+      "grad_norm": 8.22249557148097,
+      "learning_rate": 4.223899865092748e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -474.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.00066375732421875,
+      "logps/rejected": -516.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.00067138671875,
+      "loss": 0.5224,
+      "rewards/accuracies": 0.8149999380111694,
+      "rewards/chosen": -1.09375,
+      "rewards/margins": 0.796875,
+      "rewards/rejected": -1.890625,
+      "step": 1690
+    },
+    {
+      "epoch": 3.336604514229637,
+      "grad_norm": 6.131226194897482,
+      "learning_rate": 4.211442751011603e-07,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -490.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000804901123046875,
+      "logps/rejected": -516.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000774383544921875,
+      "loss": 0.4983,
+      "rewards/accuracies": 0.7699999809265137,
+      "rewards/chosen": -1.0078125,
+      "rewards/margins": 0.7265625,
+      "rewards/rejected": -1.734375,
+      "step": 1700
+    },
+    {
+      "epoch": 3.336604514229637,
+      "eval_logits/chosen": -1.3671875,
+      "eval_logits/rejected": -1.40625,
+      "eval_logps/chosen": -510.0,
+      "eval_logps/chosen_bottom_tokens": -14.125,
+      "eval_logps/chosen_top_tokens": -0.00074005126953125,
+      "eval_logps/rejected": -512.0,
+      "eval_logps/rejected_bottom_tokens": -14.0,
+      "eval_logps/rejected_top_tokens": -0.000690460205078125,
+      "eval_loss": 0.5702734589576721,
+      "eval_rewards/accuracies": 0.7014926075935364,
+      "eval_rewards/chosen": -1.1640625,
+      "eval_rewards/margins": 0.5859375,
+      "eval_rewards/rejected": -1.75,
+      "eval_runtime": 107.7445,
+      "eval_samples_per_second": 18.562,
+      "eval_steps_per_second": 0.622,
+      "step": 1700
+    },
+    {
+      "epoch": 3.356231599607458,
+      "grad_norm": 6.248071111160677,
+      "learning_rate": 4.198905147318065e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -504.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.0006866455078125,
+      "logps/rejected": -532.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.00067138671875,
+      "loss": 0.5105,
+      "rewards/accuracies": 0.7899999618530273,
+      "rewards/chosen": -0.90234375,
+      "rewards/margins": 0.72265625,
+      "rewards/rejected": -1.625,
+      "step": 1710
+    },
+    {
+      "epoch": 3.3758586849852796,
+      "grad_norm": 5.417487111723758,
+      "learning_rate": 4.1862876436590045e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -462.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000823974609375,
+      "logps/rejected": -464.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.00078582763671875,
+      "loss": 0.5037,
+      "rewards/accuracies": 0.7400000095367432,
+      "rewards/chosen": -0.99609375,
+      "rewards/margins": 0.68359375,
+      "rewards/rejected": -1.6796875,
+      "step": 1720
+    },
+    {
+      "epoch": 3.395485770363101,
+      "grad_norm": 6.486611676994055,
+      "learning_rate": 4.173590833439008e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -496.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.0006561279296875,
+      "logps/rejected": -482.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.00066375732421875,
+      "loss": 0.5146,
+      "rewards/accuracies": 0.7350000143051147,
+      "rewards/chosen": -1.046875,
+      "rewards/margins": 0.62890625,
+      "rewards/rejected": -1.6796875,
+      "step": 1730
+    },
+    {
+      "epoch": 3.4151128557409223,
+      "grad_norm": 6.780283032351629,
+      "learning_rate": 4.160815313792472e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.3828125,
+      "logps/chosen": -468.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000743865966796875,
+      "logps/rejected": -478.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000774383544921875,
+      "loss": 0.5133,
+      "rewards/accuracies": 0.7199999690055847,
+      "rewards/chosen": -1.0546875,
+      "rewards/margins": 0.5703125,
+      "rewards/rejected": -1.6171875,
+      "step": 1740
+    },
+    {
+      "epoch": 3.434739941118744,
+      "grad_norm": 7.370844460366542,
+      "learning_rate": 4.1479616855555166e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -520.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000736236572265625,
+      "logps/rejected": -506.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000759124755859375,
+      "loss": 0.529,
+      "rewards/accuracies": 0.7199999690055847,
+      "rewards/chosen": -1.03125,
+      "rewards/margins": 0.6484375,
+      "rewards/rejected": -1.6796875,
+      "step": 1750
+    },
+    {
+      "epoch": 3.4543670264965654,
+      "grad_norm": 7.3760923185749645,
+      "learning_rate": 4.1350305532377327e-07,
+      "logits/chosen": -1.3203125,
+      "logits/rejected": -1.3671875,
+      "logps/chosen": -556.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000766754150390625,
+      "logps/rejected": -560.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000759124755859375,
+      "loss": 0.5115,
+      "rewards/accuracies": 0.8050000071525574,
+      "rewards/chosen": -1.21875,
+      "rewards/margins": 0.8125,
+      "rewards/rejected": -2.03125,
+      "step": 1760
+    },
+    {
+      "epoch": 3.4739941118743864,
+      "grad_norm": 6.087167212711648,
+      "learning_rate": 4.122022524993747e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -488.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000774383544921875,
+      "logps/rejected": -498.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.0007476806640625,
+      "loss": 0.5004,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.1640625,
+      "rewards/margins": 0.6953125,
+      "rewards/rejected": -1.859375,
+      "step": 1770
+    },
+    {
+      "epoch": 3.493621197252208,
+      "grad_norm": 5.966209515920573,
+      "learning_rate": 4.1089382125946217e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000713348388671875,
+      "logps/rejected": -504.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.00072479248046875,
+      "loss": 0.4945,
+      "rewards/accuracies": 0.7199999690055847,
+      "rewards/chosen": -1.0703125,
+      "rewards/margins": 0.703125,
+      "rewards/rejected": -1.7734375,
+      "step": 1780
+    },
+    {
+      "epoch": 3.5132482826300295,
+      "grad_norm": 7.519074638967278,
+      "learning_rate": 4.0957782313990854e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -468.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000759124755859375,
+      "logps/rejected": -466.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.0007476806640625,
+      "loss": 0.5053,
+      "rewards/accuracies": 0.7649999856948853,
+      "rewards/chosen": -0.98046875,
+      "rewards/margins": 0.71875,
+      "rewards/rejected": -1.6953125,
+      "step": 1790
+    },
+    {
+      "epoch": 3.5328753680078506,
+      "grad_norm": 6.403615147944073,
+      "learning_rate": 4.0825432003245896e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -502.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.000640869140625,
+      "logps/rejected": -506.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000640869140625,
+      "loss": 0.4976,
+      "rewards/accuracies": 0.7599999904632568,
+      "rewards/chosen": -1.1484375,
+      "rewards/margins": 0.734375,
+      "rewards/rejected": -1.8828125,
+      "step": 1800
+    },
+    {
+      "epoch": 3.5328753680078506,
+      "eval_logits/chosen": -1.375,
+      "eval_logits/rejected": -1.4140625,
+      "eval_logps/chosen": -520.0,
+      "eval_logps/chosen_bottom_tokens": -14.125,
+      "eval_logps/chosen_top_tokens": -0.000732421875,
+      "eval_logps/rejected": -524.0,
+      "eval_logps/rejected_bottom_tokens": -14.0625,
+      "eval_logps/rejected_top_tokens": -0.000690460205078125,
+      "eval_loss": 0.57086181640625,
+      "eval_rewards/accuracies": 0.7253731489181519,
+      "eval_rewards/chosen": -1.265625,
+      "eval_rewards/margins": 0.61328125,
+      "eval_rewards/rejected": -1.8828125,
+      "eval_runtime": 107.8127,
+      "eval_samples_per_second": 18.551,
+      "eval_steps_per_second": 0.621,
+      "step": 1800
+    },
+    {
+      "epoch": 3.552502453385672,
+      "grad_norm": 6.030308353243273,
+      "learning_rate": 4.069233741818201e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -490.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000759124755859375,
+      "logps/rejected": -516.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000774383544921875,
+      "loss": 0.4954,
+      "rewards/accuracies": 0.7450000047683716,
+      "rewards/chosen": -1.1171875,
+      "rewards/margins": 0.734375,
+      "rewards/rejected": -1.8515625,
+      "step": 1810
+    },
+    {
+      "epoch": 3.5721295387634937,
+      "grad_norm": 6.38126467770384,
+      "learning_rate": 4.0558504818273286e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -510.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.000659942626953125,
+      "logps/rejected": -536.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.00066375732421875,
+      "loss": 0.504,
+      "rewards/accuracies": 0.7850000262260437,
+      "rewards/chosen": -1.046875,
+      "rewards/margins": 0.7734375,
+      "rewards/rejected": -1.8203125,
+      "step": 1820
+    },
+    {
+      "epoch": 3.591756624141315,
+      "grad_norm": 7.708115389756642,
+      "learning_rate": 4.0423940497702856e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -462.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00077056884765625,
+      "logps/rejected": -498.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000812530517578125,
+      "loss": 0.4979,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0,
+      "rewards/margins": 0.64453125,
+      "rewards/rejected": -1.6484375,
+      "step": 1830
+    },
+    {
+      "epoch": 3.6113837095191363,
+      "grad_norm": 6.421955849678387,
+      "learning_rate": 4.028865078506688e-07,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.3671875,
+      "logps/chosen": -516.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000736236572265625,
+      "logps/rejected": -544.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000759124755859375,
+      "loss": 0.4938,
+      "rewards/accuracies": 0.7300000786781311,
+      "rewards/chosen": -1.125,
+      "rewards/margins": 0.640625,
+      "rewards/rejected": -1.765625,
+      "step": 1840
+    },
+    {
+      "epoch": 3.631010794896958,
+      "grad_norm": 7.117336460287237,
+      "learning_rate": 4.015264204307688e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -458.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00078582763671875,
+      "logps/rejected": -494.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000782012939453125,
+      "loss": 0.5196,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.21875,
+      "rewards/margins": 0.66015625,
+      "rewards/rejected": -1.875,
+      "step": 1850
+    },
+    {
+      "epoch": 3.650637880274779,
+      "grad_norm": 6.582818402503727,
+      "learning_rate": 4.001592066826054e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.3671875,
+      "logps/chosen": -478.0,
+      "logps/chosen_bottom_tokens": -13.75,
+      "logps/chosen_top_tokens": -0.000858306884765625,
+      "logps/rejected": -532.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.00089263916015625,
+      "loss": 0.5042,
+      "rewards/accuracies": 0.7799999713897705,
+      "rewards/chosen": -1.3125,
+      "rewards/margins": 0.69140625,
+      "rewards/rejected": -2.0,
+      "step": 1860
+    },
+    {
+      "epoch": 3.6702649656526005,
+      "grad_norm": 8.021762341081473,
+      "learning_rate": 3.987849309066085e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -1.3515625,
+      "logps/chosen": -490.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00084686279296875,
+      "logps/rejected": -512.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000812530517578125,
+      "loss": 0.4992,
+      "rewards/accuracies": 0.7849999666213989,
+      "rewards/chosen": -1.171875,
+      "rewards/margins": 0.71875,
+      "rewards/rejected": -1.890625,
+      "step": 1870
+    },
+    {
+      "epoch": 3.689892051030422,
+      "grad_norm": 6.8418244724657225,
+      "learning_rate": 3.97403657735337e-07,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -1.3671875,
+      "logps/chosen": -500.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000789642333984375,
+      "logps/rejected": -532.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000789642333984375,
+      "loss": 0.5026,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.1171875,
+      "rewards/margins": 0.66015625,
+      "rewards/rejected": -1.7734375,
+      "step": 1880
+    },
+    {
+      "epoch": 3.709519136408243,
+      "grad_norm": 6.138371602438784,
+      "learning_rate": 3.9601545213043936e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -1.34375,
+      "logps/chosen": -470.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000843048095703125,
+      "logps/rejected": -510.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.00089263916015625,
+      "loss": 0.5065,
+      "rewards/accuracies": 0.7500000596046448,
+      "rewards/chosen": -1.15625,
+      "rewards/margins": 0.67578125,
+      "rewards/rejected": -1.828125,
+      "step": 1890
+    },
+    {
+      "epoch": 3.7291462217860647,
+      "grad_norm": 6.159127929581513,
+      "learning_rate": 3.9462037937959815e-07,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -498.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.000751495361328125,
+      "logps/rejected": -532.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000759124755859375,
+      "loss": 0.4956,
+      "rewards/accuracies": 0.7350000143051147,
+      "rewards/chosen": -1.171875,
+      "rewards/margins": 0.640625,
+      "rewards/rejected": -1.8203125,
+      "step": 1900
+    },
+    {
+      "epoch": 3.7291462217860647,
+      "eval_logits/chosen": -1.3671875,
+      "eval_logits/rejected": -1.40625,
+      "eval_logps/chosen": -516.0,
+      "eval_logps/chosen_bottom_tokens": -14.0625,
+      "eval_logps/chosen_top_tokens": -0.000820159912109375,
+      "eval_logps/rejected": -516.0,
+      "eval_logps/rejected_bottom_tokens": -13.9375,
+      "eval_logps/rejected_top_tokens": -0.00080108642578125,
+      "eval_loss": 0.5754028558731079,
+      "eval_rewards/accuracies": 0.7164179682731628,
+      "eval_rewards/chosen": -1.2265625,
+      "eval_rewards/margins": 0.578125,
+      "eval_rewards/rejected": -1.8046875,
+      "eval_runtime": 107.7159,
+      "eval_samples_per_second": 18.567,
+      "eval_steps_per_second": 0.622,
+      "step": 1900
+    },
+    {
+      "epoch": 3.7487733071638862,
+      "grad_norm": 6.771871270407492,
+      "learning_rate": 3.9321850509345944e-07,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.3671875,
+      "logps/chosen": -492.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000820159912109375,
+      "logps/rejected": -516.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000812530517578125,
+      "loss": 0.5049,
+      "rewards/accuracies": 0.7100000381469727,
+      "rewards/chosen": -1.109375,
+      "rewards/margins": 0.69921875,
+      "rewards/rejected": -1.8046875,
+      "step": 1910
+    },
+    {
+      "epoch": 3.7684003925417073,
+      "grad_norm": 6.091441244132361,
+      "learning_rate": 3.918098952025478e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -474.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00079345703125,
+      "logps/rejected": -482.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.00079345703125,
+      "loss": 0.5025,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1171875,
+      "rewards/margins": 0.64453125,
+      "rewards/rejected": -1.7578125,
+      "step": 1920
+    },
+    {
+      "epoch": 3.788027477919529,
+      "grad_norm": 7.065260385674628,
+      "learning_rate": 3.9039461595416467e-07,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -1.3671875,
+      "logps/chosen": -474.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.0008087158203125,
+      "logps/rejected": -496.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.00080108642578125,
+      "loss": 0.5114,
+      "rewards/accuracies": 0.7450000047683716,
+      "rewards/chosen": -1.109375,
+      "rewards/margins": 0.6953125,
+      "rewards/rejected": -1.8046875,
+      "step": 1930
+    },
+    {
+      "epoch": 3.8076545632973504,
+      "grad_norm": 7.769425748717974,
+      "learning_rate": 3.8897273390927355e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.0008392333984375,
+      "logps/rejected": -560.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.0008087158203125,
+      "loss": 0.499,
+      "rewards/accuracies": 0.7599999904632568,
+      "rewards/chosen": -1.109375,
+      "rewards/margins": 0.7265625,
+      "rewards/rejected": -1.8359375,
+      "step": 1940
+    },
+    {
+      "epoch": 3.8272816486751715,
+      "grad_norm": 9.081400742823046,
+      "learning_rate": 3.8754431593936883e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -492.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000797271728515625,
+      "logps/rejected": -492.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000789642333984375,
+      "loss": 0.5086,
+      "rewards/accuracies": 0.7649999856948853,
+      "rewards/chosen": -1.0546875,
+      "rewards/margins": 0.60546875,
+      "rewards/rejected": -1.6640625,
+      "step": 1950
+    },
+    {
+      "epoch": 3.846908734052993,
+      "grad_norm": 6.83623776664128,
+      "learning_rate": 3.8610942922333157e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -1.375,
+      "logps/chosen": -548.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000820159912109375,
+      "logps/rejected": -556.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000827789306640625,
+      "loss": 0.515,
+      "rewards/accuracies": 0.7450000047683716,
+      "rewards/chosen": -1.1484375,
+      "rewards/margins": 0.6875,
+      "rewards/rejected": -1.8359375,
+      "step": 1960
+    },
+    {
+      "epoch": 3.8665358194308146,
+      "grad_norm": 6.2883991743173775,
+      "learning_rate": 3.8466814124426937e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -1.3671875,
+      "logps/chosen": -520.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000843048095703125,
+      "logps/rejected": -536.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.0008697509765625,
+      "loss": 0.4939,
+      "rewards/accuracies": 0.7450000047683716,
+      "rewards/chosen": -1.3359375,
+      "rewards/margins": 0.66796875,
+      "rewards/rejected": -2.0,
+      "step": 1970
+    },
+    {
+      "epoch": 3.8861629048086357,
+      "grad_norm": 6.696772904643436,
+      "learning_rate": 3.832205197863432e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -492.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000766754150390625,
+      "logps/rejected": -498.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.0007781982421875,
+      "loss": 0.5156,
+      "rewards/accuracies": 0.7450000047683716,
+      "rewards/chosen": -1.265625,
+      "rewards/margins": 0.625,
+      "rewards/rejected": -1.890625,
+      "step": 1980
+    },
+    {
+      "epoch": 3.9057899901864572,
+      "grad_norm": 6.274826409211275,
+      "learning_rate": 3.817666329315792e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4453125,
+      "logps/chosen": -486.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.00074005126953125,
+      "logps/rejected": -482.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.00078582763671875,
+      "loss": 0.4877,
+      "rewards/accuracies": 0.7950000166893005,
+      "rewards/chosen": -1.046875,
+      "rewards/margins": 0.8046875,
+      "rewards/rejected": -1.8515625,
+      "step": 1990
+    },
+    {
+      "epoch": 3.9254170755642788,
+      "grad_norm": 7.773042696653781,
+      "learning_rate": 3.8030654905666663e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -508.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00079345703125,
+      "logps/rejected": -520.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000782012939453125,
+      "loss": 0.4996,
+      "rewards/accuracies": 0.7450000047683716,
+      "rewards/chosen": -1.203125,
+      "rewards/margins": 0.70703125,
+      "rewards/rejected": -1.90625,
+      "step": 2000
+    },
+    {
+      "epoch": 3.9254170755642788,
+      "eval_logits/chosen": -1.375,
+      "eval_logits/rejected": -1.40625,
+      "eval_logps/chosen": -520.0,
+      "eval_logps/chosen_bottom_tokens": -14.0625,
+      "eval_logps/chosen_top_tokens": -0.0008087158203125,
+      "eval_logps/rejected": -524.0,
+      "eval_logps/rejected_bottom_tokens": -13.9375,
+      "eval_logps/rejected_top_tokens": -0.0007781982421875,
+      "eval_loss": 0.5721948146820068,
+      "eval_rewards/accuracies": 0.7044775485992432,
+      "eval_rewards/chosen": -1.2578125,
+      "eval_rewards/margins": 0.6015625,
+      "eval_rewards/rejected": -1.8515625,
+      "eval_runtime": 107.7979,
+      "eval_samples_per_second": 18.553,
+      "eval_steps_per_second": 0.622,
+      "step": 2000
+    },
+    {
+      "epoch": 3.9450441609421,
+      "grad_norm": 6.221166448815382,
+      "learning_rate": 3.788403368297426e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -488.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.00083160400390625,
+      "logps/rejected": -532.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.00089263916015625,
+      "loss": 0.4979,
+      "rewards/accuracies": 0.7699999809265137,
+      "rewards/chosen": -1.1640625,
+      "rewards/margins": 0.765625,
+      "rewards/rejected": -1.9296875,
+      "step": 2010
+    },
+    {
+      "epoch": 3.9646712463199214,
+      "grad_norm": 6.881681342957275,
+      "learning_rate": 3.7736806520716194e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -1.359375,
+      "logps/chosen": -500.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.0008544921875,
+      "logps/rejected": -524.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000865936279296875,
+      "loss": 0.4873,
+      "rewards/accuracies": 0.7699999213218689,
+      "rewards/chosen": -1.1015625,
+      "rewards/margins": 0.69921875,
+      "rewards/rejected": -1.8046875,
+      "step": 2020
+    },
+    {
+      "epoch": 3.984298331697743,
+      "grad_norm": 7.245729337451655,
+      "learning_rate": 3.7588980343025466e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -1.375,
+      "logps/chosen": -508.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00084686279296875,
+      "logps/rejected": -548.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000843048095703125,
+      "loss": 0.486,
+      "rewards/accuracies": 0.7700000405311584,
+      "rewards/chosen": -1.25,
+      "rewards/margins": 0.81640625,
+      "rewards/rejected": -2.0625,
+      "step": 2030
+    },
+    {
+      "epoch": 4.003925417075564,
+      "grad_norm": 7.746412669207162,
+      "learning_rate": 3.744056210220692e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.3828125,
+      "logps/chosen": -532.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000919342041015625,
+      "logps/rejected": -548.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.0009002685546875,
+      "loss": 0.4992,
+      "rewards/accuracies": 0.7300000190734863,
+      "rewards/chosen": -1.4140625,
+      "rewards/margins": 0.6015625,
+      "rewards/rejected": -2.015625,
+      "step": 2040
+    },
+    {
+      "epoch": 4.023552502453386,
+      "grad_norm": 6.945255988635086,
+      "learning_rate": 3.7291558778410307e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -500.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000823974609375,
+      "logps/rejected": -510.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.0008087158203125,
+      "loss": 0.4725,
+      "rewards/accuracies": 0.7649999856948853,
+      "rewards/chosen": -1.1484375,
+      "rewards/margins": 0.72265625,
+      "rewards/rejected": -1.8671875,
+      "step": 2050
+    },
+    {
+      "epoch": 4.043179587831207,
+      "grad_norm": 6.647028975170552,
+      "learning_rate": 3.714197737930199e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.34375,
+      "logps/chosen": -458.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000827789306640625,
+      "logps/rejected": -520.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000827789306640625,
+      "loss": 0.4934,
+      "rewards/accuracies": 0.7699999809265137,
+      "rewards/chosen": -1.125,
+      "rewards/margins": 0.64453125,
+      "rewards/rejected": -1.765625,
+      "step": 2060
+    },
+    {
+      "epoch": 4.062806673209028,
+      "grad_norm": 7.140041695807948,
+      "learning_rate": 3.699182493973532e-07,
+      "logits/chosen": -1.328125,
+      "logits/rejected": -1.3828125,
+      "logps/chosen": -572.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000873565673828125,
+      "logps/rejected": -548.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.0008087158203125,
+      "loss": 0.4763,
+      "rewards/accuracies": 0.7450000047683716,
+      "rewards/chosen": -1.2421875,
+      "rewards/margins": 0.66796875,
+      "rewards/rejected": -1.90625,
+      "step": 2070
+    },
+    {
+      "epoch": 4.08243375858685,
+      "grad_norm": 7.0413962534044945,
+      "learning_rate": 3.6841108521419903e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00081634521484375,
+      "logps/rejected": -560.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000774383544921875,
+      "loss": 0.472,
+      "rewards/accuracies": 0.7849999666213989,
+      "rewards/chosen": -1.2109375,
+      "rewards/margins": 0.796875,
+      "rewards/rejected": -2.0,
+      "step": 2080
+    },
+    {
+      "epoch": 4.102060843964671,
+      "grad_norm": 7.852773860149295,
+      "learning_rate": 3.6689835212589375e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -488.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000835418701171875,
+      "logps/rejected": -480.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000827789306640625,
+      "loss": 0.4699,
+      "rewards/accuracies": 0.7350000143051147,
+      "rewards/chosen": -1.234375,
+      "rewards/margins": 0.6796875,
+      "rewards/rejected": -1.9140625,
+      "step": 2090
+    },
+    {
+      "epoch": 4.121687929342492,
+      "grad_norm": 7.332304315384899,
+      "learning_rate": 3.6538012127668095e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -1.375,
+      "logps/chosen": -510.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000858306884765625,
+      "logps/rejected": -548.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000873565673828125,
+      "loss": 0.4588,
+      "rewards/accuracies": 0.85999995470047,
+      "rewards/chosen": -1.1953125,
+      "rewards/margins": 0.9140625,
+      "rewards/rejected": -2.109375,
+      "step": 2100
+    },
+    {
+      "epoch": 4.121687929342492,
+      "eval_logits/chosen": -1.375,
+      "eval_logits/rejected": -1.40625,
+      "eval_logps/chosen": -536.0,
+      "eval_logps/chosen_bottom_tokens": -14.0,
+      "eval_logps/chosen_top_tokens": -0.00089263916015625,
+      "eval_logps/rejected": -540.0,
+      "eval_logps/rejected_bottom_tokens": -13.875,
+      "eval_logps/rejected_top_tokens": -0.000873565673828125,
+      "eval_loss": 0.5748266577720642,
+      "eval_rewards/accuracies": 0.7343283891677856,
+      "eval_rewards/chosen": -1.4140625,
+      "eval_rewards/margins": 0.62109375,
+      "eval_rewards/rejected": -2.03125,
+      "eval_runtime": 107.7433,
+      "eval_samples_per_second": 18.563,
+      "eval_steps_per_second": 0.622,
+      "step": 2100
+    },
+    {
+      "epoch": 4.141315014720314,
+      "grad_norm": 8.178777396954505,
+      "learning_rate": 3.638564640693654e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -1.375,
+      "logps/chosen": -490.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.0009002685546875,
+      "logps/rejected": -528.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00095367431640625,
+      "loss": 0.4787,
+      "rewards/accuracies": 0.7699999809265137,
+      "rewards/chosen": -1.40625,
+      "rewards/margins": 0.8359375,
+      "rewards/rejected": -2.234375,
+      "step": 2110
+    },
+    {
+      "epoch": 4.1609421000981355,
+      "grad_norm": 8.270420911364823,
+      "learning_rate": 3.623274521619549e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.3828125,
+      "logps/chosen": -528.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00086212158203125,
+      "logps/rejected": -568.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000850677490234375,
+      "loss": 0.4694,
+      "rewards/accuracies": 0.7899999618530273,
+      "rewards/chosen": -1.2265625,
+      "rewards/margins": 0.796875,
+      "rewards/rejected": -2.015625,
+      "step": 2120
+    },
+    {
+      "epoch": 4.180569185475957,
+      "grad_norm": 8.51762407281088,
+      "learning_rate": 3.6079315746429016e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00086212158203125,
+      "logps/rejected": -568.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000873565673828125,
+      "loss": 0.4691,
+      "rewards/accuracies": 0.7900000214576721,
+      "rewards/chosen": -1.2578125,
+      "rewards/margins": 0.87109375,
+      "rewards/rejected": -2.125,
+      "step": 2130
+    },
+    {
+      "epoch": 4.200196270853779,
+      "grad_norm": 7.650781283295743,
+      "learning_rate": 3.592536521346631e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -1.375,
+      "logps/chosen": -572.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00086212158203125,
+      "logps/rejected": -584.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000873565673828125,
+      "loss": 0.4688,
+      "rewards/accuracies": 0.7649999856948853,
+      "rewards/chosen": -1.1875,
+      "rewards/margins": 0.765625,
+      "rewards/rejected": -1.953125,
+      "step": 2140
+    },
+    {
+      "epoch": 4.2198233562316,
+      "grad_norm": 8.230508945281315,
+      "learning_rate": 3.5770900857642307e-07,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.375,
+      "logps/chosen": -478.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000946044921875,
+      "logps/rejected": -536.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.00096893310546875,
+      "loss": 0.4624,
+      "rewards/accuracies": 0.7699999809265137,
+      "rewards/chosen": -1.25,
+      "rewards/margins": 0.80859375,
+      "rewards/rejected": -2.0625,
+      "step": 2150
+    },
+    {
+      "epoch": 4.239450441609421,
+      "grad_norm": 7.156745919336785,
+      "learning_rate": 3.5615929943457145e-07,
+      "logits/chosen": -1.34375,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -516.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000904083251953125,
+      "logps/rejected": -528.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000911712646484375,
+      "loss": 0.4614,
+      "rewards/accuracies": 0.7799999713897705,
+      "rewards/chosen": -1.28125,
+      "rewards/margins": 0.8359375,
+      "rewards/rejected": -2.109375,
+      "step": 2160
+    },
+    {
+      "epoch": 4.259077526987243,
+      "grad_norm": 8.239581657287577,
+      "learning_rate": 3.546045975923457e-07,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.375,
+      "logps/chosen": -472.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000896453857421875,
+      "logps/rejected": -536.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000911712646484375,
+      "loss": 0.4676,
+      "rewards/accuracies": 0.8550000190734863,
+      "rewards/chosen": -1.125,
+      "rewards/margins": 0.9453125,
+      "rewards/rejected": -2.0625,
+      "step": 2170
+    },
+    {
+      "epoch": 4.278704612365064,
+      "grad_norm": 9.9311762081506,
+      "learning_rate": 3.530449761677911e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -472.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000843048095703125,
+      "logps/rejected": -490.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.0008392333984375,
+      "loss": 0.474,
+      "rewards/accuracies": 0.7799999713897705,
+      "rewards/chosen": -1.1484375,
+      "rewards/margins": 0.71484375,
+      "rewards/rejected": -1.859375,
+      "step": 2180
+    },
+    {
+      "epoch": 4.298331697742885,
+      "grad_norm": 7.430327144765855,
+      "learning_rate": 3.5148050851032217e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -516.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000946044921875,
+      "logps/rejected": -544.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000965118408203125,
+      "loss": 0.4697,
+      "rewards/accuracies": 0.7800000309944153,
+      "rewards/chosen": -1.296875,
+      "rewards/margins": 0.80078125,
+      "rewards/rejected": -2.09375,
+      "step": 2190
+    },
+    {
+      "epoch": 4.317958783120707,
+      "grad_norm": 8.411774199817168,
+      "learning_rate": 3.499112681972733e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -512.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.00091552734375,
+      "logps/rejected": -536.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000904083251953125,
+      "loss": 0.4555,
+      "rewards/accuracies": 0.7949999570846558,
+      "rewards/chosen": -1.1640625,
+      "rewards/margins": 0.859375,
+      "rewards/rejected": -2.03125,
+      "step": 2200
+    },
+    {
+      "epoch": 4.317958783120707,
+      "eval_logits/chosen": -1.390625,
+      "eval_logits/rejected": -1.421875,
+      "eval_logps/chosen": -524.0,
+      "eval_logps/chosen_bottom_tokens": -13.9375,
+      "eval_logps/chosen_top_tokens": -0.00087738037109375,
+      "eval_logps/rejected": -528.0,
+      "eval_logps/rejected_bottom_tokens": -13.8125,
+      "eval_logps/rejected_top_tokens": -0.00087738037109375,
+      "eval_loss": 0.5742602348327637,
+      "eval_rewards/accuracies": 0.7164179086685181,
+      "eval_rewards/chosen": -1.296875,
+      "eval_rewards/margins": 0.6171875,
+      "eval_rewards/rejected": -1.9140625,
+      "eval_runtime": 107.7452,
+      "eval_samples_per_second": 18.562,
+      "eval_steps_per_second": 0.622,
+      "step": 2200
+    },
+    {
+      "epoch": 4.337585868498528,
+      "grad_norm": 8.874627070136235,
+      "learning_rate": 3.48337329030438e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -536.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.0008697509765625,
+      "logps/rejected": -576.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00086212158203125,
+      "loss": 0.4699,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.4140625,
+      "rewards/margins": 0.76171875,
+      "rewards/rejected": -2.171875,
+      "step": 2210
+    },
+    {
+      "epoch": 4.357212953876349,
+      "grad_norm": 9.385129625994237,
+      "learning_rate": 3.467587650325981e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -450.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.0008544921875,
+      "logps/rejected": -500.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000865936279296875,
+      "loss": 0.4638,
+      "rewards/accuracies": 0.7649999856948853,
+      "rewards/chosen": -1.3125,
+      "rewards/margins": 0.74609375,
+      "rewards/rejected": -2.0625,
+      "step": 2220
+    },
+    {
+      "epoch": 4.376840039254171,
+      "grad_norm": 7.409577352141209,
+      "learning_rate": 3.4517565044404264e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -496.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.00081634521484375,
+      "logps/rejected": -556.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000782012939453125,
+      "loss": 0.4584,
+      "rewards/accuracies": 0.8300000429153442,
+      "rewards/chosen": -1.1640625,
+      "rewards/margins": 0.9296875,
+      "rewards/rejected": -2.09375,
+      "step": 2230
+    },
+    {
+      "epoch": 4.396467124631992,
+      "grad_norm": 7.070880799909722,
+      "learning_rate": 3.435880597190759e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -506.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.0009002685546875,
+      "logps/rejected": -548.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000911712646484375,
+      "loss": 0.4448,
+      "rewards/accuracies": 0.8349999189376831,
+      "rewards/chosen": -1.34375,
+      "rewards/margins": 0.94921875,
+      "rewards/rejected": -2.296875,
+      "step": 2240
+    },
+    {
+      "epoch": 4.416094210009813,
+      "grad_norm": 7.817845737280103,
+      "learning_rate": 3.419960675225163e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -472.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000926971435546875,
+      "logps/rejected": -536.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.0009765625,
+      "loss": 0.4482,
+      "rewards/accuracies": 0.8100000619888306,
+      "rewards/chosen": -1.3046875,
+      "rewards/margins": 0.859375,
+      "rewards/rejected": -2.15625,
+      "step": 2250
+    },
+    {
+      "epoch": 4.435721295387635,
+      "grad_norm": 6.862127053903812,
+      "learning_rate": 3.403997487261846e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -492.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000911712646484375,
+      "logps/rejected": -556.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00092315673828125,
+      "loss": 0.4831,
+      "rewards/accuracies": 0.7800000309944153,
+      "rewards/chosen": -1.171875,
+      "rewards/margins": 0.8515625,
+      "rewards/rejected": -2.015625,
+      "step": 2260
+    },
+    {
+      "epoch": 4.455348380765456,
+      "grad_norm": 7.719830836324066,
+      "learning_rate": 3.3879917840538264e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -462.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000904083251953125,
+      "logps/rejected": -516.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000926971435546875,
+      "loss": 0.4666,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.1171875,
+      "rewards/margins": 0.78515625,
+      "rewards/rejected": -1.8984375,
+      "step": 2270
+    },
+    {
+      "epoch": 4.4749754661432775,
+      "grad_norm": 7.285396802308617,
+      "learning_rate": 3.3719443183536263e-07,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -488.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000873565673828125,
+      "logps/rejected": -512.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000896453857421875,
+      "loss": 0.4662,
+      "rewards/accuracies": 0.7699999809265137,
+      "rewards/chosen": -1.1484375,
+      "rewards/margins": 0.796875,
+      "rewards/rejected": -1.9375,
+      "step": 2280
+    },
+    {
+      "epoch": 4.494602551521099,
+      "grad_norm": 8.596136962950057,
+      "learning_rate": 3.3558558448778687e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -516.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.0008392333984375,
+      "logps/rejected": -552.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.0008087158203125,
+      "loss": 0.474,
+      "rewards/accuracies": 0.7150000333786011,
+      "rewards/chosen": -1.203125,
+      "rewards/margins": 0.71484375,
+      "rewards/rejected": -1.9140625,
+      "step": 2290
+    },
+    {
+      "epoch": 4.5142296368989205,
+      "grad_norm": 6.896991963163053,
+      "learning_rate": 3.339727120271783e-07,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.3515625,
+      "logps/chosen": -460.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000896453857421875,
+      "logps/rejected": -524.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.00090789794921875,
+      "loss": 0.4625,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.2890625,
+      "rewards/margins": 0.8359375,
+      "rewards/rejected": -2.125,
+      "step": 2300
+    },
+    {
+      "epoch": 4.5142296368989205,
+      "eval_logits/chosen": -1.3828125,
+      "eval_logits/rejected": -1.4140625,
+      "eval_logps/chosen": -524.0,
+      "eval_logps/chosen_bottom_tokens": -14.0,
+      "eval_logps/chosen_top_tokens": -0.0008392333984375,
+      "eval_logps/rejected": -532.0,
+      "eval_logps/rejected_bottom_tokens": -13.875,
+      "eval_logps/rejected_top_tokens": -0.00083160400390625,
+      "eval_loss": 0.5734838843345642,
+      "eval_rewards/accuracies": 0.7134328484535217,
+      "eval_rewards/chosen": -1.3046875,
+      "eval_rewards/margins": 0.625,
+      "eval_rewards/rejected": -1.9296875,
+      "eval_runtime": 107.7862,
+      "eval_samples_per_second": 18.555,
+      "eval_steps_per_second": 0.622,
+      "step": 2300
+    },
+    {
+      "epoch": 4.533856722276742,
+      "grad_norm": 8.088656592962153,
+      "learning_rate": 3.323558903073623e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -498.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00090789794921875,
+      "logps/rejected": -536.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.00089263916015625,
+      "loss": 0.4622,
+      "rewards/accuracies": 0.7800000309944153,
+      "rewards/chosen": -1.21875,
+      "rewards/margins": 0.80859375,
+      "rewards/rejected": -2.03125,
+      "step": 2310
+    },
+    {
+      "epoch": 4.553483807654564,
+      "grad_norm": 8.835212725754523,
+      "learning_rate": 3.307351953678985e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -504.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000949859619140625,
+      "logps/rejected": -496.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000904083251953125,
+      "loss": 0.4653,
+      "rewards/accuracies": 0.7599999904632568,
+      "rewards/chosen": -1.4453125,
+      "rewards/margins": 0.703125,
+      "rewards/rejected": -2.15625,
+      "step": 2320
+    },
+    {
+      "epoch": 4.573110893032385,
+      "grad_norm": 8.83751486846289,
+      "learning_rate": 3.291107034305055e-07,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -502.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00089263916015625,
+      "logps/rejected": -536.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000888824462890625,
+      "loss": 0.4487,
+      "rewards/accuracies": 0.8050000071525574,
+      "rewards/chosen": -1.3515625,
+      "rewards/margins": 0.96484375,
+      "rewards/rejected": -2.3125,
+      "step": 2330
+    },
+    {
+      "epoch": 4.592737978410206,
+      "grad_norm": 7.538041568997226,
+      "learning_rate": 3.2748249089547555e-07,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.3359375,
+      "logps/chosen": -504.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000888824462890625,
+      "logps/rejected": -548.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.00093841552734375,
+      "loss": 0.4564,
+      "rewards/accuracies": 0.7450000047683716,
+      "rewards/chosen": -1.4453125,
+      "rewards/margins": 0.71484375,
+      "rewards/rejected": -2.15625,
+      "step": 2340
+    },
+    {
+      "epoch": 4.612365063788028,
+      "grad_norm": 7.102943658324035,
+      "learning_rate": 3.2585063433808144e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.3828125,
+      "logps/chosen": -516.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00096893310546875,
+      "logps/rejected": -544.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000934600830078125,
+      "loss": 0.4759,
+      "rewards/accuracies": 0.7450000047683716,
+      "rewards/chosen": -1.25,
+      "rewards/margins": 0.7734375,
+      "rewards/rejected": -2.015625,
+      "step": 2350
+    },
+    {
+      "epoch": 4.631992149165849,
+      "grad_norm": 7.911197835666227,
+      "learning_rate": 3.242152105049758e-07,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -1.375,
+      "logps/chosen": -498.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00083160400390625,
+      "logps/rejected": -556.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000835418701171875,
+      "loss": 0.4776,
+      "rewards/accuracies": 0.8349999189376831,
+      "rewards/chosen": -1.234375,
+      "rewards/margins": 0.84375,
+      "rewards/rejected": -2.078125,
+      "step": 2360
+    },
+    {
+      "epoch": 4.65161923454367,
+      "grad_norm": 8.320854595644885,
+      "learning_rate": 3.2257629631058065e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -486.0,
+      "logps/chosen_bottom_tokens": -13.75,
+      "logps/chosen_top_tokens": -0.000885009765625,
+      "logps/rejected": -524.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000843048095703125,
+      "loss": 0.4497,
+      "rewards/accuracies": 0.7699999809265137,
+      "rewards/chosen": -1.3359375,
+      "rewards/margins": 0.81640625,
+      "rewards/rejected": -2.15625,
+      "step": 2370
+    },
+    {
+      "epoch": 4.671246319921492,
+      "grad_norm": 7.929898566219126,
+      "learning_rate": 3.20933968833471e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -516.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000858306884765625,
+      "logps/rejected": -544.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000881195068359375,
+      "loss": 0.4584,
+      "rewards/accuracies": 0.7450000047683716,
+      "rewards/chosen": -1.3984375,
+      "rewards/margins": 0.77734375,
+      "rewards/rejected": -2.171875,
+      "step": 2380
+    },
+    {
+      "epoch": 4.690873405299313,
+      "grad_norm": 7.163855496034465,
+      "learning_rate": 3.1928830531274933e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -492.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000858306884765625,
+      "logps/rejected": -540.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000858306884765625,
+      "loss": 0.4592,
+      "rewards/accuracies": 0.8350000381469727,
+      "rewards/chosen": -1.2734375,
+      "rewards/margins": 0.9375,
+      "rewards/rejected": -2.203125,
+      "step": 2390
+    },
+    {
+      "epoch": 4.710500490677134,
+      "grad_norm": 7.977348623085118,
+      "learning_rate": 3.176393831444131e-07,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.3828125,
+      "logps/chosen": -486.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00092315673828125,
+      "logps/rejected": -564.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00093841552734375,
+      "loss": 0.469,
+      "rewards/accuracies": 0.7949999570846558,
+      "rewards/chosen": -1.296875,
+      "rewards/margins": 0.80859375,
+      "rewards/rejected": -2.109375,
+      "step": 2400
+    },
+    {
+      "epoch": 4.710500490677134,
+      "eval_logits/chosen": -1.3984375,
+      "eval_logits/rejected": -1.4375,
+      "eval_logps/chosen": -540.0,
+      "eval_logps/chosen_bottom_tokens": -14.0,
+      "eval_logps/chosen_top_tokens": -0.000873565673828125,
+      "eval_logps/rejected": -552.0,
+      "eval_logps/rejected_bottom_tokens": -13.875,
+      "eval_logps/rejected_top_tokens": -0.000858306884765625,
+      "eval_loss": 0.5743066668510437,
+      "eval_rewards/accuracies": 0.7194029688835144,
+      "eval_rewards/chosen": -1.4765625,
+      "eval_rewards/margins": 0.65625,
+      "eval_rewards/rejected": -2.140625,
+      "eval_runtime": 107.9291,
+      "eval_samples_per_second": 18.531,
+      "eval_steps_per_second": 0.621,
+      "step": 2400
+    },
+    {
+      "epoch": 4.730127576054956,
+      "grad_norm": 7.459232232628322,
+      "learning_rate": 3.1598727987771485e-07,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.453125,
+      "logps/chosen": -552.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00080108642578125,
+      "logps/rejected": -564.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.00079345703125,
+      "loss": 0.4722,
+      "rewards/accuracies": 0.7400000095367432,
+      "rewards/chosen": -1.4140625,
+      "rewards/margins": 0.8203125,
+      "rewards/rejected": -2.234375,
+      "step": 2410
+    },
+    {
+      "epoch": 4.749754661432777,
+      "grad_norm": 8.489469253151096,
+      "learning_rate": 3.1433207321151523e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000850677490234375,
+      "logps/rejected": -560.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000827789306640625,
+      "loss": 0.4536,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.390625,
+      "rewards/margins": 0.8515625,
+      "rewards/rejected": -2.234375,
+      "step": 2420
+    },
+    {
+      "epoch": 4.769381746810598,
+      "grad_norm": 8.237701367212086,
+      "learning_rate": 3.126738409906284e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -560.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.0008087158203125,
+      "logps/rejected": -620.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.00081634521484375,
+      "loss": 0.434,
+      "rewards/accuracies": 0.8450000882148743,
+      "rewards/chosen": -1.390625,
+      "rewards/margins": 0.9921875,
+      "rewards/rejected": -2.390625,
+      "step": 2430
+    },
+    {
+      "epoch": 4.78900883218842,
+      "grad_norm": 7.925340457533995,
+      "learning_rate": 3.1101266120216124e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -504.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.0008544921875,
+      "logps/rejected": -556.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000858306884765625,
+      "loss": 0.4718,
+      "rewards/accuracies": 0.7449999451637268,
+      "rewards/chosen": -1.2734375,
+      "rewards/margins": 0.84765625,
+      "rewards/rejected": -2.125,
+      "step": 2440
+    },
+    {
+      "epoch": 4.808635917566241,
+      "grad_norm": 8.462699743036197,
+      "learning_rate": 3.0934861197184547e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -520.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000904083251953125,
+      "logps/rejected": -520.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000904083251953125,
+      "loss": 0.4668,
+      "rewards/accuracies": 0.7450000047683716,
+      "rewards/chosen": -1.359375,
+      "rewards/margins": 0.8046875,
+      "rewards/rejected": -2.15625,
+      "step": 2450
+    },
+    {
+      "epoch": 4.8282630029440625,
+      "grad_norm": 8.150345327512246,
+      "learning_rate": 3.076817715603634e-07,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -1.4453125,
+      "logps/chosen": -476.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.00093841552734375,
+      "logps/rejected": -544.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.00093841552734375,
+      "loss": 0.4652,
+      "rewards/accuracies": 0.7800000309944153,
+      "rewards/chosen": -1.3984375,
+      "rewards/margins": 0.92578125,
+      "rewards/rejected": -2.328125,
+      "step": 2460
+    },
+    {
+      "epoch": 4.8478900883218845,
+      "grad_norm": 8.735080643523695,
+      "learning_rate": 3.060122183596676e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -498.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00086212158203125,
+      "logps/rejected": -524.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00086212158203125,
+      "loss": 0.4457,
+      "rewards/accuracies": 0.8399999737739563,
+      "rewards/chosen": -1.28125,
+      "rewards/margins": 0.9609375,
+      "rewards/rejected": -2.25,
+      "step": 2470
+    },
+    {
+      "epoch": 4.867517173699706,
+      "grad_norm": 10.381821722696003,
+      "learning_rate": 3.0434003088929347e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -520.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00096893310546875,
+      "logps/rejected": -560.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.0009613037109375,
+      "loss": 0.4586,
+      "rewards/accuracies": 0.7750000357627869,
+      "rewards/chosen": -1.4453125,
+      "rewards/margins": 0.921875,
+      "rewards/rejected": -2.375,
+      "step": 2480
+    },
+    {
+      "epoch": 4.887144259077527,
+      "grad_norm": 7.745597200658498,
+      "learning_rate": 3.026652877926672e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -520.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00087738037109375,
+      "logps/rejected": -540.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000850677490234375,
+      "loss": 0.4611,
+      "rewards/accuracies": 0.8050000071525574,
+      "rewards/chosen": -1.28125,
+      "rewards/margins": 0.86328125,
+      "rewards/rejected": -2.140625,
+      "step": 2490
+    },
+    {
+      "epoch": 4.906771344455349,
+      "grad_norm": 9.65042560567993,
+      "learning_rate": 3.0098806783340644e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.3828125,
+      "logps/chosen": -482.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.0008087158203125,
+      "logps/rejected": -568.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.00083160400390625,
+      "loss": 0.4796,
+      "rewards/accuracies": 0.7800000309944153,
+      "rewards/chosen": -1.1796875,
+      "rewards/margins": 0.96484375,
+      "rewards/rejected": -2.140625,
+      "step": 2500
+    },
+    {
+      "epoch": 4.906771344455349,
+      "eval_logits/chosen": -1.3984375,
+      "eval_logits/rejected": -1.4375,
+      "eval_logps/chosen": -528.0,
+      "eval_logps/chosen_bottom_tokens": -14.0,
+      "eval_logps/chosen_top_tokens": -0.00086212158203125,
+      "eval_logps/rejected": -536.0,
+      "eval_logps/rejected_bottom_tokens": -13.875,
+      "eval_logps/rejected_top_tokens": -0.000858306884765625,
+      "eval_loss": 0.574963390827179,
+      "eval_rewards/accuracies": 0.7134328484535217,
+      "eval_rewards/chosen": -1.328125,
+      "eval_rewards/margins": 0.6484375,
+      "eval_rewards/rejected": -1.9765625,
+      "eval_runtime": 107.6921,
+      "eval_samples_per_second": 18.571,
+      "eval_steps_per_second": 0.622,
+      "step": 2500
+    },
+    {
+      "epoch": 4.92639842983317,
+      "grad_norm": 9.041272943836647,
+      "learning_rate": 2.9930844989161646e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -492.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00086212158203125,
+      "logps/rejected": -536.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.0008392333984375,
+      "loss": 0.4643,
+      "rewards/accuracies": 0.7549999952316284,
+      "rewards/chosen": -1.2421875,
+      "rewards/margins": 0.83984375,
+      "rewards/rejected": -2.078125,
+      "step": 2510
+    },
+    {
+      "epoch": 4.946025515210991,
+      "grad_norm": 8.247379866929867,
+      "learning_rate": 2.9762651296018045e-07,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -564.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.0007781982421875,
+      "logps/rejected": -588.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000751495361328125,
+      "loss": 0.4637,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.2578125,
+      "rewards/margins": 1.0234375,
+      "rewards/rejected": -2.28125,
+      "step": 2520
+    },
+    {
+      "epoch": 4.965652600588813,
+      "grad_norm": 7.265609055624016,
+      "learning_rate": 2.9594233614104405e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -536.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000827789306640625,
+      "logps/rejected": -572.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00083160400390625,
+      "loss": 0.459,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.2265625,
+      "rewards/margins": 0.93359375,
+      "rewards/rejected": -2.15625,
+      "step": 2530
+    },
+    {
+      "epoch": 4.985279685966634,
+      "grad_norm": 8.284039428637186,
+      "learning_rate": 2.942559986414957e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -508.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.0008697509765625,
+      "logps/rejected": -556.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000873565673828125,
+      "loss": 0.4378,
+      "rewards/accuracies": 0.8300000429153442,
+      "rewards/chosen": -1.234375,
+      "rewards/margins": 0.90625,
+      "rewards/rejected": -2.140625,
+      "step": 2540
+    },
+    {
+      "epoch": 5.004906771344455,
+      "grad_norm": 7.587696700934878,
+      "learning_rate": 2.9256757977044104e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -528.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000858306884765625,
+      "logps/rejected": -580.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.0008544921875,
+      "loss": 0.4535,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.21875,
+      "rewards/margins": 0.84375,
+      "rewards/rejected": -2.0625,
+      "step": 2550
+    },
+    {
+      "epoch": 5.024533856722277,
+      "grad_norm": 9.079365868102483,
+      "learning_rate": 2.9087715893467305e-07,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -564.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000843048095703125,
+      "logps/rejected": -592.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000823974609375,
+      "loss": 0.4135,
+      "rewards/accuracies": 0.8700000047683716,
+      "rewards/chosen": -1.3515625,
+      "rewards/margins": 0.92578125,
+      "rewards/rejected": -2.28125,
+      "step": 2560
+    },
+    {
+      "epoch": 5.044160942100098,
+      "grad_norm": 8.135351090751177,
+      "learning_rate": 2.8918481563513796e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4453125,
+      "logps/chosen": -528.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000896453857421875,
+      "logps/rejected": -560.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000881195068359375,
+      "loss": 0.4309,
+      "rewards/accuracies": 0.8050000071525574,
+      "rewards/chosen": -1.34375,
+      "rewards/margins": 1.0234375,
+      "rewards/rejected": -2.359375,
+      "step": 2570
+    },
+    {
+      "epoch": 5.063788027477919,
+      "grad_norm": 9.685008220153263,
+      "learning_rate": 2.874906294631957e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.453125,
+      "logps/chosen": -568.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00089263916015625,
+      "logps/rejected": -564.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.00087738037109375,
+      "loss": 0.4334,
+      "rewards/accuracies": 0.8299999237060547,
+      "rewards/chosen": -1.40625,
+      "rewards/margins": 0.9453125,
+      "rewards/rejected": -2.34375,
+      "step": 2580
+    },
+    {
+      "epoch": 5.083415112855741,
+      "grad_norm": 8.303566254927492,
+      "learning_rate": 2.857946800968773e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -494.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.000896453857421875,
+      "logps/rejected": -560.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.00089263916015625,
+      "loss": 0.4368,
+      "rewards/accuracies": 0.7549999952316284,
+      "rewards/chosen": -1.40625,
+      "rewards/margins": 0.7734375,
+      "rewards/rejected": -2.1875,
+      "step": 2590
+    },
+    {
+      "epoch": 5.103042198233562,
+      "grad_norm": 8.84280428313552,
+      "learning_rate": 2.840970472971369e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -568.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.0008392333984375,
+      "logps/rejected": -580.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000820159912109375,
+      "loss": 0.4082,
+      "rewards/accuracies": 0.8199999928474426,
+      "rewards/chosen": -1.4140625,
+      "rewards/margins": 1.015625,
+      "rewards/rejected": -2.421875,
+      "step": 2600
+    },
+    {
+      "epoch": 5.103042198233562,
+      "eval_logits/chosen": -1.40625,
+      "eval_logits/rejected": -1.4453125,
+      "eval_logps/chosen": -552.0,
+      "eval_logps/chosen_bottom_tokens": -14.0,
+      "eval_logps/chosen_top_tokens": -0.000904083251953125,
+      "eval_logps/rejected": -564.0,
+      "eval_logps/rejected_bottom_tokens": -13.875,
+      "eval_logps/rejected_top_tokens": -0.0009002685546875,
+      "eval_loss": 0.5817602276802063,
+      "eval_rewards/accuracies": 0.7194029688835144,
+      "eval_rewards/chosen": -1.6015625,
+      "eval_rewards/margins": 0.66015625,
+      "eval_rewards/rejected": -2.265625,
+      "eval_runtime": 107.701,
+      "eval_samples_per_second": 18.57,
+      "eval_steps_per_second": 0.622,
+      "step": 2600
+    },
+    {
+      "epoch": 5.122669283611383,
+      "grad_norm": 8.9514089823427,
+      "learning_rate": 2.823978109041013e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.3828125,
+      "logps/chosen": -470.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00098419189453125,
+      "logps/rejected": -548.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.001007080078125,
+      "loss": 0.4238,
+      "rewards/accuracies": 0.8300000429153442,
+      "rewards/chosen": -1.40625,
+      "rewards/margins": 0.98828125,
+      "rewards/rejected": -2.390625,
+      "step": 2610
+    },
+    {
+      "epoch": 5.142296368989205,
+      "grad_norm": 8.78349947961107,
+      "learning_rate": 2.8069705083331457e-07,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -494.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.0009613037109375,
+      "logps/rejected": -544.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.0009613037109375,
+      "loss": 0.4275,
+      "rewards/accuracies": 0.7899999618530273,
+      "rewards/chosen": -1.46875,
+      "rewards/margins": 1.015625,
+      "rewards/rejected": -2.484375,
+      "step": 2620
+    },
+    {
+      "epoch": 5.1619234543670265,
+      "grad_norm": 9.596094631437214,
+      "learning_rate": 2.789948470719798e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -520.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.00092315673828125,
+      "logps/rejected": -572.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000946044921875,
+      "loss": 0.4266,
+      "rewards/accuracies": 0.8100000619888306,
+      "rewards/chosen": -1.3984375,
+      "rewards/margins": 0.96484375,
+      "rewards/rejected": -2.359375,
+      "step": 2630
+    },
+    {
+      "epoch": 5.181550539744848,
+      "grad_norm": 13.34951502784971,
+      "learning_rate": 2.7729127967519717e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -528.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.00084686279296875,
+      "logps/rejected": -580.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000858306884765625,
+      "loss": 0.4362,
+      "rewards/accuracies": 0.7949999570846558,
+      "rewards/chosen": -1.53125,
+      "rewards/margins": 0.84375,
+      "rewards/rejected": -2.375,
+      "step": 2640
+    },
+    {
+      "epoch": 5.20117762512267,
+      "grad_norm": 10.220288730846836,
+      "learning_rate": 2.7558642876219916e-07,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.4453125,
+      "logps/chosen": -576.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000812530517578125,
+      "logps/rejected": -588.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000782012939453125,
+      "loss": 0.4264,
+      "rewards/accuracies": 0.8650000691413879,
+      "rewards/chosen": -1.4453125,
+      "rewards/margins": 1.046875,
+      "rewards/rejected": -2.484375,
+      "step": 2650
+    },
+    {
+      "epoch": 5.220804710500491,
+      "grad_norm": 10.499751220520977,
+      "learning_rate": 2.7388037451258204e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -512.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00084686279296875,
+      "logps/rejected": -564.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000843048095703125,
+      "loss": 0.4175,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.40625,
+      "rewards/margins": 1.1640625,
+      "rewards/rejected": -2.578125,
+      "step": 2660
+    },
+    {
+      "epoch": 5.240431795878312,
+      "grad_norm": 8.557824000197353,
+      "learning_rate": 2.721731971625357e-07,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -520.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000949859619140625,
+      "logps/rejected": -564.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.00093841552734375,
+      "loss": 0.433,
+      "rewards/accuracies": 0.8200000524520874,
+      "rewards/chosen": -1.3359375,
+      "rewards/margins": 0.91015625,
+      "rewards/rejected": -2.25,
+      "step": 2670
+    },
+    {
+      "epoch": 5.260058881256134,
+      "grad_norm": 9.784128574871701,
+      "learning_rate": 2.704649770010696e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000885009765625,
+      "logps/rejected": -584.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.0008697509765625,
+      "loss": 0.4393,
+      "rewards/accuracies": 0.7900000214576721,
+      "rewards/chosen": -1.515625,
+      "rewards/margins": 0.953125,
+      "rewards/rejected": -2.46875,
+      "step": 2680
+    },
+    {
+      "epoch": 5.279685966633955,
+      "grad_norm": 9.663623448103827,
+      "learning_rate": 2.6875579436623674e-07,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -490.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000835418701171875,
+      "logps/rejected": -556.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000820159912109375,
+      "loss": 0.4292,
+      "rewards/accuracies": 0.8300000429153442,
+      "rewards/chosen": -1.3046875,
+      "rewards/margins": 1.015625,
+      "rewards/rejected": -2.3125,
+      "step": 2690
+    },
+    {
+      "epoch": 5.299313052011776,
+      "grad_norm": 11.510812622281577,
+      "learning_rate": 2.6704572964135574e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -520.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.00092315673828125,
+      "logps/rejected": -592.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.00091552734375,
+      "loss": 0.4193,
+      "rewards/accuracies": 0.8650000691413879,
+      "rewards/chosen": -1.3828125,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -2.640625,
+      "step": 2700
+    },
+    {
+      "epoch": 5.299313052011776,
+      "eval_logits/chosen": -1.3984375,
+      "eval_logits/rejected": -1.4375,
+      "eval_logps/chosen": -544.0,
+      "eval_logps/chosen_bottom_tokens": -14.0,
+      "eval_logps/chosen_top_tokens": -0.000885009765625,
+      "eval_logps/rejected": -552.0,
+      "eval_logps/rejected_bottom_tokens": -13.8125,
+      "eval_logps/rejected_top_tokens": -0.000873565673828125,
+      "eval_loss": 0.5803173780441284,
+      "eval_rewards/accuracies": 0.7194030284881592,
+      "eval_rewards/chosen": -1.4921875,
+      "eval_rewards/margins": 0.65234375,
+      "eval_rewards/rejected": -2.140625,
+      "eval_runtime": 107.7831,
+      "eval_samples_per_second": 18.556,
+      "eval_steps_per_second": 0.622,
+      "step": 2700
+    },
+    {
+      "epoch": 5.318940137389598,
+      "grad_norm": 10.132267872558794,
+      "learning_rate": 2.6533486325123004e-07,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.4453125,
+      "logps/chosen": -482.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000881195068359375,
+      "logps/rejected": -532.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000873565673828125,
+      "loss": 0.4207,
+      "rewards/accuracies": 0.7950000762939453,
+      "rewards/chosen": -1.296875,
+      "rewards/margins": 0.92578125,
+      "rewards/rejected": -2.21875,
+      "step": 2710
+    },
+    {
+      "epoch": 5.338567222767419,
+      "grad_norm": 8.42201810255113,
+      "learning_rate": 2.6362327565836567e-07,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -494.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000934600830078125,
+      "logps/rejected": -560.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.00095367431640625,
+      "loss": 0.4079,
+      "rewards/accuracies": 0.8400000333786011,
+      "rewards/chosen": -1.328125,
+      "rewards/margins": 1.1015625,
+      "rewards/rejected": -2.4375,
+      "step": 2720
+    },
+    {
+      "epoch": 5.35819430814524,
+      "grad_norm": 12.386861806291925,
+      "learning_rate": 2.6191104735918684e-07,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -1.453125,
+      "logps/chosen": -540.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000797271728515625,
+      "logps/rejected": -580.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000782012939453125,
+      "loss": 0.4284,
+      "rewards/accuracies": 0.7900000214576721,
+      "rewards/chosen": -1.3671875,
+      "rewards/margins": 0.90625,
+      "rewards/rejected": -2.28125,
+      "step": 2730
+    },
+    {
+      "epoch": 5.377821393523062,
+      "grad_norm": 9.323175256478242,
+      "learning_rate": 2.6019825888025066e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -512.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00086212158203125,
+      "logps/rejected": -592.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00091552734375,
+      "loss": 0.4286,
+      "rewards/accuracies": 0.8199999928474426,
+      "rewards/chosen": -1.3515625,
+      "rewards/margins": 1.0,
+      "rewards/rejected": -2.359375,
+      "step": 2740
+    },
+    {
+      "epoch": 5.397448478900883,
+      "grad_norm": 9.391284546704878,
+      "learning_rate": 2.584849907744593e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -540.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000919342041015625,
+      "logps/rejected": -572.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000919342041015625,
+      "loss": 0.4222,
+      "rewards/accuracies": 0.8799999952316284,
+      "rewards/chosen": -1.28125,
+      "rewards/margins": 1.046875,
+      "rewards/rejected": -2.328125,
+      "step": 2750
+    },
+    {
+      "epoch": 5.417075564278704,
+      "grad_norm": 9.455608857698858,
+      "learning_rate": 2.567713236172722e-07,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -1.4453125,
+      "logps/chosen": -504.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000949859619140625,
+      "logps/rejected": -556.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00093841552734375,
+      "loss": 0.4313,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.3515625,
+      "rewards/margins": 1.0546875,
+      "rewards/rejected": -2.40625,
+      "step": 2760
+    },
+    {
+      "epoch": 5.436702649656526,
+      "grad_norm": 9.934725698994821,
+      "learning_rate": 2.5505733800291616e-07,
+      "logits/chosen": -1.4375,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -516.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000858306884765625,
+      "logps/rejected": -584.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.0009002685546875,
+      "loss": 0.4344,
+      "rewards/accuracies": 0.7899999618530273,
+      "rewards/chosen": -1.359375,
+      "rewards/margins": 1.015625,
+      "rewards/rejected": -2.375,
+      "step": 2770
+    },
+    {
+      "epoch": 5.456329735034347,
+      "grad_norm": 9.969441161094824,
+      "learning_rate": 2.5334311454059505e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -532.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00084686279296875,
+      "logps/rejected": -572.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000850677490234375,
+      "loss": 0.4287,
+      "rewards/accuracies": 0.8050000071525574,
+      "rewards/chosen": -1.3984375,
+      "rewards/margins": 0.9609375,
+      "rewards/rejected": -2.359375,
+      "step": 2780
+    },
+    {
+      "epoch": 5.4759568204121685,
+      "grad_norm": 12.640410652160751,
+      "learning_rate": 2.516287338506989e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -556.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000888824462890625,
+      "logps/rejected": -572.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000865936279296875,
+      "loss": 0.4156,
+      "rewards/accuracies": 0.8250001072883606,
+      "rewards/chosen": -1.4375,
+      "rewards/margins": 1.015625,
+      "rewards/rejected": -2.453125,
+      "step": 2790
+    },
+    {
+      "epoch": 5.4955839057899905,
+      "grad_norm": 9.260418469079971,
+      "learning_rate": 2.499142765610122e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -512.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000896453857421875,
+      "logps/rejected": -560.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000911712646484375,
+      "loss": 0.419,
+      "rewards/accuracies": 0.8300000429153442,
+      "rewards/chosen": -1.375,
+      "rewards/margins": 1.0390625,
+      "rewards/rejected": -2.40625,
+      "step": 2800
+    },
+    {
+      "epoch": 5.4955839057899905,
+      "eval_logits/chosen": -1.3984375,
+      "eval_logits/rejected": -1.4375,
+      "eval_logps/chosen": -556.0,
+      "eval_logps/chosen_bottom_tokens": -14.0,
+      "eval_logps/chosen_top_tokens": -0.000949859619140625,
+      "eval_logps/rejected": -572.0,
+      "eval_logps/rejected_bottom_tokens": -13.875,
+      "eval_logps/rejected_top_tokens": -0.000926971435546875,
+      "eval_loss": 0.5795263648033142,
+      "eval_rewards/accuracies": 0.7194029688835144,
+      "eval_rewards/chosen": -1.625,
+      "eval_rewards/margins": 0.703125,
+      "eval_rewards/rejected": -2.328125,
+      "eval_runtime": 107.8021,
+      "eval_samples_per_second": 18.553,
+      "eval_steps_per_second": 0.622,
+      "step": 2800
+    },
+    {
+      "epoch": 5.5152109911678115,
+      "grad_norm": 9.426157410126075,
+      "learning_rate": 2.481998233029218e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -560.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000919342041015625,
+      "logps/rejected": -624.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000919342041015625,
+      "loss": 0.4261,
+      "rewards/accuracies": 0.8149999380111694,
+      "rewards/chosen": -1.5234375,
+      "rewards/margins": 1.125,
+      "rewards/rejected": -2.640625,
+      "step": 2810
+    },
+    {
+      "epoch": 5.534838076545633,
+      "grad_norm": 7.93562035357333,
+      "learning_rate": 2.4648545470762515e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -560.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00093841552734375,
+      "logps/rejected": -588.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.00090789794921875,
+      "loss": 0.4284,
+      "rewards/accuracies": 0.7849999666213989,
+      "rewards/chosen": -1.5,
+      "rewards/margins": 0.8828125,
+      "rewards/rejected": -2.375,
+      "step": 2820
+    },
+    {
+      "epoch": 5.554465161923455,
+      "grad_norm": 10.75663156390518,
+      "learning_rate": 2.447712514023378e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.3828125,
+      "logps/chosen": -476.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00087738037109375,
+      "logps/rejected": -552.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000888824462890625,
+      "loss": 0.4362,
+      "rewards/accuracies": 0.8199999928474426,
+      "rewards/chosen": -1.328125,
+      "rewards/margins": 0.98828125,
+      "rewards/rejected": -2.3125,
+      "step": 2830
+    },
+    {
+      "epoch": 5.574092247301276,
+      "grad_norm": 8.242316728901555,
+      "learning_rate": 2.4305729400650186e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.3828125,
+      "logps/chosen": -568.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.0008544921875,
+      "logps/rejected": -616.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00084686279296875,
+      "loss": 0.4365,
+      "rewards/accuracies": 0.7899999618530273,
+      "rewards/chosen": -1.4375,
+      "rewards/margins": 0.97265625,
+      "rewards/rejected": -2.421875,
+      "step": 2840
+    },
+    {
+      "epoch": 5.593719332679097,
+      "grad_norm": 9.729387736541266,
+      "learning_rate": 2.413436631279941e-07,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.453125,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.00077056884765625,
+      "logps/rejected": -548.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.00074005126953125,
+      "loss": 0.4472,
+      "rewards/accuracies": 0.7950000166893005,
+      "rewards/chosen": -1.375,
+      "rewards/margins": 0.92578125,
+      "rewards/rejected": -2.296875,
+      "step": 2850
+    },
+    {
+      "epoch": 5.613346418056919,
+      "grad_norm": 9.579458843278234,
+      "learning_rate": 2.3963043935933503e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -460.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00078582763671875,
+      "logps/rejected": -540.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00081634521484375,
+      "loss": 0.4314,
+      "rewards/accuracies": 0.8349999189376831,
+      "rewards/chosen": -1.2734375,
+      "rewards/margins": 1.0859375,
+      "rewards/rejected": -2.359375,
+      "step": 2860
+    },
+    {
+      "epoch": 5.63297350343474,
+      "grad_norm": 8.387186173885684,
+      "learning_rate": 2.3791770327389896e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000873565673828125,
+      "logps/rejected": -556.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000858306884765625,
+      "loss": 0.4285,
+      "rewards/accuracies": 0.8350000381469727,
+      "rewards/chosen": -1.328125,
+      "rewards/margins": 1.015625,
+      "rewards/rejected": -2.34375,
+      "step": 2870
+    },
+    {
+      "epoch": 5.652600588812561,
+      "grad_norm": 10.151030707964926,
+      "learning_rate": 2.3620553542212408e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -564.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000926971435546875,
+      "logps/rejected": -592.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00092315673828125,
+      "loss": 0.4249,
+      "rewards/accuracies": 0.7949999570846558,
+      "rewards/chosen": -1.53125,
+      "rewards/margins": 0.9609375,
+      "rewards/rejected": -2.5,
+      "step": 2880
+    },
+    {
+      "epoch": 5.672227674190383,
+      "grad_norm": 8.896938317026654,
+      "learning_rate": 2.3449401632772442e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.375,
+      "logps/chosen": -510.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00093841552734375,
+      "logps/rejected": -592.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.0009307861328125,
+      "loss": 0.4219,
+      "rewards/accuracies": 0.8600000143051147,
+      "rewards/chosen": -1.46875,
+      "rewards/margins": 1.0390625,
+      "rewards/rejected": -2.515625,
+      "step": 2890
+    },
+    {
+      "epoch": 5.691854759568204,
+      "grad_norm": 10.861567455177774,
+      "learning_rate": 2.3278322648390296e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000827789306640625,
+      "logps/rejected": -572.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000812530517578125,
+      "loss": 0.4267,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.5078125,
+      "rewards/margins": 0.9765625,
+      "rewards/rejected": -2.484375,
+      "step": 2900
+    },
+    {
+      "epoch": 5.691854759568204,
+      "eval_logits/chosen": -1.40625,
+      "eval_logits/rejected": -1.4375,
+      "eval_logps/chosen": -564.0,
+      "eval_logps/chosen_bottom_tokens": -13.9375,
+      "eval_logps/chosen_top_tokens": -0.0008697509765625,
+      "eval_logps/rejected": -576.0,
+      "eval_logps/rejected_bottom_tokens": -13.8125,
+      "eval_logps/rejected_top_tokens": -0.00084686279296875,
+      "eval_loss": 0.5779784917831421,
+      "eval_rewards/accuracies": 0.713432788848877,
+      "eval_rewards/chosen": -1.6875,
+      "eval_rewards/margins": 0.68359375,
+      "eval_rewards/rejected": -2.375,
+      "eval_runtime": 107.7752,
+      "eval_samples_per_second": 18.557,
+      "eval_steps_per_second": 0.622,
+      "step": 2900
+    },
+    {
+      "epoch": 5.711481844946025,
+      "grad_norm": 8.927136480999275,
+      "learning_rate": 2.3107324634956548e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -564.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.0008087158203125,
+      "logps/rejected": -600.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.00083160400390625,
+      "loss": 0.4294,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.65625,
+      "rewards/margins": 1.03125,
+      "rewards/rejected": -2.6875,
+      "step": 2910
+    },
+    {
+      "epoch": 5.731108930323847,
+      "grad_norm": 10.057562819842476,
+      "learning_rate": 2.2936415634553724e-07,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -536.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000820159912109375,
+      "logps/rejected": -544.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.00080108642578125,
+      "loss": 0.4347,
+      "rewards/accuracies": 0.7899999618530273,
+      "rewards/chosen": -1.3828125,
+      "rewards/margins": 0.93359375,
+      "rewards/rejected": -2.3125,
+      "step": 2920
+    },
+    {
+      "epoch": 5.750736015701668,
+      "grad_norm": 9.681666890093494,
+      "learning_rate": 2.276560368507803e-07,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -540.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.0007781982421875,
+      "logps/rejected": -624.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000827789306640625,
+      "loss": 0.4198,
+      "rewards/accuracies": 0.8300000429153442,
+      "rewards/chosen": -1.4765625,
+      "rewards/margins": 1.0546875,
+      "rewards/rejected": -2.53125,
+      "step": 2930
+    },
+    {
+      "epoch": 5.770363101079489,
+      "grad_norm": 10.195539186974106,
+      "learning_rate": 2.2594896819861342e-07,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.4453125,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00083160400390625,
+      "logps/rejected": -576.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.0008392333984375,
+      "loss": 0.4316,
+      "rewards/accuracies": 0.8550000190734863,
+      "rewards/chosen": -1.3359375,
+      "rewards/margins": 1.109375,
+      "rewards/rejected": -2.453125,
+      "step": 2940
+    },
+    {
+      "epoch": 5.789990186457311,
+      "grad_norm": 9.133704114665807,
+      "learning_rate": 2.2424303067293394e-07,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -510.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00090789794921875,
+      "logps/rejected": -580.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.0009002685546875,
+      "loss": 0.4298,
+      "rewards/accuracies": 0.8700000643730164,
+      "rewards/chosen": -1.4375,
+      "rewards/margins": 1.0703125,
+      "rewards/rejected": -2.5,
+      "step": 2950
+    },
+    {
+      "epoch": 5.809617271835132,
+      "grad_norm": 9.738248145069646,
+      "learning_rate": 2.22538304504442e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -484.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000865936279296875,
+      "logps/rejected": -540.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000873565673828125,
+      "loss": 0.4294,
+      "rewards/accuracies": 0.7900000810623169,
+      "rewards/chosen": -1.53125,
+      "rewards/margins": 0.8984375,
+      "rewards/rejected": -2.421875,
+      "step": 2960
+    },
+    {
+      "epoch": 5.8292443572129535,
+      "grad_norm": 9.963581731279755,
+      "learning_rate": 2.2083486986686737e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -568.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00093841552734375,
+      "logps/rejected": -624.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.00090789794921875,
+      "loss": 0.4318,
+      "rewards/accuracies": 0.7800000905990601,
+      "rewards/chosen": -1.546875,
+      "rewards/margins": 0.984375,
+      "rewards/rejected": -2.53125,
+      "step": 2970
+    },
+    {
+      "epoch": 5.8488714425907755,
+      "grad_norm": 9.36785850903005,
+      "learning_rate": 2.191328068731987e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -510.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00107574462890625,
+      "logps/rejected": -568.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00109100341796875,
+      "loss": 0.4178,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.5234375,
+      "rewards/margins": 1.0078125,
+      "rewards/rejected": -2.53125,
+      "step": 2980
+    },
+    {
+      "epoch": 5.868498527968597,
+      "grad_norm": 9.456033086378868,
+      "learning_rate": 2.1743219557191583e-07,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -568.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.0008392333984375,
+      "logps/rejected": -592.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00086212158203125,
+      "loss": 0.4245,
+      "rewards/accuracies": 0.8349999189376831,
+      "rewards/chosen": -1.5546875,
+      "rewards/margins": 1.0234375,
+      "rewards/rejected": -2.578125,
+      "step": 2990
+    },
+    {
+      "epoch": 5.888125613346418,
+      "grad_norm": 9.12907293722466,
+      "learning_rate": 2.1573311594322527e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -516.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.0010986328125,
+      "logps/rejected": -576.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.00109100341796875,
+      "loss": 0.402,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.4375,
+      "rewards/margins": 1.046875,
+      "rewards/rejected": -2.484375,
+      "step": 3000
+    },
+    {
+      "epoch": 5.888125613346418,
+      "eval_logits/chosen": -1.40625,
+      "eval_logits/rejected": -1.4453125,
+      "eval_logps/chosen": -560.0,
+      "eval_logps/chosen_bottom_tokens": -13.9375,
+      "eval_logps/chosen_top_tokens": -0.000919342041015625,
+      "eval_logps/rejected": -572.0,
+      "eval_logps/rejected_bottom_tokens": -13.8125,
+      "eval_logps/rejected_top_tokens": -0.000888824462890625,
+      "eval_loss": 0.5827575922012329,
+      "eval_rewards/accuracies": 0.7253730893135071,
+      "eval_rewards/chosen": -1.6484375,
+      "eval_rewards/margins": 0.7109375,
+      "eval_rewards/rejected": -2.359375,
+      "eval_runtime": 107.8231,
+      "eval_samples_per_second": 18.549,
+      "eval_steps_per_second": 0.621,
+      "step": 3000
+    },
+    {
+      "epoch": 5.90775269872424,
+      "grad_norm": 12.532499318616896,
+      "learning_rate": 2.1403564789529833e-07,
+      "logits/chosen": -1.4375,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -512.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000873565673828125,
+      "logps/rejected": -564.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.0008544921875,
+      "loss": 0.4136,
+      "rewards/accuracies": 0.8199999928474426,
+      "rewards/chosen": -1.4296875,
+      "rewards/margins": 1.0,
+      "rewards/rejected": -2.4375,
+      "step": 3010
+    },
+    {
+      "epoch": 5.927379784102061,
+      "grad_norm": 10.314331752317209,
+      "learning_rate": 2.123398712605134e-07,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -520.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000934600830078125,
+      "logps/rejected": -592.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00092315673828125,
+      "loss": 0.4162,
+      "rewards/accuracies": 0.8350000381469727,
+      "rewards/chosen": -1.46875,
+      "rewards/margins": 1.03125,
+      "rewards/rejected": -2.5,
+      "step": 3020
+    },
+    {
+      "epoch": 5.947006869479882,
+      "grad_norm": 8.665989364072294,
+      "learning_rate": 2.1064586579170121e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -482.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.0008697509765625,
+      "logps/rejected": -548.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.00086212158203125,
+      "loss": 0.4144,
+      "rewards/accuracies": 0.8050000071525574,
+      "rewards/chosen": -1.4609375,
+      "rewards/margins": 1.0234375,
+      "rewards/rejected": -2.484375,
+      "step": 3030
+    },
+    {
+      "epoch": 5.966633954857704,
+      "grad_norm": 9.103364254357956,
+      "learning_rate": 2.0895371115839412e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -528.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00079345703125,
+      "logps/rejected": -592.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000797271728515625,
+      "loss": 0.4207,
+      "rewards/accuracies": 0.8449999094009399,
+      "rewards/chosen": -1.3359375,
+      "rewards/margins": 1.0234375,
+      "rewards/rejected": -2.359375,
+      "step": 3040
+    },
+    {
+      "epoch": 5.986261040235525,
+      "grad_norm": 8.380145453097748,
+      "learning_rate": 2.0726348694307914e-07,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.453125,
+      "logps/chosen": -532.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000782012939453125,
+      "logps/rejected": -596.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.00078582763671875,
+      "loss": 0.4266,
+      "rewards/accuracies": 0.8399999737739563,
+      "rewards/chosen": -1.4453125,
+      "rewards/margins": 1.109375,
+      "rewards/rejected": -2.546875,
+      "step": 3050
+    },
+    {
+      "epoch": 6.005888125613346,
+      "grad_norm": 10.887214254749555,
+      "learning_rate": 2.0557527263745523e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -536.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00087738037109375,
+      "logps/rejected": -544.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000858306884765625,
+      "loss": 0.4071,
+      "rewards/accuracies": 0.8100000619888306,
+      "rewards/chosen": -1.3359375,
+      "rewards/margins": 1.0390625,
+      "rewards/rejected": -2.375,
+      "step": 3060
+    },
+    {
+      "epoch": 6.025515210991168,
+      "grad_norm": 9.041321401770846,
+      "learning_rate": 2.0388914763869478e-07,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.453125,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000820159912109375,
+      "logps/rejected": -576.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000812530517578125,
+      "loss": 0.396,
+      "rewards/accuracies": 0.809999942779541,
+      "rewards/chosen": -1.453125,
+      "rewards/margins": 1.0703125,
+      "rewards/rejected": -2.515625,
+      "step": 3070
+    },
+    {
+      "epoch": 6.045142296368989,
+      "grad_norm": 8.810241188671984,
+      "learning_rate": 2.0220519124570944e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.0009307861328125,
+      "logps/rejected": -556.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000888824462890625,
+      "loss": 0.3898,
+      "rewards/accuracies": 0.8449999690055847,
+      "rewards/chosen": -1.4609375,
+      "rewards/margins": 1.0703125,
+      "rewards/rejected": -2.53125,
+      "step": 3080
+    },
+    {
+      "epoch": 6.06476938174681,
+      "grad_norm": 9.444456329600422,
+      "learning_rate": 2.0052348265542086e-07,
+      "logits/chosen": -1.4296875,
+      "logits/rejected": -1.453125,
+      "logps/chosen": -512.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000835418701171875,
+      "logps/rejected": -568.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000820159912109375,
+      "loss": 0.3843,
+      "rewards/accuracies": 0.8700000047683716,
+      "rewards/chosen": -1.3203125,
+      "rewards/margins": 1.2421875,
+      "rewards/rejected": -2.5625,
+      "step": 3090
+    },
+    {
+      "epoch": 6.084396467124632,
+      "grad_norm": 9.592468437934086,
+      "learning_rate": 1.9884410095903584e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -486.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000888824462890625,
+      "logps/rejected": -564.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.00087738037109375,
+      "loss": 0.3656,
+      "rewards/accuracies": 0.8449999690055847,
+      "rewards/chosen": -1.5,
+      "rewards/margins": 1.25,
+      "rewards/rejected": -2.75,
+      "step": 3100
+    },
+    {
+      "epoch": 6.084396467124632,
+      "eval_logits/chosen": -1.40625,
+      "eval_logits/rejected": -1.4375,
+      "eval_logps/chosen": -564.0,
+      "eval_logps/chosen_bottom_tokens": -14.0,
+      "eval_logps/chosen_top_tokens": -0.00089263916015625,
+      "eval_logps/rejected": -580.0,
+      "eval_logps/rejected_bottom_tokens": -13.875,
+      "eval_logps/rejected_top_tokens": -0.0008697509765625,
+      "eval_loss": 0.5844067335128784,
+      "eval_rewards/accuracies": 0.7014926075935364,
+      "eval_rewards/chosen": -1.6875,
+      "eval_rewards/margins": 0.72265625,
+      "eval_rewards/rejected": -2.40625,
+      "eval_runtime": 107.8624,
+      "eval_samples_per_second": 18.542,
+      "eval_steps_per_second": 0.621,
+      "step": 3100
+    },
+    {
+      "epoch": 6.104023552502453,
+      "grad_norm": 11.239398087240554,
+      "learning_rate": 1.971671251383268e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -532.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00083160400390625,
+      "logps/rejected": -572.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000820159912109375,
+      "loss": 0.4034,
+      "rewards/accuracies": 0.8050000071525574,
+      "rewards/chosen": -1.5234375,
+      "rewards/margins": 0.8984375,
+      "rewards/rejected": -2.421875,
+      "step": 3110
+    },
+    {
+      "epoch": 6.123650637880274,
+      "grad_norm": 11.234299981879975,
+      "learning_rate": 1.9549263406191703e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -540.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00087738037109375,
+      "logps/rejected": -592.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000835418701171875,
+      "loss": 0.4049,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.5546875,
+      "rewards/margins": 1.140625,
+      "rewards/rejected": -2.6875,
+      "step": 3120
+    },
+    {
+      "epoch": 6.143277723258096,
+      "grad_norm": 11.498877753164736,
+      "learning_rate": 1.9382070648157184e-07,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -504.0,
+      "logps/chosen_bottom_tokens": -13.75,
+      "logps/chosen_top_tokens": -0.000911712646484375,
+      "logps/rejected": -564.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.0009307861328125,
+      "loss": 0.3899,
+      "rewards/accuracies": 0.8799999952316284,
+      "rewards/chosen": -1.3671875,
+      "rewards/margins": 1.1953125,
+      "rewards/rejected": -2.5625,
+      "step": 3130
+    },
+    {
+      "epoch": 6.1629048086359175,
+      "grad_norm": 10.004673273559828,
+      "learning_rate": 1.9215142102849448e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -552.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00112152099609375,
+      "logps/rejected": -616.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.001129150390625,
+      "loss": 0.4067,
+      "rewards/accuracies": 0.7499999403953552,
+      "rewards/chosen": -1.59375,
+      "rewards/margins": 0.9375,
+      "rewards/rejected": -2.53125,
+      "step": 3140
+    },
+    {
+      "epoch": 6.182531894013739,
+      "grad_norm": 9.220472070105401,
+      "learning_rate": 1.904848562096283e-07,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -532.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.0008697509765625,
+      "logps/rejected": -592.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000865936279296875,
+      "loss": 0.3962,
+      "rewards/accuracies": 0.8050000071525574,
+      "rewards/chosen": -1.53125,
+      "rewards/margins": 1.0625,
+      "rewards/rejected": -2.59375,
+      "step": 3150
+    },
+    {
+      "epoch": 6.202158979391561,
+      "grad_norm": 10.124298050031104,
+      "learning_rate": 1.8882109040396455e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -496.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.00087738037109375,
+      "logps/rejected": -588.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000873565673828125,
+      "loss": 0.3954,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -1.40625,
+      "rewards/margins": 1.21875,
+      "rewards/rejected": -2.625,
+      "step": 3160
+    },
+    {
+      "epoch": 6.221786064769382,
+      "grad_norm": 11.485253390722129,
+      "learning_rate": 1.8716020185885597e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -508.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.0009307861328125,
+      "logps/rejected": -556.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000904083251953125,
+      "loss": 0.3863,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.4375,
+      "rewards/margins": 1.046875,
+      "rewards/rejected": -2.484375,
+      "step": 3170
+    },
+    {
+      "epoch": 6.241413150147203,
+      "grad_norm": 9.627394491057656,
+      "learning_rate": 1.8550226868633717e-07,
+      "logits/chosen": -1.4296875,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -470.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00086212158203125,
+      "logps/rejected": -580.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.0009002685546875,
+      "loss": 0.3882,
+      "rewards/accuracies": 0.8600000143051147,
+      "rewards/chosen": -1.5,
+      "rewards/margins": 1.1328125,
+      "rewards/rejected": -2.640625,
+      "step": 3180
+    },
+    {
+      "epoch": 6.261040235525025,
+      "grad_norm": 9.841950410892725,
+      "learning_rate": 1.8384736885945059e-07,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -508.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.0007781982421875,
+      "logps/rejected": -588.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.00077056884765625,
+      "loss": 0.3849,
+      "rewards/accuracies": 0.8750001192092896,
+      "rewards/chosen": -1.484375,
+      "rewards/margins": 1.1796875,
+      "rewards/rejected": -2.671875,
+      "step": 3190
+    },
+    {
+      "epoch": 6.280667320902846,
+      "grad_norm": 11.628550681334719,
+      "learning_rate": 1.8219558020857978e-07,
+      "logits/chosen": -1.453125,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -492.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000942230224609375,
+      "logps/rejected": -612.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00096893310546875,
+      "loss": 0.3971,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.6328125,
+      "rewards/margins": 1.1796875,
+      "rewards/rejected": -2.8125,
+      "step": 3200
+    },
+    {
+      "epoch": 6.280667320902846,
+      "eval_logits/chosen": -1.4140625,
+      "eval_logits/rejected": -1.4453125,
+      "eval_logps/chosen": -556.0,
+      "eval_logps/chosen_bottom_tokens": -14.0,
+      "eval_logps/chosen_top_tokens": -0.0009002685546875,
+      "eval_logps/rejected": -572.0,
+      "eval_logps/rejected_bottom_tokens": -13.8125,
+      "eval_logps/rejected_top_tokens": -0.000865936279296875,
+      "eval_loss": 0.5873156785964966,
+      "eval_rewards/accuracies": 0.707462728023529,
+      "eval_rewards/chosen": -1.609375,
+      "eval_rewards/margins": 0.71484375,
+      "eval_rewards/rejected": -2.328125,
+      "eval_runtime": 107.9708,
+      "eval_samples_per_second": 18.524,
+      "eval_steps_per_second": 0.621,
+      "step": 3200
+    },
+    {
+      "epoch": 6.300294406280667,
+      "grad_norm": 9.556898634263252,
+      "learning_rate": 1.8054698041778877e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -552.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.0009765625,
+      "logps/rejected": -596.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.00096893310546875,
+      "loss": 0.3902,
+      "rewards/accuracies": 0.7849999666213989,
+      "rewards/chosen": -1.609375,
+      "rewards/margins": 1.0,
+      "rewards/rejected": -2.609375,
+      "step": 3210
+    },
+    {
+      "epoch": 6.319921491658489,
+      "grad_norm": 16.752069032509507,
+      "learning_rate": 1.7890164702116866e-07,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -540.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000885009765625,
+      "logps/rejected": -576.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.0008544921875,
+      "loss": 0.4008,
+      "rewards/accuracies": 0.815000057220459,
+      "rewards/chosen": -1.5859375,
+      "rewards/margins": 0.99609375,
+      "rewards/rejected": -2.578125,
+      "step": 3220
+    },
+    {
+      "epoch": 6.33954857703631,
+      "grad_norm": 11.997211906892314,
+      "learning_rate": 1.772596573991911e-07,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.453125,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.0009002685546875,
+      "logps/rejected": -608.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.0009002685546875,
+      "loss": 0.387,
+      "rewards/accuracies": 0.8449999690055847,
+      "rewards/chosen": -1.5078125,
+      "rewards/margins": 1.1640625,
+      "rewards/rejected": -2.671875,
+      "step": 3230
+    },
+    {
+      "epoch": 6.359175662414131,
+      "grad_norm": 9.293641108361177,
+      "learning_rate": 1.7562108877506917e-07,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -516.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000934600830078125,
+      "logps/rejected": -608.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000957489013671875,
+      "loss": 0.3954,
+      "rewards/accuracies": 0.8299999237060547,
+      "rewards/chosen": -1.578125,
+      "rewards/margins": 1.0703125,
+      "rewards/rejected": -2.65625,
+      "step": 3240
+    },
+    {
+      "epoch": 6.378802747791953,
+      "grad_norm": 11.246848893505113,
+      "learning_rate": 1.7398601821112552e-07,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.4453125,
+      "logps/chosen": -588.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00089263916015625,
+      "logps/rejected": -624.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000888824462890625,
+      "loss": 0.3935,
+      "rewards/accuracies": 0.8950001001358032,
+      "rewards/chosen": -1.515625,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -2.859375,
+      "step": 3250
+    },
+    {
+      "epoch": 6.398429833169774,
+      "grad_norm": 11.258833049062101,
+      "learning_rate": 1.7235452260516803e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -512.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000942230224609375,
+      "logps/rejected": -564.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000919342041015625,
+      "loss": 0.4025,
+      "rewards/accuracies": 0.8399999737739563,
+      "rewards/chosen": -1.5,
+      "rewards/margins": 1.1328125,
+      "rewards/rejected": -2.625,
+      "step": 3260
+    },
+    {
+      "epoch": 6.418056918547595,
+      "grad_norm": 8.92258190248652,
+      "learning_rate": 1.7072667868687346e-07,
+      "logits/chosen": -1.4375,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -528.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000881195068359375,
+      "logps/rejected": -604.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.00087738037109375,
+      "loss": 0.3781,
+      "rewards/accuracies": 0.8600000143051147,
+      "rewards/chosen": -1.546875,
+      "rewards/margins": 1.09375,
+      "rewards/rejected": -2.640625,
+      "step": 3270
+    },
+    {
+      "epoch": 6.437684003925417,
+      "grad_norm": 11.88168151340057,
+      "learning_rate": 1.6910256301417854e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -484.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00098419189453125,
+      "logps/rejected": -540.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000965118408203125,
+      "loss": 0.3906,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.4296875,
+      "rewards/margins": 1.140625,
+      "rewards/rejected": -2.578125,
+      "step": 3280
+    },
+    {
+      "epoch": 6.457311089303238,
+      "grad_norm": 11.633081473254599,
+      "learning_rate": 1.674822519696798e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -536.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.0008392333984375,
+      "logps/rejected": -628.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000896453857421875,
+      "loss": 0.3991,
+      "rewards/accuracies": 0.8050000071525574,
+      "rewards/chosen": -1.4921875,
+      "rewards/margins": 1.0,
+      "rewards/rejected": -2.5,
+      "step": 3290
+    },
+    {
+      "epoch": 6.4769381746810595,
+      "grad_norm": 11.362089125815567,
+      "learning_rate": 1.6586582175704088e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -552.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000835418701171875,
+      "logps/rejected": -600.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.00084686279296875,
+      "loss": 0.3923,
+      "rewards/accuracies": 0.85999995470047,
+      "rewards/chosen": -1.4140625,
+      "rewards/margins": 1.2109375,
+      "rewards/rejected": -2.625,
+      "step": 3300
+    },
+    {
+      "epoch": 6.4769381746810595,
+      "eval_logits/chosen": -1.4140625,
+      "eval_logits/rejected": -1.4453125,
+      "eval_logps/chosen": -564.0,
+      "eval_logps/chosen_bottom_tokens": -14.0,
+      "eval_logps/chosen_top_tokens": -0.000896453857421875,
+      "eval_logps/rejected": -580.0,
+      "eval_logps/rejected_bottom_tokens": -13.875,
+      "eval_logps/rejected_top_tokens": -0.000865936279296875,
+      "eval_loss": 0.590624988079071,
+      "eval_rewards/accuracies": 0.707462728023529,
+      "eval_rewards/chosen": -1.6875,
+      "eval_rewards/margins": 0.71875,
+      "eval_rewards/rejected": -2.40625,
+      "eval_runtime": 108.0588,
+      "eval_samples_per_second": 18.508,
+      "eval_steps_per_second": 0.62,
+      "step": 3300
+    },
+    {
+      "epoch": 6.4965652600588815,
+      "grad_norm": 12.198850746961845,
+      "learning_rate": 1.6425334839740912e-07,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -540.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000835418701171875,
+      "logps/rejected": -628.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000865936279296875,
+      "loss": 0.395,
+      "rewards/accuracies": 0.8700000047683716,
+      "rewards/chosen": -1.46875,
+      "rewards/margins": 1.3125,
+      "rewards/rejected": -2.78125,
+      "step": 3310
+    },
+    {
+      "epoch": 6.516192345436703,
+      "grad_norm": 11.826666381899823,
+      "learning_rate": 1.6264490772583984e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -576.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.00081634521484375,
+      "logps/rejected": -636.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000820159912109375,
+      "loss": 0.3971,
+      "rewards/accuracies": 0.8449999690055847,
+      "rewards/chosen": -1.6015625,
+      "rewards/margins": 1.1171875,
+      "rewards/rejected": -2.71875,
+      "step": 3320
+    },
+    {
+      "epoch": 6.535819430814524,
+      "grad_norm": 9.92386217929032,
+      "learning_rate": 1.6104057538772974e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -496.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00090789794921875,
+      "logps/rejected": -580.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000926971435546875,
+      "loss": 0.3988,
+      "rewards/accuracies": 0.8300000429153442,
+      "rewards/chosen": -1.5859375,
+      "rewards/margins": 1.2109375,
+      "rewards/rejected": -2.796875,
+      "step": 3330
+    },
+    {
+      "epoch": 6.555446516192346,
+      "grad_norm": 10.541347410096726,
+      "learning_rate": 1.594404268352599e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.0008392333984375,
+      "logps/rejected": -616.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000843048095703125,
+      "loss": 0.409,
+      "rewards/accuracies": 0.8399999737739563,
+      "rewards/chosen": -1.609375,
+      "rewards/margins": 0.93359375,
+      "rewards/rejected": -2.546875,
+      "step": 3340
+    },
+    {
+      "epoch": 6.575073601570167,
+      "grad_norm": 10.351560745409461,
+      "learning_rate": 1.5784453732384651e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -500.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.0008544921875,
+      "logps/rejected": -588.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000888824462890625,
+      "loss": 0.3885,
+      "rewards/accuracies": 0.8899999856948853,
+      "rewards/chosen": -1.5078125,
+      "rewards/margins": 1.1875,
+      "rewards/rejected": -2.703125,
+      "step": 3350
+    },
+    {
+      "epoch": 6.594700686947988,
+      "grad_norm": 9.362040923183626,
+      "learning_rate": 1.5625298190860226e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.3828125,
+      "logps/chosen": -552.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000885009765625,
+      "logps/rejected": -624.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000865936279296875,
+      "loss": 0.387,
+      "rewards/accuracies": 0.8899999856948853,
+      "rewards/chosen": -1.5703125,
+      "rewards/margins": 1.09375,
+      "rewards/rejected": -2.65625,
+      "step": 3360
+    },
+    {
+      "epoch": 6.61432777232581,
+      "grad_norm": 11.80049485106858,
+      "learning_rate": 1.5466583544080585e-07,
+      "logits/chosen": -1.4296875,
+      "logits/rejected": -1.4453125,
+      "logps/chosen": -560.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000850677490234375,
+      "logps/rejected": -620.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000835418701171875,
+      "loss": 0.3889,
+      "rewards/accuracies": 0.9049999117851257,
+      "rewards/chosen": -1.4453125,
+      "rewards/margins": 1.1875,
+      "rewards/rejected": -2.625,
+      "step": 3370
+    },
+    {
+      "epoch": 6.633954857703631,
+      "grad_norm": 10.067392531938895,
+      "learning_rate": 1.5308317256438203e-07,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -500.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.000888824462890625,
+      "logps/rejected": -576.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.00089263916015625,
+      "loss": 0.3836,
+      "rewards/accuracies": 0.8649999499320984,
+      "rewards/chosen": -1.40625,
+      "rewards/margins": 1.2421875,
+      "rewards/rejected": -2.640625,
+      "step": 3380
+    },
+    {
+      "epoch": 6.653581943081452,
+      "grad_norm": 8.6323831027187,
+      "learning_rate": 1.5150506771239114e-07,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -482.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00086212158203125,
+      "logps/rejected": -580.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.0008697509765625,
+      "loss": 0.3968,
+      "rewards/accuracies": 0.8350000381469727,
+      "rewards/chosen": -1.4765625,
+      "rewards/margins": 1.0625,
+      "rewards/rejected": -2.546875,
+      "step": 3390
+    },
+    {
+      "epoch": 6.673209028459274,
+      "grad_norm": 11.24553545655022,
+      "learning_rate": 1.4993159510352835e-07,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4453125,
+      "logps/chosen": -528.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00081634521484375,
+      "logps/rejected": -592.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.0008087158203125,
+      "loss": 0.4011,
+      "rewards/accuracies": 0.8700000047683716,
+      "rewards/chosen": -1.375,
+      "rewards/margins": 1.2421875,
+      "rewards/rejected": -2.625,
+      "step": 3400
+    },
+    {
+      "epoch": 6.673209028459274,
+      "eval_logits/chosen": -1.40625,
+      "eval_logits/rejected": -1.4375,
+      "eval_logps/chosen": -564.0,
+      "eval_logps/chosen_bottom_tokens": -14.0,
+      "eval_logps/chosen_top_tokens": -0.000873565673828125,
+      "eval_logps/rejected": -584.0,
+      "eval_logps/rejected_bottom_tokens": -13.875,
+      "eval_logps/rejected_top_tokens": -0.000835418701171875,
+      "eval_loss": 0.5848119854927063,
+      "eval_rewards/accuracies": 0.7253730893135071,
+      "eval_rewards/chosen": -1.7109375,
+      "eval_rewards/margins": 0.734375,
+      "eval_rewards/rejected": -2.4375,
+      "eval_runtime": 108.0607,
+      "eval_samples_per_second": 18.508,
+      "eval_steps_per_second": 0.62,
+      "step": 3400
+    },
+    {
+      "epoch": 6.692836113837095,
+      "grad_norm": 11.39250478443067,
+      "learning_rate": 1.4836282873863317e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -548.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00079345703125,
+      "logps/rejected": -592.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.0007781982421875,
+      "loss": 0.3918,
+      "rewards/accuracies": 0.8400000333786011,
+      "rewards/chosen": -1.625,
+      "rewards/margins": 1.0625,
+      "rewards/rejected": -2.6875,
+      "step": 3410
+    },
+    {
+      "epoch": 6.712463199214916,
+      "grad_norm": 10.723016747400582,
+      "learning_rate": 1.4679884239720927e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00091552734375,
+      "logps/rejected": -596.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000873565673828125,
+      "loss": 0.3919,
+      "rewards/accuracies": 0.85999995470047,
+      "rewards/chosen": -1.5,
+      "rewards/margins": 1.1171875,
+      "rewards/rejected": -2.625,
+      "step": 3420
+    },
+    {
+      "epoch": 6.732090284592738,
+      "grad_norm": 10.42018472664649,
+      "learning_rate": 1.4523970963395448e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -528.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000858306884765625,
+      "logps/rejected": -568.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.00086212158203125,
+      "loss": 0.39,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.4921875,
+      "rewards/margins": 1.0390625,
+      "rewards/rejected": -2.53125,
+      "step": 3430
+    },
+    {
+      "epoch": 6.751717369970559,
+      "grad_norm": 10.208772210201792,
+      "learning_rate": 1.436855037753016e-07,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -560.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000865936279296875,
+      "logps/rejected": -620.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000858306884765625,
+      "loss": 0.3841,
+      "rewards/accuracies": 0.8799999356269836,
+      "rewards/chosen": -1.546875,
+      "rewards/margins": 1.1171875,
+      "rewards/rejected": -2.65625,
+      "step": 3440
+    },
+    {
+      "epoch": 6.77134445534838,
+      "grad_norm": 12.551474540069226,
+      "learning_rate": 1.4213629791596948e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -548.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000946044921875,
+      "logps/rejected": -648.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.00102996826171875,
+      "loss": 0.3924,
+      "rewards/accuracies": 0.85999995470047,
+      "rewards/chosen": -1.5234375,
+      "rewards/margins": 1.0703125,
+      "rewards/rejected": -2.59375,
+      "step": 3450
+    },
+    {
+      "epoch": 6.790971540726202,
+      "grad_norm": 9.157599801693973,
+      "learning_rate": 1.405921649155262e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -532.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000774383544921875,
+      "logps/rejected": -644.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000804901123046875,
+      "loss": 0.3838,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.5078125,
+      "rewards/margins": 1.1484375,
+      "rewards/rejected": -2.65625,
+      "step": 3460
+    },
+    {
+      "epoch": 6.8105986261040234,
+      "grad_norm": 10.960666918595264,
+      "learning_rate": 1.390531773949614e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -548.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.000919342041015625,
+      "logps/rejected": -600.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.000896453857421875,
+      "loss": 0.3907,
+      "rewards/accuracies": 0.8749998807907104,
+      "rewards/chosen": -1.484375,
+      "rewards/margins": 1.234375,
+      "rewards/rejected": -2.71875,
+      "step": 3470
+    },
+    {
+      "epoch": 6.8302257114818445,
+      "grad_norm": 9.769484255696572,
+      "learning_rate": 1.3751940773327192e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -560.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000823974609375,
+      "logps/rejected": -640.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00083160400390625,
+      "loss": 0.3914,
+      "rewards/accuracies": 0.8549998998641968,
+      "rewards/chosen": -1.5078125,
+      "rewards/margins": 1.2265625,
+      "rewards/rejected": -2.734375,
+      "step": 3480
+    },
+    {
+      "epoch": 6.8498527968596665,
+      "grad_norm": 11.696207657866864,
+      "learning_rate": 1.3599092806405675e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -552.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000873565673828125,
+      "logps/rejected": -612.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.00086212158203125,
+      "loss": 0.3816,
+      "rewards/accuracies": 0.8449999690055847,
+      "rewards/chosen": -1.6875,
+      "rewards/margins": 1.09375,
+      "rewards/rejected": -2.78125,
+      "step": 3490
+    },
+    {
+      "epoch": 6.869479882237488,
+      "grad_norm": 11.278674812229928,
+      "learning_rate": 1.3446781027212562e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.0010223388671875,
+      "logps/rejected": -584.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.0010223388671875,
+      "loss": 0.3838,
+      "rewards/accuracies": 0.8649999499320984,
+      "rewards/chosen": -1.5390625,
+      "rewards/margins": 1.1640625,
+      "rewards/rejected": -2.703125,
+      "step": 3500
+    },
+    {
+      "epoch": 6.869479882237488,
+      "eval_logits/chosen": -1.3984375,
+      "eval_logits/rejected": -1.4296875,
+      "eval_logps/chosen": -568.0,
+      "eval_logps/chosen_bottom_tokens": -13.9375,
+      "eval_logps/chosen_top_tokens": -0.00087738037109375,
+      "eval_logps/rejected": -584.0,
+      "eval_logps/rejected_bottom_tokens": -13.8125,
+      "eval_logps/rejected_top_tokens": -0.00083160400390625,
+      "eval_loss": 0.5897497534751892,
+      "eval_rewards/accuracies": 0.7164179086685181,
+      "eval_rewards/chosen": -1.75,
+      "eval_rewards/margins": 0.73046875,
+      "eval_rewards/rejected": -2.484375,
+      "eval_runtime": 107.9889,
+      "eval_samples_per_second": 18.52,
+      "eval_steps_per_second": 0.62,
+      "step": 3500
+    },
+    {
+      "epoch": 6.889106967615309,
+      "grad_norm": 12.55807014514054,
+      "learning_rate": 1.3295012599011729e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -532.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000804901123046875,
+      "logps/rejected": -620.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.00081634521484375,
+      "loss": 0.4028,
+      "rewards/accuracies": 0.7950000166893005,
+      "rewards/chosen": -1.7109375,
+      "rewards/margins": 0.8984375,
+      "rewards/rejected": -2.609375,
+      "step": 3510
+    },
+    {
+      "epoch": 6.908734052993131,
+      "grad_norm": 13.054968647766866,
+      "learning_rate": 1.3143794659513152e-07,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -588.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00080108642578125,
+      "logps/rejected": -660.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000774383544921875,
+      "loss": 0.393,
+      "rewards/accuracies": 0.8399999737739563,
+      "rewards/chosen": -1.640625,
+      "rewards/margins": 1.2109375,
+      "rewards/rejected": -2.84375,
+      "step": 3520
+    },
+    {
+      "epoch": 6.928361138370952,
+      "grad_norm": 12.628737963360468,
+      "learning_rate": 1.299313432053713e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -520.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.0009307861328125,
+      "logps/rejected": -576.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000957489013671875,
+      "loss": 0.3908,
+      "rewards/accuracies": 0.8449999690055847,
+      "rewards/chosen": -1.4375,
+      "rewards/margins": 1.1015625,
+      "rewards/rejected": -2.53125,
+      "step": 3530
+    },
+    {
+      "epoch": 6.947988223748773,
+      "grad_norm": 11.20799763262095,
+      "learning_rate": 1.2843038667679903e-07,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -500.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00081634521484375,
+      "logps/rejected": -584.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00079345703125,
+      "loss": 0.4002,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.4140625,
+      "rewards/margins": 1.15625,
+      "rewards/rejected": -2.578125,
+      "step": 3540
+    },
+    {
+      "epoch": 6.967615309126595,
+      "grad_norm": 11.678629068520095,
+      "learning_rate": 1.2693514759980345e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00093841552734375,
+      "logps/rejected": -540.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000934600830078125,
+      "loss": 0.3873,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.5234375,
+      "rewards/margins": 1.0625,
+      "rewards/rejected": -2.59375,
+      "step": 3550
+    },
+    {
+      "epoch": 6.987242394504416,
+      "grad_norm": 11.651443048479592,
+      "learning_rate": 1.2544569629587994e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.3828125,
+      "logps/chosen": -506.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000919342041015625,
+      "logps/rejected": -580.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000934600830078125,
+      "loss": 0.3999,
+      "rewards/accuracies": 0.8600000143051147,
+      "rewards/chosen": -1.4140625,
+      "rewards/margins": 1.171875,
+      "rewards/rejected": -2.59375,
+      "step": 3560
+    },
+    {
+      "epoch": 7.006869479882237,
+      "grad_norm": 9.376535561651522,
+      "learning_rate": 1.2396210281432375e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -516.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.0009002685546875,
+      "logps/rejected": -604.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.00091552734375,
+      "loss": 0.3612,
+      "rewards/accuracies": 0.8399999737739563,
+      "rewards/chosen": -1.4140625,
+      "rewards/margins": 1.3515625,
+      "rewards/rejected": -2.765625,
+      "step": 3570
+    },
+    {
+      "epoch": 7.026496565260059,
+      "grad_norm": 13.057258643450153,
+      "learning_rate": 1.2248443692893462e-07,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -568.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00083160400390625,
+      "logps/rejected": -612.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000797271728515625,
+      "loss": 0.3734,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.5859375,
+      "rewards/margins": 1.0234375,
+      "rewards/rejected": -2.609375,
+      "step": 3580
+    },
+    {
+      "epoch": 7.04612365063788,
+      "grad_norm": 10.72261951423499,
+      "learning_rate": 1.210127681347364e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.3828125,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.0008544921875,
+      "logps/rejected": -600.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000888824462890625,
+      "loss": 0.3647,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.5078125,
+      "rewards/margins": 1.1640625,
+      "rewards/rejected": -2.671875,
+      "step": 3590
+    },
+    {
+      "epoch": 7.065750736015701,
+      "grad_norm": 9.306580817942635,
+      "learning_rate": 1.1954716564470771e-07,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -488.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000885009765625,
+      "logps/rejected": -556.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000896453857421875,
+      "loss": 0.3762,
+      "rewards/accuracies": 0.8899999856948853,
+      "rewards/chosen": -1.5078125,
+      "rewards/margins": 1.1171875,
+      "rewards/rejected": -2.625,
+      "step": 3600
+    },
+    {
+      "epoch": 7.065750736015701,
+      "eval_logits/chosen": -1.40625,
+      "eval_logits/rejected": -1.4375,
+      "eval_logps/chosen": -572.0,
+      "eval_logps/chosen_bottom_tokens": -13.9375,
+      "eval_logps/chosen_top_tokens": -0.000865936279296875,
+      "eval_logps/rejected": -592.0,
+      "eval_logps/rejected_bottom_tokens": -13.8125,
+      "eval_logps/rejected_top_tokens": -0.000823974609375,
+      "eval_loss": 0.5909960865974426,
+      "eval_rewards/accuracies": 0.7134329080581665,
+      "eval_rewards/chosen": -1.78125,
+      "eval_rewards/margins": 0.7421875,
+      "eval_rewards/rejected": -2.53125,
+      "eval_runtime": 108.0788,
+      "eval_samples_per_second": 18.505,
+      "eval_steps_per_second": 0.62,
+      "step": 3600
+    },
+    {
+      "epoch": 7.085377821393523,
+      "grad_norm": 11.891661500010164,
+      "learning_rate": 1.1808769838652755e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -592.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.0008392333984375,
+      "logps/rejected": -620.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000812530517578125,
+      "loss": 0.3741,
+      "rewards/accuracies": 0.8449999690055847,
+      "rewards/chosen": -1.65625,
+      "rewards/margins": 1.046875,
+      "rewards/rejected": -2.703125,
+      "step": 3610
+    },
+    {
+      "epoch": 7.105004906771344,
+      "grad_norm": 10.69499765785664,
+      "learning_rate": 1.1663443499933301e-07,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -540.0,
+      "logps/chosen_bottom_tokens": -13.75,
+      "logps/chosen_top_tokens": -0.000919342041015625,
+      "logps/rejected": -608.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.000919342041015625,
+      "loss": 0.3561,
+      "rewards/accuracies": 0.9199999570846558,
+      "rewards/chosen": -1.53125,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -2.828125,
+      "step": 3620
+    },
+    {
+      "epoch": 7.124631992149165,
+      "grad_norm": 11.278287242230846,
+      "learning_rate": 1.1518744383049187e-07,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -580.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000812530517578125,
+      "logps/rejected": -628.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.00080108642578125,
+      "loss": 0.3571,
+      "rewards/accuracies": 0.8700000047683716,
+      "rewards/chosen": -1.6640625,
+      "rewards/margins": 1.25,
+      "rewards/rejected": -2.90625,
+      "step": 3630
+    },
+    {
+      "epoch": 7.144259077526987,
+      "grad_norm": 10.714223580426642,
+      "learning_rate": 1.1374679293238731e-07,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -1.453125,
+      "logps/chosen": -568.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000850677490234375,
+      "logps/rejected": -640.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000850677490234375,
+      "loss": 0.3737,
+      "rewards/accuracies": 0.8549998998641968,
+      "rewards/chosen": -1.578125,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -2.921875,
+      "step": 3640
+    },
+    {
+      "epoch": 7.1638861629048085,
+      "grad_norm": 13.726506022363308,
+      "learning_rate": 1.1231255005921845e-07,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -1.453125,
+      "logps/chosen": -556.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000911712646484375,
+      "logps/rejected": -600.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.0008697509765625,
+      "loss": 0.3612,
+      "rewards/accuracies": 0.8649999499320984,
+      "rewards/chosen": -1.5234375,
+      "rewards/margins": 1.109375,
+      "rewards/rejected": -2.625,
+      "step": 3650
+    },
+    {
+      "epoch": 7.18351324828263,
+      "grad_norm": 13.79859784191235,
+      "learning_rate": 1.1088478266381257e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -478.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000946044921875,
+      "logps/rejected": -596.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.0009613037109375,
+      "loss": 0.3681,
+      "rewards/accuracies": 0.9050000309944153,
+      "rewards/chosen": -1.4453125,
+      "rewards/margins": 1.328125,
+      "rewards/rejected": -2.78125,
+      "step": 3660
+    },
+    {
+      "epoch": 7.203140333660452,
+      "grad_norm": 12.54064257210143,
+      "learning_rate": 1.0946355789445407e-07,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -560.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000843048095703125,
+      "logps/rejected": -628.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00083160400390625,
+      "loss": 0.3771,
+      "rewards/accuracies": 0.8199998736381531,
+      "rewards/chosen": -1.7578125,
+      "rewards/margins": 1.1171875,
+      "rewards/rejected": -2.875,
+      "step": 3670
+    },
+    {
+      "epoch": 7.222767419038273,
+      "grad_norm": 10.115831687928802,
+      "learning_rate": 1.0804894259172579e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -528.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.00086212158203125,
+      "logps/rejected": -572.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000823974609375,
+      "loss": 0.3621,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.4921875,
+      "rewards/margins": 1.2109375,
+      "rewards/rejected": -2.703125,
+      "step": 3680
+    },
+    {
+      "epoch": 7.242394504416094,
+      "grad_norm": 11.603121001252209,
+      "learning_rate": 1.0664100328536523e-07,
+      "logits/chosen": -1.4296875,
+      "logits/rejected": -1.4609375,
+      "logps/chosen": -568.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000896453857421875,
+      "logps/rejected": -616.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000888824462890625,
+      "loss": 0.3819,
+      "rewards/accuracies": 0.8799999952316284,
+      "rewards/chosen": -1.453125,
+      "rewards/margins": 1.2421875,
+      "rewards/rejected": -2.703125,
+      "step": 3690
+    },
+    {
+      "epoch": 7.262021589793916,
+      "grad_norm": 9.168794884896538,
+      "learning_rate": 1.0523980619113654e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000888824462890625,
+      "logps/rejected": -620.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000896453857421875,
+      "loss": 0.3591,
+      "rewards/accuracies": 0.8799999952316284,
+      "rewards/chosen": -1.5859375,
+      "rewards/margins": 1.234375,
+      "rewards/rejected": -2.828125,
+      "step": 3700
+    },
+    {
+      "epoch": 7.262021589793916,
+      "eval_logits/chosen": -1.40625,
+      "eval_logits/rejected": -1.4375,
+      "eval_logps/chosen": -572.0,
+      "eval_logps/chosen_bottom_tokens": -14.0,
+      "eval_logps/chosen_top_tokens": -0.00090789794921875,
+      "eval_logps/rejected": -592.0,
+      "eval_logps/rejected_bottom_tokens": -13.875,
+      "eval_logps/rejected_top_tokens": -0.0008697509765625,
+      "eval_loss": 0.5895251631736755,
+      "eval_rewards/accuracies": 0.707462728023529,
+      "eval_rewards/chosen": -1.78125,
+      "eval_rewards/margins": 0.7578125,
+      "eval_rewards/rejected": -2.53125,
+      "eval_runtime": 108.0245,
+      "eval_samples_per_second": 18.514,
+      "eval_steps_per_second": 0.62,
+      "step": 3700
+    },
+    {
+      "epoch": 7.281648675171737,
+      "grad_norm": 10.850056329380424,
+      "learning_rate": 1.0384541720771522e-07,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.0008697509765625,
+      "logps/rejected": -640.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000881195068359375,
+      "loss": 0.3514,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.5546875,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -2.84375,
+      "step": 3710
+    },
+    {
+      "epoch": 7.301275760549558,
+      "grad_norm": 11.485411143489458,
+      "learning_rate": 1.0245790191359007e-07,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4453125,
+      "logps/chosen": -560.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.00087738037109375,
+      "logps/rejected": -600.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000858306884765625,
+      "loss": 0.3556,
+      "rewards/accuracies": 0.8799999952316284,
+      "rewards/chosen": -1.578125,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -2.828125,
+      "step": 3720
+    },
+    {
+      "epoch": 7.32090284592738,
+      "grad_norm": 11.82455858645229,
+      "learning_rate": 1.0107732556397791e-07,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -510.0,
+      "logps/chosen_bottom_tokens": -13.75,
+      "logps/chosen_top_tokens": -0.00103759765625,
+      "logps/rejected": -604.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00103759765625,
+      "loss": 0.3764,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.6015625,
+      "rewards/margins": 1.1875,
+      "rewards/rejected": -2.78125,
+      "step": 3730
+    },
+    {
+      "epoch": 7.340529931305201,
+      "grad_norm": 11.475611182028112,
+      "learning_rate": 9.970375308775559e-08,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -552.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.00091552734375,
+      "logps/rejected": -636.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000904083251953125,
+      "loss": 0.3561,
+      "rewards/accuracies": 0.8600000143051147,
+      "rewards/chosen": -1.515625,
+      "rewards/margins": 1.21875,
+      "rewards/rejected": -2.734375,
+      "step": 3740
+    },
+    {
+      "epoch": 7.360157016683022,
+      "grad_norm": 12.197757170153134,
+      "learning_rate": 9.833724908440561e-08,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -532.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000812530517578125,
+      "logps/rejected": -616.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000827789306640625,
+      "loss": 0.3582,
+      "rewards/accuracies": 0.8850000500679016,
+      "rewards/chosen": -1.4453125,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -2.75,
+      "step": 3750
+    },
+    {
+      "epoch": 7.379784102060844,
+      "grad_norm": 10.738153861615361,
+      "learning_rate": 9.697787782097836e-08,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.3828125,
+      "logps/chosen": -548.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.0009002685546875,
+      "logps/rejected": -608.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.0009002685546875,
+      "loss": 0.3722,
+      "rewards/accuracies": 0.8799999952316284,
+      "rewards/chosen": -1.4765625,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": -2.796875,
+      "step": 3760
+    },
+    {
+      "epoch": 7.399411187438665,
+      "grad_norm": 21.827539496326647,
+      "learning_rate": 9.562570322906952e-08,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.453125,
+      "logps/chosen": -540.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000865936279296875,
+      "logps/rejected": -600.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.0008392333984375,
+      "loss": 0.3814,
+      "rewards/accuracies": 0.8600000143051147,
+      "rewards/chosen": -1.6484375,
+      "rewards/margins": 1.0390625,
+      "rewards/rejected": -2.6875,
+      "step": 3770
+    },
+    {
+      "epoch": 7.419038272816486,
+      "grad_norm": 10.133341733647612,
+      "learning_rate": 9.428078890181362e-08,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -1.453125,
+      "logps/chosen": -620.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.00078582763671875,
+      "logps/rejected": -672.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000762939453125,
+      "loss": 0.3636,
+      "rewards/accuracies": 0.8650000691413879,
+      "rewards/chosen": -1.7265625,
+      "rewards/margins": 1.21875,
+      "rewards/rejected": -2.953125,
+      "step": 3780
+    },
+    {
+      "epoch": 7.438665358194308,
+      "grad_norm": 11.414588203095605,
+      "learning_rate": 9.294319809089262e-08,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -552.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00092315673828125,
+      "logps/rejected": -612.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.00090789794921875,
+      "loss": 0.3786,
+      "rewards/accuracies": 0.8600000143051147,
+      "rewards/chosen": -1.546875,
+      "rewards/margins": 1.25,
+      "rewards/rejected": -2.796875,
+      "step": 3790
+    },
+    {
+      "epoch": 7.458292443572129,
+      "grad_norm": 11.04997051979466,
+      "learning_rate": 9.16129937035619e-08,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.0009002685546875,
+      "logps/rejected": -588.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.0009307861328125,
+      "loss": 0.3713,
+      "rewards/accuracies": 0.8649999499320984,
+      "rewards/chosen": -1.5,
+      "rewards/margins": 1.2265625,
+      "rewards/rejected": -2.734375,
+      "step": 3800
+    },
+    {
+      "epoch": 7.458292443572129,
+      "eval_logits/chosen": -1.3984375,
+      "eval_logits/rejected": -1.4296875,
+      "eval_logps/chosen": -572.0,
+      "eval_logps/chosen_bottom_tokens": -13.9375,
+      "eval_logps/chosen_top_tokens": -0.00089263916015625,
+      "eval_logps/rejected": -592.0,
+      "eval_logps/rejected_bottom_tokens": -13.8125,
+      "eval_logps/rejected_top_tokens": -0.000858306884765625,
+      "eval_loss": 0.5956262350082397,
+      "eval_rewards/accuracies": 0.7164179086685181,
+      "eval_rewards/chosen": -1.7734375,
+      "eval_rewards/margins": 0.75,
+      "eval_rewards/rejected": -2.53125,
+      "eval_runtime": 108.1243,
+      "eval_samples_per_second": 18.497,
+      "eval_steps_per_second": 0.62,
+      "step": 3800
+    },
+    {
+      "epoch": 7.4779195289499505,
+      "grad_norm": 13.067662208055344,
+      "learning_rate": 9.029023829969102e-08,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -568.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000896453857421875,
+      "logps/rejected": -624.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.0008544921875,
+      "loss": 0.3714,
+      "rewards/accuracies": 0.8649999499320984,
+      "rewards/chosen": -1.5390625,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -2.84375,
+      "step": 3810
+    },
+    {
+      "epoch": 7.4975466143277725,
+      "grad_norm": 12.33759924091816,
+      "learning_rate": 8.897499408882206e-08,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -552.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000850677490234375,
+      "logps/rejected": -588.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000820159912109375,
+      "loss": 0.3629,
+      "rewards/accuracies": 0.8399999737739563,
+      "rewards/chosen": -1.5859375,
+      "rewards/margins": 1.1328125,
+      "rewards/rejected": -2.71875,
+      "step": 3820
+    },
+    {
+      "epoch": 7.517173699705594,
+      "grad_norm": 13.798167173719602,
+      "learning_rate": 8.766732292724377e-08,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -540.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00081634521484375,
+      "logps/rejected": -636.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.0008087158203125,
+      "loss": 0.3653,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.6875,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -2.9375,
+      "step": 3830
+    },
+    {
+      "epoch": 7.536800785083415,
+      "grad_norm": 12.333957733743242,
+      "learning_rate": 8.636728631508198e-08,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -560.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000919342041015625,
+      "logps/rejected": -596.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000896453857421875,
+      "loss": 0.3772,
+      "rewards/accuracies": 0.8200000524520874,
+      "rewards/chosen": -1.7421875,
+      "rewards/margins": 1.1484375,
+      "rewards/rejected": -2.890625,
+      "step": 3840
+    },
+    {
+      "epoch": 7.556427870461237,
+      "grad_norm": 11.995779984555064,
+      "learning_rate": 8.507494539340804e-08,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -572.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000888824462890625,
+      "logps/rejected": -608.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.0008544921875,
+      "loss": 0.3747,
+      "rewards/accuracies": 0.8799999952316284,
+      "rewards/chosen": -1.5625,
+      "rewards/margins": 1.234375,
+      "rewards/rejected": -2.796875,
+      "step": 3850
+    },
+    {
+      "epoch": 7.576054955839058,
+      "grad_norm": 10.860318387319781,
+      "learning_rate": 8.379036094136263e-08,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -536.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.000896453857421875,
+      "logps/rejected": -612.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.000885009765625,
+      "loss": 0.3893,
+      "rewards/accuracies": 0.8849999308586121,
+      "rewards/chosen": -1.5546875,
+      "rewards/margins": 1.21875,
+      "rewards/rejected": -2.765625,
+      "step": 3860
+    },
+    {
+      "epoch": 7.595682041216879,
+      "grad_norm": 9.943400640186944,
+      "learning_rate": 8.251359337329764e-08,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -508.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.00090789794921875,
+      "logps/rejected": -588.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000911712646484375,
+      "loss": 0.3739,
+      "rewards/accuracies": 0.8400000333786011,
+      "rewards/chosen": -1.6015625,
+      "rewards/margins": 1.046875,
+      "rewards/rejected": -2.640625,
+      "step": 3870
+    },
+    {
+      "epoch": 7.615309126594701,
+      "grad_norm": 10.803806404493619,
+      "learning_rate": 8.124470273593476e-08,
+      "logits/chosen": -1.4296875,
+      "logits/rejected": -1.4453125,
+      "logps/chosen": -572.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00081634521484375,
+      "logps/rejected": -656.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000823974609375,
+      "loss": 0.3599,
+      "rewards/accuracies": 0.8799999356269836,
+      "rewards/chosen": -1.7109375,
+      "rewards/margins": 1.3671875,
+      "rewards/rejected": -3.078125,
+      "step": 3880
+    },
+    {
+      "epoch": 7.634936211972522,
+      "grad_norm": 11.581859633592725,
+      "learning_rate": 7.998374870554173e-08,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -520.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000926971435546875,
+      "logps/rejected": -608.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00093841552734375,
+      "loss": 0.371,
+      "rewards/accuracies": 0.8700000643730164,
+      "rewards/chosen": -1.59375,
+      "rewards/margins": 1.1953125,
+      "rewards/rejected": -2.796875,
+      "step": 3890
+    },
+    {
+      "epoch": 7.654563297350343,
+      "grad_norm": 11.434796576304434,
+      "learning_rate": 7.873079058512522e-08,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -552.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.0009307861328125,
+      "logps/rejected": -604.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.0009307861328125,
+      "loss": 0.381,
+      "rewards/accuracies": 0.8550000190734863,
+      "rewards/chosen": -1.6875,
+      "rewards/margins": 1.078125,
+      "rewards/rejected": -2.765625,
+      "step": 3900
+    },
+    {
+      "epoch": 7.654563297350343,
+      "eval_logits/chosen": -1.40625,
+      "eval_logits/rejected": -1.4375,
+      "eval_logps/chosen": -580.0,
+      "eval_logps/chosen_bottom_tokens": -13.9375,
+      "eval_logps/chosen_top_tokens": -0.0008697509765625,
+      "eval_logps/rejected": -600.0,
+      "eval_logps/rejected_bottom_tokens": -13.8125,
+      "eval_logps/rejected_top_tokens": -0.000843048095703125,
+      "eval_loss": 0.5948278903961182,
+      "eval_rewards/accuracies": 0.7164179086685181,
+      "eval_rewards/chosen": -1.8671875,
+      "eval_rewards/margins": 0.76953125,
+      "eval_rewards/rejected": -2.625,
+      "eval_runtime": 107.9543,
+      "eval_samples_per_second": 18.526,
+      "eval_steps_per_second": 0.621,
+      "step": 3900
+    },
+    {
+      "epoch": 7.674190382728165,
+      "grad_norm": 10.872331131797628,
+      "learning_rate": 7.74858873016424e-08,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -532.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000766754150390625,
+      "logps/rejected": -624.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000751495361328125,
+      "loss": 0.3685,
+      "rewards/accuracies": 0.8649999499320984,
+      "rewards/chosen": -1.625,
+      "rewards/margins": 1.328125,
+      "rewards/rejected": -2.953125,
+      "step": 3910
+    },
+    {
+      "epoch": 7.693817468105986,
+      "grad_norm": 9.60822682942007,
+      "learning_rate": 7.624909740322905e-08,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4453125,
+      "logps/chosen": -536.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000850677490234375,
+      "logps/rejected": -608.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000835418701171875,
+      "loss": 0.3489,
+      "rewards/accuracies": 0.8899999856948853,
+      "rewards/chosen": -1.640625,
+      "rewards/margins": 1.3359375,
+      "rewards/rejected": -2.984375,
+      "step": 3920
+    },
+    {
+      "epoch": 7.713444553483807,
+      "grad_norm": 11.059899700049831,
+      "learning_rate": 7.502047905644651e-08,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -520.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000858306884765625,
+      "logps/rejected": -608.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000843048095703125,
+      "loss": 0.3623,
+      "rewards/accuracies": 0.8850000500679016,
+      "rewards/chosen": -1.5234375,
+      "rewards/margins": 1.265625,
+      "rewards/rejected": -2.78125,
+      "step": 3930
+    },
+    {
+      "epoch": 7.733071638861629,
+      "grad_norm": 12.140940017863466,
+      "learning_rate": 7.380009004354559e-08,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -552.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000881195068359375,
+      "logps/rejected": -632.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.00089263916015625,
+      "loss": 0.3738,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.6484375,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -2.953125,
+      "step": 3940
+    },
+    {
+      "epoch": 7.75269872423945,
+      "grad_norm": 10.599955305219144,
+      "learning_rate": 7.258798775974956e-08,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -548.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000850677490234375,
+      "logps/rejected": -636.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000865936279296875,
+      "loss": 0.3768,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.6875,
+      "rewards/margins": 1.15625,
+      "rewards/rejected": -2.84375,
+      "step": 3950
+    },
+    {
+      "epoch": 7.772325809617271,
+      "grad_norm": 12.259829312485664,
+      "learning_rate": 7.138422921055437e-08,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -488.0,
+      "logps/chosen_bottom_tokens": -13.75,
+      "logps/chosen_top_tokens": -0.000881195068359375,
+      "logps/rejected": -600.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000896453857421875,
+      "loss": 0.3697,
+      "rewards/accuracies": 0.9100000262260437,
+      "rewards/chosen": -1.4375,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -2.78125,
+      "step": 3960
+    },
+    {
+      "epoch": 7.791952894995093,
+      "grad_norm": 11.673587497917335,
+      "learning_rate": 7.018887100904813e-08,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -510.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000820159912109375,
+      "logps/rejected": -608.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00081634521484375,
+      "loss": 0.3654,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.5,
+      "rewards/margins": 1.1796875,
+      "rewards/rejected": -2.671875,
+      "step": 3970
+    },
+    {
+      "epoch": 7.8115799803729145,
+      "grad_norm": 11.23626669760548,
+      "learning_rate": 6.900196937324814e-08,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.0008392333984375,
+      "logps/rejected": -628.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.0008392333984375,
+      "loss": 0.3572,
+      "rewards/accuracies": 0.8850000500679016,
+      "rewards/chosen": -1.5234375,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -2.796875,
+      "step": 3980
+    },
+    {
+      "epoch": 7.8312070657507356,
+      "grad_norm": 10.633921984388468,
+      "learning_rate": 6.782358012345715e-08,
+      "logits/chosen": -1.4296875,
+      "logits/rejected": -1.453125,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -13.75,
+      "logps/chosen_top_tokens": -0.000865936279296875,
+      "logps/rejected": -620.0,
+      "logps/rejected_bottom_tokens": -13.625,
+      "logps/rejected_top_tokens": -0.000835418701171875,
+      "loss": 0.3668,
+      "rewards/accuracies": 0.8949999809265137,
+      "rewards/chosen": -1.59375,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -2.859375,
+      "step": 3990
+    },
+    {
+      "epoch": 7.8508341511285575,
+      "grad_norm": 10.65643493105821,
+      "learning_rate": 6.665375867963832e-08,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.0009918212890625,
+      "logps/rejected": -612.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.0009613037109375,
+      "loss": 0.3639,
+      "rewards/accuracies": 0.8799999952316284,
+      "rewards/chosen": -1.6640625,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -2.921875,
+      "step": 4000
+    },
+    {
+      "epoch": 7.8508341511285575,
+      "eval_logits/chosen": -1.40625,
+      "eval_logits/rejected": -1.4375,
+      "eval_logps/chosen": -580.0,
+      "eval_logps/chosen_bottom_tokens": -13.9375,
+      "eval_logps/chosen_top_tokens": -0.00090789794921875,
+      "eval_logps/rejected": -600.0,
+      "eval_logps/rejected_bottom_tokens": -13.8125,
+      "eval_logps/rejected_top_tokens": -0.0008697509765625,
+      "eval_loss": 0.5949731469154358,
+      "eval_rewards/accuracies": 0.7194030284881592,
+      "eval_rewards/chosen": -1.8671875,
+      "eval_rewards/margins": 0.7578125,
+      "eval_rewards/rejected": -2.625,
+      "eval_runtime": 108.0285,
+      "eval_samples_per_second": 18.514,
+      "eval_steps_per_second": 0.62,
+      "step": 4000
+    },
+    {
+      "epoch": 7.870461236506379,
+      "grad_norm": 15.401979464151776,
+      "learning_rate": 6.549256005880829e-08,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4453125,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.0009613037109375,
+      "logps/rejected": -608.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000942230224609375,
+      "loss": 0.3913,
+      "rewards/accuracies": 0.8700000047683716,
+      "rewards/chosen": -1.75,
+      "rewards/margins": 1.203125,
+      "rewards/rejected": -2.953125,
+      "step": 4010
+    },
+    {
+      "epoch": 7.8900883218842,
+      "grad_norm": 12.728403003073126,
+      "learning_rate": 6.434003887245035e-08,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -576.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.0009002685546875,
+      "logps/rejected": -612.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000885009765625,
+      "loss": 0.3874,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.71875,
+      "rewards/margins": 1.015625,
+      "rewards/rejected": -2.734375,
+      "step": 4020
+    },
+    {
+      "epoch": 7.909715407262022,
+      "grad_norm": 10.70067975910462,
+      "learning_rate": 6.319624932394538e-08,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -552.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.0008544921875,
+      "logps/rejected": -584.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.0008392333984375,
+      "loss": 0.3847,
+      "rewards/accuracies": 0.8649999499320984,
+      "rewards/chosen": -1.625,
+      "rewards/margins": 1.0546875,
+      "rewards/rejected": -2.6875,
+      "step": 4030
+    },
+    {
+      "epoch": 7.929342492639843,
+      "grad_norm": 13.268929321468441,
+      "learning_rate": 6.206124520602327e-08,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -564.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000881195068359375,
+      "logps/rejected": -600.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000835418701171875,
+      "loss": 0.3719,
+      "rewards/accuracies": 0.8199999928474426,
+      "rewards/chosen": -1.6015625,
+      "rewards/margins": 1.1875,
+      "rewards/rejected": -2.78125,
+      "step": 4040
+    },
+    {
+      "epoch": 7.948969578017664,
+      "grad_norm": 16.223576138264168,
+      "learning_rate": 6.093507989823252e-08,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -536.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00080108642578125,
+      "logps/rejected": -628.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000804901123046875,
+      "loss": 0.3576,
+      "rewards/accuracies": 0.85999995470047,
+      "rewards/chosen": -1.578125,
+      "rewards/margins": 1.328125,
+      "rewards/rejected": -2.90625,
+      "step": 4050
+    },
+    {
+      "epoch": 7.968596663395486,
+      "grad_norm": 10.20306343751008,
+      "learning_rate": 5.981780636443026e-08,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000850677490234375,
+      "logps/rejected": -612.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000835418701171875,
+      "loss": 0.3692,
+      "rewards/accuracies": 0.8700000047683716,
+      "rewards/chosen": -1.546875,
+      "rewards/margins": 1.2421875,
+      "rewards/rejected": -2.796875,
+      "step": 4060
+    },
+    {
+      "epoch": 7.988223748773307,
+      "grad_norm": 11.559562412684313,
+      "learning_rate": 5.8709477150290786e-08,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -13.75,
+      "logps/chosen_top_tokens": -0.00086212158203125,
+      "logps/rejected": -600.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.000873565673828125,
+      "loss": 0.3661,
+      "rewards/accuracies": 0.8449999690055847,
+      "rewards/chosen": -1.640625,
+      "rewards/margins": 1.0546875,
+      "rewards/rejected": -2.6875,
+      "step": 4070
+    },
+    {
+      "epoch": 8.007850834151128,
+      "grad_norm": 10.95225695720274,
+      "learning_rate": 5.7610144380835003e-08,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -576.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000865936279296875,
+      "logps/rejected": -644.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.0008544921875,
+      "loss": 0.3756,
+      "rewards/accuracies": 0.869999885559082,
+      "rewards/chosen": -1.65625,
+      "rewards/margins": 1.078125,
+      "rewards/rejected": -2.734375,
+      "step": 4080
+    },
+    {
+      "epoch": 8.02747791952895,
+      "grad_norm": 10.92812643521563,
+      "learning_rate": 5.651985975797832e-08,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -540.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00087738037109375,
+      "logps/rejected": -612.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000881195068359375,
+      "loss": 0.3575,
+      "rewards/accuracies": 0.8450000882148743,
+      "rewards/chosen": -1.59375,
+      "rewards/margins": 1.171875,
+      "rewards/rejected": -2.765625,
+      "step": 4090
+    },
+    {
+      "epoch": 8.047105004906772,
+      "grad_norm": 13.88801874596346,
+      "learning_rate": 5.543867455809942e-08,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000881195068359375,
+      "logps/rejected": -632.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.0008697509765625,
+      "loss": 0.3563,
+      "rewards/accuracies": 0.8899999856948853,
+      "rewards/chosen": -1.546875,
+      "rewards/margins": 1.4453125,
+      "rewards/rejected": -2.984375,
+      "step": 4100
+    },
+    {
+      "epoch": 8.047105004906772,
+      "eval_logits/chosen": -1.3984375,
+      "eval_logits/rejected": -1.4296875,
+      "eval_logps/chosen": -576.0,
+      "eval_logps/chosen_bottom_tokens": -13.9375,
+      "eval_logps/chosen_top_tokens": -0.000885009765625,
+      "eval_logps/rejected": -596.0,
+      "eval_logps/rejected_bottom_tokens": -13.8125,
+      "eval_logps/rejected_top_tokens": -0.000850677490234375,
+      "eval_loss": 0.5938891768455505,
+      "eval_rewards/accuracies": 0.707462728023529,
+      "eval_rewards/chosen": -1.828125,
+      "eval_rewards/margins": 0.75390625,
+      "eval_rewards/rejected": -2.578125,
+      "eval_runtime": 108.0792,
+      "eval_samples_per_second": 18.505,
+      "eval_steps_per_second": 0.62,
+      "step": 4100
+    },
+    {
+      "epoch": 8.066732090284592,
+      "grad_norm": 11.318833746867718,
+      "learning_rate": 5.436663962962884e-08,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -568.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00079345703125,
+      "logps/rejected": -612.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.00078582763671875,
+      "loss": 0.3514,
+      "rewards/accuracies": 0.8700000047683716,
+      "rewards/chosen": -1.59375,
+      "rewards/margins": 1.1796875,
+      "rewards/rejected": -2.765625,
+      "step": 4110
+    },
+    {
+      "epoch": 8.086359175662414,
+      "grad_norm": 13.189282134725309,
+      "learning_rate": 5.330380539065718e-08,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -532.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000904083251953125,
+      "logps/rejected": -620.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000911712646484375,
+      "loss": 0.3489,
+      "rewards/accuracies": 0.8799999952316284,
+      "rewards/chosen": -1.625,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -2.96875,
+      "step": 4120
+    },
+    {
+      "epoch": 8.105986261040236,
+      "grad_norm": 12.834326838400363,
+      "learning_rate": 5.2250221826564385e-08,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -556.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00092315673828125,
+      "logps/rejected": -612.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000888824462890625,
+      "loss": 0.355,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.6015625,
+      "rewards/margins": 1.203125,
+      "rewards/rejected": -2.796875,
+      "step": 4130
+    },
+    {
+      "epoch": 8.125613346418056,
+      "grad_norm": 14.809258900222034,
+      "learning_rate": 5.120593848766841e-08,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -502.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.000885009765625,
+      "logps/rejected": -628.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000919342041015625,
+      "loss": 0.3752,
+      "rewards/accuracies": 0.8450000882148743,
+      "rewards/chosen": -1.625,
+      "rewards/margins": 1.2421875,
+      "rewards/rejected": -2.859375,
+      "step": 4140
+    },
+    {
+      "epoch": 8.145240431795878,
+      "grad_norm": 10.507531827005831,
+      "learning_rate": 5.017100448689538e-08,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -520.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.000904083251953125,
+      "logps/rejected": -612.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.0009307861328125,
+      "loss": 0.344,
+      "rewards/accuracies": 0.8649999499320984,
+      "rewards/chosen": -1.59375,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -2.859375,
+      "step": 4150
+    },
+    {
+      "epoch": 8.1648675171737,
+      "grad_norm": 12.259586907707716,
+      "learning_rate": 4.914546849746923e-08,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -1.4453125,
+      "logps/chosen": -548.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000812530517578125,
+      "logps/rejected": -612.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000823974609375,
+      "loss": 0.3529,
+      "rewards/accuracies": 0.8450000882148743,
+      "rewards/chosen": -1.5859375,
+      "rewards/margins": 1.25,
+      "rewards/rejected": -2.84375,
+      "step": 4160
+    },
+    {
+      "epoch": 8.18449460255152,
+      "grad_norm": 12.329389242214374,
+      "learning_rate": 4.812937875062328e-08,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -540.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000896453857421875,
+      "logps/rejected": -592.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000904083251953125,
+      "loss": 0.3521,
+      "rewards/accuracies": 0.8700000643730164,
+      "rewards/chosen": -1.59375,
+      "rewards/margins": 1.21875,
+      "rewards/rejected": -2.8125,
+      "step": 4170
+    },
+    {
+      "epoch": 8.204121687929343,
+      "grad_norm": 11.291299154109046,
+      "learning_rate": 4.71227830333312e-08,
+      "logits/chosen": -1.4296875,
+      "logits/rejected": -1.453125,
+      "logps/chosen": -536.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.0008087158203125,
+      "logps/rejected": -604.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.00080108642578125,
+      "loss": 0.3418,
+      "rewards/accuracies": 0.8899999856948853,
+      "rewards/chosen": -1.6640625,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": -2.984375,
+      "step": 4180
+    },
+    {
+      "epoch": 8.223748773307165,
+      "grad_norm": 10.114702294210609,
+      "learning_rate": 4.612572868605999e-08,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -540.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000812530517578125,
+      "logps/rejected": -604.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000797271728515625,
+      "loss": 0.3527,
+      "rewards/accuracies": 0.8649999499320984,
+      "rewards/chosen": -1.75,
+      "rewards/margins": 1.171875,
+      "rewards/rejected": -2.921875,
+      "step": 4190
+    },
+    {
+      "epoch": 8.243375858684985,
+      "grad_norm": 11.685557169535775,
+      "learning_rate": 4.513826260054357e-08,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -520.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.000926971435546875,
+      "logps/rejected": -616.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.0009307861328125,
+      "loss": 0.3484,
+      "rewards/accuracies": 0.8949999809265137,
+      "rewards/chosen": -1.6171875,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -2.96875,
+      "step": 4200
+    },
+    {
+      "epoch": 8.243375858684985,
+      "eval_logits/chosen": -1.40625,
+      "eval_logits/rejected": -1.4375,
+      "eval_logps/chosen": -580.0,
+      "eval_logps/chosen_bottom_tokens": -14.0,
+      "eval_logps/chosen_top_tokens": -0.000888824462890625,
+      "eval_logps/rejected": -600.0,
+      "eval_logps/rejected_bottom_tokens": -13.875,
+      "eval_logps/rejected_top_tokens": -0.00084686279296875,
+      "eval_loss": 0.5968652367591858,
+      "eval_rewards/accuracies": 0.7044776082038879,
+      "eval_rewards/chosen": -1.875,
+      "eval_rewards/margins": 0.765625,
+      "eval_rewards/rejected": -2.640625,
+      "eval_runtime": 107.9539,
+      "eval_samples_per_second": 18.526,
+      "eval_steps_per_second": 0.621,
+      "step": 4200
+    },
+    {
+      "epoch": 8.263002944062807,
+      "grad_norm": 11.223061664314738,
+      "learning_rate": 4.416043121757715e-08,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -580.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00083160400390625,
+      "logps/rejected": -616.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.000835418701171875,
+      "loss": 0.3615,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.734375,
+      "rewards/margins": 1.1953125,
+      "rewards/rejected": -2.921875,
+      "step": 4210
+    },
+    {
+      "epoch": 8.282630029440629,
+      "grad_norm": 13.979364868218873,
+      "learning_rate": 4.319228052483348e-08,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -1.4375,
+      "logps/chosen": -510.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.000812530517578125,
+      "logps/rejected": -580.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00079345703125,
+      "loss": 0.3312,
+      "rewards/accuracies": 0.8649999499320984,
+      "rewards/chosen": -1.421875,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -2.71875,
+      "step": 4220
+    },
+    {
+      "epoch": 8.302257114818449,
+      "grad_norm": 13.716554098357632,
+      "learning_rate": 4.223385605469962e-08,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000885009765625,
+      "logps/rejected": -636.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.0008544921875,
+      "loss": 0.3492,
+      "rewards/accuracies": 0.8949999809265137,
+      "rewards/chosen": -1.5625,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": -2.890625,
+      "step": 4230
+    },
+    {
+      "epoch": 8.321884200196271,
+      "grad_norm": 12.209448291726472,
+      "learning_rate": 4.128520288213608e-08,
+      "logits/chosen": -1.4375,
+      "logits/rejected": -1.453125,
+      "logps/chosen": -548.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000934600830078125,
+      "logps/rejected": -620.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.00093841552734375,
+      "loss": 0.3511,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.7578125,
+      "rewards/margins": 1.2265625,
+      "rewards/rejected": -2.984375,
+      "step": 4240
+    },
+    {
+      "epoch": 8.341511285574093,
+      "grad_norm": 15.422956351720499,
+      "learning_rate": 4.0346365622556276e-08,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00083160400390625,
+      "logps/rejected": -616.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00081634521484375,
+      "loss": 0.3502,
+      "rewards/accuracies": 0.8700000047683716,
+      "rewards/chosen": -1.5625,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -2.828125,
+      "step": 4250
+    },
+    {
+      "epoch": 8.361138370951913,
+      "grad_norm": 10.987525709594365,
+      "learning_rate": 3.9417388429728884e-08,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.000926971435546875,
+      "logps/rejected": -620.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.00092315673828125,
+      "loss": 0.3449,
+      "rewards/accuracies": 0.8649999499320984,
+      "rewards/chosen": -1.6171875,
+      "rewards/margins": 1.375,
+      "rewards/rejected": -2.984375,
+      "step": 4260
+    },
+    {
+      "epoch": 8.380765456329735,
+      "grad_norm": 13.277350901991092,
+      "learning_rate": 3.8498314993700764e-08,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -532.0,
+      "logps/chosen_bottom_tokens": -13.75,
+      "logps/chosen_top_tokens": -0.000957489013671875,
+      "logps/rejected": -616.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.000946044921875,
+      "loss": 0.3563,
+      "rewards/accuracies": 0.8700000047683716,
+      "rewards/chosen": -1.640625,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -2.9375,
+      "step": 4270
+    },
+    {
+      "epoch": 8.400392541707557,
+      "grad_norm": 11.773506804599066,
+      "learning_rate": 3.7589188538742654e-08,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -572.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00079345703125,
+      "logps/rejected": -628.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000804901123046875,
+      "loss": 0.3633,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.625,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -2.96875,
+      "step": 4280
+    },
+    {
+      "epoch": 8.420019627085377,
+      "grad_norm": 12.35899932980167,
+      "learning_rate": 3.66900518213159e-08,
+      "logits/chosen": -1.4296875,
+      "logits/rejected": -1.453125,
+      "logps/chosen": -576.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000843048095703125,
+      "logps/rejected": -648.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.0009765625,
+      "loss": 0.3589,
+      "rewards/accuracies": 0.9050000309944153,
+      "rewards/chosen": -1.6328125,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -2.890625,
+      "step": 4290
+    },
+    {
+      "epoch": 8.4396467124632,
+      "grad_norm": 10.566216182379264,
+      "learning_rate": 3.580094712806195e-08,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -532.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.00090789794921875,
+      "logps/rejected": -636.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000919342041015625,
+      "loss": 0.3359,
+      "rewards/accuracies": 0.8749998807907104,
+      "rewards/chosen": -1.609375,
+      "rewards/margins": 1.3828125,
+      "rewards/rejected": -2.984375,
+      "step": 4300
+    },
+    {
+      "epoch": 8.4396467124632,
+      "eval_logits/chosen": -1.40625,
+      "eval_logits/rejected": -1.4375,
+      "eval_logps/chosen": -580.0,
+      "eval_logps/chosen_bottom_tokens": -13.9375,
+      "eval_logps/chosen_top_tokens": -0.00087738037109375,
+      "eval_logps/rejected": -604.0,
+      "eval_logps/rejected_bottom_tokens": -13.8125,
+      "eval_logps/rejected_top_tokens": -0.00086212158203125,
+      "eval_loss": 0.5965551733970642,
+      "eval_rewards/accuracies": 0.7044776082038879,
+      "eval_rewards/chosen": -1.8828125,
+      "eval_rewards/margins": 0.7734375,
+      "eval_rewards/rejected": -2.65625,
+      "eval_runtime": 108.2337,
+      "eval_samples_per_second": 18.479,
+      "eval_steps_per_second": 0.619,
+      "step": 4300
+    },
+    {
+      "epoch": 8.459273797841021,
+      "grad_norm": 12.376710994660526,
+      "learning_rate": 3.49219162738133e-08,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00095367431640625,
+      "logps/rejected": -576.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000926971435546875,
+      "loss": 0.371,
+      "rewards/accuracies": 0.8200000524520874,
+      "rewards/chosen": -1.6171875,
+      "rewards/margins": 1.046875,
+      "rewards/rejected": -2.671875,
+      "step": 4310
+    },
+    {
+      "epoch": 8.478900883218841,
+      "grad_norm": 10.141612269870464,
+      "learning_rate": 3.405300059962729e-08,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -576.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00087738037109375,
+      "logps/rejected": -660.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000850677490234375,
+      "loss": 0.3598,
+      "rewards/accuracies": 0.8149999380111694,
+      "rewards/chosen": -1.859375,
+      "rewards/margins": 1.125,
+      "rewards/rejected": -2.984375,
+      "step": 4320
+    },
+    {
+      "epoch": 8.498527968596663,
+      "grad_norm": 13.581768077877944,
+      "learning_rate": 3.319424097084153e-08,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -568.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.0009002685546875,
+      "logps/rejected": -676.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000942230224609375,
+      "loss": 0.3476,
+      "rewards/accuracies": 0.8850000500679016,
+      "rewards/chosen": -1.65625,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -2.9375,
+      "step": 4330
+    },
+    {
+      "epoch": 8.518155053974485,
+      "grad_norm": 10.680900604075344,
+      "learning_rate": 3.2345677775152014e-08,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -520.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00089263916015625,
+      "logps/rejected": -592.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.0008697509765625,
+      "loss": 0.3527,
+      "rewards/accuracies": 0.8700000047683716,
+      "rewards/chosen": -1.671875,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -2.96875,
+      "step": 4340
+    },
+    {
+      "epoch": 8.537782139352306,
+      "grad_norm": 11.197449979240112,
+      "learning_rate": 3.150735092071391e-08,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -540.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.0008544921875,
+      "logps/rejected": -592.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000850677490234375,
+      "loss": 0.3679,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.6796875,
+      "rewards/margins": 1.1171875,
+      "rewards/rejected": -2.796875,
+      "step": 4350
+    },
+    {
+      "epoch": 8.557409224730128,
+      "grad_norm": 11.967712785654316,
+      "learning_rate": 3.067929983426434e-08,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -580.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.000873565673828125,
+      "logps/rejected": -636.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000843048095703125,
+      "loss": 0.3536,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.7578125,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -3.015625,
+      "step": 4360
+    },
+    {
+      "epoch": 8.57703631010795,
+      "grad_norm": 11.095146468888284,
+      "learning_rate": 2.9861563459268433e-08,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.0008697509765625,
+      "logps/rejected": -624.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.000873565673828125,
+      "loss": 0.355,
+      "rewards/accuracies": 0.8650000691413879,
+      "rewards/chosen": -1.6328125,
+      "rewards/margins": 1.3515625,
+      "rewards/rejected": -2.984375,
+      "step": 4370
+    },
+    {
+      "epoch": 8.59666339548577,
+      "grad_norm": 13.206766651368518,
+      "learning_rate": 2.9054180254087512e-08,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -580.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.0008087158203125,
+      "logps/rejected": -656.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.000820159912109375,
+      "loss": 0.365,
+      "rewards/accuracies": 0.8650000691413879,
+      "rewards/chosen": -1.6875,
+      "rewards/margins": 1.1953125,
+      "rewards/rejected": -2.875,
+      "step": 4380
+    },
+    {
+      "epoch": 8.616290480863592,
+      "grad_norm": 9.29695813066069,
+      "learning_rate": 2.8257188190170785e-08,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -608.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000789642333984375,
+      "logps/rejected": -688.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000827789306640625,
+      "loss": 0.3489,
+      "rewards/accuracies": 0.8899999856948853,
+      "rewards/chosen": -1.671875,
+      "rewards/margins": 1.2265625,
+      "rewards/rejected": -2.90625,
+      "step": 4390
+    },
+    {
+      "epoch": 8.635917566241414,
+      "grad_norm": 13.112622948489534,
+      "learning_rate": 2.7470624750269013e-08,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.000827789306640625,
+      "logps/rejected": -620.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.000823974609375,
+      "loss": 0.3639,
+      "rewards/accuracies": 0.8550000190734863,
+      "rewards/chosen": -1.578125,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -2.875,
+      "step": 4400
+    },
+    {
+      "epoch": 8.635917566241414,
+      "eval_logits/chosen": -1.3984375,
+      "eval_logits/rejected": -1.4296875,
+      "eval_logps/chosen": -580.0,
+      "eval_logps/chosen_bottom_tokens": -13.9375,
+      "eval_logps/chosen_top_tokens": -0.0009002685546875,
+      "eval_logps/rejected": -596.0,
+      "eval_logps/rejected_bottom_tokens": -13.8125,
+      "eval_logps/rejected_top_tokens": -0.0008697509765625,
+      "eval_loss": 0.5978894233703613,
+      "eval_rewards/accuracies": 0.707462728023529,
+      "eval_rewards/chosen": -1.8515625,
+      "eval_rewards/margins": 0.74609375,
+      "eval_rewards/rejected": -2.59375,
+      "eval_runtime": 108.1117,
+      "eval_samples_per_second": 18.499,
+      "eval_steps_per_second": 0.62,
+      "step": 4400
+    },
+    {
+      "epoch": 8.655544651619234,
+      "grad_norm": 14.045254103637847,
+      "learning_rate": 2.669452692667218e-08,
+      "logits/chosen": -1.453125,
+      "logits/rejected": -1.46875,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.0008087158203125,
+      "logps/rejected": -628.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.00081634521484375,
+      "loss": 0.3671,
+      "rewards/accuracies": 0.8400000333786011,
+      "rewards/chosen": -1.6328125,
+      "rewards/margins": 1.09375,
+      "rewards/rejected": -2.734375,
+      "step": 4410
+    },
+    {
+      "epoch": 8.675171736997056,
+      "grad_norm": 13.183615803977235,
+      "learning_rate": 2.5928931219469348e-08,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000858306884765625,
+      "logps/rejected": -612.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000835418701171875,
+      "loss": 0.3569,
+      "rewards/accuracies": 0.8700000643730164,
+      "rewards/chosen": -1.5390625,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -2.84375,
+      "step": 4420
+    },
+    {
+      "epoch": 8.694798822374878,
+      "grad_norm": 11.9847649039614,
+      "learning_rate": 2.517387363483242e-08,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -556.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00083160400390625,
+      "logps/rejected": -668.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.00110626220703125,
+      "loss": 0.3644,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.5078125,
+      "rewards/margins": 1.4375,
+      "rewards/rejected": -2.9375,
+      "step": 4430
+    },
+    {
+      "epoch": 8.714425907752698,
+      "grad_norm": 11.758750789803823,
+      "learning_rate": 2.4429389683322394e-08,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -506.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.00090789794921875,
+      "logps/rejected": -628.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000942230224609375,
+      "loss": 0.347,
+      "rewards/accuracies": 0.9050000309944153,
+      "rewards/chosen": -1.53125,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -2.984375,
+      "step": 4440
+    },
+    {
+      "epoch": 8.73405299313052,
+      "grad_norm": 12.063528763672434,
+      "learning_rate": 2.3695514378219593e-08,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -584.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00092315673828125,
+      "logps/rejected": -628.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000888824462890625,
+      "loss": 0.3368,
+      "rewards/accuracies": 0.8799999952316284,
+      "rewards/chosen": -1.5546875,
+      "rewards/margins": 1.328125,
+      "rewards/rejected": -2.890625,
+      "step": 4450
+    },
+    {
+      "epoch": 8.753680078508342,
+      "grad_norm": 12.302966216794538,
+      "learning_rate": 2.297228223387673e-08,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -500.0,
+      "logps/chosen_bottom_tokens": -13.75,
+      "logps/chosen_top_tokens": -0.00089263916015625,
+      "logps/rejected": -608.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.00091552734375,
+      "loss": 0.3624,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.59375,
+      "rewards/margins": 1.328125,
+      "rewards/rejected": -2.921875,
+      "step": 4460
+    },
+    {
+      "epoch": 8.773307163886162,
+      "grad_norm": 12.416744994343894,
+      "learning_rate": 2.225972726409586e-08,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.375,
+      "logps/chosen": -540.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.001007080078125,
+      "logps/rejected": -620.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.001007080078125,
+      "loss": 0.3627,
+      "rewards/accuracies": 0.8399999737739563,
+      "rewards/chosen": -1.640625,
+      "rewards/margins": 1.2109375,
+      "rewards/rejected": -2.84375,
+      "step": 4470
+    },
+    {
+      "epoch": 8.792934249263984,
+      "grad_norm": 10.542790739232196,
+      "learning_rate": 2.155788298052874e-08,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -588.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00086212158203125,
+      "logps/rejected": -600.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.0008087158203125,
+      "loss": 0.3515,
+      "rewards/accuracies": 0.85999995470047,
+      "rewards/chosen": -1.734375,
+      "rewards/margins": 1.1171875,
+      "rewards/rejected": -2.84375,
+      "step": 4480
+    },
+    {
+      "epoch": 8.812561334641806,
+      "grad_norm": 10.095206706315418,
+      "learning_rate": 2.0866782391100484e-08,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -532.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000827789306640625,
+      "logps/rejected": -648.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.0008392333984375,
+      "loss": 0.3599,
+      "rewards/accuracies": 0.8799999356269836,
+      "rewards/chosen": -1.671875,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -2.96875,
+      "step": 4490
+    },
+    {
+      "epoch": 8.832188420019627,
+      "grad_norm": 10.809244304846091,
+      "learning_rate": 2.0186457998457613e-08,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -552.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.0009002685546875,
+      "logps/rejected": -636.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000881195068359375,
+      "loss": 0.3563,
+      "rewards/accuracies": 0.9199999570846558,
+      "rewards/chosen": -1.5546875,
+      "rewards/margins": 1.40625,
+      "rewards/rejected": -2.953125,
+      "step": 4500
+    },
+    {
+      "epoch": 8.832188420019627,
+      "eval_logits/chosen": -1.3984375,
+      "eval_logits/rejected": -1.4296875,
+      "eval_logps/chosen": -580.0,
+      "eval_logps/chosen_bottom_tokens": -13.9375,
+      "eval_logps/chosen_top_tokens": -0.000904083251953125,
+      "eval_logps/rejected": -600.0,
+      "eval_logps/rejected_bottom_tokens": -13.8125,
+      "eval_logps/rejected_top_tokens": -0.000873565673828125,
+      "eval_loss": 0.5979223847389221,
+      "eval_rewards/accuracies": 0.707462728023529,
+      "eval_rewards/chosen": -1.859375,
+      "eval_rewards/margins": 0.76171875,
+      "eval_rewards/rejected": -2.625,
+      "eval_runtime": 108.3131,
+      "eval_samples_per_second": 18.465,
+      "eval_steps_per_second": 0.619,
+      "step": 4500
+    },
+    {
+      "epoch": 8.851815505397449,
+      "grad_norm": 10.472003628252363,
+      "learning_rate": 1.9516941798438935e-08,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -536.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000858306884765625,
+      "logps/rejected": -592.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.00084686279296875,
+      "loss": 0.3525,
+      "rewards/accuracies": 0.8550001382827759,
+      "rewards/chosen": -1.6796875,
+      "rewards/margins": 1.21875,
+      "rewards/rejected": -2.90625,
+      "step": 4510
+    },
+    {
+      "epoch": 8.87144259077527,
+      "grad_norm": 14.197024509603557,
+      "learning_rate": 1.8858265278571346e-08,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.3828125,
+      "logps/chosen": -512.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.0009918212890625,
+      "logps/rejected": -620.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00104522705078125,
+      "loss": 0.35,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.703125,
+      "rewards/margins": 1.1875,
+      "rewards/rejected": -2.890625,
+      "step": 4520
+    },
+    {
+      "epoch": 8.89106967615309,
+      "grad_norm": 11.730282187199597,
+      "learning_rate": 1.821045941658847e-08,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -568.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00080108642578125,
+      "logps/rejected": -624.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000782012939453125,
+      "loss": 0.3531,
+      "rewards/accuracies": 0.8550000190734863,
+      "rewards/chosen": -1.6875,
+      "rewards/margins": 1.171875,
+      "rewards/rejected": -2.859375,
+      "step": 4530
+    },
+    {
+      "epoch": 8.910696761530913,
+      "grad_norm": 12.183920255550538,
+      "learning_rate": 1.757355467897409e-08,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.0008697509765625,
+      "logps/rejected": -624.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.0008544921875,
+      "loss": 0.3586,
+      "rewards/accuracies": 0.8799999952316284,
+      "rewards/chosen": -1.5859375,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -2.859375,
+      "step": 4540
+    },
+    {
+      "epoch": 8.930323846908735,
+      "grad_norm": 16.902678047586438,
+      "learning_rate": 1.694758101952909e-08,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -500.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00087738037109375,
+      "logps/rejected": -572.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.00083160400390625,
+      "loss": 0.3646,
+      "rewards/accuracies": 0.8450000882148743,
+      "rewards/chosen": -1.5625,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -2.84375,
+      "step": 4550
+    },
+    {
+      "epoch": 8.949950932286555,
+      "grad_norm": 11.768620827284108,
+      "learning_rate": 1.6332567877962887e-08,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -560.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000858306884765625,
+      "logps/rejected": -612.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.0008544921875,
+      "loss": 0.351,
+      "rewards/accuracies": 0.8899999856948853,
+      "rewards/chosen": -1.625,
+      "rewards/margins": 1.3359375,
+      "rewards/rejected": -2.953125,
+      "step": 4560
+    },
+    {
+      "epoch": 8.969578017664377,
+      "grad_norm": 13.962472982347483,
+      "learning_rate": 1.5728544178508744e-08,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.375,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000957489013671875,
+      "logps/rejected": -620.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.0009613037109375,
+      "loss": 0.3648,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.5546875,
+      "rewards/margins": 1.265625,
+      "rewards/rejected": -2.8125,
+      "step": 4570
+    },
+    {
+      "epoch": 8.989205103042199,
+      "grad_norm": 11.885259392354866,
+      "learning_rate": 1.513553832856357e-08,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -564.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.0008697509765625,
+      "logps/rejected": -652.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.00086212158203125,
+      "loss": 0.3796,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.6796875,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -2.984375,
+      "step": 4580
+    },
+    {
+      "epoch": 9.008832188420019,
+      "grad_norm": 13.913153579158871,
+      "learning_rate": 1.455357821735173e-08,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00093841552734375,
+      "logps/rejected": -628.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.0009613037109375,
+      "loss": 0.35,
+      "rewards/accuracies": 0.8949999809265137,
+      "rewards/chosen": -1.5390625,
+      "rewards/margins": 1.40625,
+      "rewards/rejected": -2.953125,
+      "step": 4590
+    },
+    {
+      "epoch": 9.028459273797841,
+      "grad_norm": 15.123594232623052,
+      "learning_rate": 1.3982691214613678e-08,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.0009307861328125,
+      "logps/rejected": -604.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.00092315673828125,
+      "loss": 0.353,
+      "rewards/accuracies": 0.8349999189376831,
+      "rewards/chosen": -1.65625,
+      "rewards/margins": 1.171875,
+      "rewards/rejected": -2.828125,
+      "step": 4600
+    },
+    {
+      "epoch": 9.028459273797841,
+      "eval_logits/chosen": -1.3984375,
+      "eval_logits/rejected": -1.4296875,
+      "eval_logps/chosen": -580.0,
+      "eval_logps/chosen_bottom_tokens": -13.9375,
+      "eval_logps/chosen_top_tokens": -0.00087738037109375,
+      "eval_logps/rejected": -600.0,
+      "eval_logps/rejected_bottom_tokens": -13.8125,
+      "eval_logps/rejected_top_tokens": -0.000843048095703125,
+      "eval_loss": 0.5981213450431824,
+      "eval_rewards/accuracies": 0.6985074877738953,
+      "eval_rewards/chosen": -1.8671875,
+      "eval_rewards/margins": 0.76171875,
+      "eval_rewards/rejected": -2.625,
+      "eval_runtime": 108.308,
+      "eval_samples_per_second": 18.466,
+      "eval_steps_per_second": 0.619,
+      "step": 4600
+    },
+    {
+      "epoch": 9.048086359175663,
+      "grad_norm": 11.670476508295476,
+      "learning_rate": 1.3422904169318544e-08,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -552.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000797271728515625,
+      "logps/rejected": -624.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000797271728515625,
+      "loss": 0.3413,
+      "rewards/accuracies": 0.89000004529953,
+      "rewards/chosen": -1.6171875,
+      "rewards/margins": 1.359375,
+      "rewards/rejected": -2.96875,
+      "step": 4610
+    },
+    {
+      "epoch": 9.067713444553483,
+      "grad_norm": 12.360606000046072,
+      "learning_rate": 1.2874243408401376e-08,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -532.0,
+      "logps/chosen_bottom_tokens": -13.75,
+      "logps/chosen_top_tokens": -0.000873565673828125,
+      "logps/rejected": -604.0,
+      "logps/rejected_bottom_tokens": -13.5625,
+      "logps/rejected_top_tokens": -0.000850677490234375,
+      "loss": 0.3356,
+      "rewards/accuracies": 0.8899999856948853,
+      "rewards/chosen": -1.5,
+      "rewards/margins": 1.28125,
+      "rewards/rejected": -2.78125,
+      "step": 4620
+    },
+    {
+      "epoch": 9.087340529931305,
+      "grad_norm": 12.10726086361965,
+      "learning_rate": 1.2336734735525268e-08,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.453125,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00083160400390625,
+      "logps/rejected": -600.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000820159912109375,
+      "loss": 0.3409,
+      "rewards/accuracies": 0.8949999809265137,
+      "rewards/chosen": -1.5390625,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -2.8125,
+      "step": 4630
+    },
+    {
+      "epoch": 9.106967615309127,
+      "grad_norm": 11.696000638859102,
+      "learning_rate": 1.1810403429867445e-08,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -560.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00092315673828125,
+      "logps/rejected": -640.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000911712646484375,
+      "loss": 0.3481,
+      "rewards/accuracies": 0.8700000047683716,
+      "rewards/chosen": -1.75,
+      "rewards/margins": 1.1640625,
+      "rewards/rejected": -2.921875,
+      "step": 4640
+    },
+    {
+      "epoch": 9.126594700686947,
+      "grad_norm": 11.124831553826729,
+      "learning_rate": 1.129527424493068e-08,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -548.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000896453857421875,
+      "logps/rejected": -632.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.0009002685546875,
+      "loss": 0.3542,
+      "rewards/accuracies": 0.8649999499320984,
+      "rewards/chosen": -1.6640625,
+      "rewards/margins": 1.265625,
+      "rewards/rejected": -2.9375,
+      "step": 4650
+    },
+    {
+      "epoch": 9.14622178606477,
+      "grad_norm": 13.099416868432309,
+      "learning_rate": 1.0791371407378902e-08,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.453125,
+      "logps/chosen": -560.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000896453857421875,
+      "logps/rejected": -596.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.000850677490234375,
+      "loss": 0.3591,
+      "rewards/accuracies": 0.8799999952316284,
+      "rewards/chosen": -1.6484375,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -2.921875,
+      "step": 4660
+    },
+    {
+      "epoch": 9.165848871442591,
+      "grad_norm": 15.476496404936588,
+      "learning_rate": 1.0298718615897983e-08,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -516.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.0009002685546875,
+      "logps/rejected": -604.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000858306884765625,
+      "loss": 0.3592,
+      "rewards/accuracies": 0.8799999952316284,
+      "rewards/chosen": -1.546875,
+      "rewards/margins": 1.328125,
+      "rewards/rejected": -2.875,
+      "step": 4670
+    },
+    {
+      "epoch": 9.185475956820412,
+      "grad_norm": 11.22918157402849,
+      "learning_rate": 9.817339040081002e-09,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -512.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000896453857421875,
+      "logps/rejected": -576.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000888824462890625,
+      "loss": 0.3376,
+      "rewards/accuracies": 0.880000114440918,
+      "rewards/chosen": -1.6015625,
+      "rewards/margins": 1.203125,
+      "rewards/rejected": -2.8125,
+      "step": 4680
+    },
+    {
+      "epoch": 9.205103042198234,
+      "grad_norm": 11.853639781559089,
+      "learning_rate": 9.347255319338804e-09,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.000888824462890625,
+      "logps/rejected": -604.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.0008697509765625,
+      "loss": 0.3483,
+      "rewards/accuracies": 0.9199999570846558,
+      "rewards/chosen": -1.5625,
+      "rewards/margins": 1.3984375,
+      "rewards/rejected": -2.96875,
+      "step": 4690
+    },
+    {
+      "epoch": 9.224730127576056,
+      "grad_norm": 10.9646558915127,
+      "learning_rate": 8.888489561835022e-09,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -532.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.0008697509765625,
+      "logps/rejected": -636.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.000858306884765625,
+      "loss": 0.3514,
+      "rewards/accuracies": 0.8700000047683716,
+      "rewards/chosen": -1.5703125,
+      "rewards/margins": 1.4140625,
+      "rewards/rejected": -2.984375,
+      "step": 4700
+    },
+    {
+      "epoch": 9.224730127576056,
+      "eval_logits/chosen": -1.3984375,
+      "eval_logits/rejected": -1.4296875,
+      "eval_logps/chosen": -580.0,
+      "eval_logps/chosen_bottom_tokens": -13.9375,
+      "eval_logps/chosen_top_tokens": -0.00086212158203125,
+      "eval_logps/rejected": -600.0,
+      "eval_logps/rejected_bottom_tokens": -13.8125,
+      "eval_logps/rejected_top_tokens": -0.000827789306640625,
+      "eval_loss": 0.5979077219963074,
+      "eval_rewards/accuracies": 0.6985074877738953,
+      "eval_rewards/chosen": -1.859375,
+      "eval_rewards/margins": 0.765625,
+      "eval_rewards/rejected": -2.625,
+      "eval_runtime": 108.2657,
+      "eval_samples_per_second": 18.473,
+      "eval_steps_per_second": 0.619,
+      "step": 4700
+    },
+    {
+      "epoch": 9.244357212953876,
+      "grad_norm": 10.122903531296545,
+      "learning_rate": 8.441063343446548e-09,
+      "logits/chosen": -1.4375,
+      "logits/rejected": -1.4453125,
+      "logps/chosen": -482.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.000858306884765625,
+      "logps/rejected": -588.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.00086212158203125,
+      "loss": 0.3486,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.6171875,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -2.890625,
+      "step": 4710
+    },
+    {
+      "epoch": 9.263984298331698,
+      "grad_norm": 11.909843897950127,
+      "learning_rate": 8.00499770674859e-09,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -564.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00091552734375,
+      "logps/rejected": -644.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000904083251953125,
+      "loss": 0.3487,
+      "rewards/accuracies": 0.8899999856948853,
+      "rewards/chosen": -1.6640625,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -2.9375,
+      "step": 4720
+    },
+    {
+      "epoch": 9.28361138370952,
+      "grad_norm": 11.711069712684242,
+      "learning_rate": 7.58031316002522e-09,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -552.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000804901123046875,
+      "logps/rejected": -616.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.0007781982421875,
+      "loss": 0.3634,
+      "rewards/accuracies": 0.8700000047683716,
+      "rewards/chosen": -1.609375,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -2.875,
+      "step": 4730
+    },
+    {
+      "epoch": 9.30323846908734,
+      "grad_norm": 10.862219564432905,
+      "learning_rate": 7.167029676304781e-09,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -548.0,
+      "logps/chosen_bottom_tokens": -13.75,
+      "logps/chosen_top_tokens": -0.000873565673828125,
+      "logps/rejected": -604.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.0008697509765625,
+      "loss": 0.3465,
+      "rewards/accuracies": 0.8799999952316284,
+      "rewards/chosen": -1.6484375,
+      "rewards/margins": 1.2265625,
+      "rewards/rejected": -2.875,
+      "step": 4740
+    },
+    {
+      "epoch": 9.322865554465162,
+      "grad_norm": 13.451953701239999,
+      "learning_rate": 6.7651666924204984e-09,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -496.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.000926971435546875,
+      "logps/rejected": -564.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.00090789794921875,
+      "loss": 0.3474,
+      "rewards/accuracies": 0.8799999356269836,
+      "rewards/chosen": -1.609375,
+      "rewards/margins": 1.28125,
+      "rewards/rejected": -2.890625,
+      "step": 4750
+    },
+    {
+      "epoch": 9.342492639842984,
+      "grad_norm": 15.228199482134903,
+      "learning_rate": 6.374743108096547e-09,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -508.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.000881195068359375,
+      "logps/rejected": -620.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00087738037109375,
+      "loss": 0.359,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.6015625,
+      "rewards/margins": 1.28125,
+      "rewards/rejected": -2.875,
+      "step": 4760
+    },
+    {
+      "epoch": 9.362119725220804,
+      "grad_norm": 11.009933240515391,
+      "learning_rate": 5.9957772850589564e-09,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -556.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00090789794921875,
+      "logps/rejected": -604.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.00087738037109375,
+      "loss": 0.356,
+      "rewards/accuracies": 0.8650000691413879,
+      "rewards/chosen": -1.59375,
+      "rewards/margins": 1.328125,
+      "rewards/rejected": -2.90625,
+      "step": 4770
+    },
+    {
+      "epoch": 9.381746810598626,
+      "grad_norm": 10.277894448124382,
+      "learning_rate": 5.628287046172187e-09,
+      "logits/chosen": -1.3515625,
+      "logits/rejected": -1.375,
+      "logps/chosen": -528.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000835418701171875,
+      "logps/rejected": -616.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000835418701171875,
+      "loss": 0.3541,
+      "rewards/accuracies": 0.8850000500679016,
+      "rewards/chosen": -1.6015625,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -2.890625,
+      "step": 4780
+    },
+    {
+      "epoch": 9.401373895976448,
+      "grad_norm": 13.874643137091146,
+      "learning_rate": 5.272289674600916e-09,
+      "logits/chosen": -1.359375,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -536.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.0008087158203125,
+      "logps/rejected": -612.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.00080108642578125,
+      "loss": 0.3728,
+      "rewards/accuracies": 0.8600000143051147,
+      "rewards/chosen": -1.6328125,
+      "rewards/margins": 1.1953125,
+      "rewards/rejected": -2.828125,
+      "step": 4790
+    },
+    {
+      "epoch": 9.421000981354268,
+      "grad_norm": 14.853698997198226,
+      "learning_rate": 4.927801912997214e-09,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -486.0,
+      "logps/chosen_bottom_tokens": -13.75,
+      "logps/chosen_top_tokens": -0.000850677490234375,
+      "logps/rejected": -560.0,
+      "logps/rejected_bottom_tokens": -13.625,
+      "logps/rejected_top_tokens": -0.000835418701171875,
+      "loss": 0.3434,
+      "rewards/accuracies": 0.8399999737739563,
+      "rewards/chosen": -1.515625,
+      "rewards/margins": 1.2578125,
+      "rewards/rejected": -2.78125,
+      "step": 4800
+    },
+    {
+      "epoch": 9.421000981354268,
+      "eval_logits/chosen": -1.40625,
+      "eval_logits/rejected": -1.4296875,
+      "eval_logps/chosen": -580.0,
+      "eval_logps/chosen_bottom_tokens": -13.9375,
+      "eval_logps/chosen_top_tokens": -0.00087738037109375,
+      "eval_logps/rejected": -600.0,
+      "eval_logps/rejected_bottom_tokens": -13.8125,
+      "eval_logps/rejected_top_tokens": -0.00084686279296875,
+      "eval_loss": 0.5972570776939392,
+      "eval_rewards/accuracies": 0.7014925479888916,
+      "eval_rewards/chosen": -1.8671875,
+      "eval_rewards/margins": 0.765625,
+      "eval_rewards/rejected": -2.640625,
+      "eval_runtime": 108.376,
+      "eval_samples_per_second": 18.454,
+      "eval_steps_per_second": 0.618,
+      "step": 4800
+    },
+    {
+      "epoch": 9.44062806673209,
+      "grad_norm": 11.155786101466646,
+      "learning_rate": 4.594839962713009e-09,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -536.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.00090789794921875,
+      "logps/rejected": -616.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000904083251953125,
+      "loss": 0.3635,
+      "rewards/accuracies": 0.8449999690055847,
+      "rewards/chosen": -1.65625,
+      "rewards/margins": 1.3125,
+      "rewards/rejected": -2.96875,
+      "step": 4810
+    },
+    {
+      "epoch": 9.460255152109912,
+      "grad_norm": 11.415347601053586,
+      "learning_rate": 4.273419483038304e-09,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -1.4453125,
+      "logps/chosen": -560.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.00080108642578125,
+      "logps/rejected": -640.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00079345703125,
+      "loss": 0.345,
+      "rewards/accuracies": 0.8700000047683716,
+      "rewards/chosen": -1.578125,
+      "rewards/margins": 1.2109375,
+      "rewards/rejected": -2.796875,
+      "step": 4820
+    },
+    {
+      "epoch": 9.479882237487733,
+      "grad_norm": 10.958041240615906,
+      "learning_rate": 3.963555590464601e-09,
+      "logits/chosen": -1.3359375,
+      "logits/rejected": -1.3828125,
+      "logps/chosen": -512.0,
+      "logps/chosen_bottom_tokens": -13.75,
+      "logps/chosen_top_tokens": -0.00103759765625,
+      "logps/rejected": -540.0,
+      "logps/rejected_bottom_tokens": -13.625,
+      "logps/rejected_top_tokens": -0.001007080078125,
+      "loss": 0.3657,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.5390625,
+      "rewards/margins": 1.0859375,
+      "rewards/rejected": -2.625,
+      "step": 4830
+    },
+    {
+      "epoch": 9.499509322865554,
+      "grad_norm": 11.57554214332043,
+      "learning_rate": 3.6652628579740276e-09,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -528.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000926971435546875,
+      "logps/rejected": -604.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000919342041015625,
+      "loss": 0.3403,
+      "rewards/accuracies": 0.8649999499320984,
+      "rewards/chosen": -1.578125,
+      "rewards/margins": 1.28125,
+      "rewards/rejected": -2.859375,
+      "step": 4840
+    },
+    {
+      "epoch": 9.519136408243376,
+      "grad_norm": 9.297108054051924,
+      "learning_rate": 3.378555314353937e-09,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -510.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000896453857421875,
+      "logps/rejected": -580.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000865936279296875,
+      "loss": 0.3418,
+      "rewards/accuracies": 0.8149999380111694,
+      "rewards/chosen": -1.71875,
+      "rewards/margins": 1.1015625,
+      "rewards/rejected": -2.828125,
+      "step": 4850
+    },
+    {
+      "epoch": 9.538763493621197,
+      "grad_norm": 9.130763500239256,
+      "learning_rate": 3.1034464435371053e-09,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -520.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000911712646484375,
+      "logps/rejected": -608.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.0009002685546875,
+      "loss": 0.3505,
+      "rewards/accuracies": 0.8799999952316284,
+      "rewards/chosen": -1.6875,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -2.984375,
+      "step": 4860
+    },
+    {
+      "epoch": 9.558390578999019,
+      "grad_norm": 14.1811429926495,
+      "learning_rate": 2.8399491839677103e-09,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -568.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000911712646484375,
+      "logps/rejected": -616.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.00089263916015625,
+      "loss": 0.3473,
+      "rewards/accuracies": 0.89000004529953,
+      "rewards/chosen": -1.5859375,
+      "rewards/margins": 1.359375,
+      "rewards/rejected": -2.953125,
+      "step": 4870
+    },
+    {
+      "epoch": 9.57801766437684,
+      "grad_norm": 12.869889556323235,
+      "learning_rate": 2.5880759279925947e-09,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -580.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.0008392333984375,
+      "logps/rejected": -660.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000850677490234375,
+      "loss": 0.3415,
+      "rewards/accuracies": 0.8899999856948853,
+      "rewards/chosen": -1.6875,
+      "rewards/margins": 1.3203125,
+      "rewards/rejected": -3.015625,
+      "step": 4880
+    },
+    {
+      "epoch": 9.59764474975466,
+      "grad_norm": 12.548800832744114,
+      "learning_rate": 2.3478385212787055e-09,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -572.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000888824462890625,
+      "logps/rejected": -652.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.00086212158203125,
+      "loss": 0.3548,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.671875,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -2.96875,
+      "step": 4890
+    },
+    {
+      "epoch": 9.617271835132483,
+      "grad_norm": 13.023279708907932,
+      "learning_rate": 2.1192482622557904e-09,
+      "logits/chosen": -1.421875,
+      "logits/rejected": -1.453125,
+      "logps/chosen": -536.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.000850677490234375,
+      "logps/rejected": -620.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000835418701171875,
+      "loss": 0.3492,
+      "rewards/accuracies": 0.8600000143051147,
+      "rewards/chosen": -1.6484375,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -2.921875,
+      "step": 4900
+    },
+    {
+      "epoch": 9.617271835132483,
+      "eval_logits/chosen": -1.3984375,
+      "eval_logits/rejected": -1.4296875,
+      "eval_logps/chosen": -580.0,
+      "eval_logps/chosen_bottom_tokens": -13.9375,
+      "eval_logps/chosen_top_tokens": -0.00087738037109375,
+      "eval_logps/rejected": -600.0,
+      "eval_logps/rejected_bottom_tokens": -13.8125,
+      "eval_logps/rejected_top_tokens": -0.0008392333984375,
+      "eval_loss": 0.5981469750404358,
+      "eval_rewards/accuracies": 0.7044776082038879,
+      "eval_rewards/chosen": -1.875,
+      "eval_rewards/margins": 0.7578125,
+      "eval_rewards/rejected": -2.640625,
+      "eval_runtime": 108.362,
+      "eval_samples_per_second": 18.457,
+      "eval_steps_per_second": 0.618,
+      "step": 4900
+    },
+    {
+      "epoch": 9.636898920510305,
+      "grad_norm": 13.199505551719971,
+      "learning_rate": 1.902315901585155e-09,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -552.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000881195068359375,
+      "logps/rejected": -652.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.0009002685546875,
+      "loss": 0.3268,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -1.6875,
+      "rewards/margins": 1.5703125,
+      "rewards/rejected": -3.265625,
+      "step": 4910
+    },
+    {
+      "epoch": 9.656526005888125,
+      "grad_norm": 12.666272933800087,
+      "learning_rate": 1.6970516416540127e-09,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.390625,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00092315673828125,
+      "logps/rejected": -612.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000934600830078125,
+      "loss": 0.3479,
+      "rewards/accuracies": 0.89000004529953,
+      "rewards/chosen": -1.671875,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -2.96875,
+      "step": 4920
+    },
+    {
+      "epoch": 9.676153091265947,
+      "grad_norm": 15.121041451718062,
+      "learning_rate": 1.5034651360956718e-09,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -584.0,
+      "logps/chosen_bottom_tokens": -13.75,
+      "logps/chosen_top_tokens": -0.000926971435546875,
+      "logps/rejected": -640.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.00090789794921875,
+      "loss": 0.3591,
+      "rewards/accuracies": 0.8399999737739563,
+      "rewards/chosen": -1.8125,
+      "rewards/margins": 1.1640625,
+      "rewards/rejected": -2.96875,
+      "step": 4930
+    },
+    {
+      "epoch": 9.695780176643769,
+      "grad_norm": 14.043067920247658,
+      "learning_rate": 1.3215654893354578e-09,
+      "logits/chosen": -1.375,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -520.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.0008544921875,
+      "logps/rejected": -600.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.000873565673828125,
+      "loss": 0.3503,
+      "rewards/accuracies": 0.8549998998641968,
+      "rewards/chosen": -1.6015625,
+      "rewards/margins": 1.3125,
+      "rewards/rejected": -2.90625,
+      "step": 4940
+    },
+    {
+      "epoch": 9.71540726202159,
+      "grad_norm": 16.327308164510683,
+      "learning_rate": 1.1513612561626362e-09,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.0009002685546875,
+      "logps/rejected": -592.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.0008697509765625,
+      "loss": 0.3552,
+      "rewards/accuracies": 0.8949999809265137,
+      "rewards/chosen": -1.5859375,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -2.875,
+      "step": 4950
+    },
+    {
+      "epoch": 9.735034347399411,
+      "grad_norm": 11.799295708369108,
+      "learning_rate": 9.928604413280694e-10,
+      "logits/chosen": -1.4140625,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -532.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.00083160400390625,
+      "logps/rejected": -600.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.000827789306640625,
+      "loss": 0.3506,
+      "rewards/accuracies": 0.8850000500679016,
+      "rewards/chosen": -1.59375,
+      "rewards/margins": 1.2265625,
+      "rewards/rejected": -2.8125,
+      "step": 4960
+    },
+    {
+      "epoch": 9.754661432777233,
+      "grad_norm": 12.311557663406573,
+      "learning_rate": 8.460704991676004e-10,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -524.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.001007080078125,
+      "logps/rejected": -600.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.0009918212890625,
+      "loss": 0.3486,
+      "rewards/accuracies": 0.8650000691413879,
+      "rewards/chosen": -1.609375,
+      "rewards/margins": 1.2890625,
+      "rewards/rejected": -2.890625,
+      "step": 4970
+    },
+    {
+      "epoch": 9.774288518155053,
+      "grad_norm": 11.77978565668884,
+      "learning_rate": 7.109983332516945e-10,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -516.0,
+      "logps/chosen_bottom_tokens": -13.75,
+      "logps/chosen_top_tokens": -0.0008697509765625,
+      "logps/rejected": -608.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.0008697509765625,
+      "loss": 0.3436,
+      "rewards/accuracies": 0.8700000047683716,
+      "rewards/chosen": -1.6640625,
+      "rewards/margins": 1.21875,
+      "rewards/rejected": -2.875,
+      "step": 4980
+    },
+    {
+      "epoch": 9.793915603532875,
+      "grad_norm": 11.06530017719301,
+      "learning_rate": 5.876502960605878e-10,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -552.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.00086212158203125,
+      "logps/rejected": -648.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000911712646484375,
+      "loss": 0.3478,
+      "rewards/accuracies": 0.8700000047683716,
+      "rewards/chosen": -1.6796875,
+      "rewards/margins": 1.3125,
+      "rewards/rejected": -2.984375,
+      "step": 4990
+    },
+    {
+      "epoch": 9.813542688910697,
+      "grad_norm": 14.260832030615752,
+      "learning_rate": 4.7603218868561e-10,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -528.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.0009307861328125,
+      "logps/rejected": -636.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.00092315673828125,
+      "loss": 0.3487,
+      "rewards/accuracies": 0.8949999809265137,
+      "rewards/chosen": -1.6015625,
+      "rewards/margins": 1.21875,
+      "rewards/rejected": -2.8125,
+      "step": 5000
+    },
+    {
+      "epoch": 9.813542688910697,
+      "eval_logits/chosen": -1.40625,
+      "eval_logits/rejected": -1.4375,
+      "eval_logps/chosen": -580.0,
+      "eval_logps/chosen_bottom_tokens": -13.9375,
+      "eval_logps/chosen_top_tokens": -0.00086212158203125,
+      "eval_logps/rejected": -600.0,
+      "eval_logps/rejected_bottom_tokens": -13.8125,
+      "eval_logps/rejected_top_tokens": -0.0008392333984375,
+      "eval_loss": 0.5967333912849426,
+      "eval_rewards/accuracies": 0.7134328484535217,
+      "eval_rewards/chosen": -1.8671875,
+      "eval_rewards/margins": 0.7734375,
+      "eval_rewards/rejected": -2.640625,
+      "eval_runtime": 108.4167,
+      "eval_samples_per_second": 18.447,
+      "eval_steps_per_second": 0.618,
+      "step": 5000
+    },
+    {
+      "epoch": 9.833169774288518,
+      "grad_norm": 12.34808824315509,
+      "learning_rate": 3.7614926055634653e-10,
+      "logits/chosen": -1.3984375,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -540.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.0008544921875,
+      "logps/rejected": -620.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000827789306640625,
+      "loss": 0.3476,
+      "rewards/accuracies": 0.8799999952316284,
+      "rewards/chosen": -1.6640625,
+      "rewards/margins": 1.3046875,
+      "rewards/rejected": -2.96875,
+      "step": 5010
+    },
+    {
+      "epoch": 9.85279685966634,
+      "grad_norm": 12.015746876718154,
+      "learning_rate": 2.880062091937252e-10,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.3984375,
+      "logps/chosen": -520.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.000904083251953125,
+      "logps/rejected": -580.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.0009002685546875,
+      "loss": 0.3555,
+      "rewards/accuracies": 0.8899999856948853,
+      "rewards/chosen": -1.6484375,
+      "rewards/margins": 1.25,
+      "rewards/rejected": -2.90625,
+      "step": 5020
+    },
+    {
+      "epoch": 9.872423945044162,
+      "grad_norm": 13.922601928935721,
+      "learning_rate": 2.1160717998913724e-10,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.40625,
+      "logps/chosen": -580.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.00091552734375,
+      "logps/rejected": -692.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000911712646484375,
+      "loss": 0.3404,
+      "rewards/accuracies": 0.8849999308586121,
+      "rewards/chosen": -1.7421875,
+      "rewards/margins": 1.34375,
+      "rewards/rejected": -3.09375,
+      "step": 5030
+    },
+    {
+      "epoch": 9.892051030421982,
+      "grad_norm": 11.484288890209422,
+      "learning_rate": 1.4695576600948223e-10,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.4453125,
+      "logps/chosen": -564.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.0008697509765625,
+      "logps/rejected": -620.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.000865936279296875,
+      "loss": 0.3493,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.578125,
+      "rewards/margins": 1.4140625,
+      "rewards/rejected": -3.0,
+      "step": 5040
+    },
+    {
+      "epoch": 9.911678115799804,
+      "grad_norm": 14.02364411561864,
+      "learning_rate": 9.405500782808107e-11,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -568.0,
+      "logps/chosen_bottom_tokens": -13.8125,
+      "logps/chosen_top_tokens": -0.000911712646484375,
+      "logps/rejected": -636.0,
+      "logps/rejected_bottom_tokens": -13.6875,
+      "logps/rejected_top_tokens": -0.00087738037109375,
+      "loss": 0.358,
+      "rewards/accuracies": 0.8950001001358032,
+      "rewards/chosen": -1.625,
+      "rewards/margins": 1.453125,
+      "rewards/rejected": -3.078125,
+      "step": 5050
+    },
+    {
+      "epoch": 9.931305201177626,
+      "grad_norm": 11.674058347699651,
+      "learning_rate": 5.29073933818458e-11,
+      "logits/chosen": -1.40625,
+      "logits/rejected": -1.421875,
+      "logps/chosen": -544.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000911712646484375,
+      "logps/rejected": -660.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.000926971435546875,
+      "loss": 0.3439,
+      "rewards/accuracies": 0.8899999856948853,
+      "rewards/chosen": -1.7265625,
+      "rewards/margins": 1.3515625,
+      "rewards/rejected": -3.078125,
+      "step": 5060
+    },
+    {
+      "epoch": 9.950932286555446,
+      "grad_norm": 12.556977231864163,
+      "learning_rate": 2.351485785409557e-11,
+      "logits/chosen": -1.3671875,
+      "logits/rejected": -1.375,
+      "logps/chosen": -532.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.000858306884765625,
+      "logps/rejected": -644.0,
+      "logps/rejected_bottom_tokens": -13.875,
+      "logps/rejected_top_tokens": -0.0009002685546875,
+      "loss": 0.3606,
+      "rewards/accuracies": 0.8650000691413879,
+      "rewards/chosen": -1.5,
+      "rewards/margins": 1.359375,
+      "rewards/rejected": -2.859375,
+      "step": 5070
+    },
+    {
+      "epoch": 9.970559371933268,
+      "grad_norm": 11.972937841080586,
+      "learning_rate": 5.878783583712632e-12,
+      "logits/chosen": -1.390625,
+      "logits/rejected": -1.4296875,
+      "logps/chosen": -502.0,
+      "logps/chosen_bottom_tokens": -13.875,
+      "logps/chosen_top_tokens": -0.001068115234375,
+      "logps/rejected": -592.0,
+      "logps/rejected_bottom_tokens": -13.75,
+      "logps/rejected_top_tokens": -0.00106048583984375,
+      "loss": 0.3563,
+      "rewards/accuracies": 0.8449999690055847,
+      "rewards/chosen": -1.6171875,
+      "rewards/margins": 1.21875,
+      "rewards/rejected": -2.84375,
+      "step": 5080
+    },
+    {
+      "epoch": 9.99018645731109,
+      "grad_norm": 12.822609342422131,
+      "learning_rate": 0.0,
+      "logits/chosen": -1.3828125,
+      "logits/rejected": -1.4140625,
+      "logps/chosen": -536.0,
+      "logps/chosen_bottom_tokens": -13.9375,
+      "logps/chosen_top_tokens": -0.000896453857421875,
+      "logps/rejected": -620.0,
+      "logps/rejected_bottom_tokens": -13.8125,
+      "logps/rejected_top_tokens": -0.0009002685546875,
+      "loss": 0.3528,
+      "rewards/accuracies": 0.8600000143051147,
+      "rewards/chosen": -1.65625,
+      "rewards/margins": 1.296875,
+      "rewards/rejected": -2.953125,
+      "step": 5090
+    },
+    {
+      "epoch": 9.99018645731109,
+      "step": 5090,
       "total_flos": 0.0,
-      "train_loss": 0.6464882252961105,
-      "train_runtime": 8284.9703,
-      "train_samples_per_second": 7.379,
-      "train_steps_per_second": 0.061
+      "train_loss": 0.40154263242522953,
+      "train_runtime": 75277.8699,
+      "train_samples_per_second": 8.121,
+      "train_steps_per_second": 0.068
     }
   ],
   "logging_steps": 10,
-  "max_steps": 509,
+  "max_steps": 5090,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 10,
   "save_steps": 100,
   "total_flos": 0.0,
   "train_batch_size": 5,