diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,23868 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 100,
+  "global_step": 15284,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.270111183780249e-09,
+      "logits/chosen": -2.634561777114868,
+      "logits/rejected": -2.673060417175293,
+      "logps/chosen": -207.5323944091797,
+      "logps/rejected": -286.9266052246094,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.270111183780249e-08,
+      "logits/chosen": -2.217697858810425,
+      "logits/rejected": -1.9652551412582397,
+      "logps/chosen": -186.06736755371094,
+      "logps/rejected": -165.34738159179688,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.1666666716337204,
+      "rewards/chosen": -0.0014695884892717004,
+      "rewards/margins": -0.002330251270905137,
+      "rewards/rejected": 0.0008606627234257758,
+      "step": 10
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.540222367560497e-08,
+      "logits/chosen": -2.4319119453430176,
+      "logits/rejected": -2.2228429317474365,
+      "logps/chosen": -232.4527587890625,
+      "logps/rejected": -231.435546875,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -4.6026078052818775e-05,
+      "rewards/margins": 7.92687205830589e-05,
+      "rewards/rejected": -0.0001252948131877929,
+      "step": 20
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.810333551340746e-08,
+      "logits/chosen": -2.25822377204895,
+      "logits/rejected": -2.162461996078491,
+      "logps/chosen": -197.378173828125,
+      "logps/rejected": -219.074951171875,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.00045980390859767795,
+      "rewards/margins": 0.00029530542087741196,
+      "rewards/rejected": 0.0001644986041355878,
+      "step": 30
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.3080444735120995e-07,
+      "logits/chosen": -2.211453914642334,
+      "logits/rejected": -2.251152992248535,
+      "logps/chosen": -276.0306701660156,
+      "logps/rejected": -265.74371337890625,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.0004297545528970659,
+      "rewards/margins": -0.0005864914273843169,
+      "rewards/rejected": 0.00015673683083150536,
+      "step": 40
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.6350555918901243e-07,
+      "logits/chosen": -2.3490729331970215,
+      "logits/rejected": -2.1418588161468506,
+      "logps/chosen": -204.78414916992188,
+      "logps/rejected": -184.72738647460938,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0002141618897439912,
+      "rewards/margins": 0.0005374640459194779,
+      "rewards/rejected": -0.0003233022871427238,
+      "step": 50
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.9620667102681492e-07,
+      "logits/chosen": -2.30709171295166,
+      "logits/rejected": -2.0678863525390625,
+      "logps/chosen": -209.7256622314453,
+      "logps/rejected": -185.79867553710938,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": 0.0008024474373087287,
+      "rewards/margins": -0.0010456187883391976,
+      "rewards/rejected": 0.00184806645847857,
+      "step": 60
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.289077828646174e-07,
+      "logits/chosen": -2.2695107460021973,
+      "logits/rejected": -2.1568443775177,
+      "logps/chosen": -218.00942993164062,
+      "logps/rejected": -207.9250946044922,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": 0.00100115523673594,
+      "rewards/margins": -0.00015484937466681004,
+      "rewards/rejected": 0.00115600461140275,
+      "step": 70
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 2.616088947024199e-07,
+      "logits/chosen": -2.5082268714904785,
+      "logits/rejected": -2.227625608444214,
+      "logps/chosen": -258.78826904296875,
+      "logps/rejected": -213.654541015625,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": 0.0016212640330195427,
+      "rewards/margins": 0.00033446805900894105,
+      "rewards/rejected": 0.001286796061322093,
+      "step": 80
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 2.943100065402224e-07,
+      "logits/chosen": -2.25887393951416,
+      "logits/rejected": -2.173290252685547,
+      "logps/chosen": -184.6951141357422,
+      "logps/rejected": -165.476806640625,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.0008967015892267227,
+      "rewards/margins": 0.00010184949496760964,
+      "rewards/rejected": 0.0007948519778437912,
+      "step": 90
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 3.2701111837802487e-07,
+      "logits/chosen": -2.4311330318450928,
+      "logits/rejected": -2.425891399383545,
+      "logps/chosen": -168.7532196044922,
+      "logps/rejected": -183.79940795898438,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.0018166687805205584,
+      "rewards/margins": -0.00107972149271518,
+      "rewards/rejected": 0.0028963901568204165,
+      "step": 100
+    },
+    {
+      "epoch": 0.01,
+      "eval_logits/chosen": -2.348848581314087,
+      "eval_logits/rejected": -2.1603763103485107,
+      "eval_logps/chosen": -231.76463317871094,
+      "eval_logps/rejected": -211.4439239501953,
+      "eval_loss": 0.6931134462356567,
+      "eval_rewards/accuracies": 0.4950000047683716,
+      "eval_rewards/chosen": 0.0024031461216509342,
+      "eval_rewards/margins": 0.0007233246578834951,
+      "eval_rewards/rejected": 0.0016798212891444564,
+      "eval_runtime": 707.3142,
+      "eval_samples_per_second": 2.828,
+      "eval_steps_per_second": 1.414,
+      "step": 100
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 3.5971223021582736e-07,
+      "logits/chosen": -2.347839593887329,
+      "logits/rejected": -1.9999773502349854,
+      "logps/chosen": -222.35336303710938,
+      "logps/rejected": -166.99118041992188,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.004421192221343517,
+      "rewards/margins": 0.0019870258402079344,
+      "rewards/rejected": 0.002434166381135583,
+      "step": 110
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 3.9241334205362984e-07,
+      "logits/chosen": -2.340026378631592,
+      "logits/rejected": -2.244414806365967,
+      "logps/chosen": -223.99569702148438,
+      "logps/rejected": -234.1189422607422,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.3499999940395355,
+      "rewards/chosen": 0.00296420999802649,
+      "rewards/margins": 0.0004412824346218258,
+      "rewards/rejected": 0.002522927476093173,
+      "step": 120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 4.251144538914324e-07,
+      "logits/chosen": -2.2618038654327393,
+      "logits/rejected": -2.217468738555908,
+      "logps/chosen": -149.3894500732422,
+      "logps/rejected": -148.2598114013672,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0025640667881816626,
+      "rewards/margins": 0.00097900559194386,
+      "rewards/rejected": 0.0015850610798224807,
+      "step": 130
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 4.578155657292348e-07,
+      "logits/chosen": -2.3222999572753906,
+      "logits/rejected": -2.2233099937438965,
+      "logps/chosen": -225.5967254638672,
+      "logps/rejected": -159.4678955078125,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0036222212947905064,
+      "rewards/margins": 0.0013900110498070717,
+      "rewards/rejected": 0.0022322097793221474,
+      "step": 140
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 4.905166775670374e-07,
+      "logits/chosen": -2.367203950881958,
+      "logits/rejected": -2.1586971282958984,
+      "logps/chosen": -230.97109985351562,
+      "logps/rejected": -229.11178588867188,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.003845545928925276,
+      "rewards/margins": 0.003386072115972638,
+      "rewards/rejected": 0.00045947395847178996,
+      "step": 150
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.232177894048398e-07,
+      "logits/chosen": -2.2155299186706543,
+      "logits/rejected": -2.2280077934265137,
+      "logps/chosen": -260.2898864746094,
+      "logps/rejected": -224.85397338867188,
+      "loss": 0.693,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.005785978864878416,
+      "rewards/margins": 0.004330903757363558,
+      "rewards/rejected": 0.0014550751075148582,
+      "step": 160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.559189012426422e-07,
+      "logits/chosen": -2.3153624534606934,
+      "logits/rejected": -2.0374207496643066,
+      "logps/chosen": -180.40968322753906,
+      "logps/rejected": -156.81607055664062,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.004428991116583347,
+      "rewards/margins": 0.0036846441216766834,
+      "rewards/rejected": 0.0007443467038683593,
+      "step": 170
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.886200130804448e-07,
+      "logits/chosen": -2.3993406295776367,
+      "logits/rejected": -2.3392200469970703,
+      "logps/chosen": -217.6866455078125,
+      "logps/rejected": -198.7965850830078,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.005419188644737005,
+      "rewards/margins": 0.005340488161891699,
+      "rewards/rejected": 7.869987894082442e-05,
+      "step": 180
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.213211249182473e-07,
+      "logits/chosen": -2.0705583095550537,
+      "logits/rejected": -2.175136089324951,
+      "logps/chosen": -191.1099853515625,
+      "logps/rejected": -208.73690795898438,
+      "loss": 0.693,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.003862470854073763,
+      "rewards/margins": 0.004704989958554506,
+      "rewards/rejected": -0.0008425191044807434,
+      "step": 190
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.540222367560497e-07,
+      "logits/chosen": -2.27732253074646,
+      "logits/rejected": -2.24127197265625,
+      "logps/chosen": -146.89163208007812,
+      "logps/rejected": -177.7828826904297,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0032042518723756075,
+      "rewards/margins": 0.0057475874200463295,
+      "rewards/rejected": -0.0025433353148400784,
+      "step": 200
+    },
+    {
+      "epoch": 0.01,
+      "eval_logits/chosen": -2.3492345809936523,
+      "eval_logits/rejected": -2.1607820987701416,
+      "eval_logps/chosen": -231.4797821044922,
+      "eval_logps/rejected": -211.65264892578125,
+      "eval_loss": 0.6928496360778809,
+      "eval_rewards/accuracies": 0.5835000276565552,
+      "eval_rewards/chosen": 0.005251556169241667,
+      "eval_rewards/margins": 0.0056591029278934,
+      "eval_rewards/rejected": -0.0004075466131325811,
+      "eval_runtime": 708.7402,
+      "eval_samples_per_second": 2.822,
+      "eval_steps_per_second": 1.411,
+      "step": 200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.867233485938523e-07,
+      "logits/chosen": -2.4269886016845703,
+      "logits/rejected": -2.2028229236602783,
+      "logps/chosen": -218.64584350585938,
+      "logps/rejected": -188.28201293945312,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.008876695297658443,
+      "rewards/margins": 0.010037838481366634,
+      "rewards/rejected": -0.0011611439986154437,
+      "step": 210
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.194244604316547e-07,
+      "logits/chosen": -2.2129268646240234,
+      "logits/rejected": -2.0407798290252686,
+      "logps/chosen": -182.85243225097656,
+      "logps/rejected": -174.8861083984375,
+      "loss": 0.693,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.004839606583118439,
+      "rewards/margins": 0.004267896059900522,
+      "rewards/rejected": 0.000571710173971951,
+      "step": 220
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 7.521255722694571e-07,
+      "logits/chosen": -2.421215534210205,
+      "logits/rejected": -2.025574207305908,
+      "logps/chosen": -278.98101806640625,
+      "logps/rejected": -184.08096313476562,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.007983444258570671,
+      "rewards/margins": 0.005552899558097124,
+      "rewards/rejected": 0.0024305458646267653,
+      "step": 230
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 7.848266841072597e-07,
+      "logits/chosen": -2.199948310852051,
+      "logits/rejected": -2.164917469024658,
+      "logps/chosen": -214.61245727539062,
+      "logps/rejected": -206.2373504638672,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.01238412968814373,
+      "rewards/margins": 0.01192299835383892,
+      "rewards/rejected": 0.0004611331969499588,
+      "step": 240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 8.175277959450622e-07,
+      "logits/chosen": -2.1677653789520264,
+      "logits/rejected": -2.3267643451690674,
+      "logps/chosen": -218.1819610595703,
+      "logps/rejected": -220.6886444091797,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.022441856563091278,
+      "rewards/margins": 0.009667301550507545,
+      "rewards/rejected": 0.012774554081261158,
+      "step": 250
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 8.502289077828648e-07,
+      "logits/chosen": -2.5052707195281982,
+      "logits/rejected": -2.144476890563965,
+      "logps/chosen": -254.23361206054688,
+      "logps/rejected": -189.0504608154297,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.02921391651034355,
+      "rewards/margins": 0.011754143051803112,
+      "rewards/rejected": 0.017459776252508163,
+      "step": 260
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 8.829300196206672e-07,
+      "logits/chosen": -2.422799587249756,
+      "logits/rejected": -2.1278483867645264,
+      "logps/chosen": -246.3038330078125,
+      "logps/rejected": -230.50228881835938,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.02507014013826847,
+      "rewards/margins": 0.014279340393841267,
+      "rewards/rejected": 0.01079079881310463,
+      "step": 270
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.156311314584696e-07,
+      "logits/chosen": -2.3067777156829834,
+      "logits/rejected": -2.1929802894592285,
+      "logps/chosen": -159.95680236816406,
+      "logps/rejected": -146.39175415039062,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.022316502407193184,
+      "rewards/margins": 0.009042134508490562,
+      "rewards/rejected": 0.013274368830025196,
+      "step": 280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.483322432962722e-07,
+      "logits/chosen": -2.555990695953369,
+      "logits/rejected": -2.1609978675842285,
+      "logps/chosen": -282.0716857910156,
+      "logps/rejected": -225.60147094726562,
+      "loss": 0.693,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.031071314588189125,
+      "rewards/margins": 0.010234272107481956,
+      "rewards/rejected": 0.02083704061806202,
+      "step": 290
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.810333551340747e-07,
+      "logits/chosen": -2.3414573669433594,
+      "logits/rejected": -2.197089433670044,
+      "logps/chosen": -265.0999755859375,
+      "logps/rejected": -238.67355346679688,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.035596780478954315,
+      "rewards/margins": 0.022229164838790894,
+      "rewards/rejected": 0.013367618434131145,
+      "step": 300
+    },
+    {
+      "epoch": 0.02,
+      "eval_logits/chosen": -2.3535282611846924,
+      "eval_logits/rejected": -2.164868116378784,
+      "eval_logps/chosen": -228.6509246826172,
+      "eval_logps/rejected": -209.84600830078125,
+      "eval_loss": 0.6924605965614319,
+      "eval_rewards/accuracies": 0.5830000042915344,
+      "eval_rewards/chosen": 0.03354022651910782,
+      "eval_rewards/margins": 0.015881428495049477,
+      "eval_rewards/rejected": 0.017658798024058342,
+      "eval_runtime": 705.9246,
+      "eval_samples_per_second": 2.833,
+      "eval_steps_per_second": 1.417,
+      "step": 300
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0137344669718771e-06,
+      "logits/chosen": -2.350961208343506,
+      "logits/rejected": -2.365408420562744,
+      "logps/chosen": -166.96469116210938,
+      "logps/rejected": -155.90208435058594,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.025256266817450523,
+      "rewards/margins": 0.005879827309399843,
+      "rewards/rejected": 0.019376439973711967,
+      "step": 310
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0464355788096796e-06,
+      "logits/chosen": -2.455401659011841,
+      "logits/rejected": -2.0602850914001465,
+      "logps/chosen": -221.2332305908203,
+      "logps/rejected": -192.01666259765625,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.03931882977485657,
+      "rewards/margins": 0.015343300998210907,
+      "rewards/rejected": 0.02397553250193596,
+      "step": 320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.079136690647482e-06,
+      "logits/chosen": -2.4358632564544678,
+      "logits/rejected": -2.2039878368377686,
+      "logps/chosen": -203.50843811035156,
+      "logps/rejected": -175.61509704589844,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.04902844503521919,
+      "rewards/margins": 0.022538715973496437,
+      "rewards/rejected": 0.026489730924367905,
+      "step": 330
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.1118378024852844e-06,
+      "logits/chosen": -2.187950372695923,
+      "logits/rejected": -2.353523015975952,
+      "logps/chosen": -150.39413452148438,
+      "logps/rejected": -177.75634765625,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.033202774822711945,
+      "rewards/margins": -0.0008061464759521186,
+      "rewards/rejected": 0.034008920192718506,
+      "step": 340
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.144538914323087e-06,
+      "logits/chosen": -2.426027774810791,
+      "logits/rejected": -1.987079381942749,
+      "logps/chosen": -317.85028076171875,
+      "logps/rejected": -247.85317993164062,
+      "loss": 0.692,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.042603787034749985,
+      "rewards/margins": 0.02849414013326168,
+      "rewards/rejected": 0.014109638519585133,
+      "step": 350
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.1772400261608895e-06,
+      "logits/chosen": -2.4899606704711914,
+      "logits/rejected": -2.1962363719940186,
+      "logps/chosen": -220.11160278320312,
+      "logps/rejected": -192.32504272460938,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.053684353828430176,
+      "rewards/margins": 0.023779017850756645,
+      "rewards/rejected": 0.02990533970296383,
+      "step": 360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.2099411379986922e-06,
+      "logits/chosen": -2.1702442169189453,
+      "logits/rejected": -2.2464897632598877,
+      "logps/chosen": -192.40721130371094,
+      "logps/rejected": -206.269287109375,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.05089518427848816,
+      "rewards/margins": 0.010943805798888206,
+      "rewards/rejected": 0.0399513766169548,
+      "step": 370
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.2426422498364946e-06,
+      "logits/chosen": -2.3307952880859375,
+      "logits/rejected": -2.042811393737793,
+      "logps/chosen": -216.83474731445312,
+      "logps/rejected": -161.6975860595703,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.04813487082719803,
+      "rewards/margins": 0.024402152746915817,
+      "rewards/rejected": 0.02373271808028221,
+      "step": 380
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.2753433616742968e-06,
+      "logits/chosen": -2.329103946685791,
+      "logits/rejected": -2.249396562576294,
+      "logps/chosen": -181.30165100097656,
+      "logps/rejected": -244.35458374023438,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.039993561804294586,
+      "rewards/margins": 0.023708099499344826,
+      "rewards/rejected": 0.01628546044230461,
+      "step": 390
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.3080444735120995e-06,
+      "logits/chosen": -2.490332841873169,
+      "logits/rejected": -2.120997428894043,
+      "logps/chosen": -220.3374481201172,
+      "logps/rejected": -178.84103393554688,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.05155234411358833,
+      "rewards/margins": 0.0375380739569664,
+      "rewards/rejected": 0.014014266431331635,
+      "step": 400
+    },
+    {
+      "epoch": 0.03,
+      "eval_logits/chosen": -2.354814052581787,
+      "eval_logits/rejected": -2.1659581661224365,
+      "eval_logps/chosen": -227.3407745361328,
+      "eval_logps/rejected": -209.3865509033203,
+      "eval_loss": 0.6920492649078369,
+      "eval_rewards/accuracies": 0.6019999980926514,
+      "eval_rewards/chosen": 0.04664193466305733,
+      "eval_rewards/margins": 0.02438831515610218,
+      "eval_rewards/rejected": 0.022253619506955147,
+      "eval_runtime": 707.4391,
+      "eval_samples_per_second": 2.827,
+      "eval_steps_per_second": 1.414,
+      "step": 400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.3407455853499021e-06,
+      "logits/chosen": -2.4750816822052,
+      "logits/rejected": -2.324173927307129,
+      "logps/chosen": -257.1285095214844,
+      "logps/rejected": -223.27047729492188,
+      "loss": 0.692,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.05224750190973282,
+      "rewards/margins": 0.025383714586496353,
+      "rewards/rejected": 0.026863792911171913,
+      "step": 410
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.3734466971877046e-06,
+      "logits/chosen": -2.3002800941467285,
+      "logits/rejected": -2.1997017860412598,
+      "logps/chosen": -177.15582275390625,
+      "logps/rejected": -172.67115783691406,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.04763823747634888,
+      "rewards/margins": 0.029840771108865738,
+      "rewards/rejected": 0.01779746450483799,
+      "step": 420
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.406147809025507e-06,
+      "logits/chosen": -2.2951817512512207,
+      "logits/rejected": -2.1071887016296387,
+      "logps/chosen": -209.26852416992188,
+      "logps/rejected": -180.22879028320312,
+      "loss": 0.691,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.05878716707229614,
+      "rewards/margins": 0.03459661826491356,
+      "rewards/rejected": 0.024190548807382584,
+      "step": 430
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.4388489208633094e-06,
+      "logits/chosen": -2.3935062885284424,
+      "logits/rejected": -2.108060121536255,
+      "logps/chosen": -249.3080596923828,
+      "logps/rejected": -221.1660919189453,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.05189204961061478,
+      "rewards/margins": 0.017697608098387718,
+      "rewards/rejected": 0.03419443964958191,
+      "step": 440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.471550032701112e-06,
+      "logits/chosen": -2.412630558013916,
+      "logits/rejected": -2.2085201740264893,
+      "logps/chosen": -184.7351531982422,
+      "logps/rejected": -194.98269653320312,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.04912736266851425,
+      "rewards/margins": 0.04446180909872055,
+      "rewards/rejected": 0.004665557760745287,
+      "step": 450
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5042511445389143e-06,
+      "logits/chosen": -2.159377336502075,
+      "logits/rejected": -2.205676555633545,
+      "logps/chosen": -149.52809143066406,
+      "logps/rejected": -215.054931640625,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.014119003899395466,
+      "rewards/margins": 0.02100800909101963,
+      "rewards/rejected": -0.006889003328979015,
+      "step": 460
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.536952256376717e-06,
+      "logits/chosen": -2.049952983856201,
+      "logits/rejected": -2.111029624938965,
+      "logps/chosen": -199.77395629882812,
+      "logps/rejected": -250.3234405517578,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.011566747911274433,
+      "rewards/margins": 0.0498543456196785,
+      "rewards/rejected": -0.03828759863972664,
+      "step": 470
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5696533682145194e-06,
+      "logits/chosen": -2.448256731033325,
+      "logits/rejected": -2.215850353240967,
+      "logps/chosen": -178.30746459960938,
+      "logps/rejected": -151.38975524902344,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.05939297750592232,
+      "rewards/margins": 0.04141029715538025,
+      "rewards/rejected": 0.01798268035054207,
+      "step": 480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.602354480052322e-06,
+      "logits/chosen": -2.4165821075439453,
+      "logits/rejected": -2.3280694484710693,
+      "logps/chosen": -259.0218811035156,
+      "logps/rejected": -207.7999267578125,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.07617911696434021,
+      "rewards/margins": 0.03729747608304024,
+      "rewards/rejected": 0.03888164460659027,
+      "step": 490
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.6350555918901245e-06,
+      "logits/chosen": -2.21075701713562,
+      "logits/rejected": -1.8819067478179932,
+      "logps/chosen": -213.0443572998047,
+      "logps/rejected": -203.5986328125,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.05024771764874458,
+      "rewards/margins": 0.0509779155254364,
+      "rewards/rejected": -0.0007301971199922264,
+      "step": 500
+    },
+    {
+      "epoch": 0.03,
+      "eval_logits/chosen": -2.349929094314575,
+      "eval_logits/rejected": -2.1615867614746094,
+      "eval_logps/chosen": -225.62718200683594,
+      "eval_logps/rejected": -209.42613220214844,
+      "eval_loss": 0.6916049718856812,
+      "eval_rewards/accuracies": 0.6060000061988831,
+      "eval_rewards/chosen": 0.06377778202295303,
+      "eval_rewards/margins": 0.04191993921995163,
+      "eval_rewards/rejected": 0.021857835352420807,
+      "eval_runtime": 704.9072,
+      "eval_samples_per_second": 2.837,
+      "eval_steps_per_second": 1.419,
+      "step": 500
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.6677567037279269e-06,
+      "logits/chosen": -2.4886791706085205,
+      "logits/rejected": -2.138219118118286,
+      "logps/chosen": -287.70098876953125,
+      "logps/rejected": -252.02603149414062,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.085626520216465,
+      "rewards/margins": 0.04710019752383232,
+      "rewards/rejected": 0.038526326417922974,
+      "step": 510
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.7004578155657295e-06,
+      "logits/chosen": -2.2930407524108887,
+      "logits/rejected": -2.316594362258911,
+      "logps/chosen": -199.58074951171875,
+      "logps/rejected": -185.34860229492188,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0713859498500824,
+      "rewards/margins": 0.061885036528110504,
+      "rewards/rejected": 0.009500918909907341,
+      "step": 520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.7331589274035318e-06,
+      "logits/chosen": -2.1680140495300293,
+      "logits/rejected": -2.0505690574645996,
+      "logps/chosen": -180.614990234375,
+      "logps/rejected": -186.27236938476562,
+      "loss": 0.692,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0331156924366951,
+      "rewards/margins": 0.06211583688855171,
+      "rewards/rejected": -0.029000144451856613,
+      "step": 530
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.7658600392413344e-06,
+      "logits/chosen": -2.396062135696411,
+      "logits/rejected": -2.2367682456970215,
+      "logps/chosen": -199.58575439453125,
+      "logps/rejected": -187.2077178955078,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.032696597278118134,
+      "rewards/margins": 0.049915581941604614,
+      "rewards/rejected": -0.017218980938196182,
+      "step": 540
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.7985611510791368e-06,
+      "logits/chosen": -2.396876811981201,
+      "logits/rejected": -1.942486047744751,
+      "logps/chosen": -256.23150634765625,
+      "logps/rejected": -247.9710235595703,
+      "loss": 0.692,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.0731818825006485,
+      "rewards/margins": 0.08128007501363754,
+      "rewards/rejected": -0.008098193444311619,
+      "step": 550
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.8312622629169393e-06,
+      "logits/chosen": -2.3080027103424072,
+      "logits/rejected": -2.1690850257873535,
+      "logps/chosen": -257.26641845703125,
+      "logps/rejected": -226.06912231445312,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.06187974289059639,
+      "rewards/margins": 0.03323299437761307,
+      "rewards/rejected": 0.028646748512983322,
+      "step": 560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.8639633747547417e-06,
+      "logits/chosen": -2.3971519470214844,
+      "logits/rejected": -2.228024482727051,
+      "logps/chosen": -224.38430786132812,
+      "logps/rejected": -191.48605346679688,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.04760807007551193,
+      "rewards/margins": 0.027385840192437172,
+      "rewards/rejected": 0.02022222802042961,
+      "step": 570
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.8966644865925443e-06,
+      "logits/chosen": -2.246232748031616,
+      "logits/rejected": -2.271991729736328,
+      "logps/chosen": -234.4510498046875,
+      "logps/rejected": -250.18887329101562,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.04757726192474365,
+      "rewards/margins": 0.043433815240859985,
+      "rewards/rejected": 0.0041434429585933685,
+      "step": 580
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.9293655984303466e-06,
+      "logits/chosen": -2.657580852508545,
+      "logits/rejected": -2.2350101470947266,
+      "logps/chosen": -282.17510986328125,
+      "logps/rejected": -204.83383178710938,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.007911397144198418,
+      "rewards/margins": 0.056634120643138885,
+      "rewards/rejected": -0.04872272536158562,
+      "step": 590
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.9620667102681494e-06,
+      "logits/chosen": -2.393247127532959,
+      "logits/rejected": -2.4066872596740723,
+      "logps/chosen": -187.5821533203125,
+      "logps/rejected": -175.9160614013672,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": 0.016703059896826744,
+      "rewards/margins": 0.026545118540525436,
+      "rewards/rejected": -0.009842058643698692,
+      "step": 600
+    },
+    {
+      "epoch": 0.04,
+      "eval_logits/chosen": -2.3568413257598877,
+      "eval_logits/rejected": -2.1674532890319824,
+      "eval_logps/chosen": -227.02455139160156,
+      "eval_logps/rejected": -211.3561248779297,
+      "eval_loss": 0.6913270354270935,
+      "eval_rewards/accuracies": 0.597000002861023,
+      "eval_rewards/chosen": 0.04980393126606941,
+      "eval_rewards/margins": 0.04724626615643501,
+      "eval_rewards/rejected": 0.00255767023190856,
+      "eval_runtime": 708.4382,
+      "eval_samples_per_second": 2.823,
+      "eval_steps_per_second": 1.412,
+      "step": 600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.994767822105952e-06,
+      "logits/chosen": -2.306018829345703,
+      "logits/rejected": -2.208861827850342,
+      "logps/chosen": -171.64891052246094,
+      "logps/rejected": -183.595947265625,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.03554733842611313,
+      "rewards/margins": 0.055120062083005905,
+      "rewards/rejected": -0.019572719931602478,
+      "step": 610
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0274689339437543e-06,
+      "logits/chosen": -2.233059883117676,
+      "logits/rejected": -2.0187458992004395,
+      "logps/chosen": -270.9899597167969,
+      "logps/rejected": -238.4331817626953,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.07699786126613617,
+      "rewards/margins": 0.027974560856819153,
+      "rewards/rejected": 0.04902329668402672,
+      "step": 620
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0601700457815567e-06,
+      "logits/chosen": -2.382429361343384,
+      "logits/rejected": -2.015047788619995,
+      "logps/chosen": -253.5185089111328,
+      "logps/rejected": -211.54025268554688,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.05836200714111328,
+      "rewards/margins": 0.015349363908171654,
+      "rewards/rejected": 0.04301264509558678,
+      "step": 630
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.092871157619359e-06,
+      "logits/chosen": -2.4374704360961914,
+      "logits/rejected": -2.250887393951416,
+      "logps/chosen": -169.18408203125,
+      "logps/rejected": -187.0492706298828,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.07478086650371552,
+      "rewards/margins": 0.041536975651979446,
+      "rewards/rejected": 0.03324388712644577,
+      "step": 640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.1255722694571616e-06,
+      "logits/chosen": -2.44903302192688,
+      "logits/rejected": -2.05329966545105,
+      "logps/chosen": -271.42828369140625,
+      "logps/rejected": -182.59605407714844,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.07466375082731247,
+      "rewards/margins": 0.041353899985551834,
+      "rewards/rejected": 0.033309854567050934,
+      "step": 650
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.158273381294964e-06,
+      "logits/chosen": -2.3471286296844482,
+      "logits/rejected": -2.2432503700256348,
+      "logps/chosen": -207.7042236328125,
+      "logps/rejected": -214.0726776123047,
+      "loss": 0.6938,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.04722776263952255,
+      "rewards/margins": 0.008569743484258652,
+      "rewards/rejected": 0.0386580154299736,
+      "step": 660
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.190974493132767e-06,
+      "logits/chosen": -2.3504185676574707,
+      "logits/rejected": -2.0488924980163574,
+      "logps/chosen": -233.1539306640625,
+      "logps/rejected": -181.9641571044922,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.04525148868560791,
+      "rewards/margins": 0.05067021772265434,
+      "rewards/rejected": -0.005418726243078709,
+      "step": 670
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.223675604970569e-06,
+      "logits/chosen": -2.4212958812713623,
+      "logits/rejected": -2.2168402671813965,
+      "logps/chosen": -204.07693481445312,
+      "logps/rejected": -190.05075073242188,
+      "loss": 0.692,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.06657350063323975,
+      "rewards/margins": 0.033041100949048996,
+      "rewards/rejected": 0.03353239223361015,
+      "step": 680
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.2563767168083718e-06,
+      "logits/chosen": -2.495441436767578,
+      "logits/rejected": -2.002847194671631,
+      "logps/chosen": -250.1965789794922,
+      "logps/rejected": -191.36761474609375,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.07055240124464035,
+      "rewards/margins": 0.04203369468450546,
+      "rewards/rejected": 0.028518706560134888,
+      "step": 690
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.289077828646174e-06,
+      "logits/chosen": -2.32529354095459,
+      "logits/rejected": -2.090954542160034,
+      "logps/chosen": -234.6331329345703,
+      "logps/rejected": -214.04269409179688,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.04427819699048996,
+      "rewards/margins": 0.04115144535899162,
+      "rewards/rejected": 0.0031267497688531876,
+      "step": 700
+    },
+    {
+      "epoch": 0.05,
+      "eval_logits/chosen": -2.350055456161499,
+      "eval_logits/rejected": -2.161485433578491,
+      "eval_logps/chosen": -226.39279174804688,
+      "eval_logps/rejected": -210.55441284179688,
+      "eval_loss": 0.6913213133811951,
+      "eval_rewards/accuracies": 0.6144999861717224,
+      "eval_rewards/chosen": 0.056121550500392914,
+      "eval_rewards/margins": 0.04554666578769684,
+      "eval_rewards/rejected": 0.010574882850050926,
+      "eval_runtime": 706.9392,
+      "eval_samples_per_second": 2.829,
+      "eval_steps_per_second": 1.415,
+      "step": 700
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.3217789404839766e-06,
+      "logits/chosen": -2.2059988975524902,
+      "logits/rejected": -2.29160737991333,
+      "logps/chosen": -154.5279541015625,
+      "logps/rejected": -204.50648498535156,
+      "loss": 0.691,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.023031946271657944,
+      "rewards/margins": 0.03035845421254635,
+      "rewards/rejected": -0.007326505146920681,
+      "step": 710
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.354480052321779e-06,
+      "logits/chosen": -2.52923321723938,
+      "logits/rejected": -2.1005005836486816,
+      "logps/chosen": -253.52212524414062,
+      "logps/rejected": -201.50367736816406,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.048673033714294434,
+      "rewards/margins": 0.0762287974357605,
+      "rewards/rejected": -0.027555758133530617,
+      "step": 720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.3871811641595815e-06,
+      "logits/chosen": -2.3575785160064697,
+      "logits/rejected": -2.1780619621276855,
+      "logps/chosen": -252.4840087890625,
+      "logps/rejected": -198.02867126464844,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.056915633380413055,
+      "rewards/margins": 0.042005524039268494,
+      "rewards/rejected": -0.09892116487026215,
+      "step": 730
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.4198822759973843e-06,
+      "logits/chosen": -2.1879830360412598,
+      "logits/rejected": -2.2039308547973633,
+      "logps/chosen": -209.80526733398438,
+      "logps/rejected": -226.54446411132812,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.19199207425117493,
+      "rewards/margins": 0.028066366910934448,
+      "rewards/rejected": -0.220058411359787,
+      "step": 740
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.4525833878351864e-06,
+      "logits/chosen": -2.4402458667755127,
+      "logits/rejected": -2.2395517826080322,
+      "logps/chosen": -271.01220703125,
+      "logps/rejected": -208.87350463867188,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.18211308121681213,
+      "rewards/margins": 0.04141292721033096,
+      "rewards/rejected": -0.2235260307788849,
+      "step": 750
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.4852844996729892e-06,
+      "logits/chosen": -2.1985716819763184,
+      "logits/rejected": -2.10333514213562,
+      "logps/chosen": -263.1232604980469,
+      "logps/rejected": -269.29693603515625,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.23998942971229553,
+      "rewards/margins": 0.04715558513998985,
+      "rewards/rejected": -0.2871449887752533,
+      "step": 760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.5179856115107916e-06,
+      "logits/chosen": -2.3487696647644043,
+      "logits/rejected": -2.030247211456299,
+      "logps/chosen": -293.29290771484375,
+      "logps/rejected": -244.9794158935547,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.1979665905237198,
+      "rewards/margins": 0.056348543614149094,
+      "rewards/rejected": -0.2543151080608368,
+      "step": 770
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.5506867233485937e-06,
+      "logits/chosen": -2.4455208778381348,
+      "logits/rejected": -2.0224125385284424,
+      "logps/chosen": -275.57421875,
+      "logps/rejected": -228.21737670898438,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.09878290444612503,
+      "rewards/margins": 0.10141804069280624,
+      "rewards/rejected": -0.20020096004009247,
+      "step": 780
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.5833878351863965e-06,
+      "logits/chosen": -2.4715168476104736,
+      "logits/rejected": -2.403022050857544,
+      "logps/chosen": -257.7041931152344,
+      "logps/rejected": -262.29986572265625,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.05706269294023514,
+      "rewards/margins": 0.07216020673513412,
+      "rewards/rejected": -0.12922289967536926,
+      "step": 790
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.616088947024199e-06,
+      "logits/chosen": -2.155651092529297,
+      "logits/rejected": -2.314833164215088,
+      "logps/chosen": -206.03012084960938,
+      "logps/rejected": -235.5647430419922,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.09540672600269318,
+      "rewards/margins": 0.050082337111234665,
+      "rewards/rejected": -0.14548906683921814,
+      "step": 800
+    },
+    {
+      "epoch": 0.05,
+      "eval_logits/chosen": -2.330695390701294,
+      "eval_logits/rejected": -2.142754316329956,
+      "eval_logps/chosen": -242.4707794189453,
+      "eval_logps/rejected": -227.2015838623047,
+      "eval_loss": 0.6913210153579712,
+      "eval_rewards/accuracies": 0.597000002861023,
+      "eval_rewards/chosen": -0.1046583503484726,
+      "eval_rewards/margins": 0.051238518208265305,
+      "eval_rewards/rejected": -0.155896857380867,
+      "eval_runtime": 705.7692,
+      "eval_samples_per_second": 2.834,
+      "eval_steps_per_second": 1.417,
+      "step": 800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.6487900588620014e-06,
+      "logits/chosen": -2.1080448627471924,
+      "logits/rejected": -1.8789002895355225,
+      "logps/chosen": -209.4484405517578,
+      "logps/rejected": -169.5839080810547,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.09332854300737381,
+      "rewards/margins": 0.02387315407395363,
+      "rewards/rejected": -0.11720170080661774,
+      "step": 810
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.6814911706998042e-06,
+      "logits/chosen": -2.293308734893799,
+      "logits/rejected": -2.1208784580230713,
+      "logps/chosen": -232.0146484375,
+      "logps/rejected": -204.10064697265625,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.06675419211387634,
+      "rewards/margins": 0.028399985283613205,
+      "rewards/rejected": -0.09515418112277985,
+      "step": 820
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.7141922825376067e-06,
+      "logits/chosen": -2.1897878646850586,
+      "logits/rejected": -2.195380210876465,
+      "logps/chosen": -264.03271484375,
+      "logps/rejected": -280.67877197265625,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.06108593940734863,
+      "rewards/margins": 0.06715109199285507,
+      "rewards/rejected": -0.1282370388507843,
+      "step": 830
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.746893394375409e-06,
+      "logits/chosen": -2.3309171199798584,
+      "logits/rejected": -2.2094645500183105,
+      "logps/chosen": -239.76480102539062,
+      "logps/rejected": -241.4962921142578,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.053703807294368744,
+      "rewards/margins": 0.03512780740857124,
+      "rewards/rejected": -0.08883161842823029,
+      "step": 840
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.779594506213211e-06,
+      "logits/chosen": -2.236567258834839,
+      "logits/rejected": -2.002687692642212,
+      "logps/chosen": -194.21266174316406,
+      "logps/rejected": -198.6126708984375,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.04353756457567215,
+      "rewards/margins": 0.047133009880781174,
+      "rewards/rejected": -0.09067057073116302,
+      "step": 850
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.812295618051014e-06,
+      "logits/chosen": -2.4146676063537598,
+      "logits/rejected": -2.2537002563476562,
+      "logps/chosen": -283.446533203125,
+      "logps/rejected": -226.4826202392578,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.024853792041540146,
+      "rewards/margins": 0.04763239994645119,
+      "rewards/rejected": -0.07248619943857193,
+      "step": 860
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.8449967298888164e-06,
+      "logits/chosen": -2.299940347671509,
+      "logits/rejected": -2.095834732055664,
+      "logps/chosen": -180.71926879882812,
+      "logps/rejected": -155.87872314453125,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.03895248472690582,
+      "rewards/margins": 0.03217850998044014,
+      "rewards/rejected": -0.07113099843263626,
+      "step": 870
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.877697841726619e-06,
+      "logits/chosen": -2.38145112991333,
+      "logits/rejected": -2.282527208328247,
+      "logps/chosen": -221.8948211669922,
+      "logps/rejected": -208.37954711914062,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.03059009648859501,
+      "rewards/margins": 0.03299534320831299,
+      "rewards/rejected": -0.06358544528484344,
+      "step": 880
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.9103989535644217e-06,
+      "logits/chosen": -2.3034727573394775,
+      "logits/rejected": -2.4145941734313965,
+      "logps/chosen": -219.39413452148438,
+      "logps/rejected": -252.17977905273438,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.03910509869456291,
+      "rewards/margins": 0.0225521232932806,
+      "rewards/rejected": -0.06165723130106926,
+      "step": 890
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.943100065402224e-06,
+      "logits/chosen": -2.3486392498016357,
+      "logits/rejected": -2.202688455581665,
+      "logps/chosen": -297.0926208496094,
+      "logps/rejected": -285.2032470703125,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.04449423402547836,
+      "rewards/margins": 0.03427337482571602,
+      "rewards/rejected": -0.07876761257648468,
+      "step": 900
+    },
+    {
+      "epoch": 0.06,
+      "eval_logits/chosen": -2.3343303203582764,
+      "eval_logits/rejected": -2.146639347076416,
+      "eval_logps/chosen": -237.26773071289062,
+      "eval_logps/rejected": -221.73362731933594,
+      "eval_loss": 0.6909335255622864,
+      "eval_rewards/accuracies": 0.6060000061988831,
+      "eval_rewards/chosen": -0.05262775719165802,
+      "eval_rewards/margins": 0.04858950152993202,
+      "eval_rewards/rejected": -0.10121726244688034,
+      "eval_runtime": 708.5552,
+      "eval_samples_per_second": 2.823,
+      "eval_steps_per_second": 1.411,
+      "step": 900
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.9758011772400266e-06,
+      "logits/chosen": -2.2568647861480713,
+      "logits/rejected": -2.2534162998199463,
+      "logps/chosen": -263.49017333984375,
+      "logps/rejected": -273.72900390625,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.06180988624691963,
+      "rewards/margins": 0.028642665594816208,
+      "rewards/rejected": -0.09045255184173584,
+      "step": 910
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.0085022890778286e-06,
+      "logits/chosen": -2.3166985511779785,
+      "logits/rejected": -2.0645124912261963,
+      "logps/chosen": -181.6991424560547,
+      "logps/rejected": -157.57730102539062,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.05959589406847954,
+      "rewards/margins": 0.02801639400422573,
+      "rewards/rejected": -0.08761228621006012,
+      "step": 920
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.0412034009156314e-06,
+      "logits/chosen": -2.258695363998413,
+      "logits/rejected": -2.4108707904815674,
+      "logps/chosen": -240.5486297607422,
+      "logps/rejected": -240.31802368164062,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.04862401634454727,
+      "rewards/margins": 0.040310461074113846,
+      "rewards/rejected": -0.08893446624279022,
+      "step": 930
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.073904512753434e-06,
+      "logits/chosen": -2.4486923217773438,
+      "logits/rejected": -2.0814051628112793,
+      "logps/chosen": -231.5814208984375,
+      "logps/rejected": -202.97706604003906,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.01932818815112114,
+      "rewards/margins": 0.06323892623186111,
+      "rewards/rejected": -0.08256711810827255,
+      "step": 940
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.1066056245912363e-06,
+      "logits/chosen": -2.308929920196533,
+      "logits/rejected": -2.383852958679199,
+      "logps/chosen": -233.48721313476562,
+      "logps/rejected": -207.38430786132812,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.025032073259353638,
+      "rewards/margins": 0.05856107547879219,
+      "rewards/rejected": -0.08359314501285553,
+      "step": 950
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.1393067364290387e-06,
+      "logits/chosen": -2.3422508239746094,
+      "logits/rejected": -2.1356518268585205,
+      "logps/chosen": -227.9677734375,
+      "logps/rejected": -198.36422729492188,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.028410235419869423,
+      "rewards/margins": 0.08982036262750626,
+      "rewards/rejected": -0.11823059618473053,
+      "step": 960
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.1720078482668416e-06,
+      "logits/chosen": -2.2739663124084473,
+      "logits/rejected": -2.042900562286377,
+      "logps/chosen": -214.7518310546875,
+      "logps/rejected": -176.07008361816406,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.031811777502298355,
+      "rewards/margins": 0.08425115048885345,
+      "rewards/rejected": -0.11606292426586151,
+      "step": 970
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.204708960104644e-06,
+      "logits/chosen": -2.3709425926208496,
+      "logits/rejected": -2.0238471031188965,
+      "logps/chosen": -218.3297119140625,
+      "logps/rejected": -194.80873107910156,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.009269696660339832,
+      "rewards/margins": 0.09066729247570038,
+      "rewards/rejected": -0.09993697702884674,
+      "step": 980
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.237410071942446e-06,
+      "logits/chosen": -2.4116098880767822,
+      "logits/rejected": -2.1458096504211426,
+      "logps/chosen": -208.81069946289062,
+      "logps/rejected": -177.1934051513672,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.024496430531144142,
+      "rewards/margins": 0.01946501061320305,
+      "rewards/rejected": -0.04396144300699234,
+      "step": 990
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.270111183780249e-06,
+      "logits/chosen": -2.3320465087890625,
+      "logits/rejected": -2.106236219406128,
+      "logps/chosen": -201.52783203125,
+      "logps/rejected": -188.52328491210938,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.015042750164866447,
+      "rewards/margins": 0.04998449236154556,
+      "rewards/rejected": -0.06502724438905716,
+      "step": 1000
+    },
+    {
+      "epoch": 0.07,
+      "eval_logits/chosen": -2.3452556133270264,
+      "eval_logits/rejected": -2.157487154006958,
+      "eval_logps/chosen": -232.08251953125,
+      "eval_logps/rejected": -217.237060546875,
+      "eval_loss": 0.6907655000686646,
+      "eval_rewards/accuracies": 0.6184999942779541,
+      "eval_rewards/chosen": -0.0007757164421491325,
+      "eval_rewards/margins": 0.05547565594315529,
+      "eval_rewards/rejected": -0.05625137314200401,
+      "eval_runtime": 709.7867,
+      "eval_samples_per_second": 2.818,
+      "eval_steps_per_second": 1.409,
+      "step": 1000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.3028122956180513e-06,
+      "logits/chosen": -2.202904462814331,
+      "logits/rejected": -2.257141351699829,
+      "logps/chosen": -211.8687744140625,
+      "logps/rejected": -244.3396759033203,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0022913392167538404,
+      "rewards/margins": 0.05001994967460632,
+      "rewards/rejected": -0.047728605568408966,
+      "step": 1010
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.3355134074558538e-06,
+      "logits/chosen": -2.197819948196411,
+      "logits/rejected": -2.1015243530273438,
+      "logps/chosen": -222.99203491210938,
+      "logps/rejected": -202.92996215820312,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.022592799738049507,
+      "rewards/margins": 0.0552542582154274,
+      "rewards/rejected": -0.07784706354141235,
+      "step": 1020
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.368214519293656e-06,
+      "logits/chosen": -2.1490020751953125,
+      "logits/rejected": -1.9399007558822632,
+      "logps/chosen": -200.19967651367188,
+      "logps/rejected": -184.1601104736328,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.012293432839214802,
+      "rewards/margins": 0.04858936741948128,
+      "rewards/rejected": -0.060882795602083206,
+      "step": 1030
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.400915631131459e-06,
+      "logits/chosen": -2.1254327297210693,
+      "logits/rejected": -2.1563661098480225,
+      "logps/chosen": -200.5404052734375,
+      "logps/rejected": -237.29293823242188,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.025679444894194603,
+      "rewards/margins": 0.08101227134466171,
+      "rewards/rejected": -0.10669170320034027,
+      "step": 1040
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.4336167429692615e-06,
+      "logits/chosen": -2.3749256134033203,
+      "logits/rejected": -2.1596624851226807,
+      "logps/chosen": -216.2527618408203,
+      "logps/rejected": -205.42910766601562,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.017023511230945587,
+      "rewards/margins": 0.054301291704177856,
+      "rewards/rejected": -0.07132480293512344,
+      "step": 1050
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.4663178548070635e-06,
+      "logits/chosen": -2.1830973625183105,
+      "logits/rejected": -2.239147424697876,
+      "logps/chosen": -214.9247589111328,
+      "logps/rejected": -203.87339782714844,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0002921056002378464,
+      "rewards/margins": 0.053903043270111084,
+      "rewards/rejected": -0.05361093208193779,
+      "step": 1060
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.499018966644866e-06,
+      "logits/chosen": -2.3346736431121826,
+      "logits/rejected": -2.139986038208008,
+      "logps/chosen": -193.098388671875,
+      "logps/rejected": -192.88113403320312,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.012733638286590576,
+      "rewards/margins": 0.06595107167959213,
+      "rewards/rejected": -0.0786847174167633,
+      "step": 1070
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.531720078482669e-06,
+      "logits/chosen": -2.2687487602233887,
+      "logits/rejected": -1.93032968044281,
+      "logps/chosen": -225.96875,
+      "logps/rejected": -222.5755615234375,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0009903141763061285,
+      "rewards/margins": 0.07838054746389389,
+      "rewards/rejected": -0.07937086373567581,
+      "step": 1080
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.5644211903204712e-06,
+      "logits/chosen": -2.2722859382629395,
+      "logits/rejected": -2.3190114498138428,
+      "logps/chosen": -195.7224884033203,
+      "logps/rejected": -204.1152801513672,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.031599871814250946,
+      "rewards/margins": 0.05579303950071335,
+      "rewards/rejected": -0.024193167686462402,
+      "step": 1090
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.5971223021582737e-06,
+      "logits/chosen": -2.429405927658081,
+      "logits/rejected": -2.290315866470337,
+      "logps/chosen": -257.29278564453125,
+      "logps/rejected": -203.24649047851562,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.014919871464371681,
+      "rewards/margins": 0.030690353363752365,
+      "rewards/rejected": -0.045610226690769196,
+      "step": 1100
+    },
+    {
+      "epoch": 0.07,
+      "eval_logits/chosen": -2.3150722980499268,
+      "eval_logits/rejected": -2.1294045448303223,
+      "eval_logps/chosen": -232.15647888183594,
+      "eval_logps/rejected": -219.40240478515625,
+      "eval_loss": 0.691135585308075,
+      "eval_rewards/accuracies": 0.6274999976158142,
+      "eval_rewards/chosen": -0.0015151738189160824,
+      "eval_rewards/margins": 0.07638993859291077,
+      "eval_rewards/rejected": -0.07790511101484299,
+      "eval_runtime": 706.9471,
+      "eval_samples_per_second": 2.829,
+      "eval_steps_per_second": 1.415,
+      "step": 1100
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.6298234139960765e-06,
+      "logits/chosen": -2.3906145095825195,
+      "logits/rejected": -2.036782741546631,
+      "logps/chosen": -216.01156616210938,
+      "logps/rejected": -173.99920654296875,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.017574917525053024,
+      "rewards/margins": 0.07658959180116653,
+      "rewards/rejected": -0.05901466682553291,
+      "step": 1110
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.6625245258338785e-06,
+      "logits/chosen": -2.191586971282959,
+      "logits/rejected": -2.110297679901123,
+      "logps/chosen": -233.83700561523438,
+      "logps/rejected": -320.0827331542969,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0494137704372406,
+      "rewards/margins": 0.09785051643848419,
+      "rewards/rejected": -0.04843674600124359,
+      "step": 1120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.695225637671681e-06,
+      "logits/chosen": -2.442321300506592,
+      "logits/rejected": -2.241528272628784,
+      "logps/chosen": -205.39053344726562,
+      "logps/rejected": -178.85687255859375,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.018860872834920883,
+      "rewards/margins": 0.06542352586984634,
+      "rewards/rejected": -0.04656265676021576,
+      "step": 1130
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.7279267495094834e-06,
+      "logits/chosen": -2.358170986175537,
+      "logits/rejected": -2.220486879348755,
+      "logps/chosen": -159.06790161132812,
+      "logps/rejected": -177.91981506347656,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.013590176589787006,
+      "rewards/margins": 0.08726686984300613,
+      "rewards/rejected": -0.0736766904592514,
+      "step": 1140
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.7606278613472863e-06,
+      "logits/chosen": -2.4245522022247314,
+      "logits/rejected": -2.0788369178771973,
+      "logps/chosen": -277.3457946777344,
+      "logps/rejected": -205.87890625,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.04925479739904404,
+      "rewards/margins": 0.061012279242277145,
+      "rewards/rejected": -0.01175748836249113,
+      "step": 1150
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.7933289731850887e-06,
+      "logits/chosen": -2.227184772491455,
+      "logits/rejected": -2.006626605987549,
+      "logps/chosen": -220.76419067382812,
+      "logps/rejected": -200.02944946289062,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.07796461135149002,
+      "rewards/margins": 0.07523587346076965,
+      "rewards/rejected": 0.002728732768446207,
+      "step": 1160
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.826030085022891e-06,
+      "logits/chosen": -2.394782543182373,
+      "logits/rejected": -2.1593916416168213,
+      "logps/chosen": -244.3356475830078,
+      "logps/rejected": -212.73583984375,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0331416018307209,
+      "rewards/margins": 0.04552285000681877,
+      "rewards/rejected": -0.012381250038743019,
+      "step": 1170
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.858731196860693e-06,
+      "logits/chosen": -2.4333834648132324,
+      "logits/rejected": -1.9844642877578735,
+      "logps/chosen": -249.99893188476562,
+      "logps/rejected": -210.4248504638672,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.008344946429133415,
+      "rewards/margins": 0.026615191251039505,
+      "rewards/rejected": -0.01827024295926094,
+      "step": 1180
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.891432308698496e-06,
+      "logits/chosen": -2.120863437652588,
+      "logits/rejected": -2.063352108001709,
+      "logps/chosen": -193.93240356445312,
+      "logps/rejected": -226.5436553955078,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.01297982782125473,
+      "rewards/margins": 0.07365990430116653,
+      "rewards/rejected": -0.060680072754621506,
+      "step": 1190
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.924133420536299e-06,
+      "logits/chosen": -2.2395856380462646,
+      "logits/rejected": -2.041903018951416,
+      "logps/chosen": -184.75523376464844,
+      "logps/rejected": -157.53878784179688,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.026105085387825966,
+      "rewards/margins": 0.061862241476774216,
+      "rewards/rejected": -0.08796733617782593,
+      "step": 1200
+    },
+    {
+      "epoch": 0.08,
+      "eval_logits/chosen": -2.3272151947021484,
+      "eval_logits/rejected": -2.1397786140441895,
+      "eval_logps/chosen": -234.7645263671875,
+      "eval_logps/rejected": -221.4020538330078,
+      "eval_loss": 0.6907363533973694,
+      "eval_rewards/accuracies": 0.637499988079071,
+      "eval_rewards/chosen": -0.027595827355980873,
+      "eval_rewards/margins": 0.07030569016933441,
+      "eval_rewards/rejected": -0.09790151566267014,
+      "eval_runtime": 706.2261,
+      "eval_samples_per_second": 2.832,
+      "eval_steps_per_second": 1.416,
+      "step": 1200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.956834532374101e-06,
+      "logits/chosen": -2.4392762184143066,
+      "logits/rejected": -2.0458781719207764,
+      "logps/chosen": -206.30532836914062,
+      "logps/rejected": -173.46829223632812,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.010426463559269905,
+      "rewards/margins": 0.11114281415939331,
+      "rewards/rejected": -0.12156929075717926,
+      "step": 1210
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.989535644211904e-06,
+      "logits/chosen": -2.329332113265991,
+      "logits/rejected": -2.0607223510742188,
+      "logps/chosen": -216.85635375976562,
+      "logps/rejected": -202.00527954101562,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.00270930677652359,
+      "rewards/margins": 0.07769973576068878,
+      "rewards/rejected": -0.07499042898416519,
+      "step": 1220
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.022236756049706e-06,
+      "logits/chosen": -2.5103940963745117,
+      "logits/rejected": -2.1222851276397705,
+      "logps/chosen": -273.70758056640625,
+      "logps/rejected": -246.398681640625,
+      "loss": 0.691,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.03429547697305679,
+      "rewards/margins": 0.05166538804769516,
+      "rewards/rejected": -0.017369914799928665,
+      "step": 1230
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.054937867887509e-06,
+      "logits/chosen": -2.3860554695129395,
+      "logits/rejected": -1.9378671646118164,
+      "logps/chosen": -219.5018310546875,
+      "logps/rejected": -214.9021759033203,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.04764958843588829,
+      "rewards/margins": 0.06188775226473808,
+      "rewards/rejected": -0.014238161034882069,
+      "step": 1240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.087638979725311e-06,
+      "logits/chosen": -2.3056764602661133,
+      "logits/rejected": -2.371333599090576,
+      "logps/chosen": -224.9440155029297,
+      "logps/rejected": -230.5066375732422,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.02895757555961609,
+      "rewards/margins": 0.04107601195573807,
+      "rewards/rejected": -0.01211843267083168,
+      "step": 1250
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.1203400915631135e-06,
+      "logits/chosen": -2.221919059753418,
+      "logits/rejected": -2.146714687347412,
+      "logps/chosen": -237.0450439453125,
+      "logps/rejected": -214.48135375976562,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.002859347965568304,
+      "rewards/margins": 0.05717097595334053,
+      "rewards/rejected": -0.05431162193417549,
+      "step": 1260
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.153041203400916e-06,
+      "logits/chosen": -2.4315924644470215,
+      "logits/rejected": -2.265227794647217,
+      "logps/chosen": -259.8785705566406,
+      "logps/rejected": -239.6594696044922,
+      "loss": 0.693,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.04543764516711235,
+      "rewards/margins": 0.044155552983284,
+      "rewards/rejected": 0.0012820929987356067,
+      "step": 1270
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.185742315238718e-06,
+      "logits/chosen": -2.343573570251465,
+      "logits/rejected": -2.0737881660461426,
+      "logps/chosen": -194.74240112304688,
+      "logps/rejected": -188.17759704589844,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.03359198197722435,
+      "rewards/margins": 0.05302148312330246,
+      "rewards/rejected": -0.01942949742078781,
+      "step": 1280
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.218443427076521e-06,
+      "logits/chosen": -2.289780616760254,
+      "logits/rejected": -2.1402783393859863,
+      "logps/chosen": -181.81228637695312,
+      "logps/rejected": -185.71270751953125,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.00946279987692833,
+      "rewards/margins": 0.02429373934864998,
+      "rewards/rejected": -0.014830941334366798,
+      "step": 1290
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.251144538914323e-06,
+      "logits/chosen": -2.2517518997192383,
+      "logits/rejected": -2.113873243331909,
+      "logps/chosen": -218.7006378173828,
+      "logps/rejected": -213.1343536376953,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.018896425142884254,
+      "rewards/margins": 0.07629399746656418,
+      "rewards/rejected": -0.05739758163690567,
+      "step": 1300
+    },
+    {
+      "epoch": 0.09,
+      "eval_logits/chosen": -2.3501036167144775,
+      "eval_logits/rejected": -2.161346673965454,
+      "eval_logps/chosen": -230.54745483398438,
+      "eval_logps/rejected": -215.89462280273438,
+      "eval_loss": 0.6907321810722351,
+      "eval_rewards/accuracies": 0.6104999780654907,
+      "eval_rewards/chosen": 0.014575082808732986,
+      "eval_rewards/margins": 0.05740221589803696,
+      "eval_rewards/rejected": -0.04282712936401367,
+      "eval_runtime": 708.7595,
+      "eval_samples_per_second": 2.822,
+      "eval_steps_per_second": 1.411,
+      "step": 1300
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.283845650752126e-06,
+      "logits/chosen": -2.379955291748047,
+      "logits/rejected": -2.1668858528137207,
+      "logps/chosen": -285.83856201171875,
+      "logps/rejected": -227.052490234375,
+      "loss": 0.691,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.003734259633347392,
+      "rewards/margins": 0.05405404418706894,
+      "rewards/rejected": -0.05031978338956833,
+      "step": 1310
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.316546762589928e-06,
+      "logits/chosen": -2.3057024478912354,
+      "logits/rejected": -2.2258567810058594,
+      "logps/chosen": -207.4925537109375,
+      "logps/rejected": -178.53604125976562,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.004349817987531424,
+      "rewards/margins": 0.05341663956642151,
+      "rewards/rejected": -0.04906681925058365,
+      "step": 1320
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.349247874427731e-06,
+      "logits/chosen": -2.3624565601348877,
+      "logits/rejected": -2.0772993564605713,
+      "logps/chosen": -228.140869140625,
+      "logps/rejected": -275.2492980957031,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.01606837287545204,
+      "rewards/margins": 0.07025826722383499,
+      "rewards/rejected": -0.08632663637399673,
+      "step": 1330
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.381948986265534e-06,
+      "logits/chosen": -2.505383253097534,
+      "logits/rejected": -2.2410616874694824,
+      "logps/chosen": -262.94122314453125,
+      "logps/rejected": -259.1203918457031,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0009654685854911804,
+      "rewards/margins": 0.09186152368783951,
+      "rewards/rejected": -0.0928269773721695,
+      "step": 1340
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.414650098103336e-06,
+      "logits/chosen": -2.652198314666748,
+      "logits/rejected": -2.3813252449035645,
+      "logps/chosen": -260.88726806640625,
+      "logps/rejected": -242.92202758789062,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.01065239030867815,
+      "rewards/margins": 0.04458843916654587,
+      "rewards/rejected": -0.03393604978919029,
+      "step": 1350
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.447351209941138e-06,
+      "logits/chosen": -2.3565680980682373,
+      "logits/rejected": -2.0831761360168457,
+      "logps/chosen": -215.17153930664062,
+      "logps/rejected": -202.86697387695312,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.05506874993443489,
+      "rewards/margins": 0.05836993455886841,
+      "rewards/rejected": -0.1134386882185936,
+      "step": 1360
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.480052321778941e-06,
+      "logits/chosen": -2.305148124694824,
+      "logits/rejected": -2.3135273456573486,
+      "logps/chosen": -226.86849975585938,
+      "logps/rejected": -212.960205078125,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0004178326635155827,
+      "rewards/margins": 0.06787695735692978,
+      "rewards/rejected": -0.06745912879705429,
+      "step": 1370
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.5127534336167435e-06,
+      "logits/chosen": -2.3873705863952637,
+      "logits/rejected": -2.1096668243408203,
+      "logps/chosen": -252.4434814453125,
+      "logps/rejected": -214.6434783935547,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.052694983780384064,
+      "rewards/margins": 0.058600615710020065,
+      "rewards/rejected": -0.11129560321569443,
+      "step": 1380
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.5454545454545455e-06,
+      "logits/chosen": -2.356358766555786,
+      "logits/rejected": -2.299755096435547,
+      "logps/chosen": -168.05401611328125,
+      "logps/rejected": -167.7144775390625,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.014046875759959221,
+      "rewards/margins": 0.0986950695514679,
+      "rewards/rejected": -0.08464818447828293,
+      "step": 1390
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.578155657292348e-06,
+      "logits/chosen": -2.326817274093628,
+      "logits/rejected": -2.2857398986816406,
+      "logps/chosen": -253.63827514648438,
+      "logps/rejected": -270.8973083496094,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.017673691734671593,
+      "rewards/margins": 0.08337118476629257,
+      "rewards/rejected": -0.06569750607013702,
+      "step": 1400
+    },
+    {
+      "epoch": 0.09,
+      "eval_logits/chosen": -2.3205173015594482,
+      "eval_logits/rejected": -2.1350162029266357,
+      "eval_logps/chosen": -231.281494140625,
+      "eval_logps/rejected": -217.48509216308594,
+      "eval_loss": 0.69090336561203,
+      "eval_rewards/accuracies": 0.6129999756813049,
+      "eval_rewards/chosen": 0.007234419696033001,
+      "eval_rewards/margins": 0.06596639752388,
+      "eval_rewards/rejected": -0.058731988072395325,
+      "eval_runtime": 710.954,
+      "eval_samples_per_second": 2.813,
+      "eval_steps_per_second": 1.407,
+      "step": 1400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.610856769130151e-06,
+      "logits/chosen": -2.3867039680480957,
+      "logits/rejected": -2.243751049041748,
+      "logps/chosen": -238.0634002685547,
+      "logps/rejected": -215.328857421875,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.010808589868247509,
+      "rewards/margins": 0.06767071038484573,
+      "rewards/rejected": -0.05686211585998535,
+      "step": 1410
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.643557880967953e-06,
+      "logits/chosen": -2.3865418434143066,
+      "logits/rejected": -2.1880135536193848,
+      "logps/chosen": -200.36758422851562,
+      "logps/rejected": -201.47171020507812,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.015687663108110428,
+      "rewards/margins": 0.03603903204202652,
+      "rewards/rejected": -0.05172669142484665,
+      "step": 1420
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.676258992805755e-06,
+      "logits/chosen": -2.345919609069824,
+      "logits/rejected": -2.072216510772705,
+      "logps/chosen": -267.54669189453125,
+      "logps/rejected": -233.08798217773438,
+      "loss": 0.69,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.02073250710964203,
+      "rewards/margins": 0.07689642906188965,
+      "rewards/rejected": -0.05616391822695732,
+      "step": 1430
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.708960104643558e-06,
+      "logits/chosen": -2.3562963008880615,
+      "logits/rejected": -2.3137621879577637,
+      "logps/chosen": -306.24517822265625,
+      "logps/rejected": -274.78741455078125,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.00035466160625219345,
+      "rewards/margins": 0.05302319675683975,
+      "rewards/rejected": -0.05337785556912422,
+      "step": 1440
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.741661216481361e-06,
+      "logits/chosen": -2.3189618587493896,
+      "logits/rejected": -2.2773196697235107,
+      "logps/chosen": -237.4014129638672,
+      "logps/rejected": -258.08966064453125,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.010558743961155415,
+      "rewards/margins": 0.09441892802715302,
+      "rewards/rejected": -0.10497768223285675,
+      "step": 1450
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.774362328319163e-06,
+      "logits/chosen": -2.298546552658081,
+      "logits/rejected": -2.144091844558716,
+      "logps/chosen": -217.25430297851562,
+      "logps/rejected": -191.17282104492188,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.014642052352428436,
+      "rewards/margins": 0.038679804652929306,
+      "rewards/rejected": -0.05332186073064804,
+      "step": 1460
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.807063440156966e-06,
+      "logits/chosen": -2.33768367767334,
+      "logits/rejected": -2.1054131984710693,
+      "logps/chosen": -260.9105529785156,
+      "logps/rejected": -208.8505401611328,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.02398931235074997,
+      "rewards/margins": 0.09310416132211685,
+      "rewards/rejected": -0.06911484152078629,
+      "step": 1470
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.839764551994769e-06,
+      "logits/chosen": -2.4007508754730225,
+      "logits/rejected": -2.095411539077759,
+      "logps/chosen": -246.15673828125,
+      "logps/rejected": -215.8951873779297,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.025517677888274193,
+      "rewards/margins": 0.07472027838230133,
+      "rewards/rejected": -0.04920259863138199,
+      "step": 1480
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.872465663832571e-06,
+      "logits/chosen": -2.2471413612365723,
+      "logits/rejected": -2.1273739337921143,
+      "logps/chosen": -207.810302734375,
+      "logps/rejected": -194.42510986328125,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.0005180038278922439,
+      "rewards/margins": 0.07074950635433197,
+      "rewards/rejected": -0.0712675154209137,
+      "step": 1490
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.905166775670373e-06,
+      "logits/chosen": -2.372222661972046,
+      "logits/rejected": -2.034890651702881,
+      "logps/chosen": -220.8203582763672,
+      "logps/rejected": -200.08526611328125,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.00939613301306963,
+      "rewards/margins": 0.06658481061458588,
+      "rewards/rejected": -0.057188671082258224,
+      "step": 1500
+    },
+    {
+      "epoch": 0.1,
+      "eval_logits/chosen": -2.321709394454956,
+      "eval_logits/rejected": -2.1364989280700684,
+      "eval_logps/chosen": -233.14047241210938,
+      "eval_logps/rejected": -220.0596923828125,
+      "eval_loss": 0.6906663179397583,
+      "eval_rewards/accuracies": 0.6305000185966492,
+      "eval_rewards/chosen": -0.011355271562933922,
+      "eval_rewards/margins": 0.07312270253896713,
+      "eval_rewards/rejected": -0.0844779685139656,
+      "eval_runtime": 714.039,
+      "eval_samples_per_second": 2.801,
+      "eval_steps_per_second": 1.4,
+      "step": 1500
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.9378678875081756e-06,
+      "logits/chosen": -2.4643445014953613,
+      "logits/rejected": -2.1634469032287598,
+      "logps/chosen": -214.7722625732422,
+      "logps/rejected": -191.8986358642578,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.000286552298348397,
+      "rewards/margins": 0.08134503662586212,
+      "rewards/rejected": -0.081631600856781,
+      "step": 1510
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.9705689993459784e-06,
+      "logits/chosen": -2.4508216381073,
+      "logits/rejected": -2.0175795555114746,
+      "logps/chosen": -207.11288452148438,
+      "logps/rejected": -161.23849487304688,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.021875491365790367,
+      "rewards/margins": 0.11838686466217041,
+      "rewards/rejected": -0.14026235044002533,
+      "step": 1520
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999999934793849e-06,
+      "logits/chosen": -2.350825309753418,
+      "logits/rejected": -2.257450580596924,
+      "logps/chosen": -248.128173828125,
+      "logps/rejected": -223.34884643554688,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.0017488065641373396,
+      "rewards/margins": 0.04637077450752258,
+      "rewards/rejected": -0.04811957851052284,
+      "step": 1530
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999992110059814e-06,
+      "logits/chosen": -2.321943759918213,
+      "logits/rejected": -2.3055014610290527,
+      "logps/chosen": -277.02490234375,
+      "logps/rejected": -263.4502258300781,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.00662571843713522,
+      "rewards/margins": 0.06538694351911545,
+      "rewards/rejected": -0.058761220425367355,
+      "step": 1540
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999971244142299e-06,
+      "logits/chosen": -2.4726457595825195,
+      "logits/rejected": -2.1803183555603027,
+      "logps/chosen": -269.8371276855469,
+      "logps/rejected": -237.511962890625,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.03615967929363251,
+      "rewards/margins": 0.06921084225177765,
+      "rewards/rejected": -0.03305116295814514,
+      "step": 1550
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999937337150149e-06,
+      "logits/chosen": -2.155517578125,
+      "logits/rejected": -2.115971803665161,
+      "logps/chosen": -232.05459594726562,
+      "logps/rejected": -219.97708129882812,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.024602752178907394,
+      "rewards/margins": 0.03303222730755806,
+      "rewards/rejected": -0.008429473266005516,
+      "step": 1560
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.99989038926024e-06,
+      "logits/chosen": -2.11495304107666,
+      "logits/rejected": -2.245858669281006,
+      "logps/chosen": -199.39358520507812,
+      "logps/rejected": -206.3715057373047,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.009655221365392208,
+      "rewards/margins": 0.047758929431438446,
+      "rewards/rejected": -0.05741415172815323,
+      "step": 1570
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999830400717476e-06,
+      "logits/chosen": -2.3075220584869385,
+      "logits/rejected": -2.172712802886963,
+      "logps/chosen": -287.1651306152344,
+      "logps/rejected": -280.3276672363281,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0022194921039044857,
+      "rewards/margins": 0.0943801999092102,
+      "rewards/rejected": -0.09216071665287018,
+      "step": 1580
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999757371834787e-06,
+      "logits/chosen": -2.1875932216644287,
+      "logits/rejected": -2.094125747680664,
+      "logps/chosen": -241.1585693359375,
+      "logps/rejected": -238.26773071289062,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.013396786525845528,
+      "rewards/margins": 0.126246377825737,
+      "rewards/rejected": -0.13964316248893738,
+      "step": 1590
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999671302993125e-06,
+      "logits/chosen": -2.134300470352173,
+      "logits/rejected": -2.071805477142334,
+      "logps/chosen": -248.94729614257812,
+      "logps/rejected": -272.7432861328125,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.003668667282909155,
+      "rewards/margins": 0.08407886326313019,
+      "rewards/rejected": -0.08774752914905548,
+      "step": 1600
+    },
+    {
+      "epoch": 0.1,
+      "eval_logits/chosen": -2.327033042907715,
+      "eval_logits/rejected": -2.141350269317627,
+      "eval_logps/chosen": -227.57763671875,
+      "eval_logps/rejected": -214.50515747070312,
+      "eval_loss": 0.6906238794326782,
+      "eval_rewards/accuracies": 0.6259999871253967,
+      "eval_rewards/chosen": 0.04427312687039375,
+      "eval_rewards/margins": 0.07320577651262283,
+      "eval_rewards/rejected": -0.028932644054293633,
+      "eval_runtime": 706.6876,
+      "eval_samples_per_second": 2.83,
+      "eval_steps_per_second": 1.415,
+      "step": 1600
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.999572194641471e-06,
+      "logits/chosen": -2.291485071182251,
+      "logits/rejected": -2.1361746788024902,
+      "logps/chosen": -271.530029296875,
+      "logps/rejected": -228.67013549804688,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03988269716501236,
+      "rewards/margins": 0.09610097855329514,
+      "rewards/rejected": -0.056218285113573074,
+      "step": 1610
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.999460047296819e-06,
+      "logits/chosen": -2.2640249729156494,
+      "logits/rejected": -2.134519577026367,
+      "logps/chosen": -213.858154296875,
+      "logps/rejected": -200.5656280517578,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.01746007241308689,
+      "rewards/margins": 0.07616613805294037,
+      "rewards/rejected": -0.05870606750249863,
+      "step": 1620
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.999334861544186e-06,
+      "logits/chosen": -2.381791353225708,
+      "logits/rejected": -2.045926570892334,
+      "logps/chosen": -226.28970336914062,
+      "logps/rejected": -178.6829833984375,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.031635530292987823,
+      "rewards/margins": 0.0839659571647644,
+      "rewards/rejected": -0.05233042314648628,
+      "step": 1630
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.999196638036604e-06,
+      "logits/chosen": -2.4527511596679688,
+      "logits/rejected": -2.2551088333129883,
+      "logps/chosen": -281.4839782714844,
+      "logps/rejected": -249.938232421875,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.05931050330400467,
+      "rewards/margins": 0.03690633550286293,
+      "rewards/rejected": 0.02240416780114174,
+      "step": 1640
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.999045377495111e-06,
+      "logits/chosen": -2.117143154144287,
+      "logits/rejected": -2.3743062019348145,
+      "logps/chosen": -170.4903106689453,
+      "logps/rejected": -265.4920654296875,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03844357654452324,
+      "rewards/margins": 0.07219245284795761,
+      "rewards/rejected": -0.033748872578144073,
+      "step": 1650
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.998881080708759e-06,
+      "logits/chosen": -2.2666547298431396,
+      "logits/rejected": -2.1885650157928467,
+      "logps/chosen": -239.72262573242188,
+      "logps/rejected": -212.6611328125,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.00598920788615942,
+      "rewards/margins": 0.03883373737335205,
+      "rewards/rejected": -0.03284453600645065,
+      "step": 1660
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.998703748534599e-06,
+      "logits/chosen": -2.0997233390808105,
+      "logits/rejected": -1.8399873971939087,
+      "logps/chosen": -240.05831909179688,
+      "logps/rejected": -196.15187072753906,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.005714725703001022,
+      "rewards/margins": 0.06784109026193619,
+      "rewards/rejected": -0.062126368284225464,
+      "step": 1670
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.998513381897683e-06,
+      "logits/chosen": -2.338465452194214,
+      "logits/rejected": -2.107517719268799,
+      "logps/chosen": -237.6542205810547,
+      "logps/rejected": -185.0458984375,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.015895305201411247,
+      "rewards/margins": 0.04385993629693985,
+      "rewards/rejected": -0.027964631095528603,
+      "step": 1680
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.9983099817910565e-06,
+      "logits/chosen": -2.259268283843994,
+      "logits/rejected": -2.0765738487243652,
+      "logps/chosen": -243.1163330078125,
+      "logps/rejected": -246.34951782226562,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.02145574428141117,
+      "rewards/margins": 0.07711198180913925,
+      "rewards/rejected": -0.05565624311566353,
+      "step": 1690
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.998093549275754e-06,
+      "logits/chosen": -2.2384533882141113,
+      "logits/rejected": -2.2348380088806152,
+      "logps/chosen": -258.17047119140625,
+      "logps/rejected": -287.0838317871094,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.02163517475128174,
+      "rewards/margins": 0.07120385020971298,
+      "rewards/rejected": -0.049568675458431244,
+      "step": 1700
+    },
+    {
+      "epoch": 0.11,
+      "eval_logits/chosen": -2.3420772552490234,
+      "eval_logits/rejected": -2.1548047065734863,
+      "eval_logps/chosen": -228.6732940673828,
+      "eval_logps/rejected": -215.70216369628906,
+      "eval_loss": 0.6903870701789856,
+      "eval_rewards/accuracies": 0.6215000152587891,
+      "eval_rewards/chosen": 0.033316612243652344,
+      "eval_rewards/margins": 0.07421907782554626,
+      "eval_rewards/rejected": -0.04090247303247452,
+      "eval_runtime": 706.8309,
+      "eval_samples_per_second": 2.83,
+      "eval_steps_per_second": 1.415,
+      "step": 1700
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.997864085480794e-06,
+      "logits/chosen": -2.372429370880127,
+      "logits/rejected": -2.201474189758301,
+      "logps/chosen": -263.83319091796875,
+      "logps/rejected": -253.3795166015625,
+      "loss": 0.6936,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.025129973888397217,
+      "rewards/margins": 0.08083225041627884,
+      "rewards/rejected": -0.05570227652788162,
+      "step": 1710
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.997621591603171e-06,
+      "logits/chosen": -2.3386642932891846,
+      "logits/rejected": -2.1461567878723145,
+      "logps/chosen": -160.34231567382812,
+      "logps/rejected": -167.763916015625,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.023095201700925827,
+      "rewards/margins": 0.08106372505426407,
+      "rewards/rejected": -0.1041589230298996,
+      "step": 1720
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.997366068907853e-06,
+      "logits/chosen": -2.3146162033081055,
+      "logits/rejected": -2.260533094406128,
+      "logps/chosen": -257.2478942871094,
+      "logps/rejected": -243.188232421875,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.011618571355938911,
+      "rewards/margins": 0.05815444514155388,
+      "rewards/rejected": -0.04653587192296982,
+      "step": 1730
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.997097518727771e-06,
+      "logits/chosen": -2.366065263748169,
+      "logits/rejected": -2.1315550804138184,
+      "logps/chosen": -224.10079956054688,
+      "logps/rejected": -197.63809204101562,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.011443842202425003,
+      "rewards/margins": 0.06659694015979767,
+      "rewards/rejected": -0.07804077863693237,
+      "step": 1740
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.9968159424638155e-06,
+      "logits/chosen": -2.2000982761383057,
+      "logits/rejected": -2.418351650238037,
+      "logps/chosen": -218.63314819335938,
+      "logps/rejected": -273.26837158203125,
+      "loss": 0.691,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.010267667472362518,
+      "rewards/margins": 0.02790077030658722,
+      "rewards/rejected": -0.03816843777894974,
+      "step": 1750
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.9965213415848235e-06,
+      "logits/chosen": -2.284058094024658,
+      "logits/rejected": -1.9068619012832642,
+      "logps/chosen": -231.62319946289062,
+      "logps/rejected": -201.48184204101562,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.03218645602464676,
+      "rewards/margins": 0.06718595325946808,
+      "rewards/rejected": -0.09937240928411484,
+      "step": 1760
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.9962137176275805e-06,
+      "logits/chosen": -2.3636927604675293,
+      "logits/rejected": -2.164499044418335,
+      "logps/chosen": -232.9027862548828,
+      "logps/rejected": -236.94198608398438,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.013926369138062,
+      "rewards/margins": 0.04597216844558716,
+      "rewards/rejected": -0.03204580023884773,
+      "step": 1770
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.9958930721968015e-06,
+      "logits/chosen": -2.2207086086273193,
+      "logits/rejected": -2.3023934364318848,
+      "logps/chosen": -210.31875610351562,
+      "logps/rejected": -226.93685913085938,
+      "loss": 0.692,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.017164241522550583,
+      "rewards/margins": 0.0511639229953289,
+      "rewards/rejected": -0.03399968519806862,
+      "step": 1780
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.995559406965132e-06,
+      "logits/chosen": -2.4438416957855225,
+      "logits/rejected": -2.0871827602386475,
+      "logps/chosen": -229.1040802001953,
+      "logps/rejected": -204.32778930664062,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.02811221405863762,
+      "rewards/margins": 0.04839346557855606,
+      "rewards/rejected": -0.02028125710785389,
+      "step": 1790
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.995212723673131e-06,
+      "logits/chosen": -2.408142566680908,
+      "logits/rejected": -2.172886848449707,
+      "logps/chosen": -225.9892578125,
+      "logps/rejected": -183.36489868164062,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.04213835671544075,
+      "rewards/margins": 0.07358547300100327,
+      "rewards/rejected": -0.031447120010852814,
+      "step": 1800
+    },
+    {
+      "epoch": 0.12,
+      "eval_logits/chosen": -2.3330607414245605,
+      "eval_logits/rejected": -2.146653652191162,
+      "eval_logps/chosen": -227.91104125976562,
+      "eval_logps/rejected": -213.03692626953125,
+      "eval_loss": 0.6908692717552185,
+      "eval_rewards/accuracies": 0.6159999966621399,
+      "eval_rewards/chosen": 0.040939364582300186,
+      "eval_rewards/margins": 0.055189553648233414,
+      "eval_rewards/rejected": -0.014250185340642929,
+      "eval_runtime": 706.2668,
+      "eval_samples_per_second": 2.832,
+      "eval_steps_per_second": 1.416,
+      "step": 1800
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.99485302412927e-06,
+      "logits/chosen": -2.090059280395508,
+      "logits/rejected": -2.0199811458587646,
+      "logps/chosen": -204.09255981445312,
+      "logps/rejected": -211.0037078857422,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.039184752851724625,
+      "rewards/margins": 0.06218431517481804,
+      "rewards/rejected": -0.022999566048383713,
+      "step": 1810
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.994480310209918e-06,
+      "logits/chosen": -2.297668933868408,
+      "logits/rejected": -2.459190845489502,
+      "logps/chosen": -235.6026611328125,
+      "logps/rejected": -254.4353485107422,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0638035237789154,
+      "rewards/margins": 0.05078417807817459,
+      "rewards/rejected": 0.013019341044127941,
+      "step": 1820
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.994094583859332e-06,
+      "logits/chosen": -2.326498031616211,
+      "logits/rejected": -2.0701937675476074,
+      "logps/chosen": -157.9686737060547,
+      "logps/rejected": -194.639892578125,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.069440558552742,
+      "rewards/margins": 0.0464170016348362,
+      "rewards/rejected": 0.023023560643196106,
+      "step": 1830
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.9936958470896525e-06,
+      "logits/chosen": -2.334303617477417,
+      "logits/rejected": -2.089700698852539,
+      "logps/chosen": -221.90780639648438,
+      "logps/rejected": -187.50491333007812,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.018373683094978333,
+      "rewards/margins": 0.08500251919031143,
+      "rewards/rejected": -0.0666288360953331,
+      "step": 1840
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.993284101980883e-06,
+      "logits/chosen": -2.2909703254699707,
+      "logits/rejected": -2.0997474193573,
+      "logps/chosen": -244.8101348876953,
+      "logps/rejected": -217.73196411132812,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.023147549480199814,
+      "rewards/margins": 0.12554757297039032,
+      "rewards/rejected": -0.14869512617588043,
+      "step": 1850
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.9928593506808885e-06,
+      "logits/chosen": -2.40543794631958,
+      "logits/rejected": -2.2108352184295654,
+      "logps/chosen": -258.2170715332031,
+      "logps/rejected": -228.532958984375,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.014026492834091187,
+      "rewards/margins": 0.07293161749839783,
+      "rewards/rejected": -0.08695811778306961,
+      "step": 1860
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.992421595405381e-06,
+      "logits/chosen": -2.3434250354766846,
+      "logits/rejected": -2.095716714859009,
+      "logps/chosen": -224.97021484375,
+      "logps/rejected": -158.5146942138672,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.03222180902957916,
+      "rewards/margins": 0.04391375184059143,
+      "rewards/rejected": -0.011691942811012268,
+      "step": 1870
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.991970838437905e-06,
+      "logits/chosen": -2.2580971717834473,
+      "logits/rejected": -2.1631343364715576,
+      "logps/chosen": -220.3092803955078,
+      "logps/rejected": -255.4618377685547,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.0502026192843914,
+      "rewards/margins": 0.07047822326421738,
+      "rewards/rejected": -0.020275603979825974,
+      "step": 1880
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.9915070821298294e-06,
+      "logits/chosen": -2.366753339767456,
+      "logits/rejected": -2.039536237716675,
+      "logps/chosen": -162.96090698242188,
+      "logps/rejected": -157.37863159179688,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.04008902981877327,
+      "rewards/margins": 0.03506173565983772,
+      "rewards/rejected": 0.0050272950902581215,
+      "step": 1890
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.991030328900336e-06,
+      "logits/chosen": -2.295825481414795,
+      "logits/rejected": -2.040485143661499,
+      "logps/chosen": -278.2053527832031,
+      "logps/rejected": -216.57290649414062,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.06363843381404877,
+      "rewards/margins": 0.08403732627630234,
+      "rewards/rejected": -0.02039889432489872,
+      "step": 1900
+    },
+    {
+      "epoch": 0.12,
+      "eval_logits/chosen": -2.3460586071014404,
+      "eval_logits/rejected": -2.158717393875122,
+      "eval_logps/chosen": -227.45773315429688,
+      "eval_logps/rejected": -213.3264617919922,
+      "eval_loss": 0.6906041502952576,
+      "eval_rewards/accuracies": 0.6290000081062317,
+      "eval_rewards/chosen": 0.04547214135527611,
+      "eval_rewards/margins": 0.06261760741472244,
+      "eval_rewards/rejected": -0.017145469784736633,
+      "eval_runtime": 707.9289,
+      "eval_samples_per_second": 2.825,
+      "eval_steps_per_second": 1.413,
+      "step": 1900
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.9905405812364014e-06,
+      "logits/chosen": -2.3040223121643066,
+      "logits/rejected": -2.288963794708252,
+      "logps/chosen": -196.22824096679688,
+      "logps/rejected": -204.9025115966797,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.06067962199449539,
+      "rewards/margins": 0.062089789658784866,
+      "rewards/rejected": -0.0014101641718298197,
+      "step": 1910
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.990037841692791e-06,
+      "logits/chosen": -2.2711470127105713,
+      "logits/rejected": -2.027571201324463,
+      "logps/chosen": -203.07684326171875,
+      "logps/rejected": -167.22711181640625,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.02964043617248535,
+      "rewards/margins": 0.0743485689163208,
+      "rewards/rejected": -0.04470812529325485,
+      "step": 1920
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.989522112892039e-06,
+      "logits/chosen": -2.3281807899475098,
+      "logits/rejected": -2.2966067790985107,
+      "logps/chosen": -198.56414794921875,
+      "logps/rejected": -204.41529846191406,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.009347607381641865,
+      "rewards/margins": 0.06741191446781158,
+      "rewards/rejected": -0.07675951719284058,
+      "step": 1930
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.98899339752444e-06,
+      "logits/chosen": -2.370422124862671,
+      "logits/rejected": -2.1254730224609375,
+      "logps/chosen": -227.0062255859375,
+      "logps/rejected": -208.43771362304688,
+      "loss": 0.688,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.025960898026823997,
+      "rewards/margins": 0.10374845564365387,
+      "rewards/rejected": -0.07778755575418472,
+      "step": 1940
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.988451698348033e-06,
+      "logits/chosen": -2.287224769592285,
+      "logits/rejected": -2.2654972076416016,
+      "logps/chosen": -177.00662231445312,
+      "logps/rejected": -202.26292419433594,
+      "loss": 0.6936,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.013209563679993153,
+      "rewards/margins": 0.04986957833170891,
+      "rewards/rejected": -0.03666001558303833,
+      "step": 1950
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.987897018188585e-06,
+      "logits/chosen": -2.263166904449463,
+      "logits/rejected": -2.024923801422119,
+      "logps/chosen": -222.4883270263672,
+      "logps/rejected": -171.92982482910156,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0013100657379254699,
+      "rewards/margins": 0.061175812035799026,
+      "rewards/rejected": -0.05986575409770012,
+      "step": 1960
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.9873293599395814e-06,
+      "logits/chosen": -2.2896173000335693,
+      "logits/rejected": -2.1597745418548584,
+      "logps/chosen": -195.554931640625,
+      "logps/rejected": -200.31951904296875,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.02123176120221615,
+      "rewards/margins": 0.08748042583465576,
+      "rewards/rejected": -0.10871219635009766,
+      "step": 1970
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.986748726562203e-06,
+      "logits/chosen": -2.3323073387145996,
+      "logits/rejected": -2.2020726203918457,
+      "logps/chosen": -216.76565551757812,
+      "logps/rejected": -197.0693359375,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.009111806750297546,
+      "rewards/margins": 0.05196034908294678,
+      "rewards/rejected": -0.061072152107954025,
+      "step": 1980
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.98615512108532e-06,
+      "logits/chosen": -2.42622709274292,
+      "logits/rejected": -2.2754573822021484,
+      "logps/chosen": -218.6569061279297,
+      "logps/rejected": -224.9327850341797,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.028538372367620468,
+      "rewards/margins": 0.044925060123205185,
+      "rewards/rejected": -0.016386687755584717,
+      "step": 1990
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.985548546605469e-06,
+      "logits/chosen": -2.1802406311035156,
+      "logits/rejected": -2.314124584197998,
+      "logps/chosen": -215.97323608398438,
+      "logps/rejected": -237.9573974609375,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.028674548491835594,
+      "rewards/margins": 0.04343647509813309,
+      "rewards/rejected": -0.07211102545261383,
+      "step": 2000
+    },
+    {
+      "epoch": 0.13,
+      "eval_logits/chosen": -2.355757713317871,
+      "eval_logits/rejected": -2.16721248626709,
+      "eval_logps/chosen": -232.93431091308594,
+      "eval_logps/rejected": -220.59490966796875,
+      "eval_loss": 0.6904016137123108,
+      "eval_rewards/accuracies": 0.6399999856948853,
+      "eval_rewards/chosen": -0.009293550625443459,
+      "eval_rewards/margins": 0.08053648471832275,
+      "eval_rewards/rejected": -0.08983004093170166,
+      "eval_runtime": 709.7269,
+      "eval_samples_per_second": 2.818,
+      "eval_steps_per_second": 1.409,
+      "step": 2000
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.984929006286838e-06,
+      "logits/chosen": -2.2016310691833496,
+      "logits/rejected": -2.152900457382202,
+      "logps/chosen": -211.8243865966797,
+      "logps/rejected": -217.55712890625,
+      "loss": 0.6935,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.025103915482759476,
+      "rewards/margins": 0.022764649242162704,
+      "rewards/rejected": -0.04786856472492218,
+      "step": 2010
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.984296503361256e-06,
+      "logits/chosen": -2.4327239990234375,
+      "logits/rejected": -2.072183132171631,
+      "logps/chosen": -202.29269409179688,
+      "logps/rejected": -166.57730102539062,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.0037281368859112263,
+      "rewards/margins": 0.06288442015647888,
+      "rewards/rejected": -0.06661255657672882,
+      "step": 2020
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.9836510411281645e-06,
+      "logits/chosen": -2.2853875160217285,
+      "logits/rejected": -2.1659188270568848,
+      "logps/chosen": -278.9981384277344,
+      "logps/rejected": -251.56393432617188,
+      "loss": 0.6867,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.004663098603487015,
+      "rewards/margins": 0.1196819394826889,
+      "rewards/rejected": -0.12434504926204681,
+      "step": 2030
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.982992622954613e-06,
+      "logits/chosen": -2.422229766845703,
+      "logits/rejected": -2.1254818439483643,
+      "logps/chosen": -282.1938171386719,
+      "logps/rejected": -176.20748901367188,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.013343462720513344,
+      "rewards/margins": 0.08182214200496674,
+      "rewards/rejected": -0.06847867369651794,
+      "step": 2040
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.9823212522752325e-06,
+      "logits/chosen": -2.530290126800537,
+      "logits/rejected": -2.2488186359405518,
+      "logps/chosen": -275.6991882324219,
+      "logps/rejected": -252.4962615966797,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.02114512026309967,
+      "rewards/margins": 0.10833090543746948,
+      "rewards/rejected": -0.08718578517436981,
+      "step": 2050
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.981636932592222e-06,
+      "logits/chosen": -2.287421703338623,
+      "logits/rejected": -2.181699275970459,
+      "logps/chosen": -207.735107421875,
+      "logps/rejected": -210.69009399414062,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.05177903175354004,
+      "rewards/margins": 0.0697595402598381,
+      "rewards/rejected": -0.017980504781007767,
+      "step": 2060
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.980939667475328e-06,
+      "logits/chosen": -2.505133628845215,
+      "logits/rejected": -2.146200656890869,
+      "logps/chosen": -268.6704406738281,
+      "logps/rejected": -213.940673828125,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.05504737049341202,
+      "rewards/margins": 0.0586230531334877,
+      "rewards/rejected": -0.0035756707657128572,
+      "step": 2070
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.980229460561826e-06,
+      "logits/chosen": -2.353895664215088,
+      "logits/rejected": -2.2923059463500977,
+      "logps/chosen": -214.421630859375,
+      "logps/rejected": -210.04910278320312,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.03659746050834656,
+      "rewards/margins": 0.10933919250965118,
+      "rewards/rejected": -0.07274172455072403,
+      "step": 2080
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.979506315556503e-06,
+      "logits/chosen": -2.316563129425049,
+      "logits/rejected": -1.9798656702041626,
+      "logps/chosen": -284.0970153808594,
+      "logps/rejected": -241.884521484375,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.023951154202222824,
+      "rewards/margins": 0.08775506168603897,
+      "rewards/rejected": -0.06380391120910645,
+      "step": 2090
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.9787702362316395e-06,
+      "logits/chosen": -2.36979079246521,
+      "logits/rejected": -2.592101812362671,
+      "logps/chosen": -188.11888122558594,
+      "logps/rejected": -218.1317901611328,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.027083251625299454,
+      "rewards/margins": 0.04498888552188873,
+      "rewards/rejected": -0.01790563203394413,
+      "step": 2100
+    },
+    {
+      "epoch": 0.14,
+      "eval_logits/chosen": -2.3738138675689697,
+      "eval_logits/rejected": -2.1836633682250977,
+      "eval_logps/chosen": -229.5577850341797,
+      "eval_logps/rejected": -215.9217529296875,
+      "eval_loss": 0.6904562711715698,
+      "eval_rewards/accuracies": 0.6380000114440918,
+      "eval_rewards/chosen": 0.024471644312143326,
+      "eval_rewards/margins": 0.06757022440433502,
+      "eval_rewards/rejected": -0.04309859126806259,
+      "eval_runtime": 710.6569,
+      "eval_samples_per_second": 2.814,
+      "eval_steps_per_second": 1.407,
+      "step": 2100
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.9780212264269835e-06,
+      "logits/chosen": -2.3147263526916504,
+      "logits/rejected": -2.065084934234619,
+      "logps/chosen": -182.92022705078125,
+      "logps/rejected": -169.31704711914062,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.01898839697241783,
+      "rewards/margins": 0.03532510995864868,
+      "rewards/rejected": -0.01633671671152115,
+      "step": 2110
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.977259290049739e-06,
+      "logits/chosen": -2.568851947784424,
+      "logits/rejected": -1.9942678213119507,
+      "logps/chosen": -281.524658203125,
+      "logps/rejected": -226.5505828857422,
+      "loss": 0.6856,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.05417801812291145,
+      "rewards/margins": 0.14544400572776794,
+      "rewards/rejected": -0.0912659615278244,
+      "step": 2120
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.976484431074538e-06,
+      "logits/chosen": -2.3134965896606445,
+      "logits/rejected": -2.251204252243042,
+      "logps/chosen": -191.16305541992188,
+      "logps/rejected": -178.55044555664062,
+      "loss": 0.692,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.026424426585435867,
+      "rewards/margins": 0.04989726096391678,
+      "rewards/rejected": -0.02347283437848091,
+      "step": 2130
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.975696653543425e-06,
+      "logits/chosen": -2.399449586868286,
+      "logits/rejected": -2.1439669132232666,
+      "logps/chosen": -248.66311645507812,
+      "logps/rejected": -245.24496459960938,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.009857301600277424,
+      "rewards/margins": 0.09151118248701096,
+      "rewards/rejected": -0.08165387064218521,
+      "step": 2140
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.974895961565835e-06,
+      "logits/chosen": -2.325570583343506,
+      "logits/rejected": -1.9811022281646729,
+      "logps/chosen": -174.66339111328125,
+      "logps/rejected": -187.84197998046875,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.005875427275896072,
+      "rewards/margins": 0.07950626313686371,
+      "rewards/rejected": -0.07363083958625793,
+      "step": 2150
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.974082359318566e-06,
+      "logits/chosen": -2.320629596710205,
+      "logits/rejected": -2.144028902053833,
+      "logps/chosen": -252.28860473632812,
+      "logps/rejected": -216.6935577392578,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.013614351861178875,
+      "rewards/margins": 0.09884041547775269,
+      "rewards/rejected": -0.08522607386112213,
+      "step": 2160
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.973255851045769e-06,
+      "logits/chosen": -2.2949633598327637,
+      "logits/rejected": -2.3036093711853027,
+      "logps/chosen": -215.01193237304688,
+      "logps/rejected": -185.1837158203125,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.023462986573576927,
+      "rewards/margins": 0.06919287145137787,
+      "rewards/rejected": -0.04572988301515579,
+      "step": 2170
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.972416441058915e-06,
+      "logits/chosen": -2.2384517192840576,
+      "logits/rejected": -2.0826570987701416,
+      "logps/chosen": -227.3652801513672,
+      "logps/rejected": -211.1637420654297,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.023905407637357712,
+      "rewards/margins": 0.10440375655889511,
+      "rewards/rejected": -0.0804983526468277,
+      "step": 2180
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.971564133736777e-06,
+      "logits/chosen": -2.1866540908813477,
+      "logits/rejected": -2.0094194412231445,
+      "logps/chosen": -174.2550506591797,
+      "logps/rejected": -188.44947814941406,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.037455908954143524,
+      "rewards/margins": 0.09213308990001678,
+      "rewards/rejected": -0.05467717722058296,
+      "step": 2190
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.970698933525409e-06,
+      "logits/chosen": -2.478977918624878,
+      "logits/rejected": -2.19539737701416,
+      "logps/chosen": -281.2948303222656,
+      "logps/rejected": -250.3472900390625,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.01947910524904728,
+      "rewards/margins": 0.08724024891853333,
+      "rewards/rejected": -0.10671935975551605,
+      "step": 2200
+    },
+    {
+      "epoch": 0.14,
+      "eval_logits/chosen": -2.3566019535064697,
+      "eval_logits/rejected": -2.166872262954712,
+      "eval_logps/chosen": -234.11630249023438,
+      "eval_logps/rejected": -221.84378051757812,
+      "eval_loss": 0.690406084060669,
+      "eval_rewards/accuracies": 0.6259999871253967,
+      "eval_rewards/chosen": -0.021113485097885132,
+      "eval_rewards/margins": 0.08120539039373398,
+      "eval_rewards/rejected": -0.10231887549161911,
+      "eval_runtime": 713.5703,
+      "eval_samples_per_second": 2.803,
+      "eval_steps_per_second": 1.401,
+      "step": 2200
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.969820844938118e-06,
+      "logits/chosen": -2.4681389331817627,
+      "logits/rejected": -2.141389846801758,
+      "logps/chosen": -223.8988800048828,
+      "logps/rejected": -179.49386596679688,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.012956234626471996,
+      "rewards/margins": 0.0949764996767044,
+      "rewards/rejected": -0.10793273150920868,
+      "step": 2210
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.968929872555444e-06,
+      "logits/chosen": -1.9914073944091797,
+      "logits/rejected": -2.178544521331787,
+      "logps/chosen": -223.1538543701172,
+      "logps/rejected": -262.5058898925781,
+      "loss": 0.691,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.06019466370344162,
+      "rewards/margins": 0.03688964992761612,
+      "rewards/rejected": -0.09708431363105774,
+      "step": 2220
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.968026021025137e-06,
+      "logits/chosen": -2.4021904468536377,
+      "logits/rejected": -2.184993267059326,
+      "logps/chosen": -201.25662231445312,
+      "logps/rejected": -168.76951599121094,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0030830535106360912,
+      "rewards/margins": 0.07578281313180923,
+      "rewards/rejected": -0.07269976288080215,
+      "step": 2230
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.967109295062128e-06,
+      "logits/chosen": -2.2592692375183105,
+      "logits/rejected": -2.035545825958252,
+      "logps/chosen": -223.9074249267578,
+      "logps/rejected": -253.28909301757812,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.010940281674265862,
+      "rewards/margins": 0.08460705727338791,
+      "rewards/rejected": -0.0736667662858963,
+      "step": 2240
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.966179699448509e-06,
+      "logits/chosen": -2.2413432598114014,
+      "logits/rejected": -2.0473268032073975,
+      "logps/chosen": -191.69692993164062,
+      "logps/rejected": -176.70880126953125,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.009419666603207588,
+      "rewards/margins": 0.02837999537587166,
+      "rewards/rejected": -0.037799663841724396,
+      "step": 2250
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.965237239033506e-06,
+      "logits/chosen": -2.410356044769287,
+      "logits/rejected": -2.2265586853027344,
+      "logps/chosen": -286.68280029296875,
+      "logps/rejected": -257.287841796875,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.029759634286165237,
+      "rewards/margins": 0.1538696587085724,
+      "rewards/rejected": -0.12411002069711685,
+      "step": 2260
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.964281918733453e-06,
+      "logits/chosen": -2.3516123294830322,
+      "logits/rejected": -2.0975680351257324,
+      "logps/chosen": -186.3314666748047,
+      "logps/rejected": -193.1648406982422,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.03217019513249397,
+      "rewards/margins": 0.11290119588375092,
+      "rewards/rejected": -0.14507140219211578,
+      "step": 2270
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.9633137435317715e-06,
+      "logits/chosen": -2.367588758468628,
+      "logits/rejected": -1.7204262018203735,
+      "logps/chosen": -223.7427978515625,
+      "logps/rejected": -165.84214782714844,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.008477909490466118,
+      "rewards/margins": 0.09884487092494965,
+      "rewards/rejected": -0.10732278972864151,
+      "step": 2280
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.9623327184789355e-06,
+      "logits/chosen": -2.464026689529419,
+      "logits/rejected": -2.3778140544891357,
+      "logps/chosen": -216.4389190673828,
+      "logps/rejected": -217.1889190673828,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.013015327975153923,
+      "rewards/margins": 0.055762697011232376,
+      "rewards/rejected": -0.042747363448143005,
+      "step": 2290
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.9613388486924525e-06,
+      "logits/chosen": -2.069906234741211,
+      "logits/rejected": -2.2107555866241455,
+      "logps/chosen": -184.16355895996094,
+      "logps/rejected": -209.44485473632812,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.040295325219631195,
+      "rewards/margins": 0.0790117159485817,
+      "rewards/rejected": -0.11930704116821289,
+      "step": 2300
+    },
+    {
+      "epoch": 0.15,
+      "eval_logits/chosen": -2.3594470024108887,
+      "eval_logits/rejected": -2.1697957515716553,
+      "eval_logps/chosen": -235.0392608642578,
+      "eval_logps/rejected": -223.16970825195312,
+      "eval_loss": 0.6907156705856323,
+      "eval_rewards/accuracies": 0.6169999837875366,
+      "eval_rewards/chosen": -0.03034323826432228,
+      "eval_rewards/margins": 0.08523471653461456,
+      "eval_rewards/rejected": -0.11557795852422714,
+      "eval_runtime": 709.1569,
+      "eval_samples_per_second": 2.82,
+      "eval_steps_per_second": 1.41,
+      "step": 2300
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.960332139356834e-06,
+      "logits/chosen": -2.311826705932617,
+      "logits/rejected": -2.130279541015625,
+      "logps/chosen": -211.80313110351562,
+      "logps/rejected": -195.69424438476562,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.029862677678465843,
+      "rewards/margins": 0.10331012308597565,
+      "rewards/rejected": -0.13317279517650604,
+      "step": 2310
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.95931259572357e-06,
+      "logits/chosen": -2.4080193042755127,
+      "logits/rejected": -2.043299913406372,
+      "logps/chosen": -235.7884521484375,
+      "logps/rejected": -271.4584655761719,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.015484524890780449,
+      "rewards/margins": 0.09996424615383148,
+      "rewards/rejected": -0.11544877290725708,
+      "step": 2320
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.9582802231111e-06,
+      "logits/chosen": -2.2260050773620605,
+      "logits/rejected": -2.310255527496338,
+      "logps/chosen": -211.4370574951172,
+      "logps/rejected": -191.3184814453125,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.004370751790702343,
+      "rewards/margins": 0.06759954988956451,
+      "rewards/rejected": -0.06322880834341049,
+      "step": 2330
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.957235026904782e-06,
+      "logits/chosen": -2.393613815307617,
+      "logits/rejected": -2.0691580772399902,
+      "logps/chosen": -256.73382568359375,
+      "logps/rejected": -211.1546173095703,
+      "loss": 0.69,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.05418550223112106,
+      "rewards/margins": 0.055359721183776855,
+      "rewards/rejected": -0.001174215809442103,
+      "step": 2340
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.956177012556875e-06,
+      "logits/chosen": -2.478972911834717,
+      "logits/rejected": -2.2455832958221436,
+      "logps/chosen": -243.0150604248047,
+      "logps/rejected": -179.1586456298828,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.011777469888329506,
+      "rewards/margins": 0.0771162286400795,
+      "rewards/rejected": -0.06533874571323395,
+      "step": 2350
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.9551061855864976e-06,
+      "logits/chosen": -2.150123119354248,
+      "logits/rejected": -2.202948808670044,
+      "logps/chosen": -191.73294067382812,
+      "logps/rejected": -204.2236785888672,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.010951901786029339,
+      "rewards/margins": 0.0828014463186264,
+      "rewards/rejected": -0.07184954732656479,
+      "step": 2360
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.95402255157961e-06,
+      "logits/chosen": -2.189168930053711,
+      "logits/rejected": -2.2527401447296143,
+      "logps/chosen": -182.8531494140625,
+      "logps/rejected": -255.9742431640625,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.02203099988400936,
+      "rewards/margins": 0.08574902266263962,
+      "rewards/rejected": -0.0637180283665657,
+      "step": 2370
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.952926116188977e-06,
+      "logits/chosen": -2.4717514514923096,
+      "logits/rejected": -2.3997349739074707,
+      "logps/chosen": -182.4977264404297,
+      "logps/rejected": -227.0753173828125,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.00879613310098648,
+      "rewards/margins": 0.05373033136129379,
+      "rewards/rejected": -0.04493419826030731,
+      "step": 2380
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.951816885134143e-06,
+      "logits/chosen": -2.305720567703247,
+      "logits/rejected": -2.2849538326263428,
+      "logps/chosen": -200.81576538085938,
+      "logps/rejected": -206.62277221679688,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.007371959276497364,
+      "rewards/margins": 0.0666775330901146,
+      "rewards/rejected": -0.059305571019649506,
+      "step": 2390
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.950694864201399e-06,
+      "logits/chosen": -2.3164827823638916,
+      "logits/rejected": -2.2348294258117676,
+      "logps/chosen": -236.01382446289062,
+      "logps/rejected": -246.91506958007812,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.026681452989578247,
+      "rewards/margins": 0.0877201110124588,
+      "rewards/rejected": -0.06103866174817085,
+      "step": 2400
+    },
+    {
+      "epoch": 0.16,
+      "eval_logits/chosen": -2.334512710571289,
+      "eval_logits/rejected": -2.1472020149230957,
+      "eval_logps/chosen": -228.88552856445312,
+      "eval_logps/rejected": -215.46128845214844,
+      "eval_loss": 0.690426766872406,
+      "eval_rewards/accuracies": 0.6225000023841858,
+      "eval_rewards/chosen": 0.031194256618618965,
+      "eval_rewards/margins": 0.06968805193901062,
+      "eval_rewards/rejected": -0.038493797183036804,
+      "eval_runtime": 713.4104,
+      "eval_samples_per_second": 2.803,
+      "eval_steps_per_second": 1.402,
+      "step": 2400
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.9495600592437575e-06,
+      "logits/chosen": -2.3851048946380615,
+      "logits/rejected": -2.2106268405914307,
+      "logps/chosen": -228.5146484375,
+      "logps/rejected": -235.97958374023438,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.016201911494135857,
+      "rewards/margins": 0.04283355548977852,
+      "rewards/rejected": -0.059035468846559525,
+      "step": 2410
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.948412476180917e-06,
+      "logits/chosen": -2.289257526397705,
+      "logits/rejected": -2.005004405975342,
+      "logps/chosen": -185.69271850585938,
+      "logps/rejected": -171.775146484375,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.016168467700481415,
+      "rewards/margins": 0.08819150179624557,
+      "rewards/rejected": -0.10435996949672699,
+      "step": 2420
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.947252120999232e-06,
+      "logits/chosen": -2.3191158771514893,
+      "logits/rejected": -2.045474052429199,
+      "logps/chosen": -267.0201721191406,
+      "logps/rejected": -212.0325927734375,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.0008467677980661392,
+      "rewards/margins": 0.03786264732480049,
+      "rewards/rejected": -0.037015873938798904,
+      "step": 2430
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.946078999751683e-06,
+      "logits/chosen": -2.22875714302063,
+      "logits/rejected": -2.160421371459961,
+      "logps/chosen": -175.55667114257812,
+      "logps/rejected": -161.97189331054688,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.04070456326007843,
+      "rewards/margins": 0.07971414923667908,
+      "rewards/rejected": -0.039009593427181244,
+      "step": 2440
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.944893118557847e-06,
+      "logits/chosen": -2.142076015472412,
+      "logits/rejected": -2.10664439201355,
+      "logps/chosen": -203.5096893310547,
+      "logps/rejected": -157.3628692626953,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.02974940836429596,
+      "rewards/margins": 0.07543652504682541,
+      "rewards/rejected": -0.04568710923194885,
+      "step": 2450
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.943694483603861e-06,
+      "logits/chosen": -2.4463329315185547,
+      "logits/rejected": -2.032438278198242,
+      "logps/chosen": -222.4589385986328,
+      "logps/rejected": -179.25404357910156,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.035450879484415054,
+      "rewards/margins": 0.077740877866745,
+      "rewards/rejected": -0.04229000210762024,
+      "step": 2460
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.9424831011423914e-06,
+      "logits/chosen": -2.408552408218384,
+      "logits/rejected": -2.3169188499450684,
+      "logps/chosen": -288.58917236328125,
+      "logps/rejected": -253.8071746826172,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.0016259975964203477,
+      "rewards/margins": 0.02232646383345127,
+      "rewards/rejected": -0.023952458053827286,
+      "step": 2470
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.9412589774926015e-06,
+      "logits/chosen": -2.4122543334960938,
+      "logits/rejected": -2.113245964050293,
+      "logps/chosen": -273.45263671875,
+      "logps/rejected": -232.2276611328125,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0073710111901164055,
+      "rewards/margins": 0.09398536384105682,
+      "rewards/rejected": -0.08661436289548874,
+      "step": 2480
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.940022119040121e-06,
+      "logits/chosen": -2.465616226196289,
+      "logits/rejected": -2.161163806915283,
+      "logps/chosen": -287.33282470703125,
+      "logps/rejected": -265.46343994140625,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.045831646770238876,
+      "rewards/margins": 0.059488773345947266,
+      "rewards/rejected": -0.013657125644385815,
+      "step": 2490
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.93877253223701e-06,
+      "logits/chosen": -2.417496919631958,
+      "logits/rejected": -2.174833297729492,
+      "logps/chosen": -285.5562438964844,
+      "logps/rejected": -259.3797302246094,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.06715986877679825,
+      "rewards/margins": 0.053962014615535736,
+      "rewards/rejected": 0.013197846710681915,
+      "step": 2500
+    },
+    {
+      "epoch": 0.16,
+      "eval_logits/chosen": -2.3545589447021484,
+      "eval_logits/rejected": -2.165816068649292,
+      "eval_logps/chosen": -226.2342071533203,
+      "eval_logps/rejected": -212.3520965576172,
+      "eval_loss": 0.6904910802841187,
+      "eval_rewards/accuracies": 0.625,
+      "eval_rewards/chosen": 0.05770741403102875,
+      "eval_rewards/margins": 0.06510914862155914,
+      "eval_rewards/rejected": -0.007401737384498119,
+      "eval_runtime": 713.7916,
+      "eval_samples_per_second": 2.802,
+      "eval_steps_per_second": 1.401,
+      "step": 2500
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.937510223601725e-06,
+      "logits/chosen": -2.5731029510498047,
+      "logits/rejected": -2.4025304317474365,
+      "logps/chosen": -254.93716430664062,
+      "logps/rejected": -217.3448944091797,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.07588844746351242,
+      "rewards/margins": 0.04619182273745537,
+      "rewards/rejected": 0.029696622863411903,
+      "step": 2510
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.936235199719085e-06,
+      "logits/chosen": -2.3957393169403076,
+      "logits/rejected": -2.2664778232574463,
+      "logps/chosen": -164.94325256347656,
+      "logps/rejected": -142.4560089111328,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.07358762621879578,
+      "rewards/margins": 0.07691960036754608,
+      "rewards/rejected": -0.0033319753129035234,
+      "step": 2520
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.93494746724024e-06,
+      "logits/chosen": -2.384202480316162,
+      "logits/rejected": -2.121302843093872,
+      "logps/chosen": -218.81015014648438,
+      "logps/rejected": -243.3566436767578,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.0499432310461998,
+      "rewards/margins": 0.07086510956287384,
+      "rewards/rejected": -0.020921876654028893,
+      "step": 2530
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.933647032882635e-06,
+      "logits/chosen": -2.5266172885894775,
+      "logits/rejected": -2.211371898651123,
+      "logps/chosen": -236.7266082763672,
+      "logps/rejected": -201.0948486328125,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0813523381948471,
+      "rewards/margins": 0.08787768334150314,
+      "rewards/rejected": -0.0065253423526883125,
+      "step": 2540
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.932333903429969e-06,
+      "logits/chosen": -2.205667495727539,
+      "logits/rejected": -2.058753728866577,
+      "logps/chosen": -188.5623779296875,
+      "logps/rejected": -162.65000915527344,
+      "loss": 0.6944,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.045201074331998825,
+      "rewards/margins": -0.001063268631696701,
+      "rewards/rejected": 0.046264342963695526,
+      "step": 2550
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.931008085732172e-06,
+      "logits/chosen": -2.3989763259887695,
+      "logits/rejected": -1.9831393957138062,
+      "logps/chosen": -197.83444213867188,
+      "logps/rejected": -153.06373596191406,
+      "loss": 0.692,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.04988477751612663,
+      "rewards/margins": 0.05230678245425224,
+      "rewards/rejected": -0.0024220068007707596,
+      "step": 2560
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.9296695867053565e-06,
+      "logits/chosen": -2.357609510421753,
+      "logits/rejected": -2.1423609256744385,
+      "logps/chosen": -294.01385498046875,
+      "logps/rejected": -234.3355712890625,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.06587690114974976,
+      "rewards/margins": 0.06281637400388718,
+      "rewards/rejected": 0.00306052272208035,
+      "step": 2570
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.928318413331791e-06,
+      "logits/chosen": -2.4487571716308594,
+      "logits/rejected": -2.233754873275757,
+      "logps/chosen": -205.7588653564453,
+      "logps/rejected": -195.708251953125,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.054682277143001556,
+      "rewards/margins": 0.05934653431177139,
+      "rewards/rejected": -0.004664266016334295,
+      "step": 2580
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.926954572659855e-06,
+      "logits/chosen": -2.223047971725464,
+      "logits/rejected": -2.236845016479492,
+      "logps/chosen": -234.7522430419922,
+      "logps/rejected": -261.69873046875,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.07443410903215408,
+      "rewards/margins": 0.09760646522045135,
+      "rewards/rejected": -0.02317235991358757,
+      "step": 2590
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.925578071804013e-06,
+      "logits/chosen": -2.245145320892334,
+      "logits/rejected": -2.1898436546325684,
+      "logps/chosen": -227.51473999023438,
+      "logps/rejected": -292.41217041015625,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.021910278126597404,
+      "rewards/margins": 0.07485561072826385,
+      "rewards/rejected": -0.052945323288440704,
+      "step": 2600
+    },
+    {
+      "epoch": 0.17,
+      "eval_logits/chosen": -2.3452558517456055,
+      "eval_logits/rejected": -2.157046318054199,
+      "eval_logps/chosen": -226.8026885986328,
+      "eval_logps/rejected": -213.66268920898438,
+      "eval_loss": 0.6903373003005981,
+      "eval_rewards/accuracies": 0.6320000290870667,
+      "eval_rewards/chosen": 0.05202279984951019,
+      "eval_rewards/margins": 0.07253072410821915,
+      "eval_rewards/rejected": -0.020507927983999252,
+      "eval_runtime": 710.3899,
+      "eval_samples_per_second": 2.815,
+      "eval_steps_per_second": 1.408,
+      "step": 2600
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.924188917944763e-06,
+      "logits/chosen": -2.412496328353882,
+      "logits/rejected": -2.234360456466675,
+      "logps/chosen": -214.3735809326172,
+      "logps/rejected": -191.68753051757812,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.05631525442004204,
+      "rewards/margins": 0.10834388434886932,
+      "rewards/rejected": -0.052028633654117584,
+      "step": 2610
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.922787118328617e-06,
+      "logits/chosen": -2.432962656021118,
+      "logits/rejected": -2.1155600547790527,
+      "logps/chosen": -226.4817657470703,
+      "logps/rejected": -155.4060516357422,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.03535914793610573,
+      "rewards/margins": 0.052371758967638016,
+      "rewards/rejected": -0.01701260730624199,
+      "step": 2620
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.921372680268045e-06,
+      "logits/chosen": -2.411449909210205,
+      "logits/rejected": -2.0770812034606934,
+      "logps/chosen": -228.71273803710938,
+      "logps/rejected": -201.04159545898438,
+      "loss": 0.6936,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.012919628992676735,
+      "rewards/margins": 0.026976149529218674,
+      "rewards/rejected": -0.01405651867389679,
+      "step": 2630
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.919945611141451e-06,
+      "logits/chosen": -2.459571361541748,
+      "logits/rejected": -2.078706979751587,
+      "logps/chosen": -218.890869140625,
+      "logps/rejected": -165.82656860351562,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.06343764066696167,
+      "rewards/margins": 0.05455031245946884,
+      "rewards/rejected": 0.008887320756912231,
+      "step": 2640
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.918505918393125e-06,
+      "logits/chosen": -2.321850538253784,
+      "logits/rejected": -2.1715524196624756,
+      "logps/chosen": -163.71751403808594,
+      "logps/rejected": -194.63999938964844,
+      "loss": 0.689,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.05313184857368469,
+      "rewards/margins": 0.07549114525318146,
+      "rewards/rejected": -0.022359298542141914,
+      "step": 2650
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.91705360953321e-06,
+      "logits/chosen": -2.376260995864868,
+      "logits/rejected": -2.14408540725708,
+      "logps/chosen": -244.6078643798828,
+      "logps/rejected": -220.7104034423828,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.008797365240752697,
+      "rewards/margins": 0.08757869899272919,
+      "rewards/rejected": -0.07878134399652481,
+      "step": 2660
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.9155886921376615e-06,
+      "logits/chosen": -2.2897629737854004,
+      "logits/rejected": -2.244919776916504,
+      "logps/chosen": -208.5734405517578,
+      "logps/rejected": -236.90902709960938,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.026050258427858353,
+      "rewards/margins": 0.06667254120111465,
+      "rewards/rejected": -0.0927228033542633,
+      "step": 2670
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.914111173848205e-06,
+      "logits/chosen": -2.3279356956481934,
+      "logits/rejected": -2.280183792114258,
+      "logps/chosen": -237.3335418701172,
+      "logps/rejected": -226.715576171875,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.04301954060792923,
+      "rewards/margins": 0.04799992963671684,
+      "rewards/rejected": -0.09101946651935577,
+      "step": 2680
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.9126210623723e-06,
+      "logits/chosen": -2.1070661544799805,
+      "logits/rejected": -2.2766873836517334,
+      "logps/chosen": -201.4837646484375,
+      "logps/rejected": -244.10855102539062,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.011103630065917969,
+      "rewards/margins": 0.08852796256542206,
+      "rewards/rejected": -0.09963159263134003,
+      "step": 2690
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.911118365483098e-06,
+      "logits/chosen": -2.2129688262939453,
+      "logits/rejected": -2.2985918521881104,
+      "logps/chosen": -202.52978515625,
+      "logps/rejected": -220.18173217773438,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.007755742873996496,
+      "rewards/margins": 0.08225713670253754,
+      "rewards/rejected": -0.09001287072896957,
+      "step": 2700
+    },
+    {
+      "epoch": 0.18,
+      "eval_logits/chosen": -2.3249430656433105,
+      "eval_logits/rejected": -2.138172149658203,
+      "eval_logps/chosen": -231.6273651123047,
+      "eval_logps/rejected": -217.83657836914062,
+      "eval_loss": 0.6905510425567627,
+      "eval_rewards/accuracies": 0.6324999928474426,
+      "eval_rewards/chosen": 0.003775849472731352,
+      "eval_rewards/margins": 0.06602264940738678,
+      "eval_rewards/rejected": -0.06224680691957474,
+      "eval_runtime": 712.2335,
+      "eval_samples_per_second": 2.808,
+      "eval_steps_per_second": 1.404,
+      "step": 2700
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.909603091019403e-06,
+      "logits/chosen": -2.4967615604400635,
+      "logits/rejected": -2.1264119148254395,
+      "logps/chosen": -232.87014770507812,
+      "logps/rejected": -201.87669372558594,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.03507193177938461,
+      "rewards/margins": 0.05626847222447395,
+      "rewards/rejected": -0.021196534857153893,
+      "step": 2710
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.908075246885626e-06,
+      "logits/chosen": -2.2765352725982666,
+      "logits/rejected": -2.204392910003662,
+      "logps/chosen": -149.35107421875,
+      "logps/rejected": -125.10282135009766,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.01376400887966156,
+      "rewards/margins": 0.03143042325973511,
+      "rewards/rejected": -0.017666416242718697,
+      "step": 2720
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.906534841051755e-06,
+      "logits/chosen": -2.138417959213257,
+      "logits/rejected": -2.202500820159912,
+      "logps/chosen": -241.3466339111328,
+      "logps/rejected": -257.94903564453125,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.02056839130818844,
+      "rewards/margins": 0.055667709559202194,
+      "rewards/rejected": -0.03509931638836861,
+      "step": 2730
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.904981881553297e-06,
+      "logits/chosen": -2.4024128913879395,
+      "logits/rejected": -2.0706582069396973,
+      "logps/chosen": -223.4213409423828,
+      "logps/rejected": -164.6261444091797,
+      "loss": 0.691,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.007749582640826702,
+      "rewards/margins": 0.05322981998324394,
+      "rewards/rejected": -0.04548024386167526,
+      "step": 2740
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.903416376491252e-06,
+      "logits/chosen": -2.4039175510406494,
+      "logits/rejected": -2.008875608444214,
+      "logps/chosen": -277.7300720214844,
+      "logps/rejected": -245.2926788330078,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.005621733609586954,
+      "rewards/margins": 0.08154728263616562,
+      "rewards/rejected": -0.0759255513548851,
+      "step": 2750
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.90183833403206e-06,
+      "logits/chosen": -2.467904567718506,
+      "logits/rejected": -2.3105967044830322,
+      "logps/chosen": -261.31292724609375,
+      "logps/rejected": -234.25772094726562,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.029001805931329727,
+      "rewards/margins": 0.09197796136140823,
+      "rewards/rejected": -0.0629761591553688,
+      "step": 2760
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.900247762407564e-06,
+      "logits/chosen": -2.228250026702881,
+      "logits/rejected": -2.0147242546081543,
+      "logps/chosen": -173.1147003173828,
+      "logps/rejected": -203.9661407470703,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.008227216079831123,
+      "rewards/margins": 0.08808780461549759,
+      "rewards/rejected": -0.09631501138210297,
+      "step": 2770
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.898644669914965e-06,
+      "logits/chosen": -2.2914719581604004,
+      "logits/rejected": -2.2036476135253906,
+      "logps/chosen": -222.91055297851562,
+      "logps/rejected": -216.57778930664062,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.01161886565387249,
+      "rewards/margins": 0.07493311911821365,
+      "rewards/rejected": -0.06331426650285721,
+      "step": 2780
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.897029064916778e-06,
+      "logits/chosen": -2.0988786220550537,
+      "logits/rejected": -1.9013208150863647,
+      "logps/chosen": -208.904541015625,
+      "logps/rejected": -201.63589477539062,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.007546191103756428,
+      "rewards/margins": 0.051201723515987396,
+      "rewards/rejected": -0.058747924864292145,
+      "step": 2790
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.895400955840791e-06,
+      "logits/chosen": -2.459437847137451,
+      "logits/rejected": -1.807782769203186,
+      "logps/chosen": -215.23831176757812,
+      "logps/rejected": -182.17108154296875,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.05112996697425842,
+      "rewards/margins": 0.08630537241697311,
+      "rewards/rejected": -0.035175397992134094,
+      "step": 2800
+    },
+    {
+      "epoch": 0.18,
+      "eval_logits/chosen": -2.3020472526550293,
+      "eval_logits/rejected": -2.1165366172790527,
+      "eval_logps/chosen": -228.67945861816406,
+      "eval_logps/rejected": -215.24508666992188,
+      "eval_loss": 0.6903337240219116,
+      "eval_rewards/accuracies": 0.6315000057220459,
+      "eval_rewards/chosen": 0.03325507417321205,
+      "eval_rewards/margins": 0.0695870891213417,
+      "eval_rewards/rejected": -0.03633202239871025,
+      "eval_runtime": 710.6994,
+      "eval_samples_per_second": 2.814,
+      "eval_steps_per_second": 1.407,
+      "step": 2800
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.893760351180018e-06,
+      "logits/chosen": -2.3032994270324707,
+      "logits/rejected": -2.2417054176330566,
+      "logps/chosen": -195.91603088378906,
+      "logps/rejected": -206.2035675048828,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.026252543553709984,
+      "rewards/margins": 0.05186532065272331,
+      "rewards/rejected": -0.02561277151107788,
+      "step": 2810
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.892107259492657e-06,
+      "logits/chosen": -2.2603695392608643,
+      "logits/rejected": -2.022012233734131,
+      "logps/chosen": -237.22506713867188,
+      "logps/rejected": -246.58621215820312,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.026342108845710754,
+      "rewards/margins": 0.03342199698090553,
+      "rewards/rejected": -0.0070798941887915134,
+      "step": 2820
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.890441689402042e-06,
+      "logits/chosen": -2.3950798511505127,
+      "logits/rejected": -2.2143449783325195,
+      "logps/chosen": -331.57049560546875,
+      "logps/rejected": -291.9052734375,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03934413939714432,
+      "rewards/margins": 0.08334746211767197,
+      "rewards/rejected": -0.04400331899523735,
+      "step": 2830
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.888763649596606e-06,
+      "logits/chosen": -2.402956485748291,
+      "logits/rejected": -2.176975965499878,
+      "logps/chosen": -208.06912231445312,
+      "logps/rejected": -206.9716339111328,
+      "loss": 0.69,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.020925721153616905,
+      "rewards/margins": 0.07703931629657745,
+      "rewards/rejected": -0.056113600730895996,
+      "step": 2840
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.887073148829824e-06,
+      "logits/chosen": -2.3179640769958496,
+      "logits/rejected": -2.170621871948242,
+      "logps/chosen": -262.16778564453125,
+      "logps/rejected": -246.90365600585938,
+      "loss": 0.69,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.04735777527093887,
+      "rewards/margins": 0.07671411335468292,
+      "rewards/rejected": -0.029356345534324646,
+      "step": 2850
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.885370195920177e-06,
+      "logits/chosen": -2.199707508087158,
+      "logits/rejected": -2.139965057373047,
+      "logps/chosen": -185.32675170898438,
+      "logps/rejected": -182.80758666992188,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -3.044344521185849e-05,
+      "rewards/margins": 0.060539864003658295,
+      "rewards/rejected": -0.060570307075977325,
+      "step": 2860
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.883654799751101e-06,
+      "logits/chosen": -2.105257749557495,
+      "logits/rejected": -2.3305790424346924,
+      "logps/chosen": -212.72607421875,
+      "logps/rejected": -253.13320922851562,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.029830992221832275,
+      "rewards/margins": 0.056669920682907104,
+      "rewards/rejected": -0.026838932186365128,
+      "step": 2870
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.8819269692709435e-06,
+      "logits/chosen": -2.4003872871398926,
+      "logits/rejected": -2.227214813232422,
+      "logps/chosen": -263.90191650390625,
+      "logps/rejected": -199.56661987304688,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.03144312649965286,
+      "rewards/margins": 0.08026852458715439,
+      "rewards/rejected": -0.048825401812791824,
+      "step": 2880
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.880186713492915e-06,
+      "logits/chosen": -2.2545135021209717,
+      "logits/rejected": -2.066854238510132,
+      "logps/chosen": -227.1837921142578,
+      "logps/rejected": -178.45132446289062,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.005335694644600153,
+      "rewards/margins": 0.04889502376317978,
+      "rewards/rejected": -0.043559327721595764,
+      "step": 2890
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.878434041495041e-06,
+      "logits/chosen": -2.2972848415374756,
+      "logits/rejected": -2.416718006134033,
+      "logps/chosen": -233.84445190429688,
+      "logps/rejected": -244.32199096679688,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.033212922513484955,
+      "rewards/margins": 0.08615333586931229,
+      "rewards/rejected": -0.05294041708111763,
+      "step": 2900
+    },
+    {
+      "epoch": 0.19,
+      "eval_logits/chosen": -2.296010971069336,
+      "eval_logits/rejected": -2.1109728813171387,
+      "eval_logps/chosen": -230.9010009765625,
+      "eval_logps/rejected": -217.73272705078125,
+      "eval_loss": 0.6902217268943787,
+      "eval_rewards/accuracies": 0.6380000114440918,
+      "eval_rewards/chosen": 0.011039442382752895,
+      "eval_rewards/margins": 0.07224779576063156,
+      "eval_rewards/rejected": -0.06120835244655609,
+      "eval_runtime": 712.9014,
+      "eval_samples_per_second": 2.805,
+      "eval_steps_per_second": 1.403,
+      "step": 2900
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.876668962420117e-06,
+      "logits/chosen": -2.2779107093811035,
+      "logits/rejected": -2.0205576419830322,
+      "logps/chosen": -285.90301513671875,
+      "logps/rejected": -234.1432342529297,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0374809131026268,
+      "rewards/margins": 0.07856379449367523,
+      "rewards/rejected": -0.04108288139104843,
+      "step": 2910
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.87489148547566e-06,
+      "logits/chosen": -2.2894115447998047,
+      "logits/rejected": -2.142322540283203,
+      "logps/chosen": -261.393310546875,
+      "logps/rejected": -233.22802734375,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.015348220244050026,
+      "rewards/margins": 0.050076454877853394,
+      "rewards/rejected": -0.06542467325925827,
+      "step": 2920
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.873101619933862e-06,
+      "logits/chosen": -2.5051543712615967,
+      "logits/rejected": -2.1486945152282715,
+      "logps/chosen": -263.4872131347656,
+      "logps/rejected": -221.30996704101562,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.010399745777249336,
+      "rewards/margins": 0.07138345390558243,
+      "rewards/rejected": -0.06098370626568794,
+      "step": 2930
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.8712993751315385e-06,
+      "logits/chosen": -2.283648729324341,
+      "logits/rejected": -2.2247979640960693,
+      "logps/chosen": -120.84417724609375,
+      "logps/rejected": -128.2853240966797,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.001862399629317224,
+      "rewards/margins": 0.03539283573627472,
+      "rewards/rejected": -0.03725523501634598,
+      "step": 2940
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.869484760470079e-06,
+      "logits/chosen": -2.3379874229431152,
+      "logits/rejected": -2.1309611797332764,
+      "logps/chosen": -187.4462127685547,
+      "logps/rejected": -158.68853759765625,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.012146204710006714,
+      "rewards/margins": 0.07541505247354507,
+      "rewards/rejected": -0.06326885521411896,
+      "step": 2950
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.867657785415404e-06,
+      "logits/chosen": -2.2649407386779785,
+      "logits/rejected": -1.9867734909057617,
+      "logps/chosen": -250.11520385742188,
+      "logps/rejected": -217.9238739013672,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.010161913931369781,
+      "rewards/margins": 0.09553287923336029,
+      "rewards/rejected": -0.10569479316473007,
+      "step": 2960
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.865818459497911e-06,
+      "logits/chosen": -2.49599027633667,
+      "logits/rejected": -2.0337436199188232,
+      "logps/chosen": -284.77001953125,
+      "logps/rejected": -206.2366180419922,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.00591338612139225,
+      "rewards/margins": 0.0626380667090416,
+      "rewards/rejected": -0.0685514584183693,
+      "step": 2970
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.863966792312423e-06,
+      "logits/chosen": -2.382023811340332,
+      "logits/rejected": -2.142746686935425,
+      "logps/chosen": -239.0797576904297,
+      "logps/rejected": -208.9804229736328,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.023791249841451645,
+      "rewards/margins": 0.10952029377222061,
+      "rewards/rejected": -0.08572904765605927,
+      "step": 2980
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.862102793518145e-06,
+      "logits/chosen": -2.2269492149353027,
+      "logits/rejected": -2.290496349334717,
+      "logps/chosen": -194.52423095703125,
+      "logps/rejected": -209.1487274169922,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0024234852753579617,
+      "rewards/margins": 0.07699505239725113,
+      "rewards/rejected": -0.07941852509975433,
+      "step": 2990
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.8602264728386075e-06,
+      "logits/chosen": -2.3325839042663574,
+      "logits/rejected": -2.184682607650757,
+      "logps/chosen": -252.10238647460938,
+      "logps/rejected": -256.3367919921875,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.00397127028554678,
+      "rewards/margins": 0.07653030008077621,
+      "rewards/rejected": -0.07255902886390686,
+      "step": 3000
+    },
+    {
+      "epoch": 0.2,
+      "eval_logits/chosen": -2.3181509971618652,
+      "eval_logits/rejected": -2.131211280822754,
+      "eval_logps/chosen": -230.46095275878906,
+      "eval_logps/rejected": -218.1745147705078,
+      "eval_loss": 0.6903403401374817,
+      "eval_rewards/accuracies": 0.6244999766349792,
+      "eval_rewards/chosen": 0.015439935959875584,
+      "eval_rewards/margins": 0.08106595277786255,
+      "eval_rewards/rejected": -0.06562602519989014,
+      "eval_runtime": 709.3669,
+      "eval_samples_per_second": 2.819,
+      "eval_steps_per_second": 1.41,
+      "step": 3000
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.858337840061616e-06,
+      "logits/chosen": -2.309683084487915,
+      "logits/rejected": -2.230560779571533,
+      "logps/chosen": -180.07546997070312,
+      "logps/rejected": -241.62252807617188,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.030049040913581848,
+      "rewards/margins": 0.07897917181253433,
+      "rewards/rejected": -0.048930130898952484,
+      "step": 3010
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.856436905039208e-06,
+      "logits/chosen": -2.3335537910461426,
+      "logits/rejected": -2.174056053161621,
+      "logps/chosen": -207.0081024169922,
+      "logps/rejected": -181.50457763671875,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.04196963459253311,
+      "rewards/margins": 0.08883820474147797,
+      "rewards/rejected": -0.046868570148944855,
+      "step": 3020
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.854523677687588e-06,
+      "logits/chosen": -2.2027428150177,
+      "logits/rejected": -2.271785259246826,
+      "logps/chosen": -176.38583374023438,
+      "logps/rejected": -201.62588500976562,
+      "loss": 0.689,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.048596903681755066,
+      "rewards/margins": 0.06843477487564087,
+      "rewards/rejected": -0.0198378786444664,
+      "step": 3030
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.85259816798709e-06,
+      "logits/chosen": -2.4263253211975098,
+      "logits/rejected": -1.8797962665557861,
+      "logps/chosen": -281.78717041015625,
+      "logps/rejected": -212.32394409179688,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.059749238193035126,
+      "rewards/margins": 0.11092700809240341,
+      "rewards/rejected": -0.05117777734994888,
+      "step": 3040
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.850660385982114e-06,
+      "logits/chosen": -2.4107866287231445,
+      "logits/rejected": -2.2419321537017822,
+      "logps/chosen": -243.84481811523438,
+      "logps/rejected": -195.49806213378906,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0387430340051651,
+      "rewards/margins": 0.06934549659490585,
+      "rewards/rejected": -0.030602458864450455,
+      "step": 3050
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.848710341781081e-06,
+      "logits/chosen": -2.152615547180176,
+      "logits/rejected": -2.259021043777466,
+      "logps/chosen": -176.58517456054688,
+      "logps/rejected": -175.9096221923828,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.026836439967155457,
+      "rewards/margins": 0.050511687994003296,
+      "rewards/rejected": -0.07734812796115875,
+      "step": 3060
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.846748045556377e-06,
+      "logits/chosen": -2.3441312313079834,
+      "logits/rejected": -2.0264244079589844,
+      "logps/chosen": -239.79904174804688,
+      "logps/rejected": -186.3507537841797,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.003119309199973941,
+      "rewards/margins": 0.07062678039073944,
+      "rewards/rejected": -0.07374609261751175,
+      "step": 3070
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.8447735075442995e-06,
+      "logits/chosen": -2.2217564582824707,
+      "logits/rejected": -2.284585952758789,
+      "logps/chosen": -201.0135498046875,
+      "logps/rejected": -206.20608520507812,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.03451332077383995,
+      "rewards/margins": 0.08593029528856277,
+      "rewards/rejected": -0.12044362723827362,
+      "step": 3080
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.8427867380450075e-06,
+      "logits/chosen": -2.406268835067749,
+      "logits/rejected": -2.0118308067321777,
+      "logps/chosen": -234.0061798095703,
+      "logps/rejected": -197.52023315429688,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.036669012159109116,
+      "rewards/margins": 0.09763816744089127,
+      "rewards/rejected": -0.13430717587471008,
+      "step": 3090
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.840787747422462e-06,
+      "logits/chosen": -2.3698010444641113,
+      "logits/rejected": -2.114318609237671,
+      "logps/chosen": -199.02552795410156,
+      "logps/rejected": -175.43988037109375,
+      "loss": 0.692,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.033694010227918625,
+      "rewards/margins": 0.06671958416700363,
+      "rewards/rejected": -0.10041359812021255,
+      "step": 3100
+    },
+    {
+      "epoch": 0.2,
+      "eval_logits/chosen": -2.3160300254821777,
+      "eval_logits/rejected": -2.129782199859619,
+      "eval_logps/chosen": -235.4629669189453,
+      "eval_logps/rejected": -223.5566864013672,
+      "eval_loss": 0.6903056502342224,
+      "eval_rewards/accuracies": 0.6439999938011169,
+      "eval_rewards/chosen": -0.034580256789922714,
+      "eval_rewards/margins": 0.08486771583557129,
+      "eval_rewards/rejected": -0.1194479689002037,
+      "eval_runtime": 712.862,
+      "eval_samples_per_second": 2.806,
+      "eval_steps_per_second": 1.403,
+      "step": 3100
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.838776546104378e-06,
+      "logits/chosen": -2.2874035835266113,
+      "logits/rejected": -2.2883827686309814,
+      "logps/chosen": -282.4599609375,
+      "logps/rejected": -252.1973876953125,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.015067142434418201,
+      "rewards/margins": 0.10139371454715729,
+      "rewards/rejected": -0.11646085977554321,
+      "step": 3110
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.836753144582168e-06,
+      "logits/chosen": -2.2503442764282227,
+      "logits/rejected": -2.0378506183624268,
+      "logps/chosen": -245.8728790283203,
+      "logps/rejected": -234.16201782226562,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.03641275316476822,
+      "rewards/margins": 0.12401758134365082,
+      "rewards/rejected": -0.16043034195899963,
+      "step": 3120
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.834717553410884e-06,
+      "logits/chosen": -2.3153603076934814,
+      "logits/rejected": -2.077373743057251,
+      "logps/chosen": -190.7818145751953,
+      "logps/rejected": -213.3385009765625,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.015058299526572227,
+      "rewards/margins": 0.08887463808059692,
+      "rewards/rejected": -0.1039329394698143,
+      "step": 3130
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.832669783209167e-06,
+      "logits/chosen": -2.192064046859741,
+      "logits/rejected": -2.241379499435425,
+      "logps/chosen": -245.5317840576172,
+      "logps/rejected": -248.6998291015625,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.011057475581765175,
+      "rewards/margins": 0.026191571727395058,
+      "rewards/rejected": -0.03724905103445053,
+      "step": 3140
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.8306098446591895e-06,
+      "logits/chosen": -1.8934345245361328,
+      "logits/rejected": -2.0603950023651123,
+      "logps/chosen": -177.72357177734375,
+      "logps/rejected": -208.8456573486328,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.014547166414558887,
+      "rewards/margins": 0.06190754845738411,
+      "rewards/rejected": -0.07645471394062042,
+      "step": 3150
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.828537748506601e-06,
+      "logits/chosen": -2.411770820617676,
+      "logits/rejected": -2.1453702449798584,
+      "logps/chosen": -273.4091796875,
+      "logps/rejected": -221.22647094726562,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.00010936595208477229,
+      "rewards/margins": 0.04063498228788376,
+      "rewards/rejected": -0.040744349360466,
+      "step": 3160
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.826453505560469e-06,
+      "logits/chosen": -2.0967440605163574,
+      "logits/rejected": -2.0667638778686523,
+      "logps/chosen": -192.63839721679688,
+      "logps/rejected": -184.79087829589844,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.016592925414443016,
+      "rewards/margins": 0.05849025771021843,
+      "rewards/rejected": -0.07508319616317749,
+      "step": 3170
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.824357126693226e-06,
+      "logits/chosen": -2.206259250640869,
+      "logits/rejected": -1.8263355493545532,
+      "logps/chosen": -260.7179260253906,
+      "logps/rejected": -217.93618774414062,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.01974749192595482,
+      "rewards/margins": 0.05945660546422005,
+      "rewards/rejected": -0.07920410484075546,
+      "step": 3180
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.8222486228406105e-06,
+      "logits/chosen": -2.3845696449279785,
+      "logits/rejected": -2.098829746246338,
+      "logps/chosen": -211.3022003173828,
+      "logps/rejected": -183.42645263671875,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0006182378274388611,
+      "rewards/margins": 0.06805343925952911,
+      "rewards/rejected": -0.06867166608572006,
+      "step": 3190
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.820128005001612e-06,
+      "logits/chosen": -2.0764338970184326,
+      "logits/rejected": -1.9985427856445312,
+      "logps/chosen": -217.34097290039062,
+      "logps/rejected": -208.50668334960938,
+      "loss": 0.687,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.016526032239198685,
+      "rewards/margins": 0.12708571553230286,
+      "rewards/rejected": -0.11055967956781387,
+      "step": 3200
+    },
+    {
+      "epoch": 0.21,
+      "eval_logits/chosen": -2.3211934566497803,
+      "eval_logits/rejected": -2.1343932151794434,
+      "eval_logps/chosen": -233.4681854248047,
+      "eval_logps/rejected": -220.65005493164062,
+      "eval_loss": 0.6902753114700317,
+      "eval_rewards/accuracies": 0.6209999918937683,
+      "eval_rewards/chosen": -0.014632347039878368,
+      "eval_rewards/margins": 0.07574935257434845,
+      "eval_rewards/rejected": -0.0903816968202591,
+      "eval_runtime": 710.7834,
+      "eval_samples_per_second": 2.814,
+      "eval_steps_per_second": 1.407,
+      "step": 3200
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.817995284238412e-06,
+      "logits/chosen": -2.1152288913726807,
+      "logits/rejected": -2.1994452476501465,
+      "logps/chosen": -198.7126007080078,
+      "logps/rejected": -236.6934356689453,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.021882567554712296,
+      "rewards/margins": 0.10495994985103607,
+      "rewards/rejected": -0.12684252858161926,
+      "step": 3210
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.815850471676327e-06,
+      "logits/chosen": -2.2534170150756836,
+      "logits/rejected": -2.134138822555542,
+      "logps/chosen": -238.7698211669922,
+      "logps/rejected": -244.8793487548828,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.003558219876140356,
+      "rewards/margins": 0.10356787592172623,
+      "rewards/rejected": -0.10712607949972153,
+      "step": 3220
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.813693578503751e-06,
+      "logits/chosen": -2.303338050842285,
+      "logits/rejected": -2.123116970062256,
+      "logps/chosen": -295.8538513183594,
+      "logps/rejected": -247.59762573242188,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0065292296931147575,
+      "rewards/margins": 0.072014220058918,
+      "rewards/rejected": -0.06548498570919037,
+      "step": 3230
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.811524615972093e-06,
+      "logits/chosen": -2.3142409324645996,
+      "logits/rejected": -2.1741249561309814,
+      "logps/chosen": -230.9072265625,
+      "logps/rejected": -246.0470733642578,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.009716503322124481,
+      "rewards/margins": 0.06937507539987564,
+      "rewards/rejected": -0.07909159362316132,
+      "step": 3240
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.809343595395724e-06,
+      "logits/chosen": -2.51108455657959,
+      "logits/rejected": -2.3194468021392822,
+      "logps/chosen": -191.16067504882812,
+      "logps/rejected": -164.18856811523438,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.021050242707133293,
+      "rewards/margins": 0.03616604954004288,
+      "rewards/rejected": -0.05721629410982132,
+      "step": 3250
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.807150528151918e-06,
+      "logits/chosen": -2.336385726928711,
+      "logits/rejected": -2.1668715476989746,
+      "logps/chosen": -166.63986206054688,
+      "logps/rejected": -193.17758178710938,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0005269769462756813,
+      "rewards/margins": 0.0867081880569458,
+      "rewards/rejected": -0.08618120849132538,
+      "step": 3260
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.804945425680787e-06,
+      "logits/chosen": -2.288424253463745,
+      "logits/rejected": -2.3639869689941406,
+      "logps/chosen": -190.94607543945312,
+      "logps/rejected": -174.963134765625,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.025028562173247337,
+      "rewards/margins": 0.04095301777124405,
+      "rewards/rejected": -0.06598157435655594,
+      "step": 3270
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.802728299485225e-06,
+      "logits/chosen": -2.146742343902588,
+      "logits/rejected": -2.1346192359924316,
+      "logps/chosen": -153.60440063476562,
+      "logps/rejected": -180.19187927246094,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.02999986708164215,
+      "rewards/margins": 0.050558023154735565,
+      "rewards/rejected": -0.08055789768695831,
+      "step": 3280
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.8004991611308495e-06,
+      "logits/chosen": -2.4337520599365234,
+      "logits/rejected": -2.1402642726898193,
+      "logps/chosen": -240.08847045898438,
+      "logps/rejected": -231.9585418701172,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.014652663841843605,
+      "rewards/margins": 0.07920269668102264,
+      "rewards/rejected": -0.06455003470182419,
+      "step": 3290
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.798258022245937e-06,
+      "logits/chosen": -2.3729190826416016,
+      "logits/rejected": -1.9661096334457397,
+      "logps/chosen": -218.466064453125,
+      "logps/rejected": -187.5246124267578,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.00859595276415348,
+      "rewards/margins": 0.07523629814386368,
+      "rewards/rejected": -0.08383224904537201,
+      "step": 3300
+    },
+    {
+      "epoch": 0.22,
+      "eval_logits/chosen": -2.309415817260742,
+      "eval_logits/rejected": -2.1233413219451904,
+      "eval_logps/chosen": -232.6118621826172,
+      "eval_logps/rejected": -220.64340209960938,
+      "eval_loss": 0.6902174353599548,
+      "eval_rewards/accuracies": 0.6420000195503235,
+      "eval_rewards/chosen": -0.006068930495530367,
+      "eval_rewards/margins": 0.08424630761146545,
+      "eval_rewards/rejected": -0.09031523764133453,
+      "eval_runtime": 712.8632,
+      "eval_samples_per_second": 2.806,
+      "eval_steps_per_second": 1.403,
+      "step": 3300
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.796004894521365e-06,
+      "logits/chosen": -2.3003628253936768,
+      "logits/rejected": -2.1167216300964355,
+      "logps/chosen": -230.6715850830078,
+      "logps/rejected": -260.5556640625,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.021018046885728836,
+      "rewards/margins": 0.08627601712942123,
+      "rewards/rejected": -0.10729406774044037,
+      "step": 3310
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.7937397897105545e-06,
+      "logits/chosen": -2.290663242340088,
+      "logits/rejected": -2.2099320888519287,
+      "logps/chosen": -203.26271057128906,
+      "logps/rejected": -182.44137573242188,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.019701208919286728,
+      "rewards/margins": 0.043921031057834625,
+      "rewards/rejected": -0.024219822138547897,
+      "step": 3320
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.791462719629399e-06,
+      "logits/chosen": -2.3039164543151855,
+      "logits/rejected": -2.1613926887512207,
+      "logps/chosen": -183.3705596923828,
+      "logps/rejected": -171.16586303710938,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.023545963689684868,
+      "rewards/margins": 0.10206764936447144,
+      "rewards/rejected": -0.07852168381214142,
+      "step": 3330
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.789173696156212e-06,
+      "logits/chosen": -2.320606231689453,
+      "logits/rejected": -1.9661529064178467,
+      "logps/chosen": -271.17156982421875,
+      "logps/rejected": -267.260009765625,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.05269026756286621,
+      "rewards/margins": 0.14737890660762787,
+      "rewards/rejected": -0.09468863904476166,
+      "step": 3340
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.786872731231662e-06,
+      "logits/chosen": -2.3447282314300537,
+      "logits/rejected": -2.2217040061950684,
+      "logps/chosen": -214.87109375,
+      "logps/rejected": -212.9879608154297,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.02019861713051796,
+      "rewards/margins": 0.09485939145088196,
+      "rewards/rejected": -0.0746607705950737,
+      "step": 3350
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.784559836858709e-06,
+      "logits/chosen": -2.318398952484131,
+      "logits/rejected": -1.8477122783660889,
+      "logps/chosen": -234.3660888671875,
+      "logps/rejected": -210.38357543945312,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.004489635583013296,
+      "rewards/margins": 0.06438425183296204,
+      "rewards/rejected": -0.06887389719486237,
+      "step": 3360
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.782235025102542e-06,
+      "logits/chosen": -2.312790632247925,
+      "logits/rejected": -2.242957830429077,
+      "logps/chosen": -228.43661499023438,
+      "logps/rejected": -220.25283813476562,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.021173015236854553,
+      "rewards/margins": 0.092967689037323,
+      "rewards/rejected": -0.11414071172475815,
+      "step": 3370
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.779898308090519e-06,
+      "logits/chosen": -2.2664966583251953,
+      "logits/rejected": -2.0547492504119873,
+      "logps/chosen": -278.0267333984375,
+      "logps/rejected": -248.862548828125,
+      "loss": 0.689,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.03349475562572479,
+      "rewards/margins": 0.07666581869125366,
+      "rewards/rejected": -0.11016058921813965,
+      "step": 3380
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.777549698012101e-06,
+      "logits/chosen": -2.216127395629883,
+      "logits/rejected": -2.0575273036956787,
+      "logps/chosen": -244.1732177734375,
+      "logps/rejected": -233.697265625,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.009566160850226879,
+      "rewards/margins": 0.08683688193559647,
+      "rewards/rejected": -0.09640304744243622,
+      "step": 3390
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.775189207118787e-06,
+      "logits/chosen": -2.2499351501464844,
+      "logits/rejected": -2.054161548614502,
+      "logps/chosen": -271.6486511230469,
+      "logps/rejected": -258.4850158691406,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.0016902908682823181,
+      "rewards/margins": 0.09174026548862457,
+      "rewards/rejected": -0.09004998207092285,
+      "step": 3400
+    },
+    {
+      "epoch": 0.22,
+      "eval_logits/chosen": -2.3067705631256104,
+      "eval_logits/rejected": -2.121011734008789,
+      "eval_logps/chosen": -233.02999877929688,
+      "eval_logps/rejected": -220.44912719726562,
+      "eval_loss": 0.6904054880142212,
+      "eval_rewards/accuracies": 0.6345000267028809,
+      "eval_rewards/chosen": -0.010250742547214031,
+      "eval_rewards/margins": 0.07812146842479706,
+      "eval_rewards/rejected": -0.08837221562862396,
+      "eval_runtime": 711.3718,
+      "eval_samples_per_second": 2.811,
+      "eval_steps_per_second": 1.406,
+      "step": 3400
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.772816847724054e-06,
+      "logits/chosen": -2.405539035797119,
+      "logits/rejected": -2.133857011795044,
+      "logps/chosen": -224.6778106689453,
+      "logps/rejected": -227.91207885742188,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.021232225000858307,
+      "rewards/margins": 0.03736606985330582,
+      "rewards/rejected": -0.058598291128873825,
+      "step": 3410
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.770432632203294e-06,
+      "logits/chosen": -2.139242649078369,
+      "logits/rejected": -2.053284168243408,
+      "logps/chosen": -248.99319458007812,
+      "logps/rejected": -202.07363891601562,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.053460635244846344,
+      "rewards/margins": 0.03398740664124489,
+      "rewards/rejected": -0.08744804561138153,
+      "step": 3420
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.768036572993738e-06,
+      "logits/chosen": -2.1945345401763916,
+      "logits/rejected": -2.288442611694336,
+      "logps/chosen": -285.4275207519531,
+      "logps/rejected": -275.6911315917969,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.054085589945316315,
+      "rewards/margins": 0.05863531306385994,
+      "rewards/rejected": -0.11272089183330536,
+      "step": 3430
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.765628682594409e-06,
+      "logits/chosen": -2.3740832805633545,
+      "logits/rejected": -2.1946797370910645,
+      "logps/chosen": -246.08438110351562,
+      "logps/rejected": -229.9820098876953,
+      "loss": 0.689,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.023272987455129623,
+      "rewards/margins": 0.07800062000751495,
+      "rewards/rejected": -0.10127361863851547,
+      "step": 3440
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.763208973566041e-06,
+      "logits/chosen": -2.185068130493164,
+      "logits/rejected": -2.204409122467041,
+      "logps/chosen": -187.07476806640625,
+      "logps/rejected": -213.8040771484375,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.029450953006744385,
+      "rewards/margins": 0.08014042675495148,
+      "rewards/rejected": -0.10959136486053467,
+      "step": 3450
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.76077745853102e-06,
+      "logits/chosen": -2.4352564811706543,
+      "logits/rejected": -2.2802255153656006,
+      "logps/chosen": -252.4907684326172,
+      "logps/rejected": -262.39654541015625,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.024366283789277077,
+      "rewards/margins": 0.08780606091022491,
+      "rewards/rejected": -0.11217234283685684,
+      "step": 3460
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.758334150173322e-06,
+      "logits/chosen": -2.3180832862854004,
+      "logits/rejected": -2.12862491607666,
+      "logps/chosen": -261.38922119140625,
+      "logps/rejected": -242.67416381835938,
+      "loss": 0.693,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.01694723591208458,
+      "rewards/margins": 0.06338542699813843,
+      "rewards/rejected": -0.04643818736076355,
+      "step": 3470
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.755879061238439e-06,
+      "logits/chosen": -2.39463472366333,
+      "logits/rejected": -2.1686863899230957,
+      "logps/chosen": -254.03067016601562,
+      "logps/rejected": -246.9172821044922,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.02142667956650257,
+      "rewards/margins": 0.05523737147450447,
+      "rewards/rejected": -0.03381068632006645,
+      "step": 3480
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.753412204533317e-06,
+      "logits/chosen": -2.5263428688049316,
+      "logits/rejected": -2.0499589443206787,
+      "logps/chosen": -260.4190673828125,
+      "logps/rejected": -223.6571502685547,
+      "loss": 0.689,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.024389993399381638,
+      "rewards/margins": 0.0862947553396225,
+      "rewards/rejected": -0.061904750764369965,
+      "step": 3490
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.750933592926292e-06,
+      "logits/chosen": -2.398818254470825,
+      "logits/rejected": -2.0742526054382324,
+      "logps/chosen": -217.53012084960938,
+      "logps/rejected": -198.85635375976562,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.011975173838436604,
+      "rewards/margins": 0.08542615175247192,
+      "rewards/rejected": -0.073450967669487,
+      "step": 3500
+    },
+    {
+      "epoch": 0.23,
+      "eval_logits/chosen": -2.3047008514404297,
+      "eval_logits/rejected": -2.119340419769287,
+      "eval_logps/chosen": -230.0756072998047,
+      "eval_logps/rejected": -217.86997985839844,
+      "eval_loss": 0.6902625560760498,
+      "eval_rewards/accuracies": 0.6355000138282776,
+      "eval_rewards/chosen": 0.019293660297989845,
+      "eval_rewards/margins": 0.0818745344877243,
+      "eval_rewards/rejected": -0.06258086860179901,
+      "eval_runtime": 712.428,
+      "eval_samples_per_second": 2.807,
+      "eval_steps_per_second": 1.404,
+      "step": 3500
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.7484432393470124e-06,
+      "logits/chosen": -2.486417531967163,
+      "logits/rejected": -1.9664733409881592,
+      "logps/chosen": -199.4810028076172,
+      "logps/rejected": -157.95590209960938,
+      "loss": 0.6836,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.026009265333414078,
+      "rewards/margins": 0.13871736824512482,
+      "rewards/rejected": -0.11270810663700104,
+      "step": 3510
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.745941156786385e-06,
+      "logits/chosen": -2.092363119125366,
+      "logits/rejected": -2.062434434890747,
+      "logps/chosen": -150.09767150878906,
+      "logps/rejected": -195.23306274414062,
+      "loss": 0.6845,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.029414480552077293,
+      "rewards/margins": 0.1429421305656433,
+      "rewards/rejected": -0.11352765560150146,
+      "step": 3520
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.743427358296497e-06,
+      "logits/chosen": -2.2270138263702393,
+      "logits/rejected": -2.030658483505249,
+      "logps/chosen": -187.15162658691406,
+      "logps/rejected": -217.05062866210938,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.025566350668668747,
+      "rewards/margins": 0.18591654300689697,
+      "rewards/rejected": -0.16035018861293793,
+      "step": 3530
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.740901856990553e-06,
+      "logits/chosen": -2.149793863296509,
+      "logits/rejected": -1.9605810642242432,
+      "logps/chosen": -255.32052612304688,
+      "logps/rejected": -219.46920776367188,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.013713860884308815,
+      "rewards/margins": 0.08221259713172913,
+      "rewards/rejected": -0.06849874556064606,
+      "step": 3540
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.738364666042804e-06,
+      "logits/chosen": -2.3816933631896973,
+      "logits/rejected": -1.9584366083145142,
+      "logps/chosen": -286.97796630859375,
+      "logps/rejected": -231.30648803710938,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0476701520383358,
+      "rewards/margins": 0.07968376576900482,
+      "rewards/rejected": -0.03201361373066902,
+      "step": 3550
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.735815798688483e-06,
+      "logits/chosen": -2.3232216835021973,
+      "logits/rejected": -2.1071863174438477,
+      "logps/chosen": -194.87954711914062,
+      "logps/rejected": -224.50369262695312,
+      "loss": 0.6873,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.03452432155609131,
+      "rewards/margins": 0.09485017508268356,
+      "rewards/rejected": -0.06032586842775345,
+      "step": 3560
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.7332552682237285e-06,
+      "logits/chosen": -2.3406100273132324,
+      "logits/rejected": -1.8915197849273682,
+      "logps/chosen": -169.45468139648438,
+      "logps/rejected": -163.56141662597656,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.043052736669778824,
+      "rewards/margins": 0.09879221022129059,
+      "rewards/rejected": -0.05573946237564087,
+      "step": 3570
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.7306830880055234e-06,
+      "logits/chosen": -2.3042500019073486,
+      "logits/rejected": -2.2339184284210205,
+      "logps/chosen": -191.6345672607422,
+      "logps/rejected": -207.58676147460938,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.00900744367390871,
+      "rewards/margins": 0.07615131884813309,
+      "rewards/rejected": -0.08515877276659012,
+      "step": 3580
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.728099271451619e-06,
+      "logits/chosen": -2.341984510421753,
+      "logits/rejected": -2.2418646812438965,
+      "logps/chosen": -190.93679809570312,
+      "logps/rejected": -190.25389099121094,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0007123596733435988,
+      "rewards/margins": 0.06269621104001999,
+      "rewards/rejected": -0.0619838610291481,
+      "step": 3590
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.725503832040466e-06,
+      "logits/chosen": -2.1511435508728027,
+      "logits/rejected": -2.1525609493255615,
+      "logps/chosen": -148.30784606933594,
+      "logps/rejected": -181.79171752929688,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.008500255644321442,
+      "rewards/margins": 0.07396461069583893,
+      "rewards/rejected": -0.06546434760093689,
+      "step": 3600
+    },
+    {
+      "epoch": 0.24,
+      "eval_logits/chosen": -2.3040733337402344,
+      "eval_logits/rejected": -2.11887264251709,
+      "eval_logps/chosen": -230.52880859375,
+      "eval_logps/rejected": -218.516357421875,
+      "eval_loss": 0.6901616454124451,
+      "eval_rewards/accuracies": 0.6359999775886536,
+      "eval_rewards/chosen": 0.014761154539883137,
+      "eval_rewards/margins": 0.08380559831857681,
+      "eval_rewards/rejected": -0.06904443353414536,
+      "eval_runtime": 712.8273,
+      "eval_samples_per_second": 2.806,
+      "eval_steps_per_second": 1.403,
+      "step": 3600
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.722896783311152e-06,
+      "logits/chosen": -2.282073497772217,
+      "logits/rejected": -2.17645263671875,
+      "logps/chosen": -259.38861083984375,
+      "logps/rejected": -316.8056640625,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.003396064043045044,
+      "rewards/margins": 0.06283075362443924,
+      "rewards/rejected": -0.06622681021690369,
+      "step": 3610
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.720278138863318e-06,
+      "logits/chosen": -2.4280340671539307,
+      "logits/rejected": -2.218613862991333,
+      "logps/chosen": -190.4235382080078,
+      "logps/rejected": -164.82579040527344,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.003018149407580495,
+      "rewards/margins": 0.0620940737426281,
+      "rewards/rejected": -0.06511221826076508,
+      "step": 3620
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.717647912357095e-06,
+      "logits/chosen": -2.361996650695801,
+      "logits/rejected": -2.448129892349243,
+      "logps/chosen": -275.29071044921875,
+      "logps/rejected": -289.21759033203125,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.02813585475087166,
+      "rewards/margins": 0.009944294579327106,
+      "rewards/rejected": -0.03808014467358589,
+      "step": 3630
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.715006117513035e-06,
+      "logits/chosen": -2.460373640060425,
+      "logits/rejected": -2.220986843109131,
+      "logps/chosen": -321.8302307128906,
+      "logps/rejected": -273.5966796875,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.022954054176807404,
+      "rewards/margins": 0.06946495175361633,
+      "rewards/rejected": -0.04651089757680893,
+      "step": 3640
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.7123527681120326e-06,
+      "logits/chosen": -2.275266647338867,
+      "logits/rejected": -2.134054660797119,
+      "logps/chosen": -247.04855346679688,
+      "logps/rejected": -226.111328125,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0012259014183655381,
+      "rewards/margins": 0.07214462757110596,
+      "rewards/rejected": -0.0709187239408493,
+      "step": 3650
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.7096878779952594e-06,
+      "logits/chosen": -2.357933759689331,
+      "logits/rejected": -2.303584575653076,
+      "logps/chosen": -275.9615783691406,
+      "logps/rejected": -278.7004089355469,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.005482043139636517,
+      "rewards/margins": 0.05321590229868889,
+      "rewards/rejected": -0.058697957545518875,
+      "step": 3660
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.707011461064086e-06,
+      "logits/chosen": -2.159414768218994,
+      "logits/rejected": -1.9229214191436768,
+      "logps/chosen": -308.0876770019531,
+      "logps/rejected": -274.2186584472656,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.0077134473249316216,
+      "rewards/margins": 0.10321645438671112,
+      "rewards/rejected": -0.09550300985574722,
+      "step": 3670
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.704323531280016e-06,
+      "logits/chosen": -2.2135162353515625,
+      "logits/rejected": -2.040491819381714,
+      "logps/chosen": -324.78515625,
+      "logps/rejected": -248.89889526367188,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.02426346018910408,
+      "rewards/margins": 0.07309317588806152,
+      "rewards/rejected": -0.048829711973667145,
+      "step": 3680
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.701624102664606e-06,
+      "logits/chosen": -2.370241165161133,
+      "logits/rejected": -2.0312302112579346,
+      "logps/chosen": -262.2061767578125,
+      "logps/rejected": -215.11416625976562,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.010902756825089455,
+      "rewards/margins": 0.07008221745491028,
+      "rewards/rejected": -0.08098497986793518,
+      "step": 3690
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.698913189299399e-06,
+      "logits/chosen": -2.2025485038757324,
+      "logits/rejected": -2.3091206550598145,
+      "logps/chosen": -187.55035400390625,
+      "logps/rejected": -225.8077850341797,
+      "loss": 0.694,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.013985480181872845,
+      "rewards/margins": 0.05750720947980881,
+      "rewards/rejected": -0.07149268686771393,
+      "step": 3700
+    },
+    {
+      "epoch": 0.24,
+      "eval_logits/chosen": -2.281970262527466,
+      "eval_logits/rejected": -2.0983099937438965,
+      "eval_logps/chosen": -234.87881469726562,
+      "eval_logps/rejected": -221.8666534423828,
+      "eval_loss": 0.6903954744338989,
+      "eval_rewards/accuracies": 0.6389999985694885,
+      "eval_rewards/chosen": -0.028738651424646378,
+      "eval_rewards/margins": 0.0738087072968483,
+      "eval_rewards/rejected": -0.10254734754562378,
+      "eval_runtime": 711.1172,
+      "eval_samples_per_second": 2.812,
+      "eval_steps_per_second": 1.406,
+      "step": 3700
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.696190805325847e-06,
+      "logits/chosen": -2.2970728874206543,
+      "logits/rejected": -2.1544101238250732,
+      "logps/chosen": -207.85110473632812,
+      "logps/rejected": -189.63479614257812,
+      "loss": 0.69,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.014722605235874653,
+      "rewards/margins": 0.08902369439601898,
+      "rewards/rejected": -0.10374629497528076,
+      "step": 3710
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.693456964945239e-06,
+      "logits/chosen": -2.416215419769287,
+      "logits/rejected": -1.9415165185928345,
+      "logps/chosen": -298.4938049316406,
+      "logps/rejected": -207.4619598388672,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.003882091958075762,
+      "rewards/margins": 0.10060401260852814,
+      "rewards/rejected": -0.10448610782623291,
+      "step": 3720
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.6907116824186245e-06,
+      "logits/chosen": -2.3689780235290527,
+      "logits/rejected": -2.3212902545928955,
+      "logps/chosen": -226.6787109375,
+      "logps/rejected": -231.8772430419922,
+      "loss": 0.689,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.006488241255283356,
+      "rewards/margins": 0.05394769459962845,
+      "rewards/rejected": -0.047459445893764496,
+      "step": 3730
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.687954972066742e-06,
+      "logits/chosen": -2.260472297668457,
+      "logits/rejected": -1.9865401983261108,
+      "logps/chosen": -227.1370086669922,
+      "logps/rejected": -220.00808715820312,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.026792461052536964,
+      "rewards/margins": 0.1374204158782959,
+      "rewards/rejected": -0.11062794923782349,
+      "step": 3740
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.685186848269944e-06,
+      "logits/chosen": -2.2468438148498535,
+      "logits/rejected": -2.11405873298645,
+      "logps/chosen": -209.9337615966797,
+      "logps/rejected": -178.09884643554688,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.01938585564494133,
+      "rewards/margins": 0.05544018745422363,
+      "rewards/rejected": -0.0360543318092823,
+      "step": 3750
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.682407325468119e-06,
+      "logits/chosen": -2.323763370513916,
+      "logits/rejected": -1.967911720275879,
+      "logps/chosen": -214.25634765625,
+      "logps/rejected": -191.69644165039062,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.03162650763988495,
+      "rewards/margins": 0.11069830507040024,
+      "rewards/rejected": -0.07907179743051529,
+      "step": 3760
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.67961641816062e-06,
+      "logits/chosen": -2.301642417907715,
+      "logits/rejected": -2.097708225250244,
+      "logps/chosen": -271.3573303222656,
+      "logps/rejected": -236.2880096435547,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.044394414871931076,
+      "rewards/margins": 0.06260766088962555,
+      "rewards/rejected": -0.018213242292404175,
+      "step": 3770
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.676814140906188e-06,
+      "logits/chosen": -2.180407762527466,
+      "logits/rejected": -2.048719882965088,
+      "logps/chosen": -237.28604125976562,
+      "logps/rejected": -216.0989227294922,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.022501787170767784,
+      "rewards/margins": 0.09418568760156631,
+      "rewards/rejected": -0.07168390601873398,
+      "step": 3780
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.674000508322872e-06,
+      "logits/chosen": -2.022406578063965,
+      "logits/rejected": -2.098022222518921,
+      "logps/chosen": -214.1189727783203,
+      "logps/rejected": -229.61483764648438,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0341593436896801,
+      "rewards/margins": 0.06312253326177597,
+      "rewards/rejected": -0.02896319329738617,
+      "step": 3790
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.671175535087959e-06,
+      "logits/chosen": -2.194871664047241,
+      "logits/rejected": -2.153036117553711,
+      "logps/chosen": -285.90673828125,
+      "logps/rejected": -293.92242431640625,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.05420888587832451,
+      "rewards/margins": 0.11247305572032928,
+      "rewards/rejected": -0.05826416611671448,
+      "step": 3800
+    },
+    {
+      "epoch": 0.25,
+      "eval_logits/chosen": -2.2757647037506104,
+      "eval_logits/rejected": -2.092339277267456,
+      "eval_logps/chosen": -227.5012969970703,
+      "eval_logps/rejected": -213.98056030273438,
+      "eval_loss": 0.6902437806129456,
+      "eval_rewards/accuracies": 0.6320000290870667,
+      "eval_rewards/chosen": 0.04503653571009636,
+      "eval_rewards/margins": 0.06872312724590302,
+      "eval_rewards/rejected": -0.023686589673161507,
+      "eval_runtime": 711.0018,
+      "eval_samples_per_second": 2.813,
+      "eval_steps_per_second": 1.406,
+      "step": 3800
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.6683392359378924e-06,
+      "logits/chosen": -2.1588714122772217,
+      "logits/rejected": -1.9822829961776733,
+      "logps/chosen": -231.51382446289062,
+      "logps/rejected": -211.3966827392578,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.03635237738490105,
+      "rewards/margins": 0.05399390310049057,
+      "rewards/rejected": -0.017641523852944374,
+      "step": 3810
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.665491625668198e-06,
+      "logits/chosen": -2.074720859527588,
+      "logits/rejected": -2.1197152137756348,
+      "logps/chosen": -153.5623321533203,
+      "logps/rejected": -185.47169494628906,
+      "loss": 0.6873,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.02684941329061985,
+      "rewards/margins": 0.08354301750659943,
+      "rewards/rejected": -0.05669360235333443,
+      "step": 3820
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.662632719133407e-06,
+      "logits/chosen": -2.3514442443847656,
+      "logits/rejected": -2.0640666484832764,
+      "logps/chosen": -225.99966430664062,
+      "logps/rejected": -167.13034057617188,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.047978900372982025,
+      "rewards/margins": 0.08138148486614227,
+      "rewards/rejected": -0.03340258076786995,
+      "step": 3830
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.659762531246974e-06,
+      "logits/chosen": -2.2433078289031982,
+      "logits/rejected": -2.0894062519073486,
+      "logps/chosen": -214.85104370117188,
+      "logps/rejected": -186.99905395507812,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0027232493739575148,
+      "rewards/margins": 0.061046671122312546,
+      "rewards/rejected": -0.06376992166042328,
+      "step": 3840
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.656881076981207e-06,
+      "logits/chosen": -2.3131306171417236,
+      "logits/rejected": -2.1745781898498535,
+      "logps/chosen": -212.8335418701172,
+      "logps/rejected": -200.85433959960938,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.003498140024021268,
+      "rewards/margins": 0.059195131063461304,
+      "rewards/rejected": -0.055697001516819,
+      "step": 3850
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.653988371367183e-06,
+      "logits/chosen": -2.2689290046691895,
+      "logits/rejected": -2.0051369667053223,
+      "logps/chosen": -239.0817413330078,
+      "logps/rejected": -183.85263061523438,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.01722235046327114,
+      "rewards/margins": 0.05978889390826225,
+      "rewards/rejected": -0.04256654158234596,
+      "step": 3860
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.651084429494671e-06,
+      "logits/chosen": -2.3513553142547607,
+      "logits/rejected": -2.0689337253570557,
+      "logps/chosen": -272.21990966796875,
+      "logps/rejected": -197.9673614501953,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0313107892870903,
+      "rewards/margins": 0.052712440490722656,
+      "rewards/rejected": -0.021401654928922653,
+      "step": 3870
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.648169266512053e-06,
+      "logits/chosen": -2.4198365211486816,
+      "logits/rejected": -2.141869068145752,
+      "logps/chosen": -219.36965942382812,
+      "logps/rejected": -180.29354858398438,
+      "loss": 0.69,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0595845952630043,
+      "rewards/margins": 0.058780230581760406,
+      "rewards/rejected": 0.0008043628185987473,
+      "step": 3880
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.6452428976262505e-06,
+      "logits/chosen": -2.23230242729187,
+      "logits/rejected": -2.0137524604797363,
+      "logps/chosen": -199.0391387939453,
+      "logps/rejected": -166.9921112060547,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.04752303659915924,
+      "rewards/margins": 0.1227576732635498,
+      "rewards/rejected": -0.07523464411497116,
+      "step": 3890
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.642305338102633e-06,
+      "logits/chosen": -2.2830934524536133,
+      "logits/rejected": -2.3560256958007812,
+      "logps/chosen": -158.4663543701172,
+      "logps/rejected": -184.8391876220703,
+      "loss": 0.6877,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.032832108438014984,
+      "rewards/margins": 0.08227143436670303,
+      "rewards/rejected": -0.04943932965397835,
+      "step": 3900
+    },
+    {
+      "epoch": 0.26,
+      "eval_logits/chosen": -2.2935938835144043,
+      "eval_logits/rejected": -2.108933210372925,
+      "eval_logps/chosen": -229.80093383789062,
+      "eval_logps/rejected": -217.31515502929688,
+      "eval_loss": 0.6902133822441101,
+      "eval_rewards/accuracies": 0.6244999766349792,
+      "eval_rewards/chosen": 0.02204015851020813,
+      "eval_rewards/margins": 0.07907257974147797,
+      "eval_rewards/rejected": -0.05703242868185043,
+      "eval_runtime": 711.7595,
+      "eval_samples_per_second": 2.81,
+      "eval_steps_per_second": 1.405,
+      "step": 3900
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.639356603264953e-06,
+      "logits/chosen": -2.338958263397217,
+      "logits/rejected": -2.1034774780273438,
+      "logps/chosen": -240.74267578125,
+      "logps/rejected": -219.4384307861328,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.010891283862292767,
+      "rewards/margins": 0.04426593333482742,
+      "rewards/rejected": -0.03337464481592178,
+      "step": 3910
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.636396708495255e-06,
+      "logits/chosen": -2.1757044792175293,
+      "logits/rejected": -2.1606650352478027,
+      "logps/chosen": -225.34707641601562,
+      "logps/rejected": -206.83816528320312,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.03848005831241608,
+      "rewards/margins": 0.06334998458623886,
+      "rewards/rejected": -0.024869924411177635,
+      "step": 3920
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.633425669233799e-06,
+      "logits/chosen": -2.274425983428955,
+      "logits/rejected": -2.2967169284820557,
+      "logps/chosen": -230.49679565429688,
+      "logps/rejected": -234.1136474609375,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.04329800605773926,
+      "rewards/margins": 0.08064167201519012,
+      "rewards/rejected": -0.03734365105628967,
+      "step": 3930
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.6304435009789825e-06,
+      "logits/chosen": -2.2880218029022217,
+      "logits/rejected": -2.0706839561462402,
+      "logps/chosen": -233.8865509033203,
+      "logps/rejected": -172.7992401123047,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.03620678931474686,
+      "rewards/margins": 0.09822587668895721,
+      "rewards/rejected": -0.062019091099500656,
+      "step": 3940
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.627450219287256e-06,
+      "logits/chosen": -2.3368616104125977,
+      "logits/rejected": -2.1908602714538574,
+      "logps/chosen": -177.8789825439453,
+      "logps/rejected": -161.35159301757812,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.03680109232664108,
+      "rewards/margins": 0.05983690172433853,
+      "rewards/rejected": -0.02303580567240715,
+      "step": 3950
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.624445839773042e-06,
+      "logits/chosen": -2.2832131385803223,
+      "logits/rejected": -2.2111704349517822,
+      "logps/chosen": -169.68849182128906,
+      "logps/rejected": -171.02833557128906,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.003228194313123822,
+      "rewards/margins": 0.024037057533860207,
+      "rewards/rejected": -0.020808864384889603,
+      "step": 3960
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.621430378108656e-06,
+      "logits/chosen": -2.264580249786377,
+      "logits/rejected": -2.1098790168762207,
+      "logps/chosen": -257.55718994140625,
+      "logps/rejected": -260.2860412597656,
+      "loss": 0.689,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.002011381322517991,
+      "rewards/margins": 0.09493207186460495,
+      "rewards/rejected": -0.092920683324337,
+      "step": 3970
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.618403850024223e-06,
+      "logits/chosen": -2.1793527603149414,
+      "logits/rejected": -1.9493013620376587,
+      "logps/chosen": -256.003173828125,
+      "logps/rejected": -215.7708282470703,
+      "loss": 0.691,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.005130067467689514,
+      "rewards/margins": 0.06429970264434814,
+      "rewards/rejected": -0.06942977011203766,
+      "step": 3980
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.615366271307598e-06,
+      "logits/chosen": -2.3207273483276367,
+      "logits/rejected": -2.164661407470703,
+      "logps/chosen": -196.49105834960938,
+      "logps/rejected": -191.2655029296875,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.045480918139219284,
+      "rewards/margins": 0.06666766852140427,
+      "rewards/rejected": -0.11214858293533325,
+      "step": 3990
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.612317657804277e-06,
+      "logits/chosen": -2.1907215118408203,
+      "logits/rejected": -2.2445192337036133,
+      "logps/chosen": -149.4791259765625,
+      "logps/rejected": -210.52490234375,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.02246091142296791,
+      "rewards/margins": 0.09697895497083664,
+      "rewards/rejected": -0.11943986266851425,
+      "step": 4000
+    },
+    {
+      "epoch": 0.26,
+      "eval_logits/chosen": -2.2913191318511963,
+      "eval_logits/rejected": -2.106405258178711,
+      "eval_logps/chosen": -232.1314697265625,
+      "eval_logps/rejected": -219.69049072265625,
+      "eval_loss": 0.6901000738143921,
+      "eval_rewards/accuracies": 0.6359999775886536,
+      "eval_rewards/chosen": -0.0012654466554522514,
+      "eval_rewards/margins": 0.07952029258012772,
+      "eval_rewards/rejected": -0.08078574389219284,
+      "eval_runtime": 712.6087,
+      "eval_samples_per_second": 2.807,
+      "eval_steps_per_second": 1.403,
+      "step": 4000
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.6092580254173236e-06,
+      "logits/chosen": -2.1913225650787354,
+      "logits/rejected": -1.959183931350708,
+      "logps/chosen": -258.2712097167969,
+      "logps/rejected": -247.7600555419922,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.00948442704975605,
+      "rewards/margins": 0.08156983554363251,
+      "rewards/rejected": -0.09105426073074341,
+      "step": 4010
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.606187390107277e-06,
+      "logits/chosen": -2.1633963584899902,
+      "logits/rejected": -1.9803078174591064,
+      "logps/chosen": -230.2688751220703,
+      "logps/rejected": -197.57717895507812,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.053934670984745026,
+      "rewards/margins": 0.06301041692495346,
+      "rewards/rejected": -0.11694508790969849,
+      "step": 4020
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.603105767892077e-06,
+      "logits/chosen": -2.264932632446289,
+      "logits/rejected": -2.1878082752227783,
+      "logps/chosen": -195.32559204101562,
+      "logps/rejected": -221.0668182373047,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.0063558658584952354,
+      "rewards/margins": 0.07787985354661942,
+      "rewards/rejected": -0.08423570543527603,
+      "step": 4030
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.6000131748469725e-06,
+      "logits/chosen": -2.3445935249328613,
+      "logits/rejected": -1.9680637121200562,
+      "logps/chosen": -250.29660034179688,
+      "logps/rejected": -182.46685791015625,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.0015633717412129045,
+      "rewards/margins": 0.0648859366774559,
+      "rewards/rejected": -0.06644931435585022,
+      "step": 4040
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.596909627104445e-06,
+      "logits/chosen": -2.3850934505462646,
+      "logits/rejected": -2.3055262565612793,
+      "logps/chosen": -251.54226684570312,
+      "logps/rejected": -226.3249053955078,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.028039926663041115,
+      "rewards/margins": 0.07416818290948868,
+      "rewards/rejected": -0.10220811516046524,
+      "step": 4050
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.5937951408541215e-06,
+      "logits/chosen": -2.433464527130127,
+      "logits/rejected": -1.9047530889511108,
+      "logps/chosen": -254.9823760986328,
+      "logps/rejected": -220.0203857421875,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.023438826203346252,
+      "rewards/margins": 0.10415074974298477,
+      "rewards/rejected": -0.12758956849575043,
+      "step": 4060
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.590669732342685e-06,
+      "logits/chosen": -2.1566336154937744,
+      "logits/rejected": -2.012592315673828,
+      "logps/chosen": -213.21224975585938,
+      "logps/rejected": -226.32504272460938,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.010646522045135498,
+      "rewards/margins": 0.11220131814479828,
+      "rewards/rejected": -0.12284784018993378,
+      "step": 4070
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.587533417873799e-06,
+      "logits/chosen": -2.2341346740722656,
+      "logits/rejected": -2.3223414421081543,
+      "logps/chosen": -195.48965454101562,
+      "logps/rejected": -263.8111267089844,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.023255977779626846,
+      "rewards/margins": 0.08617839962244034,
+      "rewards/rejected": -0.10943436622619629,
+      "step": 4080
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.584386213808016e-06,
+      "logits/chosen": -2.2321903705596924,
+      "logits/rejected": -1.8963381052017212,
+      "logps/chosen": -222.75430297851562,
+      "logps/rejected": -183.17471313476562,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.01075290609151125,
+      "rewards/margins": 0.061965636909008026,
+      "rewards/rejected": -0.072718545794487,
+      "step": 4090
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.581228136562693e-06,
+      "logits/chosen": -2.122157096862793,
+      "logits/rejected": -2.2352585792541504,
+      "logps/chosen": -239.14389038085938,
+      "logps/rejected": -216.71829223632812,
+      "loss": 0.693,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.009634166955947876,
+      "rewards/margins": 0.03259027749300003,
+      "rewards/rejected": -0.042224448174238205,
+      "step": 4100
+    },
+    {
+      "epoch": 0.27,
+      "eval_logits/chosen": -2.3034896850585938,
+      "eval_logits/rejected": -2.117652654647827,
+      "eval_logps/chosen": -233.33334350585938,
+      "eval_logps/rejected": -219.198486328125,
+      "eval_loss": 0.6903753876686096,
+      "eval_rewards/accuracies": 0.628000020980835,
+      "eval_rewards/chosen": -0.013284044340252876,
+      "eval_rewards/margins": 0.06258184462785721,
+      "eval_rewards/rejected": -0.07586588710546494,
+      "eval_runtime": 713.5236,
+      "eval_samples_per_second": 2.803,
+      "eval_steps_per_second": 1.401,
+      "step": 4100
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.578059202611909e-06,
+      "logits/chosen": -2.3259823322296143,
+      "logits/rejected": -2.100602149963379,
+      "logps/chosen": -256.8636779785156,
+      "logps/rejected": -246.4088897705078,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.002952608745545149,
+      "rewards/margins": 0.0430120974779129,
+      "rewards/rejected": -0.04005948826670647,
+      "step": 4110
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.574879428486376e-06,
+      "logits/chosen": -2.3123717308044434,
+      "logits/rejected": -2.031857967376709,
+      "logps/chosen": -214.11416625976562,
+      "logps/rejected": -217.881591796875,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.011976310983300209,
+      "rewards/margins": 0.06689594686031342,
+      "rewards/rejected": -0.07887225598096848,
+      "step": 4120
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.571688830773352e-06,
+      "logits/chosen": -2.3346524238586426,
+      "logits/rejected": -2.226778507232666,
+      "logps/chosen": -223.7250213623047,
+      "logps/rejected": -206.4811553955078,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.016568060964345932,
+      "rewards/margins": 0.02635127305984497,
+      "rewards/rejected": -0.0429193340241909,
+      "step": 4130
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.568487426116559e-06,
+      "logits/chosen": -2.2562003135681152,
+      "logits/rejected": -2.2672030925750732,
+      "logps/chosen": -172.1905059814453,
+      "logps/rejected": -168.9148712158203,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.0021867998875677586,
+      "rewards/margins": 0.04257757216691971,
+      "rewards/rejected": -0.04476437345147133,
+      "step": 4140
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.565275231216092e-06,
+      "logits/chosen": -2.171159267425537,
+      "logits/rejected": -2.1469078063964844,
+      "logps/chosen": -151.336669921875,
+      "logps/rejected": -201.35855102539062,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 6.799399852752686e-05,
+      "rewards/margins": 0.04031980410218239,
+      "rewards/rejected": -0.04025180637836456,
+      "step": 4150
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.562052262828331e-06,
+      "logits/chosen": -2.2262353897094727,
+      "logits/rejected": -2.083603620529175,
+      "logps/chosen": -201.1298065185547,
+      "logps/rejected": -201.3617706298828,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.02318784035742283,
+      "rewards/margins": 0.055079467594623566,
+      "rewards/rejected": -0.07826730608940125,
+      "step": 4160
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.558818537765861e-06,
+      "logits/chosen": -2.4017839431762695,
+      "logits/rejected": -2.179560899734497,
+      "logps/chosen": -237.0455322265625,
+      "logps/rejected": -208.0402374267578,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.010311352089047432,
+      "rewards/margins": 0.05862750858068466,
+      "rewards/rejected": -0.06893886625766754,
+      "step": 4170
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.555574072897374e-06,
+      "logits/chosen": -2.3054423332214355,
+      "logits/rejected": -2.3093464374542236,
+      "logps/chosen": -202.890625,
+      "logps/rejected": -206.8453826904297,
+      "loss": 0.689,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.009293651208281517,
+      "rewards/margins": 0.062263913452625275,
+      "rewards/rejected": -0.07155755162239075,
+      "step": 4180
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.552318885147589e-06,
+      "logits/chosen": -2.427234649658203,
+      "logits/rejected": -2.067701816558838,
+      "logps/chosen": -240.99063110351562,
+      "logps/rejected": -188.87326049804688,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.000729889259673655,
+      "rewards/margins": 0.07272221148014069,
+      "rewards/rejected": -0.07345209270715714,
+      "step": 4190
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.549052991497159e-06,
+      "logits/chosen": -2.283116102218628,
+      "logits/rejected": -2.254042387008667,
+      "logps/chosen": -181.29025268554688,
+      "logps/rejected": -188.34085083007812,
+      "loss": 0.691,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.011230905540287495,
+      "rewards/margins": 0.06631726771593094,
+      "rewards/rejected": -0.07754816114902496,
+      "step": 4200
+    },
+    {
+      "epoch": 0.27,
+      "eval_logits/chosen": -2.296299934387207,
+      "eval_logits/rejected": -2.1112263202667236,
+      "eval_logps/chosen": -232.25408935546875,
+      "eval_logps/rejected": -218.76133728027344,
+      "eval_loss": 0.6903825402259827,
+      "eval_rewards/accuracies": 0.6359999775886536,
+      "eval_rewards/chosen": -0.002491473685950041,
+      "eval_rewards/margins": 0.0690029114484787,
+      "eval_rewards/rejected": -0.07149438560009003,
+      "eval_runtime": 711.9807,
+      "eval_samples_per_second": 2.809,
+      "eval_steps_per_second": 1.405,
+      "step": 4200
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.545776408982585e-06,
+      "logits/chosen": -2.222346782684326,
+      "logits/rejected": -2.215831756591797,
+      "logps/chosen": -230.8057403564453,
+      "logps/rejected": -228.67105102539062,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.009383128955960274,
+      "rewards/margins": 0.0687854140996933,
+      "rewards/rejected": -0.059402287006378174,
+      "step": 4210
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.542489154696128e-06,
+      "logits/chosen": -2.435891628265381,
+      "logits/rejected": -2.0726494789123535,
+      "logps/chosen": -265.7242736816406,
+      "logps/rejected": -210.0156707763672,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.019960414618253708,
+      "rewards/margins": 0.05542059987783432,
+      "rewards/rejected": -0.035460181534290314,
+      "step": 4220
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.5391912457857145e-06,
+      "logits/chosen": -2.3141016960144043,
+      "logits/rejected": -2.055931568145752,
+      "logps/chosen": -264.88525390625,
+      "logps/rejected": -223.41726684570312,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0008437506621703506,
+      "rewards/margins": 0.05641711503267288,
+      "rewards/rejected": -0.05557336285710335,
+      "step": 4230
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.535882699454854e-06,
+      "logits/chosen": -2.3128061294555664,
+      "logits/rejected": -2.189279556274414,
+      "logps/chosen": -270.4936218261719,
+      "logps/rejected": -303.23992919921875,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.005573967471718788,
+      "rewards/margins": 0.11083599179983139,
+      "rewards/rejected": -0.10526201874017715,
+      "step": 4240
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.532563532962546e-06,
+      "logits/chosen": -2.368762493133545,
+      "logits/rejected": -2.450859546661377,
+      "logps/chosen": -191.6454315185547,
+      "logps/rejected": -218.98867797851562,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.017511827871203423,
+      "rewards/margins": 0.0574830062687397,
+      "rewards/rejected": -0.07499483227729797,
+      "step": 4250
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.529233763623187e-06,
+      "logits/chosen": -2.328399896621704,
+      "logits/rejected": -2.034263849258423,
+      "logps/chosen": -203.28858947753906,
+      "logps/rejected": -163.58592224121094,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.011880872771143913,
+      "rewards/margins": 0.08256997168064117,
+      "rewards/rejected": -0.09445084631443024,
+      "step": 4260
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.5258934088064854e-06,
+      "logits/chosen": -2.2317817211151123,
+      "logits/rejected": -1.83087158203125,
+      "logps/chosen": -223.72305297851562,
+      "logps/rejected": -181.90640258789062,
+      "loss": 0.6868,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.037422824651002884,
+      "rewards/margins": 0.12880873680114746,
+      "rewards/rejected": -0.16623155772686005,
+      "step": 4270
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.522542485937369e-06,
+      "logits/chosen": -2.3460514545440674,
+      "logits/rejected": -2.08577299118042,
+      "logps/chosen": -293.10992431640625,
+      "logps/rejected": -206.9127655029297,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.024574220180511475,
+      "rewards/margins": 0.1044369488954544,
+      "rewards/rejected": -0.12901116907596588,
+      "step": 4280
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.519181012495892e-06,
+      "logits/chosen": -2.3494815826416016,
+      "logits/rejected": -2.219589948654175,
+      "logps/chosen": -247.26657104492188,
+      "logps/rejected": -227.00888061523438,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.05047302693128586,
+      "rewards/margins": 0.08816438913345337,
+      "rewards/rejected": -0.13863742351531982,
+      "step": 4290
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.515809006017147e-06,
+      "logits/chosen": -2.274042844772339,
+      "logits/rejected": -1.9699468612670898,
+      "logps/chosen": -236.0398712158203,
+      "logps/rejected": -208.1054229736328,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.026302605867385864,
+      "rewards/margins": 0.07574830204248428,
+      "rewards/rejected": -0.10205090045928955,
+      "step": 4300
+    },
+    {
+      "epoch": 0.28,
+      "eval_logits/chosen": -2.286620855331421,
+      "eval_logits/rejected": -2.1015021800994873,
+      "eval_logps/chosen": -235.3810272216797,
+      "eval_logps/rejected": -223.5635223388672,
+      "eval_loss": 0.6901422142982483,
+      "eval_rewards/accuracies": 0.6345000267028809,
+      "eval_rewards/chosen": -0.03376083821058273,
+      "eval_rewards/margins": 0.08575531840324402,
+      "eval_rewards/rejected": -0.11951615661382675,
+      "eval_runtime": 711.2011,
+      "eval_samples_per_second": 2.812,
+      "eval_steps_per_second": 1.406,
+      "step": 4300
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.512426484091171e-06,
+      "logits/chosen": -2.418959379196167,
+      "logits/rejected": -2.085226058959961,
+      "logps/chosen": -279.13177490234375,
+      "logps/rejected": -249.38064575195312,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.015505967661738396,
+      "rewards/margins": 0.06541910022497177,
+      "rewards/rejected": -0.08092506229877472,
+      "step": 4310
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.509033464362858e-06,
+      "logits/chosen": -2.122525215148926,
+      "logits/rejected": -2.1860134601593018,
+      "logps/chosen": -243.3289794921875,
+      "logps/rejected": -266.288330078125,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.020630965009331703,
+      "rewards/margins": 0.08728428930044174,
+      "rewards/rejected": -0.10791525989770889,
+      "step": 4320
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.505629964531857e-06,
+      "logits/chosen": -2.3952324390411377,
+      "logits/rejected": -2.192960500717163,
+      "logps/chosen": -226.1346435546875,
+      "logps/rejected": -203.77609252929688,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.029302721843123436,
+      "rewards/margins": 0.08454638719558716,
+      "rewards/rejected": -0.11384911835193634,
+      "step": 4330
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.502216002352492e-06,
+      "logits/chosen": -2.3942387104034424,
+      "logits/rejected": -2.1669986248016357,
+      "logps/chosen": -167.1390380859375,
+      "logps/rejected": -154.9490203857422,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.04098113626241684,
+      "rewards/margins": 0.07224190980195999,
+      "rewards/rejected": -0.11322303861379623,
+      "step": 4340
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.498791595633663e-06,
+      "logits/chosen": -2.227745771408081,
+      "logits/rejected": -1.8462340831756592,
+      "logps/chosen": -265.2237854003906,
+      "logps/rejected": -183.4816131591797,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.02839501202106476,
+      "rewards/margins": 0.054582733660936356,
+      "rewards/rejected": -0.08297775685787201,
+      "step": 4350
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.495356762238751e-06,
+      "logits/chosen": -2.472080707550049,
+      "logits/rejected": -1.9949004650115967,
+      "logps/chosen": -284.7836608886719,
+      "logps/rejected": -195.12869262695312,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.015857676044106483,
+      "rewards/margins": 0.07797005027532578,
+      "rewards/rejected": -0.09382772445678711,
+      "step": 4360
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.491911520085532e-06,
+      "logits/chosen": -2.046393394470215,
+      "logits/rejected": -1.9404007196426392,
+      "logps/chosen": -202.2923126220703,
+      "logps/rejected": -221.1287078857422,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.018845614045858383,
+      "rewards/margins": 0.08063776046037674,
+      "rewards/rejected": -0.09948337823152542,
+      "step": 4370
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.488455887146075e-06,
+      "logits/chosen": -2.159259080886841,
+      "logits/rejected": -2.141447067260742,
+      "logps/chosen": -172.12095642089844,
+      "logps/rejected": -198.18894958496094,
+      "loss": 0.6879,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.020088694989681244,
+      "rewards/margins": 0.12904280424118042,
+      "rewards/rejected": -0.14913150668144226,
+      "step": 4380
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.484989881446654e-06,
+      "logits/chosen": -2.4215025901794434,
+      "logits/rejected": -2.220041275024414,
+      "logps/chosen": -204.8853302001953,
+      "logps/rejected": -191.7876434326172,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.024093201383948326,
+      "rewards/margins": 0.0484703965485096,
+      "rewards/rejected": -0.07256358861923218,
+      "step": 4390
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.481513521067654e-06,
+      "logits/chosen": -2.3942711353302,
+      "logits/rejected": -2.039447546005249,
+      "logps/chosen": -228.5469512939453,
+      "logps/rejected": -200.28292846679688,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.03772415220737457,
+      "rewards/margins": 0.07151724398136139,
+      "rewards/rejected": -0.10924138873815536,
+      "step": 4400
+    },
+    {
+      "epoch": 0.29,
+      "eval_logits/chosen": -2.292935609817505,
+      "eval_logits/rejected": -2.107675790786743,
+      "eval_logps/chosen": -236.54518127441406,
+      "eval_logps/rejected": -223.5493927001953,
+      "eval_loss": 0.6902163028717041,
+      "eval_rewards/accuracies": 0.6274999976158142,
+      "eval_rewards/chosen": -0.04540235176682472,
+      "eval_rewards/margins": 0.07397259771823883,
+      "eval_rewards/rejected": -0.11937494575977325,
+      "eval_runtime": 713.6438,
+      "eval_samples_per_second": 2.803,
+      "eval_steps_per_second": 1.401,
+      "step": 4400
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.478026824143473e-06,
+      "logits/chosen": -2.3092598915100098,
+      "logits/rejected": -2.173832416534424,
+      "logps/chosen": -270.6745910644531,
+      "logps/rejected": -224.7923583984375,
+      "loss": 0.6862,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.046129751950502396,
+      "rewards/margins": 0.10685235261917114,
+      "rewards/rejected": -0.15298210084438324,
+      "step": 4410
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.474529808862429e-06,
+      "logits/chosen": -2.197213649749756,
+      "logits/rejected": -2.124817371368408,
+      "logps/chosen": -193.43679809570312,
+      "logps/rejected": -218.94619750976562,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.04616478085517883,
+      "rewards/margins": 0.07885146141052246,
+      "rewards/rejected": -0.1250162422657013,
+      "step": 4420
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.471022493466669e-06,
+      "logits/chosen": -2.3107991218566895,
+      "logits/rejected": -1.9796260595321655,
+      "logps/chosen": -303.61737060546875,
+      "logps/rejected": -232.7263641357422,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.016782710328698158,
+      "rewards/margins": 0.06120295077562332,
+      "rewards/rejected": -0.07798566669225693,
+      "step": 4430
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.467504896252066e-06,
+      "logits/chosen": -2.335106372833252,
+      "logits/rejected": -2.22440767288208,
+      "logps/chosen": -252.58322143554688,
+      "logps/rejected": -233.07406616210938,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.028819028288125992,
+      "rewards/margins": 0.09233070909976959,
+      "rewards/rejected": -0.12114973366260529,
+      "step": 4440
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.463977035568132e-06,
+      "logits/chosen": -2.1951041221618652,
+      "logits/rejected": -2.431762933731079,
+      "logps/chosen": -214.28427124023438,
+      "logps/rejected": -269.91864013671875,
+      "loss": 0.691,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.019015159457921982,
+      "rewards/margins": 0.039287667721509933,
+      "rewards/rejected": -0.058302827179431915,
+      "step": 4450
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.460438929817914e-06,
+      "logits/chosen": -2.264540672302246,
+      "logits/rejected": -2.11928129196167,
+      "logps/chosen": -207.63388061523438,
+      "logps/rejected": -209.83316040039062,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.012141216546297073,
+      "rewards/margins": 0.06487870216369629,
+      "rewards/rejected": -0.07701991498470306,
+      "step": 4460
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.456890597457907e-06,
+      "logits/chosen": -2.112905979156494,
+      "logits/rejected": -2.159135341644287,
+      "logps/chosen": -216.1064453125,
+      "logps/rejected": -243.3882293701172,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.03970780223608017,
+      "rewards/margins": 0.09077353030443192,
+      "rewards/rejected": -0.13048133254051208,
+      "step": 4470
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.453332056997951e-06,
+      "logits/chosen": -2.2369141578674316,
+      "logits/rejected": -2.2910995483398438,
+      "logps/chosen": -181.7244110107422,
+      "logps/rejected": -187.7960662841797,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.01764925941824913,
+      "rewards/margins": 0.10396716743707657,
+      "rewards/rejected": -0.1216164231300354,
+      "step": 4480
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.449763327001134e-06,
+      "logits/chosen": -2.2684309482574463,
+      "logits/rejected": -2.174893617630005,
+      "logps/chosen": -190.84078979492188,
+      "logps/rejected": -226.34326171875,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.020785531029105186,
+      "rewards/margins": 0.07087056338787079,
+      "rewards/rejected": -0.09165609627962112,
+      "step": 4490
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.446184426083702e-06,
+      "logits/chosen": -2.249093532562256,
+      "logits/rejected": -2.0277256965637207,
+      "logps/chosen": -195.96009826660156,
+      "logps/rejected": -216.97604370117188,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.023957300931215286,
+      "rewards/margins": 0.12648364901542664,
+      "rewards/rejected": -0.15044096112251282,
+      "step": 4500
+    },
+    {
+      "epoch": 0.29,
+      "eval_logits/chosen": -2.307412624359131,
+      "eval_logits/rejected": -2.1211140155792236,
+      "eval_logps/chosen": -234.31179809570312,
+      "eval_logps/rejected": -222.2449493408203,
+      "eval_loss": 0.6901082396507263,
+      "eval_rewards/accuracies": 0.6324999928474426,
+      "eval_rewards/chosen": -0.02306850627064705,
+      "eval_rewards/margins": 0.0832618772983551,
+      "eval_rewards/rejected": -0.10633040219545364,
+      "eval_runtime": 714.5639,
+      "eval_samples_per_second": 2.799,
+      "eval_steps_per_second": 1.399,
+      "step": 4500
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.442595372914954e-06,
+      "logits/chosen": -2.3577396869659424,
+      "logits/rejected": -2.0909037590026855,
+      "logps/chosen": -236.86007690429688,
+      "logps/rejected": -160.77267456054688,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.0012024863390251994,
+      "rewards/margins": 0.09186828881502151,
+      "rewards/rejected": -0.090665802359581,
+      "step": 4510
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.43899618621715e-06,
+      "logits/chosen": -2.303638458251953,
+      "logits/rejected": -2.097632884979248,
+      "logps/chosen": -254.14059448242188,
+      "logps/rejected": -269.0081481933594,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.030867312103509903,
+      "rewards/margins": 0.1167084202170372,
+      "rewards/rejected": -0.1475757360458374,
+      "step": 4520
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.4353868847654105e-06,
+      "logits/chosen": -2.4185733795166016,
+      "logits/rejected": -2.1811881065368652,
+      "logps/chosen": -244.586181640625,
+      "logps/rejected": -222.5398406982422,
+      "loss": 0.6862,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.012720689177513123,
+      "rewards/margins": 0.08021236956119537,
+      "rewards/rejected": -0.06749166548252106,
+      "step": 4530
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.43176748738762e-06,
+      "logits/chosen": -2.3359453678131104,
+      "logits/rejected": -2.09609055519104,
+      "logps/chosen": -233.65359497070312,
+      "logps/rejected": -248.21719360351562,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.01705637015402317,
+      "rewards/margins": 0.10175220668315887,
+      "rewards/rejected": -0.1188085675239563,
+      "step": 4540
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.4281380129643295e-06,
+      "logits/chosen": -2.2307958602905273,
+      "logits/rejected": -2.0683705806732178,
+      "logps/chosen": -229.425537109375,
+      "logps/rejected": -228.88040161132812,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0032461367081850767,
+      "rewards/margins": 0.0992891788482666,
+      "rewards/rejected": -0.09604303538799286,
+      "step": 4550
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.424498480428654e-06,
+      "logits/chosen": -2.258957862854004,
+      "logits/rejected": -2.15374755859375,
+      "logps/chosen": -249.0747528076172,
+      "logps/rejected": -211.0282745361328,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.018028225749731064,
+      "rewards/margins": 0.01978963240981102,
+      "rewards/rejected": -0.037817858159542084,
+      "step": 4560
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.420848908766178e-06,
+      "logits/chosen": -2.3698325157165527,
+      "logits/rejected": -2.2980637550354004,
+      "logps/chosen": -206.7262420654297,
+      "logps/rejected": -220.7576141357422,
+      "loss": 0.69,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.005006049759685993,
+      "rewards/margins": 0.05151135474443436,
+      "rewards/rejected": -0.04650530219078064,
+      "step": 4570
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.417189317014855e-06,
+      "logits/chosen": -2.2065937519073486,
+      "logits/rejected": -2.445247173309326,
+      "logps/chosen": -199.8828582763672,
+      "logps/rejected": -236.3675994873047,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.009130073711276054,
+      "rewards/margins": 0.057878412306308746,
+      "rewards/rejected": -0.04874832555651665,
+      "step": 4580
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.41351972426491e-06,
+      "logits/chosen": -2.129570484161377,
+      "logits/rejected": -2.161388874053955,
+      "logps/chosen": -248.59689331054688,
+      "logps/rejected": -309.24725341796875,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.018575403839349747,
+      "rewards/margins": 0.06597335636615753,
+      "rewards/rejected": -0.08454876393079758,
+      "step": 4590
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.409840149658735e-06,
+      "logits/chosen": -2.2294223308563232,
+      "logits/rejected": -1.970663070678711,
+      "logps/chosen": -284.4920654296875,
+      "logps/rejected": -242.57687377929688,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0002518877445254475,
+      "rewards/margins": 0.08236613124608994,
+      "rewards/rejected": -0.08211424201726913,
+      "step": 4600
+    },
+    {
+      "epoch": 0.3,
+      "eval_logits/chosen": -2.3078274726867676,
+      "eval_logits/rejected": -2.121540069580078,
+      "eval_logps/chosen": -231.3809051513672,
+      "eval_logps/rejected": -218.01165771484375,
+      "eval_loss": 0.6902089715003967,
+      "eval_rewards/accuracies": 0.6309999823570251,
+      "eval_rewards/chosen": 0.0062404475174844265,
+      "eval_rewards/margins": 0.07023809105157852,
+      "eval_rewards/rejected": -0.06399764865636826,
+      "eval_runtime": 711.6306,
+      "eval_samples_per_second": 2.81,
+      "eval_steps_per_second": 1.405,
+      "step": 4600
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.4061506123907925e-06,
+      "logits/chosen": -2.226529598236084,
+      "logits/rejected": -2.063323497772217,
+      "logps/chosen": -263.83251953125,
+      "logps/rejected": -228.8885040283203,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0025929573457688093,
+      "rewards/margins": 0.05362165719270706,
+      "rewards/rejected": -0.051028698682785034,
+      "step": 4610
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.402451131707519e-06,
+      "logits/chosen": -2.4300453662872314,
+      "logits/rejected": -1.9670915603637695,
+      "logps/chosen": -208.1881866455078,
+      "logps/rejected": -143.37477111816406,
+      "loss": 0.689,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.012772110290825367,
+      "rewards/margins": 0.1066797599196434,
+      "rewards/rejected": -0.09390763938426971,
+      "step": 4620
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.398741726907215e-06,
+      "logits/chosen": -2.4595742225646973,
+      "logits/rejected": -2.141775608062744,
+      "logps/chosen": -277.5190734863281,
+      "logps/rejected": -244.45352172851562,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.017383281141519547,
+      "rewards/margins": 0.07261139899492264,
+      "rewards/rejected": -0.05522811412811279,
+      "step": 4630
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.395022417339955e-06,
+      "logits/chosen": -2.2033920288085938,
+      "logits/rejected": -2.2470412254333496,
+      "logps/chosen": -208.9228973388672,
+      "logps/rejected": -223.9864044189453,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.022978752851486206,
+      "rewards/margins": 0.0657435953617096,
+      "rewards/rejected": -0.0887223556637764,
+      "step": 4640
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.391293222407479e-06,
+      "logits/chosen": -2.30222487449646,
+      "logits/rejected": -2.303806781768799,
+      "logps/chosen": -136.2609405517578,
+      "logps/rejected": -160.14111328125,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.0029621024150401354,
+      "rewards/margins": 0.05650148540735245,
+      "rewards/rejected": -0.05353938415646553,
+      "step": 4650
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.387554161563094e-06,
+      "logits/chosen": -2.3135313987731934,
+      "logits/rejected": -2.2245595455169678,
+      "logps/chosen": -200.64547729492188,
+      "logps/rejected": -199.0697479248047,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.017812874168157578,
+      "rewards/margins": 0.10243946313858032,
+      "rewards/rejected": -0.1202523335814476,
+      "step": 4660
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.383805254311575e-06,
+      "logits/chosen": -2.509479522705078,
+      "logits/rejected": -2.135781764984131,
+      "logps/chosen": -257.2196044921875,
+      "logps/rejected": -218.57418823242188,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.010785650461912155,
+      "rewards/margins": 0.06920838356018066,
+      "rewards/rejected": -0.07999403774738312,
+      "step": 4670
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.380046520209056e-06,
+      "logits/chosen": -2.3661510944366455,
+      "logits/rejected": -1.9948110580444336,
+      "logps/chosen": -202.7303924560547,
+      "logps/rejected": -186.60891723632812,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.024640483781695366,
+      "rewards/margins": 0.07717674970626831,
+      "rewards/rejected": -0.10181725025177002,
+      "step": 4680
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.376277978862936e-06,
+      "logits/chosen": -2.2196907997131348,
+      "logits/rejected": -1.9270433187484741,
+      "logps/chosen": -227.43930053710938,
+      "logps/rejected": -193.55284118652344,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.014506752602756023,
+      "rewards/margins": 0.06645031273365021,
+      "rewards/rejected": -0.0809570699930191,
+      "step": 4690
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.372499649931774e-06,
+      "logits/chosen": -2.1691818237304688,
+      "logits/rejected": -2.317289113998413,
+      "logps/chosen": -212.4759063720703,
+      "logps/rejected": -234.37451171875,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.0338435061275959,
+      "rewards/margins": 0.13942193984985352,
+      "rewards/rejected": -0.1732654571533203,
+      "step": 4700
+    },
+    {
+      "epoch": 0.31,
+      "eval_logits/chosen": -2.31925368309021,
+      "eval_logits/rejected": -2.1311213970184326,
+      "eval_logps/chosen": -235.55807495117188,
+      "eval_logps/rejected": -224.37208557128906,
+      "eval_loss": 0.6902915835380554,
+      "eval_rewards/accuracies": 0.6355000138282776,
+      "eval_rewards/chosen": -0.03553127497434616,
+      "eval_rewards/margins": 0.09207045286893845,
+      "eval_rewards/rejected": -0.1276017278432846,
+      "eval_runtime": 714.2184,
+      "eval_samples_per_second": 2.8,
+      "eval_steps_per_second": 1.4,
+      "step": 4700
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.368711553125185e-06,
+      "logits/chosen": -2.5005226135253906,
+      "logits/rejected": -2.265688180923462,
+      "logps/chosen": -281.1730041503906,
+      "logps/rejected": -229.6286163330078,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0398295521736145,
+      "rewards/margins": 0.06899070739746094,
+      "rewards/rejected": -0.10882025957107544,
+      "step": 4710
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.364913708203734e-06,
+      "logits/chosen": -2.398655891418457,
+      "logits/rejected": -2.0503017902374268,
+      "logps/chosen": -289.25067138671875,
+      "logps/rejected": -222.9422149658203,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.04354417696595192,
+      "rewards/margins": 0.09108763188123703,
+      "rewards/rejected": -0.13463182747364044,
+      "step": 4720
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.361106134978844e-06,
+      "logits/chosen": -2.277704954147339,
+      "logits/rejected": -2.071712017059326,
+      "logps/chosen": -272.6820068359375,
+      "logps/rejected": -265.93670654296875,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.022515593096613884,
+      "rewards/margins": 0.06205441802740097,
+      "rewards/rejected": -0.08457001298666,
+      "step": 4730
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.357288853312681e-06,
+      "logits/chosen": -2.3490684032440186,
+      "logits/rejected": -2.26945161819458,
+      "logps/chosen": -287.81549072265625,
+      "logps/rejected": -287.1567687988281,
+      "loss": 0.692,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.04337679222226143,
+      "rewards/margins": 0.04078169912099838,
+      "rewards/rejected": -0.08415848016738892,
+      "step": 4740
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.353461883118056e-06,
+      "logits/chosen": -2.249939203262329,
+      "logits/rejected": -2.112075090408325,
+      "logps/chosen": -232.1291961669922,
+      "logps/rejected": -214.3981475830078,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.02838616445660591,
+      "rewards/margins": 0.03420080989599228,
+      "rewards/rejected": -0.06258697807788849,
+      "step": 4750
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.34962524435832e-06,
+      "logits/chosen": -2.1387410163879395,
+      "logits/rejected": -2.0375964641571045,
+      "logps/chosen": -221.3422393798828,
+      "logps/rejected": -193.26048278808594,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.016618115827441216,
+      "rewards/margins": 0.08751632273197174,
+      "rewards/rejected": -0.10413442552089691,
+      "step": 4760
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.34577895704726e-06,
+      "logits/chosen": -2.372318744659424,
+      "logits/rejected": -2.1986355781555176,
+      "logps/chosen": -263.10650634765625,
+      "logps/rejected": -244.01565551757812,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.01492463331669569,
+      "rewards/margins": 0.06633206456899643,
+      "rewards/rejected": -0.08125670254230499,
+      "step": 4770
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.3419230412489954e-06,
+      "logits/chosen": -2.470191478729248,
+      "logits/rejected": -2.233651638031006,
+      "logps/chosen": -291.9186096191406,
+      "logps/rejected": -221.01748657226562,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.0255076102912426,
+      "rewards/margins": 0.051627278327941895,
+      "rewards/rejected": -0.07713489234447479,
+      "step": 4780
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.338057517077872e-06,
+      "logits/chosen": -2.417341709136963,
+      "logits/rejected": -2.014641284942627,
+      "logps/chosen": -193.8696746826172,
+      "logps/rejected": -168.1341094970703,
+      "loss": 0.6813,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.0017154158558696508,
+      "rewards/margins": 0.17616704106330872,
+      "rewards/rejected": -0.17445163428783417,
+      "step": 4790
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.334182404698356e-06,
+      "logits/chosen": -2.4133429527282715,
+      "logits/rejected": -1.977574110031128,
+      "logps/chosen": -234.8191680908203,
+      "logps/rejected": -160.61514282226562,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.04712152108550072,
+      "rewards/margins": 0.04195799678564072,
+      "rewards/rejected": -0.08907952159643173,
+      "step": 4800
+    },
+    {
+      "epoch": 0.31,
+      "eval_logits/chosen": -2.3064463138580322,
+      "eval_logits/rejected": -2.1199657917022705,
+      "eval_logps/chosen": -233.79531860351562,
+      "eval_logps/rejected": -220.7675323486328,
+      "eval_loss": 0.6901711225509644,
+      "eval_rewards/accuracies": 0.6384999752044678,
+      "eval_rewards/chosen": -0.017903409898281097,
+      "eval_rewards/margins": 0.07365269213914871,
+      "eval_rewards/rejected": -0.09155610203742981,
+      "eval_runtime": 711.7896,
+      "eval_samples_per_second": 2.81,
+      "eval_steps_per_second": 1.405,
+      "step": 4800
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.330297724324933e-06,
+      "logits/chosen": -2.5674805641174316,
+      "logits/rejected": -2.029761791229248,
+      "logps/chosen": -308.3761291503906,
+      "logps/rejected": -214.9816436767578,
+      "loss": 0.689,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.0003527544322423637,
+      "rewards/margins": 0.07805721461772919,
+      "rewards/rejected": -0.07770445942878723,
+      "step": 4810
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.326403496221999e-06,
+      "logits/chosen": -2.235084056854248,
+      "logits/rejected": -2.138692617416382,
+      "logps/chosen": -163.4996337890625,
+      "logps/rejected": -143.01316833496094,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.020706117153167725,
+      "rewards/margins": 0.06293468922376633,
+      "rewards/rejected": -0.08364080637693405,
+      "step": 4820
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.322499740703755e-06,
+      "logits/chosen": -2.19960618019104,
+      "logits/rejected": -2.30405592918396,
+      "logps/chosen": -193.19754028320312,
+      "logps/rejected": -224.2078094482422,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.010983394458889961,
+      "rewards/margins": 0.06486980617046356,
+      "rewards/rejected": -0.07585321366786957,
+      "step": 4830
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.318586478134101e-06,
+      "logits/chosen": -2.2257297039031982,
+      "logits/rejected": -2.188767671585083,
+      "logps/chosen": -192.26211547851562,
+      "logps/rejected": -158.5824737548828,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.004053809680044651,
+      "rewards/margins": 0.06840833276510239,
+      "rewards/rejected": -0.06435452401638031,
+      "step": 4840
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.314663728926534e-06,
+      "logits/chosen": -2.4708011150360107,
+      "logits/rejected": -2.215599536895752,
+      "logps/chosen": -259.12109375,
+      "logps/rejected": -254.83389282226562,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.022453729063272476,
+      "rewards/margins": 0.0629110336303711,
+      "rewards/rejected": -0.08536475896835327,
+      "step": 4850
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.310731513544033e-06,
+      "logits/chosen": -2.26763653755188,
+      "logits/rejected": -2.0921549797058105,
+      "logps/chosen": -245.0206756591797,
+      "logps/rejected": -206.82406616210938,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.021629411727190018,
+      "rewards/margins": 0.07882945239543915,
+      "rewards/rejected": -0.10045886039733887,
+      "step": 4860
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.30678985249896e-06,
+      "logits/chosen": -2.255072832107544,
+      "logits/rejected": -2.1915435791015625,
+      "logps/chosen": -159.80569458007812,
+      "logps/rejected": -188.56588745117188,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.026556584984064102,
+      "rewards/margins": 0.10496687889099121,
+      "rewards/rejected": -0.13152346014976501,
+      "step": 4870
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.302838766352952e-06,
+      "logits/chosen": -2.2494466304779053,
+      "logits/rejected": -2.029052495956421,
+      "logps/chosen": -259.2070007324219,
+      "logps/rejected": -231.5312042236328,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.03614037483930588,
+      "rewards/margins": 0.08115691691637039,
+      "rewards/rejected": -0.11729729175567627,
+      "step": 4880
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.298878275716806e-06,
+      "logits/chosen": -2.175429582595825,
+      "logits/rejected": -2.159177780151367,
+      "logps/chosen": -196.0753936767578,
+      "logps/rejected": -203.73773193359375,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.04195953160524368,
+      "rewards/margins": 0.10459339618682861,
+      "rewards/rejected": -0.1465529352426529,
+      "step": 4890
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.294908401250386e-06,
+      "logits/chosen": -2.3999128341674805,
+      "logits/rejected": -1.9628560543060303,
+      "logps/chosen": -217.9789276123047,
+      "logps/rejected": -182.8585968017578,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.032358746975660324,
+      "rewards/margins": 0.0941200703382492,
+      "rewards/rejected": -0.12647880613803864,
+      "step": 4900
+    },
+    {
+      "epoch": 0.32,
+      "eval_logits/chosen": -2.284270763397217,
+      "eval_logits/rejected": -2.0991475582122803,
+      "eval_logps/chosen": -234.08592224121094,
+      "eval_logps/rejected": -222.58131408691406,
+      "eval_loss": 0.6902449131011963,
+      "eval_rewards/accuracies": 0.6424999833106995,
+      "eval_rewards/chosen": -0.020809680223464966,
+      "eval_rewards/margins": 0.08888448029756546,
+      "eval_rewards/rejected": -0.10969416052103043,
+      "eval_runtime": 710.5969,
+      "eval_samples_per_second": 2.815,
+      "eval_steps_per_second": 1.407,
+      "step": 4900
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.290929163662498e-06,
+      "logits/chosen": -2.1393580436706543,
+      "logits/rejected": -1.9254634380340576,
+      "logps/chosen": -271.26580810546875,
+      "logps/rejected": -221.15762329101562,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.015305581502616405,
+      "rewards/margins": 0.076082743704319,
+      "rewards/rejected": -0.09138831496238708,
+      "step": 4910
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.286940583710796e-06,
+      "logits/chosen": -2.330704689025879,
+      "logits/rejected": -2.207934856414795,
+      "logps/chosen": -294.7912292480469,
+      "logps/rejected": -248.0767364501953,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.020425736904144287,
+      "rewards/margins": 0.09915411472320557,
+      "rewards/rejected": -0.11957985162734985,
+      "step": 4920
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.282942682201667e-06,
+      "logits/chosen": -2.2037787437438965,
+      "logits/rejected": -1.929842233657837,
+      "logps/chosen": -255.17098999023438,
+      "logps/rejected": -224.1762237548828,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.0306318998336792,
+      "rewards/margins": 0.08591778576374054,
+      "rewards/rejected": -0.11654969304800034,
+      "step": 4930
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.278935479990123e-06,
+      "logits/chosen": -2.479978322982788,
+      "logits/rejected": -2.227200508117676,
+      "logps/chosen": -207.30966186523438,
+      "logps/rejected": -171.01812744140625,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.02646293118596077,
+      "rewards/margins": 0.06679949164390564,
+      "rewards/rejected": -0.09326241165399551,
+      "step": 4940
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.274918997979695e-06,
+      "logits/chosen": -2.2375760078430176,
+      "logits/rejected": -2.2690727710723877,
+      "logps/chosen": -197.79953002929688,
+      "logps/rejected": -205.44857788085938,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0416448749601841,
+      "rewards/margins": 0.05913674831390381,
+      "rewards/rejected": -0.10078161954879761,
+      "step": 4950
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.270893257122319e-06,
+      "logits/chosen": -2.1356120109558105,
+      "logits/rejected": -1.9105371236801147,
+      "logps/chosen": -230.14260864257812,
+      "logps/rejected": -272.1700744628906,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.014206337742507458,
+      "rewards/margins": 0.12518110871315002,
+      "rewards/rejected": -0.13938744366168976,
+      "step": 4960
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.266858278418232e-06,
+      "logits/chosen": -2.1323647499084473,
+      "logits/rejected": -1.8933664560317993,
+      "logps/chosen": -238.7414093017578,
+      "logps/rejected": -223.8094940185547,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.021372055634856224,
+      "rewards/margins": 0.05419282987713814,
+      "rewards/rejected": -0.07556488364934921,
+      "step": 4970
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.26281408291586e-06,
+      "logits/chosen": -2.3737692832946777,
+      "logits/rejected": -2.0658111572265625,
+      "logps/chosen": -242.1832275390625,
+      "logps/rejected": -223.305419921875,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.0015409494517371058,
+      "rewards/margins": 0.10654549300670624,
+      "rewards/rejected": -0.10500454902648926,
+      "step": 4980
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.258760691711706e-06,
+      "logits/chosen": -2.291581392288208,
+      "logits/rejected": -2.1656148433685303,
+      "logps/chosen": -198.389404296875,
+      "logps/rejected": -202.20599365234375,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.009278281591832638,
+      "rewards/margins": 0.08409412950277328,
+      "rewards/rejected": -0.0933724194765091,
+      "step": 4990
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.254698125950247e-06,
+      "logits/chosen": -2.530463695526123,
+      "logits/rejected": -2.2810826301574707,
+      "logps/chosen": -300.74664306640625,
+      "logps/rejected": -259.6216735839844,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0026790399570018053,
+      "rewards/margins": 0.05540703609585762,
+      "rewards/rejected": -0.05272800475358963,
+      "step": 5000
+    },
+    {
+      "epoch": 0.33,
+      "eval_logits/chosen": -2.2863876819610596,
+      "eval_logits/rejected": -2.1009578704833984,
+      "eval_logps/chosen": -232.66943359375,
+      "eval_logps/rejected": -220.42218017578125,
+      "eval_loss": 0.6900946497917175,
+      "eval_rewards/accuracies": 0.6269999742507935,
+      "eval_rewards/chosen": -0.006644845940172672,
+      "eval_rewards/margins": 0.08145791292190552,
+      "eval_rewards/rejected": -0.08810276538133621,
+      "eval_runtime": 708.8089,
+      "eval_samples_per_second": 2.822,
+      "eval_steps_per_second": 1.411,
+      "step": 5000
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.250626406823815e-06,
+      "logits/chosen": -2.3481221199035645,
+      "logits/rejected": -2.086394786834717,
+      "logps/chosen": -217.6735076904297,
+      "logps/rejected": -247.91268920898438,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.008107764646410942,
+      "rewards/margins": 0.14061132073402405,
+      "rewards/rejected": -0.14871908724308014,
+      "step": 5010
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.246545555572489e-06,
+      "logits/chosen": -2.260010242462158,
+      "logits/rejected": -2.139444351196289,
+      "logps/chosen": -153.61056518554688,
+      "logps/rejected": -189.370849609375,
+      "loss": 0.688,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.005855654366314411,
+      "rewards/margins": 0.10808217525482178,
+      "rewards/rejected": -0.10222651809453964,
+      "step": 5020
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.242455593483992e-06,
+      "logits/chosen": -2.340317964553833,
+      "logits/rejected": -2.138221263885498,
+      "logps/chosen": -218.4593048095703,
+      "logps/rejected": -174.04708862304688,
+      "loss": 0.692,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.006946141831576824,
+      "rewards/margins": 0.052972644567489624,
+      "rewards/rejected": -0.04602649435400963,
+      "step": 5030
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.238356541893567e-06,
+      "logits/chosen": -2.2381134033203125,
+      "logits/rejected": -2.103583812713623,
+      "logps/chosen": -190.62313842773438,
+      "logps/rejected": -182.2062225341797,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.0049001253210008144,
+      "rewards/margins": 0.07125677168369293,
+      "rewards/rejected": -0.07615689188241959,
+      "step": 5040
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.234248422183876e-06,
+      "logits/chosen": -2.0986828804016113,
+      "logits/rejected": -2.2984180450439453,
+      "logps/chosen": -241.3805694580078,
+      "logps/rejected": -242.21878051757812,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.01554117538034916,
+      "rewards/margins": 0.0539456307888031,
+      "rewards/rejected": -0.03840445727109909,
+      "step": 5050
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.230131255784884e-06,
+      "logits/chosen": -2.5071334838867188,
+      "logits/rejected": -2.1970627307891846,
+      "logps/chosen": -253.6757354736328,
+      "logps/rejected": -244.3105926513672,
+      "loss": 0.688,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.04207398369908333,
+      "rewards/margins": 0.09431316703557968,
+      "rewards/rejected": -0.05223918706178665,
+      "step": 5060
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.226005064173748e-06,
+      "logits/chosen": -2.299450159072876,
+      "logits/rejected": -2.1493353843688965,
+      "logps/chosen": -261.6854553222656,
+      "logps/rejected": -286.0189208984375,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.029588108882308006,
+      "rewards/margins": 0.058764077723026276,
+      "rewards/rejected": -0.02917597070336342,
+      "step": 5070
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2218698688747035e-06,
+      "logits/chosen": -2.133448362350464,
+      "logits/rejected": -1.9765691757202148,
+      "logps/chosen": -233.8523712158203,
+      "logps/rejected": -196.45217895507812,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.01143039483577013,
+      "rewards/margins": 0.07730044424533844,
+      "rewards/rejected": -0.0887308269739151,
+      "step": 5080
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.217725691458957e-06,
+      "logits/chosen": -2.4555492401123047,
+      "logits/rejected": -2.2806808948516846,
+      "logps/chosen": -183.96484375,
+      "logps/rejected": -221.15011596679688,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0025927810929715633,
+      "rewards/margins": 0.10239323228597641,
+      "rewards/rejected": -0.09980045258998871,
+      "step": 5090
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.213572553544565e-06,
+      "logits/chosen": -2.343311071395874,
+      "logits/rejected": -2.1145055294036865,
+      "logps/chosen": -239.4435577392578,
+      "logps/rejected": -242.26553344726562,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.015282916836440563,
+      "rewards/margins": 0.10186527669429779,
+      "rewards/rejected": -0.08658237755298615,
+      "step": 5100
+    },
+    {
+      "epoch": 0.33,
+      "eval_logits/chosen": -2.304870843887329,
+      "eval_logits/rejected": -2.1186633110046387,
+      "eval_logps/chosen": -232.49884033203125,
+      "eval_logps/rejected": -220.59129333496094,
+      "eval_loss": 0.6902390122413635,
+      "eval_rewards/accuracies": 0.6365000009536743,
+      "eval_rewards/chosen": -0.004938941914588213,
+      "eval_rewards/margins": 0.08485515415668488,
+      "eval_rewards/rejected": -0.0897940918803215,
+      "eval_runtime": 709.3713,
+      "eval_samples_per_second": 2.819,
+      "eval_steps_per_second": 1.41,
+      "step": 5100
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.209410476796331e-06,
+      "logits/chosen": -2.2306551933288574,
+      "logits/rejected": -2.1737866401672363,
+      "logps/chosen": -172.79660034179688,
+      "logps/rejected": -172.89422607421875,
+      "loss": 0.6879,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.017051298171281815,
+      "rewards/margins": 0.10086864233016968,
+      "rewards/rejected": -0.11791994422674179,
+      "step": 5110
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.205239482925686e-06,
+      "logits/chosen": -2.1063780784606934,
+      "logits/rejected": -2.153102397918701,
+      "logps/chosen": -188.4702911376953,
+      "logps/rejected": -215.7820587158203,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.006549468729645014,
+      "rewards/margins": 0.05185345560312271,
+      "rewards/rejected": -0.05840292572975159,
+      "step": 5120
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.201059593690577e-06,
+      "logits/chosen": -2.3719985485076904,
+      "logits/rejected": -2.2848258018493652,
+      "logps/chosen": -225.9248046875,
+      "logps/rejected": -207.27273559570312,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.0033702123910188675,
+      "rewards/margins": 0.06776970624923706,
+      "rewards/rejected": -0.07113991677761078,
+      "step": 5130
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.196870830895354e-06,
+      "logits/chosen": -2.1750078201293945,
+      "logits/rejected": -2.1885952949523926,
+      "logps/chosen": -259.3610534667969,
+      "logps/rejected": -315.05426025390625,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.0161347184330225,
+      "rewards/margins": 0.06185835599899292,
+      "rewards/rejected": -0.07799308001995087,
+      "step": 5140
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.192673216390657e-06,
+      "logits/chosen": -2.3547465801239014,
+      "logits/rejected": -2.0885729789733887,
+      "logps/chosen": -235.29983520507812,
+      "logps/rejected": -204.89700317382812,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.02326892875134945,
+      "rewards/margins": 0.07924740761518478,
+      "rewards/rejected": -0.10251633077859879,
+      "step": 5150
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.188466772073296e-06,
+      "logits/chosen": -2.447526454925537,
+      "logits/rejected": -2.1424593925476074,
+      "logps/chosen": -224.17477416992188,
+      "logps/rejected": -207.6041259765625,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.03650229796767235,
+      "rewards/margins": 0.04190974682569504,
+      "rewards/rejected": -0.07841204106807709,
+      "step": 5160
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.184251519886148e-06,
+      "logits/chosen": -2.190013885498047,
+      "logits/rejected": -2.2769131660461426,
+      "logps/chosen": -201.77218627929688,
+      "logps/rejected": -237.6571807861328,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.056646548211574554,
+      "rewards/margins": 0.07229969650506973,
+      "rewards/rejected": -0.1289462298154831,
+      "step": 5170
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.180027481818033e-06,
+      "logits/chosen": -2.3080639839172363,
+      "logits/rejected": -2.2970948219299316,
+      "logps/chosen": -269.02783203125,
+      "logps/rejected": -236.8043975830078,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.023297477513551712,
+      "rewards/margins": 0.05610079690814018,
+      "rewards/rejected": -0.0793982669711113,
+      "step": 5180
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.175794679903602e-06,
+      "logits/chosen": -2.364243745803833,
+      "logits/rejected": -2.1234567165374756,
+      "logps/chosen": -227.66976928710938,
+      "logps/rejected": -164.52684020996094,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.00028926803497597575,
+      "rewards/margins": 0.09214087575674057,
+      "rewards/rejected": -0.09243014454841614,
+      "step": 5190
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.171553136223222e-06,
+      "logits/chosen": -2.3197431564331055,
+      "logits/rejected": -2.314145803451538,
+      "logps/chosen": -269.05267333984375,
+      "logps/rejected": -290.14276123046875,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.017318129539489746,
+      "rewards/margins": 0.10380220413208008,
+      "rewards/rejected": -0.12112033367156982,
+      "step": 5200
+    },
+    {
+      "epoch": 0.34,
+      "eval_logits/chosen": -2.323685884475708,
+      "eval_logits/rejected": -2.1359636783599854,
+      "eval_logps/chosen": -234.24876403808594,
+      "eval_logps/rejected": -221.4422149658203,
+      "eval_loss": 0.690197765827179,
+      "eval_rewards/accuracies": 0.6294999718666077,
+      "eval_rewards/chosen": -0.022438107058405876,
+      "eval_rewards/margins": 0.0758652612566948,
+      "eval_rewards/rejected": -0.09830336272716522,
+      "eval_runtime": 712.3382,
+      "eval_samples_per_second": 2.808,
+      "eval_steps_per_second": 1.404,
+      "step": 5200
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.167302872902865e-06,
+      "logits/chosen": -2.3222103118896484,
+      "logits/rejected": -2.2121872901916504,
+      "logps/chosen": -255.7134246826172,
+      "logps/rejected": -254.5579071044922,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.042452942579984665,
+      "rewards/margins": 0.10782526433467865,
+      "rewards/rejected": -0.1502782106399536,
+      "step": 5210
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.163043912113985e-06,
+      "logits/chosen": -2.354125499725342,
+      "logits/rejected": -2.1191887855529785,
+      "logps/chosen": -261.3983459472656,
+      "logps/rejected": -233.1907196044922,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.02358170412480831,
+      "rewards/margins": 0.05787007883191109,
+      "rewards/rejected": -0.08145178854465485,
+      "step": 5220
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.15877627607341e-06,
+      "logits/chosen": -2.1449849605560303,
+      "logits/rejected": -2.0459647178649902,
+      "logps/chosen": -218.0535430908203,
+      "logps/rejected": -200.9430694580078,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.014587024226784706,
+      "rewards/margins": 0.0711187794804573,
+      "rewards/rejected": -0.08570580184459686,
+      "step": 5230
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.154499987043217e-06,
+      "logits/chosen": -2.3814520835876465,
+      "logits/rejected": -2.1603853702545166,
+      "logps/chosen": -225.3390655517578,
+      "logps/rejected": -213.6746826171875,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.011354709044098854,
+      "rewards/margins": 0.10515379905700684,
+      "rewards/rejected": -0.11650850623846054,
+      "step": 5240
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.150215067330625e-06,
+      "logits/chosen": -2.206449508666992,
+      "logits/rejected": -2.1436514854431152,
+      "logps/chosen": -211.77523803710938,
+      "logps/rejected": -235.5076446533203,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.02726421132683754,
+      "rewards/margins": 0.09695123136043549,
+      "rewards/rejected": -0.12421544641256332,
+      "step": 5250
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.145921539287876e-06,
+      "logits/chosen": -2.2395777702331543,
+      "logits/rejected": -1.96908438205719,
+      "logps/chosen": -193.36337280273438,
+      "logps/rejected": -183.28408813476562,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.027707424014806747,
+      "rewards/margins": 0.09924031049013138,
+      "rewards/rejected": -0.12694773077964783,
+      "step": 5260
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.141619425312115e-06,
+      "logits/chosen": -2.3244357109069824,
+      "logits/rejected": -1.956451654434204,
+      "logps/chosen": -211.45498657226562,
+      "logps/rejected": -193.4413299560547,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.028037140145897865,
+      "rewards/margins": 0.06428249180316925,
+      "rewards/rejected": -0.09231962263584137,
+      "step": 5270
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.1373087478452735e-06,
+      "logits/chosen": -2.4381213188171387,
+      "logits/rejected": -2.04194974899292,
+      "logps/chosen": -223.4126739501953,
+      "logps/rejected": -183.11099243164062,
+      "loss": 0.6853,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.01827925816178322,
+      "rewards/margins": 0.14312221109867096,
+      "rewards/rejected": -0.12484294176101685,
+      "step": 5280
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.132989529373959e-06,
+      "logits/chosen": -2.352229595184326,
+      "logits/rejected": -1.917790174484253,
+      "logps/chosen": -258.89263916015625,
+      "logps/rejected": -188.7851104736328,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.029797937721014023,
+      "rewards/margins": 0.08890985697507858,
+      "rewards/rejected": -0.1187077984213829,
+      "step": 5290
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.128661792429331e-06,
+      "logits/chosen": -2.3550148010253906,
+      "logits/rejected": -2.1839187145233154,
+      "logps/chosen": -257.23114013671875,
+      "logps/rejected": -266.33551025390625,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.034605689346790314,
+      "rewards/margins": 0.042079828679561615,
+      "rewards/rejected": -0.07668552547693253,
+      "step": 5300
+    },
+    {
+      "epoch": 0.35,
+      "eval_logits/chosen": -2.310957908630371,
+      "eval_logits/rejected": -2.124268054962158,
+      "eval_logps/chosen": -235.38356018066406,
+      "eval_logps/rejected": -223.17697143554688,
+      "eval_loss": 0.6903428435325623,
+      "eval_rewards/accuracies": 0.6299999952316284,
+      "eval_rewards/chosen": -0.03378620371222496,
+      "eval_rewards/margins": 0.08186446130275726,
+      "eval_rewards/rejected": -0.11565067619085312,
+      "eval_runtime": 710.7294,
+      "eval_samples_per_second": 2.814,
+      "eval_steps_per_second": 1.407,
+      "step": 5300
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.124325559586985e-06,
+      "logits/chosen": -2.0685927867889404,
+      "logits/rejected": -2.1043925285339355,
+      "logps/chosen": -195.6849822998047,
+      "logps/rejected": -210.3332977294922,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.07628266513347626,
+      "rewards/margins": 0.017243212088942528,
+      "rewards/rejected": -0.09352587163448334,
+      "step": 5310
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.119980853466835e-06,
+      "logits/chosen": -2.27421236038208,
+      "logits/rejected": -1.881087064743042,
+      "logps/chosen": -213.4599609375,
+      "logps/rejected": -195.97486877441406,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.04305556043982506,
+      "rewards/margins": 0.09629078209400177,
+      "rewards/rejected": -0.13934634625911713,
+      "step": 5320
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.115627696732997e-06,
+      "logits/chosen": -2.199984550476074,
+      "logits/rejected": -2.0089831352233887,
+      "logps/chosen": -194.8700714111328,
+      "logps/rejected": -181.50704956054688,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0345582440495491,
+      "rewards/margins": 0.05957914516329765,
+      "rewards/rejected": -0.09413739293813705,
+      "step": 5330
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.111266112093668e-06,
+      "logits/chosen": -2.3105640411376953,
+      "logits/rejected": -2.106968402862549,
+      "logps/chosen": -206.64205932617188,
+      "logps/rejected": -236.0433349609375,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.03174377605319023,
+      "rewards/margins": 0.1100342720746994,
+      "rewards/rejected": -0.14177805185317993,
+      "step": 5340
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.1068961223010115e-06,
+      "logits/chosen": -2.2916486263275146,
+      "logits/rejected": -1.9959255456924438,
+      "logps/chosen": -273.03662109375,
+      "logps/rejected": -259.06805419921875,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.023360053077340126,
+      "rewards/margins": 0.08707686513662338,
+      "rewards/rejected": -0.11043691635131836,
+      "step": 5350
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.102517750151034e-06,
+      "logits/chosen": -2.3448870182037354,
+      "logits/rejected": -2.100322961807251,
+      "logps/chosen": -295.0011901855469,
+      "logps/rejected": -228.78164672851562,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.00043646543053910136,
+      "rewards/margins": 0.05093403905630112,
+      "rewards/rejected": -0.050497572869062424,
+      "step": 5360
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.09813101848347e-06,
+      "logits/chosen": -2.22782564163208,
+      "logits/rejected": -2.258028984069824,
+      "logps/chosen": -216.62051391601562,
+      "logps/rejected": -240.86264038085938,
+      "loss": 0.694,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.023654133081436157,
+      "rewards/margins": 0.07087962329387665,
+      "rewards/rejected": -0.04722550883889198,
+      "step": 5370
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.093735950181659e-06,
+      "logits/chosen": -2.1826157569885254,
+      "logits/rejected": -2.083122968673706,
+      "logps/chosen": -220.95883178710938,
+      "logps/rejected": -255.71231079101562,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.04514528065919876,
+      "rewards/margins": 0.10210150480270386,
+      "rewards/rejected": -0.0569562129676342,
+      "step": 5380
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.0893325681724326e-06,
+      "logits/chosen": -2.2918612957000732,
+      "logits/rejected": -2.234261989593506,
+      "logps/chosen": -258.1329040527344,
+      "logps/rejected": -252.94076538085938,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0028874489944428205,
+      "rewards/margins": 0.07087242603302002,
+      "rewards/rejected": -0.06798496842384338,
+      "step": 5390
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.084920895425988e-06,
+      "logits/chosen": -2.240539073944092,
+      "logits/rejected": -2.236994981765747,
+      "logps/chosen": -237.4442901611328,
+      "logps/rejected": -259.9585876464844,
+      "loss": 0.689,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.000796229753177613,
+      "rewards/margins": 0.0886940062046051,
+      "rewards/rejected": -0.08949021995067596,
+      "step": 5400
+    },
+    {
+      "epoch": 0.35,
+      "eval_logits/chosen": -2.296610116958618,
+      "eval_logits/rejected": -2.1113150119781494,
+      "eval_logps/chosen": -229.67491149902344,
+      "eval_logps/rejected": -216.7387237548828,
+      "eval_loss": 0.6902357339859009,
+      "eval_rewards/accuracies": 0.6334999799728394,
+      "eval_rewards/chosen": 0.023300452157855034,
+      "eval_rewards/margins": 0.07456869632005692,
+      "eval_rewards/rejected": -0.05126824975013733,
+      "eval_runtime": 710.671,
+      "eval_samples_per_second": 2.814,
+      "eval_steps_per_second": 1.407,
+      "step": 5400
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.080500954955769e-06,
+      "logits/chosen": -2.2135491371154785,
+      "logits/rejected": -1.9314069747924805,
+      "logps/chosen": -254.9764862060547,
+      "logps/rejected": -243.9836883544922,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.008954535238444805,
+      "rewards/margins": 0.07293415814638138,
+      "rewards/rejected": -0.0639796257019043,
+      "step": 5410
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.076072769818354e-06,
+      "logits/chosen": -2.4696648120880127,
+      "logits/rejected": -2.026599168777466,
+      "logps/chosen": -242.1929473876953,
+      "logps/rejected": -186.80911254882812,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.015293831005692482,
+      "rewards/margins": 0.06581826508045197,
+      "rewards/rejected": -0.05052444338798523,
+      "step": 5420
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.071636363113323e-06,
+      "logits/chosen": -2.07266902923584,
+      "logits/rejected": -1.993199110031128,
+      "logps/chosen": -264.7987976074219,
+      "logps/rejected": -217.65054321289062,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.033312998712062836,
+      "rewards/margins": 0.0689430683851242,
+      "rewards/rejected": -0.03563006594777107,
+      "step": 5430
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.067191757983146e-06,
+      "logits/chosen": -2.0450809001922607,
+      "logits/rejected": -2.091036558151245,
+      "logps/chosen": -226.9083709716797,
+      "logps/rejected": -234.5966033935547,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.02667292393743992,
+      "rewards/margins": 0.10902222245931625,
+      "rewards/rejected": -0.08234930038452148,
+      "step": 5440
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.062738977613063e-06,
+      "logits/chosen": -2.237396717071533,
+      "logits/rejected": -2.115265130996704,
+      "logps/chosen": -232.11581420898438,
+      "logps/rejected": -191.1370391845703,
+      "loss": 0.689,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.023604759946465492,
+      "rewards/margins": 0.07414443045854568,
+      "rewards/rejected": -0.05053967237472534,
+      "step": 5450
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.058278045230957e-06,
+      "logits/chosen": -2.193748950958252,
+      "logits/rejected": -2.1995644569396973,
+      "logps/chosen": -218.991455078125,
+      "logps/rejected": -224.9091796875,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.006245986558496952,
+      "rewards/margins": 0.04330100864171982,
+      "rewards/rejected": -0.049546997994184494,
+      "step": 5460
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.053808984107235e-06,
+      "logits/chosen": -2.3936734199523926,
+      "logits/rejected": -2.0713841915130615,
+      "logps/chosen": -233.24472045898438,
+      "logps/rejected": -198.1108856201172,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.003764042630791664,
+      "rewards/margins": 0.05159348249435425,
+      "rewards/rejected": -0.05535752698779106,
+      "step": 5470
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.04933181755471e-06,
+      "logits/chosen": -2.3855273723602295,
+      "logits/rejected": -2.308960437774658,
+      "logps/chosen": -206.5284423828125,
+      "logps/rejected": -206.90713500976562,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0005217326688580215,
+      "rewards/margins": 0.0759122297167778,
+      "rewards/rejected": -0.07539048790931702,
+      "step": 5480
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.044846568928477e-06,
+      "logits/chosen": -2.2790474891662598,
+      "logits/rejected": -2.3462178707122803,
+      "logps/chosen": -261.7860107421875,
+      "logps/rejected": -259.61175537109375,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.002618322381749749,
+      "rewards/margins": 0.06683683395385742,
+      "rewards/rejected": -0.06945516169071198,
+      "step": 5490
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.040353261625788e-06,
+      "logits/chosen": -2.444617748260498,
+      "logits/rejected": -2.0571980476379395,
+      "logps/chosen": -275.5302734375,
+      "logps/rejected": -246.3635711669922,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.010149752721190453,
+      "rewards/margins": 0.09463830292224884,
+      "rewards/rejected": -0.08448855578899384,
+      "step": 5500
+    },
+    {
+      "epoch": 0.36,
+      "eval_logits/chosen": -2.290515661239624,
+      "eval_logits/rejected": -2.1054341793060303,
+      "eval_logps/chosen": -232.49339294433594,
+      "eval_logps/rejected": -219.36749267578125,
+      "eval_loss": 0.6903730630874634,
+      "eval_rewards/accuracies": 0.6230000257492065,
+      "eval_rewards/chosen": -0.004884431138634682,
+      "eval_rewards/margins": 0.07267154008150101,
+      "eval_rewards/rejected": -0.07755597680807114,
+      "eval_runtime": 709.5644,
+      "eval_samples_per_second": 2.819,
+      "eval_steps_per_second": 1.409,
+      "step": 5500
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.035851919085936e-06,
+      "logits/chosen": -2.2773475646972656,
+      "logits/rejected": -2.1437253952026367,
+      "logps/chosen": -268.17034912109375,
+      "logps/rejected": -197.60702514648438,
+      "loss": 0.688,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.007762535475194454,
+      "rewards/margins": 0.08148452639579773,
+      "rewards/rejected": -0.08924706280231476,
+      "step": 5510
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.031342564790128e-06,
+      "logits/chosen": -2.2170324325561523,
+      "logits/rejected": -2.0734400749206543,
+      "logps/chosen": -204.36358642578125,
+      "logps/rejected": -210.1439666748047,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.016081126406788826,
+      "rewards/margins": 0.10553131252527237,
+      "rewards/rejected": -0.0894501805305481,
+      "step": 5520
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.026825222261367e-06,
+      "logits/chosen": -2.228926658630371,
+      "logits/rejected": -1.9831037521362305,
+      "logps/chosen": -179.34925842285156,
+      "logps/rejected": -177.38687133789062,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.06274209171533585,
+      "rewards/margins": 0.05180812627077103,
+      "rewards/rejected": -0.11455021053552628,
+      "step": 5530
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.022299915064321e-06,
+      "logits/chosen": -2.300727605819702,
+      "logits/rejected": -2.096855878829956,
+      "logps/chosen": -305.073486328125,
+      "logps/rejected": -261.6318054199219,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.007745129056274891,
+      "rewards/margins": 0.09663836658000946,
+      "rewards/rejected": -0.08889324963092804,
+      "step": 5540
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.017766666805213e-06,
+      "logits/chosen": -2.120983839035034,
+      "logits/rejected": -2.074357271194458,
+      "logps/chosen": -215.736328125,
+      "logps/rejected": -188.95654296875,
+      "loss": 0.689,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.02215953730046749,
+      "rewards/margins": 0.08581807464361191,
+      "rewards/rejected": -0.10797761380672455,
+      "step": 5550
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.013225501131684e-06,
+      "logits/chosen": -2.3249385356903076,
+      "logits/rejected": -2.043680191040039,
+      "logps/chosen": -217.39242553710938,
+      "logps/rejected": -197.59507751464844,
+      "loss": 0.691,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.00014842339442111552,
+      "rewards/margins": 0.071754589676857,
+      "rewards/rejected": -0.07190301269292831,
+      "step": 5560
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.008676441732679e-06,
+      "logits/chosen": -2.2676405906677246,
+      "logits/rejected": -1.9487760066986084,
+      "logps/chosen": -207.52322387695312,
+      "logps/rejected": -164.7802734375,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.012288715690374374,
+      "rewards/margins": 0.05257941409945488,
+      "rewards/rejected": -0.06486812978982925,
+      "step": 5570
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 4.00411951233832e-06,
+      "logits/chosen": -2.412111520767212,
+      "logits/rejected": -2.0756661891937256,
+      "logps/chosen": -220.82858276367188,
+      "logps/rejected": -184.4314727783203,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.011034145951271057,
+      "rewards/margins": 0.0827646404504776,
+      "rewards/rejected": -0.09379879385232925,
+      "step": 5580
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.999554736719785e-06,
+      "logits/chosen": -2.14727520942688,
+      "logits/rejected": -2.033402919769287,
+      "logps/chosen": -291.1112976074219,
+      "logps/rejected": -254.0940704345703,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.008655655197799206,
+      "rewards/margins": 0.06500723958015442,
+      "rewards/rejected": -0.056351590901613235,
+      "step": 5590
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.994982138689177e-06,
+      "logits/chosen": -2.4407970905303955,
+      "logits/rejected": -2.242966413497925,
+      "logps/chosen": -236.0352783203125,
+      "logps/rejected": -238.973876953125,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.013648864813148975,
+      "rewards/margins": 0.047897934913635254,
+      "rewards/rejected": -0.03424907475709915,
+      "step": 5600
+    },
+    {
+      "epoch": 0.37,
+      "eval_logits/chosen": -2.2870538234710693,
+      "eval_logits/rejected": -2.1021108627319336,
+      "eval_logps/chosen": -232.2427520751953,
+      "eval_logps/rejected": -219.23269653320312,
+      "eval_loss": 0.6902966499328613,
+      "eval_rewards/accuracies": 0.6340000033378601,
+      "eval_rewards/chosen": -0.0023779442999511957,
+      "eval_rewards/margins": 0.07382997125387192,
+      "eval_rewards/rejected": -0.07620792090892792,
+      "eval_runtime": 710.4152,
+      "eval_samples_per_second": 2.815,
+      "eval_steps_per_second": 1.408,
+      "step": 5600
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.990401742099408e-06,
+      "logits/chosen": -2.104093074798584,
+      "logits/rejected": -2.1244332790374756,
+      "logps/chosen": -179.53659057617188,
+      "logps/rejected": -176.4581298828125,
+      "loss": 0.692,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.006734983064234257,
+      "rewards/margins": 0.04270303249359131,
+      "rewards/rejected": -0.04943801462650299,
+      "step": 5610
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.985813570844072e-06,
+      "logits/chosen": -2.2303576469421387,
+      "logits/rejected": -2.086726188659668,
+      "logps/chosen": -297.89227294921875,
+      "logps/rejected": -276.9810485839844,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.03505517542362213,
+      "rewards/margins": 0.06640339642763138,
+      "rewards/rejected": -0.10145857185125351,
+      "step": 5620
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.981217648857316e-06,
+      "logits/chosen": -2.308424711227417,
+      "logits/rejected": -2.119347333908081,
+      "logps/chosen": -172.80674743652344,
+      "logps/rejected": -184.7308349609375,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.002663122955709696,
+      "rewards/margins": 0.10142280906438828,
+      "rewards/rejected": -0.10408592224121094,
+      "step": 5630
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.97661400011372e-06,
+      "logits/chosen": -2.1583411693573,
+      "logits/rejected": -2.204422950744629,
+      "logps/chosen": -244.61489868164062,
+      "logps/rejected": -238.1432342529297,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.015875743702054024,
+      "rewards/margins": 0.049512267112731934,
+      "rewards/rejected": -0.0653880164027214,
+      "step": 5640
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.972002648628174e-06,
+      "logits/chosen": -2.1991384029388428,
+      "logits/rejected": -1.8762391805648804,
+      "logps/chosen": -277.7632751464844,
+      "logps/rejected": -236.1746368408203,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.007324635982513428,
+      "rewards/margins": 0.06103179603815079,
+      "rewards/rejected": -0.06835642457008362,
+      "step": 5650
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.967383618455743e-06,
+      "logits/chosen": -2.3441319465637207,
+      "logits/rejected": -2.178091526031494,
+      "logps/chosen": -231.02035522460938,
+      "logps/rejected": -256.923095703125,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.039006225764751434,
+      "rewards/margins": 0.07712291181087494,
+      "rewards/rejected": -0.11612913757562637,
+      "step": 5660
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.9627569336915515e-06,
+      "logits/chosen": -2.477116107940674,
+      "logits/rejected": -2.1675541400909424,
+      "logps/chosen": -247.08352661132812,
+      "logps/rejected": -199.73582458496094,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.003792487783357501,
+      "rewards/margins": 0.09328923374414444,
+      "rewards/rejected": -0.08949675410985947,
+      "step": 5670
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.9581226184706555e-06,
+      "logits/chosen": -2.326864719390869,
+      "logits/rejected": -2.435724973678589,
+      "logps/chosen": -193.1772918701172,
+      "logps/rejected": -269.46685791015625,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.014881017617881298,
+      "rewards/margins": 0.055919043719768524,
+      "rewards/rejected": -0.04103802144527435,
+      "step": 5680
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.953480696967912e-06,
+      "logits/chosen": -1.9913456439971924,
+      "logits/rejected": -2.222503662109375,
+      "logps/chosen": -210.5967254638672,
+      "logps/rejected": -254.0486602783203,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.012251259759068489,
+      "rewards/margins": 0.07006217539310455,
+      "rewards/rejected": -0.0823134332895279,
+      "step": 5690
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.948831193397857e-06,
+      "logits/chosen": -2.2036399841308594,
+      "logits/rejected": -2.187148332595825,
+      "logps/chosen": -164.9729461669922,
+      "logps/rejected": -178.32603454589844,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.010424559935927391,
+      "rewards/margins": 0.07527212798595428,
+      "rewards/rejected": -0.06484757363796234,
+      "step": 5700
+    },
+    {
+      "epoch": 0.37,
+      "eval_logits/chosen": -2.2823386192321777,
+      "eval_logits/rejected": -2.0973258018493652,
+      "eval_logps/chosen": -230.5282440185547,
+      "eval_logps/rejected": -218.62940979003906,
+      "eval_loss": 0.6900830268859863,
+      "eval_rewards/accuracies": 0.6345000267028809,
+      "eval_rewards/chosen": 0.014767038635909557,
+      "eval_rewards/margins": 0.08494207262992859,
+      "eval_rewards/rejected": -0.07017502933740616,
+      "eval_runtime": 714.2649,
+      "eval_samples_per_second": 2.8,
+      "eval_steps_per_second": 1.4,
+      "step": 5700
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.94417413201458e-06,
+      "logits/chosen": -2.185957193374634,
+      "logits/rejected": -2.038264274597168,
+      "logps/chosen": -208.8540496826172,
+      "logps/rejected": -198.67984008789062,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.027174552902579308,
+      "rewards/margins": 0.07592582702636719,
+      "rewards/rejected": -0.04875127598643303,
+      "step": 5710
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.9395095371115935e-06,
+      "logits/chosen": -2.339968204498291,
+      "logits/rejected": -2.060576915740967,
+      "logps/chosen": -209.5446319580078,
+      "logps/rejected": -207.98910522460938,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.031124601140618324,
+      "rewards/margins": 0.09055650979280472,
+      "rewards/rejected": -0.059431903064250946,
+      "step": 5720
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.93483743302171e-06,
+      "logits/chosen": -2.2814464569091797,
+      "logits/rejected": -2.04420804977417,
+      "logps/chosen": -208.9713592529297,
+      "logps/rejected": -190.6579132080078,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.03820297122001648,
+      "rewards/margins": 0.0542188361287117,
+      "rewards/rejected": -0.016015859320759773,
+      "step": 5730
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.930157844116913e-06,
+      "logits/chosen": -2.1364564895629883,
+      "logits/rejected": -2.0696628093719482,
+      "logps/chosen": -202.4792938232422,
+      "logps/rejected": -195.42135620117188,
+      "loss": 0.69,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.07277990877628326,
+      "rewards/margins": 0.07660780847072601,
+      "rewards/rejected": -0.003827892942354083,
+      "step": 5740
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.925470794808229e-06,
+      "logits/chosen": -2.2870192527770996,
+      "logits/rejected": -1.9282214641571045,
+      "logps/chosen": -236.171875,
+      "logps/rejected": -211.88504028320312,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.028158491477370262,
+      "rewards/margins": 0.08530527353286743,
+      "rewards/rejected": -0.05714678019285202,
+      "step": 5750
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.920776309545606e-06,
+      "logits/chosen": -2.3560986518859863,
+      "logits/rejected": -2.188704013824463,
+      "logps/chosen": -147.86883544921875,
+      "logps/rejected": -148.76467895507812,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.04838157445192337,
+      "rewards/margins": 0.07728347182273865,
+      "rewards/rejected": -0.028901899233460426,
+      "step": 5760
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.916074412817778e-06,
+      "logits/chosen": -2.2517282962799072,
+      "logits/rejected": -1.886130690574646,
+      "logps/chosen": -239.67239379882812,
+      "logps/rejected": -250.0625457763672,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.03661612793803215,
+      "rewards/margins": 0.09867843240499496,
+      "rewards/rejected": -0.062062300741672516,
+      "step": 5770
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.911365129152139e-06,
+      "logits/chosen": -2.3809988498687744,
+      "logits/rejected": -2.224855899810791,
+      "logps/chosen": -227.54214477539062,
+      "logps/rejected": -230.674560546875,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.04891723394393921,
+      "rewards/margins": 0.08672243356704712,
+      "rewards/rejected": -0.03780519217252731,
+      "step": 5780
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.906648483114623e-06,
+      "logits/chosen": -2.2592244148254395,
+      "logits/rejected": -2.1565563678741455,
+      "logps/chosen": -195.49063110351562,
+      "logps/rejected": -174.44102478027344,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.018330033868551254,
+      "rewards/margins": 0.1153540164232254,
+      "rewards/rejected": -0.09702397882938385,
+      "step": 5790
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.901924499309564e-06,
+      "logits/chosen": -2.198864698410034,
+      "logits/rejected": -2.009559154510498,
+      "logps/chosen": -229.9792022705078,
+      "logps/rejected": -206.246337890625,
+      "loss": 0.69,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.012742845341563225,
+      "rewards/margins": 0.08873111009597778,
+      "rewards/rejected": -0.10147394984960556,
+      "step": 5800
+    },
+    {
+      "epoch": 0.38,
+      "eval_logits/chosen": -2.2906556129455566,
+      "eval_logits/rejected": -2.104820966720581,
+      "eval_logps/chosen": -233.96670532226562,
+      "eval_logps/rejected": -222.71261596679688,
+      "eval_loss": 0.6902094483375549,
+      "eval_rewards/accuracies": 0.6365000009536743,
+      "eval_rewards/chosen": -0.019617412239313126,
+      "eval_rewards/margins": 0.09138944000005722,
+      "eval_rewards/rejected": -0.11100686341524124,
+      "eval_runtime": 710.6333,
+      "eval_samples_per_second": 2.814,
+      "eval_steps_per_second": 1.407,
+      "step": 5800
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.897193202379575e-06,
+      "logits/chosen": -2.2877416610717773,
+      "logits/rejected": -2.117922306060791,
+      "logps/chosen": -201.8192596435547,
+      "logps/rejected": -193.40383911132812,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.013415491208434105,
+      "rewards/margins": 0.0855737030506134,
+      "rewards/rejected": -0.09898919612169266,
+      "step": 5810
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.8924546170054215e-06,
+      "logits/chosen": -2.2300517559051514,
+      "logits/rejected": -2.1765639781951904,
+      "logps/chosen": -216.3985595703125,
+      "logps/rejected": -204.2929229736328,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.00490927416831255,
+      "rewards/margins": 0.0736776739358902,
+      "rewards/rejected": -0.07858695089817047,
+      "step": 5820
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.887708767905883e-06,
+      "logits/chosen": -2.492140054702759,
+      "logits/rejected": -2.089107036590576,
+      "logps/chosen": -243.9584503173828,
+      "logps/rejected": -184.98101806640625,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.001912574516609311,
+      "rewards/margins": 0.0686551108956337,
+      "rewards/rejected": -0.07056768238544464,
+      "step": 5830
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.882955679837636e-06,
+      "logits/chosen": -2.250488758087158,
+      "logits/rejected": -2.1399431228637695,
+      "logps/chosen": -238.8167724609375,
+      "logps/rejected": -247.60751342773438,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.005602418445050716,
+      "rewards/margins": 0.06415437906980515,
+      "rewards/rejected": -0.058551959693431854,
+      "step": 5840
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.878195377595113e-06,
+      "logits/chosen": -2.3308000564575195,
+      "logits/rejected": -2.1495823860168457,
+      "logps/chosen": -235.9189453125,
+      "logps/rejected": -239.89706420898438,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.00013175308413337916,
+      "rewards/margins": 0.09541679173707962,
+      "rewards/rejected": -0.09554854780435562,
+      "step": 5850
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.873427886010384e-06,
+      "logits/chosen": -2.276776075363159,
+      "logits/rejected": -2.1244194507598877,
+      "logps/chosen": -189.79672241210938,
+      "logps/rejected": -179.958740234375,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.006552155129611492,
+      "rewards/margins": 0.09251097589731216,
+      "rewards/rejected": -0.0859588235616684,
+      "step": 5860
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.868653229953021e-06,
+      "logits/chosen": -2.3475050926208496,
+      "logits/rejected": -2.118560314178467,
+      "logps/chosen": -235.6221923828125,
+      "logps/rejected": -243.8154754638672,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.015926335006952286,
+      "rewards/margins": 0.10257148742675781,
+      "rewards/rejected": -0.08664515614509583,
+      "step": 5870
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.8638714343299675e-06,
+      "logits/chosen": -2.2526631355285645,
+      "logits/rejected": -2.1661479473114014,
+      "logps/chosen": -218.9933319091797,
+      "logps/rejected": -241.96755981445312,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.006506229750812054,
+      "rewards/margins": 0.0785847157239914,
+      "rewards/rejected": -0.08509095013141632,
+      "step": 5880
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.859082524085414e-06,
+      "logits/chosen": -2.2656807899475098,
+      "logits/rejected": -1.9120867252349854,
+      "logps/chosen": -273.0644226074219,
+      "logps/rejected": -219.8885498046875,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0072127147577703,
+      "rewards/margins": 0.07063201814889908,
+      "rewards/rejected": -0.07784473150968552,
+      "step": 5890
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.854286524200659e-06,
+      "logits/chosen": -2.423177480697632,
+      "logits/rejected": -2.148200511932373,
+      "logps/chosen": -276.10162353515625,
+      "logps/rejected": -229.82730102539062,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0031580955255776644,
+      "rewards/margins": 0.04362088814377785,
+      "rewards/rejected": -0.04046279191970825,
+      "step": 5900
+    },
+    {
+      "epoch": 0.39,
+      "eval_logits/chosen": -2.2796647548675537,
+      "eval_logits/rejected": -2.094618797302246,
+      "eval_logps/chosen": -231.79417419433594,
+      "eval_logps/rejected": -219.75482177734375,
+      "eval_loss": 0.6900690793991089,
+      "eval_rewards/accuracies": 0.6384999752044678,
+      "eval_rewards/chosen": 0.002107798121869564,
+      "eval_rewards/margins": 0.08353700488805771,
+      "eval_rewards/rejected": -0.08142919838428497,
+      "eval_runtime": 710.6336,
+      "eval_samples_per_second": 2.814,
+      "eval_steps_per_second": 1.407,
+      "step": 5900
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.849483459693991e-06,
+      "logits/chosen": -2.361053943634033,
+      "logits/rejected": -2.2258360385894775,
+      "logps/chosen": -208.7968292236328,
+      "logps/rejected": -182.0562744140625,
+      "loss": 0.6855,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.008798656985163689,
+      "rewards/margins": 0.11928577721118927,
+      "rewards/rejected": -0.11048711836338043,
+      "step": 5910
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.844673355620544e-06,
+      "logits/chosen": -2.253052234649658,
+      "logits/rejected": -2.1112794876098633,
+      "logps/chosen": -245.8029022216797,
+      "logps/rejected": -217.4182891845703,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.007700022310018539,
+      "rewards/margins": 0.09900878369808197,
+      "rewards/rejected": -0.09130875021219254,
+      "step": 5920
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.839856237072178e-06,
+      "logits/chosen": -2.0936896800994873,
+      "logits/rejected": -2.091200351715088,
+      "logps/chosen": -185.02664184570312,
+      "logps/rejected": -212.76174926757812,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.024525459855794907,
+      "rewards/margins": 0.10542996972799301,
+      "rewards/rejected": -0.1299554407596588,
+      "step": 5930
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.8350321291773455e-06,
+      "logits/chosen": -2.0315639972686768,
+      "logits/rejected": -1.9680635929107666,
+      "logps/chosen": -200.049560546875,
+      "logps/rejected": -164.24830627441406,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.02472389116883278,
+      "rewards/margins": 0.08395050466060638,
+      "rewards/rejected": -0.0592266209423542,
+      "step": 5940
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.830201057100953e-06,
+      "logits/chosen": -2.3199234008789062,
+      "logits/rejected": -2.351250171661377,
+      "logps/chosen": -190.0390167236328,
+      "logps/rejected": -215.13748168945312,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.01733965426683426,
+      "rewards/margins": 0.0960569754242897,
+      "rewards/rejected": -0.07871732115745544,
+      "step": 5950
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.82536304604424e-06,
+      "logits/chosen": -2.111283302307129,
+      "logits/rejected": -2.055291175842285,
+      "logps/chosen": -232.94387817382812,
+      "logps/rejected": -208.7888641357422,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.032203659415245056,
+      "rewards/margins": 0.075811468064785,
+      "rewards/rejected": -0.04360780864953995,
+      "step": 5960
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.8205181212446435e-06,
+      "logits/chosen": -2.488105297088623,
+      "logits/rejected": -2.2576441764831543,
+      "logps/chosen": -266.7863464355469,
+      "logps/rejected": -231.9990997314453,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.03772572800517082,
+      "rewards/margins": 0.05616292357444763,
+      "rewards/rejected": -0.01843719184398651,
+      "step": 5970
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.815666307975664e-06,
+      "logits/chosen": -2.29557466506958,
+      "logits/rejected": -2.167210102081299,
+      "logps/chosen": -235.0832061767578,
+      "logps/rejected": -221.5277862548828,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.022346094250679016,
+      "rewards/margins": 0.05626540258526802,
+      "rewards/rejected": -0.033919308334589005,
+      "step": 5980
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.8108076315467346e-06,
+      "logits/chosen": -2.3752923011779785,
+      "logits/rejected": -2.240790367126465,
+      "logps/chosen": -257.206787109375,
+      "logps/rejected": -189.83230590820312,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.0007598648080602288,
+      "rewards/margins": 0.0735207051038742,
+      "rewards/rejected": -0.07276083528995514,
+      "step": 5990
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.805942117303093e-06,
+      "logits/chosen": -2.5641138553619385,
+      "logits/rejected": -2.226461887359619,
+      "logps/chosen": -314.4233093261719,
+      "logps/rejected": -274.7497863769531,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.016122477129101753,
+      "rewards/margins": 0.06850017607212067,
+      "rewards/rejected": -0.05237768962979317,
+      "step": 6000
+    },
+    {
+      "epoch": 0.39,
+      "eval_logits/chosen": -2.311023235321045,
+      "eval_logits/rejected": -2.123633861541748,
+      "eval_logps/chosen": -231.44161987304688,
+      "eval_logps/rejected": -218.17413330078125,
+      "eval_loss": 0.6901270747184753,
+      "eval_rewards/accuracies": 0.6294999718666077,
+      "eval_rewards/chosen": 0.005633320193737745,
+      "eval_rewards/margins": 0.07125571370124817,
+      "eval_rewards/rejected": -0.06562238931655884,
+      "eval_runtime": 710.6252,
+      "eval_samples_per_second": 2.814,
+      "eval_steps_per_second": 1.407,
+      "step": 6000
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.8010697906256446e-06,
+      "logits/chosen": -2.1471753120422363,
+      "logits/rejected": -2.135615348815918,
+      "logps/chosen": -210.50021362304688,
+      "logps/rejected": -188.69100952148438,
+      "loss": 0.6953,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.038038115948438644,
+      "rewards/margins": 0.06020959094166756,
+      "rewards/rejected": -0.0982476994395256,
+      "step": 6010
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.7961906769308323e-06,
+      "logits/chosen": -2.19319486618042,
+      "logits/rejected": -2.009164571762085,
+      "logps/chosen": -206.8964385986328,
+      "logps/rejected": -224.5199737548828,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.006612460128962994,
+      "rewards/margins": 0.0754861980676651,
+      "rewards/rejected": -0.08209865540266037,
+      "step": 6020
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.7913048016705028e-06,
+      "logits/chosen": -2.2544267177581787,
+      "logits/rejected": -2.210512638092041,
+      "logps/chosen": -259.06982421875,
+      "logps/rejected": -262.7820739746094,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.008697474375367165,
+      "rewards/margins": 0.057602547109127045,
+      "rewards/rejected": -0.048905082046985626,
+      "step": 6030
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.786412190331775e-06,
+      "logits/chosen": -2.447171211242676,
+      "logits/rejected": -2.118438720703125,
+      "logps/chosen": -201.5836944580078,
+      "logps/rejected": -172.86338806152344,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.012768149375915527,
+      "rewards/margins": 0.08379059284925461,
+      "rewards/rejected": -0.07102244347333908,
+      "step": 6040
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.781512868436906e-06,
+      "logits/chosen": -2.435929775238037,
+      "logits/rejected": -2.2919459342956543,
+      "logps/chosen": -129.0099639892578,
+      "logps/rejected": -143.07064819335938,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.03600788861513138,
+      "rewards/margins": 0.0627172440290451,
+      "rewards/rejected": -0.02670934796333313,
+      "step": 6050
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7766068615431605e-06,
+      "logits/chosen": -2.2673373222351074,
+      "logits/rejected": -2.2142586708068848,
+      "logps/chosen": -254.04931640625,
+      "logps/rejected": -212.45474243164062,
+      "loss": 0.692,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.030377531424164772,
+      "rewards/margins": 0.07040407508611679,
+      "rewards/rejected": -0.04002653807401657,
+      "step": 6060
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.771694195242671e-06,
+      "logits/chosen": -2.4878172874450684,
+      "logits/rejected": -2.041779041290283,
+      "logps/chosen": -297.07598876953125,
+      "logps/rejected": -191.4386444091797,
+      "loss": 0.691,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.021027540788054466,
+      "rewards/margins": 0.06093548983335495,
+      "rewards/rejected": -0.039907947182655334,
+      "step": 6070
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.766774895162314e-06,
+      "logits/chosen": -2.3142666816711426,
+      "logits/rejected": -2.210665225982666,
+      "logps/chosen": -244.95858764648438,
+      "logps/rejected": -193.0878143310547,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.015811407938599586,
+      "rewards/margins": 0.04264925420284271,
+      "rewards/rejected": -0.026837846264243126,
+      "step": 6080
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7618489869635666e-06,
+      "logits/chosen": -2.227144479751587,
+      "logits/rejected": -2.172423839569092,
+      "logps/chosen": -247.10653686523438,
+      "logps/rejected": -232.07986450195312,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.000796070322394371,
+      "rewards/margins": 0.04716240614652634,
+      "rewards/rejected": -0.04795847460627556,
+      "step": 6090
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.756916496342379e-06,
+      "logits/chosen": -2.2466297149658203,
+      "logits/rejected": -2.30493426322937,
+      "logps/chosen": -184.02410888671875,
+      "logps/rejected": -198.5399932861328,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.028477992862462997,
+      "rewards/margins": 0.060630954802036285,
+      "rewards/rejected": -0.03215295821428299,
+      "step": 6100
+    },
+    {
+      "epoch": 0.4,
+      "eval_logits/chosen": -2.3196463584899902,
+      "eval_logits/rejected": -2.131615161895752,
+      "eval_logps/chosen": -228.611572265625,
+      "eval_logps/rejected": -215.37451171875,
+      "eval_loss": 0.6901247501373291,
+      "eval_rewards/accuracies": 0.6215000152587891,
+      "eval_rewards/chosen": 0.0339338555932045,
+      "eval_rewards/margins": 0.0715600922703743,
+      "eval_rewards/rejected": -0.0376262404024601,
+      "eval_runtime": 710.3418,
+      "eval_samples_per_second": 2.816,
+      "eval_steps_per_second": 1.408,
+      "step": 6100
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.751977449029039e-06,
+      "logits/chosen": -1.9810413122177124,
+      "logits/rejected": -1.983925461769104,
+      "logps/chosen": -258.67047119140625,
+      "logps/rejected": -227.87368774414062,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.014060038141906261,
+      "rewards/margins": 0.06598879396915436,
+      "rewards/rejected": -0.05192875862121582,
+      "step": 6110
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.747031870788037e-06,
+      "logits/chosen": -2.4974024295806885,
+      "logits/rejected": -2.138140916824341,
+      "logps/chosen": -313.6204833984375,
+      "logps/rejected": -238.707763671875,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03132764250040054,
+      "rewards/margins": 0.07506566494703293,
+      "rewards/rejected": -0.043738026171922684,
+      "step": 6120
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7420797874179326e-06,
+      "logits/chosen": -2.277357578277588,
+      "logits/rejected": -2.018991708755493,
+      "logps/chosen": -236.95089721679688,
+      "logps/rejected": -186.17581176757812,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0015857883263379335,
+      "rewards/margins": 0.08113957941532135,
+      "rewards/rejected": -0.07955377548933029,
+      "step": 6130
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7371212247512167e-06,
+      "logits/chosen": -2.6132965087890625,
+      "logits/rejected": -2.2684082984924316,
+      "logps/chosen": -322.7628173828125,
+      "logps/rejected": -267.7512512207031,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.03631674125790596,
+      "rewards/margins": 0.10096652805805206,
+      "rewards/rejected": -0.0646497905254364,
+      "step": 6140
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7321562086541817e-06,
+      "logits/chosen": -2.3771257400512695,
+      "logits/rejected": -2.2703232765197754,
+      "logps/chosen": -251.13504028320312,
+      "logps/rejected": -260.8123474121094,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.01845073141157627,
+      "rewards/margins": 0.08887068182229996,
+      "rewards/rejected": -0.07041995227336884,
+      "step": 6150
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7271847650267834e-06,
+      "logits/chosen": -2.168355703353882,
+      "logits/rejected": -2.0670669078826904,
+      "logps/chosen": -199.58877563476562,
+      "logps/rejected": -204.58006286621094,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.003458759281784296,
+      "rewards/margins": 0.058815740048885345,
+      "rewards/rejected": -0.05535699054598808,
+      "step": 6160
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7222069198025086e-06,
+      "logits/chosen": -2.1603808403015137,
+      "logits/rejected": -1.9670292139053345,
+      "logps/chosen": -209.85214233398438,
+      "logps/rejected": -197.63839721679688,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.013096879236400127,
+      "rewards/margins": 0.08378522843122482,
+      "rewards/rejected": -0.09688210487365723,
+      "step": 6170
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7172226989482353e-06,
+      "logits/chosen": -2.1758627891540527,
+      "logits/rejected": -1.9652674198150635,
+      "logps/chosen": -209.81430053710938,
+      "logps/rejected": -212.7050323486328,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0015852168435230851,
+      "rewards/margins": 0.05035899206995964,
+      "rewards/rejected": -0.04877377673983574,
+      "step": 6180
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7122321284641007e-06,
+      "logits/chosen": -2.4331085681915283,
+      "logits/rejected": -1.9999637603759766,
+      "logps/chosen": -363.33984375,
+      "logps/rejected": -262.52734375,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.0055319941602647305,
+      "rewards/margins": 0.09677392989397049,
+      "rewards/rejected": -0.10230592638254166,
+      "step": 6190
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.707235234383365e-06,
+      "logits/chosen": -2.3560452461242676,
+      "logits/rejected": -2.0064382553100586,
+      "logps/chosen": -258.49041748046875,
+      "logps/rejected": -178.62417602539062,
+      "loss": 0.691,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.027635321021080017,
+      "rewards/margins": 0.06912614405155182,
+      "rewards/rejected": -0.0414908304810524,
+      "step": 6200
+    },
+    {
+      "epoch": 0.41,
+      "eval_logits/chosen": -2.3145864009857178,
+      "eval_logits/rejected": -2.126389265060425,
+      "eval_logps/chosen": -229.6930694580078,
+      "eval_logps/rejected": -217.35781860351562,
+      "eval_loss": 0.689987301826477,
+      "eval_rewards/accuracies": 0.6284999847412109,
+      "eval_rewards/chosen": 0.023118959739804268,
+      "eval_rewards/margins": 0.08057821542024612,
+      "eval_rewards/rejected": -0.05745925009250641,
+      "eval_runtime": 712.4839,
+      "eval_samples_per_second": 2.807,
+      "eval_steps_per_second": 1.404,
+      "step": 6200
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.702232042772277e-06,
+      "logits/chosen": -2.1632602214813232,
+      "logits/rejected": -2.1070027351379395,
+      "logps/chosen": -201.1727294921875,
+      "logps/rejected": -189.2860565185547,
+      "loss": 0.689,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.013734388165175915,
+      "rewards/margins": 0.10467977821826935,
+      "rewards/rejected": -0.09094538539648056,
+      "step": 6210
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6972225797299325e-06,
+      "logits/chosen": -2.2812116146087646,
+      "logits/rejected": -2.31536602973938,
+      "logps/chosen": -251.81900024414062,
+      "logps/rejected": -243.4286651611328,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.013103622011840343,
+      "rewards/margins": 0.07545242458581924,
+      "rewards/rejected": -0.08855602890253067,
+      "step": 6220
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.692206871388147e-06,
+      "logits/chosen": -2.3902740478515625,
+      "logits/rejected": -1.9351933002471924,
+      "logps/chosen": -232.5216827392578,
+      "logps/rejected": -202.56259155273438,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.0013294353848323226,
+      "rewards/margins": 0.1127009242773056,
+      "rewards/rejected": -0.11403036117553711,
+      "step": 6230
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6871849439113115e-06,
+      "logits/chosen": -2.090280294418335,
+      "logits/rejected": -2.0177419185638428,
+      "logps/chosen": -228.4338836669922,
+      "logps/rejected": -225.20547485351562,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.014769596047699451,
+      "rewards/margins": 0.0631939023733139,
+      "rewards/rejected": -0.048424310982227325,
+      "step": 6240
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.682156823496259e-06,
+      "logits/chosen": -2.3378641605377197,
+      "logits/rejected": -2.049996852874756,
+      "logps/chosen": -209.41159057617188,
+      "logps/rejected": -187.8102569580078,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.045023590326309204,
+      "rewards/margins": 0.10764230787754059,
+      "rewards/rejected": -0.06261870265007019,
+      "step": 6250
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.67712253637213e-06,
+      "logits/chosen": -2.363736629486084,
+      "logits/rejected": -2.1265621185302734,
+      "logps/chosen": -291.31158447265625,
+      "logps/rejected": -208.5716552734375,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.009828880429267883,
+      "rewards/margins": 0.07172620296478271,
+      "rewards/rejected": -0.06189732626080513,
+      "step": 6260
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.672082108800231e-06,
+      "logits/chosen": -2.1761693954467773,
+      "logits/rejected": -1.9752609729766846,
+      "logps/chosen": -216.0406951904297,
+      "logps/rejected": -189.57601928710938,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.022248603403568268,
+      "rewards/margins": 0.07574795186519623,
+      "rewards/rejected": -0.0979965478181839,
+      "step": 6270
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6670355670739012e-06,
+      "logits/chosen": -2.2948927879333496,
+      "logits/rejected": -2.105381488800049,
+      "logps/chosen": -160.06436157226562,
+      "logps/rejected": -167.32986450195312,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.018890656530857086,
+      "rewards/margins": 0.09904204308986664,
+      "rewards/rejected": -0.08015139400959015,
+      "step": 6280
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6619829375183745e-06,
+      "logits/chosen": -2.451979160308838,
+      "logits/rejected": -2.256770610809326,
+      "logps/chosen": -223.16323852539062,
+      "logps/rejected": -218.20315551757812,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.02839289978146553,
+      "rewards/margins": 0.11110793054103851,
+      "rewards/rejected": -0.08271503448486328,
+      "step": 6290
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6569242464906427e-06,
+      "logits/chosen": -2.307574987411499,
+      "logits/rejected": -2.1314024925231934,
+      "logps/chosen": -202.07757568359375,
+      "logps/rejected": -233.34414672851562,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.05651476979255676,
+      "rewards/margins": 0.10217426717281342,
+      "rewards/rejected": -0.04565950110554695,
+      "step": 6300
+    },
+    {
+      "epoch": 0.41,
+      "eval_logits/chosen": -2.3068814277648926,
+      "eval_logits/rejected": -2.120007276535034,
+      "eval_logps/chosen": -227.68896484375,
+      "eval_logps/rejected": -215.39700317382812,
+      "eval_loss": 0.6900160312652588,
+      "eval_rewards/accuracies": 0.6370000243186951,
+      "eval_rewards/chosen": 0.04315978288650513,
+      "eval_rewards/margins": 0.0810108482837677,
+      "eval_rewards/rejected": -0.03785106539726257,
+      "eval_runtime": 711.6068,
+      "eval_samples_per_second": 2.811,
+      "eval_steps_per_second": 1.405,
+      "step": 6300
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6518595203793156e-06,
+      "logits/chosen": -2.1580138206481934,
+      "logits/rejected": -2.1727097034454346,
+      "logps/chosen": -251.7514190673828,
+      "logps/rejected": -275.81744384765625,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.06962844729423523,
+      "rewards/margins": 0.10591252893209457,
+      "rewards/rejected": -0.03628408536314964,
+      "step": 6310
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.646788785604485e-06,
+      "logits/chosen": -2.288708209991455,
+      "logits/rejected": -2.1455512046813965,
+      "logps/chosen": -199.6431884765625,
+      "logps/rejected": -203.20138549804688,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.060151923447847366,
+      "rewards/margins": 0.05803212523460388,
+      "rewards/rejected": 0.002119800541549921,
+      "step": 6320
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.641712068617588e-06,
+      "logits/chosen": -2.30918550491333,
+      "logits/rejected": -2.1908669471740723,
+      "logps/chosen": -255.9428253173828,
+      "logps/rejected": -208.373046875,
+      "loss": 0.693,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.054172348231077194,
+      "rewards/margins": 0.05974752828478813,
+      "rewards/rejected": -0.005575177259743214,
+      "step": 6330
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6366293959012673e-06,
+      "logits/chosen": -2.25718355178833,
+      "logits/rejected": -2.020131826400757,
+      "logps/chosen": -176.4119415283203,
+      "logps/rejected": -165.9830780029297,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.04790060594677925,
+      "rewards/margins": 0.09094378352165222,
+      "rewards/rejected": -0.04304318130016327,
+      "step": 6340
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.631540793969233e-06,
+      "logits/chosen": -2.403151035308838,
+      "logits/rejected": -2.379413604736328,
+      "logps/chosen": -183.47628784179688,
+      "logps/rejected": -190.08969116210938,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.051414500921964645,
+      "rewards/margins": 0.04935113340616226,
+      "rewards/rejected": 0.002063371241092682,
+      "step": 6350
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.626446289366127e-06,
+      "logits/chosen": -2.583311080932617,
+      "logits/rejected": -2.1373889446258545,
+      "logps/chosen": -218.4346160888672,
+      "logps/rejected": -150.66162109375,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.01714567467570305,
+      "rewards/margins": 0.037458065897226334,
+      "rewards/rejected": -0.020312385633587837,
+      "step": 6360
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.6213459086673786e-06,
+      "logits/chosen": -2.3131918907165527,
+      "logits/rejected": -2.375338315963745,
+      "logps/chosen": -168.5972900390625,
+      "logps/rejected": -186.1190185546875,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.02340294048190117,
+      "rewards/margins": 0.08148431777954102,
+      "rewards/rejected": -0.05808137729763985,
+      "step": 6370
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.6162396784790737e-06,
+      "logits/chosen": -2.1972436904907227,
+      "logits/rejected": -2.225222110748291,
+      "logps/chosen": -221.7857666015625,
+      "logps/rejected": -227.5126495361328,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.012212952598929405,
+      "rewards/margins": 0.058864694088697433,
+      "rewards/rejected": -0.04665173962712288,
+      "step": 6380
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.6111276254378095e-06,
+      "logits/chosen": -2.2998032569885254,
+      "logits/rejected": -2.295097827911377,
+      "logps/chosen": -222.982177734375,
+      "logps/rejected": -229.3250732421875,
+      "loss": 0.689,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.04974722862243652,
+      "rewards/margins": 0.10756466537714005,
+      "rewards/rejected": -0.05781743675470352,
+      "step": 6390
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.606009776210559e-06,
+      "logits/chosen": -2.290212392807007,
+      "logits/rejected": -2.0554823875427246,
+      "logps/chosen": -239.64724731445312,
+      "logps/rejected": -205.0599365234375,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.009764440357685089,
+      "rewards/margins": 0.09071458876132965,
+      "rewards/rejected": -0.08095015585422516,
+      "step": 6400
+    },
+    {
+      "epoch": 0.42,
+      "eval_logits/chosen": -2.3204777240753174,
+      "eval_logits/rejected": -2.1320362091064453,
+      "eval_logps/chosen": -229.05621337890625,
+      "eval_logps/rejected": -217.79949951171875,
+      "eval_loss": 0.6900865435600281,
+      "eval_rewards/accuracies": 0.6309999823570251,
+      "eval_rewards/chosen": 0.02948746271431446,
+      "eval_rewards/margins": 0.09136352688074112,
+      "eval_rewards/rejected": -0.06187606602907181,
+      "eval_runtime": 713.0231,
+      "eval_samples_per_second": 2.805,
+      "eval_steps_per_second": 1.402,
+      "step": 6400
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.600886157494531e-06,
+      "logits/chosen": -2.4270999431610107,
+      "logits/rejected": -2.302009105682373,
+      "logps/chosen": -260.2608642578125,
+      "logps/rejected": -257.4969787597656,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.025956381112337112,
+      "rewards/margins": 0.09018175303936005,
+      "rewards/rejected": -0.06422537565231323,
+      "step": 6410
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.5957567960170304e-06,
+      "logits/chosen": -2.5143160820007324,
+      "logits/rejected": -1.7987537384033203,
+      "logps/chosen": -286.97247314453125,
+      "logps/rejected": -178.82583618164062,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.04212607815861702,
+      "rewards/margins": 0.10511051118373871,
+      "rewards/rejected": -0.06298444420099258,
+      "step": 6420
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.590621718535319e-06,
+      "logits/chosen": -2.1314139366149902,
+      "logits/rejected": -1.941144347190857,
+      "logps/chosen": -197.22164916992188,
+      "logps/rejected": -206.35385131835938,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.004992819391191006,
+      "rewards/margins": 0.100247822701931,
+      "rewards/rejected": -0.10524062812328339,
+      "step": 6430
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.5854809518364775e-06,
+      "logits/chosen": -2.3986408710479736,
+      "logits/rejected": -2.087287425994873,
+      "logps/chosen": -241.9314422607422,
+      "logps/rejected": -209.21890258789062,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.04215322062373161,
+      "rewards/margins": 0.11800198256969452,
+      "rewards/rejected": -0.07584875077009201,
+      "step": 6440
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.580334522737262e-06,
+      "logits/chosen": -2.312293529510498,
+      "logits/rejected": -2.025383472442627,
+      "logps/chosen": -197.6604766845703,
+      "logps/rejected": -173.70423889160156,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.04283946007490158,
+      "rewards/margins": 0.09693354368209839,
+      "rewards/rejected": -0.05409408360719681,
+      "step": 6450
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.575182458083968e-06,
+      "logits/chosen": -2.2589616775512695,
+      "logits/rejected": -2.2055575847625732,
+      "logps/chosen": -239.0443878173828,
+      "logps/rejected": -221.5413818359375,
+      "loss": 0.69,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.017606602981686592,
+      "rewards/margins": 0.10502012073993683,
+      "rewards/rejected": -0.08741351217031479,
+      "step": 6460
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.5700247847522883e-06,
+      "logits/chosen": -2.358262062072754,
+      "logits/rejected": -2.2695822715759277,
+      "logps/chosen": -197.18240356445312,
+      "logps/rejected": -207.05166625976562,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.056155987083911896,
+      "rewards/margins": 0.09887482225894928,
+      "rewards/rejected": -0.04271883890032768,
+      "step": 6470
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.5648615296471743e-06,
+      "logits/chosen": -2.153874635696411,
+      "logits/rejected": -2.1209194660186768,
+      "logps/chosen": -193.21182250976562,
+      "logps/rejected": -241.32669067382812,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.04150991886854172,
+      "rewards/margins": 0.10041022300720215,
+      "rewards/rejected": -0.05890030413866043,
+      "step": 6480
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.559692719702693e-06,
+      "logits/chosen": -2.1794726848602295,
+      "logits/rejected": -1.8666191101074219,
+      "logps/chosen": -283.8521728515625,
+      "logps/rejected": -234.9342498779297,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.022929811850190163,
+      "rewards/margins": 0.07811586558818817,
+      "rewards/rejected": -0.05518605187535286,
+      "step": 6490
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.55451838188189e-06,
+      "logits/chosen": -2.293243885040283,
+      "logits/rejected": -2.1898462772369385,
+      "logps/chosen": -257.29473876953125,
+      "logps/rejected": -287.2796630859375,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.03992018103599548,
+      "rewards/margins": 0.08051706850528717,
+      "rewards/rejected": -0.04059688374400139,
+      "step": 6500
+    },
+    {
+      "epoch": 0.43,
+      "eval_logits/chosen": -2.3292880058288574,
+      "eval_logits/rejected": -2.1406772136688232,
+      "eval_logps/chosen": -229.60731506347656,
+      "eval_logps/rejected": -217.20216369628906,
+      "eval_loss": 0.6900708675384521,
+      "eval_rewards/accuracies": 0.6370000243186951,
+      "eval_rewards/chosen": 0.023976394906640053,
+      "eval_rewards/margins": 0.07987881451845169,
+      "eval_rewards/rejected": -0.05590242147445679,
+      "eval_runtime": 713.1525,
+      "eval_samples_per_second": 2.804,
+      "eval_steps_per_second": 1.402,
+      "step": 6500
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.549338543176645e-06,
+      "logits/chosen": -2.3476295471191406,
+      "logits/rejected": -2.080786943435669,
+      "logps/chosen": -302.98834228515625,
+      "logps/rejected": -277.6363220214844,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03500208631157875,
+      "rewards/margins": 0.07001151889562607,
+      "rewards/rejected": -0.035009440034627914,
+      "step": 6510
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5441532306075342e-06,
+      "logits/chosen": -2.294619083404541,
+      "logits/rejected": -2.282924175262451,
+      "logps/chosen": -231.687255859375,
+      "logps/rejected": -283.9492492675781,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.002680377336218953,
+      "rewards/margins": 0.056967295706272125,
+      "rewards/rejected": -0.05964766815304756,
+      "step": 6520
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5389624712236894e-06,
+      "logits/chosen": -2.295729637145996,
+      "logits/rejected": -2.0658695697784424,
+      "logps/chosen": -212.0193328857422,
+      "logps/rejected": -188.84963989257812,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.020164761692285538,
+      "rewards/margins": 0.037581443786621094,
+      "rewards/rejected": -0.017416680231690407,
+      "step": 6530
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.533766292102653e-06,
+      "logits/chosen": -2.2568297386169434,
+      "logits/rejected": -2.117800235748291,
+      "logps/chosen": -210.57046508789062,
+      "logps/rejected": -203.5366973876953,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.007259045727550983,
+      "rewards/margins": 0.04654930159449577,
+      "rewards/rejected": -0.053808342665433884,
+      "step": 6540
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5285647203502404e-06,
+      "logits/chosen": -2.4623608589172363,
+      "logits/rejected": -2.3327670097351074,
+      "logps/chosen": -253.4337615966797,
+      "logps/rejected": -221.22885131835938,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.038372620940208435,
+      "rewards/margins": 0.055611032992601395,
+      "rewards/rejected": -0.01723841205239296,
+      "step": 6550
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5233577831003983e-06,
+      "logits/chosen": -2.287559986114502,
+      "logits/rejected": -2.138993501663208,
+      "logps/chosen": -249.0475616455078,
+      "logps/rejected": -233.41311645507812,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.03128386288881302,
+      "rewards/margins": 0.08034192025661469,
+      "rewards/rejected": -0.04905804991722107,
+      "step": 6560
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5181455075150628e-06,
+      "logits/chosen": -2.2086570262908936,
+      "logits/rejected": -1.8272384405136108,
+      "logps/chosen": -180.17086791992188,
+      "logps/rejected": -139.23220825195312,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0177529938519001,
+      "rewards/margins": 0.05684244632720947,
+      "rewards/rejected": -0.03908945992588997,
+      "step": 6570
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.512927920784016e-06,
+      "logits/chosen": -2.37443208694458,
+      "logits/rejected": -2.232402801513672,
+      "logps/chosen": -221.47036743164062,
+      "logps/rejected": -214.24496459960938,
+      "loss": 0.6877,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.042037345468997955,
+      "rewards/margins": 0.1275465041399002,
+      "rewards/rejected": -0.08550916612148285,
+      "step": 6580
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5077050501247457e-06,
+      "logits/chosen": -2.4588403701782227,
+      "logits/rejected": -2.0066471099853516,
+      "logps/chosen": -274.39154052734375,
+      "logps/rejected": -217.39682006835938,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.06623397767543793,
+      "rewards/margins": 0.10393860191106796,
+      "rewards/rejected": -0.037704624235630035,
+      "step": 6590
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5024769227823042e-06,
+      "logits/chosen": -2.353419065475464,
+      "logits/rejected": -2.2162890434265137,
+      "logps/chosen": -161.44163513183594,
+      "logps/rejected": -128.65968322753906,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.014296752400696278,
+      "rewards/margins": 0.08015719056129456,
+      "rewards/rejected": -0.06586043536663055,
+      "step": 6600
+    },
+    {
+      "epoch": 0.43,
+      "eval_logits/chosen": -2.337346076965332,
+      "eval_logits/rejected": -2.1480324268341064,
+      "eval_logps/chosen": -228.5489959716797,
+      "eval_logps/rejected": -215.88450622558594,
+      "eval_loss": 0.6900503039360046,
+      "eval_rewards/accuracies": 0.6355000138282776,
+      "eval_rewards/chosen": 0.03455958515405655,
+      "eval_rewards/margins": 0.07728561758995056,
+      "eval_rewards/rejected": -0.042726028710603714,
+      "eval_runtime": 711.4362,
+      "eval_samples_per_second": 2.811,
+      "eval_steps_per_second": 1.406,
+      "step": 6600
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.4972435660291646e-06,
+      "logits/chosen": -2.4253835678100586,
+      "logits/rejected": -2.281532049179077,
+      "logps/chosen": -240.89852905273438,
+      "logps/rejected": -223.0607452392578,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.022457608953118324,
+      "rewards/margins": 0.07101938128471375,
+      "rewards/rejected": -0.04856177791953087,
+      "step": 6610
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.492005007165079e-06,
+      "logits/chosen": -2.2975025177001953,
+      "logits/rejected": -2.044618844985962,
+      "logps/chosen": -224.947265625,
+      "logps/rejected": -237.4292449951172,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.04589748755097389,
+      "rewards/margins": 0.06325410306453705,
+      "rewards/rejected": -0.017356622964143753,
+      "step": 6620
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.4867612735169377e-06,
+      "logits/chosen": -2.4469258785247803,
+      "logits/rejected": -2.0971310138702393,
+      "logps/chosen": -220.57882690429688,
+      "logps/rejected": -151.8156280517578,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.044555455446243286,
+      "rewards/margins": 0.09465045481920242,
+      "rewards/rejected": -0.05009499937295914,
+      "step": 6630
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.4815123924386226e-06,
+      "logits/chosen": -2.609863758087158,
+      "logits/rejected": -2.2980103492736816,
+      "logps/chosen": -301.5444030761719,
+      "logps/rejected": -238.43344116210938,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.04996975138783455,
+      "rewards/margins": 0.06561549007892609,
+      "rewards/rejected": -0.015645746141672134,
+      "step": 6640
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4762583913108696e-06,
+      "logits/chosen": -2.1854515075683594,
+      "logits/rejected": -1.9473320245742798,
+      "logps/chosen": -263.75238037109375,
+      "logps/rejected": -230.9041290283203,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.06005977839231491,
+      "rewards/margins": 0.08066942542791367,
+      "rewards/rejected": -0.020609647035598755,
+      "step": 6650
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4709992975411217e-06,
+      "logits/chosen": -2.3034565448760986,
+      "logits/rejected": -1.8144235610961914,
+      "logps/chosen": -250.67794799804688,
+      "logps/rejected": -206.841796875,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.04281472787261009,
+      "rewards/margins": 0.08430864661931992,
+      "rewards/rejected": -0.04149392247200012,
+      "step": 6660
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4657351385633886e-06,
+      "logits/chosen": -2.424379825592041,
+      "logits/rejected": -2.0678696632385254,
+      "logps/chosen": -180.195068359375,
+      "logps/rejected": -179.3686065673828,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.05095613747835159,
+      "rewards/margins": 0.11373704671859741,
+      "rewards/rejected": -0.06278089433908463,
+      "step": 6670
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4604659418381024e-06,
+      "logits/chosen": -2.3288955688476562,
+      "logits/rejected": -1.9227346181869507,
+      "logps/chosen": -218.0121612548828,
+      "logps/rejected": -198.50668334960938,
+      "loss": 0.69,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0004214264336042106,
+      "rewards/margins": 0.10025143623352051,
+      "rewards/rejected": -0.09983000159263611,
+      "step": 6680
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4551917348519744e-06,
+      "logits/chosen": -2.4326155185699463,
+      "logits/rejected": -2.1617472171783447,
+      "logps/chosen": -278.9857482910156,
+      "logps/rejected": -239.697021484375,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.025129491463303566,
+      "rewards/margins": 0.07911841571331024,
+      "rewards/rejected": -0.053988922387361526,
+      "step": 6690
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4499125451178505e-06,
+      "logits/chosen": -2.010340929031372,
+      "logits/rejected": -2.0988070964813232,
+      "logps/chosen": -204.6190643310547,
+      "logps/rejected": -235.9315948486328,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.003277752548456192,
+      "rewards/margins": 0.06699816882610321,
+      "rewards/rejected": -0.0702759176492691,
+      "step": 6700
+    },
+    {
+      "epoch": 0.44,
+      "eval_logits/chosen": -2.3288984298706055,
+      "eval_logits/rejected": -2.1398680210113525,
+      "eval_logps/chosen": -231.943115234375,
+      "eval_logps/rejected": -220.5701446533203,
+      "eval_loss": 0.6900596022605896,
+      "eval_rewards/accuracies": 0.6384999752044678,
+      "eval_rewards/chosen": 0.0006183562218211591,
+      "eval_rewards/margins": 0.09020071476697922,
+      "eval_rewards/rejected": -0.08958234637975693,
+      "eval_runtime": 710.1788,
+      "eval_samples_per_second": 2.816,
+      "eval_steps_per_second": 1.408,
+      "step": 6700
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4446284001745723e-06,
+      "logits/chosen": -2.14957857131958,
+      "logits/rejected": -1.8969390392303467,
+      "logps/chosen": -210.86453247070312,
+      "logps/rejected": -234.8885955810547,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.046308983117341995,
+      "rewards/margins": 0.07277072966098785,
+      "rewards/rejected": -0.11907969415187836,
+      "step": 6710
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.439339327586827e-06,
+      "logits/chosen": -2.2965493202209473,
+      "logits/rejected": -2.2393410205841064,
+      "logps/chosen": -188.0366973876953,
+      "logps/rejected": -191.59263610839844,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.019810201600193977,
+      "rewards/margins": 0.08919601142406464,
+      "rewards/rejected": -0.06938581168651581,
+      "step": 6720
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.434045354945008e-06,
+      "logits/chosen": -2.3867483139038086,
+      "logits/rejected": -2.1257870197296143,
+      "logps/chosen": -275.6525573730469,
+      "logps/rejected": -277.5914611816406,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.0162797961384058,
+      "rewards/margins": 0.06674771010875702,
+      "rewards/rejected": -0.08302750438451767,
+      "step": 6730
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4287465098650713e-06,
+      "logits/chosen": -2.38753604888916,
+      "logits/rejected": -2.387018918991089,
+      "logps/chosen": -249.96890258789062,
+      "logps/rejected": -239.8872833251953,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.01993386633694172,
+      "rewards/margins": 0.04759936034679413,
+      "rewards/rejected": -0.0675332322716713,
+      "step": 6740
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.423442819988387e-06,
+      "logits/chosen": -2.2356374263763428,
+      "logits/rejected": -2.1222212314605713,
+      "logps/chosen": -190.93006896972656,
+      "logps/rejected": -186.69676208496094,
+      "loss": 0.7044,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07542804628610611,
+      "rewards/margins": 0.029561137780547142,
+      "rewards/rejected": -0.1049891859292984,
+      "step": 6750
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4181343129816e-06,
+      "logits/chosen": -2.4288418292999268,
+      "logits/rejected": -2.120914936065674,
+      "logps/chosen": -175.26609802246094,
+      "logps/rejected": -167.46408081054688,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.010955859906971455,
+      "rewards/margins": 0.06488125026226044,
+      "rewards/rejected": -0.053925395011901855,
+      "step": 6760
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4128210165364837e-06,
+      "logits/chosen": -2.168002128601074,
+      "logits/rejected": -2.1744346618652344,
+      "logps/chosen": -187.72097778320312,
+      "logps/rejected": -209.94015502929688,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.03688240796327591,
+      "rewards/margins": 0.13695737719535828,
+      "rewards/rejected": -0.10007498413324356,
+      "step": 6770
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.407502958369795e-06,
+      "logits/chosen": -2.279935836791992,
+      "logits/rejected": -2.1321768760681152,
+      "logps/chosen": -240.8074951171875,
+      "logps/rejected": -220.3392333984375,
+      "loss": 0.6873,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.016402950510382652,
+      "rewards/margins": 0.08744045346975327,
+      "rewards/rejected": -0.07103750109672546,
+      "step": 6780
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4021801662231297e-06,
+      "logits/chosen": -2.3217625617980957,
+      "logits/rejected": -2.168172836303711,
+      "logps/chosen": -249.3394317626953,
+      "logps/rejected": -232.941650390625,
+      "loss": 0.692,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.004626649431884289,
+      "rewards/margins": 0.05877614766359329,
+      "rewards/rejected": -0.06340280920267105,
+      "step": 6790
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.3968526678627793e-06,
+      "logits/chosen": -2.2708561420440674,
+      "logits/rejected": -1.9311338663101196,
+      "logps/chosen": -265.8199462890625,
+      "logps/rejected": -227.452392578125,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.007894573733210564,
+      "rewards/margins": 0.0573604479432106,
+      "rewards/rejected": -0.06525502353906631,
+      "step": 6800
+    },
+    {
+      "epoch": 0.44,
+      "eval_logits/chosen": -2.326197862625122,
+      "eval_logits/rejected": -2.137258529663086,
+      "eval_logps/chosen": -233.22720336914062,
+      "eval_logps/rejected": -221.09767150878906,
+      "eval_loss": 0.6899824142456055,
+      "eval_rewards/accuracies": 0.6345000267028809,
+      "eval_rewards/chosen": -0.012222343124449253,
+      "eval_rewards/margins": 0.08263525366783142,
+      "eval_rewards/rejected": -0.0948575884103775,
+      "eval_runtime": 709.2866,
+      "eval_samples_per_second": 2.82,
+      "eval_steps_per_second": 1.41,
+      "step": 6800
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.391520491079586e-06,
+      "logits/chosen": -2.4146409034729004,
+      "logits/rejected": -2.4289791584014893,
+      "logps/chosen": -196.1842498779297,
+      "logps/rejected": -171.66029357910156,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0022164147812873125,
+      "rewards/margins": 0.06005682423710823,
+      "rewards/rejected": -0.062273234128952026,
+      "step": 6810
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3861836636887936e-06,
+      "logits/chosen": -2.3121285438537598,
+      "logits/rejected": -2.129061222076416,
+      "logps/chosen": -267.5245361328125,
+      "logps/rejected": -231.2607879638672,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.007056675851345062,
+      "rewards/margins": 0.10214730352163315,
+      "rewards/rejected": -0.1092039942741394,
+      "step": 6820
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3808422135299106e-06,
+      "logits/chosen": -2.2894935607910156,
+      "logits/rejected": -2.3739147186279297,
+      "logps/chosen": -286.0987243652344,
+      "logps/rejected": -346.9412536621094,
+      "loss": 0.691,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.004209049511700869,
+      "rewards/margins": 0.07767010480165482,
+      "rewards/rejected": -0.0818791538476944,
+      "step": 6830
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.375496168466556e-06,
+      "logits/chosen": -2.468285083770752,
+      "logits/rejected": -2.0140810012817383,
+      "logps/chosen": -214.4819793701172,
+      "logps/rejected": -164.49362182617188,
+      "loss": 0.691,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.004561436362564564,
+      "rewards/margins": 0.05938258767127991,
+      "rewards/rejected": -0.05482115224003792,
+      "step": 6840
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3701455563863205e-06,
+      "logits/chosen": -2.5253567695617676,
+      "logits/rejected": -2.0757806301116943,
+      "logps/chosen": -294.3965148925781,
+      "logps/rejected": -268.2409362792969,
+      "loss": 0.6867,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.025218481197953224,
+      "rewards/margins": 0.1016344428062439,
+      "rewards/rejected": -0.12685291469097137,
+      "step": 6850
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3647904052006174e-06,
+      "logits/chosen": -2.3288276195526123,
+      "logits/rejected": -2.264437675476074,
+      "logps/chosen": -267.79376220703125,
+      "logps/rejected": -278.85662841796875,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.024954695254564285,
+      "rewards/margins": 0.08056318014860153,
+      "rewards/rejected": -0.10551787912845612,
+      "step": 6860
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3594307428445383e-06,
+      "logits/chosen": -2.529595375061035,
+      "logits/rejected": -2.1268649101257324,
+      "logps/chosen": -333.0338439941406,
+      "logps/rejected": -309.6853942871094,
+      "loss": 0.691,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.0021690779831260443,
+      "rewards/margins": 0.08068062365055084,
+      "rewards/rejected": -0.0785115510225296,
+      "step": 6870
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.354066597276707e-06,
+      "logits/chosen": -2.184455394744873,
+      "logits/rejected": -2.18107533454895,
+      "logps/chosen": -224.90921020507812,
+      "logps/rejected": -266.7405700683594,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.013449070043861866,
+      "rewards/margins": 0.0647086501121521,
+      "rewards/rejected": -0.07815771549940109,
+      "step": 6880
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.348697996479136e-06,
+      "logits/chosen": -2.405324935913086,
+      "logits/rejected": -2.148305654525757,
+      "logps/chosen": -228.61373901367188,
+      "logps/rejected": -182.25930786132812,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.008047891780734062,
+      "rewards/margins": 0.07117293030023575,
+      "rewards/rejected": -0.07922081649303436,
+      "step": 6890
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3433249684570757e-06,
+      "logits/chosen": -2.224512815475464,
+      "logits/rejected": -2.0861592292785645,
+      "logps/chosen": -178.4225616455078,
+      "logps/rejected": -142.22152709960938,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.00593004934489727,
+      "rewards/margins": 0.09517361223697662,
+      "rewards/rejected": -0.10110366344451904,
+      "step": 6900
+    },
+    {
+      "epoch": 0.45,
+      "eval_logits/chosen": -2.3220698833465576,
+      "eval_logits/rejected": -2.133631944656372,
+      "eval_logps/chosen": -231.9954071044922,
+      "eval_logps/rejected": -219.68096923828125,
+      "eval_loss": 0.6899513006210327,
+      "eval_rewards/accuracies": 0.6309999823570251,
+      "eval_rewards/chosen": 9.537512232782319e-05,
+      "eval_rewards/margins": 0.08078599721193314,
+      "eval_rewards/rejected": -0.08069062978029251,
+      "eval_runtime": 712.9402,
+      "eval_samples_per_second": 2.805,
+      "eval_steps_per_second": 1.403,
+      "step": 6900
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3379475412388724e-06,
+      "logits/chosen": -2.405919075012207,
+      "logits/rejected": -2.2413978576660156,
+      "logps/chosen": -240.8171844482422,
+      "logps/rejected": -219.07095336914062,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.01137818954885006,
+      "rewards/margins": 0.11952348798513412,
+      "rewards/rejected": -0.1081453189253807,
+      "step": 6910
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3325657428758207e-06,
+      "logits/chosen": -2.2108242511749268,
+      "logits/rejected": -2.1874756813049316,
+      "logps/chosen": -241.8357696533203,
+      "logps/rejected": -253.5496063232422,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.018397245556116104,
+      "rewards/margins": 0.10887129604816437,
+      "rewards/rejected": -0.09047403931617737,
+      "step": 6920
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3271796014420175e-06,
+      "logits/chosen": -2.3145766258239746,
+      "logits/rejected": -2.254108428955078,
+      "logps/chosen": -214.22042846679688,
+      "logps/rejected": -208.04830932617188,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.01770118437707424,
+      "rewards/margins": 0.11536221206188202,
+      "rewards/rejected": -0.133063405752182,
+      "step": 6930
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3217891450342142e-06,
+      "logits/chosen": -2.3170406818389893,
+      "logits/rejected": -1.9900795221328735,
+      "logps/chosen": -259.0184020996094,
+      "logps/rejected": -200.655029296875,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.004199774004518986,
+      "rewards/margins": 0.10629435628652573,
+      "rewards/rejected": -0.11049413681030273,
+      "step": 6940
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3163944017716733e-06,
+      "logits/chosen": -2.4827568531036377,
+      "logits/rejected": -2.180410623550415,
+      "logps/chosen": -215.23617553710938,
+      "logps/rejected": -189.19430541992188,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.01134683657437563,
+      "rewards/margins": 0.07055879384279251,
+      "rewards/rejected": -0.05921195074915886,
+      "step": 6950
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.310995399796017e-06,
+      "logits/chosen": -2.421800374984741,
+      "logits/rejected": -2.30001163482666,
+      "logps/chosen": -274.8177185058594,
+      "logps/rejected": -272.7926940917969,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.008314954116940498,
+      "rewards/margins": 0.047770872712135315,
+      "rewards/rejected": -0.05608583241701126,
+      "step": 6960
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.305592167271085e-06,
+      "logits/chosen": -2.302924871444702,
+      "logits/rejected": -2.2044272422790527,
+      "logps/chosen": -194.39797973632812,
+      "logps/rejected": -194.43634033203125,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.028481673449277878,
+      "rewards/margins": 0.10950399935245514,
+      "rewards/rejected": -0.08102231472730637,
+      "step": 6970
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.3001847323827846e-06,
+      "logits/chosen": -2.295625925064087,
+      "logits/rejected": -2.242480516433716,
+      "logps/chosen": -266.0675354003906,
+      "logps/rejected": -272.8329162597656,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.020301584154367447,
+      "rewards/margins": 0.10184173285961151,
+      "rewards/rejected": -0.08154015243053436,
+      "step": 6980
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2947731233389447e-06,
+      "logits/chosen": -2.3801562786102295,
+      "logits/rejected": -2.004241943359375,
+      "logps/chosen": -247.1204376220703,
+      "logps/rejected": -202.22914123535156,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0038624543230980635,
+      "rewards/margins": 0.10536074638366699,
+      "rewards/rejected": -0.10149829089641571,
+      "step": 6990
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2893573683691706e-06,
+      "logits/chosen": -2.195082902908325,
+      "logits/rejected": -2.2036478519439697,
+      "logps/chosen": -204.31845092773438,
+      "logps/rejected": -198.5324249267578,
+      "loss": 0.688,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.003750187810510397,
+      "rewards/margins": 0.12109758704900742,
+      "rewards/rejected": -0.11734740436077118,
+      "step": 7000
+    },
+    {
+      "epoch": 0.46,
+      "eval_logits/chosen": -2.3213798999786377,
+      "eval_logits/rejected": -2.132957935333252,
+      "eval_logps/chosen": -232.3555450439453,
+      "eval_logps/rejected": -220.5653533935547,
+      "eval_loss": 0.689961850643158,
+      "eval_rewards/accuracies": 0.6255000233650208,
+      "eval_rewards/chosen": -0.00350601295940578,
+      "eval_rewards/margins": 0.08602865040302277,
+      "eval_rewards/rejected": -0.08953466266393661,
+      "eval_runtime": 712.7907,
+      "eval_samples_per_second": 2.806,
+      "eval_steps_per_second": 1.403,
+      "step": 7000
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2839374957246915e-06,
+      "logits/chosen": -2.4078869819641113,
+      "logits/rejected": -2.1226019859313965,
+      "logps/chosen": -268.3845520019531,
+      "logps/rejected": -181.95008850097656,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.022942641749978065,
+      "rewards/margins": 0.051037006080150604,
+      "rewards/rejected": -0.07397964596748352,
+      "step": 7010
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2785135336782187e-06,
+      "logits/chosen": -2.250103712081909,
+      "logits/rejected": -2.076584577560425,
+      "logps/chosen": -234.687744140625,
+      "logps/rejected": -267.17193603515625,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.007723203394562006,
+      "rewards/margins": 0.10171866416931152,
+      "rewards/rejected": -0.10944187641143799,
+      "step": 7020
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2730855105237952e-06,
+      "logits/chosen": -2.4183948040008545,
+      "logits/rejected": -2.2227070331573486,
+      "logps/chosen": -218.42391967773438,
+      "logps/rejected": -272.3433532714844,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.018525656312704086,
+      "rewards/margins": 0.09738930314779282,
+      "rewards/rejected": -0.07886365056037903,
+      "step": 7030
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2676534545766486e-06,
+      "logits/chosen": -2.2223055362701416,
+      "logits/rejected": -2.1927051544189453,
+      "logps/chosen": -211.2403106689453,
+      "logps/rejected": -206.35842895507812,
+      "loss": 0.691,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.012292217463254929,
+      "rewards/margins": 0.05010954663157463,
+      "rewards/rejected": -0.037817325443029404,
+      "step": 7040
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.262217394173043e-06,
+      "logits/chosen": -2.335088014602661,
+      "logits/rejected": -2.052690267562866,
+      "logps/chosen": -242.60922241210938,
+      "logps/rejected": -244.936279296875,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.017585698515176773,
+      "rewards/margins": 0.08410107344388962,
+      "rewards/rejected": -0.10168677568435669,
+      "step": 7050
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2567773576701333e-06,
+      "logits/chosen": -2.157353162765503,
+      "logits/rejected": -1.9850852489471436,
+      "logps/chosen": -257.1738586425781,
+      "logps/rejected": -238.9078826904297,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.014163834974169731,
+      "rewards/margins": 0.1204962283372879,
+      "rewards/rejected": -0.1346600502729416,
+      "step": 7060
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2513333734458154e-06,
+      "logits/chosen": -2.3726444244384766,
+      "logits/rejected": -2.276071310043335,
+      "logps/chosen": -207.281982421875,
+      "logps/rejected": -197.08543395996094,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.004406812135130167,
+      "rewards/margins": 0.06411116570234299,
+      "rewards/rejected": -0.06851796805858612,
+      "step": 7070
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.245885469898576e-06,
+      "logits/chosen": -2.2665371894836426,
+      "logits/rejected": -2.051095962524414,
+      "logps/chosen": -300.17071533203125,
+      "logps/rejected": -246.3044891357422,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.003844373393803835,
+      "rewards/margins": 0.09821848571300507,
+      "rewards/rejected": -0.09437411278486252,
+      "step": 7080
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2404336754473497e-06,
+      "logits/chosen": -2.263822317123413,
+      "logits/rejected": -2.0003104209899902,
+      "logps/chosen": -265.4076232910156,
+      "logps/rejected": -207.5909423828125,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.020993733778595924,
+      "rewards/margins": 0.069535993039608,
+      "rewards/rejected": -0.04854225367307663,
+      "step": 7090
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.234978018531367e-06,
+      "logits/chosen": -2.587207555770874,
+      "logits/rejected": -2.156212329864502,
+      "logps/chosen": -256.6408996582031,
+      "logps/rejected": -198.82559204101562,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.018593590706586838,
+      "rewards/margins": 0.06984353065490723,
+      "rewards/rejected": -0.05124994367361069,
+      "step": 7100
+    },
+    {
+      "epoch": 0.46,
+      "eval_logits/chosen": -2.31289005279541,
+      "eval_logits/rejected": -2.125515937805176,
+      "eval_logps/chosen": -231.62696838378906,
+      "eval_logps/rejected": -219.47421264648438,
+      "eval_loss": 0.689959704875946,
+      "eval_rewards/accuracies": 0.6309999823570251,
+      "eval_rewards/chosen": 0.0037797146942466497,
+      "eval_rewards/margins": 0.08240301162004471,
+      "eval_rewards/rejected": -0.07862330228090286,
+      "eval_runtime": 712.5102,
+      "eval_samples_per_second": 2.807,
+      "eval_steps_per_second": 1.403,
+      "step": 7100
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.229518527610006e-06,
+      "logits/chosen": -2.405752182006836,
+      "logits/rejected": -2.1117520332336426,
+      "logps/chosen": -291.0060119628906,
+      "logps/rejected": -252.95669555664062,
+      "loss": 0.691,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.017840737476944923,
+      "rewards/margins": 0.07126693427562714,
+      "rewards/rejected": -0.05342619866132736,
+      "step": 7110
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.2240552311626465e-06,
+      "logits/chosen": -2.3919520378112793,
+      "logits/rejected": -2.1560964584350586,
+      "logps/chosen": -239.92935180664062,
+      "logps/rejected": -231.94467163085938,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.012844247743487358,
+      "rewards/margins": 0.06960373371839523,
+      "rewards/rejected": -0.056759487837553024,
+      "step": 7120
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.2185881576885193e-06,
+      "logits/chosen": -2.403320550918579,
+      "logits/rejected": -2.052623748779297,
+      "logps/chosen": -211.9319305419922,
+      "logps/rejected": -180.37948608398438,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.0020910645835101604,
+      "rewards/margins": 0.07115035504102707,
+      "rewards/rejected": -0.07324142754077911,
+      "step": 7130
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.213117335706557e-06,
+      "logits/chosen": -2.279644727706909,
+      "logits/rejected": -2.392721176147461,
+      "logps/chosen": -256.3946838378906,
+      "logps/rejected": -271.6476135253906,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0034970403648912907,
+      "rewards/margins": 0.06046708673238754,
+      "rewards/rejected": -0.0639641284942627,
+      "step": 7140
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.2076427937552473e-06,
+      "logits/chosen": -2.3162059783935547,
+      "logits/rejected": -2.0917744636535645,
+      "logps/chosen": -244.4017791748047,
+      "logps/rejected": -236.8706512451172,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.010880010202527046,
+      "rewards/margins": 0.11592531204223633,
+      "rewards/rejected": -0.10504531860351562,
+      "step": 7150
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.2021645603924827e-06,
+      "logits/chosen": -2.1680960655212402,
+      "logits/rejected": -2.0739083290100098,
+      "logps/chosen": -137.6443634033203,
+      "logps/rejected": -159.3563995361328,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.008097528479993343,
+      "rewards/margins": 0.12347575277090073,
+      "rewards/rejected": -0.11537822335958481,
+      "step": 7160
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.196682664195412e-06,
+      "logits/chosen": -2.2917988300323486,
+      "logits/rejected": -2.0159618854522705,
+      "logps/chosen": -205.24533081054688,
+      "logps/rejected": -175.20578002929688,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.011971285566687584,
+      "rewards/margins": 0.048515014350414276,
+      "rewards/rejected": -0.06048629805445671,
+      "step": 7170
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.191197133760291e-06,
+      "logits/chosen": -2.5555553436279297,
+      "logits/rejected": -2.229135751724243,
+      "logps/chosen": -262.1866760253906,
+      "logps/rejected": -200.24937438964844,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.052167803049087524,
+      "rewards/margins": 0.1231146901845932,
+      "rewards/rejected": -0.07094688713550568,
+      "step": 7180
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.185707997702334e-06,
+      "logits/chosen": -2.212904453277588,
+      "logits/rejected": -2.0473551750183105,
+      "logps/chosen": -240.8466033935547,
+      "logps/rejected": -211.2018585205078,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0008177075651474297,
+      "rewards/margins": 0.08206796646118164,
+      "rewards/rejected": -0.08288567513227463,
+      "step": 7190
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.1802152846555624e-06,
+      "logits/chosen": -2.245023727416992,
+      "logits/rejected": -2.220525026321411,
+      "logps/chosen": -222.89584350585938,
+      "logps/rejected": -215.0545654296875,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.013471787795424461,
+      "rewards/margins": 0.07678040862083435,
+      "rewards/rejected": -0.06330861151218414,
+      "step": 7200
+    },
+    {
+      "epoch": 0.47,
+      "eval_logits/chosen": -2.3261678218841553,
+      "eval_logits/rejected": -2.1375534534454346,
+      "eval_logps/chosen": -230.54725646972656,
+      "eval_logps/rejected": -217.60206604003906,
+      "eval_loss": 0.6899964213371277,
+      "eval_rewards/accuracies": 0.621999979019165,
+      "eval_rewards/chosen": 0.014577223919332027,
+      "eval_rewards/margins": 0.07447873055934906,
+      "eval_rewards/rejected": -0.059901509433984756,
+      "eval_runtime": 713.9129,
+      "eval_samples_per_second": 2.801,
+      "eval_steps_per_second": 1.401,
+      "step": 7200
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.174719023272659e-06,
+      "logits/chosen": -2.4018406867980957,
+      "logits/rejected": -2.4472343921661377,
+      "logps/chosen": -212.6780242919922,
+      "logps/rejected": -267.15069580078125,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.010161024518311024,
+      "rewards/margins": 0.08123484998941422,
+      "rewards/rejected": -0.07107381522655487,
+      "step": 7210
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.169219242224816e-06,
+      "logits/chosen": -2.3529484272003174,
+      "logits/rejected": -2.1854114532470703,
+      "logps/chosen": -240.903564453125,
+      "logps/rejected": -241.4931640625,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.00976332277059555,
+      "rewards/margins": 0.06684872508049011,
+      "rewards/rejected": -0.07661206275224686,
+      "step": 7220
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.1637159702015837e-06,
+      "logits/chosen": -2.3580760955810547,
+      "logits/rejected": -2.0424869060516357,
+      "logps/chosen": -205.4845428466797,
+      "logps/rejected": -197.5430450439453,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.004075945820659399,
+      "rewards/margins": 0.11381276696920395,
+      "rewards/rejected": -0.10973681509494781,
+      "step": 7230
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.1582092359107263e-06,
+      "logits/chosen": -2.4041996002197266,
+      "logits/rejected": -2.184713840484619,
+      "logps/chosen": -278.28271484375,
+      "logps/rejected": -241.3439483642578,
+      "loss": 0.6935,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.9874186515808105e-05,
+      "rewards/margins": 0.08891940116882324,
+      "rewards/rejected": -0.08896928280591965,
+      "step": 7240
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.152699068078067e-06,
+      "logits/chosen": -2.2145161628723145,
+      "logits/rejected": -2.00947904586792,
+      "logps/chosen": -284.87933349609375,
+      "logps/rejected": -276.2130432128906,
+      "loss": 0.6879,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.045890286564826965,
+      "rewards/margins": 0.11874841153621674,
+      "rewards/rejected": -0.1646386981010437,
+      "step": 7250
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1471854954473415e-06,
+      "logits/chosen": -2.354721784591675,
+      "logits/rejected": -2.3982090950012207,
+      "logps/chosen": -248.9252471923828,
+      "logps/rejected": -255.275634765625,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.018414465710520744,
+      "rewards/margins": 0.10118373483419418,
+      "rewards/rejected": -0.08276927471160889,
+      "step": 7260
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1416685467800436e-06,
+      "logits/chosen": -2.1534171104431152,
+      "logits/rejected": -2.116290330886841,
+      "logps/chosen": -188.9571990966797,
+      "logps/rejected": -176.0358428955078,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.03374389559030533,
+      "rewards/margins": 0.09608308225870132,
+      "rewards/rejected": -0.12982699275016785,
+      "step": 7270
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1361482508552803e-06,
+      "logits/chosen": -2.3551740646362305,
+      "logits/rejected": -1.8543596267700195,
+      "logps/chosen": -242.98074340820312,
+      "logps/rejected": -206.5134735107422,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.010297993198037148,
+      "rewards/margins": 0.06919042766094208,
+      "rewards/rejected": -0.07948841899633408,
+      "step": 7280
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1306246364696198e-06,
+      "logits/chosen": -2.481549024581909,
+      "logits/rejected": -2.280365228652954,
+      "logps/chosen": -251.1072235107422,
+      "logps/rejected": -242.42062377929688,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.013671429827809334,
+      "rewards/margins": 0.08818572014570236,
+      "rewards/rejected": -0.07451429218053818,
+      "step": 7290
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1250977324369413e-06,
+      "logits/chosen": -2.2711830139160156,
+      "logits/rejected": -2.1912612915039062,
+      "logps/chosen": -157.5973663330078,
+      "logps/rejected": -173.55458068847656,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.013550999574363232,
+      "rewards/margins": 0.09245215356349945,
+      "rewards/rejected": -0.07890114188194275,
+      "step": 7300
+    },
+    {
+      "epoch": 0.48,
+      "eval_logits/chosen": -2.314774990081787,
+      "eval_logits/rejected": -2.127025604248047,
+      "eval_logps/chosen": -232.7455596923828,
+      "eval_logps/rejected": -220.20619201660156,
+      "eval_loss": 0.6899450421333313,
+      "eval_rewards/accuracies": 0.6290000081062317,
+      "eval_rewards/chosen": -0.007406196556985378,
+      "eval_rewards/margins": 0.07853667438030243,
+      "eval_rewards/rejected": -0.08594285696744919,
+      "eval_runtime": 714.0048,
+      "eval_samples_per_second": 2.801,
+      "eval_steps_per_second": 1.401,
+      "step": 7300
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1195675675882825e-06,
+      "logits/chosen": -2.2243969440460205,
+      "logits/rejected": -2.1143479347229004,
+      "logps/chosen": -238.1915740966797,
+      "logps/rejected": -213.28317260742188,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.030005771666765213,
+      "rewards/margins": 0.05696401745080948,
+      "rewards/rejected": -0.0869697779417038,
+      "step": 7310
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1140341707716926e-06,
+      "logits/chosen": -2.176473617553711,
+      "logits/rejected": -1.929321527481079,
+      "logps/chosen": -196.3441162109375,
+      "logps/rejected": -165.09817504882812,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.005364052020013332,
+      "rewards/margins": 0.11770284175872803,
+      "rewards/rejected": -0.12306687980890274,
+      "step": 7320
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1084975708520803e-06,
+      "logits/chosen": -2.4346864223480225,
+      "logits/rejected": -2.034972906112671,
+      "logps/chosen": -260.06732177734375,
+      "logps/rejected": -198.18801879882812,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.016565248370170593,
+      "rewards/margins": 0.08867197483778,
+      "rewards/rejected": -0.0721067264676094,
+      "step": 7330
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1029577967110625e-06,
+      "logits/chosen": -2.4259464740753174,
+      "logits/rejected": -2.2102203369140625,
+      "logps/chosen": -216.46865844726562,
+      "logps/rejected": -171.20114135742188,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0007433153805322945,
+      "rewards/margins": 0.03790837153792381,
+      "rewards/rejected": -0.038651686161756516,
+      "step": 7340
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.097414877246814e-06,
+      "logits/chosen": -2.2673535346984863,
+      "logits/rejected": -2.0183987617492676,
+      "logps/chosen": -197.57110595703125,
+      "logps/rejected": -177.40805053710938,
+      "loss": 0.6862,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.010723413899540901,
+      "rewards/margins": 0.09349813312292099,
+      "rewards/rejected": -0.08277471363544464,
+      "step": 7350
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.0918688413739197e-06,
+      "logits/chosen": -2.3423869609832764,
+      "logits/rejected": -2.030740261077881,
+      "logps/chosen": -229.8058319091797,
+      "logps/rejected": -176.20933532714844,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.028771137818694115,
+      "rewards/margins": 0.10676223039627075,
+      "rewards/rejected": -0.07799109816551208,
+      "step": 7360
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.0863197180232178e-06,
+      "logits/chosen": -2.3902642726898193,
+      "logits/rejected": -2.0197548866271973,
+      "logps/chosen": -197.1629638671875,
+      "logps/rejected": -192.62374877929688,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.025812137871980667,
+      "rewards/margins": 0.07681456953287125,
+      "rewards/rejected": -0.051002420485019684,
+      "step": 7370
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.0807675361416554e-06,
+      "logits/chosen": -2.2906887531280518,
+      "logits/rejected": -2.047722339630127,
+      "logps/chosen": -186.08908081054688,
+      "logps/rejected": -114.9020004272461,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.02791173756122589,
+      "rewards/margins": 0.0784730315208435,
+      "rewards/rejected": -0.050561290234327316,
+      "step": 7380
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.0752123246921327e-06,
+      "logits/chosen": -2.4102749824523926,
+      "logits/rejected": -2.1352226734161377,
+      "logps/chosen": -277.206298828125,
+      "logps/rejected": -218.3984832763672,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.007069566752761602,
+      "rewards/margins": 0.08123649656772614,
+      "rewards/rejected": -0.0741669163107872,
+      "step": 7390
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.069654112653353e-06,
+      "logits/chosen": -2.4256751537323,
+      "logits/rejected": -2.2184150218963623,
+      "logps/chosen": -216.2940216064453,
+      "logps/rejected": -188.50967407226562,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0019019130850210786,
+      "rewards/margins": 0.04767733812332153,
+      "rewards/rejected": -0.04577542468905449,
+      "step": 7400
+    },
+    {
+      "epoch": 0.48,
+      "eval_logits/chosen": -2.311318874359131,
+      "eval_logits/rejected": -2.123793601989746,
+      "eval_logps/chosen": -231.12094116210938,
+      "eval_logps/rejected": -218.42494201660156,
+      "eval_loss": 0.6899686455726624,
+      "eval_rewards/accuracies": 0.6284999847412109,
+      "eval_rewards/chosen": 0.008840080350637436,
+      "eval_rewards/margins": 0.07697049528360367,
+      "eval_rewards/rejected": -0.06813041865825653,
+      "eval_runtime": 713.3758,
+      "eval_samples_per_second": 2.804,
+      "eval_steps_per_second": 1.402,
+      "step": 7400
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.064092929019673e-06,
+      "logits/chosen": -2.3009819984436035,
+      "logits/rejected": -2.3356680870056152,
+      "logps/chosen": -256.2542419433594,
+      "logps/rejected": -281.6364440917969,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.012892307713627815,
+      "rewards/margins": 0.05507419630885124,
+      "rewards/rejected": -0.04218188300728798,
+      "step": 7410
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.058528802800952e-06,
+      "logits/chosen": -2.342904567718506,
+      "logits/rejected": -2.102327823638916,
+      "logps/chosen": -290.8139953613281,
+      "logps/rejected": -261.8834228515625,
+      "loss": 0.691,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.01819724775850773,
+      "rewards/margins": 0.07642240822315216,
+      "rewards/rejected": -0.05822516605257988,
+      "step": 7420
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.052961763022397e-06,
+      "logits/chosen": -2.481123685836792,
+      "logits/rejected": -2.149416208267212,
+      "logps/chosen": -183.3396453857422,
+      "logps/rejected": -155.44528198242188,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.004584819078445435,
+      "rewards/margins": 0.10829279571771622,
+      "rewards/rejected": -0.11287760734558105,
+      "step": 7430
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.047391838724415e-06,
+      "logits/chosen": -2.444658041000366,
+      "logits/rejected": -1.982791543006897,
+      "logps/chosen": -232.76083374023438,
+      "logps/rejected": -227.51760864257812,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.02757873572409153,
+      "rewards/margins": 0.10823357105255127,
+      "rewards/rejected": -0.08065483719110489,
+      "step": 7440
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0418190589624587e-06,
+      "logits/chosen": -2.3566372394561768,
+      "logits/rejected": -2.09330153465271,
+      "logps/chosen": -178.53126525878906,
+      "logps/rejected": -192.35299682617188,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.009091891348361969,
+      "rewards/margins": 0.05469350144267082,
+      "rewards/rejected": -0.06378540396690369,
+      "step": 7450
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0362434528068784e-06,
+      "logits/chosen": -2.3358893394470215,
+      "logits/rejected": -1.9141845703125,
+      "logps/chosen": -268.863037109375,
+      "logps/rejected": -194.3677978515625,
+      "loss": 0.688,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.004601252265274525,
+      "rewards/margins": 0.10998895019292831,
+      "rewards/rejected": -0.10538768768310547,
+      "step": 7460
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0306650493427657e-06,
+      "logits/chosen": -2.2316243648529053,
+      "logits/rejected": -2.127760410308838,
+      "logps/chosen": -230.8894500732422,
+      "logps/rejected": -230.5992889404297,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.007297568954527378,
+      "rewards/margins": 0.08671748638153076,
+      "rewards/rejected": -0.07941991835832596,
+      "step": 7470
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0250838776698077e-06,
+      "logits/chosen": -2.096904754638672,
+      "logits/rejected": -2.1422505378723145,
+      "logps/chosen": -184.96865844726562,
+      "logps/rejected": -193.18240356445312,
+      "loss": 0.6862,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.010493558831512928,
+      "rewards/margins": 0.10960110276937485,
+      "rewards/rejected": -0.1200946569442749,
+      "step": 7480
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0194999669021275e-06,
+      "logits/chosen": -2.098390579223633,
+      "logits/rejected": -1.7727285623550415,
+      "logps/chosen": -226.0522003173828,
+      "logps/rejected": -189.67776489257812,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.008162322454154491,
+      "rewards/margins": 0.10677523910999298,
+      "rewards/rejected": -0.09861291944980621,
+      "step": 7490
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0139133461681403e-06,
+      "logits/chosen": -2.243513822555542,
+      "logits/rejected": -2.0963521003723145,
+      "logps/chosen": -263.7023010253906,
+      "logps/rejected": -215.9556121826172,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.006919173989444971,
+      "rewards/margins": 0.08393418788909912,
+      "rewards/rejected": -0.07701500505208969,
+      "step": 7500
+    },
+    {
+      "epoch": 0.49,
+      "eval_logits/chosen": -2.286125898361206,
+      "eval_logits/rejected": -2.1006903648376465,
+      "eval_logps/chosen": -231.99969482421875,
+      "eval_logps/rejected": -219.49575805664062,
+      "eval_loss": 0.6899300813674927,
+      "eval_rewards/accuracies": 0.628000020980835,
+      "eval_rewards/chosen": 5.265325307846069e-05,
+      "eval_rewards/margins": 0.07889124006032944,
+      "eval_rewards/rejected": -0.07883859425783157,
+      "eval_runtime": 711.1311,
+      "eval_samples_per_second": 2.812,
+      "eval_steps_per_second": 1.406,
+      "step": 7500
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0083240446103965e-06,
+      "logits/chosen": -2.0148041248321533,
+      "logits/rejected": -1.978687047958374,
+      "logps/chosen": -184.3916015625,
+      "logps/rejected": -200.95066833496094,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.0035699151922017336,
+      "rewards/margins": 0.08889930695295334,
+      "rewards/rejected": -0.0924692153930664,
+      "step": 7510
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0027320913854306e-06,
+      "logits/chosen": -2.4875292778015137,
+      "logits/rejected": -2.200932025909424,
+      "logps/chosen": -291.66192626953125,
+      "logps/rejected": -237.73507690429688,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.007996944710612297,
+      "rewards/margins": 0.08782526105642319,
+      "rewards/rejected": -0.07982831448316574,
+      "step": 7520
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.997137515663609e-06,
+      "logits/chosen": -2.2359402179718018,
+      "logits/rejected": -2.1508307456970215,
+      "logps/chosen": -223.72048950195312,
+      "logps/rejected": -195.07565307617188,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.01656939461827278,
+      "rewards/margins": 0.09054360538721085,
+      "rewards/rejected": -0.07397421449422836,
+      "step": 7530
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.991540346628981e-06,
+      "logits/chosen": -2.342252492904663,
+      "logits/rejected": -2.15889573097229,
+      "logps/chosen": -238.47000122070312,
+      "logps/rejected": -218.312744140625,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.011303565464913845,
+      "rewards/margins": 0.05950998514890671,
+      "rewards/rejected": -0.048206426203250885,
+      "step": 7540
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.985940613479121e-06,
+      "logits/chosen": -2.4330556392669678,
+      "logits/rejected": -2.323356866836548,
+      "logps/chosen": -292.80767822265625,
+      "logps/rejected": -241.87033081054688,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.010419869795441628,
+      "rewards/margins": 0.08434535562992096,
+      "rewards/rejected": -0.07392548024654388,
+      "step": 7550
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.980338345424981e-06,
+      "logits/chosen": -2.2963860034942627,
+      "logits/rejected": -1.9866485595703125,
+      "logps/chosen": -248.8218536376953,
+      "logps/rejected": -206.16690063476562,
+      "loss": 0.691,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.003979469649493694,
+      "rewards/margins": 0.05621809884905815,
+      "rewards/rejected": -0.05223863199353218,
+      "step": 7560
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.974733571690735e-06,
+      "logits/chosen": -2.3758111000061035,
+      "logits/rejected": -2.0978314876556396,
+      "logps/chosen": -238.9188232421875,
+      "logps/rejected": -186.61029052734375,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.015120044350624084,
+      "rewards/margins": 0.0698397159576416,
+      "rewards/rejected": -0.08495976030826569,
+      "step": 7570
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9691263215136274e-06,
+      "logits/chosen": -2.339653730392456,
+      "logits/rejected": -2.3221707344055176,
+      "logps/chosen": -263.97918701171875,
+      "logps/rejected": -240.21945190429688,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03021194040775299,
+      "rewards/margins": 0.07894815504550934,
+      "rewards/rejected": -0.048736222088336945,
+      "step": 7580
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.963516624143823e-06,
+      "logits/chosen": -2.230799436569214,
+      "logits/rejected": -2.1142804622650146,
+      "logps/chosen": -213.874755859375,
+      "logps/rejected": -185.1417694091797,
+      "loss": 0.689,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.013080480508506298,
+      "rewards/margins": 0.0879780501127243,
+      "rewards/rejected": -0.10105852037668228,
+      "step": 7590
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9579045088442504e-06,
+      "logits/chosen": -2.1219449043273926,
+      "logits/rejected": -2.1596245765686035,
+      "logps/chosen": -189.5128631591797,
+      "logps/rejected": -221.0270538330078,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.010078911669552326,
+      "rewards/margins": 0.11765221506357193,
+      "rewards/rejected": -0.12773114442825317,
+      "step": 7600
+    },
+    {
+      "epoch": 0.5,
+      "eval_logits/chosen": -2.288795232772827,
+      "eval_logits/rejected": -2.1032533645629883,
+      "eval_logps/chosen": -232.4485321044922,
+      "eval_logps/rejected": -220.70327758789062,
+      "eval_loss": 0.6899698376655579,
+      "eval_rewards/accuracies": 0.6299999952316284,
+      "eval_rewards/chosen": -0.004435794893652201,
+      "eval_rewards/margins": 0.08647802472114563,
+      "eval_rewards/rejected": -0.0909138172864914,
+      "eval_runtime": 714.3681,
+      "eval_samples_per_second": 2.8,
+      "eval_steps_per_second": 1.4,
+      "step": 7600
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9522900048904534e-06,
+      "logits/chosen": -2.2064361572265625,
+      "logits/rejected": -2.1144938468933105,
+      "logps/chosen": -244.34390258789062,
+      "logps/rejected": -218.4773406982422,
+      "loss": 0.693,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.046057600528001785,
+      "rewards/margins": 0.041549500077962875,
+      "rewards/rejected": -0.08760710060596466,
+      "step": 7610
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9466731415704343e-06,
+      "logits/chosen": -2.2881698608398438,
+      "logits/rejected": -2.161687135696411,
+      "logps/chosen": -225.63803100585938,
+      "logps/rejected": -229.7403564453125,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.018513670191168785,
+      "rewards/margins": 0.0846969336271286,
+      "rewards/rejected": -0.10321060568094254,
+      "step": 7620
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.941053948184503e-06,
+      "logits/chosen": -2.339186668395996,
+      "logits/rejected": -2.1879472732543945,
+      "logps/chosen": -279.9281921386719,
+      "logps/rejected": -249.49502563476562,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.00010864362411666662,
+      "rewards/margins": 0.04479020833969116,
+      "rewards/rejected": -0.04468156024813652,
+      "step": 7630
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.935432454045125e-06,
+      "logits/chosen": -2.1335971355438232,
+      "logits/rejected": -2.1970443725585938,
+      "logps/chosen": -233.8820037841797,
+      "logps/rejected": -216.0299835205078,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.019176747649908066,
+      "rewards/margins": 0.033554740250110626,
+      "rewards/rejected": -0.05273149162530899,
+      "step": 7640
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.929808688476768e-06,
+      "logits/chosen": -2.363029956817627,
+      "logits/rejected": -2.2410759925842285,
+      "logps/chosen": -240.63131713867188,
+      "logps/rejected": -230.9453887939453,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.012689086608588696,
+      "rewards/margins": 0.07916554063558578,
+      "rewards/rejected": -0.09185463190078735,
+      "step": 7650
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.924182680815748e-06,
+      "logits/chosen": -2.2831203937530518,
+      "logits/rejected": -2.210198402404785,
+      "logps/chosen": -232.90261840820312,
+      "logps/rejected": -222.2987823486328,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.017279820516705513,
+      "rewards/margins": 0.12103778123855591,
+      "rewards/rejected": -0.10375796258449554,
+      "step": 7660
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9185544604100765e-06,
+      "logits/chosen": -2.063129186630249,
+      "logits/rejected": -1.9675689935684204,
+      "logps/chosen": -198.81106567382812,
+      "logps/rejected": -202.04229736328125,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.02900967001914978,
+      "rewards/margins": 0.07486601918935776,
+      "rewards/rejected": -0.10387568175792694,
+      "step": 7670
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9129240566193083e-06,
+      "logits/chosen": -2.3740992546081543,
+      "logits/rejected": -2.0523669719696045,
+      "logps/chosen": -202.94161987304688,
+      "logps/rejected": -199.41842651367188,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.00861315242946148,
+      "rewards/margins": 0.09764768928289413,
+      "rewards/rejected": -0.10626085102558136,
+      "step": 7680
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9072914988143874e-06,
+      "logits/chosen": -2.1358678340911865,
+      "logits/rejected": -2.0475707054138184,
+      "logps/chosen": -201.20492553710938,
+      "logps/rejected": -204.98117065429688,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.002248757751658559,
+      "rewards/margins": 0.12247662246227264,
+      "rewards/rejected": -0.1202278584241867,
+      "step": 7690
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9016568163774956e-06,
+      "logits/chosen": -2.360272169113159,
+      "logits/rejected": -2.1226887702941895,
+      "logps/chosen": -172.92312622070312,
+      "logps/rejected": -146.11019897460938,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.006938323378562927,
+      "rewards/margins": 0.07239948213100433,
+      "rewards/rejected": -0.07933782041072845,
+      "step": 7700
+    },
+    {
+      "epoch": 0.5,
+      "eval_logits/chosen": -2.2827038764953613,
+      "eval_logits/rejected": -2.097726583480835,
+      "eval_logps/chosen": -231.82516479492188,
+      "eval_logps/rejected": -219.7780303955078,
+      "eval_loss": 0.6899218559265137,
+      "eval_rewards/accuracies": 0.6355000138282776,
+      "eval_rewards/chosen": 0.001797709846869111,
+      "eval_rewards/margins": 0.08345920592546463,
+      "eval_rewards/rejected": -0.08166150003671646,
+      "eval_runtime": 710.0146,
+      "eval_samples_per_second": 2.817,
+      "eval_steps_per_second": 1.408,
+      "step": 7700
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.8960200387018942e-06,
+      "logits/chosen": -2.1221325397491455,
+      "logits/rejected": -2.0857224464416504,
+      "logps/chosen": -308.96600341796875,
+      "logps/rejected": -268.85888671875,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.012330361641943455,
+      "rewards/margins": 0.08727259188890457,
+      "rewards/rejected": -0.09960294514894485,
+      "step": 7710
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8903811951917792e-06,
+      "logits/chosen": -2.2766757011413574,
+      "logits/rejected": -2.124586582183838,
+      "logps/chosen": -199.05517578125,
+      "logps/rejected": -159.13063049316406,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.007754988968372345,
+      "rewards/margins": 0.07217663526535034,
+      "rewards/rejected": -0.07993160933256149,
+      "step": 7720
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.88474031526212e-06,
+      "logits/chosen": -2.2419610023498535,
+      "logits/rejected": -2.2114017009735107,
+      "logps/chosen": -203.2736053466797,
+      "logps/rejected": -222.61083984375,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.004338492639362812,
+      "rewards/margins": 0.05908681079745293,
+      "rewards/rejected": -0.06342529505491257,
+      "step": 7730
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.879097428338509e-06,
+      "logits/chosen": -2.2317774295806885,
+      "logits/rejected": -1.9235107898712158,
+      "logps/chosen": -217.020751953125,
+      "logps/rejected": -202.31373596191406,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.011257743462920189,
+      "rewards/margins": 0.0762132927775383,
+      "rewards/rejected": -0.08747103810310364,
+      "step": 7740
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8734525638570094e-06,
+      "logits/chosen": -2.234351634979248,
+      "logits/rejected": -2.1596150398254395,
+      "logps/chosen": -232.974853515625,
+      "logps/rejected": -226.3662567138672,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.012193715199828148,
+      "rewards/margins": 0.05764711648225784,
+      "rewards/rejected": -0.04545340687036514,
+      "step": 7750
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8678057512639982e-06,
+      "logits/chosen": -2.181051254272461,
+      "logits/rejected": -2.088076114654541,
+      "logps/chosen": -284.4569091796875,
+      "logps/rejected": -273.23193359375,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.016095653176307678,
+      "rewards/margins": 0.10834445059299469,
+      "rewards/rejected": -0.09224879741668701,
+      "step": 7760
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8621570200160172e-06,
+      "logits/chosen": -2.0607194900512695,
+      "logits/rejected": -1.9694864749908447,
+      "logps/chosen": -167.73159790039062,
+      "logps/rejected": -169.00257873535156,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.008374934084713459,
+      "rewards/margins": 0.09448956698179245,
+      "rewards/rejected": -0.08611463010311127,
+      "step": 7770
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.856506399579615e-06,
+      "logits/chosen": -2.414057493209839,
+      "logits/rejected": -2.0493381023406982,
+      "logps/chosen": -222.3790740966797,
+      "logps/rejected": -209.5500946044922,
+      "loss": 0.689,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.04015577957034111,
+      "rewards/margins": 0.06660310924053192,
+      "rewards/rejected": -0.10675889253616333,
+      "step": 7780
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8508539194311964e-06,
+      "logits/chosen": -2.3235208988189697,
+      "logits/rejected": -2.316335916519165,
+      "logps/chosen": -255.4383544921875,
+      "logps/rejected": -273.5601806640625,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.017049241811037064,
+      "rewards/margins": 0.052917949855327606,
+      "rewards/rejected": -0.06996718794107437,
+      "step": 7790
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8451996090568656e-06,
+      "logits/chosen": -2.2277419567108154,
+      "logits/rejected": -2.1044132709503174,
+      "logps/chosen": -189.9392547607422,
+      "logps/rejected": -181.34808349609375,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.03799670934677124,
+      "rewards/margins": 0.09370444715023041,
+      "rewards/rejected": -0.13170115649700165,
+      "step": 7800
+    },
+    {
+      "epoch": 0.51,
+      "eval_logits/chosen": -2.2713305950164795,
+      "eval_logits/rejected": -2.086493968963623,
+      "eval_logps/chosen": -235.31703186035156,
+      "eval_logps/rejected": -223.4753875732422,
+      "eval_loss": 0.6899875402450562,
+      "eval_rewards/accuracies": 0.6485000252723694,
+      "eval_rewards/chosen": -0.03312075883150101,
+      "eval_rewards/margins": 0.08551418036222458,
+      "eval_rewards/rejected": -0.11863493919372559,
+      "eval_runtime": 707.6852,
+      "eval_samples_per_second": 2.826,
+      "eval_steps_per_second": 1.413,
+      "step": 7800
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.839543497952276e-06,
+      "logits/chosen": -2.1599411964416504,
+      "logits/rejected": -2.2668697834014893,
+      "logps/chosen": -189.414306640625,
+      "logps/rejected": -190.27862548828125,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.04804200679063797,
+      "rewards/margins": 0.09568199515342712,
+      "rewards/rejected": -0.1437240093946457,
+      "step": 7810
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.833885615622474e-06,
+      "logits/chosen": -2.2124152183532715,
+      "logits/rejected": -2.0647387504577637,
+      "logps/chosen": -208.2938995361328,
+      "logps/rejected": -225.68045043945312,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.05718092992901802,
+      "rewards/margins": 0.07911114394664764,
+      "rewards/rejected": -0.13629207015037537,
+      "step": 7820
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8282259915817454e-06,
+      "logits/chosen": -1.902604341506958,
+      "logits/rejected": -2.096595287322998,
+      "logps/chosen": -144.83163452148438,
+      "logps/rejected": -194.21328735351562,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.038869358599185944,
+      "rewards/margins": 0.09170033037662506,
+      "rewards/rejected": -0.1305696964263916,
+      "step": 7830
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8225646553534614e-06,
+      "logits/chosen": -2.0661423206329346,
+      "logits/rejected": -1.9575055837631226,
+      "logps/chosen": -201.17019653320312,
+      "logps/rejected": -204.97335815429688,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.01854178123176098,
+      "rewards/margins": 0.07094518840312958,
+      "rewards/rejected": -0.0894869714975357,
+      "step": 7840
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8169016364699255e-06,
+      "logits/chosen": -2.2641472816467285,
+      "logits/rejected": -1.9965393543243408,
+      "logps/chosen": -217.72933959960938,
+      "logps/rejected": -225.2642822265625,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.05841469764709473,
+      "rewards/margins": 0.057847362011671066,
+      "rewards/rejected": -0.1162620559334755,
+      "step": 7850
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.811236964472217e-06,
+      "logits/chosen": -2.3709425926208496,
+      "logits/rejected": -2.0033254623413086,
+      "logps/chosen": -314.4047546386719,
+      "logps/rejected": -261.5574035644531,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.03998160362243652,
+      "rewards/margins": 0.07272603362798691,
+      "rewards/rejected": -0.11270763725042343,
+      "step": 7860
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.805570668910041e-06,
+      "logits/chosen": -2.0802268981933594,
+      "logits/rejected": -2.0542476177215576,
+      "logps/chosen": -177.2976837158203,
+      "logps/rejected": -247.8351593017578,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.07658363878726959,
+      "rewards/margins": 0.07037156820297241,
+      "rewards/rejected": -0.146955206990242,
+      "step": 7870
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7999027793415695e-06,
+      "logits/chosen": -2.464724063873291,
+      "logits/rejected": -1.99410879611969,
+      "logps/chosen": -250.383544921875,
+      "logps/rejected": -210.59585571289062,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.054675567895174026,
+      "rewards/margins": 0.051379382610321045,
+      "rewards/rejected": -0.10605494678020477,
+      "step": 7880
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.794233325333293e-06,
+      "logits/chosen": -2.1549906730651855,
+      "logits/rejected": -2.0477986335754395,
+      "logps/chosen": -261.9752502441406,
+      "logps/rejected": -247.65414428710938,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.03693775087594986,
+      "rewards/margins": 0.09356243908405304,
+      "rewards/rejected": -0.1305001974105835,
+      "step": 7890
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7885623364598597e-06,
+      "logits/chosen": -2.3811306953430176,
+      "logits/rejected": -2.0712475776672363,
+      "logps/chosen": -270.0716247558594,
+      "logps/rejected": -237.21182250976562,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.06379027664661407,
+      "rewards/margins": 0.09059080481529236,
+      "rewards/rejected": -0.15438108146190643,
+      "step": 7900
+    },
+    {
+      "epoch": 0.52,
+      "eval_logits/chosen": -2.269857883453369,
+      "eval_logits/rejected": -2.085240125656128,
+      "eval_logps/chosen": -236.7635498046875,
+      "eval_logps/rejected": -224.1826629638672,
+      "eval_loss": 0.6899347901344299,
+      "eval_rewards/accuracies": 0.6424999833106995,
+      "eval_rewards/chosen": -0.047585878521203995,
+      "eval_rewards/margins": 0.07812169939279556,
+      "eval_rewards/rejected": -0.12570756673812866,
+      "eval_runtime": 710.7257,
+      "eval_samples_per_second": 2.814,
+      "eval_steps_per_second": 1.407,
+      "step": 7900
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.782889842303926e-06,
+      "logits/chosen": -2.2479918003082275,
+      "logits/rejected": -2.0780441761016846,
+      "logps/chosen": -169.74075317382812,
+      "logps/rejected": -166.49923706054688,
+      "loss": 0.6938,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.09511653333902359,
+      "rewards/margins": 0.022563491016626358,
+      "rewards/rejected": -0.11768001317977905,
+      "step": 7910
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7772158724559987e-06,
+      "logits/chosen": -2.092353105545044,
+      "logits/rejected": -1.937853217124939,
+      "logps/chosen": -221.2228546142578,
+      "logps/rejected": -273.72015380859375,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.041138116270303726,
+      "rewards/margins": 0.15026527643203735,
+      "rewards/rejected": -0.1914033740758896,
+      "step": 7920
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7715404565142856e-06,
+      "logits/chosen": -2.250084161758423,
+      "logits/rejected": -2.1173858642578125,
+      "logps/chosen": -204.4228515625,
+      "logps/rejected": -204.76995849609375,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0445866696536541,
+      "rewards/margins": 0.06569734215736389,
+      "rewards/rejected": -0.11028401553630829,
+      "step": 7930
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7658636240845354e-06,
+      "logits/chosen": -2.4091989994049072,
+      "logits/rejected": -2.306553363800049,
+      "logps/chosen": -226.8249969482422,
+      "logps/rejected": -250.21707153320312,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.03376641497015953,
+      "rewards/margins": 0.08774002641439438,
+      "rewards/rejected": -0.12150643765926361,
+      "step": 7940
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7601854047798872e-06,
+      "logits/chosen": -2.1954774856567383,
+      "logits/rejected": -2.236949920654297,
+      "logps/chosen": -228.4868621826172,
+      "logps/rejected": -254.75991821289062,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.045634832233190536,
+      "rewards/margins": 0.06857772916555405,
+      "rewards/rejected": -0.11421255767345428,
+      "step": 7950
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7545058282207148e-06,
+      "logits/chosen": -2.3288655281066895,
+      "logits/rejected": -1.9428226947784424,
+      "logps/chosen": -215.27224731445312,
+      "logps/rejected": -191.93368530273438,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.03730421140789986,
+      "rewards/margins": 0.07240404933691025,
+      "rewards/rejected": -0.1097082644701004,
+      "step": 7960
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.748824924034471e-06,
+      "logits/chosen": -2.2552783489227295,
+      "logits/rejected": -2.120013475418091,
+      "logps/chosen": -226.8047637939453,
+      "logps/rejected": -217.0663299560547,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.07394564896821976,
+      "rewards/margins": 0.09158217161893845,
+      "rewards/rejected": -0.1655278354883194,
+      "step": 7970
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.743142721855536e-06,
+      "logits/chosen": -2.1140682697296143,
+      "logits/rejected": -2.0926127433776855,
+      "logps/chosen": -157.36866760253906,
+      "logps/rejected": -154.32342529296875,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.05508549138903618,
+      "rewards/margins": 0.053032286465168,
+      "rewards/rejected": -0.10811777412891388,
+      "step": 7980
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.737459251325058e-06,
+      "logits/chosen": -2.23268985748291,
+      "logits/rejected": -2.1779227256774902,
+      "logps/chosen": -271.97705078125,
+      "logps/rejected": -255.03189086914062,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.020035061985254288,
+      "rewards/margins": 0.055861860513687134,
+      "rewards/rejected": -0.07589691877365112,
+      "step": 7990
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.731774542090804e-06,
+      "logits/chosen": -2.1980490684509277,
+      "logits/rejected": -1.7955074310302734,
+      "logps/chosen": -196.6571044921875,
+      "logps/rejected": -182.37567138671875,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.03881066292524338,
+      "rewards/margins": 0.04753577709197998,
+      "rewards/rejected": -0.08634644001722336,
+      "step": 8000
+    },
+    {
+      "epoch": 0.52,
+      "eval_logits/chosen": -2.265841245651245,
+      "eval_logits/rejected": -2.081429958343506,
+      "eval_logps/chosen": -235.2987060546875,
+      "eval_logps/rejected": -223.01144409179688,
+      "eval_loss": 0.6899079084396362,
+      "eval_rewards/accuracies": 0.6345000267028809,
+      "eval_rewards/chosen": -0.03293740004301071,
+      "eval_rewards/margins": 0.08105786144733429,
+      "eval_rewards/rejected": -0.113995261490345,
+      "eval_runtime": 710.8315,
+      "eval_samples_per_second": 2.814,
+      "eval_steps_per_second": 1.407,
+      "step": 8000
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7260886238070034e-06,
+      "logits/chosen": -2.271594524383545,
+      "logits/rejected": -2.1849944591522217,
+      "logps/chosen": -198.82374572753906,
+      "logps/rejected": -202.0972442626953,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.023537836968898773,
+      "rewards/margins": 0.07981632649898529,
+      "rewards/rejected": -0.10335417091846466,
+      "step": 8010
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.72040152613419e-06,
+      "logits/chosen": -2.2961008548736572,
+      "logits/rejected": -1.8004083633422852,
+      "logps/chosen": -219.9088897705078,
+      "logps/rejected": -149.8590087890625,
+      "loss": 0.6852,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.03033808246254921,
+      "rewards/margins": 0.13175645470619202,
+      "rewards/rejected": -0.16209453344345093,
+      "step": 8020
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.7147132787390516e-06,
+      "logits/chosen": -2.286135196685791,
+      "logits/rejected": -1.993798851966858,
+      "logps/chosen": -229.1460418701172,
+      "logps/rejected": -220.80264282226562,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.01667933538556099,
+      "rewards/margins": 0.08642159402370453,
+      "rewards/rejected": -0.10310093313455582,
+      "step": 8030
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.709023911294273e-06,
+      "logits/chosen": -2.374183177947998,
+      "logits/rejected": -1.9083032608032227,
+      "logps/chosen": -242.0371551513672,
+      "logps/rejected": -225.60302734375,
+      "loss": 0.6868,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.002816893858835101,
+      "rewards/margins": 0.14465411007404327,
+      "rewards/rejected": -0.1418372094631195,
+      "step": 8040
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.7033334534783806e-06,
+      "logits/chosen": -2.263575792312622,
+      "logits/rejected": -2.3582472801208496,
+      "logps/chosen": -200.14503479003906,
+      "logps/rejected": -225.3608856201172,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.009991501457989216,
+      "rewards/margins": 0.09016770124435425,
+      "rewards/rejected": -0.10015920549631119,
+      "step": 8050
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.697641934975592e-06,
+      "logits/chosen": -2.2752552032470703,
+      "logits/rejected": -2.050177574157715,
+      "logps/chosen": -229.1072998046875,
+      "logps/rejected": -205.2366180419922,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.013208845630288124,
+      "rewards/margins": 0.08683110773563385,
+      "rewards/rejected": -0.10003993660211563,
+      "step": 8060
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.691949385475654e-06,
+      "logits/chosen": -2.3117451667785645,
+      "logits/rejected": -2.063112258911133,
+      "logps/chosen": -246.4084930419922,
+      "logps/rejected": -229.3635711669922,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.02517673373222351,
+      "rewards/margins": 0.0852632075548172,
+      "rewards/rejected": -0.11043993383646011,
+      "step": 8070
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6862558346736937e-06,
+      "logits/chosen": -2.239243268966675,
+      "logits/rejected": -2.0910251140594482,
+      "logps/chosen": -241.4723663330078,
+      "logps/rejected": -251.2477264404297,
+      "loss": 0.6856,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.031201153993606567,
+      "rewards/margins": 0.14977136254310608,
+      "rewards/rejected": -0.18097251653671265,
+      "step": 8080
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6805613122700617e-06,
+      "logits/chosen": -2.282254457473755,
+      "logits/rejected": -1.951345682144165,
+      "logps/chosen": -227.823486328125,
+      "logps/rejected": -238.18466186523438,
+      "loss": 0.688,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.04250997677445412,
+      "rewards/margins": 0.09419043362140656,
+      "rewards/rejected": -0.13670040667057037,
+      "step": 8090
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.674865847970176e-06,
+      "logits/chosen": -2.219407320022583,
+      "logits/rejected": -1.950874924659729,
+      "logps/chosen": -209.02536010742188,
+      "logps/rejected": -239.494384765625,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.014089837670326233,
+      "rewards/margins": 0.08020684868097305,
+      "rewards/rejected": -0.09429670870304108,
+      "step": 8100
+    },
+    {
+      "epoch": 0.53,
+      "eval_logits/chosen": -2.2729127407073975,
+      "eval_logits/rejected": -2.08774733543396,
+      "eval_logps/chosen": -233.5811309814453,
+      "eval_logps/rejected": -221.25350952148438,
+      "eval_loss": 0.6898881793022156,
+      "eval_rewards/accuracies": 0.6365000009536743,
+      "eval_rewards/chosen": -0.01576184667646885,
+      "eval_rewards/margins": 0.08065415918827057,
+      "eval_rewards/rejected": -0.09641600400209427,
+      "eval_runtime": 710.7109,
+      "eval_samples_per_second": 2.814,
+      "eval_steps_per_second": 1.407,
+      "step": 8100
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.669169471484368e-06,
+      "logits/chosen": -2.0301496982574463,
+      "logits/rejected": -2.0818302631378174,
+      "logps/chosen": -168.10691833496094,
+      "logps/rejected": -169.706298828125,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.03048858605325222,
+      "rewards/margins": 0.04239571467041969,
+      "rewards/rejected": -0.07288429886102676,
+      "step": 8110
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6634722125277278e-06,
+      "logits/chosen": -2.373579502105713,
+      "logits/rejected": -2.051607370376587,
+      "logps/chosen": -235.17337036132812,
+      "logps/rejected": -251.06521606445312,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.030534693971276283,
+      "rewards/margins": 0.076970174908638,
+      "rewards/rejected": -0.10750486701726913,
+      "step": 8120
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6577741008199498e-06,
+      "logits/chosen": -2.2919061183929443,
+      "logits/rejected": -1.9393638372421265,
+      "logps/chosen": -256.58197021484375,
+      "logps/rejected": -211.50302124023438,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.0042382776737213135,
+      "rewards/margins": 0.1496235430240631,
+      "rewards/rejected": -0.15386183559894562,
+      "step": 8130
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.652075166085175e-06,
+      "logits/chosen": -2.1796364784240723,
+      "logits/rejected": -2.135631561279297,
+      "logps/chosen": -235.9457244873047,
+      "logps/rejected": -273.9045715332031,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.013664024882018566,
+      "rewards/margins": 0.14413678646087646,
+      "rewards/rejected": -0.1578008234500885,
+      "step": 8140
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6463754380518395e-06,
+      "logits/chosen": -2.159858465194702,
+      "logits/rejected": -1.9525247812271118,
+      "logps/chosen": -236.5515899658203,
+      "logps/rejected": -190.86343383789062,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.05593949556350708,
+      "rewards/margins": 0.09138914197683334,
+      "rewards/rejected": -0.14732863008975983,
+      "step": 8150
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6406749464525167e-06,
+      "logits/chosen": -2.2781941890716553,
+      "logits/rejected": -1.997957468032837,
+      "logps/chosen": -233.92446899414062,
+      "logps/rejected": -196.24688720703125,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0036535251419991255,
+      "rewards/margins": 0.09554257988929749,
+      "rewards/rejected": -0.09919609874486923,
+      "step": 8160
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.634973721023762e-06,
+      "logits/chosen": -2.3184168338775635,
+      "logits/rejected": -2.161243200302124,
+      "logps/chosen": -258.32891845703125,
+      "logps/rejected": -215.8189239501953,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.03866691142320633,
+      "rewards/margins": 0.06928583234548569,
+      "rewards/rejected": -0.10795273631811142,
+      "step": 8170
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.6292717915059605e-06,
+      "logits/chosen": -2.3634932041168213,
+      "logits/rejected": -2.096513032913208,
+      "logps/chosen": -278.8802490234375,
+      "logps/rejected": -233.5529022216797,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.03250148147344589,
+      "rewards/margins": 0.10865737497806549,
+      "rewards/rejected": -0.1411588490009308,
+      "step": 8180
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.6235691876431706e-06,
+      "logits/chosen": -2.171806812286377,
+      "logits/rejected": -2.2218527793884277,
+      "logps/chosen": -221.06301879882812,
+      "logps/rejected": -234.08663940429688,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.02447427250444889,
+      "rewards/margins": 0.07424478232860565,
+      "rewards/rejected": -0.09871906042098999,
+      "step": 8190
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.6178659391829673e-06,
+      "logits/chosen": -2.3931174278259277,
+      "logits/rejected": -2.1151492595672607,
+      "logps/chosen": -236.8017120361328,
+      "logps/rejected": -204.3024139404297,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.006762659642845392,
+      "rewards/margins": 0.08343976736068726,
+      "rewards/rejected": -0.0766771137714386,
+      "step": 8200
+    },
+    {
+      "epoch": 0.54,
+      "eval_logits/chosen": -2.2691376209259033,
+      "eval_logits/rejected": -2.084272861480713,
+      "eval_logps/chosen": -234.5026397705078,
+      "eval_logps/rejected": -222.2466278076172,
+      "eval_loss": 0.6898766160011292,
+      "eval_rewards/accuracies": 0.6355000138282776,
+      "eval_rewards/chosen": -0.02497694082558155,
+      "eval_rewards/margins": 0.08137031644582748,
+      "eval_rewards/rejected": -0.10634726285934448,
+      "eval_runtime": 711.535,
+      "eval_samples_per_second": 2.811,
+      "eval_steps_per_second": 1.405,
+      "step": 8200
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.6121620758762877e-06,
+      "logits/chosen": -2.2570462226867676,
+      "logits/rejected": -2.002037525177002,
+      "logps/chosen": -194.84579467773438,
+      "logps/rejected": -200.37400817871094,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.027614828199148178,
+      "rewards/margins": 0.06327278167009354,
+      "rewards/rejected": -0.09088762104511261,
+      "step": 8210
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.606457627477277e-06,
+      "logits/chosen": -2.1911864280700684,
+      "logits/rejected": -2.134552478790283,
+      "logps/chosen": -176.81307983398438,
+      "logps/rejected": -189.58029174804688,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.011837871745228767,
+      "rewards/margins": 0.09415124356746674,
+      "rewards/rejected": -0.10598911345005035,
+      "step": 8220
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.6007526237431324e-06,
+      "logits/chosen": -2.330580234527588,
+      "logits/rejected": -2.279081344604492,
+      "logps/chosen": -182.9707489013672,
+      "logps/rejected": -206.08935546875,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.009412150830030441,
+      "rewards/margins": 0.0900057703256607,
+      "rewards/rejected": -0.09941791743040085,
+      "step": 8230
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5950470944339478e-06,
+      "logits/chosen": -2.110105514526367,
+      "logits/rejected": -2.1718857288360596,
+      "logps/chosen": -220.61978149414062,
+      "logps/rejected": -220.62734985351562,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.01182152982801199,
+      "rewards/margins": 0.03503318130970001,
+      "rewards/rejected": -0.023211652413010597,
+      "step": 8240
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.58934106931256e-06,
+      "logits/chosen": -2.2483153343200684,
+      "logits/rejected": -1.9538730382919312,
+      "logps/chosen": -221.7985382080078,
+      "logps/rejected": -208.62557983398438,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.010264934040606022,
+      "rewards/margins": 0.06004582718014717,
+      "rewards/rejected": -0.07031075656414032,
+      "step": 8250
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.58363457814439e-06,
+      "logits/chosen": -2.243074417114258,
+      "logits/rejected": -1.955980896949768,
+      "logps/chosen": -214.344970703125,
+      "logps/rejected": -209.1015625,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.03673034533858299,
+      "rewards/margins": 0.0848006159067154,
+      "rewards/rejected": -0.12153096497058868,
+      "step": 8260
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5779276506972924e-06,
+      "logits/chosen": -2.2136752605438232,
+      "logits/rejected": -2.1928346157073975,
+      "logps/chosen": -233.85415649414062,
+      "logps/rejected": -200.51458740234375,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.019013002514839172,
+      "rewards/margins": 0.0604260191321373,
+      "rewards/rejected": -0.07943902909755707,
+      "step": 8270
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5722203167413945e-06,
+      "logits/chosen": -2.336066961288452,
+      "logits/rejected": -2.0090115070343018,
+      "logps/chosen": -284.7717590332031,
+      "logps/rejected": -210.93600463867188,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.014740842394530773,
+      "rewards/margins": 0.08085087686777115,
+      "rewards/rejected": -0.0955917239189148,
+      "step": 8280
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5665126060489476e-06,
+      "logits/chosen": -2.30047607421875,
+      "logits/rejected": -2.150911569595337,
+      "logps/chosen": -190.69834899902344,
+      "logps/rejected": -225.66976928710938,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.02596314251422882,
+      "rewards/margins": 0.06124822050333023,
+      "rewards/rejected": -0.08721135556697845,
+      "step": 8290
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.560804548394165e-06,
+      "logits/chosen": -2.222855567932129,
+      "logits/rejected": -1.9643363952636719,
+      "logps/chosen": -249.7584228515625,
+      "logps/rejected": -214.04666137695312,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.02478059008717537,
+      "rewards/margins": 0.0956047847867012,
+      "rewards/rejected": -0.12038537114858627,
+      "step": 8300
+    },
+    {
+      "epoch": 0.54,
+      "eval_logits/chosen": -2.2778053283691406,
+      "eval_logits/rejected": -2.0923290252685547,
+      "eval_logps/chosen": -232.2015380859375,
+      "eval_logps/rejected": -219.4079132080078,
+      "eval_loss": 0.6899590492248535,
+      "eval_rewards/accuracies": 0.6345000267028809,
+      "eval_rewards/chosen": -0.0019656550139188766,
+      "eval_rewards/margins": 0.07599426060914993,
+      "eval_rewards/rejected": -0.07795991748571396,
+      "eval_runtime": 710.1344,
+      "eval_samples_per_second": 2.816,
+      "eval_steps_per_second": 1.408,
+      "step": 8300
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5550961735530734e-06,
+      "logits/chosen": -2.106081008911133,
+      "logits/rejected": -2.282960891723633,
+      "logps/chosen": -161.3715362548828,
+      "logps/rejected": -198.7847900390625,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.016642851755023003,
+      "rewards/margins": 0.06822551786899567,
+      "rewards/rejected": -0.051582664251327515,
+      "step": 8310
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.549387511303351e-06,
+      "logits/chosen": -2.265373706817627,
+      "logits/rejected": -2.3061885833740234,
+      "logps/chosen": -168.9114990234375,
+      "logps/rejected": -219.42587280273438,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.008253363892436028,
+      "rewards/margins": 0.062337376177310944,
+      "rewards/rejected": -0.05408401042222977,
+      "step": 8320
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.5436785914241774e-06,
+      "logits/chosen": -2.2159152030944824,
+      "logits/rejected": -2.2393479347229004,
+      "logps/chosen": -200.00836181640625,
+      "logps/rejected": -181.49374389648438,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.01049137394875288,
+      "rewards/margins": 0.1345503181219101,
+      "rewards/rejected": -0.14504170417785645,
+      "step": 8330
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.5379694436960746e-06,
+      "logits/chosen": -2.3889286518096924,
+      "logits/rejected": -2.1922972202301025,
+      "logps/chosen": -243.3367919921875,
+      "logps/rejected": -261.3111877441406,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.026742050424218178,
+      "rewards/margins": 0.07409554719924927,
+      "rewards/rejected": -0.047353483736515045,
+      "step": 8340
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.5322600979007533e-06,
+      "logits/chosen": -2.403104305267334,
+      "logits/rejected": -2.162173271179199,
+      "logps/chosen": -212.3261260986328,
+      "logps/rejected": -199.4026641845703,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0016930897254496813,
+      "rewards/margins": 0.07775741815567017,
+      "rewards/rejected": -0.0760643258690834,
+      "step": 8350
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.5265505838209592e-06,
+      "logits/chosen": -2.4180874824523926,
+      "logits/rejected": -2.0715489387512207,
+      "logps/chosen": -256.7772521972656,
+      "logps/rejected": -215.9575958251953,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0018636692548170686,
+      "rewards/margins": 0.055936507880687714,
+      "rewards/rejected": -0.05407283455133438,
+      "step": 8360
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.520840931240314e-06,
+      "logits/chosen": -2.448770046234131,
+      "logits/rejected": -1.9609102010726929,
+      "logps/chosen": -208.07290649414062,
+      "logps/rejected": -152.3336639404297,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.017356865108013153,
+      "rewards/margins": 0.07981632649898529,
+      "rewards/rejected": -0.06245948001742363,
+      "step": 8370
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.515131169943162e-06,
+      "logits/chosen": -1.9940284490585327,
+      "logits/rejected": -2.0735738277435303,
+      "logps/chosen": -258.408203125,
+      "logps/rejected": -259.9877624511719,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.0031367135234177113,
+      "rewards/margins": 0.08551900088787079,
+      "rewards/rejected": -0.08865571022033691,
+      "step": 8380
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.509421329714416e-06,
+      "logits/chosen": -2.1275012493133545,
+      "logits/rejected": -2.1602554321289062,
+      "logps/chosen": -206.140869140625,
+      "logps/rejected": -231.0389862060547,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.6808509826660156e-05,
+      "rewards/margins": 0.03706061840057373,
+      "rewards/rejected": -0.03707743063569069,
+      "step": 8390
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.5037114403393987e-06,
+      "logits/chosen": -2.247596263885498,
+      "logits/rejected": -1.9996188879013062,
+      "logps/chosen": -209.26681518554688,
+      "logps/rejected": -182.84060668945312,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.017135795205831528,
+      "rewards/margins": 0.07354326546192169,
+      "rewards/rejected": -0.05640747398138046,
+      "step": 8400
+    },
+    {
+      "epoch": 0.55,
+      "eval_logits/chosen": -2.2804696559906006,
+      "eval_logits/rejected": -2.095289468765259,
+      "eval_logps/chosen": -230.7716522216797,
+      "eval_logps/rejected": -217.13856506347656,
+      "eval_loss": 0.6900351643562317,
+      "eval_rewards/accuracies": 0.6294999718666077,
+      "eval_rewards/chosen": 0.012332833372056484,
+      "eval_rewards/margins": 0.0675993338227272,
+      "eval_rewards/rejected": -0.055266499519348145,
+      "eval_runtime": 713.3682,
+      "eval_samples_per_second": 2.804,
+      "eval_steps_per_second": 1.402,
+      "step": 8400
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4980015316036908e-06,
+      "logits/chosen": -2.116654872894287,
+      "logits/rejected": -2.1679673194885254,
+      "logps/chosen": -173.55227661132812,
+      "logps/rejected": -206.0382080078125,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.022574327886104584,
+      "rewards/margins": 0.10900095850229263,
+      "rewards/rejected": -0.08642663061618805,
+      "step": 8410
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4922916332929725e-06,
+      "logits/chosen": -2.4510018825531006,
+      "logits/rejected": -2.1898789405822754,
+      "logps/chosen": -234.3470916748047,
+      "logps/rejected": -197.39511108398438,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.010771475732326508,
+      "rewards/margins": 0.02662494219839573,
+      "rewards/rejected": -0.015853462740778923,
+      "step": 8420
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4865817751928716e-06,
+      "logits/chosen": -2.1895623207092285,
+      "logits/rejected": -2.174008369445801,
+      "logps/chosen": -193.5983123779297,
+      "logps/rejected": -231.7257537841797,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.018903840333223343,
+      "rewards/margins": 0.11705265939235687,
+      "rewards/rejected": -0.09814882278442383,
+      "step": 8430
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4808719870888037e-06,
+      "logits/chosen": -2.0574288368225098,
+      "logits/rejected": -1.983668565750122,
+      "logps/chosen": -216.3809356689453,
+      "logps/rejected": -193.36599731445312,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.012905344367027283,
+      "rewards/margins": 0.10258817672729492,
+      "rewards/rejected": -0.08968283236026764,
+      "step": 8440
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4751622987658206e-06,
+      "logits/chosen": -2.475071430206299,
+      "logits/rejected": -2.2941946983337402,
+      "logps/chosen": -235.42440795898438,
+      "logps/rejected": -230.3686065673828,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.025258636102080345,
+      "rewards/margins": 0.05684860795736313,
+      "rewards/rejected": -0.03158997744321823,
+      "step": 8450
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4694527400084546e-06,
+      "logits/chosen": -2.25466251373291,
+      "logits/rejected": -2.161506175994873,
+      "logps/chosen": -222.21908569335938,
+      "logps/rejected": -224.0065155029297,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0002555936516728252,
+      "rewards/margins": 0.07632071524858475,
+      "rewards/rejected": -0.07606511563062668,
+      "step": 8460
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4637433406005607e-06,
+      "logits/chosen": -2.4585928916931152,
+      "logits/rejected": -2.344909191131592,
+      "logps/chosen": -310.13995361328125,
+      "logps/rejected": -286.87567138671875,
+      "loss": 0.691,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.004136279225349426,
+      "rewards/margins": 0.04778838902711868,
+      "rewards/rejected": -0.05192466825246811,
+      "step": 8470
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4580341303251628e-06,
+      "logits/chosen": -2.2639238834381104,
+      "logits/rejected": -2.002631425857544,
+      "logps/chosen": -259.3542175292969,
+      "logps/rejected": -229.9150848388672,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.005969688296318054,
+      "rewards/margins": 0.06959638744592667,
+      "rewards/rejected": -0.06362669169902802,
+      "step": 8480
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4523251389642984e-06,
+      "logits/chosen": -2.16398024559021,
+      "logits/rejected": -2.036417007446289,
+      "logps/chosen": -256.45709228515625,
+      "logps/rejected": -229.12576293945312,
+      "loss": 0.6877,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.018977751955389977,
+      "rewards/margins": 0.09807170927524567,
+      "rewards/rejected": -0.11704947054386139,
+      "step": 8490
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4466163962988626e-06,
+      "logits/chosen": -2.480299711227417,
+      "logits/rejected": -2.1086299419403076,
+      "logps/chosen": -281.618896484375,
+      "logps/rejected": -193.210205078125,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.010179015807807446,
+      "rewards/margins": 0.10310627520084381,
+      "rewards/rejected": -0.11328530311584473,
+      "step": 8500
+    },
+    {
+      "epoch": 0.56,
+      "eval_logits/chosen": -2.2819478511810303,
+      "eval_logits/rejected": -2.0963072776794434,
+      "eval_logps/chosen": -231.94549560546875,
+      "eval_logps/rejected": -220.13168334960938,
+      "eval_loss": 0.689839780330658,
+      "eval_rewards/accuracies": 0.6455000042915344,
+      "eval_rewards/chosen": 0.0005945622688159347,
+      "eval_rewards/margins": 0.08579233288764954,
+      "eval_rewards/rejected": -0.08519777655601501,
+      "eval_runtime": 712.4374,
+      "eval_samples_per_second": 2.807,
+      "eval_steps_per_second": 1.404,
+      "step": 8500
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4409079321084543e-06,
+      "logits/chosen": -2.2277088165283203,
+      "logits/rejected": -2.284764051437378,
+      "logps/chosen": -213.2277374267578,
+      "logps/rejected": -252.33645629882812,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.01891925372183323,
+      "rewards/margins": 0.0916977971792221,
+      "rewards/rejected": -0.07277854532003403,
+      "step": 8510
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4351997761712184e-06,
+      "logits/chosen": -2.4851880073547363,
+      "logits/rejected": -2.031656265258789,
+      "logps/chosen": -244.4697265625,
+      "logps/rejected": -189.30319213867188,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.009185681119561195,
+      "rewards/margins": 0.11539344489574432,
+      "rewards/rejected": -0.10620777308940887,
+      "step": 8520
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4294919582636933e-06,
+      "logits/chosen": -2.274225950241089,
+      "logits/rejected": -2.129521369934082,
+      "logps/chosen": -209.3656768798828,
+      "logps/rejected": -206.8007354736328,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.017368298023939133,
+      "rewards/margins": 0.09274449944496155,
+      "rewards/rejected": -0.07537619769573212,
+      "step": 8530
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.423784508160652e-06,
+      "logits/chosen": -2.352238655090332,
+      "logits/rejected": -2.100398540496826,
+      "logps/chosen": -256.19207763671875,
+      "logps/rejected": -215.8179168701172,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.009613009169697762,
+      "rewards/margins": 0.07463245093822479,
+      "rewards/rejected": -0.084245465695858,
+      "step": 8540
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.418077455634951e-06,
+      "logits/chosen": -2.176546573638916,
+      "logits/rejected": -2.22251558303833,
+      "logps/chosen": -218.07138061523438,
+      "logps/rejected": -250.1627655029297,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.015002986416220665,
+      "rewards/margins": 0.045342180877923965,
+      "rewards/rejected": -0.06034516543149948,
+      "step": 8550
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4123708304573714e-06,
+      "logits/chosen": -2.3653371334075928,
+      "logits/rejected": -2.2171475887298584,
+      "logps/chosen": -288.91107177734375,
+      "logps/rejected": -280.00177001953125,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.008562428876757622,
+      "rewards/margins": 0.07789406925439835,
+      "rewards/rejected": -0.06933163106441498,
+      "step": 8560
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.406664662396465e-06,
+      "logits/chosen": -2.1397430896759033,
+      "logits/rejected": -1.9881635904312134,
+      "logps/chosen": -188.5435333251953,
+      "logps/rejected": -179.0817108154297,
+      "loss": 0.691,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.04161657765507698,
+      "rewards/margins": 0.06220381706953049,
+      "rewards/rejected": -0.10382040590047836,
+      "step": 8570
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4009589812184012e-06,
+      "logits/chosen": -2.3080785274505615,
+      "logits/rejected": -1.9249913692474365,
+      "logps/chosen": -205.11972045898438,
+      "logps/rejected": -160.22409057617188,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.018998144194483757,
+      "rewards/margins": 0.08215345442295074,
+      "rewards/rejected": -0.10115160048007965,
+      "step": 8580
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.3952538166868073e-06,
+      "logits/chosen": -2.0843875408172607,
+      "logits/rejected": -2.144876480102539,
+      "logps/chosen": -218.38809204101562,
+      "logps/rejected": -214.17666625976562,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.008417466655373573,
+      "rewards/margins": 0.1238173246383667,
+      "rewards/rejected": -0.13223478198051453,
+      "step": 8590
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.389549198562616e-06,
+      "logits/chosen": -2.282944917678833,
+      "logits/rejected": -1.855536699295044,
+      "logps/chosen": -225.4339599609375,
+      "logps/rejected": -205.1066131591797,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.0064767589792609215,
+      "rewards/margins": 0.09652809798717499,
+      "rewards/rejected": -0.09005134552717209,
+      "step": 8600
+    },
+    {
+      "epoch": 0.56,
+      "eval_logits/chosen": -2.289522886276245,
+      "eval_logits/rejected": -2.1033294200897217,
+      "eval_logps/chosen": -232.30738830566406,
+      "eval_logps/rejected": -220.40335083007812,
+      "eval_loss": 0.6898036003112793,
+      "eval_rewards/accuracies": 0.640999972820282,
+      "eval_rewards/chosen": -0.0030244227964431047,
+      "eval_rewards/margins": 0.08489015698432922,
+      "eval_rewards/rejected": -0.08791457116603851,
+      "eval_runtime": 714.176,
+      "eval_samples_per_second": 2.8,
+      "eval_steps_per_second": 1.4,
+      "step": 8600
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.3838451566039098e-06,
+      "logits/chosen": -2.309410333633423,
+      "logits/rejected": -2.1341471672058105,
+      "logps/chosen": -240.350830078125,
+      "logps/rejected": -234.599365234375,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.018408339470624924,
+      "rewards/margins": 0.04382626712322235,
+      "rewards/rejected": -0.062234602868556976,
+      "step": 8610
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.3781417205657662e-06,
+      "logits/chosen": -2.3165881633758545,
+      "logits/rejected": -2.01545786857605,
+      "logps/chosen": -197.41787719726562,
+      "logps/rejected": -167.50404357910156,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.003385829506441951,
+      "rewards/margins": 0.08293353766202927,
+      "rewards/rejected": -0.0795477032661438,
+      "step": 8620
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.3724389202001006e-06,
+      "logits/chosen": -2.3407962322235107,
+      "logits/rejected": -2.0748586654663086,
+      "logps/chosen": -203.34979248046875,
+      "logps/rejected": -187.19520568847656,
+      "loss": 0.692,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.010263567790389061,
+      "rewards/margins": 0.055420707911252975,
+      "rewards/rejected": -0.06568428128957748,
+      "step": 8630
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.366736785255514e-06,
+      "logits/chosen": -2.227527141571045,
+      "logits/rejected": -2.1566596031188965,
+      "logps/chosen": -200.77955627441406,
+      "logps/rejected": -196.5257110595703,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.020507413893938065,
+      "rewards/margins": 0.06966142356395721,
+      "rewards/rejected": -0.09016883373260498,
+      "step": 8640
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3610353454771355e-06,
+      "logits/chosen": -2.123077869415283,
+      "logits/rejected": -2.05281925201416,
+      "logps/chosen": -190.9486083984375,
+      "logps/rejected": -181.61386108398438,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.01001174096018076,
+      "rewards/margins": 0.06959837675094604,
+      "rewards/rejected": -0.07961011677980423,
+      "step": 8650
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.355334630606467e-06,
+      "logits/chosen": -2.493744373321533,
+      "logits/rejected": -2.0158677101135254,
+      "logps/chosen": -240.03988647460938,
+      "logps/rejected": -183.7025146484375,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0025037326849997044,
+      "rewards/margins": 0.07208188623189926,
+      "rewards/rejected": -0.06957816332578659,
+      "step": 8660
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.349634670381231e-06,
+      "logits/chosen": -2.0954480171203613,
+      "logits/rejected": -2.0449440479278564,
+      "logps/chosen": -208.32308959960938,
+      "logps/rejected": -224.8218231201172,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.01123981736600399,
+      "rewards/margins": 0.0670008659362793,
+      "rewards/rejected": -0.07824068516492844,
+      "step": 8670
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3439354945352104e-06,
+      "logits/chosen": -2.341536045074463,
+      "logits/rejected": -2.278677463531494,
+      "logps/chosen": -245.1410675048828,
+      "logps/rejected": -203.81253051757812,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.01617489382624626,
+      "rewards/margins": 0.030692163854837418,
+      "rewards/rejected": -0.04686705023050308,
+      "step": 8680
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3382371327981e-06,
+      "logits/chosen": -2.2057578563690186,
+      "logits/rejected": -2.200843334197998,
+      "logps/chosen": -230.00222778320312,
+      "logps/rejected": -225.0457000732422,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.014740320853888988,
+      "rewards/margins": 0.09338275343179703,
+      "rewards/rejected": -0.07864242792129517,
+      "step": 8690
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3325396148953456e-06,
+      "logits/chosen": -2.073983907699585,
+      "logits/rejected": -2.184781312942505,
+      "logps/chosen": -172.51876831054688,
+      "logps/rejected": -236.9097900390625,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.012173332273960114,
+      "rewards/margins": 0.10260754823684692,
+      "rewards/rejected": -0.11478088051080704,
+      "step": 8700
+    },
+    {
+      "epoch": 0.57,
+      "eval_logits/chosen": -2.2970077991485596,
+      "eval_logits/rejected": -2.1105120182037354,
+      "eval_logps/chosen": -230.84942626953125,
+      "eval_logps/rejected": -218.98678588867188,
+      "eval_loss": 0.6898258328437805,
+      "eval_rewards/accuracies": 0.6430000066757202,
+      "eval_rewards/chosen": 0.011555157601833344,
+      "eval_rewards/margins": 0.0853039100766182,
+      "eval_rewards/rejected": -0.07374875247478485,
+      "eval_runtime": 712.4551,
+      "eval_samples_per_second": 2.807,
+      "eval_steps_per_second": 1.404,
+      "step": 8700
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3268429705479915e-06,
+      "logits/chosen": -2.4747602939605713,
+      "logits/rejected": -2.1206700801849365,
+      "logps/chosen": -222.29537963867188,
+      "logps/rejected": -189.93572998046875,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.02517220936715603,
+      "rewards/margins": 0.08959371596574783,
+      "rewards/rejected": -0.06442151963710785,
+      "step": 8710
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3211472294725248e-06,
+      "logits/chosen": -2.3218271732330322,
+      "logits/rejected": -2.1841847896575928,
+      "logps/chosen": -212.7299346923828,
+      "logps/rejected": -209.63003540039062,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.03685241565108299,
+      "rewards/margins": 0.09567222744226456,
+      "rewards/rejected": -0.058819811791181564,
+      "step": 8720
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.315452421380721e-06,
+      "logits/chosen": -2.2035815715789795,
+      "logits/rejected": -1.7523466348648071,
+      "logps/chosen": -253.510986328125,
+      "logps/rejected": -214.3549346923828,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.014333168976008892,
+      "rewards/margins": 0.08837004750967026,
+      "rewards/rejected": -0.07403689622879028,
+      "step": 8730
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3097585759794886e-06,
+      "logits/chosen": -2.2899675369262695,
+      "logits/rejected": -1.9071069955825806,
+      "logps/chosen": -251.5904541015625,
+      "logps/rejected": -201.7096710205078,
+      "loss": 0.6867,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.027508467435836792,
+      "rewards/margins": 0.13511213660240173,
+      "rewards/rejected": -0.10760366916656494,
+      "step": 8740
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3040657229707155e-06,
+      "logits/chosen": -2.304961681365967,
+      "logits/rejected": -2.1966376304626465,
+      "logps/chosen": -170.78793334960938,
+      "logps/rejected": -190.66493225097656,
+      "loss": 0.69,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.012596605345606804,
+      "rewards/margins": 0.08321347087621689,
+      "rewards/rejected": -0.07061685621738434,
+      "step": 8750
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.2983738920511104e-06,
+      "logits/chosen": -2.464939832687378,
+      "logits/rejected": -1.987932801246643,
+      "logps/chosen": -265.67718505859375,
+      "logps/rejected": -223.0201416015625,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.04115743562579155,
+      "rewards/margins": 0.07645048201084137,
+      "rewards/rejected": -0.03529305011034012,
+      "step": 8760
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.2926831129120523e-06,
+      "logits/chosen": -2.120628833770752,
+      "logits/rejected": -2.073472499847412,
+      "logps/chosen": -232.22799682617188,
+      "logps/rejected": -209.72720336914062,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03563634306192398,
+      "rewards/margins": 0.0585489384829998,
+      "rewards/rejected": -0.02291259728372097,
+      "step": 8770
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.2869934152394323e-06,
+      "logits/chosen": -2.321106195449829,
+      "logits/rejected": -2.0543007850646973,
+      "logps/chosen": -268.8101501464844,
+      "logps/rejected": -220.52853393554688,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0015528525691479445,
+      "rewards/margins": 0.07320442795753479,
+      "rewards/rejected": -0.07165157794952393,
+      "step": 8780
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.281304828713501e-06,
+      "logits/chosen": -2.2122366428375244,
+      "logits/rejected": -2.1436047554016113,
+      "logps/chosen": -231.771728515625,
+      "logps/rejected": -231.8656768798828,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.012155646458268166,
+      "rewards/margins": 0.07596425712108612,
+      "rewards/rejected": -0.0638086199760437,
+      "step": 8790
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.275617383008711e-06,
+      "logits/chosen": -2.3027684688568115,
+      "logits/rejected": -2.2025887966156006,
+      "logps/chosen": -235.1232452392578,
+      "logps/rejected": -237.69161987304688,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.003641972318291664,
+      "rewards/margins": 0.05236151069402695,
+      "rewards/rejected": -0.048719536513090134,
+      "step": 8800
+    },
+    {
+      "epoch": 0.58,
+      "eval_logits/chosen": -2.304422616958618,
+      "eval_logits/rejected": -2.117183208465576,
+      "eval_logps/chosen": -229.04266357421875,
+      "eval_logps/rejected": -216.80625915527344,
+      "eval_loss": 0.6898345351219177,
+      "eval_rewards/accuracies": 0.6464999914169312,
+      "eval_rewards/chosen": 0.029622970148921013,
+      "eval_rewards/margins": 0.0815664604306221,
+      "eval_rewards/rejected": -0.05194348469376564,
+      "eval_runtime": 711.6478,
+      "eval_samples_per_second": 2.81,
+      "eval_steps_per_second": 1.405,
+      "step": 8800
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.269931107793567e-06,
+      "logits/chosen": -2.2280020713806152,
+      "logits/rejected": -2.136003255844116,
+      "logps/chosen": -208.7579803466797,
+      "logps/rejected": -222.9391326904297,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.035223446786403656,
+      "rewards/margins": 0.060886919498443604,
+      "rewards/rejected": -0.02566346526145935,
+      "step": 8810
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2642460327304655e-06,
+      "logits/chosen": -2.1614763736724854,
+      "logits/rejected": -2.2132813930511475,
+      "logps/chosen": -240.7371063232422,
+      "logps/rejected": -232.02880859375,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.038226418197155,
+      "rewards/margins": 0.07139203697443008,
+      "rewards/rejected": -0.03316562622785568,
+      "step": 8820
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.258562187475543e-06,
+      "logits/chosen": -2.15134859085083,
+      "logits/rejected": -2.1060047149658203,
+      "logps/chosen": -225.7406005859375,
+      "logps/rejected": -195.5087890625,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.012063100934028625,
+      "rewards/margins": 0.07280053198337555,
+      "rewards/rejected": -0.060737431049346924,
+      "step": 8830
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2528796016785196e-06,
+      "logits/chosen": -2.197204828262329,
+      "logits/rejected": -2.018406391143799,
+      "logps/chosen": -187.3249053955078,
+      "logps/rejected": -208.0103759765625,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.02593032643198967,
+      "rewards/margins": 0.11019430309534073,
+      "rewards/rejected": -0.08426396548748016,
+      "step": 8840
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.247198304982548e-06,
+      "logits/chosen": -2.239647388458252,
+      "logits/rejected": -2.0400216579437256,
+      "logps/chosen": -159.90098571777344,
+      "logps/rejected": -163.93194580078125,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.020705249160528183,
+      "rewards/margins": 0.07980392873287201,
+      "rewards/rejected": -0.05909866839647293,
+      "step": 8850
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2415183270240533e-06,
+      "logits/chosen": -2.512545108795166,
+      "logits/rejected": -2.268498182296753,
+      "logps/chosen": -197.340087890625,
+      "logps/rejected": -210.59591674804688,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.01989520527422428,
+      "rewards/margins": 0.08950191736221313,
+      "rewards/rejected": -0.0696067214012146,
+      "step": 8860
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2358396974325837e-06,
+      "logits/chosen": -2.317462682723999,
+      "logits/rejected": -2.1121644973754883,
+      "logps/chosen": -239.705322265625,
+      "logps/rejected": -221.26022338867188,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.025622522458434105,
+      "rewards/margins": 0.07821951061487198,
+      "rewards/rejected": -0.05259697511792183,
+      "step": 8870
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2301624458306525e-06,
+      "logits/chosen": -2.4108285903930664,
+      "logits/rejected": -2.1266798973083496,
+      "logps/chosen": -259.18951416015625,
+      "logps/rejected": -213.99862670898438,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0015859712148085237,
+      "rewards/margins": 0.061502885073423386,
+      "rewards/rejected": -0.05991692468523979,
+      "step": 8880
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2244866018335855e-06,
+      "logits/chosen": -2.2686455249786377,
+      "logits/rejected": -2.2580018043518066,
+      "logps/chosen": -215.9559326171875,
+      "logps/rejected": -237.7769012451172,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.011017450131475925,
+      "rewards/margins": 0.07413921505212784,
+      "rewards/rejected": -0.06312176585197449,
+      "step": 8890
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2188121950493648e-06,
+      "logits/chosen": -2.4074501991271973,
+      "logits/rejected": -2.0326874256134033,
+      "logps/chosen": -219.9821319580078,
+      "logps/rejected": -146.60345458984375,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.01238707359880209,
+      "rewards/margins": 0.0487365797162056,
+      "rewards/rejected": -0.06112365052103996,
+      "step": 8900
+    },
+    {
+      "epoch": 0.58,
+      "eval_logits/chosen": -2.305030584335327,
+      "eval_logits/rejected": -2.1172640323638916,
+      "eval_logps/chosen": -231.61558532714844,
+      "eval_logps/rejected": -220.36143493652344,
+      "eval_loss": 0.6898223757743835,
+      "eval_rewards/accuracies": 0.6485000252723694,
+      "eval_rewards/chosen": 0.003893795656040311,
+      "eval_rewards/margins": 0.09138916432857513,
+      "eval_rewards/rejected": -0.08749537914991379,
+      "eval_runtime": 712.4849,
+      "eval_samples_per_second": 2.807,
+      "eval_steps_per_second": 1.404,
+      "step": 8900
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2131392550784766e-06,
+      "logits/chosen": -2.4283366203308105,
+      "logits/rejected": -1.8540500402450562,
+      "logps/chosen": -285.9861145019531,
+      "logps/rejected": -198.90310668945312,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.011074522510170937,
+      "rewards/margins": 0.08481260389089584,
+      "rewards/rejected": -0.09588713943958282,
+      "step": 8910
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2074678115137533e-06,
+      "logits/chosen": -2.1023287773132324,
+      "logits/rejected": -2.0058627128601074,
+      "logps/chosen": -195.75587463378906,
+      "logps/rejected": -213.19921875,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.010389198549091816,
+      "rewards/margins": 0.12526783347129822,
+      "rewards/rejected": -0.13565704226493835,
+      "step": 8920
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.201797893940224e-06,
+      "logits/chosen": -2.190784454345703,
+      "logits/rejected": -1.9884440898895264,
+      "logps/chosen": -232.9307403564453,
+      "logps/rejected": -261.10540771484375,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0024665123783051968,
+      "rewards/margins": 0.07824081182479858,
+      "rewards/rejected": -0.07577430456876755,
+      "step": 8930
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.196129531934956e-06,
+      "logits/chosen": -2.2389838695526123,
+      "logits/rejected": -1.970949411392212,
+      "logps/chosen": -235.01522827148438,
+      "logps/rejected": -226.91268920898438,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.02898770570755005,
+      "rewards/margins": 0.09960027039051056,
+      "rewards/rejected": -0.07061255723237991,
+      "step": 8940
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.190462755066902e-06,
+      "logits/chosen": -2.251969814300537,
+      "logits/rejected": -2.020610809326172,
+      "logps/chosen": -263.11944580078125,
+      "logps/rejected": -244.75009155273438,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.007182478904724121,
+      "rewards/margins": 0.07121424376964569,
+      "rewards/rejected": -0.07839672267436981,
+      "step": 8950
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.184797592896746e-06,
+      "logits/chosen": -2.379193067550659,
+      "logits/rejected": -2.3389806747436523,
+      "logps/chosen": -233.12423706054688,
+      "logps/rejected": -215.5608673095703,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.01311335526406765,
+      "rewards/margins": 0.08969788253307343,
+      "rewards/rejected": -0.07658452540636063,
+      "step": 8960
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.17913407497675e-06,
+      "logits/chosen": -2.313098430633545,
+      "logits/rejected": -2.381880283355713,
+      "logps/chosen": -176.4674072265625,
+      "logps/rejected": -218.6103973388672,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.019145773723721504,
+      "rewards/margins": 0.0819178968667984,
+      "rewards/rejected": -0.06277212500572205,
+      "step": 8970
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.173472230850596e-06,
+      "logits/chosen": -2.4210402965545654,
+      "logits/rejected": -2.2281031608581543,
+      "logps/chosen": -201.3446044921875,
+      "logps/rejected": -165.3816680908203,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.010820412077009678,
+      "rewards/margins": 0.04480786249041557,
+      "rewards/rejected": -0.05562828108668327,
+      "step": 8980
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1678120900532375e-06,
+      "logits/chosen": -2.4019229412078857,
+      "logits/rejected": -2.1054940223693848,
+      "logps/chosen": -235.7376251220703,
+      "logps/rejected": -216.53738403320312,
+      "loss": 0.691,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.0017721873009577394,
+      "rewards/margins": 0.10060055553913116,
+      "rewards/rejected": -0.09882837533950806,
+      "step": 8990
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1621536821107412e-06,
+      "logits/chosen": -2.2904767990112305,
+      "logits/rejected": -2.159829616546631,
+      "logps/chosen": -201.26571655273438,
+      "logps/rejected": -169.5950164794922,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.013041026890277863,
+      "rewards/margins": 0.103615902364254,
+      "rewards/rejected": -0.09057489037513733,
+      "step": 9000
+    },
+    {
+      "epoch": 0.59,
+      "eval_logits/chosen": -2.3073229789733887,
+      "eval_logits/rejected": -2.1196234226226807,
+      "eval_logps/chosen": -230.892333984375,
+      "eval_logps/rejected": -219.00497436523438,
+      "eval_loss": 0.6897886991500854,
+      "eval_rewards/accuracies": 0.6399999856948853,
+      "eval_rewards/chosen": 0.011126398108899593,
+      "eval_rewards/margins": 0.08505717664957047,
+      "eval_rewards/rejected": -0.0739307776093483,
+      "eval_runtime": 715.638,
+      "eval_samples_per_second": 2.795,
+      "eval_steps_per_second": 1.397,
+      "step": 9000
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1564970365401346e-06,
+      "logits/chosen": -2.342783212661743,
+      "logits/rejected": -2.0789589881896973,
+      "logps/chosen": -183.83070373535156,
+      "logps/rejected": -154.10012817382812,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0032037317287176847,
+      "rewards/margins": 0.08277516812086105,
+      "rewards/rejected": -0.0795714408159256,
+      "step": 9010
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1508421828492527e-06,
+      "logits/chosen": -2.4671521186828613,
+      "logits/rejected": -2.1197142601013184,
+      "logps/chosen": -222.3729705810547,
+      "logps/rejected": -161.41038513183594,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.02819722518324852,
+      "rewards/margins": 0.07277707755565643,
+      "rewards/rejected": -0.04457986354827881,
+      "step": 9020
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.145189150536582e-06,
+      "logits/chosen": -2.1406853199005127,
+      "logits/rejected": -2.043962001800537,
+      "logps/chosen": -219.89431762695312,
+      "logps/rejected": -176.14846801757812,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.02275204285979271,
+      "rewards/margins": 0.05656403303146362,
+      "rewards/rejected": -0.03381199389696121,
+      "step": 9030
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.139537969091107e-06,
+      "logits/chosen": -2.1763834953308105,
+      "logits/rejected": -2.141878843307495,
+      "logps/chosen": -265.5818786621094,
+      "logps/rejected": -207.55709838867188,
+      "loss": 0.693,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 1.7970241970033385e-05,
+      "rewards/margins": 0.0414576455950737,
+      "rewards/rejected": -0.04143967479467392,
+      "step": 9040
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1338886679921603e-06,
+      "logits/chosen": -2.2526087760925293,
+      "logits/rejected": -2.184154748916626,
+      "logps/chosen": -240.57388305664062,
+      "logps/rejected": -225.2537841796875,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.009691650979220867,
+      "rewards/margins": 0.05090288445353508,
+      "rewards/rejected": -0.041211239993572235,
+      "step": 9050
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.128241276709263e-06,
+      "logits/chosen": -2.3358287811279297,
+      "logits/rejected": -2.2810444831848145,
+      "logps/chosen": -199.28738403320312,
+      "logps/rejected": -226.06655883789062,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.039743922650814056,
+      "rewards/margins": 0.06741781532764435,
+      "rewards/rejected": -0.027673888951539993,
+      "step": 9060
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1225958247019746e-06,
+      "logits/chosen": -2.3715434074401855,
+      "logits/rejected": -2.5027832984924316,
+      "logps/chosen": -184.3772430419922,
+      "logps/rejected": -209.1215362548828,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.027645772323012352,
+      "rewards/margins": 0.046046603471040726,
+      "rewards/rejected": -0.018400834873318672,
+      "step": 9070
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1169523414197383e-06,
+      "logits/chosen": -2.1513938903808594,
+      "logits/rejected": -2.152141571044922,
+      "logps/chosen": -196.04464721679688,
+      "logps/rejected": -224.1463165283203,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.027610694989562035,
+      "rewards/margins": 0.04780023545026779,
+      "rewards/rejected": -0.020189542323350906,
+      "step": 9080
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1113108563017267e-06,
+      "logits/chosen": -2.248032808303833,
+      "logits/rejected": -2.033977746963501,
+      "logps/chosen": -212.1124725341797,
+      "logps/rejected": -188.57859802246094,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.006707633379846811,
+      "rewards/margins": 0.07095328718423843,
+      "rewards/rejected": -0.06424565613269806,
+      "step": 9090
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.1056713987766905e-06,
+      "logits/chosen": -2.443134307861328,
+      "logits/rejected": -2.1070022583007812,
+      "logps/chosen": -212.5425262451172,
+      "logps/rejected": -172.75057983398438,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.02657800354063511,
+      "rewards/margins": 0.08923501521348953,
+      "rewards/rejected": -0.06265701353549957,
+      "step": 9100
+    },
+    {
+      "epoch": 0.6,
+      "eval_logits/chosen": -2.312913179397583,
+      "eval_logits/rejected": -2.1251325607299805,
+      "eval_logps/chosen": -229.99118041992188,
+      "eval_logps/rejected": -216.90176391601562,
+      "eval_loss": 0.6898962259292603,
+      "eval_rewards/accuracies": 0.6324999928474426,
+      "eval_rewards/chosen": 0.02013748697936535,
+      "eval_rewards/margins": 0.07303596287965775,
+      "eval_rewards/rejected": -0.05289847403764725,
+      "eval_runtime": 710.189,
+      "eval_samples_per_second": 2.816,
+      "eval_steps_per_second": 1.408,
+      "step": 9100
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.1000339982628022e-06,
+      "logits/chosen": -2.1159980297088623,
+      "logits/rejected": -2.1975278854370117,
+      "logps/chosen": -249.4115447998047,
+      "logps/rejected": -221.4599609375,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.01861291006207466,
+      "rewards/margins": 0.06368504464626312,
+      "rewards/rejected": -0.04507213830947876,
+      "step": 9110
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0943986841675043e-06,
+      "logits/chosen": -2.3297810554504395,
+      "logits/rejected": -2.0988831520080566,
+      "logps/chosen": -199.1885986328125,
+      "logps/rejected": -190.6521453857422,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.04022395610809326,
+      "rewards/margins": 0.10803340375423431,
+      "rewards/rejected": -0.06780944764614105,
+      "step": 9120
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.088765485887356e-06,
+      "logits/chosen": -2.3123505115509033,
+      "logits/rejected": -2.110137939453125,
+      "logps/chosen": -242.97427368164062,
+      "logps/rejected": -205.7440185546875,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.01740623079240322,
+      "rewards/margins": 0.04395180940628052,
+      "rewards/rejected": -0.026545578613877296,
+      "step": 9130
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.083134432807879e-06,
+      "logits/chosen": -2.2417685985565186,
+      "logits/rejected": -2.172234058380127,
+      "logps/chosen": -193.64578247070312,
+      "logps/rejected": -223.5891876220703,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.006263398565351963,
+      "rewards/margins": 0.08425328880548477,
+      "rewards/rejected": -0.07798988372087479,
+      "step": 9140
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.077505554303404e-06,
+      "logits/chosen": -2.3099794387817383,
+      "logits/rejected": -2.274794816970825,
+      "logps/chosen": -169.19174194335938,
+      "logps/rejected": -179.0936279296875,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.053602445870637894,
+      "rewards/margins": 0.05535256117582321,
+      "rewards/rejected": -0.0017501137917861342,
+      "step": 9150
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.071878879736918e-06,
+      "logits/chosen": -2.3148703575134277,
+      "logits/rejected": -2.1224913597106934,
+      "logps/chosen": -245.26870727539062,
+      "logps/rejected": -331.78924560546875,
+      "loss": 0.691,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.01570909097790718,
+      "rewards/margins": 0.08236773312091827,
+      "rewards/rejected": -0.06665865331888199,
+      "step": 9160
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0662544384599136e-06,
+      "logits/chosen": -2.2138946056365967,
+      "logits/rejected": -2.138765335083008,
+      "logps/chosen": -197.99423217773438,
+      "logps/rejected": -190.30654907226562,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.04112407937645912,
+      "rewards/margins": 0.08797650039196014,
+      "rewards/rejected": -0.046852417290210724,
+      "step": 9170
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0606322598122314e-06,
+      "logits/chosen": -2.2186291217803955,
+      "logits/rejected": -2.3340086936950684,
+      "logps/chosen": -185.21937561035156,
+      "logps/rejected": -208.082763671875,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.012941636145114899,
+      "rewards/margins": 0.03202500194311142,
+      "rewards/rejected": -0.019083363935351372,
+      "step": 9180
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0550123731219085e-06,
+      "logits/chosen": -2.4984421730041504,
+      "logits/rejected": -2.322842836380005,
+      "logps/chosen": -249.52920532226562,
+      "logps/rejected": -215.18814086914062,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.036506522446870804,
+      "rewards/margins": 0.06803809106349945,
+      "rewards/rejected": -0.03153156489133835,
+      "step": 9190
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0493948077050267e-06,
+      "logits/chosen": -2.162285327911377,
+      "logits/rejected": -1.980200171470642,
+      "logps/chosen": -190.65975952148438,
+      "logps/rejected": -181.0004425048828,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.025962283834815025,
+      "rewards/margins": 0.08430268615484238,
+      "rewards/rejected": -0.0583404079079628,
+      "step": 9200
+    },
+    {
+      "epoch": 0.6,
+      "eval_logits/chosen": -2.328345537185669,
+      "eval_logits/rejected": -2.139697551727295,
+      "eval_logps/chosen": -229.9346923828125,
+      "eval_logps/rejected": -217.44418334960938,
+      "eval_loss": 0.6898381114006042,
+      "eval_rewards/accuracies": 0.6355000138282776,
+      "eval_rewards/chosen": 0.020702635869383812,
+      "eval_rewards/margins": 0.07902555167675018,
+      "eval_rewards/rejected": -0.05832291021943092,
+      "eval_runtime": 713.3551,
+      "eval_samples_per_second": 2.804,
+      "eval_steps_per_second": 1.402,
+      "step": 9200
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0437795928655596e-06,
+      "logits/chosen": -2.351074457168579,
+      "logits/rejected": -2.3267416954040527,
+      "logps/chosen": -279.7415466308594,
+      "logps/rejected": -267.31201171875,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.019710825756192207,
+      "rewards/margins": 0.05754191428422928,
+      "rewards/rejected": -0.03783109039068222,
+      "step": 9210
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0381667578952184e-06,
+      "logits/chosen": -2.4281165599823,
+      "logits/rejected": -2.208249568939209,
+      "logps/chosen": -209.62942504882812,
+      "logps/rejected": -225.8242645263672,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.012434298172593117,
+      "rewards/margins": 0.10577799379825592,
+      "rewards/rejected": -0.09334369748830795,
+      "step": 9220
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0325563320732995e-06,
+      "logits/chosen": -2.4986178874969482,
+      "logits/rejected": -2.109630584716797,
+      "logps/chosen": -262.3493347167969,
+      "logps/rejected": -227.225341796875,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.009889942593872547,
+      "rewards/margins": 0.08893869817256927,
+      "rewards/rejected": -0.0790487602353096,
+      "step": 9230
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.026948344666532e-06,
+      "logits/chosen": -2.2422378063201904,
+      "logits/rejected": -2.198185443878174,
+      "logps/chosen": -199.4209747314453,
+      "logps/rejected": -207.3389434814453,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.002950438065454364,
+      "rewards/margins": 0.083625927567482,
+      "rewards/rejected": -0.08657635748386383,
+      "step": 9240
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.0213428249289257e-06,
+      "logits/chosen": -2.2211735248565674,
+      "logits/rejected": -2.1415927410125732,
+      "logps/chosen": -196.4319610595703,
+      "logps/rejected": -207.9313507080078,
+      "loss": 0.6879,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.021085821092128754,
+      "rewards/margins": 0.09691883623600006,
+      "rewards/rejected": -0.0758330225944519,
+      "step": 9250
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.0157398021016175e-06,
+      "logits/chosen": -2.2351865768432617,
+      "logits/rejected": -2.1974527835845947,
+      "logps/chosen": -144.62660217285156,
+      "logps/rejected": -199.48924255371094,
+      "loss": 0.691,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.02908751741051674,
+      "rewards/margins": 0.08657882362604141,
+      "rewards/rejected": -0.05749132111668587,
+      "step": 9260
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.010139305412719e-06,
+      "logits/chosen": -2.506202220916748,
+      "logits/rejected": -2.2831943035125732,
+      "logps/chosen": -275.13079833984375,
+      "logps/rejected": -239.76907348632812,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.014648482203483582,
+      "rewards/margins": 0.07002347707748413,
+      "rewards/rejected": -0.05537499859929085,
+      "step": 9270
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.0045413640771644e-06,
+      "logits/chosen": -2.2399230003356934,
+      "logits/rejected": -2.382997989654541,
+      "logps/chosen": -254.0187530517578,
+      "logps/rejected": -266.322021484375,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.01146581582725048,
+      "rewards/margins": 0.09047718346118927,
+      "rewards/rejected": -0.07901137322187424,
+      "step": 9280
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.998946007296558e-06,
+      "logits/chosen": -2.476470708847046,
+      "logits/rejected": -2.1356618404388428,
+      "logps/chosen": -306.3233947753906,
+      "logps/rejected": -251.1251678466797,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.02277219668030739,
+      "rewards/margins": 0.0823870599269867,
+      "rewards/rejected": -0.059614866971969604,
+      "step": 9290
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9933532642590215e-06,
+      "logits/chosen": -2.212050676345825,
+      "logits/rejected": -1.8353458642959595,
+      "logps/chosen": -184.3854522705078,
+      "logps/rejected": -143.99754333496094,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.04772832244634628,
+      "rewards/margins": 0.09772919118404388,
+      "rewards/rejected": -0.0500008650124073,
+      "step": 9300
+    },
+    {
+      "epoch": 0.61,
+      "eval_logits/chosen": -2.333347797393799,
+      "eval_logits/rejected": -2.1441400051116943,
+      "eval_logps/chosen": -231.38299560546875,
+      "eval_logps/rejected": -219.56932067871094,
+      "eval_loss": 0.689826488494873,
+      "eval_rewards/accuracies": 0.637499988079071,
+      "eval_rewards/chosen": 0.006219496019184589,
+      "eval_rewards/margins": 0.08579385280609131,
+      "eval_rewards/rejected": -0.079574353992939,
+      "eval_runtime": 709.6682,
+      "eval_samples_per_second": 2.818,
+      "eval_steps_per_second": 1.409,
+      "step": 9300
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.987763164139042e-06,
+      "logits/chosen": -2.3631813526153564,
+      "logits/rejected": -2.159517288208008,
+      "logps/chosen": -204.81552124023438,
+      "logps/rejected": -214.0128631591797,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.004395435098558664,
+      "rewards/margins": 0.06943001598119736,
+      "rewards/rejected": -0.06503458321094513,
+      "step": 9310
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.982175736097321e-06,
+      "logits/chosen": -2.047428846359253,
+      "logits/rejected": -2.0620810985565186,
+      "logps/chosen": -275.38446044921875,
+      "logps/rejected": -301.35516357421875,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.008434224873781204,
+      "rewards/margins": 0.08950887620449066,
+      "rewards/rejected": -0.09794311225414276,
+      "step": 9320
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9765910092806196e-06,
+      "logits/chosen": -2.2493948936462402,
+      "logits/rejected": -2.153007984161377,
+      "logps/chosen": -177.16024780273438,
+      "logps/rejected": -163.21905517578125,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.009152812883257866,
+      "rewards/margins": 0.05717051774263382,
+      "rewards/rejected": -0.048017702996730804,
+      "step": 9330
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9710090128216083e-06,
+      "logits/chosen": -2.3084399700164795,
+      "logits/rejected": -2.213273525238037,
+      "logps/chosen": -224.1075897216797,
+      "logps/rejected": -218.4535675048828,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.013775345869362354,
+      "rewards/margins": 0.10509101301431656,
+      "rewards/rejected": -0.11886636167764664,
+      "step": 9340
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9654297758387155e-06,
+      "logits/chosen": -2.1536035537719727,
+      "logits/rejected": -2.131392002105713,
+      "logps/chosen": -163.9993133544922,
+      "logps/rejected": -192.57957458496094,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.031292807310819626,
+      "rewards/margins": 0.07854814827442169,
+      "rewards/rejected": -0.10984095185995102,
+      "step": 9350
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9598533274359736e-06,
+      "logits/chosen": -2.3260645866394043,
+      "logits/rejected": -2.233445882797241,
+      "logps/chosen": -243.11318969726562,
+      "logps/rejected": -253.74349975585938,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.03432890772819519,
+      "rewards/margins": 0.03268744423985481,
+      "rewards/rejected": -0.0670163482427597,
+      "step": 9360
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9542796967028697e-06,
+      "logits/chosen": -2.334592819213867,
+      "logits/rejected": -2.218923330307007,
+      "logps/chosen": -219.28659057617188,
+      "logps/rejected": -207.3817596435547,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.01046024076640606,
+      "rewards/margins": 0.0633801594376564,
+      "rewards/rejected": -0.07384039461612701,
+      "step": 9370
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.948708912714192e-06,
+      "logits/chosen": -2.243697166442871,
+      "logits/rejected": -2.0024325847625732,
+      "logps/chosen": -252.65280151367188,
+      "logps/rejected": -229.02334594726562,
+      "loss": 0.692,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.05707541108131409,
+      "rewards/margins": 0.07306285202503204,
+      "rewards/rejected": -0.13013826310634613,
+      "step": 9380
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9431410045298786e-06,
+      "logits/chosen": -2.0937747955322266,
+      "logits/rejected": -2.023585796356201,
+      "logps/chosen": -219.1824951171875,
+      "logps/rejected": -225.77780151367188,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.020571475848555565,
+      "rewards/margins": 0.07571020722389221,
+      "rewards/rejected": -0.09628168493509293,
+      "step": 9390
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9375760011948654e-06,
+      "logits/chosen": -2.4148128032684326,
+      "logits/rejected": -2.2704997062683105,
+      "logps/chosen": -202.109130859375,
+      "logps/rejected": -234.7781982421875,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.024311980232596397,
+      "rewards/margins": 0.10348609834909439,
+      "rewards/rejected": -0.12779806554317474,
+      "step": 9400
+    },
+    {
+      "epoch": 0.62,
+      "eval_logits/chosen": -2.3321480751037598,
+      "eval_logits/rejected": -2.1431541442871094,
+      "eval_logps/chosen": -234.85800170898438,
+      "eval_logps/rejected": -222.5006866455078,
+      "eval_loss": 0.6898515224456787,
+      "eval_rewards/accuracies": 0.6334999799728394,
+      "eval_rewards/chosen": -0.028530515730381012,
+      "eval_rewards/margins": 0.08035717159509659,
+      "eval_rewards/rejected": -0.1088876873254776,
+      "eval_runtime": 710.5474,
+      "eval_samples_per_second": 2.815,
+      "eval_steps_per_second": 1.407,
+      "step": 9400
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.932013931738937e-06,
+      "logits/chosen": -2.310518741607666,
+      "logits/rejected": -2.0845718383789062,
+      "logps/chosen": -207.1325225830078,
+      "logps/rejected": -232.1420135498047,
+      "loss": 0.6856,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.03530799597501755,
+      "rewards/margins": 0.12127707153558731,
+      "rewards/rejected": -0.15658505260944366,
+      "step": 9410
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9264548251765717e-06,
+      "logits/chosen": -2.426779270172119,
+      "logits/rejected": -2.238455295562744,
+      "logps/chosen": -205.3507080078125,
+      "logps/rejected": -209.31491088867188,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.015480468980967999,
+      "rewards/margins": 0.07075979560613632,
+      "rewards/rejected": -0.0862402692437172,
+      "step": 9420
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9208987105067924e-06,
+      "logits/chosen": -2.2212510108947754,
+      "logits/rejected": -2.0826263427734375,
+      "logps/chosen": -216.4222412109375,
+      "logps/rejected": -198.42672729492188,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.01865190640091896,
+      "rewards/margins": 0.058921001851558685,
+      "rewards/rejected": -0.07757291197776794,
+      "step": 9430
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9153456167130154e-06,
+      "logits/chosen": -2.3300156593322754,
+      "logits/rejected": -2.324368715286255,
+      "logps/chosen": -206.3992156982422,
+      "logps/rejected": -240.15884399414062,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.023285966366529465,
+      "rewards/margins": 0.0756094753742218,
+      "rewards/rejected": -0.09889544546604156,
+      "step": 9440
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9097955727628975e-06,
+      "logits/chosen": -2.3564929962158203,
+      "logits/rejected": -2.353801965713501,
+      "logps/chosen": -196.1259765625,
+      "logps/rejected": -216.0775604248047,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.0014045886928215623,
+      "rewards/margins": 0.0801662728190422,
+      "rewards/rejected": -0.08157085627317429,
+      "step": 9450
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.904248607608187e-06,
+      "logits/chosen": -2.2641825675964355,
+      "logits/rejected": -2.2951583862304688,
+      "logps/chosen": -257.591552734375,
+      "logps/rejected": -216.25,
+      "loss": 0.692,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.003715710248798132,
+      "rewards/margins": 0.05765196681022644,
+      "rewards/rejected": -0.06136767938733101,
+      "step": 9460
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8987047501845714e-06,
+      "logits/chosen": -2.3341257572174072,
+      "logits/rejected": -2.302320957183838,
+      "logps/chosen": -166.62255859375,
+      "logps/rejected": -169.2261199951172,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0030190914403647184,
+      "rewards/margins": 0.08849085867404938,
+      "rewards/rejected": -0.08547177165746689,
+      "step": 9470
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8931640294115267e-06,
+      "logits/chosen": -2.1365644931793213,
+      "logits/rejected": -2.0182435512542725,
+      "logps/chosen": -193.88600158691406,
+      "logps/rejected": -189.7248992919922,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.0011992065701633692,
+      "rewards/margins": 0.11369111388921738,
+      "rewards/rejected": -0.11249189078807831,
+      "step": 9480
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8876264741921662e-06,
+      "logits/chosen": -2.102898120880127,
+      "logits/rejected": -2.123107433319092,
+      "logps/chosen": -189.058837890625,
+      "logps/rejected": -195.1671142578125,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.01066638994961977,
+      "rewards/margins": 0.11543774604797363,
+      "rewards/rejected": -0.10477133840322495,
+      "step": 9490
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8820921134130912e-06,
+      "logits/chosen": -2.3311290740966797,
+      "logits/rejected": -1.958857774734497,
+      "logps/chosen": -230.97128295898438,
+      "logps/rejected": -198.6006317138672,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.004591288510710001,
+      "rewards/margins": 0.12538619339466095,
+      "rewards/rejected": -0.1299774944782257,
+      "step": 9500
+    },
+    {
+      "epoch": 0.62,
+      "eval_logits/chosen": -2.332388162612915,
+      "eval_logits/rejected": -2.143458604812622,
+      "eval_logps/chosen": -232.95985412597656,
+      "eval_logps/rejected": -220.78396606445312,
+      "eval_loss": 0.6898452639579773,
+      "eval_rewards/accuracies": 0.6365000009536743,
+      "eval_rewards/chosen": -0.009548979811370373,
+      "eval_rewards/margins": 0.08217175304889679,
+      "eval_rewards/rejected": -0.09172075241804123,
+      "eval_runtime": 713.2212,
+      "eval_samples_per_second": 2.804,
+      "eval_steps_per_second": 1.402,
+      "step": 9500
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8765609759442378e-06,
+      "logits/chosen": -2.1907570362091064,
+      "logits/rejected": -2.0811009407043457,
+      "logps/chosen": -242.083251953125,
+      "logps/rejected": -238.7690887451172,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.0076196156442165375,
+      "rewards/margins": 0.07497996836900711,
+      "rewards/rejected": -0.08259958773851395,
+      "step": 9510
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8710330906387288e-06,
+      "logits/chosen": -2.3943378925323486,
+      "logits/rejected": -2.34335994720459,
+      "logps/chosen": -238.8350372314453,
+      "logps/rejected": -271.54937744140625,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.003474016208201647,
+      "rewards/margins": 0.09124413132667542,
+      "rewards/rejected": -0.09471814334392548,
+      "step": 9520
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8655084863327222e-06,
+      "logits/chosen": -2.304542303085327,
+      "logits/rejected": -2.324296474456787,
+      "logps/chosen": -183.95468139648438,
+      "logps/rejected": -195.6629180908203,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.004966600798070431,
+      "rewards/margins": 0.055461425334215164,
+      "rewards/rejected": -0.060428015887737274,
+      "step": 9530
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8599871918452603e-06,
+      "logits/chosen": -2.1602072715759277,
+      "logits/rejected": -2.1528382301330566,
+      "logps/chosen": -221.7826690673828,
+      "logps/rejected": -245.52969360351562,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.01309148408472538,
+      "rewards/margins": 0.10497549921274185,
+      "rewards/rejected": -0.09188400954008102,
+      "step": 9540
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8544692359781192e-06,
+      "logits/chosen": -2.3558590412139893,
+      "logits/rejected": -2.1197030544281006,
+      "logps/chosen": -186.4560546875,
+      "logps/rejected": -162.83966064453125,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.011286085471510887,
+      "rewards/margins": 0.07330868393182755,
+      "rewards/rejected": -0.062022604048252106,
+      "step": 9550
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8489546475156602e-06,
+      "logits/chosen": -2.5130443572998047,
+      "logits/rejected": -2.286351442337036,
+      "logps/chosen": -218.3134765625,
+      "logps/rejected": -207.4962615966797,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.011793679557740688,
+      "rewards/margins": 0.0735076516866684,
+      "rewards/rejected": -0.06171398237347603,
+      "step": 9560
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8434434552246778e-06,
+      "logits/chosen": -2.1478981971740723,
+      "logits/rejected": -2.0774495601654053,
+      "logps/chosen": -210.42529296875,
+      "logps/rejected": -207.1792755126953,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.007937637157738209,
+      "rewards/margins": 0.08016934990882874,
+      "rewards/rejected": -0.08810698240995407,
+      "step": 9570
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.837935687854251e-06,
+      "logits/chosen": -2.3582987785339355,
+      "logits/rejected": -2.111501693725586,
+      "logps/chosen": -219.9694366455078,
+      "logps/rejected": -197.4185333251953,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.015772990882396698,
+      "rewards/margins": 0.09249218553304672,
+      "rewards/rejected": -0.07671918720006943,
+      "step": 9580
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.832431374135592e-06,
+      "logits/chosen": -2.4764034748077393,
+      "logits/rejected": -2.07975697517395,
+      "logps/chosen": -249.00634765625,
+      "logps/rejected": -241.4837188720703,
+      "loss": 0.687,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.005709916818886995,
+      "rewards/margins": 0.11350512504577637,
+      "rewards/rejected": -0.11921503394842148,
+      "step": 9590
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8269305427818977e-06,
+      "logits/chosen": -2.4727559089660645,
+      "logits/rejected": -2.327918529510498,
+      "logps/chosen": -208.00125122070312,
+      "logps/rejected": -186.8267059326172,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.006150106433779001,
+      "rewards/margins": 0.06489264965057373,
+      "rewards/rejected": -0.05874254181981087,
+      "step": 9600
+    },
+    {
+      "epoch": 0.63,
+      "eval_logits/chosen": -2.3416812419891357,
+      "eval_logits/rejected": -2.1520164012908936,
+      "eval_logps/chosen": -229.97618103027344,
+      "eval_logps/rejected": -218.22509765625,
+      "eval_loss": 0.6898518204689026,
+      "eval_rewards/accuracies": 0.6384999752044678,
+      "eval_rewards/chosen": 0.02028742991387844,
+      "eval_rewards/margins": 0.08641922473907471,
+      "eval_rewards/rejected": -0.06613180041313171,
+      "eval_runtime": 712.8227,
+      "eval_samples_per_second": 2.806,
+      "eval_steps_per_second": 1.403,
+      "step": 9600
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.821433222488199e-06,
+      "logits/chosen": -2.339639663696289,
+      "logits/rejected": -2.0263071060180664,
+      "logps/chosen": -221.83694458007812,
+      "logps/rejected": -203.28822326660156,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.037778157740831375,
+      "rewards/margins": 0.09198880195617676,
+      "rewards/rejected": -0.05421064421534538,
+      "step": 9610
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8159394419312112e-06,
+      "logits/chosen": -2.377436399459839,
+      "logits/rejected": -2.1873252391815186,
+      "logps/chosen": -256.22357177734375,
+      "logps/rejected": -222.9322967529297,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.025064552202820778,
+      "rewards/margins": 0.13247425854206085,
+      "rewards/rejected": -0.10740969330072403,
+      "step": 9620
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8104492297691845e-06,
+      "logits/chosen": -2.3256943225860596,
+      "logits/rejected": -2.104449510574341,
+      "logps/chosen": -228.5015869140625,
+      "logps/rejected": -213.09912109375,
+      "loss": 0.692,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.019784385338425636,
+      "rewards/margins": 0.08710122108459473,
+      "rewards/rejected": -0.10688559710979462,
+      "step": 9630
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8049626146417562e-06,
+      "logits/chosen": -2.1221089363098145,
+      "logits/rejected": -1.9787824153900146,
+      "logps/chosen": -155.16111755371094,
+      "logps/rejected": -163.43637084960938,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.0008523104479536414,
+      "rewards/margins": 0.11988194286823273,
+      "rewards/rejected": -0.12073423713445663,
+      "step": 9640
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7994796251697983e-06,
+      "logits/chosen": -2.2313790321350098,
+      "logits/rejected": -2.09123158454895,
+      "logps/chosen": -196.5417022705078,
+      "logps/rejected": -248.27761840820312,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.04502807930111885,
+      "rewards/margins": 0.12171129137277603,
+      "rewards/rejected": -0.166739359498024,
+      "step": 9650
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.794000289955269e-06,
+      "logits/chosen": -2.2783684730529785,
+      "logits/rejected": -2.106417417526245,
+      "logps/chosen": -269.64556884765625,
+      "logps/rejected": -246.83901977539062,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.025853093713521957,
+      "rewards/margins": 0.08640275150537491,
+      "rewards/rejected": -0.11225583404302597,
+      "step": 9660
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7885246375810646e-06,
+      "logits/chosen": -2.2515721321105957,
+      "logits/rejected": -1.9777309894561768,
+      "logps/chosen": -224.28076171875,
+      "logps/rejected": -228.09130859375,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -6.888061761856079e-05,
+      "rewards/margins": 0.07572519034147263,
+      "rewards/rejected": -0.07579407840967178,
+      "step": 9670
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7830526966108713e-06,
+      "logits/chosen": -2.1138720512390137,
+      "logits/rejected": -1.9187238216400146,
+      "logps/chosen": -189.40728759765625,
+      "logps/rejected": -175.88150024414062,
+      "loss": 0.6841,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.029343629255890846,
+      "rewards/margins": 0.15085718035697937,
+      "rewards/rejected": -0.18020080029964447,
+      "step": 9680
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7775844955890129e-06,
+      "logits/chosen": -2.271742343902588,
+      "logits/rejected": -2.1086041927337646,
+      "logps/chosen": -210.3048095703125,
+      "logps/rejected": -209.052978515625,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.012415561825037003,
+      "rewards/margins": 0.103248730301857,
+      "rewards/rejected": -0.0908331573009491,
+      "step": 9690
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7721200630403046e-06,
+      "logits/chosen": -2.346208095550537,
+      "logits/rejected": -2.157820224761963,
+      "logps/chosen": -195.90628051757812,
+      "logps/rejected": -228.021240234375,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.0014390780124813318,
+      "rewards/margins": 0.06569032371044159,
+      "rewards/rejected": -0.06712940335273743,
+      "step": 9700
+    },
+    {
+      "epoch": 0.63,
+      "eval_logits/chosen": -2.3423051834106445,
+      "eval_logits/rejected": -2.1527085304260254,
+      "eval_logps/chosen": -231.52012634277344,
+      "eval_logps/rejected": -219.43946838378906,
+      "eval_loss": 0.6898437142372131,
+      "eval_rewards/accuracies": 0.6439999938011169,
+      "eval_rewards/chosen": 0.004848138429224491,
+      "eval_rewards/margins": 0.08312396705150604,
+      "eval_rewards/rejected": -0.07827582955360413,
+      "eval_runtime": 710.5463,
+      "eval_samples_per_second": 2.815,
+      "eval_steps_per_second": 1.407,
+      "step": 9700
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7666594274699037e-06,
+      "logits/chosen": -2.264838457107544,
+      "logits/rejected": -2.1133649349212646,
+      "logps/chosen": -250.9988555908203,
+      "logps/rejected": -223.15579223632812,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.0169143658131361,
+      "rewards/margins": 0.10361073911190033,
+      "rewards/rejected": -0.08669638633728027,
+      "step": 9710
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.76120261736316e-06,
+      "logits/chosen": -2.314434051513672,
+      "logits/rejected": -1.943964958190918,
+      "logps/chosen": -227.54330444335938,
+      "logps/rejected": -212.75125122070312,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.00828264094889164,
+      "rewards/margins": 0.12351206690073013,
+      "rewards/rejected": -0.13179472088813782,
+      "step": 9720
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.755749661185468e-06,
+      "logits/chosen": -2.382049322128296,
+      "logits/rejected": -1.948427438735962,
+      "logps/chosen": -290.7059631347656,
+      "logps/rejected": -242.9027557373047,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.03743572160601616,
+      "rewards/margins": 0.1103050485253334,
+      "rewards/rejected": -0.07286933809518814,
+      "step": 9730
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7503005873821183e-06,
+      "logits/chosen": -2.320734739303589,
+      "logits/rejected": -2.263913631439209,
+      "logps/chosen": -158.90725708007812,
+      "logps/rejected": -193.49929809570312,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.015210744924843311,
+      "rewards/margins": 0.08328135311603546,
+      "rewards/rejected": -0.06807061284780502,
+      "step": 9740
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.744855424378148e-06,
+      "logits/chosen": -2.1784415245056152,
+      "logits/rejected": -2.192660093307495,
+      "logps/chosen": -186.56448364257812,
+      "logps/rejected": -220.04287719726562,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.02062476985156536,
+      "rewards/margins": 0.10032866150140762,
+      "rewards/rejected": -0.07970388978719711,
+      "step": 9750
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7394142005781973e-06,
+      "logits/chosen": -2.121195077896118,
+      "logits/rejected": -2.2074551582336426,
+      "logps/chosen": -260.9930114746094,
+      "logps/rejected": -271.81256103515625,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.007180415093898773,
+      "rewards/margins": 0.07248598337173462,
+      "rewards/rejected": -0.06530557572841644,
+      "step": 9760
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7339769443663528e-06,
+      "logits/chosen": -2.3068153858184814,
+      "logits/rejected": -2.1621522903442383,
+      "logps/chosen": -142.29673767089844,
+      "logps/rejected": -153.0177764892578,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.006454641930758953,
+      "rewards/margins": 0.10017760843038559,
+      "rewards/rejected": -0.09372296929359436,
+      "step": 9770
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7285436841060078e-06,
+      "logits/chosen": -2.4975733757019043,
+      "logits/rejected": -2.209691047668457,
+      "logps/chosen": -274.84765625,
+      "logps/rejected": -240.65109252929688,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.012807024642825127,
+      "rewards/margins": 0.08757642656564713,
+      "rewards/rejected": -0.07476940006017685,
+      "step": 9780
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7231144481397083e-06,
+      "logits/chosen": -2.3894002437591553,
+      "logits/rejected": -2.312439441680908,
+      "logps/chosen": -221.06201171875,
+      "logps/rejected": -201.44149780273438,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.006636572536081076,
+      "rewards/margins": 0.058181844651699066,
+      "rewards/rejected": -0.06481841951608658,
+      "step": 9790
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7176892647890092e-06,
+      "logits/chosen": -2.457059383392334,
+      "logits/rejected": -2.2145180702209473,
+      "logps/chosen": -236.02871704101562,
+      "logps/rejected": -202.4846649169922,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.0036267780233174562,
+      "rewards/margins": 0.05337395519018173,
+      "rewards/rejected": -0.05700073391199112,
+      "step": 9800
+    },
+    {
+      "epoch": 0.64,
+      "eval_logits/chosen": -2.3302435874938965,
+      "eval_logits/rejected": -2.1416378021240234,
+      "eval_logps/chosen": -232.28138732910156,
+      "eval_logps/rejected": -219.88731384277344,
+      "eval_loss": 0.6898448467254639,
+      "eval_rewards/accuracies": 0.6420000195503235,
+      "eval_rewards/chosen": -0.002764492528513074,
+      "eval_rewards/margins": 0.07998983561992645,
+      "eval_rewards/rejected": -0.08275433629751205,
+      "eval_runtime": 711.5893,
+      "eval_samples_per_second": 2.811,
+      "eval_steps_per_second": 1.405,
+      "step": 9800
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7122681623543239e-06,
+      "logits/chosen": -2.4701716899871826,
+      "logits/rejected": -2.205655574798584,
+      "logps/chosen": -245.60400390625,
+      "logps/rejected": -244.6284637451172,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.013846084475517273,
+      "rewards/margins": 0.10238330066204071,
+      "rewards/rejected": -0.08853721618652344,
+      "step": 9810
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7068511691147788e-06,
+      "logits/chosen": -2.2074344158172607,
+      "logits/rejected": -2.2472825050354004,
+      "logps/chosen": -191.83944702148438,
+      "logps/rejected": -208.13674926757812,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.02291598729789257,
+      "rewards/margins": 0.07927681505680084,
+      "rewards/rejected": -0.05636082962155342,
+      "step": 9820
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7014383133280636e-06,
+      "logits/chosen": -2.4390132427215576,
+      "logits/rejected": -2.075756311416626,
+      "logps/chosen": -250.9913330078125,
+      "logps/rejected": -205.5496826171875,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.02356928028166294,
+      "rewards/margins": 0.07952290773391724,
+      "rewards/rejected": -0.10309220850467682,
+      "step": 9830
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.696029623230286e-06,
+      "logits/chosen": -2.4049618244171143,
+      "logits/rejected": -2.321725845336914,
+      "logps/chosen": -255.1918487548828,
+      "logps/rejected": -282.19464111328125,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.007304821163415909,
+      "rewards/margins": 0.08223724365234375,
+      "rewards/rejected": -0.07493243366479874,
+      "step": 9840
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.6906251270358229e-06,
+      "logits/chosen": -2.3903801441192627,
+      "logits/rejected": -2.23978853225708,
+      "logps/chosen": -264.2168273925781,
+      "logps/rejected": -220.81857299804688,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.013624888844788074,
+      "rewards/margins": 0.06343577802181244,
+      "rewards/rejected": -0.07706067711114883,
+      "step": 9850
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.685224852937174e-06,
+      "logits/chosen": -2.1878368854522705,
+      "logits/rejected": -2.0732967853546143,
+      "logps/chosen": -197.29331970214844,
+      "logps/rejected": -254.4430694580078,
+      "loss": 0.683,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.010714609175920486,
+      "rewards/margins": 0.15290780365467072,
+      "rewards/rejected": -0.14219316840171814,
+      "step": 9860
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6798288291048136e-06,
+      "logits/chosen": -2.151094436645508,
+      "logits/rejected": -2.0521655082702637,
+      "logps/chosen": -224.37606811523438,
+      "logps/rejected": -207.9828643798828,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.011344604194164276,
+      "rewards/margins": 0.12311786413192749,
+      "rewards/rejected": -0.13446247577667236,
+      "step": 9870
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6744370836870466e-06,
+      "logits/chosen": -2.555605888366699,
+      "logits/rejected": -2.2639527320861816,
+      "logps/chosen": -333.4253845214844,
+      "logps/rejected": -261.70599365234375,
+      "loss": 0.687,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0126813855022192,
+      "rewards/margins": 0.1179838627576828,
+      "rewards/rejected": -0.10530247539281845,
+      "step": 9880
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6690496448098576e-06,
+      "logits/chosen": -2.2291011810302734,
+      "logits/rejected": -1.9320964813232422,
+      "logps/chosen": -229.88858032226562,
+      "logps/rejected": -217.7265167236328,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.008523445576429367,
+      "rewards/margins": 0.0869704857468605,
+      "rewards/rejected": -0.09549392759799957,
+      "step": 9890
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6636665405767666e-06,
+      "logits/chosen": -2.3363430500030518,
+      "logits/rejected": -2.170015573501587,
+      "logps/chosen": -235.27737426757812,
+      "logps/rejected": -223.20523071289062,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.02493545040488243,
+      "rewards/margins": 0.06817851960659027,
+      "rewards/rejected": -0.043243080377578735,
+      "step": 9900
+    },
+    {
+      "epoch": 0.65,
+      "eval_logits/chosen": -2.327425956726074,
+      "eval_logits/rejected": -2.1391117572784424,
+      "eval_logps/chosen": -232.06900024414062,
+      "eval_logps/rejected": -220.34877014160156,
+      "eval_loss": 0.6898226141929626,
+      "eval_rewards/accuracies": 0.6434999704360962,
+      "eval_rewards/chosen": -0.0006405520252883434,
+      "eval_rewards/margins": 0.08672798424959183,
+      "eval_rewards/rejected": -0.08736853301525116,
+      "eval_runtime": 713.8035,
+      "eval_samples_per_second": 2.802,
+      "eval_steps_per_second": 1.401,
+      "step": 9900
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6582877990686827e-06,
+      "logits/chosen": -2.330827474594116,
+      "logits/rejected": -2.275949716567993,
+      "logps/chosen": -120.868408203125,
+      "logps/rejected": -154.09080505371094,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.004259592853486538,
+      "rewards/margins": 0.10891057550907135,
+      "rewards/rejected": -0.10465097427368164,
+      "step": 9910
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6529134483437562e-06,
+      "logits/chosen": -2.333859920501709,
+      "logits/rejected": -2.153388738632202,
+      "logps/chosen": -206.96316528320312,
+      "logps/rejected": -178.53024291992188,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.013889017514884472,
+      "rewards/margins": 0.11020763963460922,
+      "rewards/rejected": -0.12409665435552597,
+      "step": 9920
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.647543516437233e-06,
+      "logits/chosen": -2.296292781829834,
+      "logits/rejected": -2.237565517425537,
+      "logps/chosen": -202.46505737304688,
+      "logps/rejected": -229.85952758789062,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.034268446266651154,
+      "rewards/margins": 0.06709616631269455,
+      "rewards/rejected": -0.1013646125793457,
+      "step": 9930
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6421780313613088e-06,
+      "logits/chosen": -2.445279121398926,
+      "logits/rejected": -2.040759563446045,
+      "logps/chosen": -205.244384765625,
+      "logps/rejected": -181.84715270996094,
+      "loss": 0.691,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.005027498118579388,
+      "rewards/margins": 0.09792643785476685,
+      "rewards/rejected": -0.10295393317937851,
+      "step": 9940
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6368170211049816e-06,
+      "logits/chosen": -2.3535354137420654,
+      "logits/rejected": -1.8584728240966797,
+      "logps/chosen": -284.0115661621094,
+      "logps/rejected": -234.0505828857422,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.0021137488074600697,
+      "rewards/margins": 0.10150531679391861,
+      "rewards/rejected": -0.1036190614104271,
+      "step": 9950
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6314605136339074e-06,
+      "logits/chosen": -2.3524794578552246,
+      "logits/rejected": -2.1899707317352295,
+      "logps/chosen": -198.07577514648438,
+      "logps/rejected": -187.40274047851562,
+      "loss": 0.691,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.024890681728720665,
+      "rewards/margins": 0.07226412743330002,
+      "rewards/rejected": -0.09715481102466583,
+      "step": 9960
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6261085368902526e-06,
+      "logits/chosen": -2.551018238067627,
+      "logits/rejected": -2.202836751937866,
+      "logps/chosen": -265.8411865234375,
+      "logps/rejected": -229.21688842773438,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.009003100916743279,
+      "rewards/margins": 0.07005171477794647,
+      "rewards/rejected": -0.0790548101067543,
+      "step": 9970
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6207611187925503e-06,
+      "logits/chosen": -2.231933832168579,
+      "logits/rejected": -2.2483532428741455,
+      "logps/chosen": -209.96499633789062,
+      "logps/rejected": -265.6410827636719,
+      "loss": 0.6867,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.011381099000573158,
+      "rewards/margins": 0.09358282387256622,
+      "rewards/rejected": -0.10496392101049423,
+      "step": 9980
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6154182872355512e-06,
+      "logits/chosen": -2.2878527641296387,
+      "logits/rejected": -2.338225841522217,
+      "logps/chosen": -167.45220947265625,
+      "logps/rejected": -189.28424072265625,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.03634749725461006,
+      "rewards/margins": 0.07412412762641907,
+      "rewards/rejected": -0.11047162860631943,
+      "step": 9990
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.610080070090084e-06,
+      "logits/chosen": -2.275801181793213,
+      "logits/rejected": -2.2091450691223145,
+      "logps/chosen": -186.84176635742188,
+      "logps/rejected": -186.8584442138672,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.059204794466495514,
+      "rewards/margins": 0.09045000374317169,
+      "rewards/rejected": -0.149654820561409,
+      "step": 10000
+    },
+    {
+      "epoch": 0.65,
+      "eval_logits/chosen": -2.322659730911255,
+      "eval_logits/rejected": -2.1345160007476807,
+      "eval_logps/chosen": -233.91151428222656,
+      "eval_logps/rejected": -222.27159118652344,
+      "eval_loss": 0.6898568272590637,
+      "eval_rewards/accuracies": 0.6474999785423279,
+      "eval_rewards/chosen": -0.01906559430062771,
+      "eval_rewards/margins": 0.08753134310245514,
+      "eval_rewards/rejected": -0.1065969467163086,
+      "eval_runtime": 713.7271,
+      "eval_samples_per_second": 2.802,
+      "eval_steps_per_second": 1.401,
+      "step": 10000
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6047464952029034e-06,
+      "logits/chosen": -2.4548022747039795,
+      "logits/rejected": -2.32185697555542,
+      "logps/chosen": -254.0906219482422,
+      "logps/rejected": -272.3085021972656,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.001967624295502901,
+      "rewards/margins": 0.09991808235645294,
+      "rewards/rejected": -0.09795045852661133,
+      "step": 10010
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5994175903965486e-06,
+      "logits/chosen": -2.183396577835083,
+      "logits/rejected": -2.0627362728118896,
+      "logps/chosen": -254.3908233642578,
+      "logps/rejected": -266.36431884765625,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.04060421884059906,
+      "rewards/margins": 0.10219021886587143,
+      "rewards/rejected": -0.1427944153547287,
+      "step": 10020
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5940933834691977e-06,
+      "logits/chosen": -2.6259872913360596,
+      "logits/rejected": -1.9777504205703735,
+      "logps/chosen": -307.5574951171875,
+      "logps/rejected": -208.10971069335938,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.022915838286280632,
+      "rewards/margins": 0.08449498564004898,
+      "rewards/rejected": -0.10741082578897476,
+      "step": 10030
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.588773902194522e-06,
+      "logits/chosen": -2.1228013038635254,
+      "logits/rejected": -1.8389813899993896,
+      "logps/chosen": -208.42453002929688,
+      "logps/rejected": -221.58456420898438,
+      "loss": 0.693,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.04344822093844414,
+      "rewards/margins": 0.1740039587020874,
+      "rewards/rejected": -0.21745216846466064,
+      "step": 10040
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.583459174321541e-06,
+      "logits/chosen": -2.1011738777160645,
+      "logits/rejected": -1.9465796947479248,
+      "logps/chosen": -213.76220703125,
+      "logps/rejected": -196.18130493164062,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.04144889861345291,
+      "rewards/margins": 0.09513147920370102,
+      "rewards/rejected": -0.13658036291599274,
+      "step": 10050
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5781492275744797e-06,
+      "logits/chosen": -2.5177080631256104,
+      "logits/rejected": -2.132356882095337,
+      "logps/chosen": -295.3465881347656,
+      "logps/rejected": -280.4678039550781,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.011021384038031101,
+      "rewards/margins": 0.11339070647954941,
+      "rewards/rejected": -0.12441209703683853,
+      "step": 10060
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5728440896526215e-06,
+      "logits/chosen": -2.2357475757598877,
+      "logits/rejected": -2.054103374481201,
+      "logps/chosen": -276.3549499511719,
+      "logps/rejected": -238.15597534179688,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.02952994778752327,
+      "rewards/margins": 0.08003853261470795,
+      "rewards/rejected": -0.10956847667694092,
+      "step": 10070
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5675437882301633e-06,
+      "logits/chosen": -2.3144021034240723,
+      "logits/rejected": -2.1165995597839355,
+      "logps/chosen": -223.0723419189453,
+      "logps/rejected": -191.2317352294922,
+      "loss": 0.693,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.0402815006673336,
+      "rewards/margins": 0.02955157496035099,
+      "rewards/rejected": -0.06983307749032974,
+      "step": 10080
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5622483509560748e-06,
+      "logits/chosen": -2.194026470184326,
+      "logits/rejected": -2.2077908515930176,
+      "logps/chosen": -173.01951599121094,
+      "logps/rejected": -214.22494506835938,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.022903580218553543,
+      "rewards/margins": 0.11023157835006714,
+      "rewards/rejected": -0.13313516974449158,
+      "step": 10090
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5569578054539506e-06,
+      "logits/chosen": -2.266801357269287,
+      "logits/rejected": -1.928422212600708,
+      "logps/chosen": -279.8221130371094,
+      "logps/rejected": -209.8824920654297,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.0062502906657755375,
+      "rewards/margins": 0.13923415541648865,
+      "rewards/rejected": -0.14548444747924805,
+      "step": 10100
+    },
+    {
+      "epoch": 0.66,
+      "eval_logits/chosen": -2.3166568279266357,
+      "eval_logits/rejected": -2.12905216217041,
+      "eval_logps/chosen": -234.25631713867188,
+      "eval_logps/rejected": -222.29376220703125,
+      "eval_loss": 0.689877450466156,
+      "eval_rewards/accuracies": 0.6474999785423279,
+      "eval_rewards/chosen": -0.02251364104449749,
+      "eval_rewards/margins": 0.08430492877960205,
+      "eval_rewards/rejected": -0.10681857168674469,
+      "eval_runtime": 714.2219,
+      "eval_samples_per_second": 2.8,
+      "eval_steps_per_second": 1.4,
+      "step": 10100
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.551672179321867e-06,
+      "logits/chosen": -2.239980459213257,
+      "logits/rejected": -2.30826735496521,
+      "logps/chosen": -217.06527709960938,
+      "logps/rejected": -203.61270141601562,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.012083550915122032,
+      "rewards/margins": 0.07718555629253387,
+      "rewards/rejected": -0.08926911652088165,
+      "step": 10110
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5463915001322398e-06,
+      "logits/chosen": -2.280668258666992,
+      "logits/rejected": -2.135650157928467,
+      "logps/chosen": -249.37588500976562,
+      "logps/rejected": -242.8144073486328,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.03719538077712059,
+      "rewards/margins": 0.07806181162595749,
+      "rewards/rejected": -0.11525720357894897,
+      "step": 10120
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5411157954316784e-06,
+      "logits/chosen": -2.2775256633758545,
+      "logits/rejected": -2.181715250015259,
+      "logps/chosen": -201.49807739257812,
+      "logps/rejected": -196.48684692382812,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.003794357879087329,
+      "rewards/margins": 0.08678573369979858,
+      "rewards/rejected": -0.08299137651920319,
+      "step": 10130
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.535845092740843e-06,
+      "logits/chosen": -2.4420783519744873,
+      "logits/rejected": -2.282604217529297,
+      "logps/chosen": -237.28085327148438,
+      "logps/rejected": -254.278076171875,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.00040371305658482015,
+      "rewards/margins": 0.04601416364312172,
+      "rewards/rejected": -0.0456104576587677,
+      "step": 10140
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5305794195543005e-06,
+      "logits/chosen": -2.296903133392334,
+      "logits/rejected": -2.3559365272521973,
+      "logps/chosen": -206.9901123046875,
+      "logps/rejected": -198.97463989257812,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0018900141585618258,
+      "rewards/margins": 0.08915947377681732,
+      "rewards/rejected": -0.08726945519447327,
+      "step": 10150
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5253188033403816e-06,
+      "logits/chosen": -2.321024179458618,
+      "logits/rejected": -2.416475772857666,
+      "logps/chosen": -171.10528564453125,
+      "logps/rejected": -191.79251098632812,
+      "loss": 0.691,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.0037040214519947767,
+      "rewards/margins": 0.0402885265648365,
+      "rewards/rejected": -0.0439925454556942,
+      "step": 10160
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.520063271541037e-06,
+      "logits/chosen": -2.296424150466919,
+      "logits/rejected": -2.201472282409668,
+      "logps/chosen": -178.9801483154297,
+      "logps/rejected": -180.84454345703125,
+      "loss": 0.6852,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.00475387554615736,
+      "rewards/margins": 0.1316739171743393,
+      "rewards/rejected": -0.13642781972885132,
+      "step": 10170
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.5148128515716954e-06,
+      "logits/chosen": -2.529043674468994,
+      "logits/rejected": -1.8911247253417969,
+      "logps/chosen": -264.5617370605469,
+      "logps/rejected": -202.96563720703125,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.01967450976371765,
+      "rewards/margins": 0.10778073221445084,
+      "rewards/rejected": -0.08810621500015259,
+      "step": 10180
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.5095675708211197e-06,
+      "logits/chosen": -2.348248243331909,
+      "logits/rejected": -2.340000629425049,
+      "logps/chosen": -194.94430541992188,
+      "logps/rejected": -222.9235382080078,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.03783398121595383,
+      "rewards/margins": 0.046356312930583954,
+      "rewards/rejected": -0.08419029414653778,
+      "step": 10190
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.504327456651263e-06,
+      "logits/chosen": -2.28633189201355,
+      "logits/rejected": -2.213207244873047,
+      "logps/chosen": -262.3058166503906,
+      "logps/rejected": -244.20077514648438,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.009028220549225807,
+      "rewards/margins": 0.08859656751155853,
+      "rewards/rejected": -0.09762479364871979,
+      "step": 10200
+    },
+    {
+      "epoch": 0.67,
+      "eval_logits/chosen": -2.3150970935821533,
+      "eval_logits/rejected": -2.127429723739624,
+      "eval_logps/chosen": -231.98062133789062,
+      "eval_logps/rejected": -220.61843872070312,
+      "eval_loss": 0.6898481249809265,
+      "eval_rewards/accuracies": 0.6474999785423279,
+      "eval_rewards/chosen": 0.00024335407942999154,
+      "eval_rewards/margins": 0.09030859917402267,
+      "eval_rewards/rejected": -0.09006524831056595,
+      "eval_runtime": 712.515,
+      "eval_samples_per_second": 2.807,
+      "eval_steps_per_second": 1.403,
+      "step": 10200
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4990925363971284e-06,
+      "logits/chosen": -2.370110034942627,
+      "logits/rejected": -1.957165002822876,
+      "logps/chosen": -289.9840087890625,
+      "logps/rejected": -239.0626983642578,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.0006083324551582336,
+      "rewards/margins": 0.17567750811576843,
+      "rewards/rejected": -0.17628583312034607,
+      "step": 10210
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4938628373666236e-06,
+      "logits/chosen": -2.2375118732452393,
+      "logits/rejected": -2.292834997177124,
+      "logps/chosen": -172.5015106201172,
+      "logps/rejected": -181.63931274414062,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0007777426508255303,
+      "rewards/margins": 0.07265160232782364,
+      "rewards/rejected": -0.07187385857105255,
+      "step": 10220
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4886383868404203e-06,
+      "logits/chosen": -2.143146514892578,
+      "logits/rejected": -2.0093531608581543,
+      "logps/chosen": -163.1758575439453,
+      "logps/rejected": -161.16677856445312,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.019540909677743912,
+      "rewards/margins": 0.08020851016044617,
+      "rewards/rejected": -0.09974941611289978,
+      "step": 10230
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.483419212071813e-06,
+      "logits/chosen": -2.097904920578003,
+      "logits/rejected": -1.9259834289550781,
+      "logps/chosen": -188.40487670898438,
+      "logps/rejected": -189.32362365722656,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.000592652999330312,
+      "rewards/margins": 0.0810011625289917,
+      "rewards/rejected": -0.08040851354598999,
+      "step": 10240
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.478205340286573e-06,
+      "logits/chosen": -2.2346534729003906,
+      "logits/rejected": -2.2042670249938965,
+      "logps/chosen": -209.13229370117188,
+      "logps/rejected": -199.13385009765625,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.05523378401994705,
+      "rewards/margins": 0.08006395399570465,
+      "rewards/rejected": -0.1352977454662323,
+      "step": 10250
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4729967986828104e-06,
+      "logits/chosen": -2.3945605754852295,
+      "logits/rejected": -2.156639575958252,
+      "logps/chosen": -321.37115478515625,
+      "logps/rejected": -282.4629211425781,
+      "loss": 0.69,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.006125994957983494,
+      "rewards/margins": 0.0905977264046669,
+      "rewards/rejected": -0.08447173237800598,
+      "step": 10260
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4677936144308286e-06,
+      "logits/chosen": -2.3705971240997314,
+      "logits/rejected": -2.0719449520111084,
+      "logps/chosen": -222.4903564453125,
+      "logps/rejected": -207.2783966064453,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.017543844878673553,
+      "rewards/margins": 0.12851087749004364,
+      "rewards/rejected": -0.11096702516078949,
+      "step": 10270
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4625958146729864e-06,
+      "logits/chosen": -2.386382818222046,
+      "logits/rejected": -2.2117514610290527,
+      "logps/chosen": -220.2921142578125,
+      "logps/rejected": -212.52377319335938,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.00028302668943069875,
+      "rewards/margins": 0.08179637044668198,
+      "rewards/rejected": -0.08207939565181732,
+      "step": 10280
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4574034265235523e-06,
+      "logits/chosen": -2.474156141281128,
+      "logits/rejected": -1.9117343425750732,
+      "logps/chosen": -254.64987182617188,
+      "logps/rejected": -168.7136688232422,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.011786472983658314,
+      "rewards/margins": 0.11826670169830322,
+      "rewards/rejected": -0.10648022592067719,
+      "step": 10290
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.452216477068568e-06,
+      "logits/chosen": -2.342738389968872,
+      "logits/rejected": -1.8315894603729248,
+      "logps/chosen": -229.30612182617188,
+      "logps/rejected": -147.89767456054688,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.007388654164969921,
+      "rewards/margins": 0.11506316810846329,
+      "rewards/rejected": -0.10767451673746109,
+      "step": 10300
+    },
+    {
+      "epoch": 0.67,
+      "eval_logits/chosen": -2.3180654048919678,
+      "eval_logits/rejected": -2.130103826522827,
+      "eval_logps/chosen": -231.863525390625,
+      "eval_logps/rejected": -219.8981475830078,
+      "eval_loss": 0.6898233294487,
+      "eval_rewards/accuracies": 0.6434999704360962,
+      "eval_rewards/chosen": 0.0014140387065708637,
+      "eval_rewards/margins": 0.08427631109952927,
+      "eval_rewards/rejected": -0.08286228775978088,
+      "eval_runtime": 713.5886,
+      "eval_samples_per_second": 2.803,
+      "eval_steps_per_second": 1.401,
+      "step": 10300
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4470349933657004e-06,
+      "logits/chosen": -2.5185744762420654,
+      "logits/rejected": -2.2852203845977783,
+      "logps/chosen": -220.93069458007812,
+      "logps/rejected": -204.84841918945312,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.002010664436966181,
+      "rewards/margins": 0.07896588742733002,
+      "rewards/rejected": -0.08097656071186066,
+      "step": 10310
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4418590024441096e-06,
+      "logits/chosen": -2.41025972366333,
+      "logits/rejected": -2.01489520072937,
+      "logps/chosen": -250.54312133789062,
+      "logps/rejected": -192.83309936523438,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.029171252623200417,
+      "rewards/margins": 0.08775301277637482,
+      "rewards/rejected": -0.05858175829052925,
+      "step": 10320
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.436688531304297e-06,
+      "logits/chosen": -2.404268264770508,
+      "logits/rejected": -2.0696969032287598,
+      "logps/chosen": -216.67111206054688,
+      "logps/rejected": -219.65896606445312,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.01656418666243553,
+      "rewards/margins": 0.08477049320936203,
+      "rewards/rejected": -0.0682063102722168,
+      "step": 10330
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.431523606917974e-06,
+      "logits/chosen": -2.224674940109253,
+      "logits/rejected": -2.2193877696990967,
+      "logps/chosen": -207.0742950439453,
+      "logps/rejected": -223.9700164794922,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.019749853760004044,
+      "rewards/margins": 0.10281027853488922,
+      "rewards/rejected": -0.12256012111902237,
+      "step": 10340
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4263642562279162e-06,
+      "logits/chosen": -2.0139873027801514,
+      "logits/rejected": -1.9893707036972046,
+      "logps/chosen": -249.74459838867188,
+      "logps/rejected": -266.02679443359375,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.002572817262262106,
+      "rewards/margins": 0.07792092114686966,
+      "rewards/rejected": -0.0804937332868576,
+      "step": 10350
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4212105061478257e-06,
+      "logits/chosen": -2.0695881843566895,
+      "logits/rejected": -2.087667465209961,
+      "logps/chosen": -229.23046875,
+      "logps/rejected": -247.36196899414062,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.006600628606975079,
+      "rewards/margins": 0.10308027267456055,
+      "rewards/rejected": -0.09647965431213379,
+      "step": 10360
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4160623835621848e-06,
+      "logits/chosen": -2.4251503944396973,
+      "logits/rejected": -2.291647434234619,
+      "logps/chosen": -235.0546875,
+      "logps/rejected": -231.79080200195312,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.022564858198165894,
+      "rewards/margins": 0.08214254677295685,
+      "rewards/rejected": -0.05957768112421036,
+      "step": 10370
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4109199153261249e-06,
+      "logits/chosen": -2.2098567485809326,
+      "logits/rejected": -2.1036949157714844,
+      "logps/chosen": -273.9472961425781,
+      "logps/rejected": -253.73007202148438,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.02047353982925415,
+      "rewards/margins": 0.1009642630815506,
+      "rewards/rejected": -0.08049070835113525,
+      "step": 10380
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.405783128265278e-06,
+      "logits/chosen": -2.2883942127227783,
+      "logits/rejected": -2.265260934829712,
+      "logps/chosen": -204.190185546875,
+      "logps/rejected": -207.59140014648438,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.0005667827790603042,
+      "rewards/margins": 0.06667406111955643,
+      "rewards/rejected": -0.06724084168672562,
+      "step": 10390
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4006520491756427e-06,
+      "logits/chosen": -2.389329433441162,
+      "logits/rejected": -2.1550679206848145,
+      "logps/chosen": -194.2144012451172,
+      "logps/rejected": -140.29342651367188,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.020098352804780006,
+      "rewards/margins": 0.10719966888427734,
+      "rewards/rejected": -0.08710131794214249,
+      "step": 10400
+    },
+    {
+      "epoch": 0.68,
+      "eval_logits/chosen": -2.3169994354248047,
+      "eval_logits/rejected": -2.129032611846924,
+      "eval_logps/chosen": -230.2224578857422,
+      "eval_logps/rejected": -218.50982666015625,
+      "eval_loss": 0.6898374557495117,
+      "eval_rewards/accuracies": 0.6384999752044678,
+      "eval_rewards/chosen": 0.017825065180659294,
+      "eval_rewards/margins": 0.08680439740419388,
+      "eval_rewards/rejected": -0.06897933036088943,
+      "eval_runtime": 714.2722,
+      "eval_samples_per_second": 2.8,
+      "eval_steps_per_second": 1.4,
+      "step": 10400
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.39552670482344e-06,
+      "logits/chosen": -2.207252264022827,
+      "logits/rejected": -2.280210018157959,
+      "logps/chosen": -173.74078369140625,
+      "logps/rejected": -181.15975952148438,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.00858780462294817,
+      "rewards/margins": 0.07527503371238708,
+      "rewards/rejected": -0.06668722629547119,
+      "step": 10410
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3904071219449776e-06,
+      "logits/chosen": -2.2979884147644043,
+      "logits/rejected": -1.8623745441436768,
+      "logps/chosen": -196.11106872558594,
+      "logps/rejected": -119.69913482666016,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.037905577570199966,
+      "rewards/margins": 0.08248183131217957,
+      "rewards/rejected": -0.0445762574672699,
+      "step": 10420
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3852933272465068e-06,
+      "logits/chosen": -2.458289384841919,
+      "logits/rejected": -2.22627592086792,
+      "logps/chosen": -234.090087890625,
+      "logps/rejected": -206.09585571289062,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.03770025074481964,
+      "rewards/margins": 0.07229628413915634,
+      "rewards/rejected": -0.0345960296690464,
+      "step": 10430
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3801853474040873e-06,
+      "logits/chosen": -2.254831552505493,
+      "logits/rejected": -2.187377452850342,
+      "logps/chosen": -236.34634399414062,
+      "logps/rejected": -230.05392456054688,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.021748732775449753,
+      "rewards/margins": 0.10167870670557022,
+      "rewards/rejected": -0.07992996275424957,
+      "step": 10440
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3750832090634417e-06,
+      "logits/chosen": -2.4042470455169678,
+      "logits/rejected": -2.09578537940979,
+      "logps/chosen": -181.5123291015625,
+      "logps/rejected": -175.58468627929688,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.04660337418317795,
+      "rewards/margins": 0.08723708242177963,
+      "rewards/rejected": -0.040633708238601685,
+      "step": 10450
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3699869388398245e-06,
+      "logits/chosen": -2.2707431316375732,
+      "logits/rejected": -2.1249325275421143,
+      "logps/chosen": -209.69296264648438,
+      "logps/rejected": -199.55191040039062,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.03328350931406021,
+      "rewards/margins": 0.08312380313873291,
+      "rewards/rejected": -0.049840297549963,
+      "step": 10460
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3648965633178772e-06,
+      "logits/chosen": -2.2864489555358887,
+      "logits/rejected": -2.1960525512695312,
+      "logps/chosen": -197.67889404296875,
+      "logps/rejected": -219.58993530273438,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.030532773584127426,
+      "rewards/margins": 0.10499455034732819,
+      "rewards/rejected": -0.07446177303791046,
+      "step": 10470
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3598121090514938e-06,
+      "logits/chosen": -2.275895357131958,
+      "logits/rejected": -2.1503143310546875,
+      "logps/chosen": -180.6181640625,
+      "logps/rejected": -165.47055053710938,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.02389690652489662,
+      "rewards/margins": 0.08975638449192047,
+      "rewards/rejected": -0.06585947424173355,
+      "step": 10480
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3547336025636753e-06,
+      "logits/chosen": -2.2028050422668457,
+      "logits/rejected": -1.984891653060913,
+      "logps/chosen": -273.01007080078125,
+      "logps/rejected": -239.7392120361328,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.022864457219839096,
+      "rewards/margins": 0.08258132636547089,
+      "rewards/rejected": -0.059716880321502686,
+      "step": 10490
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3496610703464022e-06,
+      "logits/chosen": -2.351762533187866,
+      "logits/rejected": -2.1159586906433105,
+      "logps/chosen": -227.1230926513672,
+      "logps/rejected": -190.7210693359375,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.020914912223815918,
+      "rewards/margins": 0.09119327366352081,
+      "rewards/rejected": -0.0702783390879631,
+      "step": 10500
+    },
+    {
+      "epoch": 0.69,
+      "eval_logits/chosen": -2.320483684539795,
+      "eval_logits/rejected": -2.132249593734741,
+      "eval_logps/chosen": -229.9177703857422,
+      "eval_logps/rejected": -217.9020538330078,
+      "eval_loss": 0.6898301839828491,
+      "eval_rewards/accuracies": 0.6395000219345093,
+      "eval_rewards/chosen": 0.0208718404173851,
+      "eval_rewards/margins": 0.08377327024936676,
+      "eval_rewards/rejected": -0.06290142238140106,
+      "eval_runtime": 713.7053,
+      "eval_samples_per_second": 2.802,
+      "eval_steps_per_second": 1.401,
+      "step": 10500
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3445945388604848e-06,
+      "logits/chosen": -2.213752508163452,
+      "logits/rejected": -1.9681813716888428,
+      "logps/chosen": -239.2418670654297,
+      "logps/rejected": -207.7646942138672,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.010183857753872871,
+      "rewards/margins": 0.1168348640203476,
+      "rewards/rejected": -0.12701871991157532,
+      "step": 10510
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3395340345354358e-06,
+      "logits/chosen": -2.238507032394409,
+      "logits/rejected": -2.3673980236053467,
+      "logps/chosen": -223.3243408203125,
+      "logps/rejected": -249.09719848632812,
+      "loss": 0.6879,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.003734863130375743,
+      "rewards/margins": 0.08525262027978897,
+      "rewards/rejected": -0.08151774853467941,
+      "step": 10520
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.334479583769322e-06,
+      "logits/chosen": -2.470689296722412,
+      "logits/rejected": -2.1623778343200684,
+      "logps/chosen": -252.4402313232422,
+      "logps/rejected": -207.7847137451172,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.023985490202903748,
+      "rewards/margins": 0.05314163491129875,
+      "rewards/rejected": -0.029156142845749855,
+      "step": 10530
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3294312129286366e-06,
+      "logits/chosen": -2.312894344329834,
+      "logits/rejected": -2.193312168121338,
+      "logps/chosen": -263.58502197265625,
+      "logps/rejected": -255.36056518554688,
+      "loss": 0.691,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.031435929238796234,
+      "rewards/margins": 0.05446425825357437,
+      "rewards/rejected": -0.023028332740068436,
+      "step": 10540
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.324388948348153e-06,
+      "logits/chosen": -2.469510078430176,
+      "logits/rejected": -2.076355218887329,
+      "logps/chosen": -283.5762023925781,
+      "logps/rejected": -210.35165405273438,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.016617698594927788,
+      "rewards/margins": 0.0814189538359642,
+      "rewards/rejected": -0.06480126827955246,
+      "step": 10550
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.319352816330796e-06,
+      "logits/chosen": -2.5570406913757324,
+      "logits/rejected": -2.0373148918151855,
+      "logps/chosen": -281.7043151855469,
+      "logps/rejected": -184.83396911621094,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.014928947202861309,
+      "rewards/margins": 0.09585042297840118,
+      "rewards/rejected": -0.0809214860200882,
+      "step": 10560
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.314322843147494e-06,
+      "logits/chosen": -2.155149221420288,
+      "logits/rejected": -2.2533445358276367,
+      "logps/chosen": -172.06494140625,
+      "logps/rejected": -234.2112579345703,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.017391042783856392,
+      "rewards/margins": 0.07278571277856827,
+      "rewards/rejected": -0.09017674624919891,
+      "step": 10570
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3092990550370526e-06,
+      "logits/chosen": -2.4301834106445312,
+      "logits/rejected": -2.0875182151794434,
+      "logps/chosen": -345.9046325683594,
+      "logps/rejected": -267.7071228027344,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.0014622181188315153,
+      "rewards/margins": 0.07153777033090591,
+      "rewards/rejected": -0.0729999840259552,
+      "step": 10580
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3042814782060131e-06,
+      "logits/chosen": -2.3605728149414062,
+      "logits/rejected": -2.063471794128418,
+      "logps/chosen": -177.5745849609375,
+      "logps/rejected": -164.29254150390625,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.03928259015083313,
+      "rewards/margins": 0.10634209215641022,
+      "rewards/rejected": -0.06705950945615768,
+      "step": 10590
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.2992701388285112e-06,
+      "logits/chosen": -2.371594190597534,
+      "logits/rejected": -2.116765260696411,
+      "logps/chosen": -266.95880126953125,
+      "logps/rejected": -229.24685668945312,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.02390308678150177,
+      "rewards/margins": 0.06760050356388092,
+      "rewards/rejected": -0.04369742050766945,
+      "step": 10600
+    },
+    {
+      "epoch": 0.69,
+      "eval_logits/chosen": -2.317068099975586,
+      "eval_logits/rejected": -2.1291825771331787,
+      "eval_logps/chosen": -230.43104553222656,
+      "eval_logps/rejected": -218.47349548339844,
+      "eval_loss": 0.689825177192688,
+      "eval_rewards/accuracies": 0.6430000066757202,
+      "eval_rewards/chosen": 0.015739070251584053,
+      "eval_rewards/margins": 0.08435513079166412,
+      "eval_rewards/rejected": -0.06861607730388641,
+      "eval_runtime": 714.2332,
+      "eval_samples_per_second": 2.8,
+      "eval_steps_per_second": 1.4,
+      "step": 10600
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.29426506304615e-06,
+      "logits/chosen": -2.2151741981506348,
+      "logits/rejected": -2.148871898651123,
+      "logps/chosen": -225.4136199951172,
+      "logps/rejected": -209.6869354248047,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.0269019715487957,
+      "rewards/margins": 0.03751341626048088,
+      "rewards/rejected": -0.06441538780927658,
+      "step": 10610
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.289266276967855e-06,
+      "logits/chosen": -2.364128589630127,
+      "logits/rejected": -2.249577760696411,
+      "logps/chosen": -337.41046142578125,
+      "logps/rejected": -254.139892578125,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.011713572777807713,
+      "rewards/margins": 0.06887595355510712,
+      "rewards/rejected": -0.05716238543391228,
+      "step": 10620
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.284273806669745e-06,
+      "logits/chosen": -2.3198752403259277,
+      "logits/rejected": -2.095829963684082,
+      "logps/chosen": -241.2201690673828,
+      "logps/rejected": -268.35296630859375,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.018951773643493652,
+      "rewards/margins": 0.09592192620038986,
+      "rewards/rejected": -0.1148737221956253,
+      "step": 10630
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2792876781949884e-06,
+      "logits/chosen": -2.0602927207946777,
+      "logits/rejected": -1.8088810443878174,
+      "logps/chosen": -207.2493133544922,
+      "logps/rejected": -195.7181396484375,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.011766849085688591,
+      "rewards/margins": 0.07786474376916885,
+      "rewards/rejected": -0.08963160216808319,
+      "step": 10640
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.274307917553676e-06,
+      "logits/chosen": -2.3178839683532715,
+      "logits/rejected": -2.2395923137664795,
+      "logps/chosen": -196.51788330078125,
+      "logps/rejected": -232.0148162841797,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.008591088466346264,
+      "rewards/margins": 0.10897374153137207,
+      "rewards/rejected": -0.11756483465433121,
+      "step": 10650
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2693345507226767e-06,
+      "logits/chosen": -2.1067943572998047,
+      "logits/rejected": -2.191366672515869,
+      "logps/chosen": -230.4332733154297,
+      "logps/rejected": -244.55105590820312,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.004430090077221394,
+      "rewards/margins": 0.12004270404577255,
+      "rewards/rejected": -0.12447279691696167,
+      "step": 10660
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2643676036455099e-06,
+      "logits/chosen": -2.3527565002441406,
+      "logits/rejected": -2.2609760761260986,
+      "logps/chosen": -288.02587890625,
+      "logps/rejected": -241.6444854736328,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0017340302001684904,
+      "rewards/margins": 0.043596215546131134,
+      "rewards/rejected": -0.045330245047807693,
+      "step": 10670
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.259407102232203e-06,
+      "logits/chosen": -2.452852725982666,
+      "logits/rejected": -2.0722763538360596,
+      "logps/chosen": -277.25531005859375,
+      "logps/rejected": -212.7763671875,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.0037903212942183018,
+      "rewards/margins": 0.08465041220188141,
+      "rewards/rejected": -0.08086008578538895,
+      "step": 10680
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.254453072359163e-06,
+      "logits/chosen": -2.389622211456299,
+      "logits/rejected": -2.1636927127838135,
+      "logps/chosen": -232.2378387451172,
+      "logps/rejected": -217.2246551513672,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.017822107300162315,
+      "rewards/margins": 0.06853047758340836,
+      "rewards/rejected": -0.05070837587118149,
+      "step": 10690
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2495055398690337e-06,
+      "logits/chosen": -2.4718260765075684,
+      "logits/rejected": -2.228008508682251,
+      "logps/chosen": -228.7391357421875,
+      "logps/rejected": -225.41854858398438,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.011152736842632294,
+      "rewards/margins": 0.05962613224983215,
+      "rewards/rejected": -0.04847339540719986,
+      "step": 10700
+    },
+    {
+      "epoch": 0.7,
+      "eval_logits/chosen": -2.3170278072357178,
+      "eval_logits/rejected": -2.1292710304260254,
+      "eval_logps/chosen": -230.355224609375,
+      "eval_logps/rejected": -218.42803955078125,
+      "eval_loss": 0.6898084878921509,
+      "eval_rewards/accuracies": 0.6430000066757202,
+      "eval_rewards/chosen": 0.016497209668159485,
+      "eval_rewards/margins": 0.08465855568647385,
+      "eval_rewards/rejected": -0.06816134601831436,
+      "eval_runtime": 714.9419,
+      "eval_samples_per_second": 2.797,
+      "eval_steps_per_second": 1.399,
+      "step": 10700
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2445645305705718e-06,
+      "logits/chosen": -2.4910426139831543,
+      "logits/rejected": -2.1560773849487305,
+      "logps/chosen": -206.70126342773438,
+      "logps/rejected": -192.34877014160156,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.0038032480515539646,
+      "rewards/margins": 0.06290373206138611,
+      "rewards/rejected": -0.06670697778463364,
+      "step": 10710
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2396300702384995e-06,
+      "logits/chosen": -2.4755501747131348,
+      "logits/rejected": -2.2301669120788574,
+      "logps/chosen": -256.727294921875,
+      "logps/rejected": -224.15756225585938,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.010807778686285019,
+      "rewards/margins": 0.05095580965280533,
+      "rewards/rejected": -0.04014802724123001,
+      "step": 10720
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.234702184613381e-06,
+      "logits/chosen": -2.0976624488830566,
+      "logits/rejected": -2.176741600036621,
+      "logps/chosen": -212.48825073242188,
+      "logps/rejected": -221.2858428955078,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.029940495267510414,
+      "rewards/margins": 0.09133056551218033,
+      "rewards/rejected": -0.06139007955789566,
+      "step": 10730
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2297808994014793e-06,
+      "logits/chosen": -2.411856174468994,
+      "logits/rejected": -2.16345477104187,
+      "logps/chosen": -282.15887451171875,
+      "logps/rejected": -253.8297882080078,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03775983303785324,
+      "rewards/margins": 0.06563454121351242,
+      "rewards/rejected": -0.02787470817565918,
+      "step": 10740
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2248662402746314e-06,
+      "logits/chosen": -2.251694679260254,
+      "logits/rejected": -2.121366500854492,
+      "logps/chosen": -191.34400939941406,
+      "logps/rejected": -191.785400390625,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.014273548498749733,
+      "rewards/margins": 0.0655721127986908,
+      "rewards/rejected": -0.07984566688537598,
+      "step": 10750
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2199582328701045e-06,
+      "logits/chosen": -2.414492607116699,
+      "logits/rejected": -1.9229921102523804,
+      "logps/chosen": -289.251708984375,
+      "logps/rejected": -257.48602294921875,
+      "loss": 0.688,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.013818124309182167,
+      "rewards/margins": 0.08525200188159943,
+      "rewards/rejected": -0.07143385708332062,
+      "step": 10760
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2150569027904712e-06,
+      "logits/chosen": -2.3025307655334473,
+      "logits/rejected": -2.191779851913452,
+      "logps/chosen": -243.48434448242188,
+      "logps/rejected": -245.3304901123047,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.024991249665617943,
+      "rewards/margins": 0.08149056136608124,
+      "rewards/rejected": -0.056499313563108444,
+      "step": 10770
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.2101622756034688e-06,
+      "logits/chosen": -2.2971510887145996,
+      "logits/rejected": -2.281870126724243,
+      "logps/chosen": -221.53494262695312,
+      "logps/rejected": -199.281982421875,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.012190895155072212,
+      "rewards/margins": 0.08112286031246185,
+      "rewards/rejected": -0.0689319595694542,
+      "step": 10780
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.2052743768418715e-06,
+      "logits/chosen": -2.353773832321167,
+      "logits/rejected": -2.1345012187957764,
+      "logps/chosen": -243.76358032226562,
+      "logps/rejected": -218.4962158203125,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.04045509174466133,
+      "rewards/margins": 0.08739558607339859,
+      "rewards/rejected": -0.046940483152866364,
+      "step": 10790
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.2003932320033523e-06,
+      "logits/chosen": -2.463850498199463,
+      "logits/rejected": -2.155269145965576,
+      "logps/chosen": -223.14895629882812,
+      "logps/rejected": -234.60220336914062,
+      "loss": 0.6877,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.033537622541189194,
+      "rewards/margins": 0.10821393877267838,
+      "rewards/rejected": -0.07467631250619888,
+      "step": 10800
+    },
+    {
+      "epoch": 0.71,
+      "eval_logits/chosen": -2.3170695304870605,
+      "eval_logits/rejected": -2.129291534423828,
+      "eval_logps/chosen": -229.36056518554688,
+      "eval_logps/rejected": -217.1490020751953,
+      "eval_loss": 0.6898130178451538,
+      "eval_rewards/accuracies": 0.6434999704360962,
+      "eval_rewards/chosen": 0.026443878188729286,
+      "eval_rewards/margins": 0.08181502670049667,
+      "eval_rewards/rejected": -0.05537115037441254,
+      "eval_runtime": 714.9768,
+      "eval_samples_per_second": 2.797,
+      "eval_steps_per_second": 1.399,
+      "step": 10800
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1955188665503553e-06,
+      "logits/chosen": -2.1240930557250977,
+      "logits/rejected": -2.1801092624664307,
+      "logps/chosen": -205.2183074951172,
+      "logps/rejected": -190.25765991210938,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0009462200105190277,
+      "rewards/margins": 0.0582948699593544,
+      "rewards/rejected": -0.05924109369516373,
+      "step": 10810
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1906513059099566e-06,
+      "logits/chosen": -2.3073532581329346,
+      "logits/rejected": -1.9913088083267212,
+      "logps/chosen": -227.1727294921875,
+      "logps/rejected": -229.9739532470703,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.04316357523202896,
+      "rewards/margins": 0.11541776359081268,
+      "rewards/rejected": -0.07225419580936432,
+      "step": 10820
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.185790575473738e-06,
+      "logits/chosen": -2.247575283050537,
+      "logits/rejected": -2.0391554832458496,
+      "logps/chosen": -227.498046875,
+      "logps/rejected": -188.4443817138672,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.01635267771780491,
+      "rewards/margins": 0.08220528066158295,
+      "rewards/rejected": -0.06585261225700378,
+      "step": 10830
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1809367005976516e-06,
+      "logits/chosen": -2.3067519664764404,
+      "logits/rejected": -2.062891721725464,
+      "logps/chosen": -280.78778076171875,
+      "logps/rejected": -215.189697265625,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.017070407047867775,
+      "rewards/margins": 0.05449339747428894,
+      "rewards/rejected": -0.037422992289066315,
+      "step": 10840
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1760897066018842e-06,
+      "logits/chosen": -2.242995262145996,
+      "logits/rejected": -2.1077020168304443,
+      "logps/chosen": -219.93295288085938,
+      "logps/rejected": -224.17984008789062,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.031081268563866615,
+      "rewards/margins": 0.10243818908929825,
+      "rewards/rejected": -0.07135690748691559,
+      "step": 10850
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1712496187707327e-06,
+      "logits/chosen": -2.275620698928833,
+      "logits/rejected": -2.0290729999542236,
+      "logps/chosen": -240.3934783935547,
+      "logps/rejected": -239.35073852539062,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.000892448821105063,
+      "rewards/margins": 0.13023024797439575,
+      "rewards/rejected": -0.13112269341945648,
+      "step": 10860
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1664164623524646e-06,
+      "logits/chosen": -2.244417667388916,
+      "logits/rejected": -2.084033727645874,
+      "logps/chosen": -216.13034057617188,
+      "logps/rejected": -195.35897827148438,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.022446701303124428,
+      "rewards/margins": 0.0748591274023056,
+      "rewards/rejected": -0.052412427961826324,
+      "step": 10870
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1615902625591926e-06,
+      "logits/chosen": -2.2117581367492676,
+      "logits/rejected": -2.1228392124176025,
+      "logps/chosen": -227.09072875976562,
+      "logps/rejected": -223.296630859375,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.01869513839483261,
+      "rewards/margins": 0.056195490062236786,
+      "rewards/rejected": -0.0748906284570694,
+      "step": 10880
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.156771044566738e-06,
+      "logits/chosen": -2.3004848957061768,
+      "logits/rejected": -2.2247865200042725,
+      "logps/chosen": -264.3690490722656,
+      "logps/rejected": -228.86154174804688,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0024559935554862022,
+      "rewards/margins": 0.0770992636680603,
+      "rewards/rejected": -0.07464326918125153,
+      "step": 10890
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1519588335145037e-06,
+      "logits/chosen": -2.2524847984313965,
+      "logits/rejected": -2.393233060836792,
+      "logps/chosen": -205.72433471679688,
+      "logps/rejected": -228.1226806640625,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.018544599413871765,
+      "rewards/margins": 0.036989279091358185,
+      "rewards/rejected": -0.01844467595219612,
+      "step": 10900
+    },
+    {
+      "epoch": 0.71,
+      "eval_logits/chosen": -2.3110759258270264,
+      "eval_logits/rejected": -2.123832941055298,
+      "eval_logps/chosen": -230.8058624267578,
+      "eval_logps/rejected": -218.31472778320312,
+      "eval_loss": 0.6898163557052612,
+      "eval_rewards/accuracies": 0.6384999752044678,
+      "eval_rewards/chosen": 0.011990930885076523,
+      "eval_rewards/margins": 0.07901943475008011,
+      "eval_rewards/rejected": -0.06702849268913269,
+      "eval_runtime": 712.284,
+      "eval_samples_per_second": 2.808,
+      "eval_steps_per_second": 1.404,
+      "step": 10900
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1471536545053382e-06,
+      "logits/chosen": -2.343756914138794,
+      "logits/rejected": -2.3280563354492188,
+      "logps/chosen": -206.11807250976562,
+      "logps/rejected": -222.1238250732422,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.02916853502392769,
+      "rewards/margins": 0.0845358818769455,
+      "rewards/rejected": -0.05536733940243721,
+      "step": 10910
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1423555326054112e-06,
+      "logits/chosen": -2.2487311363220215,
+      "logits/rejected": -1.972808837890625,
+      "logps/chosen": -279.33294677734375,
+      "logps/rejected": -221.55191040039062,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.035016145557165146,
+      "rewards/margins": 0.1565500795841217,
+      "rewards/rejected": -0.12153393030166626,
+      "step": 10920
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1375644928440743e-06,
+      "logits/chosen": -2.3918063640594482,
+      "logits/rejected": -1.9648542404174805,
+      "logps/chosen": -234.3190155029297,
+      "logps/rejected": -176.64840698242188,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.023642176762223244,
+      "rewards/margins": 0.09833236038684845,
+      "rewards/rejected": -0.07469018548727036,
+      "step": 10930
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1327805602137396e-06,
+      "logits/chosen": -2.329948663711548,
+      "logits/rejected": -2.149320602416992,
+      "logps/chosen": -258.65191650390625,
+      "logps/rejected": -214.11959838867188,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0009221710497513413,
+      "rewards/margins": 0.08445750176906586,
+      "rewards/rejected": -0.0835353285074234,
+      "step": 10940
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1280037596697426e-06,
+      "logits/chosen": -2.181227922439575,
+      "logits/rejected": -2.1459288597106934,
+      "logps/chosen": -220.15103149414062,
+      "logps/rejected": -270.27410888671875,
+      "loss": 0.6846,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.007314229849725962,
+      "rewards/margins": 0.13196273148059845,
+      "rewards/rejected": -0.12464849650859833,
+      "step": 10950
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.123234116130216e-06,
+      "logits/chosen": -2.2392992973327637,
+      "logits/rejected": -2.20440673828125,
+      "logps/chosen": -185.53500366210938,
+      "logps/rejected": -203.2857208251953,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0035335198044776917,
+      "rewards/margins": 0.11696387827396393,
+      "rewards/rejected": -0.11343035846948624,
+      "step": 10960
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1184716544759553e-06,
+      "logits/chosen": -2.0971415042877197,
+      "logits/rejected": -2.0736968517303467,
+      "logps/chosen": -163.72232055664062,
+      "logps/rejected": -181.16928100585938,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.01641870103776455,
+      "rewards/margins": 0.05032380297780037,
+      "rewards/rejected": -0.03390509635210037,
+      "step": 10970
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1137163995502948e-06,
+      "logits/chosen": -2.5290920734405518,
+      "logits/rejected": -2.2951109409332275,
+      "logps/chosen": -218.7498779296875,
+      "logps/rejected": -193.7891387939453,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.02785148099064827,
+      "rewards/margins": 0.07741155475378036,
+      "rewards/rejected": -0.0495600700378418,
+      "step": 10980
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1089683761589717e-06,
+      "logits/chosen": -2.1559228897094727,
+      "logits/rejected": -1.998282790184021,
+      "logps/chosen": -235.5149383544922,
+      "logps/rejected": -228.6207275390625,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.04691413417458534,
+      "rewards/margins": 0.12020029127597809,
+      "rewards/rejected": -0.07328616082668304,
+      "step": 10990
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1042276090700044e-06,
+      "logits/chosen": -2.3137733936309814,
+      "logits/rejected": -2.258112668991089,
+      "logps/chosen": -211.1614227294922,
+      "logps/rejected": -247.2993927001953,
+      "loss": 0.691,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.008593087084591389,
+      "rewards/margins": 0.05968620628118515,
+      "rewards/rejected": -0.06827928870916367,
+      "step": 11000
+    },
+    {
+      "epoch": 0.72,
+      "eval_logits/chosen": -2.312530279159546,
+      "eval_logits/rejected": -2.125136613845825,
+      "eval_logps/chosen": -229.34445190429688,
+      "eval_logps/rejected": -216.98072814941406,
+      "eval_loss": 0.689818263053894,
+      "eval_rewards/accuracies": 0.6395000219345093,
+      "eval_rewards/chosen": 0.026604950428009033,
+      "eval_rewards/margins": 0.08029335737228394,
+      "eval_rewards/rejected": -0.0536884069442749,
+      "eval_runtime": 715.7804,
+      "eval_samples_per_second": 2.794,
+      "eval_steps_per_second": 1.397,
+      "step": 11000
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0994941230135536e-06,
+      "logits/chosen": -2.2787528038024902,
+      "logits/rejected": -1.983764410018921,
+      "logps/chosen": -228.21664428710938,
+      "logps/rejected": -204.8932342529297,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.04633352905511856,
+      "rewards/margins": 0.12736350297927856,
+      "rewards/rejected": -0.0810299813747406,
+      "step": 11010
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.094767942681804e-06,
+      "logits/chosen": -2.557792901992798,
+      "logits/rejected": -2.1338038444519043,
+      "logps/chosen": -249.3724365234375,
+      "logps/rejected": -218.78439331054688,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.015034383162856102,
+      "rewards/margins": 0.0861252024769783,
+      "rewards/rejected": -0.10115957260131836,
+      "step": 11020
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0900490927288248e-06,
+      "logits/chosen": -2.0898823738098145,
+      "logits/rejected": -2.044926166534424,
+      "logps/chosen": -258.9222106933594,
+      "logps/rejected": -215.9126739501953,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.015460757538676262,
+      "rewards/margins": 0.06466784328222275,
+      "rewards/rejected": -0.049207091331481934,
+      "step": 11030
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0853375977704511e-06,
+      "logits/chosen": -2.3183531761169434,
+      "logits/rejected": -2.1303043365478516,
+      "logps/chosen": -237.7437744140625,
+      "logps/rejected": -181.30435180664062,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.026807749643921852,
+      "rewards/margins": 0.07571287453174591,
+      "rewards/rejected": -0.04890512302517891,
+      "step": 11040
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0806334823841466e-06,
+      "logits/chosen": -2.1648404598236084,
+      "logits/rejected": -2.268681764602661,
+      "logps/chosen": -239.4142608642578,
+      "logps/rejected": -274.93853759765625,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.002016544807702303,
+      "rewards/margins": 0.061211831867694855,
+      "rewards/rejected": -0.05919528007507324,
+      "step": 11050
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0759367711088825e-06,
+      "logits/chosen": -2.175215244293213,
+      "logits/rejected": -2.3136703968048096,
+      "logps/chosen": -180.73880004882812,
+      "logps/rejected": -219.13919067382812,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.030937161296606064,
+      "rewards/margins": 0.049232009798288345,
+      "rewards/rejected": -0.018294844776391983,
+      "step": 11060
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0712474884450056e-06,
+      "logits/chosen": -2.2868709564208984,
+      "logits/rejected": -2.0856966972351074,
+      "logps/chosen": -200.55062866210938,
+      "logps/rejected": -180.42941284179688,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.02282613143324852,
+      "rewards/margins": 0.08665598928928375,
+      "rewards/rejected": -0.06382984668016434,
+      "step": 11070
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.066565658854112e-06,
+      "logits/chosen": -2.2322375774383545,
+      "logits/rejected": -2.195129871368408,
+      "logps/chosen": -127.13566589355469,
+      "logps/rejected": -132.34140014648438,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.006962375249713659,
+      "rewards/margins": 0.08438241481781006,
+      "rewards/rejected": -0.07742004096508026,
+      "step": 11080
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0618913067589165e-06,
+      "logits/chosen": -2.3321659564971924,
+      "logits/rejected": -2.115520715713501,
+      "logps/chosen": -213.48275756835938,
+      "logps/rejected": -184.6875457763672,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0233194287866354,
+      "rewards/margins": 0.08664701133966446,
+      "rewards/rejected": -0.06332757323980331,
+      "step": 11090
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0572244565431313e-06,
+      "logits/chosen": -2.1922779083251953,
+      "logits/rejected": -2.075507402420044,
+      "logps/chosen": -143.26010131835938,
+      "logps/rejected": -156.20030212402344,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.02156703732907772,
+      "rewards/margins": 0.07779018580913544,
+      "rewards/rejected": -0.05622314661741257,
+      "step": 11100
+    },
+    {
+      "epoch": 0.73,
+      "eval_logits/chosen": -2.313178300857544,
+      "eval_logits/rejected": -2.1258115768432617,
+      "eval_logps/chosen": -228.88185119628906,
+      "eval_logps/rejected": -216.5213623046875,
+      "eval_loss": 0.6898159384727478,
+      "eval_rewards/accuracies": 0.6359999775886536,
+      "eval_rewards/chosen": 0.031231021508574486,
+      "eval_rewards/margins": 0.08032544702291489,
+      "eval_rewards/rejected": -0.04909442365169525,
+      "eval_runtime": 712.9033,
+      "eval_samples_per_second": 2.805,
+      "eval_steps_per_second": 1.403,
+      "step": 11100
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0525651325513317e-06,
+      "logits/chosen": -2.316305637359619,
+      "logits/rejected": -2.294666290283203,
+      "logps/chosen": -327.4156188964844,
+      "logps/rejected": -310.8594970703125,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.03854013979434967,
+      "rewards/margins": 0.04599260538816452,
+      "rewards/rejected": -0.007452460937201977,
+      "step": 11110
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0479133590888351e-06,
+      "logits/chosen": -2.352234363555908,
+      "logits/rejected": -2.059476613998413,
+      "logps/chosen": -252.9855194091797,
+      "logps/rejected": -235.13720703125,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.024694135412573814,
+      "rewards/margins": 0.0912385955452919,
+      "rewards/rejected": -0.06654445827007294,
+      "step": 11120
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0432691604215695e-06,
+      "logits/chosen": -2.231292963027954,
+      "logits/rejected": -2.14487624168396,
+      "logps/chosen": -231.04995727539062,
+      "logps/rejected": -204.88375854492188,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0399702712893486,
+      "rewards/margins": 0.055469810962677,
+      "rewards/rejected": -0.015499535016715527,
+      "step": 11130
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0386325607759515e-06,
+      "logits/chosen": -2.2429559230804443,
+      "logits/rejected": -2.180182456970215,
+      "logps/chosen": -187.20619201660156,
+      "logps/rejected": -175.9566650390625,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.041839271783828735,
+      "rewards/margins": 0.08820907026529312,
+      "rewards/rejected": -0.04636979475617409,
+      "step": 11140
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0340035843387544e-06,
+      "logits/chosen": -2.34846830368042,
+      "logits/rejected": -2.021655797958374,
+      "logps/chosen": -177.23959350585938,
+      "logps/rejected": -156.76651000976562,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.024959605187177658,
+      "rewards/margins": 0.07109765708446503,
+      "rewards/rejected": -0.04613804817199707,
+      "step": 11150
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0293822552569887e-06,
+      "logits/chosen": -2.459784746170044,
+      "logits/rejected": -2.162053108215332,
+      "logps/chosen": -248.28750610351562,
+      "logps/rejected": -202.97152709960938,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.035963498055934906,
+      "rewards/margins": 0.10514490306377411,
+      "rewards/rejected": -0.0691814124584198,
+      "step": 11160
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0247685976377688e-06,
+      "logits/chosen": -2.2233853340148926,
+      "logits/rejected": -2.050699234008789,
+      "logps/chosen": -179.34494018554688,
+      "logps/rejected": -148.3097686767578,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.023400804027915,
+      "rewards/margins": 0.07796212285757065,
+      "rewards/rejected": -0.0545613169670105,
+      "step": 11170
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0201626355481939e-06,
+      "logits/chosen": -2.4143099784851074,
+      "logits/rejected": -2.1229655742645264,
+      "logps/chosen": -214.44876098632812,
+      "logps/rejected": -169.19232177734375,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.01671757362782955,
+      "rewards/margins": 0.0830346867442131,
+      "rewards/rejected": -0.0663171112537384,
+      "step": 11180
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0155643930152192e-06,
+      "logits/chosen": -2.4184117317199707,
+      "logits/rejected": -2.33054780960083,
+      "logps/chosen": -270.9144592285156,
+      "logps/rejected": -222.2581787109375,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.016928743571043015,
+      "rewards/margins": 0.059895895421504974,
+      "rewards/rejected": -0.04296715185046196,
+      "step": 11190
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0109738940255286e-06,
+      "logits/chosen": -2.244631290435791,
+      "logits/rejected": -1.9772279262542725,
+      "logps/chosen": -214.3131561279297,
+      "logps/rejected": -183.52496337890625,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.02669384703040123,
+      "rewards/margins": 0.06493046879768372,
+      "rewards/rejected": -0.038236625492572784,
+      "step": 11200
+    },
+    {
+      "epoch": 0.73,
+      "eval_logits/chosen": -2.3133840560913086,
+      "eval_logits/rejected": -2.126025438308716,
+      "eval_logps/chosen": -228.95094299316406,
+      "eval_logps/rejected": -216.6020965576172,
+      "eval_loss": 0.6898157000541687,
+      "eval_rewards/accuracies": 0.637499988079071,
+      "eval_rewards/chosen": 0.03054005466401577,
+      "eval_rewards/margins": 0.08044183254241943,
+      "eval_rewards/rejected": -0.04990177974104881,
+      "eval_runtime": 716.7898,
+      "eval_samples_per_second": 2.79,
+      "eval_steps_per_second": 1.395,
+      "step": 11200
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0063911625254155e-06,
+      "logits/chosen": -2.3134210109710693,
+      "logits/rejected": -2.174760341644287,
+      "logps/chosen": -229.67391967773438,
+      "logps/rejected": -238.20559692382812,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.06248721480369568,
+      "rewards/margins": 0.0790223479270935,
+      "rewards/rejected": -0.016535133123397827,
+      "step": 11210
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0018162224206502e-06,
+      "logits/chosen": -2.220797061920166,
+      "logits/rejected": -2.130765438079834,
+      "logps/chosen": -172.0438232421875,
+      "logps/rejected": -187.99996948242188,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.024309862405061722,
+      "rewards/margins": 0.1224609836935997,
+      "rewards/rejected": -0.09815112501382828,
+      "step": 11220
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 9.97249097576363e-07,
+      "logits/chosen": -2.467745304107666,
+      "logits/rejected": -2.184107780456543,
+      "logps/chosen": -230.2428741455078,
+      "logps/rejected": -197.06825256347656,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.028137261047959328,
+      "rewards/margins": 0.10317236185073853,
+      "rewards/rejected": -0.07503510266542435,
+      "step": 11230
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.92689811816913e-07,
+      "logits/chosen": -2.336843967437744,
+      "logits/rejected": -2.1020307540893555,
+      "logps/chosen": -213.20925903320312,
+      "logps/rejected": -173.838134765625,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0068805525079369545,
+      "rewards/margins": 0.08031884580850601,
+      "rewards/rejected": -0.08719939738512039,
+      "step": 11240
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.881383889257691e-07,
+      "logits/chosen": -2.2797598838806152,
+      "logits/rejected": -2.319304943084717,
+      "logps/chosen": -165.97084045410156,
+      "logps/rejected": -236.264404296875,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.03326042369008064,
+      "rewards/margins": 0.06215088814496994,
+      "rewards/rejected": -0.028890466317534447,
+      "step": 11250
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.835948526453817e-07,
+      "logits/chosen": -2.1445116996765137,
+      "logits/rejected": -2.3151485919952393,
+      "logps/chosen": -179.73304748535156,
+      "logps/rejected": -228.52316284179688,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0069632395170629025,
+      "rewards/margins": 0.05118337273597717,
+      "rewards/rejected": -0.044220130890607834,
+      "step": 11260
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.790592266770633e-07,
+      "logits/chosen": -2.5001702308654785,
+      "logits/rejected": -2.2158002853393555,
+      "logps/chosen": -266.3745422363281,
+      "logps/rejected": -245.5768280029297,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.02970188483595848,
+      "rewards/margins": 0.07441006600856781,
+      "rewards/rejected": -0.044708192348480225,
+      "step": 11270
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.745315346808584e-07,
+      "logits/chosen": -2.18174409866333,
+      "logits/rejected": -2.029189348220825,
+      "logps/chosen": -215.11642456054688,
+      "logps/rejected": -199.79173278808594,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.039590004831552505,
+      "rewards/margins": 0.07058382779359818,
+      "rewards/rejected": -0.03099382482469082,
+      "step": 11280
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.70011800275428e-07,
+      "logits/chosen": -2.236311674118042,
+      "logits/rejected": -2.1652560234069824,
+      "logps/chosen": -235.71142578125,
+      "logps/rejected": -255.64370727539062,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.018134601414203644,
+      "rewards/margins": 0.08486216515302658,
+      "rewards/rejected": -0.06672756373882294,
+      "step": 11290
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.655000470379206e-07,
+      "logits/chosen": -2.1597790718078613,
+      "logits/rejected": -2.0728516578674316,
+      "logps/chosen": -209.0076446533203,
+      "logps/rejected": -208.43862915039062,
+      "loss": 0.6879,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.013398419134318829,
+      "rewards/margins": 0.10113543272018433,
+      "rewards/rejected": -0.08773700892925262,
+      "step": 11300
+    },
+    {
+      "epoch": 0.74,
+      "eval_logits/chosen": -2.315458297729492,
+      "eval_logits/rejected": -2.127800226211548,
+      "eval_logps/chosen": -229.95440673828125,
+      "eval_logps/rejected": -217.7364959716797,
+      "eval_loss": 0.6897902488708496,
+      "eval_rewards/accuracies": 0.6380000114440918,
+      "eval_rewards/chosen": 0.020505422726273537,
+      "eval_rewards/margins": 0.08175148069858551,
+      "eval_rewards/rejected": -0.061246056109666824,
+      "eval_runtime": 712.1423,
+      "eval_samples_per_second": 2.808,
+      "eval_steps_per_second": 1.404,
+      "step": 11300
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.609962985038517e-07,
+      "logits/chosen": -2.427191734313965,
+      "logits/rejected": -2.028752088546753,
+      "logps/chosen": -213.2774658203125,
+      "logps/rejected": -200.72789001464844,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.020835842937231064,
+      "rewards/margins": 0.12575358152389526,
+      "rewards/rejected": -0.1049177423119545,
+      "step": 11310
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.565005781669786e-07,
+      "logits/chosen": -2.4781110286712646,
+      "logits/rejected": -2.1154332160949707,
+      "logps/chosen": -257.0857238769531,
+      "logps/rejected": -213.7594451904297,
+      "loss": 0.6879,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.0350492037832737,
+      "rewards/margins": 0.09986601769924164,
+      "rewards/rejected": -0.06481683254241943,
+      "step": 11320
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.520129094791822e-07,
+      "logits/chosen": -2.2771661281585693,
+      "logits/rejected": -2.1108384132385254,
+      "logps/chosen": -179.57064819335938,
+      "logps/rejected": -179.90342712402344,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.0007491965079680085,
+      "rewards/margins": 0.1053602546453476,
+      "rewards/rejected": -0.10610946267843246,
+      "step": 11330
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.475333158503389e-07,
+      "logits/chosen": -2.2677841186523438,
+      "logits/rejected": -1.972190499305725,
+      "logps/chosen": -215.33609008789062,
+      "logps/rejected": -175.67623901367188,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.024861928075551987,
+      "rewards/margins": 0.0549871027469635,
+      "rewards/rejected": -0.030125176534056664,
+      "step": 11340
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.430618206482053e-07,
+      "logits/chosen": -2.2450308799743652,
+      "logits/rejected": -2.185279130935669,
+      "logps/chosen": -135.9246063232422,
+      "logps/rejected": -141.6876220703125,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.044983211904764175,
+      "rewards/margins": 0.05167509242892265,
+      "rewards/rejected": -0.006691886577755213,
+      "step": 11350
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.385984471982892e-07,
+      "logits/chosen": -2.222777843475342,
+      "logits/rejected": -1.859256386756897,
+      "logps/chosen": -213.132080078125,
+      "logps/rejected": -176.3279571533203,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.016937825828790665,
+      "rewards/margins": 0.13930463790893555,
+      "rewards/rejected": -0.12236680835485458,
+      "step": 11360
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.341432187837343e-07,
+      "logits/chosen": -2.2822318077087402,
+      "logits/rejected": -2.191162586212158,
+      "logps/chosen": -187.1429443359375,
+      "logps/rejected": -212.93783569335938,
+      "loss": 0.686,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.03160820156335831,
+      "rewards/margins": 0.10926495492458344,
+      "rewards/rejected": -0.07765677571296692,
+      "step": 11370
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.29696158645193e-07,
+      "logits/chosen": -2.222790241241455,
+      "logits/rejected": -2.32561993598938,
+      "logps/chosen": -218.55014038085938,
+      "logps/rejected": -259.57879638671875,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.02552112378180027,
+      "rewards/margins": 0.12038824707269669,
+      "rewards/rejected": -0.09486713260412216,
+      "step": 11380
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.252572899807111e-07,
+      "logits/chosen": -2.2863879203796387,
+      "logits/rejected": -2.305600166320801,
+      "logps/chosen": -268.54376220703125,
+      "logps/rejected": -253.93896484375,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.020558223128318787,
+      "rewards/margins": 0.11582138389348984,
+      "rewards/rejected": -0.09526316076517105,
+      "step": 11390
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.208266359456003e-07,
+      "logits/chosen": -2.3670859336853027,
+      "logits/rejected": -2.106987237930298,
+      "logps/chosen": -202.4696807861328,
+      "logps/rejected": -219.4122772216797,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.03336096554994583,
+      "rewards/margins": 0.07371880859136581,
+      "rewards/rejected": -0.04035785049200058,
+      "step": 11400
+    },
+    {
+      "epoch": 0.75,
+      "eval_logits/chosen": -2.31719708442688,
+      "eval_logits/rejected": -2.1292054653167725,
+      "eval_logps/chosen": -230.30577087402344,
+      "eval_logps/rejected": -218.55361938476562,
+      "eval_loss": 0.6897911429405212,
+      "eval_rewards/accuracies": 0.6355000138282776,
+      "eval_rewards/chosen": 0.01699184998869896,
+      "eval_rewards/margins": 0.08640897274017334,
+      "eval_rewards/rejected": -0.06941711902618408,
+      "eval_runtime": 713.2491,
+      "eval_samples_per_second": 2.804,
+      "eval_steps_per_second": 1.402,
+      "step": 11400
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.164042196523229e-07,
+      "logits/chosen": -2.4905319213867188,
+      "logits/rejected": -2.1854054927825928,
+      "logps/chosen": -187.57745361328125,
+      "logps/rejected": -185.0448455810547,
+      "loss": 0.69,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.018236028030514717,
+      "rewards/margins": 0.1098506823182106,
+      "rewards/rejected": -0.09161464869976044,
+      "step": 11410
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.119900641703696e-07,
+      "logits/chosen": -2.4532532691955566,
+      "logits/rejected": -2.1825637817382812,
+      "logps/chosen": -211.77310180664062,
+      "logps/rejected": -174.50393676757812,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.016732508316636086,
+      "rewards/margins": 0.09065760672092438,
+      "rewards/rejected": -0.07392510026693344,
+      "step": 11420
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.075841925261364e-07,
+      "logits/chosen": -2.5480093955993652,
+      "logits/rejected": -2.3063910007476807,
+      "logps/chosen": -235.31533813476562,
+      "logps/rejected": -228.7047882080078,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.026038330048322678,
+      "rewards/margins": 0.0781828910112381,
+      "rewards/rejected": -0.05214455723762512,
+      "step": 11430
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.031866277028093e-07,
+      "logits/chosen": -2.2410953044891357,
+      "logits/rejected": -2.2712674140930176,
+      "logps/chosen": -190.3566436767578,
+      "logps/rejected": -225.5463104248047,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.007181202061474323,
+      "rewards/margins": 0.08380020409822464,
+      "rewards/rejected": -0.07661899924278259,
+      "step": 11440
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.987973926402391e-07,
+      "logits/chosen": -2.2060625553131104,
+      "logits/rejected": -2.242389440536499,
+      "logps/chosen": -209.5387420654297,
+      "logps/rejected": -218.03573608398438,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.028008287772536278,
+      "rewards/margins": 0.09084399789571762,
+      "rewards/rejected": -0.0628357082605362,
+      "step": 11450
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.944165102348273e-07,
+      "logits/chosen": -2.419955015182495,
+      "logits/rejected": -2.27038311958313,
+      "logps/chosen": -153.3720703125,
+      "logps/rejected": -171.96742248535156,
+      "loss": 0.6879,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0357503779232502,
+      "rewards/margins": 0.11337963491678238,
+      "rewards/rejected": -0.07762926071882248,
+      "step": 11460
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.900440033394018e-07,
+      "logits/chosen": -2.2393195629119873,
+      "logits/rejected": -2.2905590534210205,
+      "logps/chosen": -179.89437866210938,
+      "logps/rejected": -177.8740692138672,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.02621867135167122,
+      "rewards/margins": 0.07265409827232361,
+      "rewards/rejected": -0.04643542319536209,
+      "step": 11470
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.856798947631009e-07,
+      "logits/chosen": -2.263493537902832,
+      "logits/rejected": -2.2867515087127686,
+      "logps/chosen": -187.29135131835938,
+      "logps/rejected": -215.5435791015625,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.03584372624754906,
+      "rewards/margins": 0.11300251632928848,
+      "rewards/rejected": -0.07715878635644913,
+      "step": 11480
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.813242072712519e-07,
+      "logits/chosen": -2.0134589672088623,
+      "logits/rejected": -1.897157073020935,
+      "logps/chosen": -162.03453063964844,
+      "logps/rejected": -171.01966857910156,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.001722379820421338,
+      "rewards/margins": 0.0784909725189209,
+      "rewards/rejected": -0.08021334558725357,
+      "step": 11490
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.769769635852557e-07,
+      "logits/chosen": -2.2438220977783203,
+      "logits/rejected": -2.300549268722534,
+      "logps/chosen": -213.259521484375,
+      "logps/rejected": -182.95872497558594,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.020482342690229416,
+      "rewards/margins": 0.06456250697374344,
+      "rewards/rejected": -0.04408016428351402,
+      "step": 11500
+    },
+    {
+      "epoch": 0.75,
+      "eval_logits/chosen": -2.3182504177093506,
+      "eval_logits/rejected": -2.1302804946899414,
+      "eval_logps/chosen": -230.00030517578125,
+      "eval_logps/rejected": -217.7165069580078,
+      "eval_loss": 0.6898082494735718,
+      "eval_rewards/accuracies": 0.6294999718666077,
+      "eval_rewards/chosen": 0.02004634030163288,
+      "eval_rewards/margins": 0.08109237998723984,
+      "eval_rewards/rejected": -0.06104603409767151,
+      "eval_runtime": 713.4185,
+      "eval_samples_per_second": 2.803,
+      "eval_steps_per_second": 1.402,
+      "step": 11500
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.726381863824635e-07,
+      "logits/chosen": -2.4795243740081787,
+      "logits/rejected": -2.1508588790893555,
+      "logps/chosen": -280.99859619140625,
+      "logps/rejected": -221.33462524414062,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.04078471660614014,
+      "rewards/margins": 0.0774230808019638,
+      "rewards/rejected": -0.036638353019952774,
+      "step": 11510
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.683078982960638e-07,
+      "logits/chosen": -2.23110294342041,
+      "logits/rejected": -1.9223436117172241,
+      "logps/chosen": -224.91244506835938,
+      "logps/rejected": -185.7821044921875,
+      "loss": 0.6873,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0022748790215700865,
+      "rewards/margins": 0.1096058264374733,
+      "rewards/rejected": -0.11188069730997086,
+      "step": 11520
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.639861219149584e-07,
+      "logits/chosen": -2.0781750679016113,
+      "logits/rejected": -2.1381192207336426,
+      "logps/chosen": -263.2950134277344,
+      "logps/rejected": -229.32223510742188,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.006869921926409006,
+      "rewards/margins": 0.08951739221811295,
+      "rewards/rejected": -0.08264746516942978,
+      "step": 11530
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.596728797836532e-07,
+      "logits/chosen": -2.211719274520874,
+      "logits/rejected": -2.0574100017547607,
+      "logps/chosen": -212.7632293701172,
+      "logps/rejected": -255.2969512939453,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.025454815477132797,
+      "rewards/margins": 0.12940728664398193,
+      "rewards/rejected": -0.10395244508981705,
+      "step": 11540
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.553681944021294e-07,
+      "logits/chosen": -2.264085054397583,
+      "logits/rejected": -2.2858448028564453,
+      "logps/chosen": -237.9873809814453,
+      "logps/rejected": -227.55477905273438,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.010966275818645954,
+      "rewards/margins": 0.08914720267057419,
+      "rewards/rejected": -0.07818093150854111,
+      "step": 11550
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.510720882257365e-07,
+      "logits/chosen": -2.041898727416992,
+      "logits/rejected": -2.1856493949890137,
+      "logps/chosen": -158.22018432617188,
+      "logps/rejected": -210.814208984375,
+      "loss": 0.6862,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.030803903937339783,
+      "rewards/margins": 0.1136208325624466,
+      "rewards/rejected": -0.08281692862510681,
+      "step": 11560
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.467845836650667e-07,
+      "logits/chosen": -1.929513692855835,
+      "logits/rejected": -1.977299451828003,
+      "logps/chosen": -207.1142578125,
+      "logps/rejected": -222.85317993164062,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.011523036286234856,
+      "rewards/margins": 0.09183812886476517,
+      "rewards/rejected": -0.08031509816646576,
+      "step": 11570
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.425057030858461e-07,
+      "logits/chosen": -2.1237521171569824,
+      "logits/rejected": -1.9577020406723022,
+      "logps/chosen": -155.14407348632812,
+      "logps/rejected": -192.67381286621094,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.009118120186030865,
+      "rewards/margins": 0.08506849408149719,
+      "rewards/rejected": -0.07595036923885345,
+      "step": 11580
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.382354688088098e-07,
+      "logits/chosen": -2.2753641605377197,
+      "logits/rejected": -2.135500192642212,
+      "logps/chosen": -157.50479125976562,
+      "logps/rejected": -181.17617797851562,
+      "loss": 0.69,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.006505739875137806,
+      "rewards/margins": 0.08248453587293625,
+      "rewards/rejected": -0.07597880065441132,
+      "step": 11590
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.33973903109594e-07,
+      "logits/chosen": -2.396981954574585,
+      "logits/rejected": -2.1408324241638184,
+      "logps/chosen": -216.294677734375,
+      "logps/rejected": -194.80892944335938,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0030242991633713245,
+      "rewards/margins": 0.10075845569372177,
+      "rewards/rejected": -0.09773416072130203,
+      "step": 11600
+    },
+    {
+      "epoch": 0.76,
+      "eval_logits/chosen": -2.314687967300415,
+      "eval_logits/rejected": -2.126920223236084,
+      "eval_logps/chosen": -231.07017517089844,
+      "eval_logps/rejected": -219.44680786132812,
+      "eval_loss": 0.6898018717765808,
+      "eval_rewards/accuracies": 0.6370000243186951,
+      "eval_rewards/chosen": 0.009347718209028244,
+      "eval_rewards/margins": 0.08769676089286804,
+      "eval_rewards/rejected": -0.0783490464091301,
+      "eval_runtime": 711.5416,
+      "eval_samples_per_second": 2.811,
+      "eval_steps_per_second": 1.405,
+      "step": 11600
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.297210282186102e-07,
+      "logits/chosen": -2.1594557762145996,
+      "logits/rejected": -2.1344146728515625,
+      "logps/chosen": -232.7439727783203,
+      "logps/rejected": -261.56768798828125,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.027264848351478577,
+      "rewards/margins": 0.07457095384597778,
+      "rewards/rejected": -0.10183580219745636,
+      "step": 11610
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.254768663209397e-07,
+      "logits/chosen": -2.260918378829956,
+      "logits/rejected": -2.054774522781372,
+      "logps/chosen": -277.40802001953125,
+      "logps/rejected": -221.30020141601562,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.01171756163239479,
+      "rewards/margins": 0.0636371374130249,
+      "rewards/rejected": -0.05191957205533981,
+      "step": 11620
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.212414395562079e-07,
+      "logits/chosen": -2.1018333435058594,
+      "logits/rejected": -2.234898090362549,
+      "logps/chosen": -231.6614990234375,
+      "logps/rejected": -261.87396240234375,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.014473098330199718,
+      "rewards/margins": 0.06291624903678894,
+      "rewards/rejected": -0.07738934457302094,
+      "step": 11630
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.170147700184775e-07,
+      "logits/chosen": -2.3055419921875,
+      "logits/rejected": -2.1873691082000732,
+      "logps/chosen": -254.23110961914062,
+      "logps/rejected": -254.5798797607422,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.022202350199222565,
+      "rewards/margins": 0.09281570464372635,
+      "rewards/rejected": -0.07061335444450378,
+      "step": 11640
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.127968797561242e-07,
+      "logits/chosen": -2.353506565093994,
+      "logits/rejected": -2.0914809703826904,
+      "logps/chosen": -223.8273162841797,
+      "logps/rejected": -212.8360595703125,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.0014341063797473907,
+      "rewards/margins": 0.11431723833084106,
+      "rewards/rejected": -0.11288313567638397,
+      "step": 11650
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.085877907717338e-07,
+      "logits/chosen": -2.246596574783325,
+      "logits/rejected": -2.2063441276550293,
+      "logps/chosen": -219.5164794921875,
+      "logps/rejected": -218.2258758544922,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.013950645923614502,
+      "rewards/margins": 0.10388661921024323,
+      "rewards/rejected": -0.08993595838546753,
+      "step": 11660
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.043875250219732e-07,
+      "logits/chosen": -2.2046780586242676,
+      "logits/rejected": -2.141763687133789,
+      "logps/chosen": -231.0264434814453,
+      "logps/rejected": -213.81973266601562,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.016217241063714027,
+      "rewards/margins": 0.03862147778272629,
+      "rewards/rejected": -0.054838716983795166,
+      "step": 11670
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.001961044174881e-07,
+      "logits/chosen": -2.3708043098449707,
+      "logits/rejected": -2.1657023429870605,
+      "logps/chosen": -230.3643341064453,
+      "logps/rejected": -176.6756591796875,
+      "loss": 0.6937,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.01899898424744606,
+      "rewards/margins": 0.03499449044466019,
+      "rewards/rejected": -0.053993482142686844,
+      "step": 11680
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 7.960135508227795e-07,
+      "logits/chosen": -2.3832452297210693,
+      "logits/rejected": -2.029789686203003,
+      "logps/chosen": -294.03143310546875,
+      "logps/rejected": -236.58212280273438,
+      "loss": 0.691,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.001624174416065216,
+      "rewards/margins": 0.06866665184497833,
+      "rewards/rejected": -0.07029082626104355,
+      "step": 11690
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.91839886056098e-07,
+      "logits/chosen": -2.3958792686462402,
+      "logits/rejected": -2.139774799346924,
+      "logps/chosen": -285.03851318359375,
+      "logps/rejected": -261.7726135253906,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.012762362137436867,
+      "rewards/margins": 0.07304862141609192,
+      "rewards/rejected": -0.08581098169088364,
+      "step": 11700
+    },
+    {
+      "epoch": 0.77,
+      "eval_logits/chosen": -2.31754469871521,
+      "eval_logits/rejected": -2.1296095848083496,
+      "eval_logps/chosen": -231.76707458496094,
+      "eval_logps/rejected": -219.65855407714844,
+      "eval_loss": 0.6898021697998047,
+      "eval_rewards/accuracies": 0.6355000138282776,
+      "eval_rewards/chosen": 0.002378788311034441,
+      "eval_rewards/margins": 0.08284525573253632,
+      "eval_rewards/rejected": -0.08046647161245346,
+      "eval_runtime": 714.8864,
+      "eval_samples_per_second": 2.798,
+      "eval_steps_per_second": 1.399,
+      "step": 11700
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.876751318893217e-07,
+      "logits/chosen": -2.2193684577941895,
+      "logits/rejected": -1.9712486267089844,
+      "logps/chosen": -236.8195343017578,
+      "logps/rejected": -224.7071533203125,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.010461434721946716,
+      "rewards/margins": 0.09475782513618469,
+      "rewards/rejected": -0.08429639786481857,
+      "step": 11710
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.8351931004785e-07,
+      "logits/chosen": -2.2037353515625,
+      "logits/rejected": -1.8471559286117554,
+      "logps/chosen": -207.496337890625,
+      "logps/rejected": -191.63697814941406,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.011319467797875404,
+      "rewards/margins": 0.09023983776569366,
+      "rewards/rejected": -0.07892037183046341,
+      "step": 11720
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.793724422104834e-07,
+      "logits/chosen": -2.0594654083251953,
+      "logits/rejected": -2.2132391929626465,
+      "logps/chosen": -208.775146484375,
+      "logps/rejected": -288.7984924316406,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0004916332545690238,
+      "rewards/margins": 0.10546108335256577,
+      "rewards/rejected": -0.10496945679187775,
+      "step": 11730
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.752345500093184e-07,
+      "logits/chosen": -2.360964059829712,
+      "logits/rejected": -2.3283915519714355,
+      "logps/chosen": -213.69003295898438,
+      "logps/rejected": -188.79702758789062,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.03534995764493942,
+      "rewards/margins": 0.048714593052864075,
+      "rewards/rejected": -0.0840645581483841,
+      "step": 11740
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.711056550296253e-07,
+      "logits/chosen": -2.4189441204071045,
+      "logits/rejected": -2.2487740516662598,
+      "logps/chosen": -245.80990600585938,
+      "logps/rejected": -219.0479736328125,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.009839094243943691,
+      "rewards/margins": 0.10263122618198395,
+      "rewards/rejected": -0.09279213845729828,
+      "step": 11750
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.669857788097445e-07,
+      "logits/chosen": -2.1502368450164795,
+      "logits/rejected": -1.9063570499420166,
+      "logps/chosen": -164.16265869140625,
+      "logps/rejected": -200.6251678466797,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.025966918095946312,
+      "rewards/margins": 0.09837634116411209,
+      "rewards/rejected": -0.12434325367212296,
+      "step": 11760
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.628749428409676e-07,
+      "logits/chosen": -2.410966157913208,
+      "logits/rejected": -1.9934518337249756,
+      "logps/chosen": -239.00985717773438,
+      "logps/rejected": -188.3817138671875,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.02122049406170845,
+      "rewards/margins": 0.08123396337032318,
+      "rewards/rejected": -0.10245446860790253,
+      "step": 11770
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.587731685674288e-07,
+      "logits/chosen": -2.293890953063965,
+      "logits/rejected": -2.332033157348633,
+      "logps/chosen": -272.982666015625,
+      "logps/rejected": -288.7724304199219,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.006056091282516718,
+      "rewards/margins": 0.07952813804149628,
+      "rewards/rejected": -0.0734720379114151,
+      "step": 11780
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.546804773859931e-07,
+      "logits/chosen": -2.392029047012329,
+      "logits/rejected": -2.1629507541656494,
+      "logps/chosen": -228.03903198242188,
+      "logps/rejected": -219.1748809814453,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.009989907965064049,
+      "rewards/margins": 0.11071814596652985,
+      "rewards/rejected": -0.12070806324481964,
+      "step": 11790
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.505968906461409e-07,
+      "logits/chosen": -2.295881509780884,
+      "logits/rejected": -2.148301362991333,
+      "logps/chosen": -243.2596893310547,
+      "logps/rejected": -226.50534057617188,
+      "loss": 0.69,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.023195995017886162,
+      "rewards/margins": 0.07391957193613052,
+      "rewards/rejected": -0.09711556136608124,
+      "step": 11800
+    },
+    {
+      "epoch": 0.77,
+      "eval_logits/chosen": -2.3191685676574707,
+      "eval_logits/rejected": -2.1311001777648926,
+      "eval_logps/chosen": -232.53021240234375,
+      "eval_logps/rejected": -220.31980895996094,
+      "eval_loss": 0.6897976398468018,
+      "eval_rewards/accuracies": 0.640999972820282,
+      "eval_rewards/chosen": -0.005252572242170572,
+      "eval_rewards/margins": 0.08182655274868011,
+      "eval_rewards/rejected": -0.08707911521196365,
+      "eval_runtime": 712.4035,
+      "eval_samples_per_second": 2.807,
+      "eval_steps_per_second": 1.404,
+      "step": 11800
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.465224296498627e-07,
+      "logits/chosen": -2.3691840171813965,
+      "logits/rejected": -1.9838594198226929,
+      "logps/chosen": -233.9862823486328,
+      "logps/rejected": -198.72694396972656,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.010271742939949036,
+      "rewards/margins": 0.058696817606687546,
+      "rewards/rejected": -0.06896857172250748,
+      "step": 11810
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.424571156515412e-07,
+      "logits/chosen": -2.234841823577881,
+      "logits/rejected": -2.2243666648864746,
+      "logps/chosen": -177.17324829101562,
+      "logps/rejected": -210.3824005126953,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.005319344811141491,
+      "rewards/margins": 0.10532574355602264,
+      "rewards/rejected": -0.10000641644001007,
+      "step": 11820
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.38400969857847e-07,
+      "logits/chosen": -2.183497905731201,
+      "logits/rejected": -1.9732284545898438,
+      "logps/chosen": -189.73611450195312,
+      "logps/rejected": -206.5034637451172,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.06593702733516693,
+      "rewards/margins": 0.13615167140960693,
+      "rewards/rejected": -0.20208871364593506,
+      "step": 11830
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.343540134276225e-07,
+      "logits/chosen": -2.3149163722991943,
+      "logits/rejected": -2.23742938041687,
+      "logps/chosen": -169.7374267578125,
+      "logps/rejected": -179.54759216308594,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.009741841815412045,
+      "rewards/margins": 0.0791388601064682,
+      "rewards/rejected": -0.06939703226089478,
+      "step": 11840
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.303162674717762e-07,
+      "logits/chosen": -2.2864696979522705,
+      "logits/rejected": -1.8954169750213623,
+      "logps/chosen": -213.3994140625,
+      "logps/rejected": -165.81307983398438,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.03171641379594803,
+      "rewards/margins": 0.08249086141586304,
+      "rewards/rejected": -0.11420726776123047,
+      "step": 11850
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.26287753053167e-07,
+      "logits/chosen": -2.250080108642578,
+      "logits/rejected": -2.17789888381958,
+      "logps/chosen": -267.2779846191406,
+      "logps/rejected": -273.6764831542969,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.023018458858132362,
+      "rewards/margins": 0.0766671746969223,
+      "rewards/rejected": -0.09968564659357071,
+      "step": 11860
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.222684911865013e-07,
+      "logits/chosen": -2.3542191982269287,
+      "logits/rejected": -2.364485263824463,
+      "logps/chosen": -207.5823516845703,
+      "logps/rejected": -240.3794708251953,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.0041365777142345905,
+      "rewards/margins": 0.11111694574356079,
+      "rewards/rejected": -0.1152535229921341,
+      "step": 11870
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.182585028382166e-07,
+      "logits/chosen": -2.407355785369873,
+      "logits/rejected": -2.1081418991088867,
+      "logps/chosen": -276.42279052734375,
+      "logps/rejected": -252.2656707763672,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0027120530139654875,
+      "rewards/margins": 0.08292824774980545,
+      "rewards/rejected": -0.08564029633998871,
+      "step": 11880
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.142578089263769e-07,
+      "logits/chosen": -2.438586473464966,
+      "logits/rejected": -2.1107470989227295,
+      "logps/chosen": -319.8658447265625,
+      "logps/rejected": -268.9220886230469,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.014372703619301319,
+      "rewards/margins": 0.07856186479330063,
+      "rewards/rejected": -0.09293456375598907,
+      "step": 11890
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.102664303205611e-07,
+      "logits/chosen": -2.3172056674957275,
+      "logits/rejected": -2.0737314224243164,
+      "logps/chosen": -221.40579223632812,
+      "logps/rejected": -211.95523071289062,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.008432546630501747,
+      "rewards/margins": 0.08426036685705185,
+      "rewards/rejected": -0.09269289672374725,
+      "step": 11900
+    },
+    {
+      "epoch": 0.78,
+      "eval_logits/chosen": -2.3179848194122314,
+      "eval_logits/rejected": -2.130025625228882,
+      "eval_logps/chosen": -232.76321411132812,
+      "eval_logps/rejected": -220.74920654296875,
+      "eval_loss": 0.6897937059402466,
+      "eval_rewards/accuracies": 0.640999972820282,
+      "eval_rewards/chosen": -0.007582689169794321,
+      "eval_rewards/margins": 0.08379034698009491,
+      "eval_rewards/rejected": -0.09137304127216339,
+      "eval_runtime": 712.893,
+      "eval_samples_per_second": 2.805,
+      "eval_steps_per_second": 1.403,
+      "step": 11900
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.062843878417566e-07,
+      "logits/chosen": -2.4349122047424316,
+      "logits/rejected": -2.2990574836730957,
+      "logps/chosen": -222.23208618164062,
+      "logps/rejected": -196.22946166992188,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.012005344964563847,
+      "rewards/margins": 0.07041595876216888,
+      "rewards/rejected": -0.05841060355305672,
+      "step": 11910
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.023117022622458e-07,
+      "logits/chosen": -2.355516195297241,
+      "logits/rejected": -2.0010008811950684,
+      "logps/chosen": -242.761962890625,
+      "logps/rejected": -227.82077026367188,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.04246622696518898,
+      "rewards/margins": 0.0832718163728714,
+      "rewards/rejected": -0.12573805451393127,
+      "step": 11920
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.983483943055042e-07,
+      "logits/chosen": -2.251622200012207,
+      "logits/rejected": -2.0690550804138184,
+      "logps/chosen": -281.15789794921875,
+      "logps/rejected": -231.5454864501953,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.006913202814757824,
+      "rewards/margins": 0.07160593569278717,
+      "rewards/rejected": -0.07851915061473846,
+      "step": 11930
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.943944846460859e-07,
+      "logits/chosen": -2.2984964847564697,
+      "logits/rejected": -2.2344911098480225,
+      "logps/chosen": -218.5690460205078,
+      "logps/rejected": -177.935791015625,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.001852800720371306,
+      "rewards/margins": 0.06108871102333069,
+      "rewards/rejected": -0.059235911816358566,
+      "step": 11940
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.904499939095225e-07,
+      "logits/chosen": -2.264219045639038,
+      "logits/rejected": -2.2117581367492676,
+      "logps/chosen": -222.3902587890625,
+      "logps/rejected": -217.29684448242188,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.00043100136099383235,
+      "rewards/margins": 0.1023484319448471,
+      "rewards/rejected": -0.10277943313121796,
+      "step": 11950
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.865149426722079e-07,
+      "logits/chosen": -2.233142852783203,
+      "logits/rejected": -2.1767783164978027,
+      "logps/chosen": -274.33160400390625,
+      "logps/rejected": -252.09359741210938,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.03146423026919365,
+      "rewards/margins": 0.0845610499382019,
+      "rewards/rejected": -0.11602529138326645,
+      "step": 11960
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.825893514612985e-07,
+      "logits/chosen": -2.071587324142456,
+      "logits/rejected": -2.1735153198242188,
+      "logps/chosen": -236.6058807373047,
+      "logps/rejected": -243.20828247070312,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0003904670593328774,
+      "rewards/margins": 0.08785964548587799,
+      "rewards/rejected": -0.08825010061264038,
+      "step": 11970
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.786732407546001e-07,
+      "logits/chosen": -2.084094285964966,
+      "logits/rejected": -2.0345733165740967,
+      "logps/chosen": -201.45138549804688,
+      "logps/rejected": -169.8759765625,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.005701950751245022,
+      "rewards/margins": 0.08140133321285248,
+      "rewards/rejected": -0.08710329234600067,
+      "step": 11980
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.747666309804654e-07,
+      "logits/chosen": -2.4778218269348145,
+      "logits/rejected": -2.1540145874023438,
+      "logps/chosen": -287.5825500488281,
+      "logps/rejected": -215.3527374267578,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.0009467907366342843,
+      "rewards/margins": 0.07305634021759033,
+      "rewards/rejected": -0.07210955023765564,
+      "step": 11990
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.708695425176831e-07,
+      "logits/chosen": -2.1013684272766113,
+      "logits/rejected": -2.102804660797119,
+      "logps/chosen": -170.0596466064453,
+      "logps/rejected": -200.979248046875,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.010208925232291222,
+      "rewards/margins": 0.1016424298286438,
+      "rewards/rejected": -0.11185135692358017,
+      "step": 12000
+    },
+    {
+      "epoch": 0.79,
+      "eval_logits/chosen": -2.3212053775787354,
+      "eval_logits/rejected": -2.132889986038208,
+      "eval_logps/chosen": -232.20034790039062,
+      "eval_logps/rejected": -220.30197143554688,
+      "eval_loss": 0.6897847652435303,
+      "eval_rewards/accuracies": 0.6420000195503235,
+      "eval_rewards/chosen": -0.001953852828592062,
+      "eval_rewards/margins": 0.08494684100151062,
+      "eval_rewards/rejected": -0.08690068125724792,
+      "eval_runtime": 712.847,
+      "eval_samples_per_second": 2.806,
+      "eval_steps_per_second": 1.403,
+      "step": 12000
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.669819956953768e-07,
+      "logits/chosen": -2.1582138538360596,
+      "logits/rejected": -2.0927162170410156,
+      "logps/chosen": -170.88861083984375,
+      "logps/rejected": -186.5122833251953,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.005573832895606756,
+      "rewards/margins": 0.07083684206008911,
+      "rewards/rejected": -0.07641066610813141,
+      "step": 12010
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.631040107928957e-07,
+      "logits/chosen": -2.4988465309143066,
+      "logits/rejected": -2.1730918884277344,
+      "logps/chosen": -268.5477294921875,
+      "logps/rejected": -180.80491638183594,
+      "loss": 0.691,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.0036344178952276707,
+      "rewards/margins": 0.08555683493614197,
+      "rewards/rejected": -0.0891912430524826,
+      "step": 12020
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.592356080397072e-07,
+      "logits/chosen": -2.393764019012451,
+      "logits/rejected": -1.8318722248077393,
+      "logps/chosen": -227.7593994140625,
+      "logps/rejected": -179.75608825683594,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0052979374304413795,
+      "rewards/margins": 0.08460094779729843,
+      "rewards/rejected": -0.07930301129817963,
+      "step": 12030
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.553768076152963e-07,
+      "logits/chosen": -2.2826318740844727,
+      "logits/rejected": -2.3857228755950928,
+      "logps/chosen": -157.96484375,
+      "logps/rejected": -197.0670928955078,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0013273532968014479,
+      "rewards/margins": 0.11998225748538971,
+      "rewards/rejected": -0.11865489184856415,
+      "step": 12040
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.51527629649055e-07,
+      "logits/chosen": -2.419004201889038,
+      "logits/rejected": -2.2709298133850098,
+      "logps/chosen": -254.87680053710938,
+      "logps/rejected": -231.243408203125,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.012937125749886036,
+      "rewards/margins": 0.05513680726289749,
+      "rewards/rejected": -0.06807393580675125,
+      "step": 12050
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.476880942201824e-07,
+      "logits/chosen": -2.5580201148986816,
+      "logits/rejected": -2.1555469036102295,
+      "logps/chosen": -238.6651611328125,
+      "logps/rejected": -189.90174865722656,
+      "loss": 0.688,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.026725223287940025,
+      "rewards/margins": 0.09860799461603165,
+      "rewards/rejected": -0.07188276946544647,
+      "step": 12060
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.438582213575748e-07,
+      "logits/chosen": -2.245471477508545,
+      "logits/rejected": -2.217454195022583,
+      "logps/chosen": -229.4067840576172,
+      "logps/rejected": -247.80722045898438,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.005097637884318829,
+      "rewards/margins": 0.0725497156381607,
+      "rewards/rejected": -0.0674520879983902,
+      "step": 12070
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.400380310397267e-07,
+      "logits/chosen": -2.1843056678771973,
+      "logits/rejected": -2.1890132427215576,
+      "logps/chosen": -232.0767822265625,
+      "logps/rejected": -268.4687805175781,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.00033568107755854726,
+      "rewards/margins": 0.04980158433318138,
+      "rewards/rejected": -0.04946590214967728,
+      "step": 12080
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.362275431946202e-07,
+      "logits/chosen": -2.168041467666626,
+      "logits/rejected": -2.201719284057617,
+      "logps/chosen": -235.95803833007812,
+      "logps/rejected": -248.0696563720703,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.003955559339374304,
+      "rewards/margins": 0.06063423305749893,
+      "rewards/rejected": -0.05667867138981819,
+      "step": 12090
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.324267776996285e-07,
+      "logits/chosen": -2.389530658721924,
+      "logits/rejected": -2.0050384998321533,
+      "logps/chosen": -375.6055908203125,
+      "logps/rejected": -265.6221618652344,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.0102651696652174,
+      "rewards/margins": 0.11997060477733612,
+      "rewards/rejected": -0.13023580610752106,
+      "step": 12100
+    },
+    {
+      "epoch": 0.79,
+      "eval_logits/chosen": -2.3229787349700928,
+      "eval_logits/rejected": -2.134584426879883,
+      "eval_logps/chosen": -231.93675231933594,
+      "eval_logps/rejected": -219.7613525390625,
+      "eval_loss": 0.6897767782211304,
+      "eval_rewards/accuracies": 0.6384999752044678,
+      "eval_rewards/chosen": 0.0006819861009716988,
+      "eval_rewards/margins": 0.0821765884757042,
+      "eval_rewards/rejected": -0.08149460703134537,
+      "eval_runtime": 714.3235,
+      "eval_samples_per_second": 2.8,
+      "eval_steps_per_second": 1.4,
+      "step": 12100
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.286357543814045e-07,
+      "logits/chosen": -2.2420029640197754,
+      "logits/rejected": -2.154069662094116,
+      "logps/chosen": -193.7761993408203,
+      "logps/rejected": -272.2079772949219,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.002463629934936762,
+      "rewards/margins": 0.10420586168766022,
+      "rewards/rejected": -0.10666950047016144,
+      "step": 12110
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.248544930157838e-07,
+      "logits/chosen": -2.3656675815582275,
+      "logits/rejected": -2.122084617614746,
+      "logps/chosen": -183.5366973876953,
+      "logps/rejected": -178.4772186279297,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.005488743539899588,
+      "rewards/margins": 0.12721005082130432,
+      "rewards/rejected": -0.12172132730484009,
+      "step": 12120
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.21083013327678e-07,
+      "logits/chosen": -2.3051021099090576,
+      "logits/rejected": -2.202974796295166,
+      "logps/chosen": -300.95269775390625,
+      "logps/rejected": -251.5149383544922,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.032880254089832306,
+      "rewards/margins": 0.06488000601530075,
+      "rewards/rejected": -0.03199975565075874,
+      "step": 12130
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.17321334990973e-07,
+      "logits/chosen": -2.2721495628356934,
+      "logits/rejected": -2.188018798828125,
+      "logps/chosen": -200.94345092773438,
+      "logps/rejected": -172.30050659179688,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0013894874136894941,
+      "rewards/margins": 0.06594385951757431,
+      "rewards/rejected": -0.06733334064483643,
+      "step": 12140
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.135694776284243e-07,
+      "logits/chosen": -2.431753635406494,
+      "logits/rejected": -2.160250425338745,
+      "logps/chosen": -269.80584716796875,
+      "logps/rejected": -216.8905487060547,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.009851393289864063,
+      "rewards/margins": 0.11856885999441147,
+      "rewards/rejected": -0.10871747881174088,
+      "step": 12150
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.098274608115595e-07,
+      "logits/chosen": -2.2147669792175293,
+      "logits/rejected": -2.0826354026794434,
+      "logps/chosen": -204.7724151611328,
+      "logps/rejected": -179.89138793945312,
+      "loss": 0.693,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.008782130666077137,
+      "rewards/margins": 0.037513960152864456,
+      "rewards/rejected": -0.028731834143400192,
+      "step": 12160
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.060953040605697e-07,
+      "logits/chosen": -2.4421582221984863,
+      "logits/rejected": -1.913739562034607,
+      "logps/chosen": -334.14593505859375,
+      "logps/rejected": -275.56756591796875,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.04983791708946228,
+      "rewards/margins": 0.11316549777984619,
+      "rewards/rejected": -0.0633275955915451,
+      "step": 12170
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.023730268442144e-07,
+      "logits/chosen": -2.23698091506958,
+      "logits/rejected": -2.057175636291504,
+      "logps/chosen": -202.0666046142578,
+      "logps/rejected": -185.1787109375,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.027542103081941605,
+      "rewards/margins": 0.11556919664144516,
+      "rewards/rejected": -0.08802708983421326,
+      "step": 12180
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.986606485797131e-07,
+      "logits/chosen": -2.234809160232544,
+      "logits/rejected": -2.023869752883911,
+      "logps/chosen": -199.58416748046875,
+      "logps/rejected": -211.5013885498047,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.003064130200073123,
+      "rewards/margins": 0.06851175427436829,
+      "rewards/rejected": -0.0654476210474968,
+      "step": 12190
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.949581886326511e-07,
+      "logits/chosen": -2.352273941040039,
+      "logits/rejected": -2.33347749710083,
+      "logps/chosen": -295.5454406738281,
+      "logps/rejected": -271.19891357421875,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.022466326132416725,
+      "rewards/margins": 0.0456775538623333,
+      "rewards/rejected": -0.023211227729916573,
+      "step": 12200
+    },
+    {
+      "epoch": 0.8,
+      "eval_logits/chosen": -2.3217251300811768,
+      "eval_logits/rejected": -2.1334657669067383,
+      "eval_logps/chosen": -230.84371948242188,
+      "eval_logps/rejected": -218.59002685546875,
+      "eval_loss": 0.6897699236869812,
+      "eval_rewards/accuracies": 0.6340000033378601,
+      "eval_rewards/chosen": 0.011612382717430592,
+      "eval_rewards/margins": 0.08139365911483765,
+      "eval_rewards/rejected": -0.06978128105401993,
+      "eval_runtime": 713.062,
+      "eval_samples_per_second": 2.805,
+      "eval_steps_per_second": 1.402,
+      "step": 12200
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.912656663168717e-07,
+      "logits/chosen": -2.4126431941986084,
+      "logits/rejected": -2.3553099632263184,
+      "logps/chosen": -221.67831420898438,
+      "logps/rejected": -221.48641967773438,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.023309772834181786,
+      "rewards/margins": 0.06180506944656372,
+      "rewards/rejected": -0.03849529102444649,
+      "step": 12210
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.875831008943817e-07,
+      "logits/chosen": -2.158846616744995,
+      "logits/rejected": -2.1420199871063232,
+      "logps/chosen": -177.19949340820312,
+      "logps/rejected": -160.8948516845703,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0044457814656198025,
+      "rewards/margins": 0.06295563280582428,
+      "rewards/rejected": -0.058509863913059235,
+      "step": 12220
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.839105115752442e-07,
+      "logits/chosen": -2.284562110900879,
+      "logits/rejected": -2.0982518196105957,
+      "logps/chosen": -224.8004150390625,
+      "logps/rejected": -193.5330352783203,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.01822907105088234,
+      "rewards/margins": 0.08740357309579849,
+      "rewards/rejected": -0.10563264042139053,
+      "step": 12230
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.802479175174855e-07,
+      "logits/chosen": -2.2706997394561768,
+      "logits/rejected": -2.1211254596710205,
+      "logps/chosen": -163.5981903076172,
+      "logps/rejected": -178.07614135742188,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.023960810154676437,
+      "rewards/margins": 0.08550871908664703,
+      "rewards/rejected": -0.061547912657260895,
+      "step": 12240
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.765953378269901e-07,
+      "logits/chosen": -2.189150333404541,
+      "logits/rejected": -2.127336025238037,
+      "logps/chosen": -206.731689453125,
+      "logps/rejected": -248.9289093017578,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0026171256322413683,
+      "rewards/margins": 0.11153455078601837,
+      "rewards/rejected": -0.10891741514205933,
+      "step": 12250
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.729527915574037e-07,
+      "logits/chosen": -2.343411922454834,
+      "logits/rejected": -2.2044577598571777,
+      "logps/chosen": -220.0668487548828,
+      "logps/rejected": -229.0086669921875,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.0035170826595276594,
+      "rewards/margins": 0.08602721989154816,
+      "rewards/rejected": -0.08251012861728668,
+      "step": 12260
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.693202977100304e-07,
+      "logits/chosen": -2.354865074157715,
+      "logits/rejected": -2.067964553833008,
+      "logps/chosen": -168.99929809570312,
+      "logps/rejected": -172.4113311767578,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.011458302848041058,
+      "rewards/margins": 0.06748346984386444,
+      "rewards/rejected": -0.05602516978979111,
+      "step": 12270
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.656978752337389e-07,
+      "logits/chosen": -2.3720412254333496,
+      "logits/rejected": -2.168274402618408,
+      "logps/chosen": -200.18475341796875,
+      "logps/rejected": -212.4142303466797,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.00529628898948431,
+      "rewards/margins": 0.10829710960388184,
+      "rewards/rejected": -0.11359341442584991,
+      "step": 12280
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.620855430248581e-07,
+      "logits/chosen": -2.269085645675659,
+      "logits/rejected": -2.128979206085205,
+      "logps/chosen": -160.27426147460938,
+      "logps/rejected": -168.23971557617188,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.017207933589816093,
+      "rewards/margins": 0.10143836587667465,
+      "rewards/rejected": -0.0842304453253746,
+      "step": 12290
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.584833199270837e-07,
+      "logits/chosen": -2.33870267868042,
+      "logits/rejected": -2.1915290355682373,
+      "logps/chosen": -228.42819213867188,
+      "logps/rejected": -227.838623046875,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0020407275296747684,
+      "rewards/margins": 0.07103622704744339,
+      "rewards/rejected": -0.06899549812078476,
+      "step": 12300
+    },
+    {
+      "epoch": 0.8,
+      "eval_logits/chosen": -2.3225796222686768,
+      "eval_logits/rejected": -2.1341781616210938,
+      "eval_logps/chosen": -231.32383728027344,
+      "eval_logps/rejected": -219.537353515625,
+      "eval_loss": 0.6897599101066589,
+      "eval_rewards/accuracies": 0.6365000009536743,
+      "eval_rewards/chosen": 0.006811096332967281,
+      "eval_rewards/margins": 0.08606572449207306,
+      "eval_rewards/rejected": -0.0792546421289444,
+      "eval_runtime": 712.8422,
+      "eval_samples_per_second": 2.806,
+      "eval_steps_per_second": 1.403,
+      "step": 12300
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.548912247313742e-07,
+      "logits/chosen": -2.5504541397094727,
+      "logits/rejected": -2.120837450027466,
+      "logps/chosen": -284.99554443359375,
+      "logps/rejected": -243.7223663330078,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.011955673806369305,
+      "rewards/margins": 0.06302481144666672,
+      "rewards/rejected": -0.0749804899096489,
+      "step": 12310
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.513092761758596e-07,
+      "logits/chosen": -2.367363452911377,
+      "logits/rejected": -2.1712703704833984,
+      "logps/chosen": -271.68951416015625,
+      "logps/rejected": -208.8588104248047,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0047961072996258736,
+      "rewards/margins": 0.0415019765496254,
+      "rewards/rejected": -0.0462980791926384,
+      "step": 12320
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.477374929457363e-07,
+      "logits/chosen": -2.2922816276550293,
+      "logits/rejected": -2.291826009750366,
+      "logps/chosen": -201.52459716796875,
+      "logps/rejected": -188.91586303710938,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.005365630611777306,
+      "rewards/margins": 0.06563162058591843,
+      "rewards/rejected": -0.06026599556207657,
+      "step": 12330
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.441758936731772e-07,
+      "logits/chosen": -2.3044772148132324,
+      "logits/rejected": -2.1925055980682373,
+      "logps/chosen": -234.58203125,
+      "logps/rejected": -225.813720703125,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.007632553577423096,
+      "rewards/margins": 0.09021967649459839,
+      "rewards/rejected": -0.08258712291717529,
+      "step": 12340
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.406244969372273e-07,
+      "logits/chosen": -2.225092887878418,
+      "logits/rejected": -2.0609354972839355,
+      "logps/chosen": -199.0269012451172,
+      "logps/rejected": -219.9811248779297,
+      "loss": 0.6867,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.006959347520023584,
+      "rewards/margins": 0.14452257752418518,
+      "rewards/rejected": -0.13756322860717773,
+      "step": 12350
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.370833212637122e-07,
+      "logits/chosen": -2.2868409156799316,
+      "logits/rejected": -1.9941694736480713,
+      "logps/chosen": -218.37545776367188,
+      "logps/rejected": -213.766357421875,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0013551099691540003,
+      "rewards/margins": 0.09176047146320343,
+      "rewards/rejected": -0.09040535986423492,
+      "step": 12360
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.335523851251392e-07,
+      "logits/chosen": -2.239475727081299,
+      "logits/rejected": -2.166835069656372,
+      "logps/chosen": -206.76382446289062,
+      "logps/rejected": -195.5325469970703,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0062995306216180325,
+      "rewards/margins": 0.09566928446292877,
+      "rewards/rejected": -0.10196882486343384,
+      "step": 12370
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.300317069406003e-07,
+      "logits/chosen": -2.219008445739746,
+      "logits/rejected": -2.191943407058716,
+      "logps/chosen": -158.67398071289062,
+      "logps/rejected": -173.11231994628906,
+      "loss": 0.687,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.020719021558761597,
+      "rewards/margins": 0.10671563446521759,
+      "rewards/rejected": -0.085996612906456,
+      "step": 12380
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.265213050756782e-07,
+      "logits/chosen": -2.44820499420166,
+      "logits/rejected": -2.2743122577667236,
+      "logps/chosen": -229.58837890625,
+      "logps/rejected": -236.1202850341797,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.024628793820738792,
+      "rewards/margins": 0.08663028478622437,
+      "rewards/rejected": -0.06200150400400162,
+      "step": 12390
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.230211978423477e-07,
+      "logits/chosen": -2.3545405864715576,
+      "logits/rejected": -2.235017776489258,
+      "logps/chosen": -220.39273071289062,
+      "logps/rejected": -212.700439453125,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.01592477597296238,
+      "rewards/margins": 0.07476507127285004,
+      "rewards/rejected": -0.09068983793258667,
+      "step": 12400
+    },
+    {
+      "epoch": 0.81,
+      "eval_logits/chosen": -2.3246352672576904,
+      "eval_logits/rejected": -2.1361496448516846,
+      "eval_logps/chosen": -230.83546447753906,
+      "eval_logps/rejected": -218.64422607421875,
+      "eval_loss": 0.6897544264793396,
+      "eval_rewards/accuracies": 0.6349999904632568,
+      "eval_rewards/chosen": 0.0116947703063488,
+      "eval_rewards/margins": 0.08201787620782852,
+      "eval_rewards/rejected": -0.07032310217618942,
+      "eval_runtime": 715.5021,
+      "eval_samples_per_second": 2.795,
+      "eval_steps_per_second": 1.398,
+      "step": 12400
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.195314034988835e-07,
+      "logits/chosen": -2.4760658740997314,
+      "logits/rejected": -2.2121901512145996,
+      "logps/chosen": -217.1764678955078,
+      "logps/rejected": -159.36187744140625,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.021106228232383728,
+      "rewards/margins": 0.09224637597799301,
+      "rewards/rejected": -0.07114015519618988,
+      "step": 12410
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.160519402497616e-07,
+      "logits/chosen": -2.3596584796905518,
+      "logits/rejected": -2.209519147872925,
+      "logps/chosen": -230.1166229248047,
+      "logps/rejected": -234.6483612060547,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0006260558729991317,
+      "rewards/margins": 0.0859164297580719,
+      "rewards/rejected": -0.08654247969388962,
+      "step": 12420
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.125828262455679e-07,
+      "logits/chosen": -2.256119728088379,
+      "logits/rejected": -2.056142807006836,
+      "logps/chosen": -254.59011840820312,
+      "logps/rejected": -233.541748046875,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.011229803785681725,
+      "rewards/margins": 0.09291192889213562,
+      "rewards/rejected": -0.08168213069438934,
+      "step": 12430
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.091240795828992e-07,
+      "logits/chosen": -2.019127368927002,
+      "logits/rejected": -2.2054896354675293,
+      "logps/chosen": -200.9918212890625,
+      "logps/rejected": -231.53662109375,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.014986000955104828,
+      "rewards/margins": 0.10714125633239746,
+      "rewards/rejected": -0.09215524047613144,
+      "step": 12440
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.056757183042732e-07,
+      "logits/chosen": -2.24973464012146,
+      "logits/rejected": -2.1625046730041504,
+      "logps/chosen": -232.5579071044922,
+      "logps/rejected": -222.85122680664062,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.009982489980757236,
+      "rewards/margins": 0.10170789062976837,
+      "rewards/rejected": -0.09172537922859192,
+      "step": 12450
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 5.022377603980308e-07,
+      "logits/chosen": -2.4154021739959717,
+      "logits/rejected": -2.081878185272217,
+      "logps/chosen": -249.6466522216797,
+      "logps/rejected": -192.86483764648438,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.0036199470050632954,
+      "rewards/margins": 0.09689504653215408,
+      "rewards/rejected": -0.10051499307155609,
+      "step": 12460
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.988102237982454e-07,
+      "logits/chosen": -2.36234974861145,
+      "logits/rejected": -2.286999225616455,
+      "logps/chosen": -223.90420532226562,
+      "logps/rejected": -188.7877655029297,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.017237504944205284,
+      "rewards/margins": 0.05458803102374077,
+      "rewards/rejected": -0.0718255490064621,
+      "step": 12470
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.953931263846251e-07,
+      "logits/chosen": -2.3473331928253174,
+      "logits/rejected": -2.0705606937408447,
+      "logps/chosen": -261.57763671875,
+      "logps/rejected": -227.3242645263672,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0013126353733241558,
+      "rewards/margins": 0.0970732644200325,
+      "rewards/rejected": -0.09576062858104706,
+      "step": 12480
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.919864859824266e-07,
+      "logits/chosen": -2.291419267654419,
+      "logits/rejected": -2.145946979522705,
+      "logps/chosen": -238.6917724609375,
+      "logps/rejected": -202.22686767578125,
+      "loss": 0.691,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.01257591973990202,
+      "rewards/margins": 0.08570267260074615,
+      "rewards/rejected": -0.09827860444784164,
+      "step": 12490
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.885903203623532e-07,
+      "logits/chosen": -2.481529712677002,
+      "logits/rejected": -2.0849225521087646,
+      "logps/chosen": -283.2362365722656,
+      "logps/rejected": -227.9224090576172,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.017776403576135635,
+      "rewards/margins": 0.0845954492688179,
+      "rewards/rejected": -0.06681904196739197,
+      "step": 12500
+    },
+    {
+      "epoch": 0.82,
+      "eval_logits/chosen": -2.325686454772949,
+      "eval_logits/rejected": -2.137054681777954,
+      "eval_logps/chosen": -231.05908203125,
+      "eval_logps/rejected": -218.7409210205078,
+      "eval_loss": 0.6897637844085693,
+      "eval_rewards/accuracies": 0.6324999928474426,
+      "eval_rewards/chosen": 0.009458942338824272,
+      "eval_rewards/margins": 0.08074919879436493,
+      "eval_rewards/rejected": -0.07129025459289551,
+      "eval_runtime": 712.8253,
+      "eval_samples_per_second": 2.806,
+      "eval_steps_per_second": 1.403,
+      "step": 12500
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.852046472404695e-07,
+      "logits/chosen": -2.4758987426757812,
+      "logits/rejected": -1.738454818725586,
+      "logps/chosen": -294.7528381347656,
+      "logps/rejected": -177.1123504638672,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.031358979642391205,
+      "rewards/margins": 0.08032914996147156,
+      "rewards/rejected": -0.04897017404437065,
+      "step": 12510
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.818294842781035e-07,
+      "logits/chosen": -2.3906936645507812,
+      "logits/rejected": -2.208167552947998,
+      "logps/chosen": -224.6951141357422,
+      "logps/rejected": -184.04940795898438,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0151332076638937,
+      "rewards/margins": 0.11712169647216797,
+      "rewards/rejected": -0.10198847949504852,
+      "step": 12520
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.784648490817601e-07,
+      "logits/chosen": -2.388882875442505,
+      "logits/rejected": -2.094998836517334,
+      "logps/chosen": -218.9868621826172,
+      "logps/rejected": -182.89913940429688,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.011199096217751503,
+      "rewards/margins": 0.07170800119638443,
+      "rewards/rejected": -0.060508906841278076,
+      "step": 12530
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.751107592030235e-07,
+      "logits/chosen": -2.399803400039673,
+      "logits/rejected": -2.1163930892944336,
+      "logps/chosen": -167.94920349121094,
+      "logps/rejected": -162.90350341796875,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.01902575045824051,
+      "rewards/margins": 0.12054232507944107,
+      "rewards/rejected": -0.10151656717061996,
+      "step": 12540
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.717672321384703e-07,
+      "logits/chosen": -2.294178009033203,
+      "logits/rejected": -2.0486741065979004,
+      "logps/chosen": -220.0991973876953,
+      "logps/rejected": -187.33387756347656,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.03146491199731827,
+      "rewards/margins": 0.094304159283638,
+      "rewards/rejected": -0.06283925473690033,
+      "step": 12550
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.684342853295748e-07,
+      "logits/chosen": -2.250814199447632,
+      "logits/rejected": -2.1270248889923096,
+      "logps/chosen": -185.5677032470703,
+      "logps/rejected": -191.38328552246094,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.02129070833325386,
+      "rewards/margins": 0.09739609807729721,
+      "rewards/rejected": -0.07610537856817245,
+      "step": 12560
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.651119361626213e-07,
+      "logits/chosen": -2.5402443408966064,
+      "logits/rejected": -2.2059483528137207,
+      "logps/chosen": -238.58627319335938,
+      "logps/rejected": -199.43728637695312,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.025323236361145973,
+      "rewards/margins": 0.07079877704381943,
+      "rewards/rejected": -0.045475538820028305,
+      "step": 12570
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.618002019686091e-07,
+      "logits/chosen": -2.3017804622650146,
+      "logits/rejected": -2.1221537590026855,
+      "logps/chosen": -278.8421630859375,
+      "logps/rejected": -230.077880859375,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0037017776630818844,
+      "rewards/margins": 0.07871778309345245,
+      "rewards/rejected": -0.07501600682735443,
+      "step": 12580
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.5849910002316757e-07,
+      "logits/chosen": -2.3771467208862305,
+      "logits/rejected": -2.034799575805664,
+      "logps/chosen": -182.04556274414062,
+      "logps/rejected": -158.250732421875,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.015334153547883034,
+      "rewards/margins": 0.09469417482614517,
+      "rewards/rejected": -0.11002832651138306,
+      "step": 12590
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.5520864754645984e-07,
+      "logits/chosen": -2.421297550201416,
+      "logits/rejected": -2.2816054821014404,
+      "logps/chosen": -279.0403747558594,
+      "logps/rejected": -242.0817108154297,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.023093996569514275,
+      "rewards/margins": 0.06866296380758286,
+      "rewards/rejected": -0.04556896537542343,
+      "step": 12600
+    },
+    {
+      "epoch": 0.82,
+      "eval_logits/chosen": -2.3262624740600586,
+      "eval_logits/rejected": -2.1376304626464844,
+      "eval_logps/chosen": -231.39772033691406,
+      "eval_logps/rejected": -219.05184936523438,
+      "eval_loss": 0.6897605061531067,
+      "eval_rewards/accuracies": 0.6365000009536743,
+      "eval_rewards/chosen": 0.006072386633604765,
+      "eval_rewards/margins": 0.08047185838222504,
+      "eval_rewards/rejected": -0.07439946383237839,
+      "eval_runtime": 714.0825,
+      "eval_samples_per_second": 2.801,
+      "eval_steps_per_second": 1.4,
+      "step": 12600
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.5192886170309896e-07,
+      "logits/chosen": -2.250743865966797,
+      "logits/rejected": -2.1508920192718506,
+      "logps/chosen": -190.7171630859375,
+      "logps/rejected": -195.17050170898438,
+      "loss": 0.692,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0005129704950377345,
+      "rewards/margins": 0.042692478746175766,
+      "rewards/rejected": -0.04217951372265816,
+      "step": 12610
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.486597596020548e-07,
+      "logits/chosen": -2.3476834297180176,
+      "logits/rejected": -2.0744900703430176,
+      "logps/chosen": -219.5320281982422,
+      "logps/rejected": -190.0487518310547,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.0004929341375827789,
+      "rewards/margins": 0.08838485181331635,
+      "rewards/rejected": -0.08789192140102386,
+      "step": 12620
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.454013582965644e-07,
+      "logits/chosen": -2.2743661403656006,
+      "logits/rejected": -1.8610731363296509,
+      "logps/chosen": -263.5701904296875,
+      "logps/rejected": -219.12197875976562,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.001134876860305667,
+      "rewards/margins": 0.059951018542051315,
+      "rewards/rejected": -0.06108590215444565,
+      "step": 12630
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.4215367478404605e-07,
+      "logits/chosen": -2.125401020050049,
+      "logits/rejected": -2.1011133193969727,
+      "logps/chosen": -272.4917297363281,
+      "logps/rejected": -303.33233642578125,
+      "loss": 0.691,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.01731831021606922,
+      "rewards/margins": 0.06468813121318817,
+      "rewards/rejected": -0.08200643211603165,
+      "step": 12640
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.389167260060068e-07,
+      "logits/chosen": -2.4071826934814453,
+      "logits/rejected": -2.16564679145813,
+      "logps/chosen": -205.21432495117188,
+      "logps/rejected": -178.94972229003906,
+      "loss": 0.6879,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.03475916385650635,
+      "rewards/margins": 0.11909898370504379,
+      "rewards/rejected": -0.08433983474969864,
+      "step": 12650
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.356905288479579e-07,
+      "logits/chosen": -2.2683780193328857,
+      "logits/rejected": -2.039124011993408,
+      "logps/chosen": -227.39346313476562,
+      "logps/rejected": -212.4646453857422,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.008726147934794426,
+      "rewards/margins": 0.13835129141807556,
+      "rewards/rejected": -0.14707742631435394,
+      "step": 12660
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.3247510013932377e-07,
+      "logits/chosen": -2.217339515686035,
+      "logits/rejected": -2.0521254539489746,
+      "logps/chosen": -252.25942993164062,
+      "logps/rejected": -260.9900817871094,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.0007096766494214535,
+      "rewards/margins": 0.08910763263702393,
+      "rewards/rejected": -0.08981730788946152,
+      "step": 12670
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.2927045665335594e-07,
+      "logits/chosen": -1.9187930822372437,
+      "logits/rejected": -1.8410171270370483,
+      "logps/chosen": -167.76393127441406,
+      "logps/rejected": -172.5765838623047,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.0288618765771389,
+      "rewards/margins": 0.08367923647165298,
+      "rewards/rejected": -0.11254110187292099,
+      "step": 12680
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.260766151070439e-07,
+      "logits/chosen": -2.1377556324005127,
+      "logits/rejected": -2.1839041709899902,
+      "logps/chosen": -224.4342803955078,
+      "logps/rejected": -222.44125366210938,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -7.813423508196138e-06,
+      "rewards/margins": 0.09267593920230865,
+      "rewards/rejected": -0.09268374741077423,
+      "step": 12690
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.228935921610308e-07,
+      "logits/chosen": -2.3698036670684814,
+      "logits/rejected": -2.0076329708099365,
+      "logps/chosen": -262.02777099609375,
+      "logps/rejected": -204.87368774414062,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.016908859834074974,
+      "rewards/margins": 0.06897404789924622,
+      "rewards/rejected": -0.052065182477235794,
+      "step": 12700
+    },
+    {
+      "epoch": 0.83,
+      "eval_logits/chosen": -2.3263168334960938,
+      "eval_logits/rejected": -2.1376240253448486,
+      "eval_logps/chosen": -231.38568115234375,
+      "eval_logps/rejected": -219.1470947265625,
+      "eval_loss": 0.689755916595459,
+      "eval_rewards/accuracies": 0.6334999799728394,
+      "eval_rewards/chosen": 0.0061928038485348225,
+      "eval_rewards/margins": 0.08154484629631042,
+      "eval_rewards/rejected": -0.07535204291343689,
+      "eval_runtime": 712.3303,
+      "eval_samples_per_second": 2.808,
+      "eval_steps_per_second": 1.404,
+      "step": 12700
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.1972140441952246e-07,
+      "logits/chosen": -2.178255081176758,
+      "logits/rejected": -2.146824359893799,
+      "logps/chosen": -236.79483032226562,
+      "logps/rejected": -246.6728973388672,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.027884885668754578,
+      "rewards/margins": 0.07868941873311996,
+      "rewards/rejected": -0.05080454424023628,
+      "step": 12710
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.165600684302046e-07,
+      "logits/chosen": -2.2848753929138184,
+      "logits/rejected": -2.33852481842041,
+      "logps/chosen": -173.56161499023438,
+      "logps/rejected": -192.32496643066406,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.022223882377147675,
+      "rewards/margins": 0.08805432170629501,
+      "rewards/rejected": -0.06583045423030853,
+      "step": 12720
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.13409600684154e-07,
+      "logits/chosen": -2.392894744873047,
+      "logits/rejected": -2.1074211597442627,
+      "logps/chosen": -215.2464599609375,
+      "logps/rejected": -195.3648681640625,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.0022683297283947468,
+      "rewards/margins": 0.09301736205816269,
+      "rewards/rejected": -0.09528569132089615,
+      "step": 12730
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.102700176157548e-07,
+      "logits/chosen": -2.457080364227295,
+      "logits/rejected": -2.087562084197998,
+      "logps/chosen": -324.733642578125,
+      "logps/rejected": -234.39254760742188,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.00021561775065492839,
+      "rewards/margins": 0.07445430010557175,
+      "rewards/rejected": -0.07423867285251617,
+      "step": 12740
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.0714133560260884e-07,
+      "logits/chosen": -2.3434674739837646,
+      "logits/rejected": -2.1800377368927,
+      "logps/chosen": -259.04107666015625,
+      "logps/rejected": -207.67910766601562,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.004920002073049545,
+      "rewards/margins": 0.06868889182806015,
+      "rewards/rejected": -0.0637688934803009,
+      "step": 12750
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.0402357096545527e-07,
+      "logits/chosen": -2.2215254306793213,
+      "logits/rejected": -2.1798095703125,
+      "logps/chosen": -250.39193725585938,
+      "logps/rejected": -248.3852081298828,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.013696703128516674,
+      "rewards/margins": 0.0816536545753479,
+      "rewards/rejected": -0.06795695424079895,
+      "step": 12760
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 4.0091673996808025e-07,
+      "logits/chosen": -2.4354679584503174,
+      "logits/rejected": -2.2336437702178955,
+      "logps/chosen": -198.1351776123047,
+      "logps/rejected": -181.49725341796875,
+      "loss": 0.689,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.010751022025942802,
+      "rewards/margins": 0.07989239692687988,
+      "rewards/rejected": -0.09064342081546783,
+      "step": 12770
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.9782085881723776e-07,
+      "logits/chosen": -2.2519314289093018,
+      "logits/rejected": -2.1054606437683105,
+      "logps/chosen": -164.67416381835938,
+      "logps/rejected": -188.26077270507812,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.009234304539859295,
+      "rewards/margins": 0.11619944870471954,
+      "rewards/rejected": -0.10696514695882797,
+      "step": 12780
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.947359436625592e-07,
+      "logits/chosen": -2.2633697986602783,
+      "logits/rejected": -2.157527446746826,
+      "logps/chosen": -232.0320281982422,
+      "logps/rejected": -205.00436401367188,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.02630910649895668,
+      "rewards/margins": 0.1053650975227356,
+      "rewards/rejected": -0.07905599474906921,
+      "step": 12790
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.9166201059647386e-07,
+      "logits/chosen": -2.399799346923828,
+      "logits/rejected": -2.2643110752105713,
+      "logps/chosen": -259.34234619140625,
+      "logps/rejected": -227.6080780029297,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.02507871761918068,
+      "rewards/margins": 0.051967114210128784,
+      "rewards/rejected": -0.026888396590948105,
+      "step": 12800
+    },
+    {
+      "epoch": 0.84,
+      "eval_logits/chosen": -2.3278772830963135,
+      "eval_logits/rejected": -2.138990640640259,
+      "eval_logps/chosen": -230.71701049804688,
+      "eval_logps/rejected": -218.49427795410156,
+      "eval_loss": 0.6897500157356262,
+      "eval_rewards/accuracies": 0.6359999775886536,
+      "eval_rewards/chosen": 0.012879305519163609,
+      "eval_rewards/margins": 0.08170315623283386,
+      "eval_rewards/rejected": -0.06882384419441223,
+      "eval_runtime": 711.356,
+      "eval_samples_per_second": 2.812,
+      "eval_steps_per_second": 1.406,
+      "step": 12800
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.8859907565412194e-07,
+      "logits/chosen": -2.208301544189453,
+      "logits/rejected": -2.289896249771118,
+      "logps/chosen": -179.2988739013672,
+      "logps/rejected": -186.98802185058594,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0035035633482038975,
+      "rewards/margins": 0.08267536014318466,
+      "rewards/rejected": -0.0791717916727066,
+      "step": 12810
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.8554715481327303e-07,
+      "logits/chosen": -2.365440845489502,
+      "logits/rejected": -1.9772167205810547,
+      "logps/chosen": -233.1007843017578,
+      "logps/rejected": -207.98318481445312,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0015824921429157257,
+      "rewards/margins": 0.10280168056488037,
+      "rewards/rejected": -0.10121919214725494,
+      "step": 12820
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.8250626399424007e-07,
+      "logits/chosen": -2.3790652751922607,
+      "logits/rejected": -2.1290512084960938,
+      "logps/chosen": -249.7417449951172,
+      "logps/rejected": -239.1602020263672,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.017364097759127617,
+      "rewards/margins": 0.08268658816814423,
+      "rewards/rejected": -0.06532249599695206,
+      "step": 12830
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.7947641905980104e-07,
+      "logits/chosen": -2.203456163406372,
+      "logits/rejected": -2.2128920555114746,
+      "logps/chosen": -205.8812713623047,
+      "logps/rejected": -179.02224731445312,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.014371681027114391,
+      "rewards/margins": 0.07821665704250336,
+      "rewards/rejected": -0.0638449639081955,
+      "step": 12840
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.764576358151098e-07,
+      "logits/chosen": -2.238459825515747,
+      "logits/rejected": -2.233910322189331,
+      "logps/chosen": -173.391845703125,
+      "logps/rejected": -167.51803588867188,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.01425178349018097,
+      "rewards/margins": 0.059704847633838654,
+      "rewards/rejected": -0.045453060418367386,
+      "step": 12850
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.7344993000761944e-07,
+      "logits/chosen": -2.349444627761841,
+      "logits/rejected": -2.242591381072998,
+      "logps/chosen": -178.75843811035156,
+      "logps/rejected": -230.89614868164062,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.014432880096137524,
+      "rewards/margins": 0.0916595309972763,
+      "rewards/rejected": -0.10609239339828491,
+      "step": 12860
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.7045331732699585e-07,
+      "logits/chosen": -2.3606972694396973,
+      "logits/rejected": -2.171160936355591,
+      "logps/chosen": -203.33251953125,
+      "logps/rejected": -180.00003051757812,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.01865537092089653,
+      "rewards/margins": 0.12366944551467896,
+      "rewards/rejected": -0.10501406341791153,
+      "step": 12870
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.6746781340503993e-07,
+      "logits/chosen": -2.2035956382751465,
+      "logits/rejected": -2.104219436645508,
+      "logps/chosen": -231.5043487548828,
+      "logps/rejected": -232.2509307861328,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.029114311560988426,
+      "rewards/margins": 0.0877356305718422,
+      "rewards/rejected": -0.058621324598789215,
+      "step": 12880
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.6449343381560116e-07,
+      "logits/chosen": -2.2976372241973877,
+      "logits/rejected": -2.071730613708496,
+      "logps/chosen": -232.3043212890625,
+      "logps/rejected": -229.9260711669922,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.008786660619080067,
+      "rewards/margins": 0.11011286079883575,
+      "rewards/rejected": -0.11889950931072235,
+      "step": 12890
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.615301940745017e-07,
+      "logits/chosen": -2.5623362064361572,
+      "logits/rejected": -1.9647992849349976,
+      "logps/chosen": -314.986083984375,
+      "logps/rejected": -217.2484130859375,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.022090371698141098,
+      "rewards/margins": 0.07439263164997101,
+      "rewards/rejected": -0.05230225995182991,
+      "step": 12900
+    },
+    {
+      "epoch": 0.84,
+      "eval_logits/chosen": -2.3258814811706543,
+      "eval_logits/rejected": -2.1371657848358154,
+      "eval_logps/chosen": -230.18865966796875,
+      "eval_logps/rejected": -218.1457061767578,
+      "eval_loss": 0.6897482872009277,
+      "eval_rewards/accuracies": 0.6334999799728394,
+      "eval_rewards/chosen": 0.0181629229336977,
+      "eval_rewards/margins": 0.0835009291768074,
+      "eval_rewards/rejected": -0.06533800810575485,
+      "eval_runtime": 710.8607,
+      "eval_samples_per_second": 2.813,
+      "eval_steps_per_second": 1.407,
+      "step": 12900
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.5857810963945084e-07,
+      "logits/chosen": -2.1857872009277344,
+      "logits/rejected": -1.9668527841567993,
+      "logps/chosen": -214.0235595703125,
+      "logps/rejected": -207.2059783935547,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.004855555482208729,
+      "rewards/margins": 0.08005331456661224,
+      "rewards/rejected": -0.07519775629043579,
+      "step": 12910
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.556371959099678e-07,
+      "logits/chosen": -2.3944733142852783,
+      "logits/rejected": -2.1525301933288574,
+      "logps/chosen": -294.125732421875,
+      "logps/rejected": -261.60772705078125,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.030485982075333595,
+      "rewards/margins": 0.07452499121427536,
+      "rewards/rejected": -0.04403900355100632,
+      "step": 12920
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.5270746822729797e-07,
+      "logits/chosen": -2.280972957611084,
+      "logits/rejected": -2.2035224437713623,
+      "logps/chosen": -246.06930541992188,
+      "logps/rejected": -267.5752258300781,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.014127634465694427,
+      "rewards/margins": 0.08930746465921402,
+      "rewards/rejected": -0.07517983019351959,
+      "step": 12930
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.4978894187433746e-07,
+      "logits/chosen": -2.3760852813720703,
+      "logits/rejected": -2.23614501953125,
+      "logps/chosen": -152.7420196533203,
+      "logps/rejected": -146.85598754882812,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.016667893156409264,
+      "rewards/margins": 0.05022420361638069,
+      "rewards/rejected": -0.0668920949101448,
+      "step": 12940
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.468816320755486e-07,
+      "logits/chosen": -2.163353443145752,
+      "logits/rejected": -1.9819806814193726,
+      "logps/chosen": -220.0610809326172,
+      "logps/rejected": -186.81573486328125,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.03866659849882126,
+      "rewards/margins": 0.06364385038614273,
+      "rewards/rejected": -0.02497725561261177,
+      "step": 12950
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.4398555399688336e-07,
+      "logits/chosen": -2.425262212753296,
+      "logits/rejected": -2.0546271800994873,
+      "logps/chosen": -211.6660919189453,
+      "logps/rejected": -196.01766967773438,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0031560491770505905,
+      "rewards/margins": 0.05015747994184494,
+      "rewards/rejected": -0.05331353470683098,
+      "step": 12960
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.411007227457047e-07,
+      "logits/chosen": -2.327322006225586,
+      "logits/rejected": -2.270181179046631,
+      "logps/chosen": -244.98068237304688,
+      "logps/rejected": -223.50830078125,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.02780618704855442,
+      "rewards/margins": 0.10802390426397324,
+      "rewards/rejected": -0.08021771907806396,
+      "step": 12970
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.382271533707043e-07,
+      "logits/chosen": -2.24385404586792,
+      "logits/rejected": -2.227750301361084,
+      "logps/chosen": -189.8992919921875,
+      "logps/rejected": -173.1783447265625,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.01705920323729515,
+      "rewards/margins": 0.0616273507475853,
+      "rewards/rejected": -0.044568147510290146,
+      "step": 12980
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.353648608618287e-07,
+      "logits/chosen": -2.3153414726257324,
+      "logits/rejected": -2.06382417678833,
+      "logps/chosen": -167.10702514648438,
+      "logps/rejected": -164.1302490234375,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.00764242559671402,
+      "rewards/margins": 0.0699392706155777,
+      "rewards/rejected": -0.06229684501886368,
+      "step": 12990
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.3251386015019676e-07,
+      "logits/chosen": -2.3504998683929443,
+      "logits/rejected": -2.136061191558838,
+      "logps/chosen": -201.58786010742188,
+      "logps/rejected": -176.71900939941406,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.011715460568666458,
+      "rewards/margins": 0.0887567475438118,
+      "rewards/rejected": -0.07704129070043564,
+      "step": 13000
+    },
+    {
+      "epoch": 0.85,
+      "eval_logits/chosen": -2.3278446197509766,
+      "eval_logits/rejected": -2.138990640640259,
+      "eval_logps/chosen": -230.5150146484375,
+      "eval_logps/rejected": -218.6830596923828,
+      "eval_loss": 0.6897379159927368,
+      "eval_rewards/accuracies": 0.6365000009536743,
+      "eval_rewards/chosen": 0.014899209141731262,
+      "eval_rewards/margins": 0.0856110006570816,
+      "eval_rewards/rejected": -0.07071178406476974,
+      "eval_runtime": 711.2988,
+      "eval_samples_per_second": 2.812,
+      "eval_steps_per_second": 1.406,
+      "step": 13000
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.296741661080255e-07,
+      "logits/chosen": -2.2785840034484863,
+      "logits/rejected": -2.186216115951538,
+      "logps/chosen": -232.2931671142578,
+      "logps/rejected": -235.34976196289062,
+      "loss": 0.688,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.008452139794826508,
+      "rewards/margins": 0.09597768634557724,
+      "rewards/rejected": -0.08752553910017014,
+      "step": 13010
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.2684579354854974e-07,
+      "logits/chosen": -2.3801522254943848,
+      "logits/rejected": -2.2735419273376465,
+      "logps/chosen": -294.13189697265625,
+      "logps/rejected": -312.67303466796875,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.016732942312955856,
+      "rewards/margins": 0.08282653987407684,
+      "rewards/rejected": -0.099559485912323,
+      "step": 13020
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.2402875722594653e-07,
+      "logits/chosen": -2.408092737197876,
+      "logits/rejected": -2.130885362625122,
+      "logps/chosen": -165.7288360595703,
+      "logps/rejected": -181.1701202392578,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.03316589817404747,
+      "rewards/margins": 0.09289722144603729,
+      "rewards/rejected": -0.05973132699728012,
+      "step": 13030
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.212230718352566e-07,
+      "logits/chosen": -2.2618842124938965,
+      "logits/rejected": -2.2778449058532715,
+      "logps/chosen": -224.6171875,
+      "logps/rejected": -162.24813842773438,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.004326606169342995,
+      "rewards/margins": 0.013279316015541553,
+      "rewards/rejected": -0.008952709846198559,
+      "step": 13040
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.1842875201231025e-07,
+      "logits/chosen": -2.3244717121124268,
+      "logits/rejected": -2.0360231399536133,
+      "logps/chosen": -220.89639282226562,
+      "logps/rejected": -202.62872314453125,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.008436797186732292,
+      "rewards/margins": 0.07507555186748505,
+      "rewards/rejected": -0.0666387528181076,
+      "step": 13050
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.156458123336478e-07,
+      "logits/chosen": -2.1647417545318604,
+      "logits/rejected": -1.9881470203399658,
+      "logps/chosen": -156.07313537597656,
+      "logps/rejected": -158.6283721923828,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.024387424811720848,
+      "rewards/margins": 0.12704019248485565,
+      "rewards/rejected": -0.10265277326107025,
+      "step": 13060
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.128742673164459e-07,
+      "logits/chosen": -2.402404308319092,
+      "logits/rejected": -2.034379720687866,
+      "logps/chosen": -280.46630859375,
+      "logps/rejected": -245.130615234375,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.024609588086605072,
+      "rewards/margins": 0.09180278331041336,
+      "rewards/rejected": -0.06719318777322769,
+      "step": 13070
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.101141314184414e-07,
+      "logits/chosen": -2.511915683746338,
+      "logits/rejected": -2.2664966583251953,
+      "logps/chosen": -203.702880859375,
+      "logps/rejected": -200.3907928466797,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.016816768795251846,
+      "rewards/margins": 0.058991938829422,
+      "rewards/rejected": -0.04217516630887985,
+      "step": 13080
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.0736541903785526e-07,
+      "logits/chosen": -2.164177417755127,
+      "logits/rejected": -2.129770517349243,
+      "logps/chosen": -207.080322265625,
+      "logps/rejected": -264.8565368652344,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.019345903769135475,
+      "rewards/margins": 0.08316637575626373,
+      "rewards/rejected": -0.06382046639919281,
+      "step": 13090
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.0462814451331704e-07,
+      "logits/chosen": -2.218174934387207,
+      "logits/rejected": -2.0847418308258057,
+      "logps/chosen": -229.975341796875,
+      "logps/rejected": -235.6430206298828,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.007010665722191334,
+      "rewards/margins": 0.05762631446123123,
+      "rewards/rejected": -0.05061563849449158,
+      "step": 13100
+    },
+    {
+      "epoch": 0.86,
+      "eval_logits/chosen": -2.326002836227417,
+      "eval_logits/rejected": -2.1373050212860107,
+      "eval_logps/chosen": -230.6532745361328,
+      "eval_logps/rejected": -218.62350463867188,
+      "eval_loss": 0.6897422075271606,
+      "eval_rewards/accuracies": 0.6355000138282776,
+      "eval_rewards/chosen": 0.013516743667423725,
+      "eval_rewards/margins": 0.0836327001452446,
+      "eval_rewards/rejected": -0.07011596858501434,
+      "eval_runtime": 712.647,
+      "eval_samples_per_second": 2.806,
+      "eval_steps_per_second": 1.403,
+      "step": 13100
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.019023221237927e-07,
+      "logits/chosen": -2.2801291942596436,
+      "logits/rejected": -2.1055219173431396,
+      "logps/chosen": -236.72677612304688,
+      "logps/rejected": -187.6962890625,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 5.53287572984118e-05,
+      "rewards/margins": 0.08536889404058456,
+      "rewards/rejected": -0.08531356602907181,
+      "step": 13110
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.991879660885058e-07,
+      "logits/chosen": -2.4416868686676025,
+      "logits/rejected": -2.1827034950256348,
+      "logps/chosen": -261.5027770996094,
+      "logps/rejected": -253.5607147216797,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.020441105589270592,
+      "rewards/margins": 0.08418162912130356,
+      "rewards/rejected": -0.06374052911996841,
+      "step": 13120
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.9648509056686786e-07,
+      "logits/chosen": -2.3484883308410645,
+      "logits/rejected": -2.1797919273376465,
+      "logps/chosen": -175.03335571289062,
+      "logps/rejected": -158.2578887939453,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.01908428594470024,
+      "rewards/margins": 0.08800263702869415,
+      "rewards/rejected": -0.06891834735870361,
+      "step": 13130
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.937937096584012e-07,
+      "logits/chosen": -2.3359837532043457,
+      "logits/rejected": -2.114428758621216,
+      "logps/chosen": -287.9132385253906,
+      "logps/rejected": -234.6566619873047,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.024876803159713745,
+      "rewards/margins": 0.07227373123168945,
+      "rewards/rejected": -0.04739692062139511,
+      "step": 13140
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.9111383740266756e-07,
+      "logits/chosen": -2.134230136871338,
+      "logits/rejected": -2.000624179840088,
+      "logps/chosen": -234.1415557861328,
+      "logps/rejected": -235.3389434814453,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.016426388174295425,
+      "rewards/margins": 0.06985460221767426,
+      "rewards/rejected": -0.05342821404337883,
+      "step": 13150
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.8844548777919255e-07,
+      "logits/chosen": -2.374894618988037,
+      "logits/rejected": -2.0936150550842285,
+      "logps/chosen": -201.53781127929688,
+      "logps/rejected": -187.21641540527344,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.02346022054553032,
+      "rewards/margins": 0.0795883983373642,
+      "rewards/rejected": -0.056128181517124176,
+      "step": 13160
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.8578867470739594e-07,
+      "logits/chosen": -2.2102534770965576,
+      "logits/rejected": -2.069348096847534,
+      "logps/chosen": -184.6317138671875,
+      "logps/rejected": -164.10693359375,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.001433709287084639,
+      "rewards/margins": 0.1089792400598526,
+      "rewards/rejected": -0.11041294038295746,
+      "step": 13170
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.8314341204651484e-07,
+      "logits/chosen": -2.4396963119506836,
+      "logits/rejected": -2.1671500205993652,
+      "logps/chosen": -273.39495849609375,
+      "logps/rejected": -211.6111602783203,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.026441499590873718,
+      "rewards/margins": 0.1099875345826149,
+      "rewards/rejected": -0.08354604244232178,
+      "step": 13180
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.805097135955362e-07,
+      "logits/chosen": -2.3553264141082764,
+      "logits/rejected": -2.145498037338257,
+      "logps/chosen": -210.2996368408203,
+      "logps/rejected": -186.16250610351562,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.022243741899728775,
+      "rewards/margins": 0.10788760334253311,
+      "rewards/rejected": -0.08564385771751404,
+      "step": 13190
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.778875930931213e-07,
+      "logits/chosen": -2.3537840843200684,
+      "logits/rejected": -2.0290145874023438,
+      "logps/chosen": -233.2071075439453,
+      "logps/rejected": -225.6391143798828,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.019117634743452072,
+      "rewards/margins": 0.09794165939092636,
+      "rewards/rejected": -0.07882402837276459,
+      "step": 13200
+    },
+    {
+      "epoch": 0.86,
+      "eval_logits/chosen": -2.3253469467163086,
+      "eval_logits/rejected": -2.136681318283081,
+      "eval_logps/chosen": -230.8813018798828,
+      "eval_logps/rejected": -218.95068359375,
+      "eval_loss": 0.6897428035736084,
+      "eval_rewards/accuracies": 0.6370000243186951,
+      "eval_rewards/chosen": 0.011236421763896942,
+      "eval_rewards/margins": 0.08462419360876083,
+      "eval_rewards/rejected": -0.0733877643942833,
+      "eval_runtime": 711.1811,
+      "eval_samples_per_second": 2.812,
+      "eval_steps_per_second": 1.406,
+      "step": 13200
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.7527706421753426e-07,
+      "logits/chosen": -2.320481777191162,
+      "logits/rejected": -2.259533643722534,
+      "logps/chosen": -198.22946166992188,
+      "logps/rejected": -209.1739044189453,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0017170917708426714,
+      "rewards/margins": 0.06455695629119873,
+      "rewards/rejected": -0.06283987313508987,
+      "step": 13210
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.726781405865736e-07,
+      "logits/chosen": -2.4160947799682617,
+      "logits/rejected": -1.8638538122177124,
+      "logps/chosen": -302.37939453125,
+      "logps/rejected": -195.12802124023438,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0028636218048632145,
+      "rewards/margins": 0.08517131209373474,
+      "rewards/rejected": -0.08230768889188766,
+      "step": 13220
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.7009083575749687e-07,
+      "logits/chosen": -2.3083367347717285,
+      "logits/rejected": -2.20975661277771,
+      "logps/chosen": -243.7759246826172,
+      "logps/rejected": -248.42660522460938,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0008438148652203381,
+      "rewards/margins": 0.06286215782165527,
+      "rewards/rejected": -0.062018342316150665,
+      "step": 13230
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.6751516322695457e-07,
+      "logits/chosen": -2.380704402923584,
+      "logits/rejected": -2.3123157024383545,
+      "logps/chosen": -188.8930206298828,
+      "logps/rejected": -188.26637268066406,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.013505371287465096,
+      "rewards/margins": 0.054132528603076935,
+      "rewards/rejected": -0.04062715917825699,
+      "step": 13240
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.649511364309154e-07,
+      "logits/chosen": -2.315520763397217,
+      "logits/rejected": -2.2880232334136963,
+      "logps/chosen": -201.6102294921875,
+      "logps/rejected": -189.54742431640625,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.006794482469558716,
+      "rewards/margins": 0.08818355947732925,
+      "rewards/rejected": -0.08138908445835114,
+      "step": 13250
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.6239876874460003e-07,
+      "logits/chosen": -2.4259033203125,
+      "logits/rejected": -2.2980501651763916,
+      "logps/chosen": -282.17254638671875,
+      "logps/rejected": -264.3243713378906,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.022426238283514977,
+      "rewards/margins": 0.12707999348640442,
+      "rewards/rejected": -0.10465376079082489,
+      "step": 13260
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.5985807348240744e-07,
+      "logits/chosen": -2.4714465141296387,
+      "logits/rejected": -2.0047621726989746,
+      "logps/chosen": -228.8441925048828,
+      "logps/rejected": -192.6537322998047,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.034993596374988556,
+      "rewards/margins": 0.11756626516580582,
+      "rewards/rejected": -0.08257267624139786,
+      "step": 13270
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.5732906389785014e-07,
+      "logits/chosen": -2.3537182807922363,
+      "logits/rejected": -2.1851718425750732,
+      "logps/chosen": -274.45111083984375,
+      "logps/rejected": -248.9810333251953,
+      "loss": 0.6862,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0325496569275856,
+      "rewards/margins": 0.12675470113754272,
+      "rewards/rejected": -0.09420505911111832,
+      "step": 13280
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.5481175318347956e-07,
+      "logits/chosen": -2.2177436351776123,
+      "logits/rejected": -2.26359224319458,
+      "logps/chosen": -225.0997314453125,
+      "logps/rejected": -252.0758819580078,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.023978976532816887,
+      "rewards/margins": 0.09080135077238083,
+      "rewards/rejected": -0.06682237982749939,
+      "step": 13290
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.5230615447082246e-07,
+      "logits/chosen": -2.3211140632629395,
+      "logits/rejected": -1.975968360900879,
+      "logps/chosen": -250.9895477294922,
+      "logps/rejected": -244.07345581054688,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.003271388355642557,
+      "rewards/margins": 0.07869114726781845,
+      "rewards/rejected": -0.07541977614164352,
+      "step": 13300
+    },
+    {
+      "epoch": 0.87,
+      "eval_logits/chosen": -2.324570894241333,
+      "eval_logits/rejected": -2.1359615325927734,
+      "eval_logps/chosen": -230.75726318359375,
+      "eval_logps/rejected": -218.9420623779297,
+      "eval_loss": 0.6897307634353638,
+      "eval_rewards/accuracies": 0.640500009059906,
+      "eval_rewards/chosen": 0.012476898729801178,
+      "eval_rewards/margins": 0.08577845990657806,
+      "eval_rewards/rejected": -0.07330156117677689,
+      "eval_runtime": 715.3491,
+      "eval_samples_per_second": 2.796,
+      "eval_steps_per_second": 1.398,
+      "step": 13300
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.49812280830308e-07,
+      "logits/chosen": -2.3523142337799072,
+      "logits/rejected": -1.8849273920059204,
+      "logps/chosen": -222.9706268310547,
+      "logps/rejected": -205.4813995361328,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.02066592127084732,
+      "rewards/margins": 0.17366810142993927,
+      "rewards/rejected": -0.15300217270851135,
+      "step": 13310
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.4733014527120457e-07,
+      "logits/chosen": -2.2234625816345215,
+      "logits/rejected": -2.0490236282348633,
+      "logps/chosen": -206.6344451904297,
+      "logps/rejected": -194.33306884765625,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.04219576343894005,
+      "rewards/margins": 0.10591878741979599,
+      "rewards/rejected": -0.14811456203460693,
+      "step": 13320
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.4485976074154565e-07,
+      "logits/chosen": -2.285674571990967,
+      "logits/rejected": -2.3419785499572754,
+      "logps/chosen": -208.6743927001953,
+      "logps/rejected": -240.1510009765625,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": 0.003236269112676382,
+      "rewards/margins": 0.01433448027819395,
+      "rewards/rejected": -0.011098211631178856,
+      "step": 13330
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.4240114012806763e-07,
+      "logits/chosen": -2.310711622238159,
+      "logits/rejected": -2.287083148956299,
+      "logps/chosen": -205.61257934570312,
+      "logps/rejected": -194.80186462402344,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.02185620740056038,
+      "rewards/margins": 0.06259147822856903,
+      "rewards/rejected": -0.04073526710271835,
+      "step": 13340
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.399542962561399e-07,
+      "logits/chosen": -2.22048282623291,
+      "logits/rejected": -2.0561389923095703,
+      "logps/chosen": -222.4910125732422,
+      "logps/rejected": -185.62081909179688,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.029649171978235245,
+      "rewards/margins": 0.10539106279611588,
+      "rewards/rejected": -0.07574189454317093,
+      "step": 13350
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.3751924188969876e-07,
+      "logits/chosen": -2.2612175941467285,
+      "logits/rejected": -2.1327712535858154,
+      "logps/chosen": -249.13388061523438,
+      "logps/rejected": -239.14697265625,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.026046359911561012,
+      "rewards/margins": 0.09847725927829742,
+      "rewards/rejected": -0.07243090122938156,
+      "step": 13360
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.3509598973118024e-07,
+      "logits/chosen": -2.448444366455078,
+      "logits/rejected": -2.249556064605713,
+      "logps/chosen": -219.6066436767578,
+      "logps/rejected": -162.51773071289062,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.013987274840474129,
+      "rewards/margins": 0.05698896199464798,
+      "rewards/rejected": -0.043001689016819,
+      "step": 13370
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.326845524214555e-07,
+      "logits/chosen": -2.1156511306762695,
+      "logits/rejected": -2.193941354751587,
+      "logps/chosen": -234.92654418945312,
+      "logps/rejected": -209.80361938476562,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.001277850242331624,
+      "rewards/margins": 0.010018276050686836,
+      "rewards/rejected": -0.008740425109863281,
+      "step": 13380
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.3028494253976158e-07,
+      "logits/chosen": -2.3883180618286133,
+      "logits/rejected": -2.1776084899902344,
+      "logps/chosen": -340.9706115722656,
+      "logps/rejected": -282.01092529296875,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0029482836835086346,
+      "rewards/margins": 0.056216467171907425,
+      "rewards/rejected": -0.0532681830227375,
+      "step": 13390
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.2789717260364026e-07,
+      "logits/chosen": -2.376600980758667,
+      "logits/rejected": -2.19740891456604,
+      "logps/chosen": -164.39688110351562,
+      "logps/rejected": -148.5932159423828,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.007092096842825413,
+      "rewards/margins": 0.059240736067295074,
+      "rewards/rejected": -0.05214863270521164,
+      "step": 13400
+    },
+    {
+      "epoch": 0.88,
+      "eval_logits/chosen": -2.3266842365264893,
+      "eval_logits/rejected": -2.1379003524780273,
+      "eval_logps/chosen": -230.48577880859375,
+      "eval_logps/rejected": -218.5886993408203,
+      "eval_loss": 0.6897343993186951,
+      "eval_rewards/accuracies": 0.6305000185966492,
+      "eval_rewards/chosen": 0.015191725455224514,
+      "eval_rewards/margins": 0.08495970070362091,
+      "eval_rewards/rejected": -0.06976797431707382,
+      "eval_runtime": 711.3263,
+      "eval_samples_per_second": 2.812,
+      "eval_steps_per_second": 1.406,
+      "step": 13400
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.255212550688682e-07,
+      "logits/chosen": -2.2746529579162598,
+      "logits/rejected": -2.3803787231445312,
+      "logps/chosen": -216.0994110107422,
+      "logps/rejected": -289.5954895019531,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.018555883318185806,
+      "rewards/margins": 0.09309352934360504,
+      "rewards/rejected": -0.07453764975070953,
+      "step": 13410
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.2315720232939598e-07,
+      "logits/chosen": -2.6417040824890137,
+      "logits/rejected": -2.17865252494812,
+      "logps/chosen": -258.8695373535156,
+      "logps/rejected": -180.49835205078125,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03244171291589737,
+      "rewards/margins": 0.1123114600777626,
+      "rewards/rejected": -0.07986976206302643,
+      "step": 13420
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.2080502671727956e-07,
+      "logits/chosen": -2.428260087966919,
+      "logits/rejected": -2.08381724357605,
+      "logps/chosen": -218.2057342529297,
+      "logps/rejected": -204.05075073242188,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.018760915845632553,
+      "rewards/margins": 0.07672649621963501,
+      "rewards/rejected": -0.05796556919813156,
+      "step": 13430
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.1846474050262078e-07,
+      "logits/chosen": -2.3987889289855957,
+      "logits/rejected": -2.263538360595703,
+      "logps/chosen": -243.8972930908203,
+      "logps/rejected": -184.60328674316406,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.026634354144334793,
+      "rewards/margins": 0.06503110378980637,
+      "rewards/rejected": -0.03839675337076187,
+      "step": 13440
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.1613635589349756e-07,
+      "logits/chosen": -1.9668877124786377,
+      "logits/rejected": -2.058589458465576,
+      "logps/chosen": -194.289306640625,
+      "logps/rejected": -232.1760711669922,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.028739606961607933,
+      "rewards/margins": 0.10366035997867584,
+      "rewards/rejected": -0.07492075115442276,
+      "step": 13450
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.1381988503590578e-07,
+      "logits/chosen": -2.0607008934020996,
+      "logits/rejected": -2.126495599746704,
+      "logps/chosen": -213.87832641601562,
+      "logps/rejected": -218.08837890625,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.019935574382543564,
+      "rewards/margins": 0.10359902679920197,
+      "rewards/rejected": -0.0836634561419487,
+      "step": 13460
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.11515340013691e-07,
+      "logits/chosen": -2.41133975982666,
+      "logits/rejected": -2.3844552040100098,
+      "logps/chosen": -228.10009765625,
+      "logps/rejected": -219.16781616210938,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.030724655836820602,
+      "rewards/margins": 0.12647958099842072,
+      "rewards/rejected": -0.09575492143630981,
+      "step": 13470
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.092227328484897e-07,
+      "logits/chosen": -2.187948226928711,
+      "logits/rejected": -2.1468586921691895,
+      "logps/chosen": -202.74105834960938,
+      "logps/rejected": -243.099609375,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.012676420621573925,
+      "rewards/margins": 0.09701034426689148,
+      "rewards/rejected": -0.08433392643928528,
+      "step": 13480
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.0694207549966345e-07,
+      "logits/chosen": -2.2124342918395996,
+      "logits/rejected": -2.0843167304992676,
+      "logps/chosen": -208.91720581054688,
+      "logps/rejected": -197.19125366210938,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -8.745789818931371e-05,
+      "rewards/margins": 0.051435112953186035,
+      "rewards/rejected": -0.05152256414294243,
+      "step": 13490
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.0467337986423864e-07,
+      "logits/chosen": -2.462228298187256,
+      "logits/rejected": -2.1654422283172607,
+      "logps/chosen": -302.31915283203125,
+      "logps/rejected": -275.478759765625,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.0340435728430748,
+      "rewards/margins": 0.06372375041246414,
+      "rewards/rejected": -0.029680173844099045,
+      "step": 13500
+    },
+    {
+      "epoch": 0.88,
+      "eval_logits/chosen": -2.326474189758301,
+      "eval_logits/rejected": -2.1377792358398438,
+      "eval_logps/chosen": -230.06185913085938,
+      "eval_logps/rejected": -218.02516174316406,
+      "eval_loss": 0.6897424459457397,
+      "eval_rewards/accuracies": 0.6359999775886536,
+      "eval_rewards/chosen": 0.019430968910455704,
+      "eval_rewards/margins": 0.0835636630654335,
+      "eval_rewards/rejected": -0.0641326829791069,
+      "eval_runtime": 712.2388,
+      "eval_samples_per_second": 2.808,
+      "eval_steps_per_second": 1.404,
+      "step": 13500
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.0241665777684272e-07,
+      "logits/chosen": -2.3777573108673096,
+      "logits/rejected": -2.2866101264953613,
+      "logps/chosen": -265.6940002441406,
+      "logps/rejected": -234.2218017578125,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.025989696383476257,
+      "rewards/margins": 0.12143020331859589,
+      "rewards/rejected": -0.09544049948453903,
+      "step": 13510
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.0017192100964366e-07,
+      "logits/chosen": -2.0364346504211426,
+      "logits/rejected": -2.1206259727478027,
+      "logps/chosen": -199.53173828125,
+      "logps/rejected": -210.08749389648438,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.004242539405822754,
+      "rewards/margins": 0.08438628911972046,
+      "rewards/rejected": -0.08014374226331711,
+      "step": 13520
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.9793918127228777e-07,
+      "logits/chosen": -2.411618232727051,
+      "logits/rejected": -2.04459547996521,
+      "logps/chosen": -314.3971862792969,
+      "logps/rejected": -269.76873779296875,
+      "loss": 0.688,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.019022373482584953,
+      "rewards/margins": 0.09453563392162323,
+      "rewards/rejected": -0.07551325857639313,
+      "step": 13530
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.9571845021184005e-07,
+      "logits/chosen": -2.174851894378662,
+      "logits/rejected": -2.0823864936828613,
+      "logps/chosen": -232.1296844482422,
+      "logps/rejected": -243.97640991210938,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.011437867768108845,
+      "rewards/margins": 0.07728725671768188,
+      "rewards/rejected": -0.08872512727975845,
+      "step": 13540
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.9350973941272027e-07,
+      "logits/chosen": -2.308931350708008,
+      "logits/rejected": -2.2825989723205566,
+      "logps/chosen": -203.7218017578125,
+      "logps/rejected": -191.3800506591797,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0005322685465216637,
+      "rewards/margins": 0.08159051835536957,
+      "rewards/rejected": -0.08105824887752533,
+      "step": 13550
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.9131306039664676e-07,
+      "logits/chosen": -2.1821532249450684,
+      "logits/rejected": -2.1290283203125,
+      "logps/chosen": -198.3598175048828,
+      "logps/rejected": -232.4375457763672,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.015442472882568836,
+      "rewards/margins": 0.09721332788467407,
+      "rewards/rejected": -0.08177085965871811,
+      "step": 13560
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.8912842462257358e-07,
+      "logits/chosen": -2.2131478786468506,
+      "logits/rejected": -2.144645929336548,
+      "logps/chosen": -217.9169464111328,
+      "logps/rejected": -212.69229125976562,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.022523250430822372,
+      "rewards/margins": 0.11762279272079468,
+      "rewards/rejected": -0.0950995534658432,
+      "step": 13570
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.869558434866303e-07,
+      "logits/chosen": -2.2858431339263916,
+      "logits/rejected": -2.352550745010376,
+      "logps/chosen": -180.52920532226562,
+      "logps/rejected": -207.8330078125,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0056061758659780025,
+      "rewards/margins": 0.0927395448088646,
+      "rewards/rejected": -0.08713337033987045,
+      "step": 13580
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.847953283220652e-07,
+      "logits/chosen": -2.4596104621887207,
+      "logits/rejected": -2.134106397628784,
+      "logps/chosen": -252.9336700439453,
+      "logps/rejected": -188.153076171875,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.03176042437553406,
+      "rewards/margins": 0.1385561227798462,
+      "rewards/rejected": -0.10679570585489273,
+      "step": 13590
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.8264689039918265e-07,
+      "logits/chosen": -2.424584150314331,
+      "logits/rejected": -2.084427833557129,
+      "logps/chosen": -258.3966369628906,
+      "logps/rejected": -238.4198760986328,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.014337467029690742,
+      "rewards/margins": 0.07449439913034439,
+      "rewards/rejected": -0.0601569339632988,
+      "step": 13600
+    },
+    {
+      "epoch": 0.89,
+      "eval_logits/chosen": -2.3269259929656982,
+      "eval_logits/rejected": -2.138150930404663,
+      "eval_logps/chosen": -230.37106323242188,
+      "eval_logps/rejected": -218.5099639892578,
+      "eval_loss": 0.6897370219230652,
+      "eval_rewards/accuracies": 0.6380000114440918,
+      "eval_rewards/chosen": 0.016339082270860672,
+      "eval_rewards/margins": 0.08531977236270905,
+      "eval_rewards/rejected": -0.06898068636655807,
+      "eval_runtime": 711.7818,
+      "eval_samples_per_second": 2.81,
+      "eval_steps_per_second": 1.405,
+      "step": 13600
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.8051054092528857e-07,
+      "logits/chosen": -2.351792573928833,
+      "logits/rejected": -2.163576602935791,
+      "logps/chosen": -257.79010009765625,
+      "logps/rejected": -259.30328369140625,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.03311315178871155,
+      "rewards/margins": 0.11273415386676788,
+      "rewards/rejected": -0.07962099462747574,
+      "step": 13610
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.783862910446271e-07,
+      "logits/chosen": -1.985805869102478,
+      "logits/rejected": -2.1247270107269287,
+      "logps/chosen": -171.29833984375,
+      "logps/rejected": -174.77696228027344,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.01554956752806902,
+      "rewards/margins": 0.12793493270874023,
+      "rewards/rejected": -0.1123853549361229,
+      "step": 13620
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.762741518383271e-07,
+      "logits/chosen": -2.367798328399658,
+      "logits/rejected": -2.2053184509277344,
+      "logps/chosen": -220.3152618408203,
+      "logps/rejected": -201.3122100830078,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.015171055682003498,
+      "rewards/margins": 0.08661060780286789,
+      "rewards/rejected": -0.07143954932689667,
+      "step": 13630
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.7417413432434082e-07,
+      "logits/chosen": -2.435668468475342,
+      "logits/rejected": -2.041511058807373,
+      "logps/chosen": -252.56173706054688,
+      "logps/rejected": -206.9319305419922,
+      "loss": 0.691,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0009791527409106493,
+      "rewards/margins": 0.07137512415647507,
+      "rewards/rejected": -0.07039596140384674,
+      "step": 13640
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.7208624945738855e-07,
+      "logits/chosen": -2.435779571533203,
+      "logits/rejected": -2.284393310546875,
+      "logps/chosen": -216.6245574951172,
+      "logps/rejected": -228.45632934570312,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.014597500674426556,
+      "rewards/margins": 0.04394357651472092,
+      "rewards/rejected": -0.02934606932103634,
+      "step": 13650
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.7001050812889995e-07,
+      "logits/chosen": -2.412449598312378,
+      "logits/rejected": -2.0942466259002686,
+      "logps/chosen": -256.7345886230469,
+      "logps/rejected": -224.16006469726562,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.01201794296503067,
+      "rewards/margins": 0.0767635852098465,
+      "rewards/rejected": -0.08878152817487717,
+      "step": 13660
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.679469211669596e-07,
+      "logits/chosen": -2.3339667320251465,
+      "logits/rejected": -2.177741527557373,
+      "logps/chosen": -224.02359008789062,
+      "logps/rejected": -183.2321319580078,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.009215526282787323,
+      "rewards/margins": 0.10812918096780777,
+      "rewards/rejected": -0.09891365468502045,
+      "step": 13670
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.6589549933624715e-07,
+      "logits/chosen": -2.3178319931030273,
+      "logits/rejected": -2.134033679962158,
+      "logps/chosen": -236.70999145507812,
+      "logps/rejected": -199.88436889648438,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.03497043251991272,
+      "rewards/margins": 0.14659801125526428,
+      "rewards/rejected": -0.11162757873535156,
+      "step": 13680
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.638562533379845e-07,
+      "logits/chosen": -2.3221664428710938,
+      "logits/rejected": -2.150352954864502,
+      "logps/chosen": -253.377197265625,
+      "logps/rejected": -197.33758544921875,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.02325398102402687,
+      "rewards/margins": 0.062283407896757126,
+      "rewards/rejected": -0.039029426872730255,
+      "step": 13690
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.6182919380987676e-07,
+      "logits/chosen": -2.365807056427002,
+      "logits/rejected": -2.283456325531006,
+      "logps/chosen": -224.37783813476562,
+      "logps/rejected": -214.51156616210938,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.013401429168879986,
+      "rewards/margins": 0.055108923465013504,
+      "rewards/rejected": -0.041707489639520645,
+      "step": 13700
+    },
+    {
+      "epoch": 0.9,
+      "eval_logits/chosen": -2.3265719413757324,
+      "eval_logits/rejected": -2.1378707885742188,
+      "eval_logps/chosen": -230.2803497314453,
+      "eval_logps/rejected": -218.3449249267578,
+      "eval_loss": 0.6897291541099548,
+      "eval_rewards/accuracies": 0.6359999775886536,
+      "eval_rewards/chosen": 0.017245886847376823,
+      "eval_rewards/margins": 0.08457593619823456,
+      "eval_rewards/rejected": -0.06733004748821259,
+      "eval_runtime": 711.018,
+      "eval_samples_per_second": 2.813,
+      "eval_steps_per_second": 1.406,
+      "step": 13700
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.598143313260603e-07,
+      "logits/chosen": -2.2812981605529785,
+      "logits/rejected": -2.1852867603302,
+      "logps/chosen": -186.65716552734375,
+      "logps/rejected": -181.11973571777344,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.013965973630547523,
+      "rewards/margins": 0.0647711455821991,
+      "rewards/rejected": -0.05080517381429672,
+      "step": 13710
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.5781167639704415e-07,
+      "logits/chosen": -2.5151546001434326,
+      "logits/rejected": -1.9689128398895264,
+      "logps/chosen": -329.40081787109375,
+      "logps/rejected": -211.47140502929688,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.027660047635436058,
+      "rewards/margins": 0.0631406158208847,
+      "rewards/rejected": -0.0354805663228035,
+      "step": 13720
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.5582123946965787e-07,
+      "logits/chosen": -2.1882617473602295,
+      "logits/rejected": -2.0734565258026123,
+      "logps/chosen": -231.71762084960938,
+      "logps/rejected": -250.3955535888672,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.026773914694786072,
+      "rewards/margins": 0.08530018478631973,
+      "rewards/rejected": -0.05852626636624336,
+      "step": 13730
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.5384303092699504e-07,
+      "logits/chosen": -2.391803503036499,
+      "logits/rejected": -2.1646900177001953,
+      "logps/chosen": -284.1597900390625,
+      "logps/rejected": -295.56170654296875,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.02824859321117401,
+      "rewards/margins": 0.13083064556121826,
+      "rewards/rejected": -0.10258202254772186,
+      "step": 13740
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.518770610883613e-07,
+      "logits/chosen": -2.2725253105163574,
+      "logits/rejected": -2.001561403274536,
+      "logps/chosen": -215.4679412841797,
+      "logps/rejected": -199.0486297607422,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.009293107315897942,
+      "rewards/margins": 0.12994512915611267,
+      "rewards/rejected": -0.13923820853233337,
+      "step": 13750
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.4992334020921735e-07,
+      "logits/chosen": -2.275468111038208,
+      "logits/rejected": -2.1718432903289795,
+      "logps/chosen": -168.7460174560547,
+      "logps/rejected": -151.58731079101562,
+      "loss": 0.688,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.04226404055953026,
+      "rewards/margins": 0.12455103546380997,
+      "rewards/rejected": -0.08228699862957001,
+      "step": 13760
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.4798187848112905e-07,
+      "logits/chosen": -2.203535556793213,
+      "logits/rejected": -2.2031960487365723,
+      "logps/chosen": -224.80953979492188,
+      "logps/rejected": -193.4915313720703,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.018388142809271812,
+      "rewards/margins": 0.09539445489645004,
+      "rewards/rejected": -0.113782599568367,
+      "step": 13770
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.460526860317113e-07,
+      "logits/chosen": -2.3986401557922363,
+      "logits/rejected": -2.332296848297119,
+      "logps/chosen": -173.04156494140625,
+      "logps/rejected": -216.95974731445312,
+      "loss": 0.686,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0024310871958732605,
+      "rewards/margins": 0.12171381711959839,
+      "rewards/rejected": -0.11928270757198334,
+      "step": 13780
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.441357729245771e-07,
+      "logits/chosen": -2.5229907035827637,
+      "logits/rejected": -1.9477760791778564,
+      "logps/chosen": -248.5333709716797,
+      "logps/rejected": -202.63084411621094,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0054153092205524445,
+      "rewards/margins": 0.10084688663482666,
+      "rewards/rejected": -0.10626220703125,
+      "step": 13790
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.4223114915928482e-07,
+      "logits/chosen": -2.1714558601379395,
+      "logits/rejected": -1.9200232028961182,
+      "logps/chosen": -225.0757293701172,
+      "logps/rejected": -238.7044219970703,
+      "loss": 0.69,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.008468665182590485,
+      "rewards/margins": 0.06861492991447449,
+      "rewards/rejected": -0.060146261006593704,
+      "step": 13800
+    },
+    {
+      "epoch": 0.9,
+      "eval_logits/chosen": -2.3265576362609863,
+      "eval_logits/rejected": -2.137892961502075,
+      "eval_logps/chosen": -230.2596893310547,
+      "eval_logps/rejected": -218.37973022460938,
+      "eval_loss": 0.6897318363189697,
+      "eval_rewards/accuracies": 0.6389999985694885,
+      "eval_rewards/chosen": 0.017452586442232132,
+      "eval_rewards/margins": 0.08513098210096359,
+      "eval_rewards/rejected": -0.06767839938402176,
+      "eval_runtime": 712.4298,
+      "eval_samples_per_second": 2.807,
+      "eval_steps_per_second": 1.404,
+      "step": 13800
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.403388246712842e-07,
+      "logits/chosen": -2.2311573028564453,
+      "logits/rejected": -1.99333918094635,
+      "logps/chosen": -164.9827423095703,
+      "logps/rejected": -163.3975830078125,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.00731696467846632,
+      "rewards/margins": 0.055666130036115646,
+      "rewards/rejected": -0.06298309564590454,
+      "step": 13810
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.3845880933186757e-07,
+      "logits/chosen": -2.4922029972076416,
+      "logits/rejected": -2.229177951812744,
+      "logps/chosen": -237.0489959716797,
+      "logps/rejected": -206.5233154296875,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.01689792238175869,
+      "rewards/margins": 0.037102360278367996,
+      "rewards/rejected": -0.020204436033964157,
+      "step": 13820
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.3659111294811457e-07,
+      "logits/chosen": -2.3271474838256836,
+      "logits/rejected": -2.1840929985046387,
+      "logps/chosen": -193.53775024414062,
+      "logps/rejected": -181.85487365722656,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.016945457085967064,
+      "rewards/margins": 0.062133751809597015,
+      "rewards/rejected": -0.07907922565937042,
+      "step": 13830
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.347357452628459e-07,
+      "logits/chosen": -2.4782280921936035,
+      "logits/rejected": -2.344364643096924,
+      "logps/chosen": -241.06307983398438,
+      "logps/rejected": -240.8955078125,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.04234471544623375,
+      "rewards/margins": 0.07334139943122864,
+      "rewards/rejected": -0.030996689572930336,
+      "step": 13840
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.3289271595456732e-07,
+      "logits/chosen": -2.2512059211730957,
+      "logits/rejected": -2.064351797103882,
+      "logps/chosen": -204.22561645507812,
+      "logps/rejected": -189.5396270751953,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.0010356527054682374,
+      "rewards/margins": 0.1110767275094986,
+      "rewards/rejected": -0.1100410670042038,
+      "step": 13850
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.310620346374228e-07,
+      "logits/chosen": -2.1624104976654053,
+      "logits/rejected": -2.0331850051879883,
+      "logps/chosen": -229.8220672607422,
+      "logps/rejected": -205.7158660888672,
+      "loss": 0.6867,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0239988062530756,
+      "rewards/margins": 0.12696941196918488,
+      "rewards/rejected": -0.10297061502933502,
+      "step": 13860
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.2924371086114274e-07,
+      "logits/chosen": -2.213139057159424,
+      "logits/rejected": -1.9702332019805908,
+      "logps/chosen": -234.7813262939453,
+      "logps/rejected": -237.0749053955078,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.01280174870043993,
+      "rewards/margins": 0.0832042545080185,
+      "rewards/rejected": -0.07040251046419144,
+      "step": 13870
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.274377541109953e-07,
+      "logits/chosen": -2.175238847732544,
+      "logits/rejected": -2.2496845722198486,
+      "logps/chosen": -163.90567016601562,
+      "logps/rejected": -248.86221313476562,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.004448303487151861,
+      "rewards/margins": 0.07032819092273712,
+      "rewards/rejected": -0.06587988883256912,
+      "step": 13880
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.2564417380773435e-07,
+      "logits/chosen": -2.1063437461853027,
+      "logits/rejected": -1.9700400829315186,
+      "logps/chosen": -177.04061889648438,
+      "logps/rejected": -210.00146484375,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.014564013108611107,
+      "rewards/margins": 0.10478832572698593,
+      "rewards/rejected": -0.09022431075572968,
+      "step": 13890
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.2386297930755436e-07,
+      "logits/chosen": -2.3223581314086914,
+      "logits/rejected": -2.3294119834899902,
+      "logps/chosen": -249.2999267578125,
+      "logps/rejected": -253.54208374023438,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.014141452498733997,
+      "rewards/margins": 0.0906859040260315,
+      "rewards/rejected": -0.10482735931873322,
+      "step": 13900
+    },
+    {
+      "epoch": 0.91,
+      "eval_logits/chosen": -2.3256688117980957,
+      "eval_logits/rejected": -2.137094497680664,
+      "eval_logps/chosen": -230.19509887695312,
+      "eval_logps/rejected": -218.29586791992188,
+      "eval_loss": 0.6897297501564026,
+      "eval_rewards/accuracies": 0.6399999856948853,
+      "eval_rewards/chosen": 0.01809842139482498,
+      "eval_rewards/margins": 0.08493825048208237,
+      "eval_rewards/rejected": -0.06683983653783798,
+      "eval_runtime": 714.4928,
+      "eval_samples_per_second": 2.799,
+      "eval_steps_per_second": 1.4,
+      "step": 13900
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.220941799020378e-07,
+      "logits/chosen": -2.109891176223755,
+      "logits/rejected": -2.0245959758758545,
+      "logps/chosen": -217.385498046875,
+      "logps/rejected": -203.8630828857422,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.016956061124801636,
+      "rewards/margins": 0.08830462396144867,
+      "rewards/rejected": -0.07134857028722763,
+      "step": 13910
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.2033778481810975e-07,
+      "logits/chosen": -2.38193416595459,
+      "logits/rejected": -2.114203453063965,
+      "logps/chosen": -217.64212036132812,
+      "logps/rejected": -189.1055145263672,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.02351401373744011,
+      "rewards/margins": 0.08809584379196167,
+      "rewards/rejected": -0.06458182632923126,
+      "step": 13920
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1859380321798591e-07,
+      "logits/chosen": -2.3214306831359863,
+      "logits/rejected": -2.388051748275757,
+      "logps/chosen": -200.78286743164062,
+      "logps/rejected": -222.736083984375,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.017784133553504944,
+      "rewards/margins": 0.07208283245563507,
+      "rewards/rejected": -0.05429869890213013,
+      "step": 13930
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1686224419912989e-07,
+      "logits/chosen": -2.2252655029296875,
+      "logits/rejected": -2.0139126777648926,
+      "logps/chosen": -251.2513885498047,
+      "logps/rejected": -235.0546112060547,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.016212433576583862,
+      "rewards/margins": 0.13094016909599304,
+      "rewards/rejected": -0.11472772061824799,
+      "step": 13940
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1514311679420104e-07,
+      "logits/chosen": -2.0391550064086914,
+      "logits/rejected": -2.1126651763916016,
+      "logps/chosen": -154.56124877929688,
+      "logps/rejected": -220.6207733154297,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.005872879642993212,
+      "rewards/margins": 0.10239820182323456,
+      "rewards/rejected": -0.09652532637119293,
+      "step": 13950
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1343642997101029e-07,
+      "logits/chosen": -2.3179874420166016,
+      "logits/rejected": -2.2247185707092285,
+      "logps/chosen": -199.3680877685547,
+      "logps/rejected": -196.05459594726562,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.029852483421564102,
+      "rewards/margins": 0.09908358752727509,
+      "rewards/rejected": -0.06923110783100128,
+      "step": 13960
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1174219263247188e-07,
+      "logits/chosen": -2.0668439865112305,
+      "logits/rejected": -1.9567053318023682,
+      "logps/chosen": -198.8998260498047,
+      "logps/rejected": -190.6134033203125,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.000822742294985801,
+      "rewards/margins": 0.0924471914768219,
+      "rewards/rejected": -0.09326992928981781,
+      "step": 13970
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1006041361655839e-07,
+      "logits/chosen": -2.4929490089416504,
+      "logits/rejected": -2.028357982635498,
+      "logps/chosen": -208.04507446289062,
+      "logps/rejected": -175.75112915039062,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.003921913914382458,
+      "rewards/margins": 0.07643552869558334,
+      "rewards/rejected": -0.07251361012458801,
+      "step": 13980
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.0839110169625189e-07,
+      "logits/chosen": -2.05533504486084,
+      "logits/rejected": -2.352057933807373,
+      "logps/chosen": -205.25399780273438,
+      "logps/rejected": -211.28079223632812,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.015882687643170357,
+      "rewards/margins": 0.133821040391922,
+      "rewards/rejected": -0.1179383248090744,
+      "step": 13990
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.06734265579502e-07,
+      "logits/chosen": -2.337198495864868,
+      "logits/rejected": -2.033975124359131,
+      "logps/chosen": -262.0823974609375,
+      "logps/rejected": -206.0341796875,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.005657001864165068,
+      "rewards/margins": 0.10456991195678711,
+      "rewards/rejected": -0.11022691428661346,
+      "step": 14000
+    },
+    {
+      "epoch": 0.92,
+      "eval_logits/chosen": -2.326231002807617,
+      "eval_logits/rejected": -2.137554168701172,
+      "eval_logps/chosen": -230.58172607421875,
+      "eval_logps/rejected": -218.70066833496094,
+      "eval_loss": 0.689732551574707,
+      "eval_rewards/accuracies": 0.6380000114440918,
+      "eval_rewards/chosen": 0.01423216424882412,
+      "eval_rewards/margins": 0.0851198136806488,
+      "eval_rewards/rejected": -0.07088765501976013,
+      "eval_runtime": 712.0104,
+      "eval_samples_per_second": 2.809,
+      "eval_steps_per_second": 1.404,
+      "step": 14000
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.050899139091771e-07,
+      "logits/chosen": -2.416721820831299,
+      "logits/rejected": -2.0669798851013184,
+      "logps/chosen": -279.1082763671875,
+      "logps/rejected": -241.64761352539062,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.0048080976121127605,
+      "rewards/margins": 0.09057492017745972,
+      "rewards/rejected": -0.09538300335407257,
+      "step": 14010
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.0345805526302072e-07,
+      "logits/chosen": -2.242600917816162,
+      "logits/rejected": -2.3467628955841064,
+      "logps/chosen": -198.83761596679688,
+      "logps/rejected": -201.98965454101562,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.023247262462973595,
+      "rewards/margins": 0.10389737784862518,
+      "rewards/rejected": -0.08065011352300644,
+      "step": 14020
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.0183869815360764e-07,
+      "logits/chosen": -2.231644868850708,
+      "logits/rejected": -2.350635290145874,
+      "logps/chosen": -189.2916717529297,
+      "logps/rejected": -228.9336700439453,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.02621796727180481,
+      "rewards/margins": 0.07001911103725433,
+      "rewards/rejected": -0.04380114749073982,
+      "step": 14030
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.0023185102829763e-07,
+      "logits/chosen": -2.0455517768859863,
+      "logits/rejected": -2.2643027305603027,
+      "logps/chosen": -228.81332397460938,
+      "logps/rejected": -242.60061645507812,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.020088955760002136,
+      "rewards/margins": 0.09602537006139755,
+      "rewards/rejected": -0.07593640685081482,
+      "step": 14040
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.863752226919182e-08,
+      "logits/chosen": -2.2488272190093994,
+      "logits/rejected": -1.7611221075057983,
+      "logps/chosen": -235.35617065429688,
+      "logps/rejected": -182.02853393554688,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.02376001887023449,
+      "rewards/margins": 0.12250945717096329,
+      "rewards/rejected": -0.09874944388866425,
+      "step": 14050
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.705572019309107e-08,
+      "logits/chosen": -2.169804096221924,
+      "logits/rejected": -2.2068305015563965,
+      "logps/chosen": -266.7760925292969,
+      "logps/rejected": -246.24984741210938,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.012513126246631145,
+      "rewards/margins": 0.10957686603069305,
+      "rewards/rejected": -0.09706376492977142,
+      "step": 14060
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.548645305144849e-08,
+      "logits/chosen": -2.3847720623016357,
+      "logits/rejected": -2.2535669803619385,
+      "logps/chosen": -171.47378540039062,
+      "logps/rejected": -184.33763122558594,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.020213961601257324,
+      "rewards/margins": 0.08890150487422943,
+      "rewards/rejected": -0.0686875432729721,
+      "step": 14070
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.392972903033149e-08,
+      "logits/chosen": -2.306051254272461,
+      "logits/rejected": -2.159519672393799,
+      "logps/chosen": -222.64682006835938,
+      "logps/rejected": -218.47372436523438,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.004844355396926403,
+      "rewards/margins": 0.03370397537946701,
+      "rewards/rejected": -0.028859620913863182,
+      "step": 14080
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.238555625037449e-08,
+      "logits/chosen": -2.328538417816162,
+      "logits/rejected": -2.1120870113372803,
+      "logps/chosen": -188.9663848876953,
+      "logps/rejected": -166.45822143554688,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.017418090254068375,
+      "rewards/margins": 0.0717364102602005,
+      "rewards/rejected": -0.05431831628084183,
+      "step": 14090
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.085394276673903e-08,
+      "logits/chosen": -2.3301329612731934,
+      "logits/rejected": -2.0303778648376465,
+      "logps/chosen": -267.52117919921875,
+      "logps/rejected": -261.9759216308594,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.006449407432228327,
+      "rewards/margins": 0.0890425592660904,
+      "rewards/rejected": -0.0825931578874588,
+      "step": 14100
+    },
+    {
+      "epoch": 0.92,
+      "eval_logits/chosen": -2.325167417526245,
+      "eval_logits/rejected": -2.1365697383880615,
+      "eval_logps/chosen": -230.42181396484375,
+      "eval_logps/rejected": -218.4661865234375,
+      "eval_loss": 0.6897357106208801,
+      "eval_rewards/accuracies": 0.637499988079071,
+      "eval_rewards/chosen": 0.0158314798027277,
+      "eval_rewards/margins": 0.08437444269657135,
+      "eval_rewards/rejected": -0.0685429498553276,
+      "eval_runtime": 710.8403,
+      "eval_samples_per_second": 2.814,
+      "eval_steps_per_second": 1.407,
+      "step": 14100
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.933489656907157e-08,
+      "logits/chosen": -2.308310031890869,
+      "logits/rejected": -2.2029194831848145,
+      "logps/chosen": -217.09829711914062,
+      "logps/rejected": -247.48666381835938,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.00237136147916317,
+      "rewards/margins": 0.05447987839579582,
+      "rewards/rejected": -0.0521085187792778,
+      "step": 14110
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.782842558146127e-08,
+      "logits/chosen": -2.3630995750427246,
+      "logits/rejected": -2.2901546955108643,
+      "logps/chosen": -159.1778564453125,
+      "logps/rejected": -167.31874084472656,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.02706066146492958,
+      "rewards/margins": 0.09847380220890045,
+      "rewards/rejected": -0.07141314446926117,
+      "step": 14120
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.633453766239836e-08,
+      "logits/chosen": -2.415795087814331,
+      "logits/rejected": -2.186148166656494,
+      "logps/chosen": -230.60061645507812,
+      "logps/rejected": -205.28359985351562,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.04520539939403534,
+      "rewards/margins": 0.06538231670856476,
+      "rewards/rejected": -0.020176919177174568,
+      "step": 14130
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 8.485324060473448e-08,
+      "logits/chosen": -2.249516248703003,
+      "logits/rejected": -2.1395606994628906,
+      "logps/chosen": -237.13107299804688,
+      "logps/rejected": -232.41909790039062,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.031510137021541595,
+      "rewards/margins": 0.07678806781768799,
+      "rewards/rejected": -0.04527793079614639,
+      "step": 14140
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 8.338454213564052e-08,
+      "logits/chosen": -2.291496753692627,
+      "logits/rejected": -2.0189805030822754,
+      "logps/chosen": -230.54672241210938,
+      "logps/rejected": -223.07864379882812,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.0035024010576307774,
+      "rewards/margins": 0.09726149588823318,
+      "rewards/rejected": -0.09375908970832825,
+      "step": 14150
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 8.192844991656679e-08,
+      "logits/chosen": -2.2867093086242676,
+      "logits/rejected": -2.0489730834960938,
+      "logps/chosen": -237.1662139892578,
+      "logps/rejected": -208.5237579345703,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.015504756942391396,
+      "rewards/margins": 0.07765550911426544,
+      "rewards/rejected": -0.0621507465839386,
+      "step": 14160
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 8.048497154320434e-08,
+      "logits/chosen": -2.3233590126037598,
+      "logits/rejected": -2.3688528537750244,
+      "logps/chosen": -130.73397827148438,
+      "logps/rejected": -147.06256103515625,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.011844434775412083,
+      "rewards/margins": 0.07231110334396362,
+      "rewards/rejected": -0.08415552228689194,
+      "step": 14170
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.905411454544265e-08,
+      "logits/chosen": -2.324502468109131,
+      "logits/rejected": -2.174683094024658,
+      "logps/chosen": -236.509765625,
+      "logps/rejected": -246.95889282226562,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -5.5506454373244196e-05,
+      "rewards/margins": 0.057068269699811935,
+      "rewards/rejected": -0.057123780250549316,
+      "step": 14180
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.763588638733332e-08,
+      "logits/chosen": -2.3418097496032715,
+      "logits/rejected": -2.2953555583953857,
+      "logps/chosen": -260.10223388671875,
+      "logps/rejected": -249.2476348876953,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.023179076611995697,
+      "rewards/margins": 0.08969531953334808,
+      "rewards/rejected": -0.06651624292135239,
+      "step": 14190
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.623029446704899e-08,
+      "logits/chosen": -2.1991162300109863,
+      "logits/rejected": -2.3756251335144043,
+      "logps/chosen": -308.20355224609375,
+      "logps/rejected": -282.3921813964844,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.03237896040081978,
+      "rewards/margins": 0.11083276569843292,
+      "rewards/rejected": -0.07845381647348404,
+      "step": 14200
+    },
+    {
+      "epoch": 0.93,
+      "eval_logits/chosen": -2.325500965118408,
+      "eval_logits/rejected": -2.1368813514709473,
+      "eval_logps/chosen": -230.51712036132812,
+      "eval_logps/rejected": -218.59410095214844,
+      "eval_loss": 0.6897341012954712,
+      "eval_rewards/accuracies": 0.637499988079071,
+      "eval_rewards/chosen": 0.014878012239933014,
+      "eval_rewards/margins": 0.08469977974891663,
+      "eval_rewards/rejected": -0.06982176750898361,
+      "eval_runtime": 711.1188,
+      "eval_samples_per_second": 2.812,
+      "eval_steps_per_second": 1.406,
+      "step": 14200
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.483734611684557e-08,
+      "logits/chosen": -2.124391555786133,
+      "logits/rejected": -1.9591686725616455,
+      "logps/chosen": -254.62777709960938,
+      "logps/rejected": -209.1081085205078,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.022580375894904137,
+      "rewards/margins": 0.08641272783279419,
+      "rewards/rejected": -0.0638323426246643,
+      "step": 14210
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.345704860302366e-08,
+      "logits/chosen": -2.399385690689087,
+      "logits/rejected": -2.3872694969177246,
+      "logps/chosen": -246.51803588867188,
+      "logps/rejected": -255.66213989257812,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.017346328124403954,
+      "rewards/margins": 0.09822475165128708,
+      "rewards/rejected": -0.08087843656539917,
+      "step": 14220
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.208940912589224e-08,
+      "logits/chosen": -2.334130048751831,
+      "logits/rejected": -1.9988048076629639,
+      "logps/chosen": -210.1862335205078,
+      "logps/rejected": -185.14744567871094,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.0059993029572069645,
+      "rewards/margins": 0.12339667230844498,
+      "rewards/rejected": -0.1293959617614746,
+      "step": 14230
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.073443481972753e-08,
+      "logits/chosen": -2.1471316814422607,
+      "logits/rejected": -2.125434637069702,
+      "logps/chosen": -184.0389404296875,
+      "logps/rejected": -211.4552459716797,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0011679657036438584,
+      "rewards/margins": 0.07914597541093826,
+      "rewards/rejected": -0.07797800749540329,
+      "step": 14240
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.939213275274027e-08,
+      "logits/chosen": -2.2806684970855713,
+      "logits/rejected": -2.240402936935425,
+      "logps/chosen": -238.58798217773438,
+      "logps/rejected": -225.0943603515625,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.006657888181507587,
+      "rewards/margins": 0.06109069660305977,
+      "rewards/rejected": -0.054432809352874756,
+      "step": 14250
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.806250992703461e-08,
+      "logits/chosen": -2.3053228855133057,
+      "logits/rejected": -2.15181303024292,
+      "logps/chosen": -214.410400390625,
+      "logps/rejected": -195.6936798095703,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.022841984406113625,
+      "rewards/margins": 0.07493428885936737,
+      "rewards/rejected": -0.0520923025906086,
+      "step": 14260
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.674557327857572e-08,
+      "logits/chosen": -2.2985949516296387,
+      "logits/rejected": -2.311959743499756,
+      "logps/chosen": -251.68814086914062,
+      "logps/rejected": -254.1820831298828,
+      "loss": 0.6867,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.025825273245573044,
+      "rewards/margins": 0.11581947654485703,
+      "rewards/rejected": -0.08999422192573547,
+      "step": 14270
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.544132967714917e-08,
+      "logits/chosen": -2.0479283332824707,
+      "logits/rejected": -2.0568835735321045,
+      "logps/chosen": -246.87765502929688,
+      "logps/rejected": -244.86984252929688,
+      "loss": 0.688,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.00017667413339950144,
+      "rewards/margins": 0.11704652011394501,
+      "rewards/rejected": -0.11722320318222046,
+      "step": 14280
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.414978592632932e-08,
+      "logits/chosen": -2.3786206245422363,
+      "logits/rejected": -1.9369407892227173,
+      "logps/chosen": -260.21160888671875,
+      "logps/rejected": -222.98580932617188,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.008877063170075417,
+      "rewards/margins": 0.09229589253664017,
+      "rewards/rejected": -0.0834188237786293,
+      "step": 14290
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.287094876344046e-08,
+      "logits/chosen": -2.3294384479522705,
+      "logits/rejected": -2.362797260284424,
+      "logps/chosen": -168.59146118164062,
+      "logps/rejected": -183.0385284423828,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03828797861933708,
+      "rewards/margins": 0.07808025181293488,
+      "rewards/rejected": -0.0397922620177269,
+      "step": 14300
+    },
+    {
+      "epoch": 0.94,
+      "eval_logits/chosen": -2.325085163116455,
+      "eval_logits/rejected": -2.1365013122558594,
+      "eval_logps/chosen": -230.55078125,
+      "eval_logps/rejected": -218.63143920898438,
+      "eval_loss": 0.6897284388542175,
+      "eval_rewards/accuracies": 0.6399999856948853,
+      "eval_rewards/chosen": 0.014541618525981903,
+      "eval_rewards/margins": 0.08473705500364304,
+      "eval_rewards/rejected": -0.07019543647766113,
+      "eval_runtime": 710.3597,
+      "eval_samples_per_second": 2.815,
+      "eval_steps_per_second": 1.408,
+      "step": 14300
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.160482485952413e-08,
+      "logits/chosen": -2.46873140335083,
+      "logits/rejected": -2.203112840652466,
+      "logps/chosen": -236.5559539794922,
+      "logps/rejected": -211.58816528320312,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.001061995280906558,
+      "rewards/margins": 0.0682806596159935,
+      "rewards/rejected": -0.06934265792369843,
+      "step": 14310
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.035142081930234e-08,
+      "logits/chosen": -2.333582639694214,
+      "logits/rejected": -1.9607412815093994,
+      "logps/chosen": -263.0543518066406,
+      "logps/rejected": -192.89089965820312,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.002580016851425171,
+      "rewards/margins": 0.0685344785451889,
+      "rewards/rejected": -0.07111448794603348,
+      "step": 14320
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.911074318114496e-08,
+      "logits/chosen": -2.16135311126709,
+      "logits/rejected": -2.2769253253936768,
+      "logps/chosen": -202.2639617919922,
+      "logps/rejected": -251.79052734375,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.01324182003736496,
+      "rewards/margins": 0.07401735335588455,
+      "rewards/rejected": -0.060775529593229294,
+      "step": 14330
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.788279841703381e-08,
+      "logits/chosen": -2.3670153617858887,
+      "logits/rejected": -2.1148641109466553,
+      "logps/chosen": -182.29180908203125,
+      "logps/rejected": -184.1455535888672,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.01617772877216339,
+      "rewards/margins": 0.09520837664604187,
+      "rewards/rejected": -0.07903064042329788,
+      "step": 14340
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.66675929325311e-08,
+      "logits/chosen": -2.369096279144287,
+      "logits/rejected": -2.1264519691467285,
+      "logps/chosen": -218.7493438720703,
+      "logps/rejected": -212.5388641357422,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.019151246175169945,
+      "rewards/margins": 0.05138329789042473,
+      "rewards/rejected": -0.032232046127319336,
+      "step": 14350
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.546513306674301e-08,
+      "logits/chosen": -2.2958080768585205,
+      "logits/rejected": -1.9007478952407837,
+      "logps/chosen": -276.90521240234375,
+      "logps/rejected": -211.9548797607422,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.016114667057991028,
+      "rewards/margins": 0.10132592916488647,
+      "rewards/rejected": -0.08521126955747604,
+      "step": 14360
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.4275425092290004e-08,
+      "logits/chosen": -2.4178357124328613,
+      "logits/rejected": -2.370293140411377,
+      "logps/chosen": -252.72412109375,
+      "logps/rejected": -242.9341278076172,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.020720353350043297,
+      "rewards/margins": 0.08652675151824951,
+      "rewards/rejected": -0.06580640375614166,
+      "step": 14370
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.309847521527078e-08,
+      "logits/chosen": -2.2703206539154053,
+      "logits/rejected": -1.9016317129135132,
+      "logps/chosen": -283.227294921875,
+      "logps/rejected": -247.84408569335938,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.013460688292980194,
+      "rewards/margins": 0.07545115798711777,
+      "rewards/rejected": -0.06199047714471817,
+      "step": 14380
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.1934289575233385e-08,
+      "logits/chosen": -2.147324323654175,
+      "logits/rejected": -1.8097765445709229,
+      "logps/chosen": -240.4329071044922,
+      "logps/rejected": -215.96237182617188,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.009034966118633747,
+      "rewards/margins": 0.09896841645240784,
+      "rewards/rejected": -0.10800337791442871,
+      "step": 14390
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.078287424513994e-08,
+      "logits/chosen": -2.390627384185791,
+      "logits/rejected": -2.2773404121398926,
+      "logps/chosen": -270.3782653808594,
+      "logps/rejected": -206.30615234375,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.003305424703285098,
+      "rewards/margins": 0.10758145898580551,
+      "rewards/rejected": -0.11088689416646957,
+      "step": 14400
+    },
+    {
+      "epoch": 0.94,
+      "eval_logits/chosen": -2.3247244358062744,
+      "eval_logits/rejected": -2.1361277103424072,
+      "eval_logps/chosen": -230.61825561523438,
+      "eval_logps/rejected": -218.7084503173828,
+      "eval_loss": 0.6897271871566772,
+      "eval_rewards/accuracies": 0.640999972820282,
+      "eval_rewards/chosen": 0.013867066241800785,
+      "eval_rewards/margins": 0.08483249694108963,
+      "eval_rewards/rejected": -0.07096543163061142,
+      "eval_runtime": 710.9987,
+      "eval_samples_per_second": 2.813,
+      "eval_steps_per_second": 1.406,
+      "step": 14400
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 4.964423523133671e-08,
+      "logits/chosen": -2.3970162868499756,
+      "logits/rejected": -2.158749580383301,
+      "logps/chosen": -215.6207275390625,
+      "logps/rejected": -187.32325744628906,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.021107520908117294,
+      "rewards/margins": 0.060516953468322754,
+      "rewards/rejected": -0.03940943256020546,
+      "step": 14410
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 4.8518378473522976e-08,
+      "logits/chosen": -2.262331008911133,
+      "logits/rejected": -2.106858253479004,
+      "logps/chosen": -250.4281768798828,
+      "logps/rejected": -251.893310546875,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.000816689629573375,
+      "rewards/margins": 0.08192334324121475,
+      "rewards/rejected": -0.08110664784908295,
+      "step": 14420
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 4.7405309844718584e-08,
+      "logits/chosen": -2.184021472930908,
+      "logits/rejected": -2.0803000926971436,
+      "logps/chosen": -192.67074584960938,
+      "logps/rejected": -213.4363250732422,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.0017172780353575945,
+      "rewards/margins": 0.11569315195083618,
+      "rewards/rejected": -0.11741043627262115,
+      "step": 14430
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 4.630503515123508e-08,
+      "logits/chosen": -2.426945209503174,
+      "logits/rejected": -2.128527879714966,
+      "logps/chosen": -200.57327270507812,
+      "logps/rejected": -160.0536651611328,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0029270625673234463,
+      "rewards/margins": 0.09346749633550644,
+      "rewards/rejected": -0.09054042398929596,
+      "step": 14440
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.5217560132644056e-08,
+      "logits/chosen": -2.2494208812713623,
+      "logits/rejected": -2.1792380809783936,
+      "logps/chosen": -148.93338012695312,
+      "logps/rejected": -174.1099395751953,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.016862403601408005,
+      "rewards/margins": 0.06775657087564468,
+      "rewards/rejected": -0.05089417099952698,
+      "step": 14450
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.41428904617483e-08,
+      "logits/chosen": -2.2892022132873535,
+      "logits/rejected": -2.2819294929504395,
+      "logps/chosen": -177.44277954101562,
+      "logps/rejected": -189.7292938232422,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.0033408640883862972,
+      "rewards/margins": 0.07625994831323624,
+      "rewards/rejected": -0.07960081100463867,
+      "step": 14460
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.3081031744550696e-08,
+      "logits/chosen": -2.3867905139923096,
+      "logits/rejected": -2.3108649253845215,
+      "logps/chosen": -250.4009552001953,
+      "logps/rejected": -236.97607421875,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.027443695813417435,
+      "rewards/margins": 0.08902832120656967,
+      "rewards/rejected": -0.061584629118442535,
+      "step": 14470
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.2031989520227025e-08,
+      "logits/chosen": -2.3677258491516113,
+      "logits/rejected": -2.1632471084594727,
+      "logps/chosen": -225.92098999023438,
+      "logps/rejected": -210.75198364257812,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.023934394121170044,
+      "rewards/margins": 0.06718473136425018,
+      "rewards/rejected": -0.04325033351778984,
+      "step": 14480
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.099576926109461e-08,
+      "logits/chosen": -2.445075273513794,
+      "logits/rejected": -1.9394609928131104,
+      "logps/chosen": -237.6958465576172,
+      "logps/rejected": -165.7293243408203,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0208599753677845,
+      "rewards/margins": 0.07187938690185547,
+      "rewards/rejected": -0.05101940780878067,
+      "step": 14490
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.997237637258705e-08,
+      "logits/chosen": -2.277336835861206,
+      "logits/rejected": -2.330341339111328,
+      "logps/chosen": -313.8475341796875,
+      "logps/rejected": -279.31109619140625,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.03405776619911194,
+      "rewards/margins": 0.08806699514389038,
+      "rewards/rejected": -0.05400923639535904,
+      "step": 14500
+    },
+    {
+      "epoch": 0.95,
+      "eval_logits/chosen": -2.3249900341033936,
+      "eval_logits/rejected": -2.136406660079956,
+      "eval_logps/chosen": -230.61793518066406,
+      "eval_logps/rejected": -218.70700073242188,
+      "eval_loss": 0.6897269487380981,
+      "eval_rewards/accuracies": 0.6370000243186951,
+      "eval_rewards/chosen": 0.013869978487491608,
+      "eval_rewards/margins": 0.08482073247432709,
+      "eval_rewards/rejected": -0.07095075398683548,
+      "eval_runtime": 710.297,
+      "eval_samples_per_second": 2.816,
+      "eval_steps_per_second": 1.408,
+      "step": 14500
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.8961816193222035e-08,
+      "logits/chosen": -2.4175612926483154,
+      "logits/rejected": -2.200261354446411,
+      "logps/chosen": -237.2295379638672,
+      "logps/rejected": -184.6630096435547,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.013792415149509907,
+      "rewards/margins": 0.04766743257641792,
+      "rewards/rejected": -0.061459846794605255,
+      "step": 14510
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.79640939945769e-08,
+      "logits/chosen": -2.3744473457336426,
+      "logits/rejected": -2.240773916244507,
+      "logps/chosen": -283.0396728515625,
+      "logps/rejected": -200.9557647705078,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.03181237354874611,
+      "rewards/margins": 0.04180184006690979,
+      "rewards/rejected": -0.009989465586841106,
+      "step": 14520
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.697921498125895e-08,
+      "logits/chosen": -2.109978199005127,
+      "logits/rejected": -2.2066032886505127,
+      "logps/chosen": -211.96316528320312,
+      "logps/rejected": -225.83584594726562,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0201013945043087,
+      "rewards/margins": 0.07992889732122421,
+      "rewards/rejected": -0.10003030300140381,
+      "step": 14530
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.6007184290880456e-08,
+      "logits/chosen": -2.3271656036376953,
+      "logits/rejected": -2.227792739868164,
+      "logps/chosen": -211.0209197998047,
+      "logps/rejected": -203.0982666015625,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.009520738385617733,
+      "rewards/margins": 0.08740987628698349,
+      "rewards/rejected": -0.07788912951946259,
+      "step": 14540
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.504800699402872e-08,
+      "logits/chosen": -2.557425022125244,
+      "logits/rejected": -2.2593464851379395,
+      "logps/chosen": -342.3351135253906,
+      "logps/rejected": -273.93585205078125,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.03319120034575462,
+      "rewards/margins": 0.04967629909515381,
+      "rewards/rejected": -0.016485098749399185,
+      "step": 14550
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.4101688094242967e-08,
+      "logits/chosen": -2.264317035675049,
+      "logits/rejected": -2.1588878631591797,
+      "logps/chosen": -270.6173095703125,
+      "logps/rejected": -257.766357421875,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.010563389398157597,
+      "rewards/margins": 0.12325240671634674,
+      "rewards/rejected": -0.13381578028202057,
+      "step": 14560
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.3168232527985564e-08,
+      "logits/chosen": -2.2050108909606934,
+      "logits/rejected": -1.9072158336639404,
+      "logps/chosen": -241.6295166015625,
+      "logps/rejected": -194.75392150878906,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.00010975040640914813,
+      "rewards/margins": 0.07874272018671036,
+      "rewards/rejected": -0.07885247468948364,
+      "step": 14570
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.224764516461892e-08,
+      "logits/chosen": -2.3158745765686035,
+      "logits/rejected": -2.1368792057037354,
+      "logps/chosen": -252.09890747070312,
+      "logps/rejected": -232.61062622070312,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.028999557718634605,
+      "rewards/margins": 0.1126040369272232,
+      "rewards/rejected": -0.08360447734594345,
+      "step": 14580
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.133993080637665e-08,
+      "logits/chosen": -2.2999939918518066,
+      "logits/rejected": -2.108703374862671,
+      "logps/chosen": -201.35792541503906,
+      "logps/rejected": -199.75430297851562,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.015230001881718636,
+      "rewards/margins": 0.10807321965694427,
+      "rewards/rejected": -0.09284321218729019,
+      "step": 14590
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.0445094188342186e-08,
+      "logits/chosen": -2.112121343612671,
+      "logits/rejected": -1.8166097402572632,
+      "logps/chosen": -253.4682159423828,
+      "logps/rejected": -182.1717987060547,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 8.347779657924548e-05,
+      "rewards/margins": 0.08967778086662292,
+      "rewards/rejected": -0.0895942971110344,
+      "step": 14600
+    },
+    {
+      "epoch": 0.96,
+      "eval_logits/chosen": -2.3248980045318604,
+      "eval_logits/rejected": -2.1363399028778076,
+      "eval_logps/chosen": -230.62680053710938,
+      "eval_logps/rejected": -218.67767333984375,
+      "eval_loss": 0.6897242665290833,
+      "eval_rewards/accuracies": 0.6355000138282776,
+      "eval_rewards/chosen": 0.013781617395579815,
+      "eval_rewards/margins": 0.08443937450647354,
+      "eval_rewards/rejected": -0.07065775245428085,
+      "eval_runtime": 710.5131,
+      "eval_samples_per_second": 2.815,
+      "eval_steps_per_second": 1.407,
+      "step": 14600
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.9563139978421028e-08,
+      "logits/chosen": -2.2371926307678223,
+      "logits/rejected": -2.259657382965088,
+      "logps/chosen": -223.68856811523438,
+      "logps/rejected": -221.4380340576172,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.021833080798387527,
+      "rewards/margins": 0.05533973500132561,
+      "rewards/rejected": -0.03350665792822838,
+      "step": 14610
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.869407277731939e-08,
+      "logits/chosen": -2.2002997398376465,
+      "logits/rejected": -2.1275038719177246,
+      "logps/chosen": -183.22903442382812,
+      "logps/rejected": -168.91224670410156,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.02028103545308113,
+      "rewards/margins": 0.08852804452180862,
+      "rewards/rejected": -0.06824701279401779,
+      "step": 14620
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.783789711851642e-08,
+      "logits/chosen": -2.3309357166290283,
+      "logits/rejected": -2.1030194759368896,
+      "logps/chosen": -157.8726043701172,
+      "logps/rejected": -147.86341857910156,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.03978399187326431,
+      "rewards/margins": 0.12423492968082428,
+      "rewards/rejected": -0.08445094525814056,
+      "step": 14630
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.6994617468244778e-08,
+      "logits/chosen": -2.4272868633270264,
+      "logits/rejected": -1.9481617212295532,
+      "logps/chosen": -210.47348022460938,
+      "logps/rejected": -158.01644897460938,
+      "loss": 0.689,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.02547174133360386,
+      "rewards/margins": 0.11461669206619263,
+      "rewards/rejected": -0.08914494514465332,
+      "step": 14640
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.6164238225463155e-08,
+      "logits/chosen": -2.264310598373413,
+      "logits/rejected": -1.9083404541015625,
+      "logps/chosen": -281.5527648925781,
+      "logps/rejected": -212.22207641601562,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0087438328191638,
+      "rewards/margins": 0.09177269041538239,
+      "rewards/rejected": -0.08302884548902512,
+      "step": 14650
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.534676372183742e-08,
+      "logits/chosen": -2.288485527038574,
+      "logits/rejected": -2.1355769634246826,
+      "logps/chosen": -282.95892333984375,
+      "logps/rejected": -237.60372924804688,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.009953884407877922,
+      "rewards/margins": 0.06971423327922821,
+      "rewards/rejected": -0.05976034328341484,
+      "step": 14660
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.4542198221714218e-08,
+      "logits/chosen": -2.1728930473327637,
+      "logits/rejected": -1.9003547430038452,
+      "logps/chosen": -137.03646850585938,
+      "logps/rejected": -146.15728759765625,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.010704811662435532,
+      "rewards/margins": 0.08893907815217972,
+      "rewards/rejected": -0.07823427021503448,
+      "step": 14670
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.3750545922101854e-08,
+      "logits/chosen": -2.5925240516662598,
+      "logits/rejected": -2.207447052001953,
+      "logps/chosen": -308.4715270996094,
+      "logps/rejected": -248.4219207763672,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.009196789935231209,
+      "rewards/margins": 0.08344466239213943,
+      "rewards/rejected": -0.07424787431955338,
+      "step": 14680
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2971810952646112e-08,
+      "logits/chosen": -2.3056700229644775,
+      "logits/rejected": -2.2151083946228027,
+      "logps/chosen": -256.753662109375,
+      "logps/rejected": -214.6802215576172,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.01609097793698311,
+      "rewards/margins": 0.056323904544115067,
+      "rewards/rejected": -0.04023292288184166,
+      "step": 14690
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2205997375610576e-08,
+      "logits/chosen": -2.1445729732513428,
+      "logits/rejected": -2.0863611698150635,
+      "logps/chosen": -181.83251953125,
+      "logps/rejected": -196.3846435546875,
+      "loss": 0.691,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.03673393279314041,
+      "rewards/margins": 0.09392055124044418,
+      "rewards/rejected": -0.057186610996723175,
+      "step": 14700
+    },
+    {
+      "epoch": 0.96,
+      "eval_logits/chosen": -2.324772834777832,
+      "eval_logits/rejected": -2.1362075805664062,
+      "eval_logps/chosen": -230.6251678466797,
+      "eval_logps/rejected": -218.66000366210938,
+      "eval_loss": 0.6897311806678772,
+      "eval_rewards/accuracies": 0.6365000009536743,
+      "eval_rewards/chosen": 0.013797725550830364,
+      "eval_rewards/margins": 0.08427882194519043,
+      "eval_rewards/rejected": -0.07048109173774719,
+      "eval_runtime": 709.9702,
+      "eval_samples_per_second": 2.817,
+      "eval_steps_per_second": 1.409,
+      "step": 14700
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1453109185853304e-08,
+      "logits/chosen": -2.3570213317871094,
+      "logits/rejected": -2.299567461013794,
+      "logps/chosen": -199.55894470214844,
+      "logps/rejected": -209.33212280273438,
+      "loss": 0.689,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.02519531175494194,
+      "rewards/margins": 0.08025936782360077,
+      "rewards/rejected": -0.05506405234336853,
+      "step": 14710
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0713150310808784e-08,
+      "logits/chosen": -2.121694564819336,
+      "logits/rejected": -2.446938991546631,
+      "logps/chosen": -226.5384979248047,
+      "logps/rejected": -235.46133422851562,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.010506866499781609,
+      "rewards/margins": 0.031594760715961456,
+      "rewards/rejected": -0.042101629078388214,
+      "step": 14720
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9986124610464064e-08,
+      "logits/chosen": -2.2158148288726807,
+      "logits/rejected": -1.9099407196044922,
+      "logps/chosen": -285.78350830078125,
+      "logps/rejected": -228.42324829101562,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.014465957880020142,
+      "rewards/margins": 0.12452026456594467,
+      "rewards/rejected": -0.11005431413650513,
+      "step": 14730
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.927203587734211e-08,
+      "logits/chosen": -2.19920015335083,
+      "logits/rejected": -1.8051159381866455,
+      "logps/chosen": -250.2556915283203,
+      "logps/rejected": -210.61441040039062,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.027360107749700546,
+      "rewards/margins": 0.08885184675455093,
+      "rewards/rejected": -0.06149173900485039,
+      "step": 14740
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.8570887836479034e-08,
+      "logits/chosen": -2.3139779567718506,
+      "logits/rejected": -2.1160130500793457,
+      "logps/chosen": -195.26492309570312,
+      "logps/rejected": -255.01797485351562,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.016417790204286575,
+      "rewards/margins": 0.06937674432992935,
+      "rewards/rejected": -0.08579452335834503,
+      "step": 14750
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7882684145406616e-08,
+      "logits/chosen": -2.3864521980285645,
+      "logits/rejected": -2.3232614994049072,
+      "logps/chosen": -292.4091796875,
+      "logps/rejected": -298.75286865234375,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.04011436551809311,
+      "rewards/margins": 0.0823252946138382,
+      "rewards/rejected": -0.04221092164516449,
+      "step": 14760
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7207428394132865e-08,
+      "logits/chosen": -2.5386404991149902,
+      "logits/rejected": -2.052577018737793,
+      "logps/chosen": -268.1858825683594,
+      "logps/rejected": -225.7917938232422,
+      "loss": 0.6873,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.01594383455812931,
+      "rewards/margins": 0.12293130159378052,
+      "rewards/rejected": -0.10698747634887695,
+      "step": 14770
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.654512410512177e-08,
+      "logits/chosen": -2.2604587078094482,
+      "logits/rejected": -2.0169408321380615,
+      "logps/chosen": -253.12109375,
+      "logps/rejected": -196.11129760742188,
+      "loss": 0.692,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.004558461718261242,
+      "rewards/margins": 0.0537867471575737,
+      "rewards/rejected": -0.04922827333211899,
+      "step": 14780
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5895774733277468e-08,
+      "logits/chosen": -2.3233249187469482,
+      "logits/rejected": -1.9812166690826416,
+      "logps/chosen": -277.13775634765625,
+      "logps/rejected": -236.5921630859375,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.026791805401444435,
+      "rewards/margins": 0.09690927714109421,
+      "rewards/rejected": -0.07011748105287552,
+      "step": 14790
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5259383665924e-08,
+      "logits/chosen": -2.584911823272705,
+      "logits/rejected": -2.1934475898742676,
+      "logps/chosen": -337.8364562988281,
+      "logps/rejected": -252.3568572998047,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.042554210871458054,
+      "rewards/margins": 0.08529296517372131,
+      "rewards/rejected": -0.04273875802755356,
+      "step": 14800
+    },
+    {
+      "epoch": 0.97,
+      "eval_logits/chosen": -2.324951171875,
+      "eval_logits/rejected": -2.136404037475586,
+      "eval_logps/chosen": -230.61355590820312,
+      "eval_logps/rejected": -218.665283203125,
+      "eval_loss": 0.6897242665290833,
+      "eval_rewards/accuracies": 0.6340000033378601,
+      "eval_rewards/chosen": 0.013913972303271294,
+      "eval_rewards/margins": 0.08444766700267792,
+      "eval_rewards/rejected": -0.07053370773792267,
+      "eval_runtime": 710.025,
+      "eval_samples_per_second": 2.817,
+      "eval_steps_per_second": 1.408,
+      "step": 14800
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4635954222789461e-08,
+      "logits/chosen": -2.2716784477233887,
+      "logits/rejected": -2.1852469444274902,
+      "logps/chosen": -214.40774536132812,
+      "logps/rejected": -227.9254150390625,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.026998598128557205,
+      "rewards/margins": 0.08409784734249115,
+      "rewards/rejected": -0.05709924176335335,
+      "step": 14810
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.402548965598688e-08,
+      "logits/chosen": -2.210674285888672,
+      "logits/rejected": -2.305459499359131,
+      "logps/chosen": -202.4854736328125,
+      "logps/rejected": -205.67385864257812,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.0214177705347538,
+      "rewards/margins": 0.06404820829629898,
+      "rewards/rejected": -0.04263044521212578,
+      "step": 14820
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3427993149998375e-08,
+      "logits/chosen": -2.4624884128570557,
+      "logits/rejected": -2.211487293243408,
+      "logps/chosen": -238.8626251220703,
+      "logps/rejected": -194.0426483154297,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.03174557164311409,
+      "rewards/margins": 0.10550177097320557,
+      "rewards/rejected": -0.07375619560480118,
+      "step": 14830
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.2843467821658518e-08,
+      "logits/chosen": -2.4166269302368164,
+      "logits/rejected": -2.350491523742676,
+      "logps/chosen": -222.0076141357422,
+      "logps/rejected": -231.8187713623047,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.04215265437960625,
+      "rewards/margins": 0.1017962247133255,
+      "rewards/rejected": -0.05964357778429985,
+      "step": 14840
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.2271916720137666e-08,
+      "logits/chosen": -2.52500581741333,
+      "logits/rejected": -2.208137035369873,
+      "logps/chosen": -285.6896057128906,
+      "logps/rejected": -245.7957000732422,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0007992383325472474,
+      "rewards/margins": 0.06241076439619064,
+      "rewards/rejected": -0.06161152571439743,
+      "step": 14850
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.171334282692671e-08,
+      "logits/chosen": -2.3669638633728027,
+      "logits/rejected": -2.2914958000183105,
+      "logps/chosen": -280.2817077636719,
+      "logps/rejected": -260.38970947265625,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.023167315870523453,
+      "rewards/margins": 0.10739920288324356,
+      "rewards/rejected": -0.084231898188591,
+      "step": 14860
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.116774905582041e-08,
+      "logits/chosen": -2.4192698001861572,
+      "logits/rejected": -2.087759017944336,
+      "logps/chosen": -183.83871459960938,
+      "logps/rejected": -177.45944213867188,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.00979800708591938,
+      "rewards/margins": 0.06323973834514618,
+      "rewards/rejected": -0.05344173312187195,
+      "step": 14870
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.0635138252902966e-08,
+      "logits/chosen": -2.504063129425049,
+      "logits/rejected": -2.2471530437469482,
+      "logps/chosen": -229.2535400390625,
+      "logps/rejected": -215.491455078125,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.011604288592934608,
+      "rewards/margins": 0.08366361260414124,
+      "rewards/rejected": -0.07205932587385178,
+      "step": 14880
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.0115513196533589e-08,
+      "logits/chosen": -2.338160991668701,
+      "logits/rejected": -2.171861410140991,
+      "logps/chosen": -261.0177001953125,
+      "logps/rejected": -249.5078125,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.015924451872706413,
+      "rewards/margins": 0.05798298865556717,
+      "rewards/rejected": -0.04205853492021561,
+      "step": 14890
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 9.608876597330952e-09,
+      "logits/chosen": -2.3476691246032715,
+      "logits/rejected": -2.022096633911133,
+      "logps/chosen": -288.65777587890625,
+      "logps/rejected": -285.6450500488281,
+      "loss": 0.6892,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.020350560545921326,
+      "rewards/margins": 0.08672511577606201,
+      "rewards/rejected": -0.10707566887140274,
+      "step": 14900
+    },
+    {
+      "epoch": 0.97,
+      "eval_logits/chosen": -2.3250153064727783,
+      "eval_logits/rejected": -2.1365039348602295,
+      "eval_logps/chosen": -230.6240997314453,
+      "eval_logps/rejected": -218.6448974609375,
+      "eval_loss": 0.6897345185279846,
+      "eval_rewards/accuracies": 0.6380000114440918,
+      "eval_rewards/chosen": 0.013808542862534523,
+      "eval_rewards/margins": 0.08413854986429214,
+      "eval_rewards/rejected": -0.07033000141382217,
+      "eval_runtime": 709.7106,
+      "eval_samples_per_second": 2.818,
+      "eval_steps_per_second": 1.409,
+      "step": 14900
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.115231098159594e-09,
+      "logits/chosen": -2.4037442207336426,
+      "logits/rejected": -2.292450189590454,
+      "logps/chosen": -251.7987518310547,
+      "logps/rejected": -238.9403076171875,
+      "loss": 0.691,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.015149926766753197,
+      "rewards/margins": 0.074435293674469,
+      "rewards/rejected": -0.05928536504507065,
+      "step": 14910
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.634579274116317e-09,
+      "logits/chosen": -2.260741710662842,
+      "logits/rejected": -2.1925673484802246,
+      "logps/chosen": -188.34481811523438,
+      "logps/rejected": -219.61099243164062,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.01714429259300232,
+      "rewards/margins": 0.10656633228063583,
+      "rewards/rejected": -0.08942203223705292,
+      "step": 14920
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.166923632516865e-09,
+      "logits/chosen": -2.439234733581543,
+      "logits/rejected": -2.150946617126465,
+      "logps/chosen": -225.3989715576172,
+      "logps/rejected": -278.0585021972656,
+      "loss": 0.6849,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.02332211658358574,
+      "rewards/margins": 0.1503240466117859,
+      "rewards/rejected": -0.12700191140174866,
+      "step": 14930
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 7.712266612881492e-09,
+      "logits/chosen": -2.1845946311950684,
+      "logits/rejected": -2.0539097785949707,
+      "logps/chosen": -181.65396118164062,
+      "logps/rejected": -185.66207885742188,
+      "loss": 0.69,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0358271598815918,
+      "rewards/margins": 0.088839091360569,
+      "rewards/rejected": -0.0530119314789772,
+      "step": 14940
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 7.270610586924687e-09,
+      "logits/chosen": -2.450291872024536,
+      "logits/rejected": -2.2107715606689453,
+      "logps/chosen": -260.78521728515625,
+      "logps/rejected": -224.33633422851562,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.041800957173109055,
+      "rewards/margins": 0.07631386816501617,
+      "rewards/rejected": -0.03451291099190712,
+      "step": 14950
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 6.841957858539916e-09,
+      "logits/chosen": -2.2664036750793457,
+      "logits/rejected": -2.1220388412475586,
+      "logps/chosen": -168.59519958496094,
+      "logps/rejected": -185.31361389160156,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.019396457821130753,
+      "rewards/margins": 0.05867626518011093,
+      "rewards/rejected": -0.07807272672653198,
+      "step": 14960
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 6.426310663790181e-09,
+      "logits/chosen": -2.1963348388671875,
+      "logits/rejected": -2.1024935245513916,
+      "logps/chosen": -236.64208984375,
+      "logps/rejected": -211.93154907226562,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0033364570699632168,
+      "rewards/margins": 0.06757830083370209,
+      "rewards/rejected": -0.06424184143543243,
+      "step": 14970
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 6.023671170894696e-09,
+      "logits/chosen": -2.5223042964935303,
+      "logits/rejected": -1.931014060974121,
+      "logps/chosen": -288.4738464355469,
+      "logps/rejected": -213.46670532226562,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.03854692354798317,
+      "rewards/margins": 0.11772453784942627,
+      "rewards/rejected": -0.0791776180267334,
+      "step": 14980
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 5.634041480218344e-09,
+      "logits/chosen": -2.4757115840911865,
+      "logits/rejected": -2.137505054473877,
+      "logps/chosen": -250.7669677734375,
+      "logps/rejected": -253.52487182617188,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.023066259920597076,
+      "rewards/margins": 0.07198301702737808,
+      "rewards/rejected": -0.0489167645573616,
+      "step": 14990
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 5.257423624260849e-09,
+      "logits/chosen": -2.520918369293213,
+      "logits/rejected": -2.0873606204986572,
+      "logps/chosen": -260.77056884765625,
+      "logps/rejected": -224.59854125976562,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.025528425350785255,
+      "rewards/margins": 0.07071218639612198,
+      "rewards/rejected": -0.045183759182691574,
+      "step": 15000
+    },
+    {
+      "epoch": 0.98,
+      "eval_logits/chosen": -2.325486660003662,
+      "eval_logits/rejected": -2.136941432952881,
+      "eval_logps/chosen": -230.58958435058594,
+      "eval_logps/rejected": -218.62281799316406,
+      "eval_loss": 0.6897296905517578,
+      "eval_rewards/accuracies": 0.6384999752044678,
+      "eval_rewards/chosen": 0.014153635129332542,
+      "eval_rewards/margins": 0.08426273614168167,
+      "eval_rewards/rejected": -0.07010909914970398,
+      "eval_runtime": 709.7214,
+      "eval_samples_per_second": 2.818,
+      "eval_steps_per_second": 1.409,
+      "step": 15000
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 4.893819567644564e-09,
+      "logits/chosen": -2.2012670040130615,
+      "logits/rejected": -2.1785387992858887,
+      "logps/chosen": -188.42910766601562,
+      "logps/rejected": -204.0313720703125,
+      "loss": 0.6902,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.004323553293943405,
+      "rewards/margins": 0.054262660443782806,
+      "rewards/rejected": -0.05858622118830681,
+      "step": 15010
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 4.543231207107257e-09,
+      "logits/chosen": -2.2762811183929443,
+      "logits/rejected": -2.059688091278076,
+      "logps/chosen": -257.8048400878906,
+      "logps/rejected": -240.6076202392578,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.00016754865646362305,
+      "rewards/margins": 0.0641152560710907,
+      "rewards/rejected": -0.06394769996404648,
+      "step": 15020
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 4.205660371488785e-09,
+      "logits/chosen": -2.5772321224212646,
+      "logits/rejected": -2.2152597904205322,
+      "logps/chosen": -282.9827575683594,
+      "logps/rejected": -245.66513061523438,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.011705792509019375,
+      "rewards/margins": 0.04472345858812332,
+      "rewards/rejected": -0.03301766887307167,
+      "step": 15030
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 3.88110882172471e-09,
+      "logits/chosen": -2.2456133365631104,
+      "logits/rejected": -2.1853480339050293,
+      "logps/chosen": -220.9130859375,
+      "logps/rejected": -221.7021026611328,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.0021468736231327057,
+      "rewards/margins": 0.05425567179918289,
+      "rewards/rejected": -0.0564025342464447,
+      "step": 15040
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 3.569578250834371e-09,
+      "logits/chosen": -2.3457345962524414,
+      "logits/rejected": -2.0774590969085693,
+      "logps/chosen": -298.05279541015625,
+      "logps/rejected": -271.5147705078125,
+      "loss": 0.6879,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.03925010561943054,
+      "rewards/margins": 0.1122470498085022,
+      "rewards/rejected": -0.07299693673849106,
+      "step": 15050
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.2710702839139353e-09,
+      "logits/chosen": -2.373473644256592,
+      "logits/rejected": -2.229529857635498,
+      "logps/chosen": -202.07412719726562,
+      "logps/rejected": -216.16708374023438,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.025251392275094986,
+      "rewards/margins": 0.0481976643204689,
+      "rewards/rejected": -0.022946273908019066,
+      "step": 15060
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.9855864781272448e-09,
+      "logits/chosen": -2.3376505374908447,
+      "logits/rejected": -2.3747403621673584,
+      "logps/chosen": -209.34140014648438,
+      "logps/rejected": -250.8092498779297,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.023322973400354385,
+      "rewards/margins": 0.06673813611268997,
+      "rewards/rejected": -0.04341515898704529,
+      "step": 15070
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.7131283226977665e-09,
+      "logits/chosen": -2.3406639099121094,
+      "logits/rejected": -2.4078879356384277,
+      "logps/chosen": -217.74288940429688,
+      "logps/rejected": -240.52072143554688,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.017053885385394096,
+      "rewards/margins": 0.08927709609270096,
+      "rewards/rejected": -0.07222320139408112,
+      "step": 15080
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.4536972389008205e-09,
+      "logits/chosen": -2.2868685722351074,
+      "logits/rejected": -2.058229446411133,
+      "logps/chosen": -233.2720489501953,
+      "logps/rejected": -207.40798950195312,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.038629334419965744,
+      "rewards/margins": 0.118269182741642,
+      "rewards/rejected": -0.07963985949754715,
+      "step": 15090
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.20729458005553e-09,
+      "logits/chosen": -2.201646089553833,
+      "logits/rejected": -2.040139675140381,
+      "logps/chosen": -191.54600524902344,
+      "logps/rejected": -181.17190551757812,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.028632348403334618,
+      "rewards/margins": 0.11827573925256729,
+      "rewards/rejected": -0.08964338153600693,
+      "step": 15100
+    },
+    {
+      "epoch": 0.99,
+      "eval_logits/chosen": -2.3254594802856445,
+      "eval_logits/rejected": -2.1368985176086426,
+      "eval_logps/chosen": -230.5937042236328,
+      "eval_logps/rejected": -218.6256866455078,
+      "eval_loss": 0.6897294521331787,
+      "eval_rewards/accuracies": 0.6389999985694885,
+      "eval_rewards/chosen": 0.014112350530922413,
+      "eval_rewards/margins": 0.08425014466047287,
+      "eval_rewards/rejected": -0.07013778388500214,
+      "eval_runtime": 710.0809,
+      "eval_samples_per_second": 2.817,
+      "eval_steps_per_second": 1.408,
+      "step": 15100
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.9739216315192712e-09,
+      "logits/chosen": -2.3063950538635254,
+      "logits/rejected": -2.096644163131714,
+      "logps/chosen": -227.7020263671875,
+      "logps/rejected": -210.0640106201172,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.016067935153841972,
+      "rewards/margins": 0.06059398502111435,
+      "rewards/rejected": -0.044526055455207825,
+      "step": 15110
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.7535796106796231e-09,
+      "logits/chosen": -2.3938944339752197,
+      "logits/rejected": -2.072892665863037,
+      "logps/chosen": -276.0992431640625,
+      "logps/rejected": -202.15260314941406,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.005628110375255346,
+      "rewards/margins": 0.06384317576885223,
+      "rewards/rejected": -0.06947128474712372,
+      "step": 15120
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.5462696669482636e-09,
+      "logits/chosen": -2.359903335571289,
+      "logits/rejected": -2.2396528720855713,
+      "logps/chosen": -219.18911743164062,
+      "logps/rejected": -233.8794403076172,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.020530302077531815,
+      "rewards/margins": 0.07046308368444443,
+      "rewards/rejected": -0.04993278905749321,
+      "step": 15130
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.3519928817556927e-09,
+      "logits/chosen": -2.21510648727417,
+      "logits/rejected": -2.1697449684143066,
+      "logps/chosen": -171.1583251953125,
+      "logps/rejected": -178.51443481445312,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.02235172688961029,
+      "rewards/margins": 0.0680917277932167,
+      "rewards/rejected": -0.045739997178316116,
+      "step": 15140
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.1707502685448512e-09,
+      "logits/chosen": -2.445695638656616,
+      "logits/rejected": -2.1078381538391113,
+      "logps/chosen": -222.18508911132812,
+      "logps/rejected": -176.2418975830078,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0008492677588947117,
+      "rewards/margins": 0.12245283275842667,
+      "rewards/rejected": -0.12160356342792511,
+      "step": 15150
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.002542772765569e-09,
+      "logits/chosen": -2.2757375240325928,
+      "logits/rejected": -1.9920152425765991,
+      "logps/chosen": -187.7380828857422,
+      "logps/rejected": -158.5767059326172,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.008657276630401611,
+      "rewards/margins": 0.09146241843700409,
+      "rewards/rejected": -0.08280514925718307,
+      "step": 15160
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 8.473712718709559e-10,
+      "logits/chosen": -2.1393208503723145,
+      "logits/rejected": -2.1515212059020996,
+      "logps/chosen": -193.9955596923828,
+      "logps/rejected": -187.0467987060547,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.002506362274289131,
+      "rewards/margins": 0.03829359635710716,
+      "rewards/rejected": -0.040799956768751144,
+      "step": 15170
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.052365753112966e-10,
+      "logits/chosen": -2.1680989265441895,
+      "logits/rejected": -1.8749319314956665,
+      "logps/chosen": -234.80142211914062,
+      "logps/rejected": -226.61306762695312,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.0051465281285345554,
+      "rewards/margins": 0.10587289184331894,
+      "rewards/rejected": -0.10072635114192963,
+      "step": 15180
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.761394245307195e-10,
+      "logits/chosen": -2.155925989151001,
+      "logits/rejected": -2.18461275100708,
+      "logps/chosen": -235.2099609375,
+      "logps/rejected": -238.4646759033203,
+      "loss": 0.692,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.018722042441368103,
+      "rewards/margins": 0.04466164484620094,
+      "rewards/rejected": -0.02593959867954254,
+      "step": 15190
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.6008049296358826e-10,
+      "logits/chosen": -2.229979991912842,
+      "logits/rejected": -2.126420736312866,
+      "logps/chosen": -185.88441467285156,
+      "logps/rejected": -165.7515411376953,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0013815786223858595,
+      "rewards/margins": 0.08110791444778442,
+      "rewards/rejected": -0.07972635328769684,
+      "step": 15200
+    },
+    {
+      "epoch": 0.99,
+      "eval_logits/chosen": -2.325113296508789,
+      "eval_logits/rejected": -2.1365654468536377,
+      "eval_logps/chosen": -230.59994506835938,
+      "eval_logps/rejected": -218.62445068359375,
+      "eval_loss": 0.6897304654121399,
+      "eval_rewards/accuracies": 0.6365000009536743,
+      "eval_rewards/chosen": 0.014050180092453957,
+      "eval_rewards/margins": 0.08417567610740662,
+      "eval_rewards/rejected": -0.07012549042701721,
+      "eval_runtime": 709.6375,
+      "eval_samples_per_second": 2.818,
+      "eval_steps_per_second": 1.409,
+      "step": 15200
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.5706038603006146e-10,
+      "logits/chosen": -2.4180667400360107,
+      "logits/rejected": -2.3709511756896973,
+      "logps/chosen": -284.6794128417969,
+      "logps/rejected": -279.5739440917969,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.04930936545133591,
+      "rewards/margins": 0.09137637913227081,
+      "rewards/rejected": -0.04206700250506401,
+      "step": 15210
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.670796411333165e-10,
+      "logits/chosen": -2.588311195373535,
+      "logits/rejected": -2.297461986541748,
+      "logps/chosen": -219.0515594482422,
+      "logps/rejected": -215.50772094726562,
+      "loss": 0.6888,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.00759897381067276,
+      "rewards/margins": 0.08762288093566895,
+      "rewards/rejected": -0.08002390712499619,
+      "step": 15220
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.9013872765677455e-10,
+      "logits/chosen": -2.3167214393615723,
+      "logits/rejected": -2.108008861541748,
+      "logps/chosen": -216.2996368408203,
+      "logps/rejected": -207.8689422607422,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.03213484585285187,
+      "rewards/margins": 0.04103900119662285,
+      "rewards/rejected": -0.008904160931706429,
+      "step": 15230
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.262380469624347e-10,
+      "logits/chosen": -2.2934911251068115,
+      "logits/rejected": -2.121281862258911,
+      "logps/chosen": -202.36239624023438,
+      "logps/rejected": -185.4467010498047,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.01644532009959221,
+      "rewards/margins": 0.05994013696908951,
+      "rewards/rejected": -0.0434948205947876,
+      "step": 15240
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 7.53779323872661e-11,
+      "logits/chosen": -2.1865756511688232,
+      "logits/rejected": -2.289961576461792,
+      "logps/chosen": -190.6034698486328,
+      "logps/rejected": -205.8804931640625,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.012599390931427479,
+      "rewards/margins": 0.09233128279447556,
+      "rewards/rejected": -0.07973189651966095,
+      "step": 15250
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.7558649242652734e-11,
+      "logits/chosen": -2.4936611652374268,
+      "logits/rejected": -2.2574238777160645,
+      "logps/chosen": -395.0047912597656,
+      "logps/rejected": -322.8507995605469,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.016790423542261124,
+      "rewards/margins": 0.07416915148496628,
+      "rewards/rejected": -0.05737873911857605,
+      "step": 15260
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.2780394812450526e-11,
+      "logits/chosen": -2.1327195167541504,
+      "logits/rejected": -2.085716724395752,
+      "logps/chosen": -233.3424072265625,
+      "logps/rejected": -242.1055145263672,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.011986208148300648,
+      "rewards/margins": 0.08845750242471695,
+      "rewards/rejected": -0.10044372081756592,
+      "step": 15270
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.0432983521546646e-12,
+      "logits/chosen": -2.170581817626953,
+      "logits/rejected": -2.0781712532043457,
+      "logps/chosen": -182.6875,
+      "logps/rejected": -211.4102325439453,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.007787027861922979,
+      "rewards/margins": 0.09877107292413712,
+      "rewards/rejected": -0.0909840390086174,
+      "step": 15280
+    },
+    {
+      "epoch": 1.0,
+      "step": 15284,
+      "total_flos": 0.0,
+      "train_loss": 0.6900739747015976,
+      "train_runtime": 171639.7836,
+      "train_samples_per_second": 0.356,
+      "train_steps_per_second": 0.089
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 15284,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}