diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,10110 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 7.952662721893491,
+  "eval_steps": 1,
+  "global_step": 672,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.011834319526627219,
+      "grad_norm": 55.38106439710106,
+      "learning_rate": 7.352941176470588e-09,
+      "logits/chosen": -0.587167501449585,
+      "logits/rejected": -0.6672874093055725,
+      "logps/chosen": -39.686065673828125,
+      "logps/rejected": -46.94537353515625,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.023668639053254437,
+      "grad_norm": 54.65079993565704,
+      "learning_rate": 1.4705882352941176e-08,
+      "logits/chosen": -0.3381628394126892,
+      "logits/rejected": -0.2981947958469391,
+      "logps/chosen": -38.55506134033203,
+      "logps/rejected": -47.09852600097656,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 2
+    },
+    {
+      "epoch": 0.03550295857988166,
+      "grad_norm": 54.205478847015584,
+      "learning_rate": 2.2058823529411764e-08,
+      "logits/chosen": -0.47191303968429565,
+      "logits/rejected": -0.5924246311187744,
+      "logps/chosen": -36.32940673828125,
+      "logps/rejected": -37.75663375854492,
+      "loss": 0.6975,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.004663002677261829,
+      "rewards/margins": 0.020077597349882126,
+      "rewards/rejected": -0.02474059723317623,
+      "step": 3
+    },
+    {
+      "epoch": 0.047337278106508875,
+      "grad_norm": 53.90641855823955,
+      "learning_rate": 2.941176470588235e-08,
+      "logits/chosen": -0.715237021446228,
+      "logits/rejected": -0.8035542964935303,
+      "logps/chosen": -39.215999603271484,
+      "logps/rejected": -47.370750427246094,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.01616680435836315,
+      "rewards/margins": 0.024854015558958054,
+      "rewards/rejected": -0.04102082550525665,
+      "step": 4
+    },
+    {
+      "epoch": 0.05917159763313609,
+      "grad_norm": 52.25558555929606,
+      "learning_rate": 3.676470588235294e-08,
+      "logits/chosen": -0.8661510944366455,
+      "logits/rejected": -0.7564424276351929,
+      "logps/chosen": -46.795005798339844,
+      "logps/rejected": -44.85298538208008,
+      "loss": 0.6964,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.021712014451622963,
+      "rewards/margins": -0.05135791748762131,
+      "rewards/rejected": 0.0296458937227726,
+      "step": 5
+    },
+    {
+      "epoch": 0.07100591715976332,
+      "grad_norm": 61.52227993189212,
+      "learning_rate": 4.411764705882353e-08,
+      "logits/chosen": -0.6120268106460571,
+      "logits/rejected": -0.5849899053573608,
+      "logps/chosen": -38.418251037597656,
+      "logps/rejected": -42.02568054199219,
+      "loss": 0.7025,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.042597055435180664,
+      "rewards/margins": -0.009472893550992012,
+      "rewards/rejected": -0.0331241637468338,
+      "step": 6
+    },
+    {
+      "epoch": 0.08284023668639054,
+      "grad_norm": 54.74581505857575,
+      "learning_rate": 5.147058823529411e-08,
+      "logits/chosen": -0.382029265165329,
+      "logits/rejected": -0.3890838623046875,
+      "logps/chosen": -38.0916633605957,
+      "logps/rejected": -48.64350509643555,
+      "loss": 0.6941,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0039642686024308205,
+      "rewards/margins": 0.00704039353877306,
+      "rewards/rejected": -0.01100466400384903,
+      "step": 7
+    },
+    {
+      "epoch": 0.09467455621301775,
+      "grad_norm": 55.737912792059,
+      "learning_rate": 5.88235294117647e-08,
+      "logits/chosen": -0.4821730852127075,
+      "logits/rejected": -0.497173935174942,
+      "logps/chosen": -41.562705993652344,
+      "logps/rejected": -36.803367614746094,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.02364335022866726,
+      "rewards/margins": -0.013885259628295898,
+      "rewards/rejected": -0.009758088737726212,
+      "step": 8
+    },
+    {
+      "epoch": 0.10650887573964497,
+      "grad_norm": 57.095682285700036,
+      "learning_rate": 6.617647058823529e-08,
+      "logits/chosen": -0.3777148723602295,
+      "logits/rejected": -0.5628172755241394,
+      "logps/chosen": -39.834068298339844,
+      "logps/rejected": -40.3427734375,
+      "loss": 0.6969,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.025170041248202324,
+      "rewards/margins": -0.03208901360630989,
+      "rewards/rejected": 0.006918976083397865,
+      "step": 9
+    },
+    {
+      "epoch": 0.11834319526627218,
+      "grad_norm": 54.157479939562684,
+      "learning_rate": 7.352941176470588e-08,
+      "logits/chosen": -0.7878235578536987,
+      "logits/rejected": -0.9192472696304321,
+      "logps/chosen": -42.91743469238281,
+      "logps/rejected": -41.28839874267578,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.03524742275476456,
+      "rewards/margins": 0.016644442453980446,
+      "rewards/rejected": -0.051891863346099854,
+      "step": 10
+    },
+    {
+      "epoch": 0.1301775147928994,
+      "grad_norm": 57.291604252818274,
+      "learning_rate": 8.088235294117647e-08,
+      "logits/chosen": -0.8743460774421692,
+      "logits/rejected": -0.8040152788162231,
+      "logps/chosen": -46.50861358642578,
+      "logps/rejected": -48.608097076416016,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.04767334461212158,
+      "rewards/margins": 0.07948525249958038,
+      "rewards/rejected": -0.0318119041621685,
+      "step": 11
+    },
+    {
+      "epoch": 0.14201183431952663,
+      "grad_norm": 54.33703237230041,
+      "learning_rate": 8.823529411764706e-08,
+      "logits/chosen": -0.7734875679016113,
+      "logits/rejected": -0.7457428574562073,
+      "logps/chosen": -39.402000427246094,
+      "logps/rejected": -47.64637756347656,
+      "loss": 0.6986,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.00954131968319416,
+      "rewards/margins": -0.019432254135608673,
+      "rewards/rejected": 0.009890936315059662,
+      "step": 12
+    },
+    {
+      "epoch": 0.15384615384615385,
+      "grad_norm": 52.16368650444846,
+      "learning_rate": 9.558823529411763e-08,
+      "logits/chosen": -0.6576703786849976,
+      "logits/rejected": -0.6208328008651733,
+      "logps/chosen": -38.19804000854492,
+      "logps/rejected": -49.989341735839844,
+      "loss": 0.6868,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.007916189730167389,
+      "rewards/margins": 0.02689201757311821,
+      "rewards/rejected": -0.01897583156824112,
+      "step": 13
+    },
+    {
+      "epoch": 0.16568047337278108,
+      "grad_norm": 59.87799896888102,
+      "learning_rate": 1.0294117647058822e-07,
+      "logits/chosen": -0.4240450859069824,
+      "logits/rejected": -0.5086762309074402,
+      "logps/chosen": -31.308774948120117,
+      "logps/rejected": -37.28623580932617,
+      "loss": 0.6977,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.005666483659297228,
+      "rewards/margins": -0.00042060669511556625,
+      "rewards/rejected": -0.005245877429842949,
+      "step": 14
+    },
+    {
+      "epoch": 0.17751479289940827,
+      "grad_norm": 51.93559977619766,
+      "learning_rate": 1.1029411764705881e-07,
+      "logits/chosen": -0.6712979674339294,
+      "logits/rejected": -0.40396082401275635,
+      "logps/chosen": -32.445457458496094,
+      "logps/rejected": -51.878196716308594,
+      "loss": 0.6975,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.0392225980758667,
+      "rewards/margins": -0.022353485226631165,
+      "rewards/rejected": -0.016869116574525833,
+      "step": 15
+    },
+    {
+      "epoch": 0.1893491124260355,
+      "grad_norm": 54.783459080693845,
+      "learning_rate": 1.176470588235294e-07,
+      "logits/chosen": -0.5053819417953491,
+      "logits/rejected": -0.7511165738105774,
+      "logps/chosen": -39.814659118652344,
+      "logps/rejected": -38.76427459716797,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.004495692439377308,
+      "rewards/margins": 0.00818572100251913,
+      "rewards/rejected": -0.01268141157925129,
+      "step": 16
+    },
+    {
+      "epoch": 0.20118343195266272,
+      "grad_norm": 50.13716138258182,
+      "learning_rate": 1.25e-07,
+      "logits/chosen": -0.38788411021232605,
+      "logits/rejected": -0.5259600877761841,
+      "logps/chosen": -39.752655029296875,
+      "logps/rejected": -35.34233856201172,
+      "loss": 0.6873,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.0529358834028244,
+      "rewards/margins": 0.05223493278026581,
+      "rewards/rejected": 0.0007009506225585938,
+      "step": 17
+    },
+    {
+      "epoch": 0.21301775147928995,
+      "grad_norm": 53.56051736985278,
+      "learning_rate": 1.3235294117647057e-07,
+      "logits/chosen": -0.6249361038208008,
+      "logits/rejected": -0.2918693721294403,
+      "logps/chosen": -37.1358528137207,
+      "logps/rejected": -54.117340087890625,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.01282811164855957,
+      "rewards/margins": 0.035147905349731445,
+      "rewards/rejected": -0.047976016998291016,
+      "step": 18
+    },
+    {
+      "epoch": 0.22485207100591717,
+      "grad_norm": 55.4929889677955,
+      "learning_rate": 1.3970588235294117e-07,
+      "logits/chosen": -0.7626937627792358,
+      "logits/rejected": -0.6267987489700317,
+      "logps/chosen": -34.65644073486328,
+      "logps/rejected": -41.60660934448242,
+      "loss": 0.687,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.019578397274017334,
+      "rewards/margins": 0.030142582952976227,
+      "rewards/rejected": -0.010564185678958893,
+      "step": 19
+    },
+    {
+      "epoch": 0.23668639053254437,
+      "grad_norm": 53.81366773458671,
+      "learning_rate": 1.4705882352941175e-07,
+      "logits/chosen": -0.9788577556610107,
+      "logits/rejected": -0.9218689203262329,
+      "logps/chosen": -35.290775299072266,
+      "logps/rejected": -40.17462158203125,
+      "loss": 0.6949,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.02336425706744194,
+      "rewards/margins": -0.008133504539728165,
+      "rewards/rejected": -0.015230750665068626,
+      "step": 20
+    },
+    {
+      "epoch": 0.2485207100591716,
+      "grad_norm": 51.609039023069855,
+      "learning_rate": 1.5441176470588236e-07,
+      "logits/chosen": -0.7643724083900452,
+      "logits/rejected": -0.8248336911201477,
+      "logps/chosen": -39.32087707519531,
+      "logps/rejected": -44.490169525146484,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.023657750338315964,
+      "rewards/margins": 0.06776070594787598,
+      "rewards/rejected": -0.04410295560956001,
+      "step": 21
+    },
+    {
+      "epoch": 0.2603550295857988,
+      "grad_norm": 52.42721385787476,
+      "learning_rate": 1.6176470588235293e-07,
+      "logits/chosen": -0.6979169249534607,
+      "logits/rejected": -0.5965849161148071,
+      "logps/chosen": -35.5829963684082,
+      "logps/rejected": -46.63249206542969,
+      "loss": 0.686,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.017513036727905273,
+      "rewards/margins": 0.06794863194227219,
+      "rewards/rejected": -0.05043559521436691,
+      "step": 22
+    },
+    {
+      "epoch": 0.27218934911242604,
+      "grad_norm": 52.14376613382141,
+      "learning_rate": 1.6911764705882354e-07,
+      "logits/chosen": -0.26172494888305664,
+      "logits/rejected": -0.34854474663734436,
+      "logps/chosen": -43.53734588623047,
+      "logps/rejected": -47.87934494018555,
+      "loss": 0.684,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.08613558113574982,
+      "rewards/margins": -0.0002110544592142105,
+      "rewards/rejected": -0.08592452853918076,
+      "step": 23
+    },
+    {
+      "epoch": 0.28402366863905326,
+      "grad_norm": 50.932647643811045,
+      "learning_rate": 1.764705882352941e-07,
+      "logits/chosen": -0.5405491590499878,
+      "logits/rejected": -0.6527445316314697,
+      "logps/chosen": -41.2894287109375,
+      "logps/rejected": -42.569549560546875,
+      "loss": 0.6868,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.017691707238554955,
+      "rewards/margins": 0.07997651398181915,
+      "rewards/rejected": -0.09766822308301926,
+      "step": 24
+    },
+    {
+      "epoch": 0.2958579881656805,
+      "grad_norm": 52.98859914460327,
+      "learning_rate": 1.8382352941176472e-07,
+      "logits/chosen": -0.8704635500907898,
+      "logits/rejected": -0.8379695415496826,
+      "logps/chosen": -28.815597534179688,
+      "logps/rejected": -40.756103515625,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.02325289323925972,
+      "rewards/margins": 0.040268998593091965,
+      "rewards/rejected": -0.06352189183235168,
+      "step": 25
+    },
+    {
+      "epoch": 0.3076923076923077,
+      "grad_norm": 48.75103011196724,
+      "learning_rate": 1.9117647058823527e-07,
+      "logits/chosen": -0.7580403089523315,
+      "logits/rejected": -0.5783815979957581,
+      "logps/chosen": -37.27444839477539,
+      "logps/rejected": -50.22969055175781,
+      "loss": 0.679,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.04801575839519501,
+      "rewards/margins": 0.08580491691827774,
+      "rewards/rejected": -0.13382068276405334,
+      "step": 26
+    },
+    {
+      "epoch": 0.31952662721893493,
+      "grad_norm": 50.933141633314996,
+      "learning_rate": 1.9852941176470587e-07,
+      "logits/chosen": -0.5684085488319397,
+      "logits/rejected": -0.41421839594841003,
+      "logps/chosen": -29.699729919433594,
+      "logps/rejected": -45.36609649658203,
+      "loss": 0.6783,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.1085064709186554,
+      "rewards/margins": -0.0001874007284641266,
+      "rewards/rejected": -0.10831907391548157,
+      "step": 27
+    },
+    {
+      "epoch": 0.33136094674556216,
+      "grad_norm": 51.922779020745416,
+      "learning_rate": 2.0588235294117645e-07,
+      "logits/chosen": -0.6725336909294128,
+      "logits/rejected": -0.6964073777198792,
+      "logps/chosen": -35.67041015625,
+      "logps/rejected": -41.71068572998047,
+      "loss": 0.6749,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.059418633580207825,
+      "rewards/margins": 0.05292558670043945,
+      "rewards/rejected": -0.11234420537948608,
+      "step": 28
+    },
+    {
+      "epoch": 0.3431952662721893,
+      "grad_norm": 50.897993802285555,
+      "learning_rate": 2.1323529411764705e-07,
+      "logits/chosen": -0.45363789796829224,
+      "logits/rejected": -0.5877288579940796,
+      "logps/chosen": -41.74443817138672,
+      "logps/rejected": -40.014991760253906,
+      "loss": 0.6687,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.09418225288391113,
+      "rewards/margins": 0.036634661257267,
+      "rewards/rejected": -0.13081692159175873,
+      "step": 29
+    },
+    {
+      "epoch": 0.35502958579881655,
+      "grad_norm": 47.17244651587936,
+      "learning_rate": 2.2058823529411763e-07,
+      "logits/chosen": -0.2768702805042267,
+      "logits/rejected": -0.2787284255027771,
+      "logps/chosen": -42.12376403808594,
+      "logps/rejected": -51.230594635009766,
+      "loss": 0.6608,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.11093667894601822,
+      "rewards/margins": 0.1094408631324768,
+      "rewards/rejected": -0.22037753462791443,
+      "step": 30
+    },
+    {
+      "epoch": 0.3668639053254438,
+      "grad_norm": 47.9586113592662,
+      "learning_rate": 2.2794117647058823e-07,
+      "logits/chosen": -0.4938819110393524,
+      "logits/rejected": -0.5225380659103394,
+      "logps/chosen": -32.536563873291016,
+      "logps/rejected": -38.28359603881836,
+      "loss": 0.6474,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.12956911325454712,
+      "rewards/margins": 0.13196122646331787,
+      "rewards/rejected": -0.261530339717865,
+      "step": 31
+    },
+    {
+      "epoch": 0.378698224852071,
+      "grad_norm": 52.2429961485054,
+      "learning_rate": 2.352941176470588e-07,
+      "logits/chosen": -0.7137393951416016,
+      "logits/rejected": -0.7953929305076599,
+      "logps/chosen": -34.03690719604492,
+      "logps/rejected": -42.32886505126953,
+      "loss": 0.6439,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.13849107921123505,
+      "rewards/margins": 0.14229899644851685,
+      "rewards/rejected": -0.2807900607585907,
+      "step": 32
+    },
+    {
+      "epoch": 0.3905325443786982,
+      "grad_norm": 48.070320304458484,
+      "learning_rate": 2.426470588235294e-07,
+      "logits/chosen": -0.709818959236145,
+      "logits/rejected": -0.4917900562286377,
+      "logps/chosen": -32.813262939453125,
+      "logps/rejected": -52.584774017333984,
+      "loss": 0.6496,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.10162033885717392,
+      "rewards/margins": 0.21768181025981903,
+      "rewards/rejected": -0.31930214166641235,
+      "step": 33
+    },
+    {
+      "epoch": 0.40236686390532544,
+      "grad_norm": 50.501375226344486,
+      "learning_rate": 2.5e-07,
+      "logits/chosen": -1.2210612297058105,
+      "logits/rejected": -1.069713830947876,
+      "logps/chosen": -28.096343994140625,
+      "logps/rejected": -42.77171325683594,
+      "loss": 0.6609,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.06562051922082901,
+      "rewards/margins": 0.1742829531431198,
+      "rewards/rejected": -0.23990347981452942,
+      "step": 34
+    },
+    {
+      "epoch": 0.41420118343195267,
+      "grad_norm": 50.18256699637281,
+      "learning_rate": 2.5735294117647057e-07,
+      "logits/chosen": -0.5130794048309326,
+      "logits/rejected": -0.5719175934791565,
+      "logps/chosen": -37.741825103759766,
+      "logps/rejected": -45.18247985839844,
+      "loss": 0.6448,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.18347424268722534,
+      "rewards/margins": 0.13004478812217712,
+      "rewards/rejected": -0.31351903080940247,
+      "step": 35
+    },
+    {
+      "epoch": 0.4260355029585799,
+      "grad_norm": 49.643545907741306,
+      "learning_rate": 2.6470588235294114e-07,
+      "logits/chosen": -0.9145612716674805,
+      "logits/rejected": -0.8272578120231628,
+      "logps/chosen": -34.39950180053711,
+      "logps/rejected": -54.22731018066406,
+      "loss": 0.6389,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.22830158472061157,
+      "rewards/margins": 0.19368687272071838,
+      "rewards/rejected": -0.42198845744132996,
+      "step": 36
+    },
+    {
+      "epoch": 0.4378698224852071,
+      "grad_norm": 46.37858153447604,
+      "learning_rate": 2.720588235294117e-07,
+      "logits/chosen": -0.8228033781051636,
+      "logits/rejected": -0.6848031282424927,
+      "logps/chosen": -30.318634033203125,
+      "logps/rejected": -42.5218505859375,
+      "loss": 0.6339,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1820099651813507,
+      "rewards/margins": 0.23087123036384583,
+      "rewards/rejected": -0.41288119554519653,
+      "step": 37
+    },
+    {
+      "epoch": 0.44970414201183434,
+      "grad_norm": 47.68646874364111,
+      "learning_rate": 2.7941176470588235e-07,
+      "logits/chosen": -0.41094350814819336,
+      "logits/rejected": -0.457292765378952,
+      "logps/chosen": -40.93132019042969,
+      "logps/rejected": -44.33328628540039,
+      "loss": 0.6219,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2081282138824463,
+      "rewards/margins": 0.17258334159851074,
+      "rewards/rejected": -0.38071155548095703,
+      "step": 38
+    },
+    {
+      "epoch": 0.46153846153846156,
+      "grad_norm": 46.99139408792661,
+      "learning_rate": 2.8676470588235293e-07,
+      "logits/chosen": -0.9174866676330566,
+      "logits/rejected": -0.8664580583572388,
+      "logps/chosen": -34.22110366821289,
+      "logps/rejected": -45.25912857055664,
+      "loss": 0.6077,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.24102823436260223,
+      "rewards/margins": 0.3032039701938629,
+      "rewards/rejected": -0.544232189655304,
+      "step": 39
+    },
+    {
+      "epoch": 0.47337278106508873,
+      "grad_norm": 44.92025317341631,
+      "learning_rate": 2.941176470588235e-07,
+      "logits/chosen": -0.5698567628860474,
+      "logits/rejected": -0.6562440395355225,
+      "logps/chosen": -35.16241455078125,
+      "logps/rejected": -43.87369155883789,
+      "loss": 0.6074,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3447490930557251,
+      "rewards/margins": 0.1720276027917862,
+      "rewards/rejected": -0.5167766809463501,
+      "step": 40
+    },
+    {
+      "epoch": 0.48520710059171596,
+      "grad_norm": 43.61954903949639,
+      "learning_rate": 3.014705882352941e-07,
+      "logits/chosen": -0.9510785341262817,
+      "logits/rejected": -1.0125081539154053,
+      "logps/chosen": -43.066551208496094,
+      "logps/rejected": -50.81156921386719,
+      "loss": 0.5926,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.42604392766952515,
+      "rewards/margins": 0.2472018599510193,
+      "rewards/rejected": -0.6732457876205444,
+      "step": 41
+    },
+    {
+      "epoch": 0.4970414201183432,
+      "grad_norm": 48.79189553867012,
+      "learning_rate": 3.088235294117647e-07,
+      "logits/chosen": -1.0401208400726318,
+      "logits/rejected": -0.9916763305664062,
+      "logps/chosen": -36.0028076171875,
+      "logps/rejected": -47.314449310302734,
+      "loss": 0.6188,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3239697813987732,
+      "rewards/margins": 0.3483524024486542,
+      "rewards/rejected": -0.672322154045105,
+      "step": 42
+    },
+    {
+      "epoch": 0.5088757396449705,
+      "grad_norm": 45.29293394827699,
+      "learning_rate": 3.161764705882353e-07,
+      "logits/chosen": -0.6100953817367554,
+      "logits/rejected": -0.5119140148162842,
+      "logps/chosen": -36.110836029052734,
+      "logps/rejected": -55.10944366455078,
+      "loss": 0.5727,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.42954936623573303,
+      "rewards/margins": 0.548741340637207,
+      "rewards/rejected": -0.9782906770706177,
+      "step": 43
+    },
+    {
+      "epoch": 0.5207100591715976,
+      "grad_norm": 43.75505094492696,
+      "learning_rate": 3.2352941176470586e-07,
+      "logits/chosen": -0.5729663372039795,
+      "logits/rejected": -0.4085710644721985,
+      "logps/chosen": -37.87840270996094,
+      "logps/rejected": -50.89206314086914,
+      "loss": 0.6021,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.3693719506263733,
+      "rewards/margins": 0.343736857175827,
+      "rewards/rejected": -0.7131087779998779,
+      "step": 44
+    },
+    {
+      "epoch": 0.5325443786982249,
+      "grad_norm": 46.651471884498214,
+      "learning_rate": 3.3088235294117644e-07,
+      "logits/chosen": -0.5654891133308411,
+      "logits/rejected": -0.689947783946991,
+      "logps/chosen": -41.97024154663086,
+      "logps/rejected": -46.030967712402344,
+      "loss": 0.573,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.35547372698783875,
+      "rewards/margins": 0.4130653738975525,
+      "rewards/rejected": -0.7685391902923584,
+      "step": 45
+    },
+    {
+      "epoch": 0.5443786982248521,
+      "grad_norm": 43.74968458385381,
+      "learning_rate": 3.3823529411764707e-07,
+      "logits/chosen": -0.6354714632034302,
+      "logits/rejected": -0.7248853445053101,
+      "logps/chosen": -34.041595458984375,
+      "logps/rejected": -39.55751037597656,
+      "loss": 0.5944,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.3268294930458069,
+      "rewards/margins": 0.2843489646911621,
+      "rewards/rejected": -0.611178457736969,
+      "step": 46
+    },
+    {
+      "epoch": 0.5562130177514792,
+      "grad_norm": 44.663170642243635,
+      "learning_rate": 3.4558823529411765e-07,
+      "logits/chosen": -0.43914204835891724,
+      "logits/rejected": -0.41449761390686035,
+      "logps/chosen": -32.82707595825195,
+      "logps/rejected": -37.95698547363281,
+      "loss": 0.5942,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3479520082473755,
+      "rewards/margins": 0.3092345595359802,
+      "rewards/rejected": -0.6571865677833557,
+      "step": 47
+    },
+    {
+      "epoch": 0.5680473372781065,
+      "grad_norm": 43.36900384363229,
+      "learning_rate": 3.529411764705882e-07,
+      "logits/chosen": -0.6816117167472839,
+      "logits/rejected": -0.6095008254051208,
+      "logps/chosen": -33.6269416809082,
+      "logps/rejected": -47.18854522705078,
+      "loss": 0.5445,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.40539008378982544,
+      "rewards/margins": 0.5039654970169067,
+      "rewards/rejected": -0.909355640411377,
+      "step": 48
+    },
+    {
+      "epoch": 0.5798816568047337,
+      "grad_norm": 44.70080178640567,
+      "learning_rate": 3.602941176470588e-07,
+      "logits/chosen": -0.49769917130470276,
+      "logits/rejected": -0.33341550827026367,
+      "logps/chosen": -37.26922607421875,
+      "logps/rejected": -45.382469177246094,
+      "loss": 0.5435,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.41994184255599976,
+      "rewards/margins": 0.42326265573501587,
+      "rewards/rejected": -0.8432044982910156,
+      "step": 49
+    },
+    {
+      "epoch": 0.591715976331361,
+      "grad_norm": 42.832062592121694,
+      "learning_rate": 3.6764705882352943e-07,
+      "logits/chosen": -0.6227316856384277,
+      "logits/rejected": -0.6291834712028503,
+      "logps/chosen": -37.443336486816406,
+      "logps/rejected": -56.363712310791016,
+      "loss": 0.564,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.3350003659725189,
+      "rewards/margins": 0.5508270263671875,
+      "rewards/rejected": -0.8858274817466736,
+      "step": 50
+    },
+    {
+      "epoch": 0.6035502958579881,
+      "grad_norm": 40.537726641128835,
+      "learning_rate": 3.75e-07,
+      "logits/chosen": -0.5767950415611267,
+      "logits/rejected": -0.40233951807022095,
+      "logps/chosen": -34.15302658081055,
+      "logps/rejected": -41.203514099121094,
+      "loss": 0.5447,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.4783581495285034,
+      "rewards/margins": 0.36022478342056274,
+      "rewards/rejected": -0.8385828733444214,
+      "step": 51
+    },
+    {
+      "epoch": 0.6153846153846154,
+      "grad_norm": 37.3320312919307,
+      "learning_rate": 3.8235294117647053e-07,
+      "logits/chosen": -0.33203282952308655,
+      "logits/rejected": -0.39482319355010986,
+      "logps/chosen": -35.42144012451172,
+      "logps/rejected": -38.94422149658203,
+      "loss": 0.52,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3685012459754944,
+      "rewards/margins": 0.5317042469978333,
+      "rewards/rejected": -0.9002054929733276,
+      "step": 52
+    },
+    {
+      "epoch": 0.6272189349112426,
+      "grad_norm": 44.962671519597414,
+      "learning_rate": 3.8970588235294116e-07,
+      "logits/chosen": -0.7768542766571045,
+      "logits/rejected": -0.7662684917449951,
+      "logps/chosen": -40.90711975097656,
+      "logps/rejected": -43.82652282714844,
+      "loss": 0.5597,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5888789892196655,
+      "rewards/margins": 0.7121487259864807,
+      "rewards/rejected": -1.3010276556015015,
+      "step": 53
+    },
+    {
+      "epoch": 0.6390532544378699,
+      "grad_norm": 40.11509893356046,
+      "learning_rate": 3.9705882352941174e-07,
+      "logits/chosen": -0.44021129608154297,
+      "logits/rejected": -0.33102425932884216,
+      "logps/chosen": -35.29465103149414,
+      "logps/rejected": -48.99385070800781,
+      "loss": 0.4633,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.4893972873687744,
+      "rewards/margins": 0.4504549503326416,
+      "rewards/rejected": -0.939852237701416,
+      "step": 54
+    },
+    {
+      "epoch": 0.650887573964497,
+      "grad_norm": 43.113444451145575,
+      "learning_rate": 4.044117647058823e-07,
+      "logits/chosen": -0.546736478805542,
+      "logits/rejected": -0.6314712762832642,
+      "logps/chosen": -43.929542541503906,
+      "logps/rejected": -49.84693908691406,
+      "loss": 0.5246,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.6373113393783569,
+      "rewards/margins": 0.9987615346908569,
+      "rewards/rejected": -1.6360729932785034,
+      "step": 55
+    },
+    {
+      "epoch": 0.6627218934911243,
+      "grad_norm": 44.49441569027861,
+      "learning_rate": 4.117647058823529e-07,
+      "logits/chosen": -0.5550782084465027,
+      "logits/rejected": -0.5997810959815979,
+      "logps/chosen": -39.631263732910156,
+      "logps/rejected": -41.65299987792969,
+      "loss": 0.5692,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.5492139458656311,
+      "rewards/margins": 0.6075623631477356,
+      "rewards/rejected": -1.1567763090133667,
+      "step": 56
+    },
+    {
+      "epoch": 0.6745562130177515,
+      "grad_norm": 49.99500479519036,
+      "learning_rate": 4.191176470588235e-07,
+      "logits/chosen": -0.7147572040557861,
+      "logits/rejected": -0.809965193271637,
+      "logps/chosen": -39.72154998779297,
+      "logps/rejected": -44.34455871582031,
+      "loss": 0.5461,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.4619465172290802,
+      "rewards/margins": 0.43853867053985596,
+      "rewards/rejected": -0.9004851579666138,
+      "step": 57
+    },
+    {
+      "epoch": 0.6863905325443787,
+      "grad_norm": 41.686424338259535,
+      "learning_rate": 4.264705882352941e-07,
+      "logits/chosen": -0.4276542663574219,
+      "logits/rejected": -0.6359944939613342,
+      "logps/chosen": -38.81695556640625,
+      "logps/rejected": -44.787322998046875,
+      "loss": 0.4877,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.31248077750205994,
+      "rewards/margins": 0.9359371662139893,
+      "rewards/rejected": -1.248417854309082,
+      "step": 58
+    },
+    {
+      "epoch": 0.6982248520710059,
+      "grad_norm": 39.36173522050298,
+      "learning_rate": 4.338235294117647e-07,
+      "logits/chosen": -0.6473718285560608,
+      "logits/rejected": -0.7196489572525024,
+      "logps/chosen": -39.105743408203125,
+      "logps/rejected": -43.301734924316406,
+      "loss": 0.4509,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5941877365112305,
+      "rewards/margins": 0.8893916010856628,
+      "rewards/rejected": -1.483579397201538,
+      "step": 59
+    },
+    {
+      "epoch": 0.7100591715976331,
+      "grad_norm": 37.06656262082701,
+      "learning_rate": 4.4117647058823526e-07,
+      "logits/chosen": -0.4076049029827118,
+      "logits/rejected": -0.20640525221824646,
+      "logps/chosen": -32.933773040771484,
+      "logps/rejected": -50.366722106933594,
+      "loss": 0.4341,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.39851048588752747,
+      "rewards/margins": 0.7430359125137329,
+      "rewards/rejected": -1.141546368598938,
+      "step": 60
+    },
+    {
+      "epoch": 0.7218934911242604,
+      "grad_norm": 36.19815173586054,
+      "learning_rate": 4.485294117647059e-07,
+      "logits/chosen": -0.7895621061325073,
+      "logits/rejected": -0.8529470562934875,
+      "logps/chosen": -41.30886459350586,
+      "logps/rejected": -52.98210906982422,
+      "loss": 0.4463,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.4878369867801666,
+      "rewards/margins": 1.4172559976577759,
+      "rewards/rejected": -1.9050930738449097,
+      "step": 61
+    },
+    {
+      "epoch": 0.7337278106508875,
+      "grad_norm": 37.646252584686685,
+      "learning_rate": 4.5588235294117646e-07,
+      "logits/chosen": -0.46224623918533325,
+      "logits/rejected": -0.5132017731666565,
+      "logps/chosen": -35.61504364013672,
+      "logps/rejected": -45.02849578857422,
+      "loss": 0.446,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.31568989157676697,
+      "rewards/margins": 1.0938689708709717,
+      "rewards/rejected": -1.409558892250061,
+      "step": 62
+    },
+    {
+      "epoch": 0.7455621301775148,
+      "grad_norm": 40.99920932267914,
+      "learning_rate": 4.6323529411764704e-07,
+      "logits/chosen": -0.24282173812389374,
+      "logits/rejected": -0.41654685139656067,
+      "logps/chosen": -40.87584686279297,
+      "logps/rejected": -45.074466705322266,
+      "loss": 0.4864,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.5393080711364746,
+      "rewards/margins": 0.729945182800293,
+      "rewards/rejected": -1.2692532539367676,
+      "step": 63
+    },
+    {
+      "epoch": 0.757396449704142,
+      "grad_norm": 39.08132618780929,
+      "learning_rate": 4.705882352941176e-07,
+      "logits/chosen": -0.7257353067398071,
+      "logits/rejected": -0.6613651514053345,
+      "logps/chosen": -31.330947875976562,
+      "logps/rejected": -46.061580657958984,
+      "loss": 0.431,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.2516774535179138,
+      "rewards/margins": 1.220921277999878,
+      "rewards/rejected": -1.472598910331726,
+      "step": 64
+    },
+    {
+      "epoch": 0.7692307692307693,
+      "grad_norm": 41.475907121673465,
+      "learning_rate": 4.779411764705882e-07,
+      "logits/chosen": -0.609380841255188,
+      "logits/rejected": -0.6923696994781494,
+      "logps/chosen": -42.626522064208984,
+      "logps/rejected": -47.41869354248047,
+      "loss": 0.4345,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.4935339689254761,
+      "rewards/margins": 0.8159966468811035,
+      "rewards/rejected": -1.30953049659729,
+      "step": 65
+    },
+    {
+      "epoch": 0.7810650887573964,
+      "grad_norm": 40.932537298379174,
+      "learning_rate": 4.852941176470588e-07,
+      "logits/chosen": -1.1143670082092285,
+      "logits/rejected": -1.1686656475067139,
+      "logps/chosen": -49.556392669677734,
+      "logps/rejected": -38.85927200317383,
+      "loss": 0.4568,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.46525275707244873,
+      "rewards/margins": 0.4739494323730469,
+      "rewards/rejected": -0.9392022490501404,
+      "step": 66
+    },
+    {
+      "epoch": 0.7928994082840237,
+      "grad_norm": 36.93572313017648,
+      "learning_rate": 4.926470588235295e-07,
+      "logits/chosen": -0.659454882144928,
+      "logits/rejected": -0.7761635184288025,
+      "logps/chosen": -36.797096252441406,
+      "logps/rejected": -46.951560974121094,
+      "loss": 0.4149,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.47968119382858276,
+      "rewards/margins": 1.5291297435760498,
+      "rewards/rejected": -2.0088109970092773,
+      "step": 67
+    },
+    {
+      "epoch": 0.8047337278106509,
+      "grad_norm": 44.10212062947347,
+      "learning_rate": 5e-07,
+      "logits/chosen": -0.22745545208454132,
+      "logits/rejected": -0.3337535858154297,
+      "logps/chosen": -43.72065734863281,
+      "logps/rejected": -46.58799743652344,
+      "loss": 0.451,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.11050184071063995,
+      "rewards/margins": 1.0298746824264526,
+      "rewards/rejected": -1.1403765678405762,
+      "step": 68
+    },
+    {
+      "epoch": 0.8165680473372781,
+      "grad_norm": 41.21338166081281,
+      "learning_rate": 4.999966183013662e-07,
+      "logits/chosen": -0.5445861220359802,
+      "logits/rejected": -0.8019598126411438,
+      "logps/chosen": -41.4810791015625,
+      "logps/rejected": -36.41514587402344,
+      "loss": 0.4593,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.505042314529419,
+      "rewards/margins": 0.4535430669784546,
+      "rewards/rejected": -0.9585853815078735,
+      "step": 69
+    },
+    {
+      "epoch": 0.8284023668639053,
+      "grad_norm": 35.095204565269476,
+      "learning_rate": 4.999864732969518e-07,
+      "logits/chosen": -0.7983517050743103,
+      "logits/rejected": -0.7316067814826965,
+      "logps/chosen": -34.78556442260742,
+      "logps/rejected": -48.34429931640625,
+      "loss": 0.409,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.520796537399292,
+      "rewards/margins": 1.445544958114624,
+      "rewards/rejected": -1.966341495513916,
+      "step": 70
+    },
+    {
+      "epoch": 0.8402366863905325,
+      "grad_norm": 40.30265829692534,
+      "learning_rate": 4.999695652612155e-07,
+      "logits/chosen": -0.9361473917961121,
+      "logits/rejected": -1.0400917530059814,
+      "logps/chosen": -34.00445556640625,
+      "logps/rejected": -45.20707702636719,
+      "loss": 0.4569,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.12054769694805145,
+      "rewards/margins": 1.341543197631836,
+      "rewards/rejected": -1.4620908498764038,
+      "step": 71
+    },
+    {
+      "epoch": 0.8520710059171598,
+      "grad_norm": 40.415075736728824,
+      "learning_rate": 4.999458946515807e-07,
+      "logits/chosen": -0.5303232669830322,
+      "logits/rejected": -0.6230794191360474,
+      "logps/chosen": -39.0274658203125,
+      "logps/rejected": -45.97068786621094,
+      "loss": 0.4644,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.48386603593826294,
+      "rewards/margins": 0.9363454580307007,
+      "rewards/rejected": -1.4202115535736084,
+      "step": 72
+    },
+    {
+      "epoch": 0.863905325443787,
+      "grad_norm": 38.77891075432772,
+      "learning_rate": 4.999154621084221e-07,
+      "logits/chosen": -0.6193030476570129,
+      "logits/rejected": -0.6146333813667297,
+      "logps/chosen": -43.54295349121094,
+      "logps/rejected": -48.63895034790039,
+      "loss": 0.4151,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.49407535791397095,
+      "rewards/margins": 1.051137924194336,
+      "rewards/rejected": -1.5452133417129517,
+      "step": 73
+    },
+    {
+      "epoch": 0.8757396449704142,
+      "grad_norm": 37.77568612282111,
+      "learning_rate": 4.998782684550491e-07,
+      "logits/chosen": -0.36244261264801025,
+      "logits/rejected": -0.44385403394699097,
+      "logps/chosen": -42.133628845214844,
+      "logps/rejected": -43.378028869628906,
+      "loss": 0.4466,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.2831823229789734,
+      "rewards/margins": 0.7465813159942627,
+      "rewards/rejected": -1.0297636985778809,
+      "step": 74
+    },
+    {
+      "epoch": 0.8875739644970414,
+      "grad_norm": 41.66281833701898,
+      "learning_rate": 4.998343146976837e-07,
+      "logits/chosen": -0.9405574202537537,
+      "logits/rejected": -0.9447466135025024,
+      "logps/chosen": -37.16999053955078,
+      "logps/rejected": -50.3101921081543,
+      "loss": 0.4843,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.34810084104537964,
+      "rewards/margins": 1.4797228574752808,
+      "rewards/rejected": -1.8278236389160156,
+      "step": 75
+    },
+    {
+      "epoch": 0.8994082840236687,
+      "grad_norm": 41.38364147290228,
+      "learning_rate": 4.997836020254328e-07,
+      "logits/chosen": -0.760882556438446,
+      "logits/rejected": -0.8380026817321777,
+      "logps/chosen": -32.650291442871094,
+      "logps/rejected": -44.52061462402344,
+      "loss": 0.4716,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.0004928633570671082,
+      "rewards/margins": 1.002596378326416,
+      "rewards/rejected": -1.003089189529419,
+      "step": 76
+    },
+    {
+      "epoch": 0.9112426035502958,
+      "grad_norm": 37.663962046996495,
+      "learning_rate": 4.99726131810256e-07,
+      "logits/chosen": -0.6014434099197388,
+      "logits/rejected": -0.8074021935462952,
+      "logps/chosen": -40.66423797607422,
+      "logps/rejected": -30.966102600097656,
+      "loss": 0.414,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.06758510321378708,
+      "rewards/margins": 0.7224689722061157,
+      "rewards/rejected": -0.7900540232658386,
+      "step": 77
+    },
+    {
+      "epoch": 0.9230769230769231,
+      "grad_norm": 35.62116536902,
+      "learning_rate": 4.996619056069291e-07,
+      "logits/chosen": -0.8005455732345581,
+      "logits/rejected": -1.0737569332122803,
+      "logps/chosen": -47.74522399902344,
+      "logps/rejected": -40.22245788574219,
+      "loss": 0.3874,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.11318114399909973,
+      "rewards/margins": 0.9375712275505066,
+      "rewards/rejected": -0.8243900537490845,
+      "step": 78
+    },
+    {
+      "epoch": 0.9349112426035503,
+      "grad_norm": 41.14929411833454,
+      "learning_rate": 4.995909251530013e-07,
+      "logits/chosen": -0.764075517654419,
+      "logits/rejected": -0.7951244115829468,
+      "logps/chosen": -37.42706298828125,
+      "logps/rejected": -49.54686737060547,
+      "loss": 0.4372,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.2584840655326843,
+      "rewards/margins": 1.3500514030456543,
+      "rewards/rejected": -1.6085355281829834,
+      "step": 79
+    },
+    {
+      "epoch": 0.9467455621301775,
+      "grad_norm": 37.43972651882339,
+      "learning_rate": 4.995131923687487e-07,
+      "logits/chosen": -0.6362431049346924,
+      "logits/rejected": -0.4747117757797241,
+      "logps/chosen": -37.49502944946289,
+      "logps/rejected": -55.17448806762695,
+      "loss": 0.3999,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.2551206946372986,
+      "rewards/margins": 0.9605333209037781,
+      "rewards/rejected": -1.2156541347503662,
+      "step": 80
+    },
+    {
+      "epoch": 0.9585798816568047,
+      "grad_norm": 43.94031843678492,
+      "learning_rate": 4.994287093571221e-07,
+      "logits/chosen": -0.7600383758544922,
+      "logits/rejected": -0.537135660648346,
+      "logps/chosen": -39.631553649902344,
+      "logps/rejected": -59.48577880859375,
+      "loss": 0.4709,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.30705738067626953,
+      "rewards/margins": 1.0295747518539429,
+      "rewards/rejected": -1.3366321325302124,
+      "step": 81
+    },
+    {
+      "epoch": 0.9704142011834319,
+      "grad_norm": 43.25344667732382,
+      "learning_rate": 4.993374784036901e-07,
+      "logits/chosen": -0.8365252017974854,
+      "logits/rejected": -0.7885805368423462,
+      "logps/chosen": -38.859161376953125,
+      "logps/rejected": -54.36302947998047,
+      "loss": 0.3922,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.3023694157600403,
+      "rewards/margins": 2.4610185623168945,
+      "rewards/rejected": -2.763388156890869,
+      "step": 82
+    },
+    {
+      "epoch": 0.9822485207100592,
+      "grad_norm": 37.685612874220524,
+      "learning_rate": 4.992395019765775e-07,
+      "logits/chosen": -0.6076855659484863,
+      "logits/rejected": -0.6970850229263306,
+      "logps/chosen": -37.532691955566406,
+      "logps/rejected": -53.90351486206055,
+      "loss": 0.3857,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5352523326873779,
+      "rewards/margins": 1.9930164813995361,
+      "rewards/rejected": -2.528268814086914,
+      "step": 83
+    },
+    {
+      "epoch": 0.9940828402366864,
+      "grad_norm": 36.826262132209905,
+      "learning_rate": 4.991347827263982e-07,
+      "logits/chosen": -0.8733373880386353,
+      "logits/rejected": -0.7132407426834106,
+      "logps/chosen": -33.2780647277832,
+      "logps/rejected": -47.66956329345703,
+      "loss": 0.4261,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.13416826725006104,
+      "rewards/margins": 1.7117326259613037,
+      "rewards/rejected": -1.5775643587112427,
+      "step": 84
+    },
+    {
+      "epoch": 1.0059171597633136,
+      "grad_norm": 40.77876491907824,
+      "learning_rate": 4.990233234861839e-07,
+      "logits/chosen": -0.6671120524406433,
+      "logits/rejected": -0.961428701877594,
+      "logps/chosen": -37.66517639160156,
+      "logps/rejected": -42.085426330566406,
+      "loss": 0.375,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.03282582014799118,
+      "rewards/margins": 2.404395580291748,
+      "rewards/rejected": -2.4372215270996094,
+      "step": 85
+    },
+    {
+      "epoch": 1.017751479289941,
+      "grad_norm": 35.343447731733626,
+      "learning_rate": 4.989051272713069e-07,
+      "logits/chosen": -0.9536569118499756,
+      "logits/rejected": -1.1731913089752197,
+      "logps/chosen": -35.637535095214844,
+      "logps/rejected": -42.364036560058594,
+      "loss": 0.3999,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.12619183957576752,
+      "rewards/margins": 2.5188794136047363,
+      "rewards/rejected": -2.645071268081665,
+      "step": 86
+    },
+    {
+      "epoch": 1.029585798816568,
+      "grad_norm": 33.223106342915955,
+      "learning_rate": 4.987801972793993e-07,
+      "logits/chosen": -0.7712712287902832,
+      "logits/rejected": -0.9531198740005493,
+      "logps/chosen": -45.71641540527344,
+      "logps/rejected": -58.85322952270508,
+      "loss": 0.3053,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.2449014037847519,
+      "rewards/margins": 3.286306142807007,
+      "rewards/rejected": -3.0414042472839355,
+      "step": 87
+    },
+    {
+      "epoch": 1.0414201183431953,
+      "grad_norm": 35.64530002995125,
+      "learning_rate": 4.986485368902656e-07,
+      "logits/chosen": -0.6423381567001343,
+      "logits/rejected": -0.6168457865715027,
+      "logps/chosen": -31.16773223876953,
+      "logps/rejected": -45.484893798828125,
+      "loss": 0.3375,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.22324074804782867,
+      "rewards/margins": 1.2495863437652588,
+      "rewards/rejected": -1.4728271961212158,
+      "step": 88
+    },
+    {
+      "epoch": 1.0532544378698225,
+      "grad_norm": 37.20259605794752,
+      "learning_rate": 4.985101496657918e-07,
+      "logits/chosen": -1.1634238958358765,
+      "logits/rejected": -1.103615164756775,
+      "logps/chosen": -41.237342834472656,
+      "logps/rejected": -62.546531677246094,
+      "loss": 0.344,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.3175942599773407,
+      "rewards/margins": 2.2205255031585693,
+      "rewards/rejected": -2.5381197929382324,
+      "step": 89
+    },
+    {
+      "epoch": 1.0650887573964498,
+      "grad_norm": 32.598385557495284,
+      "learning_rate": 4.983650393498489e-07,
+      "logits/chosen": -0.922864556312561,
+      "logits/rejected": -0.8289706707000732,
+      "logps/chosen": -29.635332107543945,
+      "logps/rejected": -45.355323791503906,
+      "loss": 0.3229,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.7630589008331299,
+      "rewards/margins": 1.704155683517456,
+      "rewards/rejected": -2.467214584350586,
+      "step": 90
+    },
+    {
+      "epoch": 1.0769230769230769,
+      "grad_norm": 38.26125209425027,
+      "learning_rate": 4.982132098681923e-07,
+      "logits/chosen": -1.0017441511154175,
+      "logits/rejected": -0.9490557909011841,
+      "logps/chosen": -40.40355682373047,
+      "logps/rejected": -53.06212615966797,
+      "loss": 0.3536,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.7460950016975403,
+      "rewards/margins": 2.146270513534546,
+      "rewards/rejected": -2.8923654556274414,
+      "step": 91
+    },
+    {
+      "epoch": 1.0887573964497042,
+      "grad_norm": 35.48709030046864,
+      "learning_rate": 4.980546653283537e-07,
+      "logits/chosen": -0.8610115051269531,
+      "logits/rejected": -0.8497661352157593,
+      "logps/chosen": -33.734352111816406,
+      "logps/rejected": -51.21432113647461,
+      "loss": 0.3493,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.08930137753486633,
+      "rewards/margins": 2.130716562271118,
+      "rewards/rejected": -2.041415214538574,
+      "step": 92
+    },
+    {
+      "epoch": 1.1005917159763314,
+      "grad_norm": 33.05555278643355,
+      "learning_rate": 4.978894100195324e-07,
+      "logits/chosen": -0.7339059114456177,
+      "logits/rejected": -0.7007895112037659,
+      "logps/chosen": -43.019500732421875,
+      "logps/rejected": -54.64815902709961,
+      "loss": 0.2908,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.5389220118522644,
+      "rewards/margins": 2.059274673461914,
+      "rewards/rejected": -2.5981967449188232,
+      "step": 93
+    },
+    {
+      "epoch": 1.1124260355029585,
+      "grad_norm": 32.64806139806569,
+      "learning_rate": 4.977174484124775e-07,
+      "logits/chosen": -0.6110660433769226,
+      "logits/rejected": -0.6840114593505859,
+      "logps/chosen": -39.90541458129883,
+      "logps/rejected": -41.75571060180664,
+      "loss": 0.322,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3695860803127289,
+      "rewards/margins": 1.257794737815857,
+      "rewards/rejected": -1.6273807287216187,
+      "step": 94
+    },
+    {
+      "epoch": 1.1242603550295858,
+      "grad_norm": 28.73456224275126,
+      "learning_rate": 4.975387851593676e-07,
+      "logits/chosen": -1.0169286727905273,
+      "logits/rejected": -0.9107025861740112,
+      "logps/chosen": -38.64891052246094,
+      "logps/rejected": -54.81282043457031,
+      "loss": 0.3026,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.09714408218860626,
+      "rewards/margins": 1.6703901290893555,
+      "rewards/rejected": -1.7675341367721558,
+      "step": 95
+    },
+    {
+      "epoch": 1.136094674556213,
+      "grad_norm": 40.46066004537338,
+      "learning_rate": 4.97353425093685e-07,
+      "logits/chosen": -0.5108687877655029,
+      "logits/rejected": -0.5603746175765991,
+      "logps/chosen": -38.765052795410156,
+      "logps/rejected": -44.74797058105469,
+      "loss": 0.3836,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.35121989250183105,
+      "rewards/margins": 1.2221122980117798,
+      "rewards/rejected": -1.5733323097229004,
+      "step": 96
+    },
+    {
+      "epoch": 1.1479289940828403,
+      "grad_norm": 33.115171654161905,
+      "learning_rate": 4.971613732300848e-07,
+      "logits/chosen": -0.5614181756973267,
+      "logits/rejected": -0.44806018471717834,
+      "logps/chosen": -34.740657806396484,
+      "logps/rejected": -51.19026184082031,
+      "loss": 0.3033,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.37993931770324707,
+      "rewards/margins": 2.2012622356414795,
+      "rewards/rejected": -2.5812015533447266,
+      "step": 97
+    },
+    {
+      "epoch": 1.1597633136094674,
+      "grad_norm": 30.314211745480826,
+      "learning_rate": 4.96962634764259e-07,
+      "logits/chosen": -0.6665958762168884,
+      "logits/rejected": -0.9045260548591614,
+      "logps/chosen": -50.40956115722656,
+      "logps/rejected": -52.3349494934082,
+      "loss": 0.2727,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.28259751200675964,
+      "rewards/margins": 2.2699124813079834,
+      "rewards/rejected": -2.5525100231170654,
+      "step": 98
+    },
+    {
+      "epoch": 1.1715976331360947,
+      "grad_norm": 35.25579514857924,
+      "learning_rate": 4.967572150727964e-07,
+      "logits/chosen": -0.6161059737205505,
+      "logits/rejected": -0.619467556476593,
+      "logps/chosen": -41.40134048461914,
+      "logps/rejected": -47.676292419433594,
+      "loss": 0.3182,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.002119541168213,
+      "rewards/margins": 1.5968635082244873,
+      "rewards/rejected": -2.598982810974121,
+      "step": 99
+    },
+    {
+      "epoch": 1.183431952662722,
+      "grad_norm": 31.87031408729149,
+      "learning_rate": 4.965451197130372e-07,
+      "logits/chosen": -0.6040835380554199,
+      "logits/rejected": -0.6801787614822388,
+      "logps/chosen": -35.400917053222656,
+      "logps/rejected": -46.94811248779297,
+      "loss": 0.3003,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.07005947083234787,
+      "rewards/margins": 2.207547903060913,
+      "rewards/rejected": -2.2776074409484863,
+      "step": 100
+    },
+    {
+      "epoch": 1.195266272189349,
+      "grad_norm": 34.95313563934469,
+      "learning_rate": 4.963263544229219e-07,
+      "logits/chosen": -0.7886058688163757,
+      "logits/rejected": -0.8541610240936279,
+      "logps/chosen": -37.76707077026367,
+      "logps/rejected": -52.185367584228516,
+      "loss": 0.2959,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.6315910816192627,
+      "rewards/margins": 2.7951231002807617,
+      "rewards/rejected": -3.4267144203186035,
+      "step": 101
+    },
+    {
+      "epoch": 1.2071005917159763,
+      "grad_norm": 32.38499457275457,
+      "learning_rate": 4.961009251208367e-07,
+      "logits/chosen": -0.9285825490951538,
+      "logits/rejected": -0.9554150700569153,
+      "logps/chosen": -34.80419921875,
+      "logps/rejected": -44.33222961425781,
+      "loss": 0.3286,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.12241765856742859,
+      "rewards/margins": 1.6503454446792603,
+      "rewards/rejected": -1.7727631330490112,
+      "step": 102
+    },
+    {
+      "epoch": 1.2189349112426036,
+      "grad_norm": 35.88300586588521,
+      "learning_rate": 4.958688379054535e-07,
+      "logits/chosen": -0.717126727104187,
+      "logits/rejected": -0.5765900611877441,
+      "logps/chosen": -32.79739761352539,
+      "logps/rejected": -56.2498893737793,
+      "loss": 0.338,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.11334749311208725,
+      "rewards/margins": 2.23618745803833,
+      "rewards/rejected": -2.3495349884033203,
+      "step": 103
+    },
+    {
+      "epoch": 1.2307692307692308,
+      "grad_norm": 32.32700164175202,
+      "learning_rate": 4.956300990555643e-07,
+      "logits/chosen": -0.8154267072677612,
+      "logits/rejected": -0.9468405246734619,
+      "logps/chosen": -38.78935623168945,
+      "logps/rejected": -56.32616424560547,
+      "loss": 0.2922,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.04591973125934601,
+      "rewards/margins": 3.170891761779785,
+      "rewards/rejected": -3.2168116569519043,
+      "step": 104
+    },
+    {
+      "epoch": 1.242603550295858,
+      "grad_norm": 31.39200334822418,
+      "learning_rate": 4.953847150299118e-07,
+      "logits/chosen": -0.6333639621734619,
+      "logits/rejected": -0.5471930503845215,
+      "logps/chosen": -36.49355697631836,
+      "logps/rejected": -41.39311218261719,
+      "loss": 0.3301,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8302067518234253,
+      "rewards/margins": 0.9772886037826538,
+      "rewards/rejected": -1.8074952363967896,
+      "step": 105
+    },
+    {
+      "epoch": 1.2544378698224852,
+      "grad_norm": 28.680251994430197,
+      "learning_rate": 4.951326924670147e-07,
+      "logits/chosen": -0.7039244771003723,
+      "logits/rejected": -0.6648294925689697,
+      "logps/chosen": -34.96596145629883,
+      "logps/rejected": -54.543174743652344,
+      "loss": 0.2705,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4160834550857544,
+      "rewards/margins": 4.053628444671631,
+      "rewards/rejected": -4.469712257385254,
+      "step": 106
+    },
+    {
+      "epoch": 1.2662721893491125,
+      "grad_norm": 34.660972787041885,
+      "learning_rate": 4.948740381849879e-07,
+      "logits/chosen": -0.436050683259964,
+      "logits/rejected": -0.49883347749710083,
+      "logps/chosen": -42.65693664550781,
+      "logps/rejected": -48.37257766723633,
+      "loss": 0.2924,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.3416898846626282,
+      "rewards/margins": 2.6665897369384766,
+      "rewards/rejected": -3.00827956199646,
+      "step": 107
+    },
+    {
+      "epoch": 1.2781065088757395,
+      "grad_norm": 35.73754376882297,
+      "learning_rate": 4.94608759181358e-07,
+      "logits/chosen": -1.3829025030136108,
+      "logits/rejected": -1.270089864730835,
+      "logps/chosen": -42.553672790527344,
+      "logps/rejected": -68.03410339355469,
+      "loss": 0.3177,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.4059150815010071,
+      "rewards/margins": 2.707810878753662,
+      "rewards/rejected": -3.1137256622314453,
+      "step": 108
+    },
+    {
+      "epoch": 1.2899408284023668,
+      "grad_norm": 29.030913535413347,
+      "learning_rate": 4.943368626328741e-07,
+      "logits/chosen": -0.7719374895095825,
+      "logits/rejected": -0.8931103348731995,
+      "logps/chosen": -37.83287048339844,
+      "logps/rejected": -55.789669036865234,
+      "loss": 0.2498,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5285115838050842,
+      "rewards/margins": 3.6779208183288574,
+      "rewards/rejected": -4.206432819366455,
+      "step": 109
+    },
+    {
+      "epoch": 1.301775147928994,
+      "grad_norm": 32.18624732441535,
+      "learning_rate": 4.940583558953137e-07,
+      "logits/chosen": -0.8106911778450012,
+      "logits/rejected": -0.6266003847122192,
+      "logps/chosen": -27.738155364990234,
+      "logps/rejected": -53.347442626953125,
+      "loss": 0.2545,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.49206268787384033,
+      "rewards/margins": 3.2540321350097656,
+      "rewards/rejected": -3.7460951805114746,
+      "step": 110
+    },
+    {
+      "epoch": 1.3136094674556213,
+      "grad_norm": 29.81498338818304,
+      "learning_rate": 4.937732465032838e-07,
+      "logits/chosen": -0.9487054347991943,
+      "logits/rejected": -0.7686504125595093,
+      "logps/chosen": -37.48987579345703,
+      "logps/rejected": -61.35884475708008,
+      "loss": 0.2985,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.44103536009788513,
+      "rewards/margins": 2.5024337768554688,
+      "rewards/rejected": -2.9434690475463867,
+      "step": 111
+    },
+    {
+      "epoch": 1.3254437869822486,
+      "grad_norm": 31.489716700318738,
+      "learning_rate": 4.934815421700164e-07,
+      "logits/chosen": -0.6628305315971375,
+      "logits/rejected": -0.5205198526382446,
+      "logps/chosen": -38.25563430786133,
+      "logps/rejected": -62.12502670288086,
+      "loss": 0.2841,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.37647631764411926,
+      "rewards/margins": 2.712799549102783,
+      "rewards/rejected": -3.08927583694458,
+      "step": 112
+    },
+    {
+      "epoch": 1.3372781065088757,
+      "grad_norm": 26.849776060745437,
+      "learning_rate": 4.93183250787161e-07,
+      "logits/chosen": -0.8486281633377075,
+      "logits/rejected": -0.6799747347831726,
+      "logps/chosen": -36.62342834472656,
+      "logps/rejected": -51.13971710205078,
+      "loss": 0.2563,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.19857852160930634,
+      "rewards/margins": 2.1364779472351074,
+      "rewards/rejected": -2.3350563049316406,
+      "step": 113
+    },
+    {
+      "epoch": 1.349112426035503,
+      "grad_norm": 30.82600982658151,
+      "learning_rate": 4.928783804245699e-07,
+      "logits/chosen": -0.9438729286193848,
+      "logits/rejected": -1.0174190998077393,
+      "logps/chosen": -40.94575119018555,
+      "logps/rejected": -47.26662826538086,
+      "loss": 0.2841,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.4203495681285858,
+      "rewards/margins": 2.5113162994384766,
+      "rewards/rejected": -2.9316658973693848,
+      "step": 114
+    },
+    {
+      "epoch": 1.3609467455621302,
+      "grad_norm": 37.63814438854822,
+      "learning_rate": 4.925669393300807e-07,
+      "logits/chosen": -0.5685192942619324,
+      "logits/rejected": -0.6837583184242249,
+      "logps/chosen": -35.69676208496094,
+      "logps/rejected": -42.57113265991211,
+      "loss": 0.3434,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.2215765118598938,
+      "rewards/margins": 2.7080235481262207,
+      "rewards/rejected": -2.9296000003814697,
+      "step": 115
+    },
+    {
+      "epoch": 1.3727810650887573,
+      "grad_norm": 27.187951013233324,
+      "learning_rate": 4.922489359292927e-07,
+      "logits/chosen": -0.8806582689285278,
+      "logits/rejected": -0.8417138457298279,
+      "logps/chosen": -48.617950439453125,
+      "logps/rejected": -63.693763732910156,
+      "loss": 0.258,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.31812140345573425,
+      "rewards/margins": 3.5926265716552734,
+      "rewards/rejected": -3.910747766494751,
+      "step": 116
+    },
+    {
+      "epoch": 1.3846153846153846,
+      "grad_norm": 33.37578095279378,
+      "learning_rate": 4.919243788253393e-07,
+      "logits/chosen": -0.6644065380096436,
+      "logits/rejected": -0.8404392004013062,
+      "logps/chosen": -34.850852966308594,
+      "logps/rejected": -44.51880645751953,
+      "loss": 0.3009,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.36010777950286865,
+      "rewards/margins": 2.36444091796875,
+      "rewards/rejected": -2.724548816680908,
+      "step": 117
+    },
+    {
+      "epoch": 1.3964497041420119,
+      "grad_norm": 31.874390557999053,
+      "learning_rate": 4.915932767986551e-07,
+      "logits/chosen": -0.3398258686065674,
+      "logits/rejected": -0.35200411081314087,
+      "logps/chosen": -29.706180572509766,
+      "logps/rejected": -36.340633392333984,
+      "loss": 0.2804,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.17402225732803345,
+      "rewards/margins": 1.5369799137115479,
+      "rewards/rejected": -1.3629577159881592,
+      "step": 118
+    },
+    {
+      "epoch": 1.4082840236686391,
+      "grad_norm": 31.69981967422697,
+      "learning_rate": 4.912556388067381e-07,
+      "logits/chosen": -0.8166269063949585,
+      "logits/rejected": -0.867180347442627,
+      "logps/chosen": -31.888137817382812,
+      "logps/rejected": -43.59930419921875,
+      "loss": 0.2591,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.0748639851808548,
+      "rewards/margins": 1.9631797075271606,
+      "rewards/rejected": -2.038043737411499,
+      "step": 119
+    },
+    {
+      "epoch": 1.4201183431952662,
+      "grad_norm": 31.59528234355252,
+      "learning_rate": 4.909114739839079e-07,
+      "logits/chosen": -0.6452093124389648,
+      "logits/rejected": -0.5552669763565063,
+      "logps/chosen": -29.506380081176758,
+      "logps/rejected": -50.04014587402344,
+      "loss": 0.2972,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.0014930292963981628,
+      "rewards/margins": 2.9174089431762695,
+      "rewards/rejected": -2.9159157276153564,
+      "step": 120
+    },
+    {
+      "epoch": 1.4319526627218935,
+      "grad_norm": 29.665894281084412,
+      "learning_rate": 4.90560791641058e-07,
+      "logits/chosen": -0.9231401085853577,
+      "logits/rejected": -0.9149960279464722,
+      "logps/chosen": -39.093299865722656,
+      "logps/rejected": -58.45146942138672,
+      "loss": 0.2505,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.0397074818611145,
+      "rewards/margins": 3.4828078746795654,
+      "rewards/rejected": -3.4431004524230957,
+      "step": 121
+    },
+    {
+      "epoch": 1.4437869822485208,
+      "grad_norm": 26.842826013555836,
+      "learning_rate": 4.902036012654048e-07,
+      "logits/chosen": -0.743898332118988,
+      "logits/rejected": -0.85213303565979,
+      "logps/chosen": -37.486961364746094,
+      "logps/rejected": -45.735897064208984,
+      "loss": 0.2211,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.2370671182870865,
+      "rewards/margins": 2.456345558166504,
+      "rewards/rejected": -2.219278335571289,
+      "step": 122
+    },
+    {
+      "epoch": 1.4556213017751478,
+      "grad_norm": 33.17228544783056,
+      "learning_rate": 4.898399125202295e-07,
+      "logits/chosen": -0.6975520849227905,
+      "logits/rejected": -0.5711613893508911,
+      "logps/chosen": -34.24563217163086,
+      "logps/rejected": -54.74011993408203,
+      "loss": 0.2873,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.04336928948760033,
+      "rewards/margins": 2.063955307006836,
+      "rewards/rejected": -2.0205860137939453,
+      "step": 123
+    },
+    {
+      "epoch": 1.467455621301775,
+      "grad_norm": 23.661270990308882,
+      "learning_rate": 4.894697352446182e-07,
+      "logits/chosen": -1.0904819965362549,
+      "logits/rejected": -0.9323499798774719,
+      "logps/chosen": -32.2510986328125,
+      "logps/rejected": -49.083892822265625,
+      "loss": 0.2113,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.38007357716560364,
+      "rewards/margins": 2.8477623462677,
+      "rewards/rejected": -3.2278361320495605,
+      "step": 124
+    },
+    {
+      "epoch": 1.4792899408284024,
+      "grad_norm": 34.604687836314845,
+      "learning_rate": 4.890930794531947e-07,
+      "logits/chosen": -0.8184336423873901,
+      "logits/rejected": -0.7664436101913452,
+      "logps/chosen": -33.786197662353516,
+      "logps/rejected": -51.256500244140625,
+      "loss": 0.3319,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.057498496025800705,
+      "rewards/margins": 2.2278025150299072,
+      "rewards/rejected": -2.2853007316589355,
+      "step": 125
+    },
+    {
+      "epoch": 1.4911242603550297,
+      "grad_norm": 32.152626338805746,
+      "learning_rate": 4.887099553358501e-07,
+      "logits/chosen": -0.557933509349823,
+      "logits/rejected": -0.46904832124710083,
+      "logps/chosen": -32.54315948486328,
+      "logps/rejected": -43.39186477661133,
+      "loss": 0.2547,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3194792866706848,
+      "rewards/margins": 2.087386131286621,
+      "rewards/rejected": -1.767906665802002,
+      "step": 126
+    },
+    {
+      "epoch": 1.502958579881657,
+      "grad_norm": 27.486858140339642,
+      "learning_rate": 4.883203732574667e-07,
+      "logits/chosen": -0.6726119518280029,
+      "logits/rejected": -0.9127836227416992,
+      "logps/chosen": -37.70806884765625,
+      "logps/rejected": -49.78596878051758,
+      "loss": 0.2222,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.04751332104206085,
+      "rewards/margins": 4.030301094055176,
+      "rewards/rejected": -4.07781457901001,
+      "step": 127
+    },
+    {
+      "epoch": 1.514792899408284,
+      "grad_norm": 31.048864646685523,
+      "learning_rate": 4.879243437576383e-07,
+      "logits/chosen": -0.7582688927650452,
+      "logits/rejected": -0.5859317183494568,
+      "logps/chosen": -36.134395599365234,
+      "logps/rejected": -54.693885803222656,
+      "loss": 0.2645,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.12503023445606232,
+      "rewards/margins": 2.837843894958496,
+      "rewards/rejected": -2.712813377380371,
+      "step": 128
+    },
+    {
+      "epoch": 1.5266272189349113,
+      "grad_norm": 24.406363199351727,
+      "learning_rate": 4.875218775503837e-07,
+      "logits/chosen": -0.8559905290603638,
+      "logits/rejected": -0.5925788879394531,
+      "logps/chosen": -30.623798370361328,
+      "logps/rejected": -54.57696533203125,
+      "loss": 0.1985,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.09095414727926254,
+      "rewards/margins": 2.6515233516693115,
+      "rewards/rejected": -2.7424774169921875,
+      "step": 129
+    },
+    {
+      "epoch": 1.5384615384615383,
+      "grad_norm": 29.935002372070645,
+      "learning_rate": 4.871129855238588e-07,
+      "logits/chosen": -0.6436349749565125,
+      "logits/rejected": -0.6043447852134705,
+      "logps/chosen": -39.27113723754883,
+      "logps/rejected": -53.498077392578125,
+      "loss": 0.2637,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1115611121058464,
+      "rewards/margins": 2.6680796146392822,
+      "rewards/rejected": -2.7796406745910645,
+      "step": 130
+    },
+    {
+      "epoch": 1.5502958579881656,
+      "grad_norm": 34.141742145577254,
+      "learning_rate": 4.866976787400601e-07,
+      "logits/chosen": -0.6176570653915405,
+      "logits/rejected": -0.6334538459777832,
+      "logps/chosen": -32.522979736328125,
+      "logps/rejected": -41.17702865600586,
+      "loss": 0.3054,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.09118051826953888,
+      "rewards/margins": 1.9793744087219238,
+      "rewards/rejected": -1.8881936073303223,
+      "step": 131
+    },
+    {
+      "epoch": 1.5621301775147929,
+      "grad_norm": 32.31778466187239,
+      "learning_rate": 4.862759684345269e-07,
+      "logits/chosen": -0.8636021614074707,
+      "logits/rejected": -0.8571298122406006,
+      "logps/chosen": -27.641830444335938,
+      "logps/rejected": -47.43977737426758,
+      "loss": 0.2662,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.15791228413581848,
+      "rewards/margins": 3.5890026092529297,
+      "rewards/rejected": -3.746914863586426,
+      "step": 132
+    },
+    {
+      "epoch": 1.5739644970414202,
+      "grad_norm": 32.025278441483124,
+      "learning_rate": 4.858478660160363e-07,
+      "logits/chosen": -0.7859846353530884,
+      "logits/rejected": -0.855846643447876,
+      "logps/chosen": -42.2872314453125,
+      "logps/rejected": -59.02558135986328,
+      "loss": 0.2718,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.20162326097488403,
+      "rewards/margins": 4.159268856048584,
+      "rewards/rejected": -4.360892295837402,
+      "step": 133
+    },
+    {
+      "epoch": 1.5857988165680474,
+      "grad_norm": 31.88463706660638,
+      "learning_rate": 4.854133830662955e-07,
+      "logits/chosen": -1.0088858604431152,
+      "logits/rejected": -1.0345858335494995,
+      "logps/chosen": -36.01057052612305,
+      "logps/rejected": -40.89894104003906,
+      "loss": 0.2697,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.32271209359169006,
+      "rewards/margins": 1.9244492053985596,
+      "rewards/rejected": -2.247161388397217,
+      "step": 134
+    },
+    {
+      "epoch": 1.5976331360946747,
+      "grad_norm": 32.09028994814745,
+      "learning_rate": 4.849725313396274e-07,
+      "logits/chosen": -0.8029739856719971,
+      "logits/rejected": -0.668880820274353,
+      "logps/chosen": -31.962514877319336,
+      "logps/rejected": -56.866539001464844,
+      "loss": 0.2921,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.030226286500692368,
+      "rewards/margins": 3.690258026123047,
+      "rewards/rejected": -3.660031795501709,
+      "step": 135
+    },
+    {
+      "epoch": 1.6094674556213018,
+      "grad_norm": 25.510937214506086,
+      "learning_rate": 4.845253227626536e-07,
+      "logits/chosen": -1.067764401435852,
+      "logits/rejected": -1.126332402229309,
+      "logps/chosen": -29.248348236083984,
+      "logps/rejected": -38.563045501708984,
+      "loss": 0.221,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.5202600359916687,
+      "rewards/margins": 2.1755950450897217,
+      "rewards/rejected": -2.695855140686035,
+      "step": 136
+    },
+    {
+      "epoch": 1.6213017751479288,
+      "grad_norm": 35.45937914643292,
+      "learning_rate": 4.84071769433971e-07,
+      "logits/chosen": -0.9243749380111694,
+      "logits/rejected": -1.0414302349090576,
+      "logps/chosen": -43.77132797241211,
+      "logps/rejected": -46.58074951171875,
+      "loss": 0.2904,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.23414787650108337,
+      "rewards/margins": 2.354924201965332,
+      "rewards/rejected": -2.5890719890594482,
+      "step": 137
+    },
+    {
+      "epoch": 1.6331360946745561,
+      "grad_norm": 27.49463867525898,
+      "learning_rate": 4.836118836238252e-07,
+      "logits/chosen": -0.8824262619018555,
+      "logits/rejected": -0.8479146361351013,
+      "logps/chosen": -38.267608642578125,
+      "logps/rejected": -54.601646423339844,
+      "loss": 0.2142,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.2675169110298157,
+      "rewards/margins": 3.23266339302063,
+      "rewards/rejected": -3.500180244445801,
+      "step": 138
+    },
+    {
+      "epoch": 1.6449704142011834,
+      "grad_norm": 25.03352720952285,
+      "learning_rate": 4.831456777737779e-07,
+      "logits/chosen": -0.87726891040802,
+      "logits/rejected": -0.6373116374015808,
+      "logps/chosen": -37.56227111816406,
+      "logps/rejected": -62.3519287109375,
+      "loss": 0.2154,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.22444042563438416,
+      "rewards/margins": 2.3676767349243164,
+      "rewards/rejected": -2.5921175479888916,
+      "step": 139
+    },
+    {
+      "epoch": 1.6568047337278107,
+      "grad_norm": 30.77350000619414,
+      "learning_rate": 4.826731644963704e-07,
+      "logits/chosen": -0.7138141989707947,
+      "logits/rejected": -0.746364951133728,
+      "logps/chosen": -56.473121643066406,
+      "logps/rejected": -67.74131774902344,
+      "loss": 0.2472,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.1767007112503052,
+      "rewards/margins": 3.261204719543457,
+      "rewards/rejected": -4.437905311584473,
+      "step": 140
+    },
+    {
+      "epoch": 1.668639053254438,
+      "grad_norm": 28.0953853311355,
+      "learning_rate": 4.82194356574783e-07,
+      "logits/chosen": -1.0583032369613647,
+      "logits/rejected": -1.1049402952194214,
+      "logps/chosen": -34.80393981933594,
+      "logps/rejected": -44.68275451660156,
+      "loss": 0.2266,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5045955181121826,
+      "rewards/margins": 1.9453740119934082,
+      "rewards/rejected": -2.4499692916870117,
+      "step": 141
+    },
+    {
+      "epoch": 1.6804733727810652,
+      "grad_norm": 29.806685142730963,
+      "learning_rate": 4.817092669624882e-07,
+      "logits/chosen": -0.9740761518478394,
+      "logits/rejected": -0.8185986876487732,
+      "logps/chosen": -39.042179107666016,
+      "logps/rejected": -60.931121826171875,
+      "loss": 0.2278,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.528903067111969,
+      "rewards/margins": 3.3279364109039307,
+      "rewards/rejected": -3.856839656829834,
+      "step": 142
+    },
+    {
+      "epoch": 1.6923076923076923,
+      "grad_norm": 29.649829246510613,
+      "learning_rate": 4.812179087829012e-07,
+      "logits/chosen": -0.6173474788665771,
+      "logits/rejected": -0.5431764125823975,
+      "logps/chosen": -27.147438049316406,
+      "logps/rejected": -43.8935546875,
+      "loss": 0.2404,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.24039342999458313,
+      "rewards/margins": 2.5034737586975098,
+      "rewards/rejected": -2.7438669204711914,
+      "step": 143
+    },
+    {
+      "epoch": 1.7041420118343196,
+      "grad_norm": 30.07302253382677,
+      "learning_rate": 4.807202953290243e-07,
+      "logits/chosen": -0.523086428642273,
+      "logits/rejected": -0.5003029108047485,
+      "logps/chosen": -37.44425582885742,
+      "logps/rejected": -49.62798309326172,
+      "loss": 0.2756,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.09302498400211334,
+      "rewards/margins": 1.9907047748565674,
+      "rewards/rejected": -1.8976799249649048,
+      "step": 144
+    },
+    {
+      "epoch": 1.7159763313609466,
+      "grad_norm": 30.457439377224762,
+      "learning_rate": 4.802164400630872e-07,
+      "logits/chosen": -1.0345890522003174,
+      "logits/rejected": -1.0042998790740967,
+      "logps/chosen": -41.94634246826172,
+      "logps/rejected": -54.282379150390625,
+      "loss": 0.2416,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.555735170841217,
+      "rewards/margins": 3.464998960494995,
+      "rewards/rejected": -4.020733833312988,
+      "step": 145
+    },
+    {
+      "epoch": 1.727810650887574,
+      "grad_norm": 29.365588226716095,
+      "learning_rate": 4.797063566161834e-07,
+      "logits/chosen": -0.5482521653175354,
+      "logits/rejected": -0.8150917291641235,
+      "logps/chosen": -45.255279541015625,
+      "logps/rejected": -44.48121643066406,
+      "loss": 0.2289,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.004989638924598694,
+      "rewards/margins": 2.8408713340759277,
+      "rewards/rejected": -2.845860719680786,
+      "step": 146
+    },
+    {
+      "epoch": 1.7396449704142012,
+      "grad_norm": 23.051187654817465,
+      "learning_rate": 4.791900587879009e-07,
+      "logits/chosen": -1.079697608947754,
+      "logits/rejected": -1.1642544269561768,
+      "logps/chosen": -35.86004638671875,
+      "logps/rejected": -55.203956604003906,
+      "loss": 0.2049,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.20117734372615814,
+      "rewards/margins": 4.757822513580322,
+      "rewards/rejected": -4.959000110626221,
+      "step": 147
+    },
+    {
+      "epoch": 1.7514792899408285,
+      "grad_norm": 23.873165296629157,
+      "learning_rate": 4.786675605459487e-07,
+      "logits/chosen": -0.6171930432319641,
+      "logits/rejected": -0.7254853844642639,
+      "logps/chosen": -37.1898193359375,
+      "logps/rejected": -58.64811706542969,
+      "loss": 0.1909,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.25659316778182983,
+      "rewards/margins": 4.113015174865723,
+      "rewards/rejected": -3.856421947479248,
+      "step": 148
+    },
+    {
+      "epoch": 1.7633136094674557,
+      "grad_norm": 28.365749172580184,
+      "learning_rate": 4.781388760257799e-07,
+      "logits/chosen": -0.6407098174095154,
+      "logits/rejected": -0.5406571626663208,
+      "logps/chosen": -32.85944366455078,
+      "logps/rejected": -45.028846740722656,
+      "loss": 0.2134,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.13824699819087982,
+      "rewards/margins": 1.9902299642562866,
+      "rewards/rejected": -1.8519829511642456,
+      "step": 149
+    },
+    {
+      "epoch": 1.7751479289940828,
+      "grad_norm": 26.001580942778393,
+      "learning_rate": 4.776040195302079e-07,
+      "logits/chosen": -0.7967870235443115,
+      "logits/rejected": -0.925701916217804,
+      "logps/chosen": -34.21118927001953,
+      "logps/rejected": -47.52817916870117,
+      "loss": 0.1779,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.07275627553462982,
+      "rewards/margins": 3.335747003555298,
+      "rewards/rejected": -3.262990951538086,
+      "step": 150
+    },
+    {
+      "epoch": 1.78698224852071,
+      "grad_norm": 31.836026789786995,
+      "learning_rate": 4.770630055290208e-07,
+      "logits/chosen": -1.0263866186141968,
+      "logits/rejected": -0.8518810868263245,
+      "logps/chosen": -40.71949768066406,
+      "logps/rejected": -57.82533264160156,
+      "loss": 0.2399,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.18406735360622406,
+      "rewards/margins": 2.9664227962493896,
+      "rewards/rejected": -3.1504902839660645,
+      "step": 151
+    },
+    {
+      "epoch": 1.7988165680473371,
+      "grad_norm": 28.63311269352365,
+      "learning_rate": 4.76515848658589e-07,
+      "logits/chosen": -0.5594848394393921,
+      "logits/rejected": -0.8863150477409363,
+      "logps/chosen": -45.70458221435547,
+      "logps/rejected": -38.8409538269043,
+      "loss": 0.2137,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.2182983160018921,
+      "rewards/margins": 2.2750673294067383,
+      "rewards/rejected": -2.4933652877807617,
+      "step": 152
+    },
+    {
+      "epoch": 1.8106508875739644,
+      "grad_norm": 34.34513187647427,
+      "learning_rate": 4.759625637214696e-07,
+      "logits/chosen": -0.7560346126556396,
+      "logits/rejected": -0.8882582187652588,
+      "logps/chosen": -30.11992073059082,
+      "logps/rejected": -39.5860595703125,
+      "loss": 0.2751,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3838905096054077,
+      "rewards/margins": 1.65618097782135,
+      "rewards/rejected": -2.040071487426758,
+      "step": 153
+    },
+    {
+      "epoch": 1.8224852071005917,
+      "grad_norm": 32.10962096929911,
+      "learning_rate": 4.754031656860059e-07,
+      "logits/chosen": -0.8416492342948914,
+      "logits/rejected": -0.6819140911102295,
+      "logps/chosen": -33.661521911621094,
+      "logps/rejected": -44.78226852416992,
+      "loss": 0.2833,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.014047026634216309,
+      "rewards/margins": 2.2703661918640137,
+      "rewards/rejected": -2.256319046020508,
+      "step": 154
+    },
+    {
+      "epoch": 1.834319526627219,
+      "grad_norm": 30.000217414301275,
+      "learning_rate": 4.748376696859226e-07,
+      "logits/chosen": -0.9286520481109619,
+      "logits/rejected": -0.9778493642807007,
+      "logps/chosen": -41.20292663574219,
+      "logps/rejected": -48.67192077636719,
+      "loss": 0.2596,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.4870182275772095,
+      "rewards/margins": 2.4510464668273926,
+      "rewards/rejected": -1.9640284776687622,
+      "step": 155
+    },
+    {
+      "epoch": 1.8461538461538463,
+      "grad_norm": 32.4118395869003,
+      "learning_rate": 4.74266091019916e-07,
+      "logits/chosen": -0.8907891511917114,
+      "logits/rejected": -0.9750989079475403,
+      "logps/chosen": -45.65682601928711,
+      "logps/rejected": -50.134220123291016,
+      "loss": 0.2678,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.07820607721805573,
+      "rewards/margins": 3.1200551986694336,
+      "rewards/rejected": -3.198261260986328,
+      "step": 156
+    },
+    {
+      "epoch": 1.8579881656804735,
+      "grad_norm": 28.928479525804008,
+      "learning_rate": 4.7368844515124046e-07,
+      "logits/chosen": -0.6818917393684387,
+      "logits/rejected": -0.8876403570175171,
+      "logps/chosen": -38.4929084777832,
+      "logps/rejected": -44.042869567871094,
+      "loss": 0.2156,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.11902601271867752,
+      "rewards/margins": 3.1182360649108887,
+      "rewards/rejected": -2.9992103576660156,
+      "step": 157
+    },
+    {
+      "epoch": 1.8698224852071006,
+      "grad_norm": 29.65958384914858,
+      "learning_rate": 4.7310474770728996e-07,
+      "logits/chosen": -0.6357210278511047,
+      "logits/rejected": -0.5424870252609253,
+      "logps/chosen": -37.36800765991211,
+      "logps/rejected": -58.64324951171875,
+      "loss": 0.2376,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.6739649176597595,
+      "rewards/margins": 3.992888927459717,
+      "rewards/rejected": -4.666853904724121,
+      "step": 158
+    },
+    {
+      "epoch": 1.8816568047337277,
+      "grad_norm": 34.26684990746155,
+      "learning_rate": 4.725150144791753e-07,
+      "logits/chosen": -0.46081531047821045,
+      "logits/rejected": -0.5821120142936707,
+      "logps/chosen": -37.67076110839844,
+      "logps/rejected": -41.890037536621094,
+      "loss": 0.3107,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.4266676902770996,
+      "rewards/margins": 1.9253289699554443,
+      "rewards/rejected": -2.351996660232544,
+      "step": 159
+    },
+    {
+      "epoch": 1.893491124260355,
+      "grad_norm": 35.05902021201059,
+      "learning_rate": 4.719192614212969e-07,
+      "logits/chosen": -0.6496328711509705,
+      "logits/rejected": -0.5829223990440369,
+      "logps/chosen": -36.085716247558594,
+      "logps/rejected": -51.334327697753906,
+      "loss": 0.2594,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.33468520641326904,
+      "rewards/margins": 2.861034631729126,
+      "rewards/rejected": -3.1957197189331055,
+      "step": 160
+    },
+    {
+      "epoch": 1.9053254437869822,
+      "grad_norm": 30.04841620365748,
+      "learning_rate": 4.713175046509131e-07,
+      "logits/chosen": -0.9227581024169922,
+      "logits/rejected": -0.8245525360107422,
+      "logps/chosen": -39.57933807373047,
+      "logps/rejected": -66.19513702392578,
+      "loss": 0.23,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.03423337638378143,
+      "rewards/margins": 5.112496376037598,
+      "rewards/rejected": -5.146729469299316,
+      "step": 161
+    },
+    {
+      "epoch": 1.9171597633136095,
+      "grad_norm": 31.04728022665797,
+      "learning_rate": 4.707097604477045e-07,
+      "logits/chosen": -1.0367375612258911,
+      "logits/rejected": -1.0377088785171509,
+      "logps/chosen": -46.939781188964844,
+      "logps/rejected": -56.965232849121094,
+      "loss": 0.2494,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.24659229815006256,
+      "rewards/margins": 2.464108467102051,
+      "rewards/rejected": -2.710700750350952,
+      "step": 162
+    },
+    {
+      "epoch": 1.9289940828402368,
+      "grad_norm": 28.85191727468816,
+      "learning_rate": 4.700960452533328e-07,
+      "logits/chosen": -0.5171566605567932,
+      "logits/rejected": -0.5847671627998352,
+      "logps/chosen": -41.256561279296875,
+      "logps/rejected": -48.218727111816406,
+      "loss": 0.2072,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.48802250623703003,
+      "rewards/margins": 2.6758077144622803,
+      "rewards/rejected": -3.163830280303955,
+      "step": 163
+    },
+    {
+      "epoch": 1.940828402366864,
+      "grad_norm": 34.99511678470972,
+      "learning_rate": 4.694763756709967e-07,
+      "logits/chosen": -0.9630662202835083,
+      "logits/rejected": -1.0143767595291138,
+      "logps/chosen": -33.528255462646484,
+      "logps/rejected": -41.041446685791016,
+      "loss": 0.2857,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.00032395869493484497,
+      "rewards/margins": 2.951721429824829,
+      "rewards/rejected": -2.952045440673828,
+      "step": 164
+    },
+    {
+      "epoch": 1.952662721893491,
+      "grad_norm": 32.6916182628487,
+      "learning_rate": 4.688507684649825e-07,
+      "logits/chosen": -0.5867289304733276,
+      "logits/rejected": -0.6317815184593201,
+      "logps/chosen": -34.935386657714844,
+      "logps/rejected": -50.50403594970703,
+      "loss": 0.299,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.3650425374507904,
+      "rewards/margins": 3.305384635925293,
+      "rewards/rejected": -2.9403419494628906,
+      "step": 165
+    },
+    {
+      "epoch": 1.9644970414201184,
+      "grad_norm": 25.572043392572215,
+      "learning_rate": 4.6821924056021053e-07,
+      "logits/chosen": -0.8966530561447144,
+      "logits/rejected": -0.9081934094429016,
+      "logps/chosen": -39.484596252441406,
+      "logps/rejected": -59.22516632080078,
+      "loss": 0.2167,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0013990327715873718,
+      "rewards/margins": 4.235882759094238,
+      "rewards/rejected": -4.237281799316406,
+      "step": 166
+    },
+    {
+      "epoch": 1.9763313609467454,
+      "grad_norm": 33.32850848147448,
+      "learning_rate": 4.6758180904177715e-07,
+      "logits/chosen": -0.9043580293655396,
+      "logits/rejected": -0.9197327494621277,
+      "logps/chosen": -30.749338150024414,
+      "logps/rejected": -46.336402893066406,
+      "loss": 0.2978,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.061946846544742584,
+      "rewards/margins": 3.1583476066589355,
+      "rewards/rejected": -3.0964009761810303,
+      "step": 167
+    },
+    {
+      "epoch": 1.9881656804733727,
+      "grad_norm": 30.12339676116848,
+      "learning_rate": 4.669384911544926e-07,
+      "logits/chosen": -0.7334628701210022,
+      "logits/rejected": -0.6194922924041748,
+      "logps/chosen": -33.2913818359375,
+      "logps/rejected": -58.153072357177734,
+      "loss": 0.2359,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.35717520117759705,
+      "rewards/margins": 3.8730297088623047,
+      "rewards/rejected": -4.230205059051514,
+      "step": 168
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 21.4554135691754,
+      "learning_rate": 4.6628930430241495e-07,
+      "logits/chosen": -0.5781211853027344,
+      "logits/rejected": -0.6272490620613098,
+      "logps/chosen": -33.10559844970703,
+      "logps/rejected": -47.765830993652344,
+      "loss": 0.1717,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.07726180553436279,
+      "rewards/margins": 3.996021270751953,
+      "rewards/rejected": -4.0732831954956055,
+      "step": 169
+    },
+    {
+      "epoch": 2.0118343195266273,
+      "grad_norm": 17.804389758687186,
+      "learning_rate": 4.6563426604837817e-07,
+      "logits/chosen": -0.7812290787696838,
+      "logits/rejected": -0.6464096307754517,
+      "logps/chosen": -37.48862838745117,
+      "logps/rejected": -57.36015319824219,
+      "loss": 0.1516,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.08392468094825745,
+      "rewards/margins": 3.2438626289367676,
+      "rewards/rejected": -3.327787160873413,
+      "step": 170
+    },
+    {
+      "epoch": 2.0236686390532546,
+      "grad_norm": 16.40347419407978,
+      "learning_rate": 4.649733941135183e-07,
+      "logits/chosen": -0.9235811233520508,
+      "logits/rejected": -0.966956615447998,
+      "logps/chosen": -46.7747802734375,
+      "logps/rejected": -61.03520584106445,
+      "loss": 0.1229,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.24425256252288818,
+      "rewards/margins": 3.003178834915161,
+      "rewards/rejected": -3.2474312782287598,
+      "step": 171
+    },
+    {
+      "epoch": 2.035502958579882,
+      "grad_norm": 17.315280346491143,
+      "learning_rate": 4.6430670637679294e-07,
+      "logits/chosen": -0.8347846865653992,
+      "logits/rejected": -0.8632474541664124,
+      "logps/chosen": -40.783226013183594,
+      "logps/rejected": -54.76463317871094,
+      "loss": 0.1394,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.18245816230773926,
+      "rewards/margins": 3.924067497253418,
+      "rewards/rejected": -4.106525421142578,
+      "step": 172
+    },
+    {
+      "epoch": 2.0473372781065087,
+      "grad_norm": 17.457800428549817,
+      "learning_rate": 4.636342208744981e-07,
+      "logits/chosen": -0.6723726987838745,
+      "logits/rejected": -0.7173234820365906,
+      "logps/chosen": -28.166179656982422,
+      "logps/rejected": -44.342166900634766,
+      "loss": 0.1436,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3225460350513458,
+      "rewards/margins": 3.5667474269866943,
+      "rewards/rejected": -3.244201421737671,
+      "step": 173
+    },
+    {
+      "epoch": 2.059171597633136,
+      "grad_norm": 15.061594900995148,
+      "learning_rate": 4.629559557997804e-07,
+      "logits/chosen": -0.6233261823654175,
+      "logits/rejected": -0.7046049237251282,
+      "logps/chosen": -51.49015426635742,
+      "logps/rejected": -72.83427429199219,
+      "loss": 0.1249,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3748975396156311,
+      "rewards/margins": 4.756686210632324,
+      "rewards/rejected": -5.1315836906433105,
+      "step": 174
+    },
+    {
+      "epoch": 2.0710059171597632,
+      "grad_norm": 15.61277777512069,
+      "learning_rate": 4.6227192950214435e-07,
+      "logits/chosen": -0.8263496160507202,
+      "logits/rejected": -0.7221701145172119,
+      "logps/chosen": -37.889495849609375,
+      "logps/rejected": -54.355918884277344,
+      "loss": 0.1236,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.1404503583908081,
+      "rewards/margins": 3.4195923805236816,
+      "rewards/rejected": -3.279142379760742,
+      "step": 175
+    },
+    {
+      "epoch": 2.0828402366863905,
+      "grad_norm": 15.2773228213431,
+      "learning_rate": 4.615821604869563e-07,
+      "logits/chosen": -0.9737514853477478,
+      "logits/rejected": -0.8237433433532715,
+      "logps/chosen": -39.199546813964844,
+      "logps/rejected": -62.34404754638672,
+      "loss": 0.111,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.041255027055740356,
+      "rewards/margins": 3.107170581817627,
+      "rewards/rejected": -3.148425579071045,
+      "step": 176
+    },
+    {
+      "epoch": 2.094674556213018,
+      "grad_norm": 15.829900533787184,
+      "learning_rate": 4.6088666741494384e-07,
+      "logits/chosen": -1.1444345712661743,
+      "logits/rejected": -1.1528784036636353,
+      "logps/chosen": -38.0962028503418,
+      "logps/rejected": -62.006046295166016,
+      "loss": 0.1109,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.7900293469429016,
+      "rewards/margins": 4.543603420257568,
+      "rewards/rejected": -5.3336334228515625,
+      "step": 177
+    },
+    {
+      "epoch": 2.106508875739645,
+      "grad_norm": 16.116982532695665,
+      "learning_rate": 4.6018546910169067e-07,
+      "logits/chosen": -1.0515778064727783,
+      "logits/rejected": -0.9833648800849915,
+      "logps/chosen": -40.967002868652344,
+      "logps/rejected": -65.30772399902344,
+      "loss": 0.1273,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.15716314315795898,
+      "rewards/margins": 3.4091856479644775,
+      "rewards/rejected": -3.5663485527038574,
+      "step": 178
+    },
+    {
+      "epoch": 2.1183431952662723,
+      "grad_norm": 15.361490349201445,
+      "learning_rate": 4.5947858451712773e-07,
+      "logits/chosen": -0.819975733757019,
+      "logits/rejected": -0.8770939111709595,
+      "logps/chosen": -33.2042236328125,
+      "logps/rejected": -50.4263916015625,
+      "loss": 0.1302,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.4746937155723572,
+      "rewards/margins": 4.132750511169434,
+      "rewards/rejected": -3.6580564975738525,
+      "step": 179
+    },
+    {
+      "epoch": 2.1301775147928996,
+      "grad_norm": 14.5720865184576,
+      "learning_rate": 4.5876603278502027e-07,
+      "logits/chosen": -0.7717497944831848,
+      "logits/rejected": -0.7804837822914124,
+      "logps/chosen": -43.37432861328125,
+      "logps/rejected": -59.71336364746094,
+      "loss": 0.1184,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3754761517047882,
+      "rewards/margins": 3.8695340156555176,
+      "rewards/rejected": -4.2450103759765625,
+      "step": 180
+    },
+    {
+      "epoch": 2.1420118343195265,
+      "grad_norm": 17.24169816378187,
+      "learning_rate": 4.580478331824498e-07,
+      "logits/chosen": -0.7291906476020813,
+      "logits/rejected": -0.7426820993423462,
+      "logps/chosen": -35.4974365234375,
+      "logps/rejected": -49.904014587402344,
+      "loss": 0.1635,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2558782994747162,
+      "rewards/margins": 2.3701655864715576,
+      "rewards/rejected": -2.1142873764038086,
+      "step": 181
+    },
+    {
+      "epoch": 2.1538461538461537,
+      "grad_norm": 17.93219012330719,
+      "learning_rate": 4.573240051392935e-07,
+      "logits/chosen": -0.6299488544464111,
+      "logits/rejected": -0.5285680294036865,
+      "logps/chosen": -27.371360778808594,
+      "logps/rejected": -41.3109245300293,
+      "loss": 0.1323,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.1316337138414383,
+      "rewards/margins": 2.6835739612579346,
+      "rewards/rejected": -2.8152074813842773,
+      "step": 182
+    },
+    {
+      "epoch": 2.165680473372781,
+      "grad_norm": 19.89980345891806,
+      "learning_rate": 4.565945682376977e-07,
+      "logits/chosen": -0.9120879173278809,
+      "logits/rejected": -0.6737322211265564,
+      "logps/chosen": -43.5380859375,
+      "logps/rejected": -75.94633483886719,
+      "loss": 0.1551,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.55577552318573,
+      "rewards/margins": 4.484833717346191,
+      "rewards/rejected": -5.040609359741211,
+      "step": 183
+    },
+    {
+      "epoch": 2.1775147928994083,
+      "grad_norm": 16.09665942372561,
+      "learning_rate": 4.5585954221154853e-07,
+      "logits/chosen": -0.8184518814086914,
+      "logits/rejected": -0.7758468985557556,
+      "logps/chosen": -30.036758422851562,
+      "logps/rejected": -45.972900390625,
+      "loss": 0.1244,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.20884810388088226,
+      "rewards/margins": 3.8934361934661865,
+      "rewards/rejected": -3.6845881938934326,
+      "step": 184
+    },
+    {
+      "epoch": 2.1893491124260356,
+      "grad_norm": 16.783451535199216,
+      "learning_rate": 4.551189469459382e-07,
+      "logits/chosen": -0.9438289403915405,
+      "logits/rejected": -0.7987072467803955,
+      "logps/chosen": -31.17508888244629,
+      "logps/rejected": -50.338836669921875,
+      "loss": 0.1361,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.023317746818065643,
+      "rewards/margins": 2.8164429664611816,
+      "rewards/rejected": -2.8397610187530518,
+      "step": 185
+    },
+    {
+      "epoch": 2.201183431952663,
+      "grad_norm": 20.143036888697704,
+      "learning_rate": 4.5437280247662646e-07,
+      "logits/chosen": -0.7536525130271912,
+      "logits/rejected": -0.855215311050415,
+      "logps/chosen": -54.403778076171875,
+      "logps/rejected": -66.35308074951172,
+      "loss": 0.1466,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.0882877111434937,
+      "rewards/margins": 4.7153778076171875,
+      "rewards/rejected": -5.803666114807129,
+      "step": 186
+    },
+    {
+      "epoch": 2.21301775147929,
+      "grad_norm": 14.808692042518373,
+      "learning_rate": 4.5362112898949947e-07,
+      "logits/chosen": -0.48705536127090454,
+      "logits/rejected": -0.572258710861206,
+      "logps/chosen": -37.71576690673828,
+      "logps/rejected": -45.96680450439453,
+      "loss": 0.1157,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.4106942117214203,
+      "rewards/margins": 4.168295860290527,
+      "rewards/rejected": -4.5789899826049805,
+      "step": 187
+    },
+    {
+      "epoch": 2.224852071005917,
+      "grad_norm": 18.376017289584023,
+      "learning_rate": 4.528639468200226e-07,
+      "logits/chosen": -0.9109969735145569,
+      "logits/rejected": -0.9377925992012024,
+      "logps/chosen": -35.95753479003906,
+      "logps/rejected": -53.1741943359375,
+      "loss": 0.1315,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.26368990540504456,
+      "rewards/margins": 4.901925563812256,
+      "rewards/rejected": -5.165615558624268,
+      "step": 188
+    },
+    {
+      "epoch": 2.2366863905325443,
+      "grad_norm": 17.965404516840803,
+      "learning_rate": 4.5210127645269125e-07,
+      "logits/chosen": -0.7325922846794128,
+      "logits/rejected": -0.7497273683547974,
+      "logps/chosen": -32.5103759765625,
+      "logps/rejected": -51.267616271972656,
+      "loss": 0.1185,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3522591292858124,
+      "rewards/margins": 4.243406295776367,
+      "rewards/rejected": -4.595664978027344,
+      "step": 189
+    },
+    {
+      "epoch": 2.2485207100591715,
+      "grad_norm": 16.570545177365638,
+      "learning_rate": 4.5133313852047613e-07,
+      "logits/chosen": -0.351344496011734,
+      "logits/rejected": -0.3544999659061432,
+      "logps/chosen": -30.0961971282959,
+      "logps/rejected": -47.018310546875,
+      "loss": 0.1119,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.17489728331565857,
+      "rewards/margins": 3.530579090118408,
+      "rewards/rejected": -3.7054765224456787,
+      "step": 190
+    },
+    {
+      "epoch": 2.260355029585799,
+      "grad_norm": 15.819997196077969,
+      "learning_rate": 4.5055955380426514e-07,
+      "logits/chosen": -0.6617011427879333,
+      "logits/rejected": -0.6919267773628235,
+      "logps/chosen": -31.14508819580078,
+      "logps/rejected": -49.937660217285156,
+      "loss": 0.1371,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.010032139718532562,
+      "rewards/margins": 4.074734687805176,
+      "rewards/rejected": -4.06470251083374,
+      "step": 191
+    },
+    {
+      "epoch": 2.272189349112426,
+      "grad_norm": 14.990393051749372,
+      "learning_rate": 4.4978054323230144e-07,
+      "logits/chosen": -0.8150711059570312,
+      "logits/rejected": -0.8408107757568359,
+      "logps/chosen": -27.763072967529297,
+      "logps/rejected": -43.43778991699219,
+      "loss": 0.1182,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2238546460866928,
+      "rewards/margins": 3.774090051651001,
+      "rewards/rejected": -3.5502355098724365,
+      "step": 192
+    },
+    {
+      "epoch": 2.2840236686390534,
+      "grad_norm": 16.48968206240205,
+      "learning_rate": 4.489961278796167e-07,
+      "logits/chosen": -1.1060757637023926,
+      "logits/rejected": -0.9984903335571289,
+      "logps/chosen": -39.745208740234375,
+      "logps/rejected": -61.31498718261719,
+      "loss": 0.1238,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.17969991266727448,
+      "rewards/margins": 4.915201663970947,
+      "rewards/rejected": -5.0949015617370605,
+      "step": 193
+    },
+    {
+      "epoch": 2.2958579881656807,
+      "grad_norm": 19.190560048976156,
+      "learning_rate": 4.482063289674618e-07,
+      "logits/chosen": -0.8471282720565796,
+      "logits/rejected": -0.7247289419174194,
+      "logps/chosen": -32.47812271118164,
+      "logps/rejected": -54.26726150512695,
+      "loss": 0.1459,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.25966179370880127,
+      "rewards/margins": 3.339050531387329,
+      "rewards/rejected": -3.0793888568878174,
+      "step": 194
+    },
+    {
+      "epoch": 2.3076923076923075,
+      "grad_norm": 14.39242536769752,
+      "learning_rate": 4.4741116786273176e-07,
+      "logits/chosen": -0.8266146779060364,
+      "logits/rejected": -0.815448522567749,
+      "logps/chosen": -34.32430648803711,
+      "logps/rejected": -54.35994338989258,
+      "loss": 0.1023,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.08938950300216675,
+      "rewards/margins": 4.4216461181640625,
+      "rewards/rejected": -4.511035919189453,
+      "step": 195
+    },
+    {
+      "epoch": 2.3195266272189348,
+      "grad_norm": 15.55346828770515,
+      "learning_rate": 4.466106660773884e-07,
+      "logits/chosen": -0.7416298389434814,
+      "logits/rejected": -0.7652521133422852,
+      "logps/chosen": -38.31879425048828,
+      "logps/rejected": -59.721343994140625,
+      "loss": 0.103,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3820110857486725,
+      "rewards/margins": 4.027256488800049,
+      "rewards/rejected": -3.645245313644409,
+      "step": 196
+    },
+    {
+      "epoch": 2.331360946745562,
+      "grad_norm": 20.33506239450445,
+      "learning_rate": 4.4580484526787807e-07,
+      "logits/chosen": -0.9200114011764526,
+      "logits/rejected": -0.8991610407829285,
+      "logps/chosen": -34.80711364746094,
+      "logps/rejected": -54.14820861816406,
+      "loss": 0.1412,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4271819293498993,
+      "rewards/margins": 4.234493732452393,
+      "rewards/rejected": -4.661675930023193,
+      "step": 197
+    },
+    {
+      "epoch": 2.3431952662721893,
+      "grad_norm": 16.966597309776112,
+      "learning_rate": 4.44993727234546e-07,
+      "logits/chosen": -0.9545549154281616,
+      "logits/rejected": -0.9994832277297974,
+      "logps/chosen": -41.37195587158203,
+      "logps/rejected": -53.36943817138672,
+      "loss": 0.1497,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6568608283996582,
+      "rewards/margins": 4.235930442810059,
+      "rewards/rejected": -4.892791748046875,
+      "step": 198
+    },
+    {
+      "epoch": 2.3550295857988166,
+      "grad_norm": 16.008353794022174,
+      "learning_rate": 4.4417733392104585e-07,
+      "logits/chosen": -0.9546022415161133,
+      "logits/rejected": -1.0305328369140625,
+      "logps/chosen": -37.549339294433594,
+      "logps/rejected": -52.69127655029297,
+      "loss": 0.1028,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.14469043910503387,
+      "rewards/margins": 4.475038528442383,
+      "rewards/rejected": -4.619729042053223,
+      "step": 199
+    },
+    {
+      "epoch": 2.366863905325444,
+      "grad_norm": 12.501500717461248,
+      "learning_rate": 4.4335568741374695e-07,
+      "logits/chosen": -0.9863907098770142,
+      "logits/rejected": -0.9025635123252869,
+      "logps/chosen": -31.974454879760742,
+      "logps/rejected": -47.34270477294922,
+      "loss": 0.0902,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.17588046193122864,
+      "rewards/margins": 4.036709308624268,
+      "rewards/rejected": -4.212589740753174,
+      "step": 200
+    },
+    {
+      "epoch": 2.378698224852071,
+      "grad_norm": 12.198066199001053,
+      "learning_rate": 4.425288099411364e-07,
+      "logits/chosen": -1.135861873626709,
+      "logits/rejected": -1.1184909343719482,
+      "logps/chosen": -42.55821228027344,
+      "logps/rejected": -58.36648178100586,
+      "loss": 0.077,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.1261935532093048,
+      "rewards/margins": 6.448820114135742,
+      "rewards/rejected": -6.575014591217041,
+      "step": 201
+    },
+    {
+      "epoch": 2.390532544378698,
+      "grad_norm": 18.343824652909003,
+      "learning_rate": 4.4169672387321735e-07,
+      "logits/chosen": -0.7927027344703674,
+      "logits/rejected": -0.8605716228485107,
+      "logps/chosen": -44.98955535888672,
+      "logps/rejected": -52.32347106933594,
+      "loss": 0.1207,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.08800630271434784,
+      "rewards/margins": 4.026900768280029,
+      "rewards/rejected": -4.1149067878723145,
+      "step": 202
+    },
+    {
+      "epoch": 2.4023668639053253,
+      "grad_norm": 14.754318186294096,
+      "learning_rate": 4.408594517209045e-07,
+      "logits/chosen": -0.9587286710739136,
+      "logits/rejected": -0.9846871495246887,
+      "logps/chosen": -34.01704025268555,
+      "logps/rejected": -53.6927375793457,
+      "loss": 0.108,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4293210208415985,
+      "rewards/margins": 5.7534356117248535,
+      "rewards/rejected": -6.182756423950195,
+      "step": 203
+    },
+    {
+      "epoch": 2.4142011834319526,
+      "grad_norm": 16.259714501612795,
+      "learning_rate": 4.4001701613541454e-07,
+      "logits/chosen": -0.8172876834869385,
+      "logits/rejected": -1.0052968263626099,
+      "logps/chosen": -50.09556198120117,
+      "logps/rejected": -48.03728103637695,
+      "loss": 0.1187,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.36984017491340637,
+      "rewards/margins": 3.9901065826416016,
+      "rewards/rejected": -4.3599467277526855,
+      "step": 204
+    },
+    {
+      "epoch": 2.42603550295858,
+      "grad_norm": 19.017744132865513,
+      "learning_rate": 4.391694399076536e-07,
+      "logits/chosen": -0.9216486215591431,
+      "logits/rejected": -0.8954623937606812,
+      "logps/chosen": -29.45151710510254,
+      "logps/rejected": -53.07656478881836,
+      "loss": 0.131,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.30826300382614136,
+      "rewards/margins": 5.172004699707031,
+      "rewards/rejected": -5.4802680015563965,
+      "step": 205
+    },
+    {
+      "epoch": 2.437869822485207,
+      "grad_norm": 16.95344970048546,
+      "learning_rate": 4.383167459676008e-07,
+      "logits/chosen": -0.8062804341316223,
+      "logits/rejected": -0.5907716155052185,
+      "logps/chosen": -37.306583404541016,
+      "logps/rejected": -65.82327270507812,
+      "loss": 0.1077,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.03246502950787544,
+      "rewards/margins": 3.6565213203430176,
+      "rewards/rejected": -3.624056100845337,
+      "step": 206
+    },
+    {
+      "epoch": 2.4497041420118344,
+      "grad_norm": 14.56061263170088,
+      "learning_rate": 4.374589573836874e-07,
+      "logits/chosen": -0.9072690606117249,
+      "logits/rejected": -1.0106024742126465,
+      "logps/chosen": -39.9183349609375,
+      "logps/rejected": -51.78164291381836,
+      "loss": 0.0865,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.8933297395706177,
+      "rewards/margins": 4.566455841064453,
+      "rewards/rejected": -5.459786415100098,
+      "step": 207
+    },
+    {
+      "epoch": 2.4615384615384617,
+      "grad_norm": 14.148619010746572,
+      "learning_rate": 4.365960973621734e-07,
+      "logits/chosen": -0.9763280749320984,
+      "logits/rejected": -1.2256582975387573,
+      "logps/chosen": -43.78369903564453,
+      "logps/rejected": -54.03715896606445,
+      "loss": 0.1012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2019457072019577,
+      "rewards/margins": 4.386034965515137,
+      "rewards/rejected": -4.587980270385742,
+      "step": 208
+    },
+    {
+      "epoch": 2.4733727810650885,
+      "grad_norm": 11.952569068353256,
+      "learning_rate": 4.357281892465191e-07,
+      "logits/chosen": -0.850165069103241,
+      "logits/rejected": -0.9085839986801147,
+      "logps/chosen": -30.494304656982422,
+      "logps/rejected": -45.906429290771484,
+      "loss": 0.0831,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.06118796020746231,
+      "rewards/margins": 4.031259536743164,
+      "rewards/rejected": -3.970071792602539,
+      "step": 209
+    },
+    {
+      "epoch": 2.485207100591716,
+      "grad_norm": 14.94650375178369,
+      "learning_rate": 4.348552565167542e-07,
+      "logits/chosen": -0.9337579011917114,
+      "logits/rejected": -0.8570343255996704,
+      "logps/chosen": -37.36121368408203,
+      "logps/rejected": -63.51266098022461,
+      "loss": 0.1052,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.1832229197025299,
+      "rewards/margins": 5.160886287689209,
+      "rewards/rejected": -5.344109058380127,
+      "step": 210
+    },
+    {
+      "epoch": 2.497041420118343,
+      "grad_norm": 17.82179694890819,
+      "learning_rate": 4.3397732278884194e-07,
+      "logits/chosen": -0.7793615460395813,
+      "logits/rejected": -0.7347290515899658,
+      "logps/chosen": -38.538475036621094,
+      "logps/rejected": -51.74705123901367,
+      "loss": 0.1133,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.012450069189071655,
+      "rewards/margins": 2.9992494583129883,
+      "rewards/rejected": -2.9867992401123047,
+      "step": 211
+    },
+    {
+      "epoch": 2.5088757396449703,
+      "grad_norm": 17.00316151116976,
+      "learning_rate": 4.330944118140406e-07,
+      "logits/chosen": -0.671549916267395,
+      "logits/rejected": -0.530448317527771,
+      "logps/chosen": -36.966880798339844,
+      "logps/rejected": -57.631778717041016,
+      "loss": 0.0971,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0925275981426239,
+      "rewards/margins": 5.078682899475098,
+      "rewards/rejected": -5.171210289001465,
+      "step": 212
+    },
+    {
+      "epoch": 2.5207100591715976,
+      "grad_norm": 12.753485442227316,
+      "learning_rate": 4.322065474782609e-07,
+      "logits/chosen": -0.9630086421966553,
+      "logits/rejected": -0.7986509799957275,
+      "logps/chosen": -34.51360321044922,
+      "logps/rejected": -55.286319732666016,
+      "loss": 0.0986,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.17616373300552368,
+      "rewards/margins": 4.3915815353393555,
+      "rewards/rejected": -4.567745208740234,
+      "step": 213
+    },
+    {
+      "epoch": 2.532544378698225,
+      "grad_norm": 19.996942513731593,
+      "learning_rate": 4.313137538014198e-07,
+      "logits/chosen": -0.6898477077484131,
+      "logits/rejected": -0.636679470539093,
+      "logps/chosen": -35.616844177246094,
+      "logps/rejected": -56.1121826171875,
+      "loss": 0.119,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.24041543900966644,
+      "rewards/margins": 4.271088600158691,
+      "rewards/rejected": -4.51150369644165,
+      "step": 214
+    },
+    {
+      "epoch": 2.544378698224852,
+      "grad_norm": 16.48295901806472,
+      "learning_rate": 4.304160549367906e-07,
+      "logits/chosen": -1.0932834148406982,
+      "logits/rejected": -1.1098268032073975,
+      "logps/chosen": -29.987680435180664,
+      "logps/rejected": -46.118587493896484,
+      "loss": 0.0842,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.31450319290161133,
+      "rewards/margins": 3.8299648761749268,
+      "rewards/rejected": -3.5154621601104736,
+      "step": 215
+    },
+    {
+      "epoch": 2.556213017751479,
+      "grad_norm": 17.61093634952303,
+      "learning_rate": 4.295134751703492e-07,
+      "logits/chosen": -0.9821122884750366,
+      "logits/rejected": -1.0032066106796265,
+      "logps/chosen": -29.145097732543945,
+      "logps/rejected": -53.80962371826172,
+      "loss": 0.1108,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3220353424549103,
+      "rewards/margins": 5.978228569030762,
+      "rewards/rejected": -6.300264358520508,
+      "step": 216
+    },
+    {
+      "epoch": 2.5680473372781067,
+      "grad_norm": 12.734227646855976,
+      "learning_rate": 4.28606038920118e-07,
+      "logits/chosen": -0.8004301190376282,
+      "logits/rejected": -0.6481240391731262,
+      "logps/chosen": -33.56897735595703,
+      "logps/rejected": -59.778018951416016,
+      "loss": 0.0721,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.31138908863067627,
+      "rewards/margins": 5.115996837615967,
+      "rewards/rejected": -5.427386283874512,
+      "step": 217
+    },
+    {
+      "epoch": 2.5798816568047336,
+      "grad_norm": 19.316829502268995,
+      "learning_rate": 4.276937707355044e-07,
+      "logits/chosen": -0.8966995477676392,
+      "logits/rejected": -0.970150887966156,
+      "logps/chosen": -37.73095703125,
+      "logps/rejected": -48.6308708190918,
+      "loss": 0.1289,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.6531825065612793,
+      "rewards/margins": 4.240229606628418,
+      "rewards/rejected": -4.893411636352539,
+      "step": 218
+    },
+    {
+      "epoch": 2.591715976331361,
+      "grad_norm": 15.35867876097276,
+      "learning_rate": 4.2677669529663686e-07,
+      "logits/chosen": -0.7338119745254517,
+      "logits/rejected": -0.7964376211166382,
+      "logps/chosen": -35.940223693847656,
+      "logps/rejected": -49.16367721557617,
+      "loss": 0.0979,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5470548868179321,
+      "rewards/margins": 4.513436317443848,
+      "rewards/rejected": -5.060491561889648,
+      "step": 219
+    },
+    {
+      "epoch": 2.603550295857988,
+      "grad_norm": 15.463745366871002,
+      "learning_rate": 4.2585483741369755e-07,
+      "logits/chosen": -0.7903125882148743,
+      "logits/rejected": -0.6668514609336853,
+      "logps/chosen": -39.22986602783203,
+      "logps/rejected": -56.949462890625,
+      "loss": 0.109,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4734441637992859,
+      "rewards/margins": 3.438096523284912,
+      "rewards/rejected": -3.9115407466888428,
+      "step": 220
+    },
+    {
+      "epoch": 2.6153846153846154,
+      "grad_norm": 11.656065312741738,
+      "learning_rate": 4.2492822202625065e-07,
+      "logits/chosen": -0.746453046798706,
+      "logits/rejected": -0.5671579837799072,
+      "logps/chosen": -29.767318725585938,
+      "logps/rejected": -55.21434020996094,
+      "loss": 0.0675,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5708121657371521,
+      "rewards/margins": 4.064294338226318,
+      "rewards/rejected": -4.635106563568115,
+      "step": 221
+    },
+    {
+      "epoch": 2.6272189349112427,
+      "grad_norm": 21.80300920307137,
+      "learning_rate": 4.239968742025684e-07,
+      "logits/chosen": -1.0824371576309204,
+      "logits/rejected": -1.0959596633911133,
+      "logps/chosen": -43.30390548706055,
+      "logps/rejected": -61.727516174316406,
+      "loss": 0.1193,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.6028754711151123,
+      "rewards/margins": 4.46554708480835,
+      "rewards/rejected": -5.068422317504883,
+      "step": 222
+    },
+    {
+      "epoch": 2.63905325443787,
+      "grad_norm": 14.049260804964101,
+      "learning_rate": 4.2306081913895177e-07,
+      "logits/chosen": -0.9191200137138367,
+      "logits/rejected": -1.0407882928848267,
+      "logps/chosen": -32.684852600097656,
+      "logps/rejected": -46.769744873046875,
+      "loss": 0.0912,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8108103275299072,
+      "rewards/margins": 4.522951126098633,
+      "rewards/rejected": -5.333761215209961,
+      "step": 223
+    },
+    {
+      "epoch": 2.6508875739644973,
+      "grad_norm": 15.994418099284374,
+      "learning_rate": 4.2212008215905e-07,
+      "logits/chosen": -0.8396817445755005,
+      "logits/rejected": -0.7051962018013,
+      "logps/chosen": -41.968048095703125,
+      "logps/rejected": -68.28170776367188,
+      "loss": 0.0873,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.5884815454483032,
+      "rewards/margins": 5.965177536010742,
+      "rewards/rejected": -6.553658962249756,
+      "step": 224
+    },
+    {
+      "epoch": 2.662721893491124,
+      "grad_norm": 17.35636622879192,
+      "learning_rate": 4.2117468871317465e-07,
+      "logits/chosen": -1.0077329874038696,
+      "logits/rejected": -1.021416425704956,
+      "logps/chosen": -33.865623474121094,
+      "logps/rejected": -55.850704193115234,
+      "loss": 0.1052,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.3826393485069275,
+      "rewards/margins": 5.580774784088135,
+      "rewards/rejected": -5.963414192199707,
+      "step": 225
+    },
+    {
+      "epoch": 2.6745562130177514,
+      "grad_norm": 12.60270784440611,
+      "learning_rate": 4.2022466437761154e-07,
+      "logits/chosen": -1.1540577411651611,
+      "logits/rejected": -1.0161852836608887,
+      "logps/chosen": -32.255516052246094,
+      "logps/rejected": -64.34480285644531,
+      "loss": 0.0715,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.15079544484615326,
+      "rewards/margins": 4.2565717697143555,
+      "rewards/rejected": -4.105776786804199,
+      "step": 226
+    },
+    {
+      "epoch": 2.6863905325443787,
+      "grad_norm": 11.837128157661406,
+      "learning_rate": 4.1927003485392873e-07,
+      "logits/chosen": -1.1032685041427612,
+      "logits/rejected": -1.2223795652389526,
+      "logps/chosen": -30.195100784301758,
+      "logps/rejected": -43.28514099121094,
+      "loss": 0.0599,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.1709037721157074,
+      "rewards/margins": 3.4937398433685303,
+      "rewards/rejected": -3.6646437644958496,
+      "step": 227
+    },
+    {
+      "epoch": 2.698224852071006,
+      "grad_norm": 16.871047034781974,
+      "learning_rate": 4.18310825968281e-07,
+      "logits/chosen": -1.022660732269287,
+      "logits/rejected": -1.1288559436798096,
+      "logps/chosen": -40.125972747802734,
+      "logps/rejected": -48.385841369628906,
+      "loss": 0.1092,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.03397001326084137,
+      "rewards/margins": 3.533397674560547,
+      "rewards/rejected": -3.4994277954101562,
+      "step": 228
+    },
+    {
+      "epoch": 2.710059171597633,
+      "grad_norm": 14.91535959862739,
+      "learning_rate": 4.173470636707115e-07,
+      "logits/chosen": -1.0440349578857422,
+      "logits/rejected": -1.0170382261276245,
+      "logps/chosen": -31.154685974121094,
+      "logps/rejected": -54.95928192138672,
+      "loss": 0.0941,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.514030933380127,
+      "rewards/margins": 5.214803695678711,
+      "rewards/rejected": -5.72883415222168,
+      "step": 229
+    },
+    {
+      "epoch": 2.7218934911242605,
+      "grad_norm": 16.4841578502459,
+      "learning_rate": 4.1637877403444923e-07,
+      "logits/chosen": -0.771393358707428,
+      "logits/rejected": -0.7641423940658569,
+      "logps/chosen": -42.56052017211914,
+      "logps/rejected": -58.539730072021484,
+      "loss": 0.0918,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2780624628067017,
+      "rewards/margins": 5.585483551025391,
+      "rewards/rejected": -6.863546371459961,
+      "step": 230
+    },
+    {
+      "epoch": 2.7337278106508878,
+      "grad_norm": 12.082861048786787,
+      "learning_rate": 4.1540598325520406e-07,
+      "logits/chosen": -0.8163421154022217,
+      "logits/rejected": -0.8682371377944946,
+      "logps/chosen": -26.423309326171875,
+      "logps/rejected": -41.69921875,
+      "loss": 0.0766,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.16670477390289307,
+      "rewards/margins": 4.770580291748047,
+      "rewards/rejected": -4.93728494644165,
+      "step": 231
+    },
+    {
+      "epoch": 2.7455621301775146,
+      "grad_norm": 13.206244471660067,
+      "learning_rate": 4.144287176504582e-07,
+      "logits/chosen": -1.0770142078399658,
+      "logits/rejected": -1.100901484489441,
+      "logps/chosen": -43.73130416870117,
+      "logps/rejected": -58.910987854003906,
+      "loss": 0.0782,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.775425136089325,
+      "rewards/margins": 5.230464935302734,
+      "rewards/rejected": -6.005890369415283,
+      "step": 232
+    },
+    {
+      "epoch": 2.757396449704142,
+      "grad_norm": 18.831608964137178,
+      "learning_rate": 4.1344700365875353e-07,
+      "logits/chosen": -1.2907536029815674,
+      "logits/rejected": -1.1133619546890259,
+      "logps/chosen": -42.22511672973633,
+      "logps/rejected": -80.1695327758789,
+      "loss": 0.1184,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.7888930439949036,
+      "rewards/margins": 4.728762149810791,
+      "rewards/rejected": -5.517655372619629,
+      "step": 233
+    },
+    {
+      "epoch": 2.769230769230769,
+      "grad_norm": 18.13801394049149,
+      "learning_rate": 4.1246086783897713e-07,
+      "logits/chosen": -0.7159754037857056,
+      "logits/rejected": -0.9155410528182983,
+      "logps/chosen": -38.486732482910156,
+      "logps/rejected": -46.4632568359375,
+      "loss": 0.1165,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.28693825006484985,
+      "rewards/margins": 4.148380756378174,
+      "rewards/rejected": -4.435318946838379,
+      "step": 234
+    },
+    {
+      "epoch": 2.7810650887573964,
+      "grad_norm": 14.561324524299124,
+      "learning_rate": 4.1147033686964213e-07,
+      "logits/chosen": -1.0087186098098755,
+      "logits/rejected": -1.0201172828674316,
+      "logps/chosen": -38.35453796386719,
+      "logps/rejected": -61.89058303833008,
+      "loss": 0.082,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6061041951179504,
+      "rewards/margins": 6.066676139831543,
+      "rewards/rejected": -6.6727800369262695,
+      "step": 235
+    },
+    {
+      "epoch": 2.7928994082840237,
+      "grad_norm": 18.20449643988736,
+      "learning_rate": 4.104754375481664e-07,
+      "logits/chosen": -1.0226837396621704,
+      "logits/rejected": -1.1159076690673828,
+      "logps/chosen": -33.471160888671875,
+      "logps/rejected": -49.130104064941406,
+      "loss": 0.1018,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9132391214370728,
+      "rewards/margins": 4.530313491821289,
+      "rewards/rejected": -5.443552494049072,
+      "step": 236
+    },
+    {
+      "epoch": 2.804733727810651,
+      "grad_norm": 18.714482911950206,
+      "learning_rate": 4.0947619679014733e-07,
+      "logits/chosen": -1.3095102310180664,
+      "logits/rejected": -1.3122498989105225,
+      "logps/chosen": -37.17940902709961,
+      "logps/rejected": -56.99421691894531,
+      "loss": 0.0986,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.6820850372314453,
+      "rewards/margins": 5.377782821655273,
+      "rewards/rejected": -6.059867858886719,
+      "step": 237
+    },
+    {
+      "epoch": 2.8165680473372783,
+      "grad_norm": 11.370290693432937,
+      "learning_rate": 4.084726416286337e-07,
+      "logits/chosen": -1.0340602397918701,
+      "logits/rejected": -0.8559252023696899,
+      "logps/chosen": -36.95317840576172,
+      "logps/rejected": -69.14118957519531,
+      "loss": 0.0699,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.7623500227928162,
+      "rewards/margins": 6.243594169616699,
+      "rewards/rejected": -7.00594425201416,
+      "step": 238
+    },
+    {
+      "epoch": 2.828402366863905,
+      "grad_norm": 18.470159692479328,
+      "learning_rate": 4.0746479921339456e-07,
+      "logits/chosen": -1.0048712491989136,
+      "logits/rejected": -1.0136165618896484,
+      "logps/chosen": -51.31761169433594,
+      "logps/rejected": -61.482666015625,
+      "loss": 0.102,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9726583361625671,
+      "rewards/margins": 3.5378739833831787,
+      "rewards/rejected": -4.510532379150391,
+      "step": 239
+    },
+    {
+      "epoch": 2.8402366863905324,
+      "grad_norm": 11.915657733560364,
+      "learning_rate": 4.0645269681018434e-07,
+      "logits/chosen": -1.22614324092865,
+      "logits/rejected": -1.3402845859527588,
+      "logps/chosen": -37.68153381347656,
+      "logps/rejected": -55.563926696777344,
+      "loss": 0.0739,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6234626173973083,
+      "rewards/margins": 4.600487232208252,
+      "rewards/rejected": -5.223949909210205,
+      "step": 240
+    },
+    {
+      "epoch": 2.8520710059171597,
+      "grad_norm": 15.928696873524284,
+      "learning_rate": 4.054363618000057e-07,
+      "logits/chosen": -1.0241750478744507,
+      "logits/rejected": -0.9920932650566101,
+      "logps/chosen": -41.79422378540039,
+      "logps/rejected": -73.42777252197266,
+      "loss": 0.1032,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.0987391397356987,
+      "rewards/margins": 6.280758857727051,
+      "rewards/rejected": -6.18202018737793,
+      "step": 241
+    },
+    {
+      "epoch": 2.863905325443787,
+      "grad_norm": 14.965124803326795,
+      "learning_rate": 4.044158216783684e-07,
+      "logits/chosen": -1.2023519277572632,
+      "logits/rejected": -1.1554661989212036,
+      "logps/chosen": -50.22661590576172,
+      "logps/rejected": -60.445709228515625,
+      "loss": 0.0811,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.7524704933166504,
+      "rewards/margins": 4.429455280303955,
+      "rewards/rejected": -5.1819257736206055,
+      "step": 242
+    },
+    {
+      "epoch": 2.8757396449704142,
+      "grad_norm": 15.090679655603102,
+      "learning_rate": 4.033911040545453e-07,
+      "logits/chosen": -1.0586270093917847,
+      "logits/rejected": -1.1135869026184082,
+      "logps/chosen": -39.405941009521484,
+      "logps/rejected": -55.257102966308594,
+      "loss": 0.0836,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.5358131527900696,
+      "rewards/margins": 4.630053520202637,
+      "rewards/rejected": -5.165866374969482,
+      "step": 243
+    },
+    {
+      "epoch": 2.8875739644970415,
+      "grad_norm": 15.379447226520636,
+      "learning_rate": 4.0236223665082605e-07,
+      "logits/chosen": -0.9784256219863892,
+      "logits/rejected": -1.0398386716842651,
+      "logps/chosen": -40.73570251464844,
+      "logps/rejected": -48.330848693847656,
+      "loss": 0.0896,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.9628621935844421,
+      "rewards/margins": 3.23721981048584,
+      "rewards/rejected": -4.200081825256348,
+      "step": 244
+    },
+    {
+      "epoch": 2.899408284023669,
+      "grad_norm": 16.74378459980107,
+      "learning_rate": 4.0132924730176653e-07,
+      "logits/chosen": -0.723244309425354,
+      "logits/rejected": -0.7906000018119812,
+      "logps/chosen": -33.3350944519043,
+      "logps/rejected": -44.06977844238281,
+      "loss": 0.1087,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.1651308238506317,
+      "rewards/margins": 4.324864864349365,
+      "rewards/rejected": -4.489995956420898,
+      "step": 245
+    },
+    {
+      "epoch": 2.9112426035502956,
+      "grad_norm": 16.171542896949546,
+      "learning_rate": 4.0029216395343617e-07,
+      "logits/chosen": -0.9244989156723022,
+      "logits/rejected": -0.7894719839096069,
+      "logps/chosen": -38.35075759887695,
+      "logps/rejected": -63.83277893066406,
+      "loss": 0.104,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6405995488166809,
+      "rewards/margins": 5.280316352844238,
+      "rewards/rejected": -5.9209160804748535,
+      "step": 246
+    },
+    {
+      "epoch": 2.9230769230769234,
+      "grad_norm": 16.103173612778058,
+      "learning_rate": 3.992510146626617e-07,
+      "logits/chosen": -1.072364091873169,
+      "logits/rejected": -1.219006061553955,
+      "logps/chosen": -52.24778366088867,
+      "logps/rejected": -54.28422546386719,
+      "loss": 0.088,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5743333101272583,
+      "rewards/margins": 4.490229606628418,
+      "rewards/rejected": -5.064562797546387,
+      "step": 247
+    },
+    {
+      "epoch": 2.93491124260355,
+      "grad_norm": 21.934919226614063,
+      "learning_rate": 3.982058275962682e-07,
+      "logits/chosen": -1.0643049478530884,
+      "logits/rejected": -1.0713043212890625,
+      "logps/chosen": -38.724098205566406,
+      "logps/rejected": -49.14268112182617,
+      "loss": 0.1283,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.003525674343109131,
+      "rewards/margins": 3.779031991958618,
+      "rewards/rejected": -3.775506019592285,
+      "step": 248
+    },
+    {
+      "epoch": 2.9467455621301775,
+      "grad_norm": 13.845420751689739,
+      "learning_rate": 3.9715663103031706e-07,
+      "logits/chosen": -0.8697773218154907,
+      "logits/rejected": -1.0021997690200806,
+      "logps/chosen": -48.516197204589844,
+      "logps/rejected": -63.481204986572266,
+      "loss": 0.0863,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.2427257299423218,
+      "rewards/margins": 5.817460060119629,
+      "rewards/rejected": -7.06018590927124,
+      "step": 249
+    },
+    {
+      "epoch": 2.9585798816568047,
+      "grad_norm": 15.20831572963414,
+      "learning_rate": 3.9610345334934094e-07,
+      "logits/chosen": -0.9560275673866272,
+      "logits/rejected": -0.7608417272567749,
+      "logps/chosen": -27.386804580688477,
+      "logps/rejected": -56.76541519165039,
+      "loss": 0.0957,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.055940259248018265,
+      "rewards/margins": 4.628420829772949,
+      "rewards/rejected": -4.57248067855835,
+      "step": 250
+    },
+    {
+      "epoch": 2.970414201183432,
+      "grad_norm": 13.509957666437309,
+      "learning_rate": 3.950463230455761e-07,
+      "logits/chosen": -0.9403877258300781,
+      "logits/rejected": -1.0766938924789429,
+      "logps/chosen": -46.44561004638672,
+      "logps/rejected": -55.376487731933594,
+      "loss": 0.0769,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.6139943599700928,
+      "rewards/margins": 5.4038591384887695,
+      "rewards/rejected": -7.017853260040283,
+      "step": 251
+    },
+    {
+      "epoch": 2.9822485207100593,
+      "grad_norm": 13.281356988917038,
+      "learning_rate": 3.939852687181915e-07,
+      "logits/chosen": -1.5572373867034912,
+      "logits/rejected": -1.509653091430664,
+      "logps/chosen": -38.27780532836914,
+      "logps/rejected": -59.58174514770508,
+      "loss": 0.0765,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.13283273577690125,
+      "rewards/margins": 6.1197829246521,
+      "rewards/rejected": -6.252615451812744,
+      "step": 252
+    },
+    {
+      "epoch": 2.994082840236686,
+      "grad_norm": 13.363993336432463,
+      "learning_rate": 3.9292031907251464e-07,
+      "logits/chosen": -0.9404221773147583,
+      "logits/rejected": -0.8659825921058655,
+      "logps/chosen": -47.44232177734375,
+      "logps/rejected": -72.00565338134766,
+      "loss": 0.0827,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.288888692855835,
+      "rewards/margins": 6.671911239624023,
+      "rewards/rejected": -8.960800170898438,
+      "step": 253
+    },
+    {
+      "epoch": 3.0059171597633134,
+      "grad_norm": 13.191144507547312,
+      "learning_rate": 3.9185150291925585e-07,
+      "logits/chosen": -0.9490239024162292,
+      "logits/rejected": -0.7950053215026855,
+      "logps/chosen": -27.338720321655273,
+      "logps/rejected": -54.10641860961914,
+      "loss": 0.085,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2650642395019531,
+      "rewards/margins": 4.751713275909424,
+      "rewards/rejected": -5.016777992248535,
+      "step": 254
+    },
+    {
+      "epoch": 3.0177514792899407,
+      "grad_norm": 10.096716674793884,
+      "learning_rate": 3.9077884917372806e-07,
+      "logits/chosen": -0.9493421912193298,
+      "logits/rejected": -0.9330881834030151,
+      "logps/chosen": -33.75062561035156,
+      "logps/rejected": -58.77503967285156,
+      "loss": 0.045,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.49208763241767883,
+      "rewards/margins": 5.006468296051025,
+      "rewards/rejected": -5.498556137084961,
+      "step": 255
+    },
+    {
+      "epoch": 3.029585798816568,
+      "grad_norm": 8.187374294888702,
+      "learning_rate": 3.8970238685506486e-07,
+      "logits/chosen": -1.0067384243011475,
+      "logits/rejected": -1.0715603828430176,
+      "logps/chosen": -29.130413055419922,
+      "logps/rejected": -55.432456970214844,
+      "loss": 0.0465,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.37083467841148376,
+      "rewards/margins": 6.350147247314453,
+      "rewards/rejected": -6.720982551574707,
+      "step": 256
+    },
+    {
+      "epoch": 3.0414201183431953,
+      "grad_norm": 10.400956778483433,
+      "learning_rate": 3.8862214508543544e-07,
+      "logits/chosen": -1.1537256240844727,
+      "logits/rejected": -1.2295022010803223,
+      "logps/chosen": -44.73936080932617,
+      "logps/rejected": -56.7740478515625,
+      "loss": 0.0558,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8702179789543152,
+      "rewards/margins": 4.224546432495117,
+      "rewards/rejected": -5.09476375579834,
+      "step": 257
+    },
+    {
+      "epoch": 3.0532544378698225,
+      "grad_norm": 9.746601784268003,
+      "learning_rate": 3.8753815308925685e-07,
+      "logits/chosen": -0.9428281784057617,
+      "logits/rejected": -0.7745850682258606,
+      "logps/chosen": -42.13050842285156,
+      "logps/rejected": -81.40808868408203,
+      "loss": 0.0576,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.15858441591262817,
+      "rewards/margins": 5.615185260772705,
+      "rewards/rejected": -5.773769378662109,
+      "step": 258
+    },
+    {
+      "epoch": 3.06508875739645,
+      "grad_norm": 8.307502810968026,
+      "learning_rate": 3.864504401924031e-07,
+      "logits/chosen": -0.750975489616394,
+      "logits/rejected": -0.8657329082489014,
+      "logps/chosen": -42.026031494140625,
+      "logps/rejected": -60.68046951293945,
+      "loss": 0.0497,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.10235822200775146,
+      "rewards/margins": 5.683484077453613,
+      "rewards/rejected": -5.7858428955078125,
+      "step": 259
+    },
+    {
+      "epoch": 3.076923076923077,
+      "grad_norm": 10.229530562143605,
+      "learning_rate": 3.8535903582141184e-07,
+      "logits/chosen": -0.9478952288627625,
+      "logits/rejected": -1.0307767391204834,
+      "logps/chosen": -55.219512939453125,
+      "logps/rejected": -65.23870849609375,
+      "loss": 0.0592,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.5834192633628845,
+      "rewards/margins": 5.146862983703613,
+      "rewards/rejected": -5.730282306671143,
+      "step": 260
+    },
+    {
+      "epoch": 3.088757396449704,
+      "grad_norm": 11.360767151960536,
+      "learning_rate": 3.8426396950268846e-07,
+      "logits/chosen": -0.9507812261581421,
+      "logits/rejected": -1.0435519218444824,
+      "logps/chosen": -37.444190979003906,
+      "logps/rejected": -53.09665298461914,
+      "loss": 0.0683,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.7893316745758057,
+      "rewards/margins": 4.056205749511719,
+      "rewards/rejected": -4.845537185668945,
+      "step": 261
+    },
+    {
+      "epoch": 3.100591715976331,
+      "grad_norm": 10.428147481093854,
+      "learning_rate": 3.8316527086170727e-07,
+      "logits/chosen": -1.1136678457260132,
+      "logits/rejected": -1.1166181564331055,
+      "logps/chosen": -46.36027526855469,
+      "logps/rejected": -61.403350830078125,
+      "loss": 0.0644,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.6686680316925049,
+      "rewards/margins": 5.635904312133789,
+      "rewards/rejected": -7.304572582244873,
+      "step": 262
+    },
+    {
+      "epoch": 3.1124260355029585,
+      "grad_norm": 9.067198800478028,
+      "learning_rate": 3.820629696222096e-07,
+      "logits/chosen": -0.6134490370750427,
+      "logits/rejected": -0.6816850900650024,
+      "logps/chosen": -37.14327621459961,
+      "logps/rejected": -45.768043518066406,
+      "loss": 0.0516,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.10417760908603668,
+      "rewards/margins": 4.2560577392578125,
+      "rewards/rejected": -4.36023473739624,
+      "step": 263
+    },
+    {
+      "epoch": 3.1242603550295858,
+      "grad_norm": 9.142972342246532,
+      "learning_rate": 3.809570956054003e-07,
+      "logits/chosen": -1.110413908958435,
+      "logits/rejected": -1.2050219774246216,
+      "logps/chosen": -46.59968185424805,
+      "logps/rejected": -67.91047668457031,
+      "loss": 0.0521,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.3769968748092651,
+      "rewards/margins": 4.8837504386901855,
+      "rewards/rejected": -6.260746955871582,
+      "step": 264
+    },
+    {
+      "epoch": 3.136094674556213,
+      "grad_norm": 9.722445350426577,
+      "learning_rate": 3.798476787291407e-07,
+      "logits/chosen": -1.204038143157959,
+      "logits/rejected": -1.2214388847351074,
+      "logps/chosen": -55.631988525390625,
+      "logps/rejected": -70.27022552490234,
+      "loss": 0.0545,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.43508243560791,
+      "rewards/margins": 5.983229637145996,
+      "rewards/rejected": -8.418312072753906,
+      "step": 265
+    },
+    {
+      "epoch": 3.1479289940828403,
+      "grad_norm": 12.165954862754088,
+      "learning_rate": 3.787347490071389e-07,
+      "logits/chosen": -1.0962128639221191,
+      "logits/rejected": -1.0446038246154785,
+      "logps/chosen": -30.58979034423828,
+      "logps/rejected": -57.98908233642578,
+      "loss": 0.0751,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3746563792228699,
+      "rewards/margins": 6.635012626647949,
+      "rewards/rejected": -7.009669303894043,
+      "step": 266
+    },
+    {
+      "epoch": 3.1597633136094676,
+      "grad_norm": 9.45931606123372,
+      "learning_rate": 3.776183365481385e-07,
+      "logits/chosen": -1.1556551456451416,
+      "logits/rejected": -1.1371455192565918,
+      "logps/chosen": -29.80742835998535,
+      "logps/rejected": -65.14445495605469,
+      "loss": 0.046,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.65096515417099,
+      "rewards/margins": 8.631277084350586,
+      "rewards/rejected": -9.282241821289062,
+      "step": 267
+    },
+    {
+      "epoch": 3.171597633136095,
+      "grad_norm": 8.216009521089312,
+      "learning_rate": 3.764984715551031e-07,
+      "logits/chosen": -0.7823802828788757,
+      "logits/rejected": -0.8771790266036987,
+      "logps/chosen": -33.291709899902344,
+      "logps/rejected": -53.34422302246094,
+      "loss": 0.0455,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4557185471057892,
+      "rewards/margins": 5.85953426361084,
+      "rewards/rejected": -6.315252780914307,
+      "step": 268
+    },
+    {
+      "epoch": 3.1834319526627217,
+      "grad_norm": 8.44405243827575,
+      "learning_rate": 3.753751843244003e-07,
+      "logits/chosen": -0.6730700731277466,
+      "logits/rejected": -0.7033834457397461,
+      "logps/chosen": -38.24939727783203,
+      "logps/rejected": -50.691505432128906,
+      "loss": 0.0506,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.48223841190338135,
+      "rewards/margins": 5.0005269050598145,
+      "rewards/rejected": -5.482766151428223,
+      "step": 269
+    },
+    {
+      "epoch": 3.195266272189349,
+      "grad_norm": 12.678568405782729,
+      "learning_rate": 3.7424850524498113e-07,
+      "logits/chosen": -1.1234819889068604,
+      "logits/rejected": -1.0680205821990967,
+      "logps/chosen": -49.61648178100586,
+      "logps/rejected": -62.6021728515625,
+      "loss": 0.0669,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4030035734176636,
+      "rewards/margins": 6.600266456604004,
+      "rewards/rejected": -8.003270149230957,
+      "step": 270
+    },
+    {
+      "epoch": 3.2071005917159763,
+      "grad_norm": 8.769925116184863,
+      "learning_rate": 3.731184647975584e-07,
+      "logits/chosen": -1.0641188621520996,
+      "logits/rejected": -1.0786867141723633,
+      "logps/chosen": -39.86817169189453,
+      "logps/rejected": -59.44279479980469,
+      "loss": 0.0539,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2523530125617981,
+      "rewards/margins": 5.5049519538879395,
+      "rewards/rejected": -5.757305145263672,
+      "step": 271
+    },
+    {
+      "epoch": 3.2189349112426036,
+      "grad_norm": 9.128279564046098,
+      "learning_rate": 3.7198509355378207e-07,
+      "logits/chosen": -1.1525990962982178,
+      "logits/rejected": -1.1050852537155151,
+      "logps/chosen": -31.691631317138672,
+      "logps/rejected": -58.040374755859375,
+      "loss": 0.0547,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3830980956554413,
+      "rewards/margins": 5.858831882476807,
+      "rewards/rejected": -6.24193000793457,
+      "step": 272
+    },
+    {
+      "epoch": 3.230769230769231,
+      "grad_norm": 5.945965238283508,
+      "learning_rate": 3.7084842217541196e-07,
+      "logits/chosen": -1.1103689670562744,
+      "logits/rejected": -1.2655141353607178,
+      "logps/chosen": -43.58381652832031,
+      "logps/rejected": -63.32284927368164,
+      "loss": 0.0271,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6699415445327759,
+      "rewards/margins": 7.9212541580200195,
+      "rewards/rejected": -8.591195106506348,
+      "step": 273
+    },
+    {
+      "epoch": 3.242603550295858,
+      "grad_norm": 9.302647208034704,
+      "learning_rate": 3.6970848141348855e-07,
+      "logits/chosen": -1.2158900499343872,
+      "logits/rejected": -1.2491604089736938,
+      "logps/chosen": -43.763771057128906,
+      "logps/rejected": -65.52889251708984,
+      "loss": 0.0642,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5131170153617859,
+      "rewards/margins": 5.668525218963623,
+      "rewards/rejected": -6.181642532348633,
+      "step": 274
+    },
+    {
+      "epoch": 3.2544378698224854,
+      "grad_norm": 8.636078483244923,
+      "learning_rate": 3.685653021075006e-07,
+      "logits/chosen": -1.3144724369049072,
+      "logits/rejected": -1.1518325805664062,
+      "logps/chosen": -32.579593658447266,
+      "logps/rejected": -59.97221374511719,
+      "loss": 0.0518,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.165370911359787,
+      "rewards/margins": 6.193992614746094,
+      "rewards/rejected": -6.359362602233887,
+      "step": 275
+    },
+    {
+      "epoch": 3.2662721893491122,
+      "grad_norm": 8.631432982295077,
+      "learning_rate": 3.6741891518455146e-07,
+      "logits/chosen": -1.0432945489883423,
+      "logits/rejected": -0.9308419227600098,
+      "logps/chosen": -38.913291931152344,
+      "logps/rejected": -64.8463363647461,
+      "loss": 0.0483,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2648208141326904,
+      "rewards/margins": 7.041728496551514,
+      "rewards/rejected": -8.306549072265625,
+      "step": 276
+    },
+    {
+      "epoch": 3.2781065088757395,
+      "grad_norm": 8.627648180187752,
+      "learning_rate": 3.6626935165852183e-07,
+      "logits/chosen": -1.1467763185501099,
+      "logits/rejected": -0.95106440782547,
+      "logps/chosen": -45.197444915771484,
+      "logps/rejected": -77.6880874633789,
+      "loss": 0.0575,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.7282447218894958,
+      "rewards/margins": 6.463216304779053,
+      "rewards/rejected": -7.191461086273193,
+      "step": 277
+    },
+    {
+      "epoch": 3.289940828402367,
+      "grad_norm": 10.925286523452975,
+      "learning_rate": 3.6511664262923094e-07,
+      "logits/chosen": -1.0780082941055298,
+      "logits/rejected": -1.060866355895996,
+      "logps/chosen": -41.23103332519531,
+      "logps/rejected": -64.39918518066406,
+      "loss": 0.0655,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5627799034118652,
+      "rewards/margins": 4.834949970245361,
+      "rewards/rejected": -5.397729873657227,
+      "step": 278
+    },
+    {
+      "epoch": 3.301775147928994,
+      "grad_norm": 7.9347499078081745,
+      "learning_rate": 3.639608192815951e-07,
+      "logits/chosen": -1.1390736103057861,
+      "logits/rejected": -1.1505769491195679,
+      "logps/chosen": -31.97028923034668,
+      "logps/rejected": -52.920997619628906,
+      "loss": 0.0435,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.059164464473724365,
+      "rewards/margins": 5.323459148406982,
+      "rewards/rejected": -5.264294624328613,
+      "step": 279
+    },
+    {
+      "epoch": 3.3136094674556213,
+      "grad_norm": 9.658317222064715,
+      "learning_rate": 3.6280191288478435e-07,
+      "logits/chosen": -0.3561999797821045,
+      "logits/rejected": -0.4131832420825958,
+      "logps/chosen": -49.74396514892578,
+      "logps/rejected": -51.83927917480469,
+      "loss": 0.0488,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.05161542445421219,
+      "rewards/margins": 3.9036967754364014,
+      "rewards/rejected": -3.852081298828125,
+      "step": 280
+    },
+    {
+      "epoch": 3.3254437869822486,
+      "grad_norm": 7.0333602841005565,
+      "learning_rate": 3.61639954791376e-07,
+      "logits/chosen": -0.7871007919311523,
+      "logits/rejected": -0.7978352308273315,
+      "logps/chosen": -37.140010833740234,
+      "logps/rejected": -54.86417007446289,
+      "loss": 0.0392,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.12373484671115875,
+      "rewards/margins": 5.170371055603027,
+      "rewards/rejected": -5.2941060066223145,
+      "step": 281
+    },
+    {
+      "epoch": 3.337278106508876,
+      "grad_norm": 7.845433796623778,
+      "learning_rate": 3.604749764365069e-07,
+      "logits/chosen": -0.9799227714538574,
+      "logits/rejected": -0.933765709400177,
+      "logps/chosen": -41.03892517089844,
+      "logps/rejected": -58.41741943359375,
+      "loss": 0.0466,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3890104293823242,
+      "rewards/margins": 4.3942551612854,
+      "rewards/rejected": -4.783265113830566,
+      "step": 282
+    },
+    {
+      "epoch": 3.3491124260355027,
+      "grad_norm": 8.104393341578428,
+      "learning_rate": 3.593070093370226e-07,
+      "logits/chosen": -1.1032558679580688,
+      "logits/rejected": -1.1244325637817383,
+      "logps/chosen": -37.84381866455078,
+      "logps/rejected": -54.38085174560547,
+      "loss": 0.0493,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.03373198211193085,
+      "rewards/margins": 5.32047176361084,
+      "rewards/rejected": -5.286739349365234,
+      "step": 283
+    },
+    {
+      "epoch": 3.36094674556213,
+      "grad_norm": 7.636891980111439,
+      "learning_rate": 3.5813608509062526e-07,
+      "logits/chosen": -1.0172635316848755,
+      "logits/rejected": -1.1440664529800415,
+      "logps/chosen": -46.45831298828125,
+      "logps/rejected": -58.084495544433594,
+      "loss": 0.0364,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.6164510250091553,
+      "rewards/margins": 5.46311092376709,
+      "rewards/rejected": -7.079561233520508,
+      "step": 284
+    },
+    {
+      "epoch": 3.3727810650887573,
+      "grad_norm": 8.575228042301212,
+      "learning_rate": 3.569622353750181e-07,
+      "logits/chosen": -1.1266000270843506,
+      "logits/rejected": -1.0841931104660034,
+      "logps/chosen": -30.485980987548828,
+      "logps/rejected": -66.34876251220703,
+      "loss": 0.0429,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.12574508786201477,
+      "rewards/margins": 6.036714553833008,
+      "rewards/rejected": -5.910969257354736,
+      "step": 285
+    },
+    {
+      "epoch": 3.3846153846153846,
+      "grad_norm": 14.644268169698243,
+      "learning_rate": 3.557854919470491e-07,
+      "logits/chosen": -0.8823134899139404,
+      "logits/rejected": -0.9364801645278931,
+      "logps/chosen": -42.28858947753906,
+      "logps/rejected": -47.119564056396484,
+      "loss": 0.0808,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.7562182545661926,
+      "rewards/margins": 3.6561923027038574,
+      "rewards/rejected": -4.412410736083984,
+      "step": 286
+    },
+    {
+      "epoch": 3.396449704142012,
+      "grad_norm": 8.345961744927349,
+      "learning_rate": 3.546058866418513e-07,
+      "logits/chosen": -1.3047679662704468,
+      "logits/rejected": -1.2904943227767944,
+      "logps/chosen": -32.27897644042969,
+      "logps/rejected": -55.80604553222656,
+      "loss": 0.0401,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.10520702600479126,
+      "rewards/margins": 6.855145454406738,
+      "rewards/rejected": -6.960352420806885,
+      "step": 287
+    },
+    {
+      "epoch": 3.408284023668639,
+      "grad_norm": 6.8991491567748895,
+      "learning_rate": 3.5342345137198206e-07,
+      "logits/chosen": -1.0065886974334717,
+      "logits/rejected": -0.9561058282852173,
+      "logps/chosen": -36.960044860839844,
+      "logps/rejected": -61.72477340698242,
+      "loss": 0.0353,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1544219255447388,
+      "rewards/margins": 6.217504978179932,
+      "rewards/rejected": -7.371927261352539,
+      "step": 288
+    },
+    {
+      "epoch": 3.4201183431952664,
+      "grad_norm": 10.230292373857246,
+      "learning_rate": 3.5223821812655903e-07,
+      "logits/chosen": -0.9817609190940857,
+      "logits/rejected": -1.056479811668396,
+      "logps/chosen": -42.391998291015625,
+      "logps/rejected": -52.13397979736328,
+      "loss": 0.0515,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1262445449829102,
+      "rewards/margins": 5.622671604156494,
+      "rewards/rejected": -6.748915672302246,
+      "step": 289
+    },
+    {
+      "epoch": 3.4319526627218933,
+      "grad_norm": 11.408367741978596,
+      "learning_rate": 3.510502189703954e-07,
+      "logits/chosen": -0.8357728719711304,
+      "logits/rejected": -0.7616229057312012,
+      "logps/chosen": -42.24382781982422,
+      "logps/rejected": -66.47443389892578,
+      "loss": 0.0657,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8911622762680054,
+      "rewards/margins": 6.7061848640441895,
+      "rewards/rejected": -7.597347259521484,
+      "step": 290
+    },
+    {
+      "epoch": 3.4437869822485205,
+      "grad_norm": 12.561936917149565,
+      "learning_rate": 3.4985948604313237e-07,
+      "logits/chosen": -0.8755354285240173,
+      "logits/rejected": -0.8271730542182922,
+      "logps/chosen": -31.485605239868164,
+      "logps/rejected": -58.347190856933594,
+      "loss": 0.0752,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.19215503334999084,
+      "rewards/margins": 5.973523139953613,
+      "rewards/rejected": -5.781367301940918,
+      "step": 291
+    },
+    {
+      "epoch": 3.455621301775148,
+      "grad_norm": 11.955176243577917,
+      "learning_rate": 3.486660515583691e-07,
+      "logits/chosen": -1.228103518486023,
+      "logits/rejected": -1.2567815780639648,
+      "logps/chosen": -41.65220260620117,
+      "logps/rejected": -58.665496826171875,
+      "loss": 0.0598,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.3302292823791504,
+      "rewards/margins": 5.375921726226807,
+      "rewards/rejected": -6.706151008605957,
+      "step": 292
+    },
+    {
+      "epoch": 3.467455621301775,
+      "grad_norm": 10.442695814707806,
+      "learning_rate": 3.474699478027918e-07,
+      "logits/chosen": -1.121075987815857,
+      "logits/rejected": -1.162339687347412,
+      "logps/chosen": -40.622459411621094,
+      "logps/rejected": -55.069026947021484,
+      "loss": 0.0514,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2291250228881836,
+      "rewards/margins": 5.634562015533447,
+      "rewards/rejected": -5.863687515258789,
+      "step": 293
+    },
+    {
+      "epoch": 3.4792899408284024,
+      "grad_norm": 9.212432774516737,
+      "learning_rate": 3.4627120713529983e-07,
+      "logits/chosen": -1.1274569034576416,
+      "logits/rejected": -1.0957834720611572,
+      "logps/chosen": -29.82750129699707,
+      "logps/rejected": -60.58903503417969,
+      "loss": 0.0459,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8590860366821289,
+      "rewards/margins": 6.390963554382324,
+      "rewards/rejected": -7.250050067901611,
+      "step": 294
+    },
+    {
+      "epoch": 3.4911242603550297,
+      "grad_norm": 8.987438061891513,
+      "learning_rate": 3.4506986198613077e-07,
+      "logits/chosen": -1.041571855545044,
+      "logits/rejected": -1.0537890195846558,
+      "logps/chosen": -41.13783264160156,
+      "logps/rejected": -72.97576904296875,
+      "loss": 0.0419,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5465377569198608,
+      "rewards/margins": 7.734911918640137,
+      "rewards/rejected": -8.281450271606445,
+      "step": 295
+    },
+    {
+      "epoch": 3.502958579881657,
+      "grad_norm": 7.117159631612635,
+      "learning_rate": 3.438659448559825e-07,
+      "logits/chosen": -1.280914306640625,
+      "logits/rejected": -1.3170225620269775,
+      "logps/chosen": -36.09680938720703,
+      "logps/rejected": -57.332427978515625,
+      "loss": 0.0397,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9314979910850525,
+      "rewards/margins": 7.998897075653076,
+      "rewards/rejected": -8.930395126342773,
+      "step": 296
+    },
+    {
+      "epoch": 3.5147928994082838,
+      "grad_norm": 9.213072401647429,
+      "learning_rate": 3.4265948831513434e-07,
+      "logits/chosen": -1.0224130153656006,
+      "logits/rejected": -1.0106931924819946,
+      "logps/chosen": -51.425453186035156,
+      "logps/rejected": -59.91952896118164,
+      "loss": 0.0492,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6506232023239136,
+      "rewards/margins": 6.213413715362549,
+      "rewards/rejected": -6.864037036895752,
+      "step": 297
+    },
+    {
+      "epoch": 3.5266272189349115,
+      "grad_norm": 8.34764646763322,
+      "learning_rate": 3.414505250025659e-07,
+      "logits/chosen": -0.5949307680130005,
+      "logits/rejected": -0.6818762421607971,
+      "logps/chosen": -31.321874618530273,
+      "logps/rejected": -45.574928283691406,
+      "loss": 0.0393,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6341886520385742,
+      "rewards/margins": 4.197469234466553,
+      "rewards/rejected": -4.831657886505127,
+      "step": 298
+    },
+    {
+      "epoch": 3.5384615384615383,
+      "grad_norm": 9.531733325613454,
+      "learning_rate": 3.402390876250737e-07,
+      "logits/chosen": -0.9537699222564697,
+      "logits/rejected": -0.8830502033233643,
+      "logps/chosen": -43.555381774902344,
+      "logps/rejected": -60.58173751831055,
+      "loss": 0.0451,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.3789546489715576,
+      "rewards/margins": 5.775010585784912,
+      "rewards/rejected": -7.153965473175049,
+      "step": 299
+    },
+    {
+      "epoch": 3.5502958579881656,
+      "grad_norm": 9.379884768180517,
+      "learning_rate": 3.390252089563867e-07,
+      "logits/chosen": -1.3647562265396118,
+      "logits/rejected": -1.2569482326507568,
+      "logps/chosen": -32.759376525878906,
+      "logps/rejected": -46.86608123779297,
+      "loss": 0.0402,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.663446307182312,
+      "rewards/margins": 4.781562805175781,
+      "rewards/rejected": -5.445009231567383,
+      "step": 300
+    },
+    {
+      "epoch": 3.562130177514793,
+      "grad_norm": 6.852898201829805,
+      "learning_rate": 3.3780892183627974e-07,
+      "logits/chosen": -1.2863893508911133,
+      "logits/rejected": -1.256026029586792,
+      "logps/chosen": -45.72820281982422,
+      "logps/rejected": -76.36709594726562,
+      "loss": 0.0311,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.662516713142395,
+      "rewards/margins": 7.760738372802734,
+      "rewards/rejected": -8.423254013061523,
+      "step": 301
+    },
+    {
+      "epoch": 3.57396449704142,
+      "grad_norm": 9.656007482244595,
+      "learning_rate": 3.3659025916968475e-07,
+      "logits/chosen": -1.1637496948242188,
+      "logits/rejected": -1.0999524593353271,
+      "logps/chosen": -40.303245544433594,
+      "logps/rejected": -65.28178405761719,
+      "loss": 0.048,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.500770092010498,
+      "rewards/margins": 6.479090690612793,
+      "rewards/rejected": -7.979861259460449,
+      "step": 302
+    },
+    {
+      "epoch": 3.5857988165680474,
+      "grad_norm": 6.56810361487223,
+      "learning_rate": 3.353692539258006e-07,
+      "logits/chosen": -1.1888104677200317,
+      "logits/rejected": -1.1970776319503784,
+      "logps/chosen": -55.63449478149414,
+      "logps/rejected": -79.99554443359375,
+      "loss": 0.0402,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.9003673791885376,
+      "rewards/margins": 7.262213706970215,
+      "rewards/rejected": -9.162581443786621,
+      "step": 303
+    },
+    {
+      "epoch": 3.5976331360946747,
+      "grad_norm": 10.490981950638373,
+      "learning_rate": 3.3414593913720155e-07,
+      "logits/chosen": -0.9099432229995728,
+      "logits/rejected": -0.8601805567741394,
+      "logps/chosen": -41.21411895751953,
+      "logps/rejected": -65.25252532958984,
+      "loss": 0.0569,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1893513202667236,
+      "rewards/margins": 5.567013740539551,
+      "rewards/rejected": -6.756364822387695,
+      "step": 304
+    },
+    {
+      "epoch": 3.609467455621302,
+      "grad_norm": 6.610277331758854,
+      "learning_rate": 3.329203478989431e-07,
+      "logits/chosen": -1.0593793392181396,
+      "logits/rejected": -1.0052438974380493,
+      "logps/chosen": -36.38404083251953,
+      "logps/rejected": -59.90925598144531,
+      "loss": 0.0423,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4144278764724731,
+      "rewards/margins": 5.430604457855225,
+      "rewards/rejected": -6.845032215118408,
+      "step": 305
+    },
+    {
+      "epoch": 3.621301775147929,
+      "grad_norm": 9.012951266000211,
+      "learning_rate": 3.3169251336766697e-07,
+      "logits/chosen": -1.0458168983459473,
+      "logits/rejected": -1.012871265411377,
+      "logps/chosen": -35.66429138183594,
+      "logps/rejected": -57.909767150878906,
+      "loss": 0.0446,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.3210654258728027,
+      "rewards/margins": 6.225252151489258,
+      "rewards/rejected": -7.546317100524902,
+      "step": 306
+    },
+    {
+      "epoch": 3.633136094674556,
+      "grad_norm": 10.588611564681699,
+      "learning_rate": 3.3046246876070405e-07,
+      "logits/chosen": -1.1131824254989624,
+      "logits/rejected": -1.117433786392212,
+      "logps/chosen": -41.028900146484375,
+      "logps/rejected": -58.1406135559082,
+      "loss": 0.0481,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.07064366340637207,
+      "rewards/margins": 6.420350551605225,
+      "rewards/rejected": -6.490993976593018,
+      "step": 307
+    },
+    {
+      "epoch": 3.6449704142011834,
+      "grad_norm": 10.357494650059015,
+      "learning_rate": 3.2923024735517567e-07,
+      "logits/chosen": -1.2510991096496582,
+      "logits/rejected": -1.169487476348877,
+      "logps/chosen": -49.32808303833008,
+      "logps/rejected": -77.26386260986328,
+      "loss": 0.0458,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1415975093841553,
+      "rewards/margins": 8.103403091430664,
+      "rewards/rejected": -9.245000839233398,
+      "step": 308
+    },
+    {
+      "epoch": 3.6568047337278107,
+      "grad_norm": 10.469013609843655,
+      "learning_rate": 3.279958824870934e-07,
+      "logits/chosen": -0.921501100063324,
+      "logits/rejected": -0.8510603308677673,
+      "logps/chosen": -37.00920486450195,
+      "logps/rejected": -50.785682678222656,
+      "loss": 0.0538,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2592521011829376,
+      "rewards/margins": 3.4533708095550537,
+      "rewards/rejected": -3.1941187381744385,
+      "step": 309
+    },
+    {
+      "epoch": 3.668639053254438,
+      "grad_norm": 8.143849115083505,
+      "learning_rate": 3.2675940755045713e-07,
+      "logits/chosen": -1.1272180080413818,
+      "logits/rejected": -1.1566120386123657,
+      "logps/chosen": -41.058563232421875,
+      "logps/rejected": -59.50581741333008,
+      "loss": 0.0535,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3426307439804077,
+      "rewards/margins": 5.675926685333252,
+      "rewards/rejected": -6.018557548522949,
+      "step": 310
+    },
+    {
+      "epoch": 3.6804733727810652,
+      "grad_norm": 7.797821676664401,
+      "learning_rate": 3.2552085599635167e-07,
+      "logits/chosen": -1.2833009958267212,
+      "logits/rejected": -1.1694531440734863,
+      "logps/chosen": -32.42615509033203,
+      "logps/rejected": -64.32304382324219,
+      "loss": 0.0376,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2963796854019165,
+      "rewards/margins": 6.591299533843994,
+      "rewards/rejected": -7.887678623199463,
+      "step": 311
+    },
+    {
+      "epoch": 3.6923076923076925,
+      "grad_norm": 6.89356963474552,
+      "learning_rate": 3.242802613320418e-07,
+      "logits/chosen": -1.393490195274353,
+      "logits/rejected": -1.3847185373306274,
+      "logps/chosen": -41.63416290283203,
+      "logps/rejected": -70.63141632080078,
+      "loss": 0.0327,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9141947031021118,
+      "rewards/margins": 7.590588569641113,
+      "rewards/rejected": -8.504783630371094,
+      "step": 312
+    },
+    {
+      "epoch": 3.7041420118343193,
+      "grad_norm": 11.222653962104593,
+      "learning_rate": 3.2303765712006585e-07,
+      "logits/chosen": -1.1832377910614014,
+      "logits/rejected": -1.1547398567199707,
+      "logps/chosen": -40.68590545654297,
+      "logps/rejected": -70.75530242919922,
+      "loss": 0.0511,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.7116194367408752,
+      "rewards/margins": 5.869533538818359,
+      "rewards/rejected": -6.58115291595459,
+      "step": 313
+    },
+    {
+      "epoch": 3.7159763313609466,
+      "grad_norm": 6.708240807361137,
+      "learning_rate": 3.217930769773275e-07,
+      "logits/chosen": -1.2435672283172607,
+      "logits/rejected": -1.035918951034546,
+      "logps/chosen": -50.29835891723633,
+      "logps/rejected": -74.53111267089844,
+      "loss": 0.0359,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.17033040523529053,
+      "rewards/margins": 6.697113513946533,
+      "rewards/rejected": -6.867444038391113,
+      "step": 314
+    },
+    {
+      "epoch": 3.727810650887574,
+      "grad_norm": 9.42387101375972,
+      "learning_rate": 3.2054655457418647e-07,
+      "logits/chosen": -1.0515731573104858,
+      "logits/rejected": -1.0346689224243164,
+      "logps/chosen": -35.189369201660156,
+      "logps/rejected": -55.9654655456543,
+      "loss": 0.0497,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5117961168289185,
+      "rewards/margins": 5.10178804397583,
+      "rewards/rejected": -5.613584041595459,
+      "step": 315
+    },
+    {
+      "epoch": 3.739644970414201,
+      "grad_norm": 6.516712675879623,
+      "learning_rate": 3.1929812363354764e-07,
+      "logits/chosen": -1.3077701330184937,
+      "logits/rejected": -1.1945271492004395,
+      "logps/chosen": -35.00279235839844,
+      "logps/rejected": -54.25492477416992,
+      "loss": 0.0313,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9486129283905029,
+      "rewards/margins": 5.082883834838867,
+      "rewards/rejected": -6.031496524810791,
+      "step": 316
+    },
+    {
+      "epoch": 3.7514792899408285,
+      "grad_norm": 8.087941236249774,
+      "learning_rate": 3.1804781792994867e-07,
+      "logits/chosen": -1.1108276844024658,
+      "logits/rejected": -1.2471861839294434,
+      "logps/chosen": -41.833309173583984,
+      "logps/rejected": -61.84939193725586,
+      "loss": 0.0482,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.742074728012085,
+      "rewards/margins": 7.079897403717041,
+      "rewards/rejected": -8.821971893310547,
+      "step": 317
+    },
+    {
+      "epoch": 3.7633136094674557,
+      "grad_norm": 10.801590352605164,
+      "learning_rate": 3.167956712886463e-07,
+      "logits/chosen": -1.373018503189087,
+      "logits/rejected": -1.1945630311965942,
+      "logps/chosen": -30.055692672729492,
+      "logps/rejected": -56.957427978515625,
+      "loss": 0.053,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4738597273826599,
+      "rewards/margins": 5.641894817352295,
+      "rewards/rejected": -6.115754127502441,
+      "step": 318
+    },
+    {
+      "epoch": 3.775147928994083,
+      "grad_norm": 10.24310089782514,
+      "learning_rate": 3.155417175847011e-07,
+      "logits/chosen": -1.0630590915679932,
+      "logits/rejected": -1.0347498655319214,
+      "logps/chosen": -32.52122497558594,
+      "logps/rejected": -51.32162857055664,
+      "loss": 0.0591,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5487163066864014,
+      "rewards/margins": 4.911801815032959,
+      "rewards/rejected": -5.460517883300781,
+      "step": 319
+    },
+    {
+      "epoch": 3.78698224852071,
+      "grad_norm": 6.870366909127477,
+      "learning_rate": 3.142859907420615e-07,
+      "logits/chosen": -1.1960430145263672,
+      "logits/rejected": -1.1303733587265015,
+      "logps/chosen": -41.18871307373047,
+      "logps/rejected": -58.43561553955078,
+      "loss": 0.0309,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.28152644634246826,
+      "rewards/margins": 5.013343811035156,
+      "rewards/rejected": -5.294870376586914,
+      "step": 320
+    },
+    {
+      "epoch": 3.798816568047337,
+      "grad_norm": 8.050195637302277,
+      "learning_rate": 3.1302852473264537e-07,
+      "logits/chosen": -0.8659465312957764,
+      "logits/rejected": -0.8467557430267334,
+      "logps/chosen": -33.895164489746094,
+      "logps/rejected": -50.80845642089844,
+      "loss": 0.0497,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.08278264105319977,
+      "rewards/margins": 4.752477169036865,
+      "rewards/rejected": -4.669694900512695,
+      "step": 321
+    },
+    {
+      "epoch": 3.8106508875739644,
+      "grad_norm": 8.069576088623236,
+      "learning_rate": 3.117693535754213e-07,
+      "logits/chosen": -0.9457200765609741,
+      "logits/rejected": -1.0279572010040283,
+      "logps/chosen": -37.87699890136719,
+      "logps/rejected": -58.80052185058594,
+      "loss": 0.0452,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.5841734409332275,
+      "rewards/margins": 5.943948268890381,
+      "rewards/rejected": -7.528121471405029,
+      "step": 322
+    },
+    {
+      "epoch": 3.8224852071005917,
+      "grad_norm": 9.494312396980359,
+      "learning_rate": 3.105085113354885e-07,
+      "logits/chosen": -1.0865830183029175,
+      "logits/rejected": -1.06438410282135,
+      "logps/chosen": -34.925296783447266,
+      "logps/rejected": -50.69199752807617,
+      "loss": 0.0619,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6583223342895508,
+      "rewards/margins": 5.800050735473633,
+      "rewards/rejected": -6.458373069763184,
+      "step": 323
+    },
+    {
+      "epoch": 3.834319526627219,
+      "grad_norm": 10.61028311721643,
+      "learning_rate": 3.092460321231547e-07,
+      "logits/chosen": -1.1025853157043457,
+      "logits/rejected": -1.3029682636260986,
+      "logps/chosen": -45.37340545654297,
+      "logps/rejected": -57.43135452270508,
+      "loss": 0.0482,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.6712946891784668,
+      "rewards/margins": 6.542860984802246,
+      "rewards/rejected": -8.214155197143555,
+      "step": 324
+    },
+    {
+      "epoch": 3.8461538461538463,
+      "grad_norm": 9.127446196094581,
+      "learning_rate": 3.079819500930138e-07,
+      "logits/chosen": -1.025221824645996,
+      "logits/rejected": -1.021698236465454,
+      "logps/chosen": -35.102596282958984,
+      "logps/rejected": -62.06328582763672,
+      "loss": 0.0451,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.45298677682876587,
+      "rewards/margins": 5.769662857055664,
+      "rewards/rejected": -6.222650051116943,
+      "step": 325
+    },
+    {
+      "epoch": 3.8579881656804735,
+      "grad_norm": 12.29389361531357,
+      "learning_rate": 3.0671629944302164e-07,
+      "logits/chosen": -1.0510854721069336,
+      "logits/rejected": -0.951848566532135,
+      "logps/chosen": -36.842681884765625,
+      "logps/rejected": -58.03953552246094,
+      "loss": 0.0686,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3583628535270691,
+      "rewards/margins": 6.295431613922119,
+      "rewards/rejected": -6.653794288635254,
+      "step": 326
+    },
+    {
+      "epoch": 3.8698224852071004,
+      "grad_norm": 6.750607959887297,
+      "learning_rate": 3.054491144135707e-07,
+      "logits/chosen": -1.186820387840271,
+      "logits/rejected": -1.2949779033660889,
+      "logps/chosen": -38.38534164428711,
+      "logps/rejected": -59.20850372314453,
+      "loss": 0.0295,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5839760303497314,
+      "rewards/margins": 7.092982292175293,
+      "rewards/rejected": -7.6769585609436035,
+      "step": 327
+    },
+    {
+      "epoch": 3.8816568047337277,
+      "grad_norm": 11.006787084671119,
+      "learning_rate": 3.0418042928656415e-07,
+      "logits/chosen": -0.9911553859710693,
+      "logits/rejected": -1.0456457138061523,
+      "logps/chosen": -46.48586654663086,
+      "logps/rejected": -65.22984313964844,
+      "loss": 0.0537,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.38870108127593994,
+      "rewards/margins": 6.2278571128845215,
+      "rewards/rejected": -6.616558074951172,
+      "step": 328
+    },
+    {
+      "epoch": 3.893491124260355,
+      "grad_norm": 10.159592627032264,
+      "learning_rate": 3.029102783844879e-07,
+      "logits/chosen": -0.9870089292526245,
+      "logits/rejected": -1.0301330089569092,
+      "logps/chosen": -43.52060317993164,
+      "logps/rejected": -66.05502319335938,
+      "loss": 0.0515,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.344921350479126,
+      "rewards/margins": 7.7191667556762695,
+      "rewards/rejected": -9.064088821411133,
+      "step": 329
+    },
+    {
+      "epoch": 3.905325443786982,
+      "grad_norm": 9.8965394155921,
+      "learning_rate": 3.016386960694827e-07,
+      "logits/chosen": -1.0783404111862183,
+      "logits/rejected": -1.1657443046569824,
+      "logps/chosen": -35.86804962158203,
+      "logps/rejected": -49.75996017456055,
+      "loss": 0.0476,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5152837038040161,
+      "rewards/margins": 5.677330017089844,
+      "rewards/rejected": -6.1926140785217285,
+      "step": 330
+    },
+    {
+      "epoch": 3.9171597633136095,
+      "grad_norm": 7.647123014651004,
+      "learning_rate": 3.003657167424139e-07,
+      "logits/chosen": -1.116632103919983,
+      "logits/rejected": -1.007010579109192,
+      "logps/chosen": -27.30445671081543,
+      "logps/rejected": -46.01488494873047,
+      "loss": 0.0448,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.475461483001709,
+      "rewards/margins": 4.353217601776123,
+      "rewards/rejected": -4.828679084777832,
+      "step": 331
+    },
+    {
+      "epoch": 3.9289940828402368,
+      "grad_norm": 7.370007475987176,
+      "learning_rate": 2.990913748419411e-07,
+      "logits/chosen": -1.0164567232131958,
+      "logits/rejected": -1.099671483039856,
+      "logps/chosen": -42.57475662231445,
+      "logps/rejected": -57.868221282958984,
+      "loss": 0.0329,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8491787910461426,
+      "rewards/margins": 5.017266750335693,
+      "rewards/rejected": -5.866445541381836,
+      "step": 332
+    },
+    {
+      "epoch": 3.940828402366864,
+      "grad_norm": 11.13700718226015,
+      "learning_rate": 2.978157048435863e-07,
+      "logits/chosen": -1.2751820087432861,
+      "logits/rejected": -1.3596327304840088,
+      "logps/chosen": -44.9207878112793,
+      "logps/rejected": -69.44329833984375,
+      "loss": 0.0585,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.0238862037658691,
+      "rewards/margins": 6.633727073669434,
+      "rewards/rejected": -7.657613754272461,
+      "step": 333
+    },
+    {
+      "epoch": 3.952662721893491,
+      "grad_norm": 9.53101226669104,
+      "learning_rate": 2.9653874125880167e-07,
+      "logits/chosen": -1.184804081916809,
+      "logits/rejected": -1.2233521938323975,
+      "logps/chosen": -33.34724807739258,
+      "logps/rejected": -57.77789306640625,
+      "loss": 0.041,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6291203498840332,
+      "rewards/margins": 6.075474739074707,
+      "rewards/rejected": -6.704594612121582,
+      "step": 334
+    },
+    {
+      "epoch": 3.9644970414201186,
+      "grad_norm": 7.607623067226138,
+      "learning_rate": 2.9526051863403517e-07,
+      "logits/chosen": -0.9798343181610107,
+      "logits/rejected": -1.0512641668319702,
+      "logps/chosen": -33.80629348754883,
+      "logps/rejected": -59.37635803222656,
+      "loss": 0.0379,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.18926861882209778,
+      "rewards/margins": 6.627708435058594,
+      "rewards/rejected": -6.438440322875977,
+      "step": 335
+    },
+    {
+      "epoch": 3.9763313609467454,
+      "grad_norm": 8.807254950481648,
+      "learning_rate": 2.9398107154979634e-07,
+      "logits/chosen": -1.0378540754318237,
+      "logits/rejected": -1.236304759979248,
+      "logps/chosen": -53.999755859375,
+      "logps/rejected": -71.97964477539062,
+      "loss": 0.048,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.30962872505188,
+      "rewards/margins": 8.442242622375488,
+      "rewards/rejected": -10.751871109008789,
+      "step": 336
+    },
+    {
+      "epoch": 3.9881656804733727,
+      "grad_norm": 7.885596351737529,
+      "learning_rate": 2.9270043461972097e-07,
+      "logits/chosen": -1.066207766532898,
+      "logits/rejected": -1.1475964784622192,
+      "logps/chosen": -51.98231506347656,
+      "logps/rejected": -71.3375244140625,
+      "loss": 0.0408,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8781924247741699,
+      "rewards/margins": 7.75531005859375,
+      "rewards/rejected": -8.633502006530762,
+      "step": 337
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 9.210155965565953,
+      "learning_rate": 2.9141864248963427e-07,
+      "logits/chosen": -1.214263916015625,
+      "logits/rejected": -1.1584827899932861,
+      "logps/chosen": -45.34033966064453,
+      "logps/rejected": -73.4906005859375,
+      "loss": 0.0458,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.079200029373169,
+      "rewards/margins": 5.90239143371582,
+      "rewards/rejected": -6.98159122467041,
+      "step": 338
+    },
+    {
+      "epoch": 4.011834319526627,
+      "grad_norm": 6.508446991164881,
+      "learning_rate": 2.9013572983661375e-07,
+      "logits/chosen": -1.2331136465072632,
+      "logits/rejected": -1.3492088317871094,
+      "logps/chosen": -40.93061828613281,
+      "logps/rejected": -59.18800354003906,
+      "loss": 0.0279,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.27319905161857605,
+      "rewards/margins": 7.39341926574707,
+      "rewards/rejected": -7.666618347167969,
+      "step": 339
+    },
+    {
+      "epoch": 4.023668639053255,
+      "grad_norm": 7.338060837553236,
+      "learning_rate": 2.8885173136805125e-07,
+      "logits/chosen": -1.226414442062378,
+      "logits/rejected": -1.2364081144332886,
+      "logps/chosen": -43.55427551269531,
+      "logps/rejected": -65.36073303222656,
+      "loss": 0.037,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.8232107162475586,
+      "rewards/margins": 7.595679759979248,
+      "rewards/rejected": -9.418889999389648,
+      "step": 340
+    },
+    {
+      "epoch": 4.035502958579881,
+      "grad_norm": 4.628023712795517,
+      "learning_rate": 2.8756668182071357e-07,
+      "logits/chosen": -1.180713176727295,
+      "logits/rejected": -1.1071914434432983,
+      "logps/chosen": -41.24338150024414,
+      "logps/rejected": -62.83121109008789,
+      "loss": 0.0232,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.6221990585327148,
+      "rewards/margins": 6.7731523513793945,
+      "rewards/rejected": -8.39535140991211,
+      "step": 341
+    },
+    {
+      "epoch": 4.047337278106509,
+      "grad_norm": 6.104842751233805,
+      "learning_rate": 2.862806159598032e-07,
+      "logits/chosen": -0.9698923826217651,
+      "logits/rejected": -0.9841170310974121,
+      "logps/chosen": -39.91763687133789,
+      "logps/rejected": -61.13340759277344,
+      "loss": 0.0302,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4580131769180298,
+      "rewards/margins": 5.857044219970703,
+      "rewards/rejected": -7.315057754516602,
+      "step": 342
+    },
+    {
+      "epoch": 4.059171597633136,
+      "grad_norm": 6.103306838069507,
+      "learning_rate": 2.8499356857801744e-07,
+      "logits/chosen": -1.0496821403503418,
+      "logits/rejected": -1.0485060214996338,
+      "logps/chosen": -36.92170715332031,
+      "logps/rejected": -52.14490509033203,
+      "loss": 0.0283,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.119423747062683,
+      "rewards/margins": 5.038466453552246,
+      "rewards/rejected": -6.1578898429870605,
+      "step": 343
+    },
+    {
+      "epoch": 4.071005917159764,
+      "grad_norm": 5.23129093382033,
+      "learning_rate": 2.837055744946072e-07,
+      "logits/chosen": -1.3063690662384033,
+      "logits/rejected": -1.1161246299743652,
+      "logps/chosen": -37.174598693847656,
+      "logps/rejected": -70.67867279052734,
+      "loss": 0.0267,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.7968075275421143,
+      "rewards/margins": 8.80527114868164,
+      "rewards/rejected": -10.602078437805176,
+      "step": 344
+    },
+    {
+      "epoch": 4.0828402366863905,
+      "grad_norm": 5.119583740628856,
+      "learning_rate": 2.8241666855443526e-07,
+      "logits/chosen": -1.1465834379196167,
+      "logits/rejected": -0.9921385049819946,
+      "logps/chosen": -43.1416015625,
+      "logps/rejected": -62.68724822998047,
+      "loss": 0.0263,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9080947637557983,
+      "rewards/margins": 6.91895866394043,
+      "rewards/rejected": -7.827053070068359,
+      "step": 345
+    },
+    {
+      "epoch": 4.094674556213017,
+      "grad_norm": 7.489855061260909,
+      "learning_rate": 2.811268856270332e-07,
+      "logits/chosen": -1.083118200302124,
+      "logits/rejected": -1.002679467201233,
+      "logps/chosen": -38.36323547363281,
+      "logps/rejected": -67.2934799194336,
+      "loss": 0.0394,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.746131181716919,
+      "rewards/margins": 6.556519508361816,
+      "rewards/rejected": -7.302650451660156,
+      "step": 346
+    },
+    {
+      "epoch": 4.106508875739645,
+      "grad_norm": 5.683245039375868,
+      "learning_rate": 2.798362606056583e-07,
+      "logits/chosen": -1.0683708190917969,
+      "logits/rejected": -1.0461918115615845,
+      "logps/chosen": -44.84065628051758,
+      "logps/rejected": -65.26290130615234,
+      "loss": 0.024,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4837861061096191,
+      "rewards/margins": 6.347103118896484,
+      "rewards/rejected": -7.8308892250061035,
+      "step": 347
+    },
+    {
+      "epoch": 4.118343195266272,
+      "grad_norm": 6.990533138399409,
+      "learning_rate": 2.7854482840634965e-07,
+      "logits/chosen": -1.2399723529815674,
+      "logits/rejected": -1.242792010307312,
+      "logps/chosen": -35.35354995727539,
+      "logps/rejected": -50.29901123046875,
+      "loss": 0.0362,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.814429759979248,
+      "rewards/margins": 4.70147180557251,
+      "rewards/rejected": -5.5159010887146,
+      "step": 348
+    },
+    {
+      "epoch": 4.1301775147929,
+      "grad_norm": 6.319829009469765,
+      "learning_rate": 2.772526239669831e-07,
+      "logits/chosen": -1.2442165613174438,
+      "logits/rejected": -1.2576425075531006,
+      "logps/chosen": -41.020626068115234,
+      "logps/rejected": -71.2080078125,
+      "loss": 0.0333,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.1832388937473297,
+      "rewards/margins": 6.497980117797852,
+      "rewards/rejected": -6.681219100952148,
+      "step": 349
+    },
+    {
+      "epoch": 4.1420118343195265,
+      "grad_norm": 7.137478103836617,
+      "learning_rate": 2.759596822463267e-07,
+      "logits/chosen": -1.0815571546554565,
+      "logits/rejected": -1.0727592706680298,
+      "logps/chosen": -45.82701110839844,
+      "logps/rejected": -74.0084228515625,
+      "loss": 0.032,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.5913039445877075,
+      "rewards/margins": 7.610474586486816,
+      "rewards/rejected": -9.201778411865234,
+      "step": 350
+    },
+    {
+      "epoch": 4.153846153846154,
+      "grad_norm": 6.127060479740991,
+      "learning_rate": 2.746660382230944e-07,
+      "logits/chosen": -1.1690300703048706,
+      "logits/rejected": -1.1895644664764404,
+      "logps/chosen": -31.055179595947266,
+      "logps/rejected": -52.40230178833008,
+      "loss": 0.0282,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1996252536773682,
+      "rewards/margins": 6.068203926086426,
+      "rewards/rejected": -7.267828941345215,
+      "step": 351
+    },
+    {
+      "epoch": 4.165680473372781,
+      "grad_norm": 6.941829346693308,
+      "learning_rate": 2.73371726895e-07,
+      "logits/chosen": -1.4027228355407715,
+      "logits/rejected": -1.4094792604446411,
+      "logps/chosen": -36.16400909423828,
+      "logps/rejected": -54.41309356689453,
+      "loss": 0.0423,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.0032942295074463,
+      "rewards/margins": 5.790349960327148,
+      "rewards/rejected": -6.793643951416016,
+      "step": 352
+    },
+    {
+      "epoch": 4.177514792899408,
+      "grad_norm": 5.664169120525594,
+      "learning_rate": 2.7207678327781036e-07,
+      "logits/chosen": -1.0156735181808472,
+      "logits/rejected": -1.0846226215362549,
+      "logps/chosen": -41.97883605957031,
+      "logps/rejected": -58.83436584472656,
+      "loss": 0.0326,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6752943396568298,
+      "rewards/margins": 6.577206611633301,
+      "rewards/rejected": -7.252501487731934,
+      "step": 353
+    },
+    {
+      "epoch": 4.189349112426036,
+      "grad_norm": 4.278910321706378,
+      "learning_rate": 2.7078124240439793e-07,
+      "logits/chosen": -1.4016739130020142,
+      "logits/rejected": -1.3881572484970093,
+      "logps/chosen": -40.908294677734375,
+      "logps/rejected": -81.16354370117188,
+      "loss": 0.0216,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1897321939468384,
+      "rewards/margins": 9.733206748962402,
+      "rewards/rejected": -10.92293930053711,
+      "step": 354
+    },
+    {
+      "epoch": 4.201183431952662,
+      "grad_norm": 5.662906607186885,
+      "learning_rate": 2.6948513932379307e-07,
+      "logits/chosen": -1.127701997756958,
+      "logits/rejected": -1.2144131660461426,
+      "logps/chosen": -40.303466796875,
+      "logps/rejected": -53.539791107177734,
+      "loss": 0.031,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6169775724411011,
+      "rewards/margins": 5.929780006408691,
+      "rewards/rejected": -6.54675817489624,
+      "step": 355
+    },
+    {
+      "epoch": 4.21301775147929,
+      "grad_norm": 5.869401589278453,
+      "learning_rate": 2.68188509100236e-07,
+      "logits/chosen": -1.1310701370239258,
+      "logits/rejected": -0.9885028004646301,
+      "logps/chosen": -43.84556579589844,
+      "logps/rejected": -67.79801940917969,
+      "loss": 0.0368,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5635640621185303,
+      "rewards/margins": 6.635869979858398,
+      "rewards/rejected": -7.199433326721191,
+      "step": 356
+    },
+    {
+      "epoch": 4.224852071005917,
+      "grad_norm": 6.4938093501862335,
+      "learning_rate": 2.668913868122279e-07,
+      "logits/chosen": -1.145686388015747,
+      "logits/rejected": -1.1048319339752197,
+      "logps/chosen": -38.25225830078125,
+      "logps/rejected": -84.31645965576172,
+      "loss": 0.0317,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9592878222465515,
+      "rewards/margins": 8.200289726257324,
+      "rewards/rejected": -9.159577369689941,
+      "step": 357
+    },
+    {
+      "epoch": 4.236686390532545,
+      "grad_norm": 5.3318941750637805,
+      "learning_rate": 2.6559380755158206e-07,
+      "logits/chosen": -1.306983470916748,
+      "logits/rejected": -1.3083993196487427,
+      "logps/chosen": -32.1518440246582,
+      "logps/rejected": -57.46953582763672,
+      "loss": 0.0212,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.3908557891845703,
+      "rewards/margins": 6.340449810028076,
+      "rewards/rejected": -7.731306076049805,
+      "step": 358
+    },
+    {
+      "epoch": 4.2485207100591715,
+      "grad_norm": 4.583272241534079,
+      "learning_rate": 2.642958064224747e-07,
+      "logits/chosen": -1.1370916366577148,
+      "logits/rejected": -1.223602056503296,
+      "logps/chosen": -45.049015045166016,
+      "logps/rejected": -67.99398803710938,
+      "loss": 0.0222,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5301861763000488,
+      "rewards/margins": 7.891375541687012,
+      "rewards/rejected": -8.421562194824219,
+      "step": 359
+    },
+    {
+      "epoch": 4.260355029585799,
+      "grad_norm": 5.8869583103527505,
+      "learning_rate": 2.629974185404951e-07,
+      "logits/chosen": -1.142185926437378,
+      "logits/rejected": -1.0663723945617676,
+      "logps/chosen": -54.66786193847656,
+      "logps/rejected": -72.61735534667969,
+      "loss": 0.0292,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.0223376750946045,
+      "rewards/margins": 5.790317535400391,
+      "rewards/rejected": -7.812655448913574,
+      "step": 360
+    },
+    {
+      "epoch": 4.272189349112426,
+      "grad_norm": 5.434403797417726,
+      "learning_rate": 2.616986790316952e-07,
+      "logits/chosen": -1.084727168083191,
+      "logits/rejected": -1.000258207321167,
+      "logps/chosen": -41.456581115722656,
+      "logps/rejected": -65.51860046386719,
+      "loss": 0.0308,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9817674160003662,
+      "rewards/margins": 6.707640647888184,
+      "rewards/rejected": -7.689408302307129,
+      "step": 361
+    },
+    {
+      "epoch": 4.284023668639053,
+      "grad_norm": 7.370310021015268,
+      "learning_rate": 2.603996230316402e-07,
+      "logits/chosen": -0.9062331318855286,
+      "logits/rejected": -1.0064326524734497,
+      "logps/chosen": -37.973838806152344,
+      "logps/rejected": -57.57244873046875,
+      "loss": 0.0387,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.2495323419570923,
+      "rewards/margins": 6.539309501647949,
+      "rewards/rejected": -7.78884220123291,
+      "step": 362
+    },
+    {
+      "epoch": 4.295857988165681,
+      "grad_norm": 4.712912974501213,
+      "learning_rate": 2.5910028568445716e-07,
+      "logits/chosen": -1.0892990827560425,
+      "logits/rejected": -0.9764453172683716,
+      "logps/chosen": -36.994937896728516,
+      "logps/rejected": -60.98323059082031,
+      "loss": 0.0238,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5305968523025513,
+      "rewards/margins": 6.173823356628418,
+      "rewards/rejected": -6.704420566558838,
+      "step": 363
+    },
+    {
+      "epoch": 4.3076923076923075,
+      "grad_norm": 8.269082594101617,
+      "learning_rate": 2.5780070214188474e-07,
+      "logits/chosen": -1.2240197658538818,
+      "logits/rejected": -1.2403483390808105,
+      "logps/chosen": -44.62925720214844,
+      "logps/rejected": -68.63853454589844,
+      "loss": 0.0348,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.3063997030258179,
+      "rewards/margins": 9.024595260620117,
+      "rewards/rejected": -10.330994606018066,
+      "step": 364
+    },
+    {
+      "epoch": 4.319526627218935,
+      "grad_norm": 7.007032072350081,
+      "learning_rate": 2.5650090756232226e-07,
+      "logits/chosen": -0.9315057992935181,
+      "logits/rejected": -0.9429672360420227,
+      "logps/chosen": -41.07881546020508,
+      "logps/rejected": -55.53737258911133,
+      "loss": 0.0291,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.0261346101760864,
+      "rewards/margins": 5.170999526977539,
+      "rewards/rejected": -6.197134017944336,
+      "step": 365
+    },
+    {
+      "epoch": 4.331360946745562,
+      "grad_norm": 6.395251337450572,
+      "learning_rate": 2.552009371098778e-07,
+      "logits/chosen": -1.404557466506958,
+      "logits/rejected": -1.3076186180114746,
+      "logps/chosen": -31.61805534362793,
+      "logps/rejected": -48.43040084838867,
+      "loss": 0.0315,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.18829268217086792,
+      "rewards/margins": 5.8097405433654785,
+      "rewards/rejected": -5.998033046722412,
+      "step": 366
+    },
+    {
+      "epoch": 4.34319526627219,
+      "grad_norm": 3.8677234721220275,
+      "learning_rate": 2.5390082595341816e-07,
+      "logits/chosen": -1.1186158657073975,
+      "logits/rejected": -1.1845828294754028,
+      "logps/chosen": -30.134925842285156,
+      "logps/rejected": -55.77180480957031,
+      "loss": 0.0175,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2170889675617218,
+      "rewards/margins": 6.981093883514404,
+      "rewards/rejected": -6.764005661010742,
+      "step": 367
+    },
+    {
+      "epoch": 4.355029585798817,
+      "grad_norm": 6.159683736078311,
+      "learning_rate": 2.5260060926561604e-07,
+      "logits/chosen": -1.056211233139038,
+      "logits/rejected": -1.0722953081130981,
+      "logps/chosen": -38.946746826171875,
+      "logps/rejected": -69.67351531982422,
+      "loss": 0.0319,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.082647681236267,
+      "rewards/margins": 6.603187561035156,
+      "rewards/rejected": -7.685835361480713,
+      "step": 368
+    },
+    {
+      "epoch": 4.366863905325443,
+      "grad_norm": 6.5279554127734745,
+      "learning_rate": 2.5130032222199954e-07,
+      "logits/chosen": -1.2777047157287598,
+      "logits/rejected": -1.230380654335022,
+      "logps/chosen": -44.25871658325195,
+      "logps/rejected": -68.12265014648438,
+      "loss": 0.0321,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.8238017559051514,
+      "rewards/margins": 7.426830291748047,
+      "rewards/rejected": -9.250631332397461,
+      "step": 369
+    },
+    {
+      "epoch": 4.378698224852071,
+      "grad_norm": 5.951315298701114,
+      "learning_rate": 2.5e-07,
+      "logits/chosen": -1.1897178888320923,
+      "logits/rejected": -1.1760835647583008,
+      "logps/chosen": -50.80073928833008,
+      "logps/rejected": -85.21668243408203,
+      "loss": 0.029,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2594873905181885,
+      "rewards/margins": 10.383180618286133,
+      "rewards/rejected": -11.642667770385742,
+      "step": 370
+    },
+    {
+      "epoch": 4.390532544378698,
+      "grad_norm": 6.15592831885902,
+      "learning_rate": 2.4869967777800055e-07,
+      "logits/chosen": -0.6888167858123779,
+      "logits/rejected": -0.6404827833175659,
+      "logps/chosen": -41.95843505859375,
+      "logps/rejected": -54.23292922973633,
+      "loss": 0.0292,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.676899790763855,
+      "rewards/margins": 4.1613569259643555,
+      "rewards/rejected": -4.8382568359375,
+      "step": 371
+    },
+    {
+      "epoch": 4.402366863905326,
+      "grad_norm": 6.891198413290914,
+      "learning_rate": 2.4739939073438393e-07,
+      "logits/chosen": -0.82053142786026,
+      "logits/rejected": -0.8729037046432495,
+      "logps/chosen": -42.464149475097656,
+      "logps/rejected": -56.32053756713867,
+      "loss": 0.0373,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1332708597183228,
+      "rewards/margins": 5.776480674743652,
+      "rewards/rejected": -6.909751892089844,
+      "step": 372
+    },
+    {
+      "epoch": 4.414201183431953,
+      "grad_norm": 5.861758579748894,
+      "learning_rate": 2.460991740465819e-07,
+      "logits/chosen": -1.1168274879455566,
+      "logits/rejected": -1.2398712635040283,
+      "logps/chosen": -42.28007507324219,
+      "logps/rejected": -57.90584945678711,
+      "loss": 0.026,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.5763722658157349,
+      "rewards/margins": 5.462160110473633,
+      "rewards/rejected": -7.038532733917236,
+      "step": 373
+    },
+    {
+      "epoch": 4.42603550295858,
+      "grad_norm": 6.486927362087198,
+      "learning_rate": 2.4479906289012216e-07,
+      "logits/chosen": -1.2565290927886963,
+      "logits/rejected": -1.167330026626587,
+      "logps/chosen": -33.31645584106445,
+      "logps/rejected": -67.92594909667969,
+      "loss": 0.0316,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6366786956787109,
+      "rewards/margins": 7.325068473815918,
+      "rewards/rejected": -7.961746692657471,
+      "step": 374
+    },
+    {
+      "epoch": 4.437869822485207,
+      "grad_norm": 7.945577732784187,
+      "learning_rate": 2.434990924376778e-07,
+      "logits/chosen": -1.3346067667007446,
+      "logits/rejected": -1.3618366718292236,
+      "logps/chosen": -40.330810546875,
+      "logps/rejected": -75.98135375976562,
+      "loss": 0.0393,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.036665678024292,
+      "rewards/margins": 8.026451110839844,
+      "rewards/rejected": -9.063117027282715,
+      "step": 375
+    },
+    {
+      "epoch": 4.449704142011834,
+      "grad_norm": 4.958938820398859,
+      "learning_rate": 2.421992978581152e-07,
+      "logits/chosen": -0.9798704981803894,
+      "logits/rejected": -0.9934732913970947,
+      "logps/chosen": -55.855712890625,
+      "logps/rejected": -73.17900085449219,
+      "loss": 0.02,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.960662841796875,
+      "rewards/margins": 6.544261932373047,
+      "rewards/rejected": -9.504924774169922,
+      "step": 376
+    },
+    {
+      "epoch": 4.461538461538462,
+      "grad_norm": 7.067788432308369,
+      "learning_rate": 2.4089971431554287e-07,
+      "logits/chosen": -1.0426850318908691,
+      "logits/rejected": -1.1020509004592896,
+      "logps/chosen": -38.878639221191406,
+      "logps/rejected": -50.67585754394531,
+      "loss": 0.038,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.268679678440094,
+      "rewards/margins": 6.866507053375244,
+      "rewards/rejected": -7.135187149047852,
+      "step": 377
+    },
+    {
+      "epoch": 4.4733727810650885,
+      "grad_norm": 6.288836568067266,
+      "learning_rate": 2.3960037696835987e-07,
+      "logits/chosen": -1.304239273071289,
+      "logits/rejected": -1.282547950744629,
+      "logps/chosen": -39.218841552734375,
+      "logps/rejected": -60.83658218383789,
+      "loss": 0.0302,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.095663070678711,
+      "rewards/margins": 7.307448863983154,
+      "rewards/rejected": -8.403112411499023,
+      "step": 378
+    },
+    {
+      "epoch": 4.485207100591716,
+      "grad_norm": 6.123368205316238,
+      "learning_rate": 2.3830132096830475e-07,
+      "logits/chosen": -1.1514697074890137,
+      "logits/rejected": -1.2038421630859375,
+      "logps/chosen": -38.228721618652344,
+      "logps/rejected": -63.91864776611328,
+      "loss": 0.027,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9770803451538086,
+      "rewards/margins": 7.351076602935791,
+      "rewards/rejected": -8.328157424926758,
+      "step": 379
+    },
+    {
+      "epoch": 4.497041420118343,
+      "grad_norm": 6.073409726703173,
+      "learning_rate": 2.3700258145950493e-07,
+      "logits/chosen": -1.1492507457733154,
+      "logits/rejected": -1.1283336877822876,
+      "logps/chosen": -34.70807647705078,
+      "logps/rejected": -53.41638946533203,
+      "loss": 0.027,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4615098237991333,
+      "rewards/margins": 5.6219682693481445,
+      "rewards/rejected": -7.083477973937988,
+      "step": 380
+    },
+    {
+      "epoch": 4.508875739644971,
+      "grad_norm": 6.859789156730043,
+      "learning_rate": 2.3570419357752518e-07,
+      "logits/chosen": -1.1793534755706787,
+      "logits/rejected": -1.1703987121582031,
+      "logps/chosen": -35.941184997558594,
+      "logps/rejected": -62.35126495361328,
+      "loss": 0.0281,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.987781286239624,
+      "rewards/margins": 7.1669921875,
+      "rewards/rejected": -9.154773712158203,
+      "step": 381
+    },
+    {
+      "epoch": 4.520710059171598,
+      "grad_norm": 5.867928411886104,
+      "learning_rate": 2.3440619244841794e-07,
+      "logits/chosen": -1.33339262008667,
+      "logits/rejected": -1.2492295503616333,
+      "logps/chosen": -34.28276062011719,
+      "logps/rejected": -59.37235641479492,
+      "loss": 0.0294,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.813497006893158,
+      "rewards/margins": 6.642629623413086,
+      "rewards/rejected": -7.456126689910889,
+      "step": 382
+    },
+    {
+      "epoch": 4.5325443786982245,
+      "grad_norm": 7.114872273799716,
+      "learning_rate": 2.3310861318777214e-07,
+      "logits/chosen": -0.9742549657821655,
+      "logits/rejected": -0.9942373037338257,
+      "logps/chosen": -35.96149826049805,
+      "logps/rejected": -58.944549560546875,
+      "loss": 0.0363,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.3453497886657715,
+      "rewards/margins": 6.538041114807129,
+      "rewards/rejected": -7.883390426635742,
+      "step": 383
+    },
+    {
+      "epoch": 4.544378698224852,
+      "grad_norm": 6.454893576265482,
+      "learning_rate": 2.3181149089976404e-07,
+      "logits/chosen": -0.9551500082015991,
+      "logits/rejected": -0.8459126949310303,
+      "logps/chosen": -35.49050521850586,
+      "logps/rejected": -59.616416931152344,
+      "loss": 0.0252,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.7028827667236328,
+      "rewards/margins": 5.758362293243408,
+      "rewards/rejected": -7.461245536804199,
+      "step": 384
+    },
+    {
+      "epoch": 4.556213017751479,
+      "grad_norm": 4.604917778075366,
+      "learning_rate": 2.30514860676207e-07,
+      "logits/chosen": -0.9264880418777466,
+      "logits/rejected": -0.8983692526817322,
+      "logps/chosen": -37.952232360839844,
+      "logps/rejected": -66.89598846435547,
+      "loss": 0.0198,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.458532452583313,
+      "rewards/margins": 7.424208641052246,
+      "rewards/rejected": -8.88274097442627,
+      "step": 385
+    },
+    {
+      "epoch": 4.568047337278107,
+      "grad_norm": 5.972753790143483,
+      "learning_rate": 2.2921875759560207e-07,
+      "logits/chosen": -1.2438300848007202,
+      "logits/rejected": -1.4885196685791016,
+      "logps/chosen": -48.01713943481445,
+      "logps/rejected": -50.086448669433594,
+      "loss": 0.0317,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.3077518939971924,
+      "rewards/margins": 5.58108377456665,
+      "rewards/rejected": -6.888835906982422,
+      "step": 386
+    },
+    {
+      "epoch": 4.579881656804734,
+      "grad_norm": 5.6047301713387485,
+      "learning_rate": 2.2792321672218967e-07,
+      "logits/chosen": -1.1834698915481567,
+      "logits/rejected": -1.1805692911148071,
+      "logps/chosen": -37.24559020996094,
+      "logps/rejected": -68.1692886352539,
+      "loss": 0.024,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.3298698663711548,
+      "rewards/margins": 8.419723510742188,
+      "rewards/rejected": -9.749593734741211,
+      "step": 387
+    },
+    {
+      "epoch": 4.591715976331361,
+      "grad_norm": 7.525291987243171,
+      "learning_rate": 2.2662827310499995e-07,
+      "logits/chosen": -1.016861081123352,
+      "logits/rejected": -1.143229603767395,
+      "logps/chosen": -43.143497467041016,
+      "logps/rejected": -63.837677001953125,
+      "loss": 0.043,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9594184756278992,
+      "rewards/margins": 6.90932559967041,
+      "rewards/rejected": -7.868744850158691,
+      "step": 388
+    },
+    {
+      "epoch": 4.603550295857988,
+      "grad_norm": 7.015133341827511,
+      "learning_rate": 2.2533396177690562e-07,
+      "logits/chosen": -1.2056879997253418,
+      "logits/rejected": -1.3042309284210205,
+      "logps/chosen": -46.522300720214844,
+      "logps/rejected": -61.494346618652344,
+      "loss": 0.0287,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.9979819059371948,
+      "rewards/margins": 6.070592880249023,
+      "rewards/rejected": -7.068574905395508,
+      "step": 389
+    },
+    {
+      "epoch": 4.615384615384615,
+      "grad_norm": 5.78008137850188,
+      "learning_rate": 2.2404031775367332e-07,
+      "logits/chosen": -1.2291995286941528,
+      "logits/rejected": -1.314921259880066,
+      "logps/chosen": -45.853309631347656,
+      "logps/rejected": -55.867095947265625,
+      "loss": 0.0246,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.3121535778045654,
+      "rewards/margins": 5.488016605377197,
+      "rewards/rejected": -6.800169944763184,
+      "step": 390
+    },
+    {
+      "epoch": 4.627218934911243,
+      "grad_norm": 6.252483858832655,
+      "learning_rate": 2.227473760330169e-07,
+      "logits/chosen": -1.239790439605713,
+      "logits/rejected": -1.1441867351531982,
+      "logps/chosen": -37.55449676513672,
+      "logps/rejected": -68.65978240966797,
+      "loss": 0.027,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.0223238468170166,
+      "rewards/margins": 8.488706588745117,
+      "rewards/rejected": -9.511030197143555,
+      "step": 391
+    },
+    {
+      "epoch": 4.6390532544378695,
+      "grad_norm": 5.571831888079325,
+      "learning_rate": 2.2145517159365043e-07,
+      "logits/chosen": -1.147141456604004,
+      "logits/rejected": -1.0945308208465576,
+      "logps/chosen": -40.555084228515625,
+      "logps/rejected": -71.9697265625,
+      "loss": 0.0237,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.045267105102539,
+      "rewards/margins": 7.398416519165039,
+      "rewards/rejected": -9.443683624267578,
+      "step": 392
+    },
+    {
+      "epoch": 4.650887573964497,
+      "grad_norm": 6.698497149981605,
+      "learning_rate": 2.2016373939434166e-07,
+      "logits/chosen": -1.2282354831695557,
+      "logits/rejected": -1.2330670356750488,
+      "logps/chosen": -41.47081756591797,
+      "logps/rejected": -56.92009353637695,
+      "loss": 0.0356,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8390868902206421,
+      "rewards/margins": 5.762375831604004,
+      "rewards/rejected": -6.601463317871094,
+      "step": 393
+    },
+    {
+      "epoch": 4.662721893491124,
+      "grad_norm": 5.665101121799137,
+      "learning_rate": 2.1887311437296684e-07,
+      "logits/chosen": -1.3237837553024292,
+      "logits/rejected": -1.4168846607208252,
+      "logps/chosen": -43.45185470581055,
+      "logps/rejected": -69.13044738769531,
+      "loss": 0.0231,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4442262649536133,
+      "rewards/margins": 8.553362846374512,
+      "rewards/rejected": -9.997589111328125,
+      "step": 394
+    },
+    {
+      "epoch": 4.674556213017752,
+      "grad_norm": 4.857938949164866,
+      "learning_rate": 2.175833314455647e-07,
+      "logits/chosen": -1.044704794883728,
+      "logits/rejected": -1.047666072845459,
+      "logps/chosen": -57.778099060058594,
+      "logps/rejected": -86.46309661865234,
+      "loss": 0.021,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1337398290634155,
+      "rewards/margins": 7.24013614654541,
+      "rewards/rejected": -8.373876571655273,
+      "step": 395
+    },
+    {
+      "epoch": 4.686390532544379,
+      "grad_norm": 4.452615646822525,
+      "learning_rate": 2.162944255053928e-07,
+      "logits/chosen": -1.4428414106369019,
+      "logits/rejected": -1.300477147102356,
+      "logps/chosen": -43.00053405761719,
+      "logps/rejected": -78.71292877197266,
+      "loss": 0.0179,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2613164186477661,
+      "rewards/margins": 7.334977149963379,
+      "rewards/rejected": -8.596292495727539,
+      "step": 396
+    },
+    {
+      "epoch": 4.6982248520710055,
+      "grad_norm": 4.714776633335646,
+      "learning_rate": 2.1500643142198264e-07,
+      "logits/chosen": -1.146743893623352,
+      "logits/rejected": -1.1370177268981934,
+      "logps/chosen": -37.42723846435547,
+      "logps/rejected": -47.88990020751953,
+      "loss": 0.0208,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.5033150911331177,
+      "rewards/margins": 4.86285400390625,
+      "rewards/rejected": -6.366168975830078,
+      "step": 397
+    },
+    {
+      "epoch": 4.710059171597633,
+      "grad_norm": 6.0816315897712725,
+      "learning_rate": 2.137193840401968e-07,
+      "logits/chosen": -1.2402558326721191,
+      "logits/rejected": -1.2731809616088867,
+      "logps/chosen": -44.44050598144531,
+      "logps/rejected": -54.368404388427734,
+      "loss": 0.0227,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6547656059265137,
+      "rewards/margins": 5.59221887588501,
+      "rewards/rejected": -6.246984481811523,
+      "step": 398
+    },
+    {
+      "epoch": 4.72189349112426,
+      "grad_norm": 5.278736402112801,
+      "learning_rate": 2.1243331817928643e-07,
+      "logits/chosen": -1.2595813274383545,
+      "logits/rejected": -1.263629674911499,
+      "logps/chosen": -44.460487365722656,
+      "logps/rejected": -73.69570922851562,
+      "loss": 0.0192,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5480247735977173,
+      "rewards/margins": 8.522817611694336,
+      "rewards/rejected": -9.070842742919922,
+      "step": 399
+    },
+    {
+      "epoch": 4.733727810650888,
+      "grad_norm": 7.040411325600633,
+      "learning_rate": 2.1114826863194878e-07,
+      "logits/chosen": -1.3222355842590332,
+      "logits/rejected": -1.3484004735946655,
+      "logps/chosen": -34.907257080078125,
+      "logps/rejected": -60.81079864501953,
+      "loss": 0.0259,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.18377304077148438,
+      "rewards/margins": 8.166070938110352,
+      "rewards/rejected": -8.349843978881836,
+      "step": 400
+    },
+    {
+      "epoch": 4.745562130177515,
+      "grad_norm": 5.317019025575069,
+      "learning_rate": 2.0986427016338623e-07,
+      "logits/chosen": -0.9710571765899658,
+      "logits/rejected": -0.9310001730918884,
+      "logps/chosen": -39.45679473876953,
+      "logps/rejected": -56.64714050292969,
+      "loss": 0.0241,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3554608225822449,
+      "rewards/margins": 6.091450214385986,
+      "rewards/rejected": -6.446911811828613,
+      "step": 401
+    },
+    {
+      "epoch": 4.757396449704142,
+      "grad_norm": 6.760332535967884,
+      "learning_rate": 2.0858135751036568e-07,
+      "logits/chosen": -1.2228119373321533,
+      "logits/rejected": -1.1839139461517334,
+      "logps/chosen": -30.775981903076172,
+      "logps/rejected": -47.54846954345703,
+      "loss": 0.0313,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.206694558262825,
+      "rewards/margins": 5.442242622375488,
+      "rewards/rejected": -5.23554801940918,
+      "step": 402
+    },
+    {
+      "epoch": 4.769230769230769,
+      "grad_norm": 5.929464844555724,
+      "learning_rate": 2.0729956538027904e-07,
+      "logits/chosen": -1.2554802894592285,
+      "logits/rejected": -1.1986385583877563,
+      "logps/chosen": -29.697330474853516,
+      "logps/rejected": -48.260414123535156,
+      "loss": 0.0266,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.7226145267486572,
+      "rewards/margins": 5.005193710327148,
+      "rewards/rejected": -5.727808475494385,
+      "step": 403
+    },
+    {
+      "epoch": 4.781065088757396,
+      "grad_norm": 7.814308902771925,
+      "learning_rate": 2.060189284502037e-07,
+      "logits/chosen": -1.1625741720199585,
+      "logits/rejected": -1.014739751815796,
+      "logps/chosen": -44.1807861328125,
+      "logps/rejected": -75.00509643554688,
+      "loss": 0.0417,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.093876600265503,
+      "rewards/margins": 7.723724842071533,
+      "rewards/rejected": -9.817602157592773,
+      "step": 404
+    },
+    {
+      "epoch": 4.792899408284024,
+      "grad_norm": 5.110756280509404,
+      "learning_rate": 2.0473948136596486e-07,
+      "logits/chosen": -1.2573013305664062,
+      "logits/rejected": -1.1042354106903076,
+      "logps/chosen": -37.25166320800781,
+      "logps/rejected": -62.03949737548828,
+      "loss": 0.0188,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.6967883110046387,
+      "rewards/margins": 7.865792751312256,
+      "rewards/rejected": -9.562580108642578,
+      "step": 405
+    },
+    {
+      "epoch": 4.804733727810651,
+      "grad_norm": 6.7687604973263635,
+      "learning_rate": 2.0346125874119838e-07,
+      "logits/chosen": -1.3532781600952148,
+      "logits/rejected": -1.336191177368164,
+      "logps/chosen": -39.134239196777344,
+      "logps/rejected": -60.64654541015625,
+      "loss": 0.0287,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8724418878555298,
+      "rewards/margins": 7.370305061340332,
+      "rewards/rejected": -8.242748260498047,
+      "step": 406
+    },
+    {
+      "epoch": 4.816568047337278,
+      "grad_norm": 5.980695574776132,
+      "learning_rate": 2.0218429515641368e-07,
+      "logits/chosen": -0.9892873167991638,
+      "logits/rejected": -0.639751672744751,
+      "logps/chosen": -29.87440299987793,
+      "logps/rejected": -68.78720092773438,
+      "loss": 0.0305,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.17198152840137482,
+      "rewards/margins": 8.037885665893555,
+      "rewards/rejected": -8.209866523742676,
+      "step": 407
+    },
+    {
+      "epoch": 4.828402366863905,
+      "grad_norm": 6.2455457286356815,
+      "learning_rate": 2.0090862515805895e-07,
+      "logits/chosen": -1.1471354961395264,
+      "logits/rejected": -1.0667625665664673,
+      "logps/chosen": -34.445716857910156,
+      "logps/rejected": -61.40612030029297,
+      "loss": 0.0287,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.5051982402801514,
+      "rewards/margins": 6.787008285522461,
+      "rewards/rejected": -8.292206764221191,
+      "step": 408
+    },
+    {
+      "epoch": 4.840236686390533,
+      "grad_norm": 6.051206947220886,
+      "learning_rate": 1.9963428325758613e-07,
+      "logits/chosen": -1.2832268476486206,
+      "logits/rejected": -1.2529618740081787,
+      "logps/chosen": -48.28749465942383,
+      "logps/rejected": -68.13203430175781,
+      "loss": 0.025,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.748518466949463,
+      "rewards/margins": 7.8393754959106445,
+      "rewards/rejected": -9.58789348602295,
+      "step": 409
+    },
+    {
+      "epoch": 4.85207100591716,
+      "grad_norm": 6.3012658749958765,
+      "learning_rate": 1.983613039305173e-07,
+      "logits/chosen": -1.134077787399292,
+      "logits/rejected": -1.1521306037902832,
+      "logps/chosen": -49.527854919433594,
+      "logps/rejected": -67.16100311279297,
+      "loss": 0.0307,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.6244655847549438,
+      "rewards/margins": 6.405965805053711,
+      "rewards/rejected": -8.030430793762207,
+      "step": 410
+    },
+    {
+      "epoch": 4.8639053254437865,
+      "grad_norm": 4.470631833677289,
+      "learning_rate": 1.9708972161551213e-07,
+      "logits/chosen": -1.132239580154419,
+      "logits/rejected": -1.0069799423217773,
+      "logps/chosen": -39.385215759277344,
+      "logps/rejected": -58.923709869384766,
+      "loss": 0.0226,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6855692863464355,
+      "rewards/margins": 6.034358024597168,
+      "rewards/rejected": -6.7199273109436035,
+      "step": 411
+    },
+    {
+      "epoch": 4.875739644970414,
+      "grad_norm": 6.376861071339664,
+      "learning_rate": 1.9581957071343588e-07,
+      "logits/chosen": -0.9943496584892273,
+      "logits/rejected": -0.9143755435943604,
+      "logps/chosen": -48.05913543701172,
+      "logps/rejected": -69.44911193847656,
+      "loss": 0.0287,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.8955947160720825,
+      "rewards/margins": 6.447237968444824,
+      "rewards/rejected": -8.342832565307617,
+      "step": 412
+    },
+    {
+      "epoch": 4.887573964497041,
+      "grad_norm": 6.431311497276744,
+      "learning_rate": 1.9455088558642932e-07,
+      "logits/chosen": -1.2816779613494873,
+      "logits/rejected": -1.1103105545043945,
+      "logps/chosen": -28.044443130493164,
+      "logps/rejected": -67.90215301513672,
+      "loss": 0.0215,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.3063042163848877,
+      "rewards/margins": 9.634206771850586,
+      "rewards/rejected": -10.940509796142578,
+      "step": 413
+    },
+    {
+      "epoch": 4.899408284023669,
+      "grad_norm": 4.779737157402348,
+      "learning_rate": 1.9328370055697832e-07,
+      "logits/chosen": -1.1393996477127075,
+      "logits/rejected": -1.108349323272705,
+      "logps/chosen": -34.88286590576172,
+      "logps/rejected": -57.27365493774414,
+      "loss": 0.021,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.0275461673736572,
+      "rewards/margins": 6.514031410217285,
+      "rewards/rejected": -7.541577339172363,
+      "step": 414
+    },
+    {
+      "epoch": 4.911242603550296,
+      "grad_norm": 4.144080990787552,
+      "learning_rate": 1.9201804990698616e-07,
+      "logits/chosen": -1.0242797136306763,
+      "logits/rejected": -1.0819731950759888,
+      "logps/chosen": -62.003173828125,
+      "logps/rejected": -75.39320373535156,
+      "loss": 0.0157,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.2295634746551514,
+      "rewards/margins": 6.729301452636719,
+      "rewards/rejected": -8.95886516571045,
+      "step": 415
+    },
+    {
+      "epoch": 4.923076923076923,
+      "grad_norm": 7.28582515595878,
+      "learning_rate": 1.907539678768453e-07,
+      "logits/chosen": -1.2485833168029785,
+      "logits/rejected": -1.2470812797546387,
+      "logps/chosen": -33.02880859375,
+      "logps/rejected": -60.55603790283203,
+      "loss": 0.0274,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4287913739681244,
+      "rewards/margins": 8.188567161560059,
+      "rewards/rejected": -8.617358207702637,
+      "step": 416
+    },
+    {
+      "epoch": 4.93491124260355,
+      "grad_norm": 5.63867046125462,
+      "learning_rate": 1.8949148866451152e-07,
+      "logits/chosen": -1.3377947807312012,
+      "logits/rejected": -1.2713524103164673,
+      "logps/chosen": -43.48049545288086,
+      "logps/rejected": -66.24922943115234,
+      "loss": 0.029,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.6896387338638306,
+      "rewards/margins": 7.300614356994629,
+      "rewards/rejected": -8.990253448486328,
+      "step": 417
+    },
+    {
+      "epoch": 4.946745562130177,
+      "grad_norm": 7.296218456889154,
+      "learning_rate": 1.8823064642457876e-07,
+      "logits/chosen": -1.2259938716888428,
+      "logits/rejected": -1.05912184715271,
+      "logps/chosen": -27.457061767578125,
+      "logps/rejected": -54.369407653808594,
+      "loss": 0.0318,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2313295304775238,
+      "rewards/margins": 6.432798862457275,
+      "rewards/rejected": -6.664128303527832,
+      "step": 418
+    },
+    {
+      "epoch": 4.958579881656805,
+      "grad_norm": 6.2793049860116215,
+      "learning_rate": 1.8697147526735466e-07,
+      "logits/chosen": -1.1357454061508179,
+      "logits/rejected": -1.172181487083435,
+      "logps/chosen": -39.59063720703125,
+      "logps/rejected": -70.14149475097656,
+      "loss": 0.0311,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2256022691726685,
+      "rewards/margins": 7.56755256652832,
+      "rewards/rejected": -8.793155670166016,
+      "step": 419
+    },
+    {
+      "epoch": 4.970414201183432,
+      "grad_norm": 5.690239912229541,
+      "learning_rate": 1.8571400925793852e-07,
+      "logits/chosen": -1.1824370622634888,
+      "logits/rejected": -1.0679001808166504,
+      "logps/chosen": -39.977928161621094,
+      "logps/rejected": -63.33866500854492,
+      "loss": 0.0291,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.93265700340271,
+      "rewards/margins": 7.5748395919799805,
+      "rewards/rejected": -8.507495880126953,
+      "step": 420
+    },
+    {
+      "epoch": 4.982248520710059,
+      "grad_norm": 6.961259730499676,
+      "learning_rate": 1.844582824152988e-07,
+      "logits/chosen": -1.2937397956848145,
+      "logits/rejected": -1.4227337837219238,
+      "logps/chosen": -47.419578552246094,
+      "logps/rejected": -62.564239501953125,
+      "loss": 0.0327,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.698372483253479,
+      "rewards/margins": 8.21216869354248,
+      "rewards/rejected": -8.910541534423828,
+      "step": 421
+    },
+    {
+      "epoch": 4.994082840236686,
+      "grad_norm": 7.002318301330378,
+      "learning_rate": 1.8320432871135376e-07,
+      "logits/chosen": -1.068471074104309,
+      "logits/rejected": -0.9389731884002686,
+      "logps/chosen": -39.13528823852539,
+      "logps/rejected": -73.06786346435547,
+      "loss": 0.0272,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.0938169956207275,
+      "rewards/margins": 7.050632476806641,
+      "rewards/rejected": -8.144449234008789,
+      "step": 422
+    },
+    {
+      "epoch": 5.005917159763314,
+      "grad_norm": 6.3621819487176205,
+      "learning_rate": 1.8195218207005136e-07,
+      "logits/chosen": -1.3071154356002808,
+      "logits/rejected": -1.2660558223724365,
+      "logps/chosen": -41.71501541137695,
+      "logps/rejected": -58.077205657958984,
+      "loss": 0.0257,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.21319010853767395,
+      "rewards/margins": 5.4651103019714355,
+      "rewards/rejected": -5.678299903869629,
+      "step": 423
+    },
+    {
+      "epoch": 5.017751479289941,
+      "grad_norm": 5.688784125844276,
+      "learning_rate": 1.8070187636645237e-07,
+      "logits/chosen": -1.1194195747375488,
+      "logits/rejected": -1.1475024223327637,
+      "logps/chosen": -39.484375,
+      "logps/rejected": -53.20079803466797,
+      "loss": 0.0301,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6331821084022522,
+      "rewards/margins": 5.97371768951416,
+      "rewards/rejected": -6.606899261474609,
+      "step": 424
+    },
+    {
+      "epoch": 5.029585798816568,
+      "grad_norm": 4.08606565715224,
+      "learning_rate": 1.7945344542581353e-07,
+      "logits/chosen": -1.2313172817230225,
+      "logits/rejected": -1.2285120487213135,
+      "logps/chosen": -34.221275329589844,
+      "logps/rejected": -71.1649398803711,
+      "loss": 0.0187,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6170086860656738,
+      "rewards/margins": 9.6792573928833,
+      "rewards/rejected": -10.296265602111816,
+      "step": 425
+    },
+    {
+      "epoch": 5.041420118343195,
+      "grad_norm": 4.101168582532201,
+      "learning_rate": 1.782069230226725e-07,
+      "logits/chosen": -1.331200122833252,
+      "logits/rejected": -1.2716310024261475,
+      "logps/chosen": -32.71428298950195,
+      "logps/rejected": -57.31195068359375,
+      "loss": 0.0175,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.05249008536338806,
+      "rewards/margins": 6.174403190612793,
+      "rewards/rejected": -6.121913433074951,
+      "step": 426
+    },
+    {
+      "epoch": 5.053254437869822,
+      "grad_norm": 3.885129904982064,
+      "learning_rate": 1.7696234287993413e-07,
+      "logits/chosen": -1.0245940685272217,
+      "logits/rejected": -0.809888482093811,
+      "logps/chosen": -26.134567260742188,
+      "logps/rejected": -58.40096664428711,
+      "loss": 0.017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.018941545858979225,
+      "rewards/margins": 6.3871893882751465,
+      "rewards/rejected": -6.3682475090026855,
+      "step": 427
+    },
+    {
+      "epoch": 5.06508875739645,
+      "grad_norm": 4.487578296923204,
+      "learning_rate": 1.7571973866795813e-07,
+      "logits/chosen": -1.1871979236602783,
+      "logits/rejected": -1.1301292181015015,
+      "logps/chosen": -47.12839889526367,
+      "logps/rejected": -72.7445068359375,
+      "loss": 0.0229,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.4960553646087646,
+      "rewards/margins": 8.162572860717773,
+      "rewards/rejected": -10.658628463745117,
+      "step": 428
+    },
+    {
+      "epoch": 5.076923076923077,
+      "grad_norm": 4.259353249136821,
+      "learning_rate": 1.7447914400364833e-07,
+      "logits/chosen": -1.3441885709762573,
+      "logits/rejected": -1.4836678504943848,
+      "logps/chosen": -56.2959098815918,
+      "logps/rejected": -63.773292541503906,
+      "loss": 0.0202,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.281679391860962,
+      "rewards/margins": 6.2985944747924805,
+      "rewards/rejected": -7.580273628234863,
+      "step": 429
+    },
+    {
+      "epoch": 5.088757396449704,
+      "grad_norm": 6.225232745781602,
+      "learning_rate": 1.7324059244954292e-07,
+      "logits/chosen": -1.179904580116272,
+      "logits/rejected": -1.0788750648498535,
+      "logps/chosen": -31.996553421020508,
+      "logps/rejected": -57.39022445678711,
+      "loss": 0.0317,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4775885343551636,
+      "rewards/margins": 5.636683464050293,
+      "rewards/rejected": -7.114272117614746,
+      "step": 430
+    },
+    {
+      "epoch": 5.100591715976331,
+      "grad_norm": 5.489476203604287,
+      "learning_rate": 1.720041175129066e-07,
+      "logits/chosen": -1.0896636247634888,
+      "logits/rejected": -0.9166202545166016,
+      "logps/chosen": -37.66105651855469,
+      "logps/rejected": -66.37045288085938,
+      "loss": 0.0315,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2693698406219482,
+      "rewards/margins": 6.162668228149414,
+      "rewards/rejected": -7.432038307189941,
+      "step": 431
+    },
+    {
+      "epoch": 5.112426035502959,
+      "grad_norm": 5.243666393695587,
+      "learning_rate": 1.7076975264482433e-07,
+      "logits/chosen": -1.1870818138122559,
+      "logits/rejected": -1.1362037658691406,
+      "logps/chosen": -36.95043182373047,
+      "logps/rejected": -65.01616668701172,
+      "loss": 0.025,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.3586033582687378,
+      "rewards/margins": 8.19964599609375,
+      "rewards/rejected": -9.558250427246094,
+      "step": 432
+    },
+    {
+      "epoch": 5.124260355029586,
+      "grad_norm": 5.459471747574669,
+      "learning_rate": 1.6953753123929595e-07,
+      "logits/chosen": -0.9043876528739929,
+      "logits/rejected": -1.0556137561798096,
+      "logps/chosen": -42.625064849853516,
+      "logps/rejected": -65.89883422851562,
+      "loss": 0.0255,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.0012390613555908,
+      "rewards/margins": 5.687243938446045,
+      "rewards/rejected": -6.688483238220215,
+      "step": 433
+    },
+    {
+      "epoch": 5.136094674556213,
+      "grad_norm": 4.0735562348358725,
+      "learning_rate": 1.6830748663233303e-07,
+      "logits/chosen": -1.276261806488037,
+      "logits/rejected": -1.059903860092163,
+      "logps/chosen": -37.19121551513672,
+      "logps/rejected": -67.30182647705078,
+      "loss": 0.019,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8689858913421631,
+      "rewards/margins": 7.131950378417969,
+      "rewards/rejected": -8.000936508178711,
+      "step": 434
+    },
+    {
+      "epoch": 5.14792899408284,
+      "grad_norm": 5.750153182234357,
+      "learning_rate": 1.6707965210105687e-07,
+      "logits/chosen": -1.0981320142745972,
+      "logits/rejected": -1.1993542909622192,
+      "logps/chosen": -45.06623077392578,
+      "logps/rejected": -59.895782470703125,
+      "loss": 0.0258,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1987268924713135,
+      "rewards/margins": 7.3387932777404785,
+      "rewards/rejected": -8.537520408630371,
+      "step": 435
+    },
+    {
+      "epoch": 5.159763313609467,
+      "grad_norm": 4.092389515181816,
+      "learning_rate": 1.6585406086279846e-07,
+      "logits/chosen": -1.2612695693969727,
+      "logits/rejected": -1.0451221466064453,
+      "logps/chosen": -36.8587646484375,
+      "logps/rejected": -79.8933334350586,
+      "loss": 0.0208,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8900283575057983,
+      "rewards/margins": 9.542950630187988,
+      "rewards/rejected": -10.432977676391602,
+      "step": 436
+    },
+    {
+      "epoch": 5.171597633136095,
+      "grad_norm": 5.775549033301964,
+      "learning_rate": 1.6463074607419942e-07,
+      "logits/chosen": -1.1313787698745728,
+      "logits/rejected": -1.1185730695724487,
+      "logps/chosen": -39.299861907958984,
+      "logps/rejected": -55.985435485839844,
+      "loss": 0.0253,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1983766555786133,
+      "rewards/margins": 5.455132484436035,
+      "rewards/rejected": -6.65350866317749,
+      "step": 437
+    },
+    {
+      "epoch": 5.183431952662722,
+      "grad_norm": 4.591857419480486,
+      "learning_rate": 1.6340974083031523e-07,
+      "logits/chosen": -1.1867667436599731,
+      "logits/rejected": -1.101377248764038,
+      "logps/chosen": -37.22975158691406,
+      "logps/rejected": -59.464149475097656,
+      "loss": 0.0189,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3749149739742279,
+      "rewards/margins": 7.300636291503906,
+      "rewards/rejected": -7.675551414489746,
+      "step": 438
+    },
+    {
+      "epoch": 5.195266272189349,
+      "grad_norm": 4.200678818481589,
+      "learning_rate": 1.6219107816372024e-07,
+      "logits/chosen": -1.307371973991394,
+      "logits/rejected": -1.1307945251464844,
+      "logps/chosen": -45.63922119140625,
+      "logps/rejected": -96.90985107421875,
+      "loss": 0.0213,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.9187942743301392,
+      "rewards/margins": 10.127294540405273,
+      "rewards/rejected": -12.046089172363281,
+      "step": 439
+    },
+    {
+      "epoch": 5.207100591715976,
+      "grad_norm": 4.736427790505553,
+      "learning_rate": 1.6097479104361326e-07,
+      "logits/chosen": -1.4351165294647217,
+      "logits/rejected": -1.3055188655853271,
+      "logps/chosen": -33.065948486328125,
+      "logps/rejected": -62.625099182128906,
+      "loss": 0.0207,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6623735427856445,
+      "rewards/margins": 7.489253044128418,
+      "rewards/rejected": -8.151626586914062,
+      "step": 440
+    },
+    {
+      "epoch": 5.218934911242604,
+      "grad_norm": 5.4353100679782385,
+      "learning_rate": 1.5976091237492634e-07,
+      "logits/chosen": -1.1784343719482422,
+      "logits/rejected": -1.1669944524765015,
+      "logps/chosen": -43.70185852050781,
+      "logps/rejected": -61.90728759765625,
+      "loss": 0.0237,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9407563805580139,
+      "rewards/margins": 5.61828088760376,
+      "rewards/rejected": -6.559036731719971,
+      "step": 441
+    },
+    {
+      "epoch": 5.230769230769231,
+      "grad_norm": 5.5234881107996,
+      "learning_rate": 1.5854947499743413e-07,
+      "logits/chosen": -1.3166191577911377,
+      "logits/rejected": -1.2368249893188477,
+      "logps/chosen": -42.619972229003906,
+      "logps/rejected": -83.75607299804688,
+      "loss": 0.0312,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4741311073303223,
+      "rewards/margins": 8.95831298828125,
+      "rewards/rejected": -10.432443618774414,
+      "step": 442
+    },
+    {
+      "epoch": 5.242603550295858,
+      "grad_norm": 5.027875638098313,
+      "learning_rate": 1.573405116848656e-07,
+      "logits/chosen": -1.2968052625656128,
+      "logits/rejected": -1.1763187646865845,
+      "logps/chosen": -41.92218017578125,
+      "logps/rejected": -65.44755554199219,
+      "loss": 0.0218,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.754935622215271,
+      "rewards/margins": 5.887596607208252,
+      "rewards/rejected": -7.6425323486328125,
+      "step": 443
+    },
+    {
+      "epoch": 5.254437869822485,
+      "grad_norm": 4.9663450415072035,
+      "learning_rate": 1.5613405514401757e-07,
+      "logits/chosen": -1.1399455070495605,
+      "logits/rejected": -1.21956205368042,
+      "logps/chosen": -39.965789794921875,
+      "logps/rejected": -55.96144485473633,
+      "loss": 0.018,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.283247709274292,
+      "rewards/margins": 6.501648902893066,
+      "rewards/rejected": -7.7848968505859375,
+      "step": 444
+    },
+    {
+      "epoch": 5.266272189349112,
+      "grad_norm": 5.858342159433277,
+      "learning_rate": 1.5493013801386923e-07,
+      "logits/chosen": -1.2090262174606323,
+      "logits/rejected": -1.2344286441802979,
+      "logps/chosen": -46.805938720703125,
+      "logps/rejected": -75.00871276855469,
+      "loss": 0.0259,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.800337791442871,
+      "rewards/margins": 8.601978302001953,
+      "rewards/rejected": -10.402315139770508,
+      "step": 445
+    },
+    {
+      "epoch": 5.27810650887574,
+      "grad_norm": 3.9282604177697107,
+      "learning_rate": 1.537287928647002e-07,
+      "logits/chosen": -1.1603989601135254,
+      "logits/rejected": -1.1315605640411377,
+      "logps/chosen": -56.917301177978516,
+      "logps/rejected": -87.98289489746094,
+      "loss": 0.0168,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.3748254776000977,
+      "rewards/margins": 9.121573448181152,
+      "rewards/rejected": -11.49639892578125,
+      "step": 446
+    },
+    {
+      "epoch": 5.289940828402367,
+      "grad_norm": 4.17827764711654,
+      "learning_rate": 1.525300521972082e-07,
+      "logits/chosen": -1.1303211450576782,
+      "logits/rejected": -1.0561895370483398,
+      "logps/chosen": -34.059898376464844,
+      "logps/rejected": -58.381256103515625,
+      "loss": 0.0186,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6126248836517334,
+      "rewards/margins": 6.0588579177856445,
+      "rewards/rejected": -6.671482086181641,
+      "step": 447
+    },
+    {
+      "epoch": 5.3017751479289945,
+      "grad_norm": 5.0074680639809745,
+      "learning_rate": 1.513339484416309e-07,
+      "logits/chosen": -1.169161319732666,
+      "logits/rejected": -1.327141284942627,
+      "logps/chosen": -44.55299377441406,
+      "logps/rejected": -59.21006774902344,
+      "loss": 0.0225,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9135736227035522,
+      "rewards/margins": 6.587090492248535,
+      "rewards/rejected": -7.500664710998535,
+      "step": 448
+    },
+    {
+      "epoch": 5.313609467455621,
+      "grad_norm": 5.786312787732582,
+      "learning_rate": 1.5014051395686766e-07,
+      "logits/chosen": -1.314427137374878,
+      "logits/rejected": -1.2852325439453125,
+      "logps/chosen": -48.46540832519531,
+      "logps/rejected": -74.1810531616211,
+      "loss": 0.026,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.380225658416748,
+      "rewards/margins": 7.730525016784668,
+      "rewards/rejected": -10.110750198364258,
+      "step": 449
+    },
+    {
+      "epoch": 5.325443786982248,
+      "grad_norm": 4.769640674846225,
+      "learning_rate": 1.489497810296046e-07,
+      "logits/chosen": -1.3386704921722412,
+      "logits/rejected": -1.4422738552093506,
+      "logps/chosen": -50.036563873291016,
+      "logps/rejected": -78.25895690917969,
+      "loss": 0.0179,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -3.123276710510254,
+      "rewards/margins": 8.760514259338379,
+      "rewards/rejected": -11.883790969848633,
+      "step": 450
+    },
+    {
+      "epoch": 5.337278106508876,
+      "grad_norm": 4.460218619596584,
+      "learning_rate": 1.4776178187344105e-07,
+      "logits/chosen": -0.9677804112434387,
+      "logits/rejected": -1.0331177711486816,
+      "logps/chosen": -59.2320442199707,
+      "logps/rejected": -78.76850891113281,
+      "loss": 0.0205,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -3.6641674041748047,
+      "rewards/margins": 6.445460319519043,
+      "rewards/rejected": -10.109628677368164,
+      "step": 451
+    },
+    {
+      "epoch": 5.349112426035503,
+      "grad_norm": 3.802978212761357,
+      "learning_rate": 1.4657654862801797e-07,
+      "logits/chosen": -1.4019286632537842,
+      "logits/rejected": -1.3270961046218872,
+      "logps/chosen": -50.94618606567383,
+      "logps/rejected": -73.76014709472656,
+      "loss": 0.0161,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.508474826812744,
+      "rewards/margins": 7.509592056274414,
+      "rewards/rejected": -10.01806640625,
+      "step": 452
+    },
+    {
+      "epoch": 5.3609467455621305,
+      "grad_norm": 5.615210502971369,
+      "learning_rate": 1.4539411335814866e-07,
+      "logits/chosen": -1.0637266635894775,
+      "logits/rejected": -1.0000033378601074,
+      "logps/chosen": -41.48188018798828,
+      "logps/rejected": -63.91345977783203,
+      "loss": 0.0245,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4643651247024536,
+      "rewards/margins": 6.114117622375488,
+      "rewards/rejected": -6.578482627868652,
+      "step": 453
+    },
+    {
+      "epoch": 5.372781065088757,
+      "grad_norm": 5.198548513900638,
+      "learning_rate": 1.4421450805295082e-07,
+      "logits/chosen": -1.2260617017745972,
+      "logits/rejected": -1.0831642150878906,
+      "logps/chosen": -35.77007293701172,
+      "logps/rejected": -64.84323120117188,
+      "loss": 0.025,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.987048864364624,
+      "rewards/margins": 7.374147415161133,
+      "rewards/rejected": -8.361196517944336,
+      "step": 454
+    },
+    {
+      "epoch": 5.384615384615385,
+      "grad_norm": 4.832767981066584,
+      "learning_rate": 1.4303776462498186e-07,
+      "logits/chosen": -1.160849690437317,
+      "logits/rejected": -1.2087219953536987,
+      "logps/chosen": -40.54812240600586,
+      "logps/rejected": -65.13203430175781,
+      "loss": 0.0224,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.0010249614715576,
+      "rewards/margins": 8.726207733154297,
+      "rewards/rejected": -10.727232933044434,
+      "step": 455
+    },
+    {
+      "epoch": 5.396449704142012,
+      "grad_norm": 5.177069682037774,
+      "learning_rate": 1.418639149093748e-07,
+      "logits/chosen": -1.1638591289520264,
+      "logits/rejected": -1.1559276580810547,
+      "logps/chosen": -40.623634338378906,
+      "logps/rejected": -68.79119873046875,
+      "loss": 0.0258,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -1.1391851902008057,
+      "rewards/margins": 8.373973846435547,
+      "rewards/rejected": -9.513158798217773,
+      "step": 456
+    },
+    {
+      "epoch": 5.408284023668639,
+      "grad_norm": 4.365063167846994,
+      "learning_rate": 1.406929906629774e-07,
+      "logits/chosen": -1.0570735931396484,
+      "logits/rejected": -1.124732255935669,
+      "logps/chosen": -43.47991943359375,
+      "logps/rejected": -74.1714096069336,
+      "loss": 0.0172,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.225761890411377,
+      "rewards/margins": 8.236499786376953,
+      "rewards/rejected": -10.462261199951172,
+      "step": 457
+    },
+    {
+      "epoch": 5.420118343195266,
+      "grad_norm": 3.589902288780942,
+      "learning_rate": 1.3952502356349323e-07,
+      "logits/chosen": -1.3269389867782593,
+      "logits/rejected": -1.233825922012329,
+      "logps/chosen": -34.30451202392578,
+      "logps/rejected": -52.173431396484375,
+      "loss": 0.0164,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.0334144830703735,
+      "rewards/margins": 5.583642482757568,
+      "rewards/rejected": -6.617057800292969,
+      "step": 458
+    },
+    {
+      "epoch": 5.431952662721893,
+      "grad_norm": 4.55181870052482,
+      "learning_rate": 1.38360045208624e-07,
+      "logits/chosen": -1.2689670324325562,
+      "logits/rejected": -1.2837406396865845,
+      "logps/chosen": -40.163639068603516,
+      "logps/rejected": -56.57490539550781,
+      "loss": 0.0191,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6674104928970337,
+      "rewards/margins": 6.7150068283081055,
+      "rewards/rejected": -7.382417678833008,
+      "step": 459
+    },
+    {
+      "epoch": 5.443786982248521,
+      "grad_norm": 5.077146204999947,
+      "learning_rate": 1.371980871152157e-07,
+      "logits/chosen": -1.154836893081665,
+      "logits/rejected": -0.9715840816497803,
+      "logps/chosen": -45.965538024902344,
+      "logps/rejected": -83.77821350097656,
+      "loss": 0.0182,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.9867550134658813,
+      "rewards/margins": 7.073174953460693,
+      "rewards/rejected": -9.059929847717285,
+      "step": 460
+    },
+    {
+      "epoch": 5.455621301775148,
+      "grad_norm": 5.101492700995248,
+      "learning_rate": 1.3603918071840486e-07,
+      "logits/chosen": -1.316448450088501,
+      "logits/rejected": -1.2533689737319946,
+      "logps/chosen": -42.93837356567383,
+      "logps/rejected": -62.27436065673828,
+      "loss": 0.0223,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.6039561033248901,
+      "rewards/margins": 8.3004732131958,
+      "rewards/rejected": -9.90442943572998,
+      "step": 461
+    },
+    {
+      "epoch": 5.4674556213017755,
+      "grad_norm": 3.581286281901749,
+      "learning_rate": 1.3488335737076911e-07,
+      "logits/chosen": -1.05559504032135,
+      "logits/rejected": -1.045451283454895,
+      "logps/chosen": -39.23100280761719,
+      "logps/rejected": -68.82288360595703,
+      "loss": 0.0143,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.3661481142044067,
+      "rewards/margins": 7.688545227050781,
+      "rewards/rejected": -9.054693222045898,
+      "step": 462
+    },
+    {
+      "epoch": 5.479289940828402,
+      "grad_norm": 4.630228404526749,
+      "learning_rate": 1.3373064834147817e-07,
+      "logits/chosen": -1.1598727703094482,
+      "logits/rejected": -1.264807939529419,
+      "logps/chosen": -34.88840103149414,
+      "logps/rejected": -62.14350891113281,
+      "loss": 0.0221,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.034010887145996,
+      "rewards/margins": 8.205092430114746,
+      "rewards/rejected": -9.239103317260742,
+      "step": 463
+    },
+    {
+      "epoch": 5.491124260355029,
+      "grad_norm": 5.16650939084812,
+      "learning_rate": 1.3258108481544847e-07,
+      "logits/chosen": -1.1226065158843994,
+      "logits/rejected": -1.1048481464385986,
+      "logps/chosen": -43.532371520996094,
+      "logps/rejected": -56.43935775756836,
+      "loss": 0.023,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.7918784618377686,
+      "rewards/margins": 5.705506324768066,
+      "rewards/rejected": -7.497385025024414,
+      "step": 464
+    },
+    {
+      "epoch": 5.502958579881657,
+      "grad_norm": 3.8686941600847433,
+      "learning_rate": 1.314346978924994e-07,
+      "logits/chosen": -1.229318380355835,
+      "logits/rejected": -1.2423676252365112,
+      "logps/chosen": -39.24761199951172,
+      "logps/rejected": -61.67021179199219,
+      "loss": 0.0118,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.169360876083374,
+      "rewards/margins": 7.238840103149414,
+      "rewards/rejected": -9.408201217651367,
+      "step": 465
+    },
+    {
+      "epoch": 5.514792899408284,
+      "grad_norm": 5.296943513917031,
+      "learning_rate": 1.3029151858651143e-07,
+      "logits/chosen": -1.3978345394134521,
+      "logits/rejected": -1.3712904453277588,
+      "logps/chosen": -33.012115478515625,
+      "logps/rejected": -56.899269104003906,
+      "loss": 0.0219,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2505531311035156,
+      "rewards/margins": 6.717418670654297,
+      "rewards/rejected": -7.9679718017578125,
+      "step": 466
+    },
+    {
+      "epoch": 5.5266272189349115,
+      "grad_norm": 4.646575940746333,
+      "learning_rate": 1.2915157782458802e-07,
+      "logits/chosen": -1.171278715133667,
+      "logits/rejected": -1.205727458000183,
+      "logps/chosen": -47.129981994628906,
+      "logps/rejected": -68.88152313232422,
+      "loss": 0.02,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.6739788055419922,
+      "rewards/margins": 7.506168842315674,
+      "rewards/rejected": -9.180148124694824,
+      "step": 467
+    },
+    {
+      "epoch": 5.538461538461538,
+      "grad_norm": 6.123565299064301,
+      "learning_rate": 1.2801490644621788e-07,
+      "logits/chosen": -1.1880264282226562,
+      "logits/rejected": -1.1200838088989258,
+      "logps/chosen": -38.7286491394043,
+      "logps/rejected": -48.82095718383789,
+      "loss": 0.0248,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8554790616035461,
+      "rewards/margins": 4.912729740142822,
+      "rewards/rejected": -5.7682085037231445,
+      "step": 468
+    },
+    {
+      "epoch": 5.550295857988166,
+      "grad_norm": 4.461044775468357,
+      "learning_rate": 1.268815352024416e-07,
+      "logits/chosen": -1.3372142314910889,
+      "logits/rejected": -1.2087198495864868,
+      "logps/chosen": -39.35560607910156,
+      "logps/rejected": -66.68772888183594,
+      "loss": 0.0191,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.420445203781128,
+      "rewards/margins": 7.610683441162109,
+      "rewards/rejected": -10.0311279296875,
+      "step": 469
+    },
+    {
+      "epoch": 5.562130177514793,
+      "grad_norm": 3.4572482970403624,
+      "learning_rate": 1.257514947550189e-07,
+      "logits/chosen": -1.3124825954437256,
+      "logits/rejected": -1.2549489736557007,
+      "logps/chosen": -34.87763214111328,
+      "logps/rejected": -71.9126968383789,
+      "loss": 0.0133,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.7214508652687073,
+      "rewards/margins": 9.442049026489258,
+      "rewards/rejected": -10.16349983215332,
+      "step": 470
+    },
+    {
+      "epoch": 5.57396449704142,
+      "grad_norm": 4.197929133254421,
+      "learning_rate": 1.2462481567559966e-07,
+      "logits/chosen": -1.253973126411438,
+      "logits/rejected": -1.0877537727355957,
+      "logps/chosen": -39.30159378051758,
+      "logps/rejected": -77.09483337402344,
+      "loss": 0.0191,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9544857740402222,
+      "rewards/margins": 6.852291107177734,
+      "rewards/rejected": -7.806777000427246,
+      "step": 471
+    },
+    {
+      "epoch": 5.585798816568047,
+      "grad_norm": 5.704469040817279,
+      "learning_rate": 1.2350152844489688e-07,
+      "logits/chosen": -1.2464970350265503,
+      "logits/rejected": -1.2669868469238281,
+      "logps/chosen": -34.1337890625,
+      "logps/rejected": -61.517852783203125,
+      "loss": 0.0246,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.8590599298477173,
+      "rewards/margins": 8.024746894836426,
+      "rewards/rejected": -9.883807182312012,
+      "step": 472
+    },
+    {
+      "epoch": 5.597633136094674,
+      "grad_norm": 3.345069840685552,
+      "learning_rate": 1.2238166345186152e-07,
+      "logits/chosen": -1.1105680465698242,
+      "logits/rejected": -1.1130532026290894,
+      "logps/chosen": -44.979270935058594,
+      "logps/rejected": -61.89778137207031,
+      "loss": 0.0135,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2353321313858032,
+      "rewards/margins": 6.803773880004883,
+      "rewards/rejected": -8.039106369018555,
+      "step": 473
+    },
+    {
+      "epoch": 5.609467455621302,
+      "grad_norm": 5.571254370649255,
+      "learning_rate": 1.2126525099286108e-07,
+      "logits/chosen": -1.5405449867248535,
+      "logits/rejected": -1.5201728343963623,
+      "logps/chosen": -35.71417236328125,
+      "logps/rejected": -76.58299255371094,
+      "loss": 0.0279,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.5148788690567017,
+      "rewards/margins": 8.865453720092773,
+      "rewards/rejected": -10.380332946777344,
+      "step": 474
+    },
+    {
+      "epoch": 5.621301775147929,
+      "grad_norm": 6.171366106305816,
+      "learning_rate": 1.201523212708593e-07,
+      "logits/chosen": -1.2863376140594482,
+      "logits/rejected": -1.0735771656036377,
+      "logps/chosen": -31.64728546142578,
+      "logps/rejected": -67.95651245117188,
+      "loss": 0.0294,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.8103072047233582,
+      "rewards/margins": 8.086475372314453,
+      "rewards/rejected": -8.896783828735352,
+      "step": 475
+    },
+    {
+      "epoch": 5.633136094674557,
+      "grad_norm": 5.562476478094527,
+      "learning_rate": 1.1904290439459971e-07,
+      "logits/chosen": -1.3088757991790771,
+      "logits/rejected": -1.3743656873703003,
+      "logps/chosen": -44.74784851074219,
+      "logps/rejected": -75.913818359375,
+      "loss": 0.0253,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.7934825420379639,
+      "rewards/margins": 8.956192016601562,
+      "rewards/rejected": -10.749673843383789,
+      "step": 476
+    },
+    {
+      "epoch": 5.644970414201183,
+      "grad_norm": 4.9137843056433494,
+      "learning_rate": 1.1793703037779055e-07,
+      "logits/chosen": -1.2263046503067017,
+      "logits/rejected": -1.2299702167510986,
+      "logps/chosen": -52.14278793334961,
+      "logps/rejected": -83.45344543457031,
+      "loss": 0.02,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.6467747688293457,
+      "rewards/margins": 9.627193450927734,
+      "rewards/rejected": -12.273969650268555,
+      "step": 477
+    },
+    {
+      "epoch": 5.65680473372781,
+      "grad_norm": 4.209643150852624,
+      "learning_rate": 1.1683472913829284e-07,
+      "logits/chosen": -1.1544851064682007,
+      "logits/rejected": -1.0651438236236572,
+      "logps/chosen": -43.3776741027832,
+      "logps/rejected": -73.96808624267578,
+      "loss": 0.0192,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.589303970336914,
+      "rewards/margins": 7.712847709655762,
+      "rewards/rejected": -9.302152633666992,
+      "step": 478
+    },
+    {
+      "epoch": 5.668639053254438,
+      "grad_norm": 4.463200843657177,
+      "learning_rate": 1.1573603049731153e-07,
+      "logits/chosen": -1.2334978580474854,
+      "logits/rejected": -1.2861613035202026,
+      "logps/chosen": -46.0629997253418,
+      "logps/rejected": -64.01387023925781,
+      "loss": 0.0177,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.542292058467865,
+      "rewards/margins": 7.245844841003418,
+      "rewards/rejected": -7.788136959075928,
+      "step": 479
+    },
+    {
+      "epoch": 5.680473372781065,
+      "grad_norm": 4.733353453788434,
+      "learning_rate": 1.146409641785882e-07,
+      "logits/chosen": -1.3681819438934326,
+      "logits/rejected": -1.1802641153335571,
+      "logps/chosen": -49.00510025024414,
+      "logps/rejected": -74.6863784790039,
+      "loss": 0.0171,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9393169283866882,
+      "rewards/margins": 8.013870239257812,
+      "rewards/rejected": -8.953186988830566,
+      "step": 480
+    },
+    {
+      "epoch": 5.6923076923076925,
+      "grad_norm": 4.192888321849314,
+      "learning_rate": 1.1354955980759689e-07,
+      "logits/chosen": -1.2294647693634033,
+      "logits/rejected": -1.1312428712844849,
+      "logps/chosen": -32.66830825805664,
+      "logps/rejected": -62.29680252075195,
+      "loss": 0.0161,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.7126701474189758,
+      "rewards/margins": 7.077181816101074,
+      "rewards/rejected": -7.789851665496826,
+      "step": 481
+    },
+    {
+      "epoch": 5.704142011834319,
+      "grad_norm": 4.6911466794721255,
+      "learning_rate": 1.1246184691074314e-07,
+      "logits/chosen": -1.097620964050293,
+      "logits/rejected": -1.0427839756011963,
+      "logps/chosen": -36.93230056762695,
+      "logps/rejected": -56.27244567871094,
+      "loss": 0.0224,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5761284232139587,
+      "rewards/margins": 6.3100385665893555,
+      "rewards/rejected": -6.886167049407959,
+      "step": 482
+    },
+    {
+      "epoch": 5.715976331360947,
+      "grad_norm": 4.510848016909919,
+      "learning_rate": 1.1137785491456453e-07,
+      "logits/chosen": -1.135402798652649,
+      "logits/rejected": -1.1884230375289917,
+      "logps/chosen": -42.09026336669922,
+      "logps/rejected": -65.27650451660156,
+      "loss": 0.0157,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.159618377685547,
+      "rewards/margins": 7.496487617492676,
+      "rewards/rejected": -9.656106948852539,
+      "step": 483
+    },
+    {
+      "epoch": 5.727810650887574,
+      "grad_norm": 6.019538895167783,
+      "learning_rate": 1.1029761314493518e-07,
+      "logits/chosen": -1.4300427436828613,
+      "logits/rejected": -1.311929702758789,
+      "logps/chosen": -35.11430358886719,
+      "logps/rejected": -61.214385986328125,
+      "loss": 0.0257,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1705330610275269,
+      "rewards/margins": 6.273542404174805,
+      "rewards/rejected": -7.444075584411621,
+      "step": 484
+    },
+    {
+      "epoch": 5.739644970414201,
+      "grad_norm": 4.484361080757079,
+      "learning_rate": 1.0922115082627196e-07,
+      "logits/chosen": -1.2917143106460571,
+      "logits/rejected": -1.2676193714141846,
+      "logps/chosen": -39.90666580200195,
+      "logps/rejected": -63.61042404174805,
+      "loss": 0.0201,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.078958511352539,
+      "rewards/margins": 7.296936511993408,
+      "rewards/rejected": -9.375894546508789,
+      "step": 485
+    },
+    {
+      "epoch": 5.7514792899408285,
+      "grad_norm": 3.7963083798265678,
+      "learning_rate": 1.0814849708074414e-07,
+      "logits/chosen": -1.2180938720703125,
+      "logits/rejected": -1.2313947677612305,
+      "logps/chosen": -38.8764762878418,
+      "logps/rejected": -58.5401496887207,
+      "loss": 0.0163,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2903863787651062,
+      "rewards/margins": 6.708767890930176,
+      "rewards/rejected": -6.999154090881348,
+      "step": 486
+    },
+    {
+      "epoch": 5.763313609467455,
+      "grad_norm": 3.9184087371836953,
+      "learning_rate": 1.070796809274853e-07,
+      "logits/chosen": -1.410796880722046,
+      "logits/rejected": -1.51241934299469,
+      "logps/chosen": -39.3426513671875,
+      "logps/rejected": -64.799560546875,
+      "loss": 0.0121,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2968083620071411,
+      "rewards/margins": 8.182130813598633,
+      "rewards/rejected": -9.478939056396484,
+      "step": 487
+    },
+    {
+      "epoch": 5.775147928994083,
+      "grad_norm": 4.695034688833399,
+      "learning_rate": 1.0601473128180854e-07,
+      "logits/chosen": -1.1525176763534546,
+      "logits/rejected": -1.1728888750076294,
+      "logps/chosen": -29.180187225341797,
+      "logps/rejected": -47.42591094970703,
+      "loss": 0.025,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2968808710575104,
+      "rewards/margins": 5.646054267883301,
+      "rewards/rejected": -5.349173545837402,
+      "step": 488
+    },
+    {
+      "epoch": 5.78698224852071,
+      "grad_norm": 3.9629884838666793,
+      "learning_rate": 1.0495367695442392e-07,
+      "logits/chosen": -1.4741270542144775,
+      "logits/rejected": -1.4874193668365479,
+      "logps/chosen": -37.341251373291016,
+      "logps/rejected": -64.96741485595703,
+      "loss": 0.0176,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.7409987449645996,
+      "rewards/margins": 8.580930709838867,
+      "rewards/rejected": -10.321928977966309,
+      "step": 489
+    },
+    {
+      "epoch": 5.798816568047338,
+      "grad_norm": 4.579911167657576,
+      "learning_rate": 1.0389654665065908e-07,
+      "logits/chosen": -1.3020949363708496,
+      "logits/rejected": -1.1568399667739868,
+      "logps/chosen": -37.12276840209961,
+      "logps/rejected": -74.50867462158203,
+      "loss": 0.0188,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.156496047973633,
+      "rewards/margins": 8.242414474487305,
+      "rewards/rejected": -10.398909568786621,
+      "step": 490
+    },
+    {
+      "epoch": 5.810650887573964,
+      "grad_norm": 3.8373743576852046,
+      "learning_rate": 1.0284336896968304e-07,
+      "logits/chosen": -1.4103869199752808,
+      "logits/rejected": -1.2831324338912964,
+      "logps/chosen": -27.218812942504883,
+      "logps/rejected": -64.12830352783203,
+      "loss": 0.0129,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5227464437484741,
+      "rewards/margins": 7.4249677658081055,
+      "rewards/rejected": -7.947714328765869,
+      "step": 491
+    },
+    {
+      "epoch": 5.822485207100591,
+      "grad_norm": 3.95946268366364,
+      "learning_rate": 1.0179417240373182e-07,
+      "logits/chosen": -1.0852227210998535,
+      "logits/rejected": -1.0003684759140015,
+      "logps/chosen": -34.86237716674805,
+      "logps/rejected": -55.69224166870117,
+      "loss": 0.0168,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.195955753326416,
+      "rewards/margins": 6.8462629318237305,
+      "rewards/rejected": -8.042219161987305,
+      "step": 492
+    },
+    {
+      "epoch": 5.834319526627219,
+      "grad_norm": 4.561320933520321,
+      "learning_rate": 1.0074898533733833e-07,
+      "logits/chosen": -1.3027982711791992,
+      "logits/rejected": -1.272012710571289,
+      "logps/chosen": -44.73501205444336,
+      "logps/rejected": -74.4944076538086,
+      "loss": 0.0233,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.941219687461853,
+      "rewards/margins": 7.836876392364502,
+      "rewards/rejected": -8.778095245361328,
+      "step": 493
+    },
+    {
+      "epoch": 5.846153846153846,
+      "grad_norm": 3.4880073627324113,
+      "learning_rate": 9.970783604656383e-08,
+      "logits/chosen": -1.1432762145996094,
+      "logits/rejected": -1.182610034942627,
+      "logps/chosen": -45.404197692871094,
+      "logps/rejected": -68.64156341552734,
+      "loss": 0.0132,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.063544273376465,
+      "rewards/margins": 7.183533668518066,
+      "rewards/rejected": -9.247077941894531,
+      "step": 494
+    },
+    {
+      "epoch": 5.8579881656804735,
+      "grad_norm": 4.263751098046147,
+      "learning_rate": 9.867075269823353e-08,
+      "logits/chosen": -1.2150077819824219,
+      "logits/rejected": -1.156544804573059,
+      "logps/chosen": -31.194244384765625,
+      "logps/rejected": -62.26982498168945,
+      "loss": 0.0163,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5434834957122803,
+      "rewards/margins": 7.384872913360596,
+      "rewards/rejected": -7.928356647491455,
+      "step": 495
+    },
+    {
+      "epoch": 5.8698224852071,
+      "grad_norm": 3.5712261428305476,
+      "learning_rate": 9.763776334917398e-08,
+      "logits/chosen": -1.137619972229004,
+      "logits/rejected": -1.1010360717773438,
+      "logps/chosen": -42.5244140625,
+      "logps/rejected": -64.01480865478516,
+      "loss": 0.0147,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.5774519443511963,
+      "rewards/margins": 7.529844284057617,
+      "rewards/rejected": -9.107295989990234,
+      "step": 496
+    },
+    {
+      "epoch": 5.881656804733728,
+      "grad_norm": 3.9520695580072838,
+      "learning_rate": 9.660889594545469e-08,
+      "logits/chosen": -1.3965240716934204,
+      "logits/rejected": -1.2792103290557861,
+      "logps/chosen": -35.29607391357422,
+      "logps/rejected": -62.42978286743164,
+      "loss": 0.0165,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.475753664970398,
+      "rewards/margins": 7.789628982543945,
+      "rewards/rejected": -9.265382766723633,
+      "step": 497
+    },
+    {
+      "epoch": 5.893491124260355,
+      "grad_norm": 2.9741347782819343,
+      "learning_rate": 9.558417832163162e-08,
+      "logits/chosen": -1.3339378833770752,
+      "logits/rejected": -1.3494333028793335,
+      "logps/chosen": -34.82639694213867,
+      "logps/rejected": -60.99176788330078,
+      "loss": 0.0113,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.567624568939209,
+      "rewards/margins": 7.62171745300293,
+      "rewards/rejected": -8.189342498779297,
+      "step": 498
+    },
+    {
+      "epoch": 5.905325443786982,
+      "grad_norm": 3.571047214468529,
+      "learning_rate": 9.456363819999419e-08,
+      "logits/chosen": -1.3118352890014648,
+      "logits/rejected": -1.2693629264831543,
+      "logps/chosen": -41.321712493896484,
+      "logps/rejected": -56.57762145996094,
+      "loss": 0.0158,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.858654797077179,
+      "rewards/margins": 6.5985426902771,
+      "rewards/rejected": -7.457197666168213,
+      "step": 499
+    },
+    {
+      "epoch": 5.9171597633136095,
+      "grad_norm": 5.851871018530032,
+      "learning_rate": 9.354730318981561e-08,
+      "logits/chosen": -1.362858772277832,
+      "logits/rejected": -1.2244151830673218,
+      "logps/chosen": -42.49793243408203,
+      "logps/rejected": -82.23344421386719,
+      "loss": 0.0193,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.179011344909668,
+      "rewards/margins": 9.12831974029541,
+      "rewards/rejected": -11.307331085205078,
+      "step": 500
+    },
+    {
+      "epoch": 5.928994082840236,
+      "grad_norm": 5.383838413833312,
+      "learning_rate": 9.25352007866054e-08,
+      "logits/chosen": -1.2556869983673096,
+      "logits/rejected": -1.443664789199829,
+      "logps/chosen": -40.52064514160156,
+      "logps/rejected": -56.383453369140625,
+      "loss": 0.0227,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.034975528717041,
+      "rewards/margins": 6.7539753913879395,
+      "rewards/rejected": -8.78895092010498,
+      "step": 501
+    },
+    {
+      "epoch": 5.940828402366864,
+      "grad_norm": 5.05266297652339,
+      "learning_rate": 9.15273583713663e-08,
+      "logits/chosen": -1.4486913681030273,
+      "logits/rejected": -1.5014913082122803,
+      "logps/chosen": -48.460594177246094,
+      "logps/rejected": -62.99694061279297,
+      "loss": 0.0287,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.5520663261413574,
+      "rewards/margins": 5.633842468261719,
+      "rewards/rejected": -7.185909271240234,
+      "step": 502
+    },
+    {
+      "epoch": 5.952662721893491,
+      "grad_norm": 3.687156022372964,
+      "learning_rate": 9.052380320985273e-08,
+      "logits/chosen": -1.378886103630066,
+      "logits/rejected": -1.4398740530014038,
+      "logps/chosen": -45.16747283935547,
+      "logps/rejected": -63.0939826965332,
+      "loss": 0.0167,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.6366732120513916,
+      "rewards/margins": 7.194758892059326,
+      "rewards/rejected": -8.83143138885498,
+      "step": 503
+    },
+    {
+      "epoch": 5.964497041420119,
+      "grad_norm": 4.578124297729498,
+      "learning_rate": 8.95245624518336e-08,
+      "logits/chosen": -1.1423323154449463,
+      "logits/rejected": -1.1454185247421265,
+      "logps/chosen": -36.498924255371094,
+      "logps/rejected": -51.14469528198242,
+      "loss": 0.0201,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.339404821395874,
+      "rewards/margins": 5.312580585479736,
+      "rewards/rejected": -6.651985168457031,
+      "step": 504
+    },
+    {
+      "epoch": 5.976331360946745,
+      "grad_norm": 3.6118523674964518,
+      "learning_rate": 8.85296631303579e-08,
+      "logits/chosen": -1.2390810251235962,
+      "logits/rejected": -1.1867014169692993,
+      "logps/chosen": -32.07006072998047,
+      "logps/rejected": -58.290775299072266,
+      "loss": 0.0152,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.3951120972633362,
+      "rewards/margins": 6.040757656097412,
+      "rewards/rejected": -6.435870170593262,
+      "step": 505
+    },
+    {
+      "epoch": 5.988165680473373,
+      "grad_norm": 4.13649603264138,
+      "learning_rate": 8.753913216102285e-08,
+      "logits/chosen": -1.404435634613037,
+      "logits/rejected": -1.3154963254928589,
+      "logps/chosen": -47.064483642578125,
+      "logps/rejected": -74.79150390625,
+      "loss": 0.0173,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -3.253176212310791,
+      "rewards/margins": 7.4068121910095215,
+      "rewards/rejected": -10.659988403320312,
+      "step": 506
+    },
+    {
+      "epoch": 6.0,
+      "grad_norm": 6.57455134237728,
+      "learning_rate": 8.655299634124646e-08,
+      "logits/chosen": -1.1584608554840088,
+      "logits/rejected": -1.1112971305847168,
+      "logps/chosen": -35.10588455200195,
+      "logps/rejected": -55.12195587158203,
+      "loss": 0.0193,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5993782877922058,
+      "rewards/margins": 7.258352279663086,
+      "rewards/rejected": -7.857730865478516,
+      "step": 507
+    },
+    {
+      "epoch": 6.011834319526627,
+      "grad_norm": 5.072270968295128,
+      "learning_rate": 8.557128234954189e-08,
+      "logits/chosen": -1.2677009105682373,
+      "logits/rejected": -1.2384376525878906,
+      "logps/chosen": -26.764663696289062,
+      "logps/rejected": -54.153812408447266,
+      "loss": 0.0226,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.4589073061943054,
+      "rewards/margins": 7.230363845825195,
+      "rewards/rejected": -7.689270973205566,
+      "step": 508
+    },
+    {
+      "epoch": 6.023668639053255,
+      "grad_norm": 3.823553652256516,
+      "learning_rate": 8.459401674479594e-08,
+      "logits/chosen": -1.1779940128326416,
+      "logits/rejected": -1.098586082458496,
+      "logps/chosen": -37.93672180175781,
+      "logps/rejected": -65.87274932861328,
+      "loss": 0.0169,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2578346729278564,
+      "rewards/margins": 7.301403999328613,
+      "rewards/rejected": -8.55923843383789,
+      "step": 509
+    },
+    {
+      "epoch": 6.035502958579881,
+      "grad_norm": 3.2726696014561516,
+      "learning_rate": 8.362122596555088e-08,
+      "logits/chosen": -1.3603450059890747,
+      "logits/rejected": -1.190643548965454,
+      "logps/chosen": -31.422203063964844,
+      "logps/rejected": -67.04301452636719,
+      "loss": 0.0129,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9887487888336182,
+      "rewards/margins": 9.028081893920898,
+      "rewards/rejected": -10.016830444335938,
+      "step": 510
+    },
+    {
+      "epoch": 6.047337278106509,
+      "grad_norm": 3.455246506585187,
+      "learning_rate": 8.265293632928854e-08,
+      "logits/chosen": -1.155658483505249,
+      "logits/rejected": -1.0525039434432983,
+      "logps/chosen": -38.18634796142578,
+      "logps/rejected": -54.874427795410156,
+      "loss": 0.0138,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.180433750152588,
+      "rewards/margins": 5.439917087554932,
+      "rewards/rejected": -6.6203508377075195,
+      "step": 511
+    },
+    {
+      "epoch": 6.059171597633136,
+      "grad_norm": 3.350614417751688,
+      "learning_rate": 8.16891740317189e-08,
+      "logits/chosen": -1.2656223773956299,
+      "logits/rejected": -1.3540056943893433,
+      "logps/chosen": -35.06914138793945,
+      "logps/rejected": -60.15445327758789,
+      "loss": 0.0148,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.5391483306884766,
+      "rewards/margins": 8.150188446044922,
+      "rewards/rejected": -9.689336776733398,
+      "step": 512
+    },
+    {
+      "epoch": 6.071005917159764,
+      "grad_norm": 3.484209239680023,
+      "learning_rate": 8.072996514607124e-08,
+      "logits/chosen": -1.2974562644958496,
+      "logits/rejected": -1.1921536922454834,
+      "logps/chosen": -55.431785583496094,
+      "logps/rejected": -79.14292907714844,
+      "loss": 0.0184,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.612431526184082,
+      "rewards/margins": 9.743474960327148,
+      "rewards/rejected": -12.355907440185547,
+      "step": 513
+    },
+    {
+      "epoch": 6.0828402366863905,
+      "grad_norm": 4.446328914142042,
+      "learning_rate": 7.977533562238838e-08,
+      "logits/chosen": -1.277441143989563,
+      "logits/rejected": -1.2345993518829346,
+      "logps/chosen": -40.02715301513672,
+      "logps/rejected": -76.5589599609375,
+      "loss": 0.0193,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1496319770812988,
+      "rewards/margins": 10.045013427734375,
+      "rewards/rejected": -11.194644927978516,
+      "step": 514
+    },
+    {
+      "epoch": 6.094674556213017,
+      "grad_norm": 3.4828911535899496,
+      "learning_rate": 7.882531128682538e-08,
+      "logits/chosen": -1.220058798789978,
+      "logits/rejected": -1.2398343086242676,
+      "logps/chosen": -48.92205810546875,
+      "logps/rejected": -75.846923828125,
+      "loss": 0.0137,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.44828462600708,
+      "rewards/margins": 8.325919151306152,
+      "rewards/rejected": -10.77420425415039,
+      "step": 515
+    },
+    {
+      "epoch": 6.106508875739645,
+      "grad_norm": 3.974217510492179,
+      "learning_rate": 7.787991784094999e-08,
+      "logits/chosen": -1.1681749820709229,
+      "logits/rejected": -1.2502659559249878,
+      "logps/chosen": -40.98628616333008,
+      "logps/rejected": -69.3888168334961,
+      "loss": 0.0181,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.623548746109009,
+      "rewards/margins": 9.416035652160645,
+      "rewards/rejected": -12.03958511352539,
+      "step": 516
+    },
+    {
+      "epoch": 6.118343195266272,
+      "grad_norm": 3.087635521774272,
+      "learning_rate": 7.693918086104825e-08,
+      "logits/chosen": -1.2360180616378784,
+      "logits/rejected": -1.1360399723052979,
+      "logps/chosen": -37.70630645751953,
+      "logps/rejected": -69.30216979980469,
+      "loss": 0.0139,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.35728567838668823,
+      "rewards/margins": 8.243356704711914,
+      "rewards/rejected": -8.600642204284668,
+      "step": 517
+    },
+    {
+      "epoch": 6.1301775147929,
+      "grad_norm": 4.697710852543094,
+      "learning_rate": 7.60031257974316e-08,
+      "logits/chosen": -1.4873653650283813,
+      "logits/rejected": -1.365291953086853,
+      "logps/chosen": -36.08180618286133,
+      "logps/rejected": -69.22196960449219,
+      "loss": 0.02,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.034714698791504,
+      "rewards/margins": 7.524087905883789,
+      "rewards/rejected": -9.558802604675293,
+      "step": 518
+    },
+    {
+      "epoch": 6.1420118343195265,
+      "grad_norm": 4.048751031295707,
+      "learning_rate": 7.507177797374927e-08,
+      "logits/chosen": -1.3827829360961914,
+      "logits/rejected": -1.363875389099121,
+      "logps/chosen": -43.49116516113281,
+      "logps/rejected": -78.3033447265625,
+      "loss": 0.0148,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.8128806352615356,
+      "rewards/margins": 10.288780212402344,
+      "rewards/rejected": -12.101661682128906,
+      "step": 519
+    },
+    {
+      "epoch": 6.153846153846154,
+      "grad_norm": 3.0144143135282286,
+      "learning_rate": 7.414516258630244e-08,
+      "logits/chosen": -1.3672518730163574,
+      "logits/rejected": -1.294494390487671,
+      "logps/chosen": -28.87794303894043,
+      "logps/rejected": -49.833717346191406,
+      "loss": 0.0112,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.42372238636016846,
+      "rewards/margins": 6.666773796081543,
+      "rewards/rejected": -7.090496063232422,
+      "step": 520
+    },
+    {
+      "epoch": 6.165680473372781,
+      "grad_norm": 5.7375168904982745,
+      "learning_rate": 7.322330470336313e-08,
+      "logits/chosen": -1.1266202926635742,
+      "logits/rejected": -1.1897248029708862,
+      "logps/chosen": -36.070762634277344,
+      "logps/rejected": -67.00115203857422,
+      "loss": 0.0237,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.276731491088867,
+      "rewards/margins": 8.913595199584961,
+      "rewards/rejected": -11.190326690673828,
+      "step": 521
+    },
+    {
+      "epoch": 6.177514792899408,
+      "grad_norm": 3.9341322070007227,
+      "learning_rate": 7.230622926449564e-08,
+      "logits/chosen": -1.2871345281600952,
+      "logits/rejected": -1.3150429725646973,
+      "logps/chosen": -32.48534393310547,
+      "logps/rejected": -56.244903564453125,
+      "loss": 0.0183,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2191728353500366,
+      "rewards/margins": 7.289381980895996,
+      "rewards/rejected": -8.508554458618164,
+      "step": 522
+    },
+    {
+      "epoch": 6.189349112426036,
+      "grad_norm": 3.8384098316045776,
+      "learning_rate": 7.139396107988193e-08,
+      "logits/chosen": -1.0808664560317993,
+      "logits/rejected": -1.051466464996338,
+      "logps/chosen": -36.810298919677734,
+      "logps/rejected": -50.8199462890625,
+      "loss": 0.0211,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.059516020119190216,
+      "rewards/margins": 5.3354082107543945,
+      "rewards/rejected": -5.27589225769043,
+      "step": 523
+    },
+    {
+      "epoch": 6.201183431952662,
+      "grad_norm": 3.7617803402555965,
+      "learning_rate": 7.048652482965078e-08,
+      "logits/chosen": -1.5526072978973389,
+      "logits/rejected": -1.4698734283447266,
+      "logps/chosen": -35.63499450683594,
+      "logps/rejected": -58.799835205078125,
+      "loss": 0.0162,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1897969245910645,
+      "rewards/margins": 6.380156517028809,
+      "rewards/rejected": -7.569952964782715,
+      "step": 524
+    },
+    {
+      "epoch": 6.21301775147929,
+      "grad_norm": 6.280571337534133,
+      "learning_rate": 6.958394506320947e-08,
+      "logits/chosen": -1.082352876663208,
+      "logits/rejected": -0.9739059805870056,
+      "logps/chosen": -59.19343566894531,
+      "logps/rejected": -85.16358184814453,
+      "loss": 0.0226,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -3.377283811569214,
+      "rewards/margins": 8.0195951461792,
+      "rewards/rejected": -11.396878242492676,
+      "step": 525
+    },
+    {
+      "epoch": 6.224852071005917,
+      "grad_norm": 3.6878257721370926,
+      "learning_rate": 6.868624619858021e-08,
+      "logits/chosen": -1.1773961782455444,
+      "logits/rejected": -1.069741129875183,
+      "logps/chosen": -35.47507095336914,
+      "logps/rejected": -71.48643493652344,
+      "loss": 0.014,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.48855772614479065,
+      "rewards/margins": 9.38101863861084,
+      "rewards/rejected": -9.869575500488281,
+      "step": 526
+    },
+    {
+      "epoch": 6.236686390532545,
+      "grad_norm": 5.553600246448852,
+      "learning_rate": 6.779345252173906e-08,
+      "logits/chosen": -0.9772287011146545,
+      "logits/rejected": -1.2621002197265625,
+      "logps/chosen": -66.20740509033203,
+      "logps/rejected": -61.06924819946289,
+      "loss": 0.0241,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.746315598487854,
+      "rewards/margins": 5.9837775230407715,
+      "rewards/rejected": -6.730093002319336,
+      "step": 527
+    },
+    {
+      "epoch": 6.2485207100591715,
+      "grad_norm": 4.0981125572939785,
+      "learning_rate": 6.690558818595943e-08,
+      "logits/chosen": -1.3861331939697266,
+      "logits/rejected": -1.297480821609497,
+      "logps/chosen": -43.87446975708008,
+      "logps/rejected": -63.237091064453125,
+      "loss": 0.0163,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.5427628755569458,
+      "rewards/margins": 7.580643653869629,
+      "rewards/rejected": -9.123406410217285,
+      "step": 528
+    },
+    {
+      "epoch": 6.260355029585799,
+      "grad_norm": 3.147025133945202,
+      "learning_rate": 6.602267721115806e-08,
+      "logits/chosen": -1.410119891166687,
+      "logits/rejected": -1.324759840965271,
+      "logps/chosen": -42.89299011230469,
+      "logps/rejected": -70.10383605957031,
+      "loss": 0.012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9887200593948364,
+      "rewards/margins": 6.539437770843506,
+      "rewards/rejected": -7.5281572341918945,
+      "step": 529
+    },
+    {
+      "epoch": 6.272189349112426,
+      "grad_norm": 3.4923988387654714,
+      "learning_rate": 6.514474348324581e-08,
+      "logits/chosen": -1.1285738945007324,
+      "logits/rejected": -1.0195645093917847,
+      "logps/chosen": -39.756072998046875,
+      "logps/rejected": -64.98381042480469,
+      "loss": 0.0142,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.783644437789917,
+      "rewards/margins": 7.37436056137085,
+      "rewards/rejected": -9.158004760742188,
+      "step": 530
+    },
+    {
+      "epoch": 6.284023668639053,
+      "grad_norm": 4.081843420506292,
+      "learning_rate": 6.427181075348084e-08,
+      "logits/chosen": -1.1205132007598877,
+      "logits/rejected": -1.0184344053268433,
+      "logps/chosen": -29.305950164794922,
+      "logps/rejected": -60.268062591552734,
+      "loss": 0.0181,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.907474160194397,
+      "rewards/margins": 6.502952575683594,
+      "rewards/rejected": -7.410426139831543,
+      "step": 531
+    },
+    {
+      "epoch": 6.295857988165681,
+      "grad_norm": 3.8839404909626585,
+      "learning_rate": 6.340390263782655e-08,
+      "logits/chosen": -1.3858798742294312,
+      "logits/rejected": -1.2407383918762207,
+      "logps/chosen": -41.1785888671875,
+      "logps/rejected": -76.4423828125,
+      "loss": 0.0159,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.216499090194702,
+      "rewards/margins": 8.766088485717773,
+      "rewards/rejected": -10.982587814331055,
+      "step": 532
+    },
+    {
+      "epoch": 6.3076923076923075,
+      "grad_norm": 4.4263228598532365,
+      "learning_rate": 6.254104261631254e-08,
+      "logits/chosen": -1.1712279319763184,
+      "logits/rejected": -1.214058518409729,
+      "logps/chosen": -34.68932342529297,
+      "logps/rejected": -55.6457633972168,
+      "loss": 0.0212,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2202101945877075,
+      "rewards/margins": 6.376070022583008,
+      "rewards/rejected": -7.596280574798584,
+      "step": 533
+    },
+    {
+      "epoch": 6.319526627218935,
+      "grad_norm": 3.9131998116199918,
+      "learning_rate": 6.168325403239913e-08,
+      "logits/chosen": -1.3168898820877075,
+      "logits/rejected": -1.4670816659927368,
+      "logps/chosen": -45.90602111816406,
+      "logps/rejected": -59.57246398925781,
+      "loss": 0.0159,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.582594633102417,
+      "rewards/margins": 7.453899383544922,
+      "rewards/rejected": -9.036494255065918,
+      "step": 534
+    },
+    {
+      "epoch": 6.331360946745562,
+      "grad_norm": 4.764780460541576,
+      "learning_rate": 6.08305600923463e-08,
+      "logits/chosen": -1.1949267387390137,
+      "logits/rejected": -1.23961341381073,
+      "logps/chosen": -36.51057434082031,
+      "logps/rejected": -72.36233520507812,
+      "loss": 0.0216,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.984785795211792,
+      "rewards/margins": 8.723003387451172,
+      "rewards/rejected": -11.707788467407227,
+      "step": 535
+    },
+    {
+      "epoch": 6.34319526627219,
+      "grad_norm": 4.149947688223677,
+      "learning_rate": 5.998298386458545e-08,
+      "logits/chosen": -1.1838600635528564,
+      "logits/rejected": -1.0856642723083496,
+      "logps/chosen": -42.33216857910156,
+      "logps/rejected": -66.8110122680664,
+      "loss": 0.0171,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.4710516929626465,
+      "rewards/margins": 6.801319599151611,
+      "rewards/rejected": -9.272371292114258,
+      "step": 536
+    },
+    {
+      "epoch": 6.355029585798817,
+      "grad_norm": 4.1571001554854465,
+      "learning_rate": 5.914054827909548e-08,
+      "logits/chosen": -1.094924807548523,
+      "logits/rejected": -1.1296252012252808,
+      "logps/chosen": -46.14147186279297,
+      "logps/rejected": -86.65098571777344,
+      "loss": 0.0143,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.7291374206542969,
+      "rewards/margins": 10.872190475463867,
+      "rewards/rejected": -12.601327896118164,
+      "step": 537
+    },
+    {
+      "epoch": 6.366863905325443,
+      "grad_norm": 4.127454820039479,
+      "learning_rate": 5.830327612678265e-08,
+      "logits/chosen": -1.187494158744812,
+      "logits/rejected": -1.201699137687683,
+      "logps/chosen": -43.62383270263672,
+      "logps/rejected": -69.97433471679688,
+      "loss": 0.0177,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.9448773860931396,
+      "rewards/margins": 9.503475189208984,
+      "rewards/rejected": -11.448352813720703,
+      "step": 538
+    },
+    {
+      "epoch": 6.378698224852071,
+      "grad_norm": 5.236200973476889,
+      "learning_rate": 5.747119005886361e-08,
+      "logits/chosen": -1.3696651458740234,
+      "logits/rejected": -1.2970082759857178,
+      "logps/chosen": -35.38874816894531,
+      "logps/rejected": -63.076866149902344,
+      "loss": 0.018,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.804175615310669,
+      "rewards/margins": 7.7187957763671875,
+      "rewards/rejected": -8.522972106933594,
+      "step": 539
+    },
+    {
+      "epoch": 6.390532544378698,
+      "grad_norm": 3.4361194305765226,
+      "learning_rate": 5.6644312586253044e-08,
+      "logits/chosen": -1.3181718587875366,
+      "logits/rejected": -1.4402356147766113,
+      "logps/chosen": -47.9390754699707,
+      "logps/rejected": -69.83265686035156,
+      "loss": 0.0132,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2599098682403564,
+      "rewards/margins": 6.955543518066406,
+      "rewards/rejected": -8.215453147888184,
+      "step": 540
+    },
+    {
+      "epoch": 6.402366863905326,
+      "grad_norm": 5.247988440802347,
+      "learning_rate": 5.582266607895422e-08,
+      "logits/chosen": -1.2013980150222778,
+      "logits/rejected": -1.2277536392211914,
+      "logps/chosen": -32.803062438964844,
+      "logps/rejected": -51.46520233154297,
+      "loss": 0.019,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.23354974389076233,
+      "rewards/margins": 7.179046154022217,
+      "rewards/rejected": -7.412596702575684,
+      "step": 541
+    },
+    {
+      "epoch": 6.414201183431953,
+      "grad_norm": 4.074351591350812,
+      "learning_rate": 5.5006272765454056e-08,
+      "logits/chosen": -1.6633051633834839,
+      "logits/rejected": -1.3203377723693848,
+      "logps/chosen": -43.05129623413086,
+      "logps/rejected": -89.4016342163086,
+      "loss": 0.017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.711348056793213,
+      "rewards/margins": 10.217995643615723,
+      "rewards/rejected": -11.929344177246094,
+      "step": 542
+    },
+    {
+      "epoch": 6.42603550295858,
+      "grad_norm": 4.882941673599114,
+      "learning_rate": 5.419515473212191e-08,
+      "logits/chosen": -1.0465339422225952,
+      "logits/rejected": -0.9034765958786011,
+      "logps/chosen": -39.69237518310547,
+      "logps/rejected": -64.68145751953125,
+      "loss": 0.0221,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.7466580867767334,
+      "rewards/margins": 5.554471969604492,
+      "rewards/rejected": -7.301130294799805,
+      "step": 543
+    },
+    {
+      "epoch": 6.437869822485207,
+      "grad_norm": 3.1692395598442804,
+      "learning_rate": 5.338933392261158e-08,
+      "logits/chosen": -1.3215035200119019,
+      "logits/rejected": -1.341964602470398,
+      "logps/chosen": -40.1537971496582,
+      "logps/rejected": -63.64967346191406,
+      "loss": 0.0167,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.646998405456543,
+      "rewards/margins": 8.139799118041992,
+      "rewards/rejected": -10.786796569824219,
+      "step": 544
+    },
+    {
+      "epoch": 6.449704142011834,
+      "grad_norm": 5.949554370120341,
+      "learning_rate": 5.258883213726828e-08,
+      "logits/chosen": -1.194183588027954,
+      "logits/rejected": -1.2076101303100586,
+      "logps/chosen": -44.26350402832031,
+      "logps/rejected": -64.31948852539062,
+      "loss": 0.0322,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1567471027374268,
+      "rewards/margins": 7.238821506500244,
+      "rewards/rejected": -8.39556884765625,
+      "step": 545
+    },
+    {
+      "epoch": 6.461538461538462,
+      "grad_norm": 3.7663787692141995,
+      "learning_rate": 5.1793671032538206e-08,
+      "logits/chosen": -1.4605358839035034,
+      "logits/rejected": -1.4506992101669312,
+      "logps/chosen": -35.8072395324707,
+      "logps/rejected": -57.40998458862305,
+      "loss": 0.0191,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.3689743280410767,
+      "rewards/margins": 7.345977783203125,
+      "rewards/rejected": -8.714951515197754,
+      "step": 546
+    },
+    {
+      "epoch": 6.4733727810650885,
+      "grad_norm": 2.9635523735572247,
+      "learning_rate": 5.100387212038324e-08,
+      "logits/chosen": -1.2727254629135132,
+      "logits/rejected": -1.374696135520935,
+      "logps/chosen": -39.98680114746094,
+      "logps/rejected": -69.32289123535156,
+      "loss": 0.0105,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.05542254447937,
+      "rewards/margins": 8.780436515808105,
+      "rewards/rejected": -10.835859298706055,
+      "step": 547
+    },
+    {
+      "epoch": 6.485207100591716,
+      "grad_norm": 5.072194913958191,
+      "learning_rate": 5.021945676769859e-08,
+      "logits/chosen": -1.2008986473083496,
+      "logits/rejected": -1.1709994077682495,
+      "logps/chosen": -46.82018280029297,
+      "logps/rejected": -68.16090393066406,
+      "loss": 0.0225,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.5476022958755493,
+      "rewards/margins": 7.798556804656982,
+      "rewards/rejected": -9.346158981323242,
+      "step": 548
+    },
+    {
+      "epoch": 6.497041420118343,
+      "grad_norm": 4.114324411595974,
+      "learning_rate": 4.9440446195734817e-08,
+      "logits/chosen": -1.4089347124099731,
+      "logits/rejected": -1.346145749092102,
+      "logps/chosen": -31.944503784179688,
+      "logps/rejected": -61.80097198486328,
+      "loss": 0.018,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2490514516830444,
+      "rewards/margins": 8.163646697998047,
+      "rewards/rejected": -9.412696838378906,
+      "step": 549
+    },
+    {
+      "epoch": 6.508875739644971,
+      "grad_norm": 5.13609604269027,
+      "learning_rate": 4.866686147952387e-08,
+      "logits/chosen": -1.5707539319992065,
+      "logits/rejected": -1.5316340923309326,
+      "logps/chosen": -42.37779235839844,
+      "logps/rejected": -67.51331329345703,
+      "loss": 0.0238,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2908955812454224,
+      "rewards/margins": 8.278606414794922,
+      "rewards/rejected": -9.569501876831055,
+      "step": 550
+    },
+    {
+      "epoch": 6.520710059171598,
+      "grad_norm": 5.255591810225031,
+      "learning_rate": 4.789872354730873e-08,
+      "logits/chosen": -1.5177299976348877,
+      "logits/rejected": -1.4273862838745117,
+      "logps/chosen": -32.705142974853516,
+      "logps/rejected": -63.322547912597656,
+      "loss": 0.02,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9528019428253174,
+      "rewards/margins": 7.921998023986816,
+      "rewards/rejected": -8.874799728393555,
+      "step": 551
+    },
+    {
+      "epoch": 6.5325443786982245,
+      "grad_norm": 3.6838956833536938,
+      "learning_rate": 4.71360531799774e-08,
+      "logits/chosen": -1.2748504877090454,
+      "logits/rejected": -1.2340025901794434,
+      "logps/chosen": -38.841758728027344,
+      "logps/rejected": -68.60884094238281,
+      "loss": 0.0168,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4062817096710205,
+      "rewards/margins": 8.743720054626465,
+      "rewards/rejected": -10.150002479553223,
+      "step": 552
+    },
+    {
+      "epoch": 6.544378698224852,
+      "grad_norm": 3.742498897516179,
+      "learning_rate": 4.637887101050053e-08,
+      "logits/chosen": -1.331305742263794,
+      "logits/rejected": -1.2110060453414917,
+      "logps/chosen": -35.87771224975586,
+      "logps/rejected": -66.48898315429688,
+      "loss": 0.0124,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5530174374580383,
+      "rewards/margins": 7.692140579223633,
+      "rewards/rejected": -8.245157241821289,
+      "step": 553
+    },
+    {
+      "epoch": 6.556213017751479,
+      "grad_norm": 4.145756635966609,
+      "learning_rate": 4.562719752337349e-08,
+      "logits/chosen": -1.241339087486267,
+      "logits/rejected": -1.238210916519165,
+      "logps/chosen": -39.84772491455078,
+      "logps/rejected": -65.00758361816406,
+      "loss": 0.0171,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1539757251739502,
+      "rewards/margins": 7.031815528869629,
+      "rewards/rejected": -8.185791015625,
+      "step": 554
+    },
+    {
+      "epoch": 6.568047337278107,
+      "grad_norm": 4.724411466421152,
+      "learning_rate": 4.488105305406187e-08,
+      "logits/chosen": -1.0200409889221191,
+      "logits/rejected": -0.9774606823921204,
+      "logps/chosen": -38.79374694824219,
+      "logps/rejected": -61.03683090209961,
+      "loss": 0.0175,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.9451122283935547,
+      "rewards/margins": 7.298187732696533,
+      "rewards/rejected": -9.24329948425293,
+      "step": 555
+    },
+    {
+      "epoch": 6.579881656804734,
+      "grad_norm": 4.840401053211273,
+      "learning_rate": 4.4140457788451434e-08,
+      "logits/chosen": -1.3930317163467407,
+      "logits/rejected": -1.3531696796417236,
+      "logps/chosen": -51.00494384765625,
+      "logps/rejected": -69.20004272460938,
+      "loss": 0.0199,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.2470803260803223,
+      "rewards/margins": 7.987146854400635,
+      "rewards/rejected": -10.234228134155273,
+      "step": 556
+    },
+    {
+      "epoch": 6.591715976331361,
+      "grad_norm": 4.058919938184551,
+      "learning_rate": 4.340543176230232e-08,
+      "logits/chosen": -1.1343650817871094,
+      "logits/rejected": -1.149098515510559,
+      "logps/chosen": -37.745506286621094,
+      "logps/rejected": -62.56489562988281,
+      "loss": 0.0142,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.3520774841308594,
+      "rewards/margins": 7.742616176605225,
+      "rewards/rejected": -9.094694137573242,
+      "step": 557
+    },
+    {
+      "epoch": 6.603550295857988,
+      "grad_norm": 3.8922355564931865,
+      "learning_rate": 4.267599486070647e-08,
+      "logits/chosen": -1.1515233516693115,
+      "logits/rejected": -1.1255171298980713,
+      "logps/chosen": -33.81602478027344,
+      "logps/rejected": -61.61820983886719,
+      "loss": 0.0177,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.054863691329956,
+      "rewards/margins": 8.12367057800293,
+      "rewards/rejected": -9.178534507751465,
+      "step": 558
+    },
+    {
+      "epoch": 6.615384615384615,
+      "grad_norm": 2.9916154212804025,
+      "learning_rate": 4.1952166817550176e-08,
+      "logits/chosen": -1.3873323202133179,
+      "logits/rejected": -1.270090937614441,
+      "logps/chosen": -37.5373649597168,
+      "logps/rejected": -70.85179138183594,
+      "loss": 0.0115,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.6242471933364868,
+      "rewards/margins": 9.16792106628418,
+      "rewards/rejected": -10.792167663574219,
+      "step": 559
+    },
+    {
+      "epoch": 6.627218934911243,
+      "grad_norm": 3.497280688298504,
+      "learning_rate": 4.1233967214979764e-08,
+      "logits/chosen": -1.2440836429595947,
+      "logits/rejected": -1.182302713394165,
+      "logps/chosen": -36.41077423095703,
+      "logps/rejected": -67.35704803466797,
+      "loss": 0.0142,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9620167016983032,
+      "rewards/margins": 7.311346054077148,
+      "rewards/rejected": -8.27336311340332,
+      "step": 560
+    },
+    {
+      "epoch": 6.6390532544378695,
+      "grad_norm": 5.110607066618349,
+      "learning_rate": 4.05214154828723e-08,
+      "logits/chosen": -1.1929218769073486,
+      "logits/rejected": -1.1870311498641968,
+      "logps/chosen": -41.97100830078125,
+      "logps/rejected": -72.02296447753906,
+      "loss": 0.022,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.625248670578003,
+      "rewards/margins": 8.42895793914795,
+      "rewards/rejected": -10.054206848144531,
+      "step": 561
+    },
+    {
+      "epoch": 6.650887573964497,
+      "grad_norm": 4.697261168017771,
+      "learning_rate": 3.9814530898309356e-08,
+      "logits/chosen": -1.4104831218719482,
+      "logits/rejected": -1.1492834091186523,
+      "logps/chosen": -40.75250244140625,
+      "logps/rejected": -86.0766372680664,
+      "loss": 0.0237,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.9909330606460571,
+      "rewards/margins": 10.393043518066406,
+      "rewards/rejected": -12.383977890014648,
+      "step": 562
+    },
+    {
+      "epoch": 6.662721893491124,
+      "grad_norm": 3.86163495686982,
+      "learning_rate": 3.9113332585056166e-08,
+      "logits/chosen": -1.0711450576782227,
+      "logits/rejected": -1.1439751386642456,
+      "logps/chosen": -47.77074432373047,
+      "logps/rejected": -57.20773696899414,
+      "loss": 0.0134,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.2142012119293213,
+      "rewards/margins": 5.361945152282715,
+      "rewards/rejected": -7.576145648956299,
+      "step": 563
+    },
+    {
+      "epoch": 6.674556213017752,
+      "grad_norm": 3.142265753154405,
+      "learning_rate": 3.8417839513043646e-08,
+      "logits/chosen": -1.3805079460144043,
+      "logits/rejected": -1.2833032608032227,
+      "logps/chosen": -34.34812927246094,
+      "logps/rejected": -61.53770446777344,
+      "loss": 0.0149,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.6832209825515747,
+      "rewards/margins": 7.745155334472656,
+      "rewards/rejected": -9.428376197814941,
+      "step": 564
+    },
+    {
+      "epoch": 6.686390532544379,
+      "grad_norm": 4.537864993317601,
+      "learning_rate": 3.7728070497855594e-08,
+      "logits/chosen": -1.4104318618774414,
+      "logits/rejected": -1.3599334955215454,
+      "logps/chosen": -37.086971282958984,
+      "logps/rejected": -67.63109588623047,
+      "loss": 0.0176,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.2368394136428833,
+      "rewards/margins": 7.939535140991211,
+      "rewards/rejected": -8.176374435424805,
+      "step": 565
+    },
+    {
+      "epoch": 6.6982248520710055,
+      "grad_norm": 3.938202778272971,
+      "learning_rate": 3.704404420021956e-08,
+      "logits/chosen": -1.3482962846755981,
+      "logits/rejected": -1.3277989625930786,
+      "logps/chosen": -42.97172546386719,
+      "logps/rejected": -66.13438415527344,
+      "loss": 0.0152,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.497891902923584,
+      "rewards/margins": 7.281721115112305,
+      "rewards/rejected": -9.77961254119873,
+      "step": 566
+    },
+    {
+      "epoch": 6.710059171597633,
+      "grad_norm": 4.295649983212432,
+      "learning_rate": 3.636577912550187e-08,
+      "logits/chosen": -1.3801615238189697,
+      "logits/rejected": -1.3534774780273438,
+      "logps/chosen": -48.245174407958984,
+      "logps/rejected": -75.80772399902344,
+      "loss": 0.0186,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.976886510848999,
+      "rewards/margins": 8.30002212524414,
+      "rewards/rejected": -11.276908874511719,
+      "step": 567
+    },
+    {
+      "epoch": 6.72189349112426,
+      "grad_norm": 3.3047241439197497,
+      "learning_rate": 3.569329362320708e-08,
+      "logits/chosen": -1.323474407196045,
+      "logits/rejected": -1.2122368812561035,
+      "logps/chosen": -38.685638427734375,
+      "logps/rejected": -81.03096771240234,
+      "loss": 0.012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.7437925934791565,
+      "rewards/margins": 10.262330055236816,
+      "rewards/rejected": -11.006122589111328,
+      "step": 568
+    },
+    {
+      "epoch": 6.733727810650888,
+      "grad_norm": 3.1683374020519315,
+      "learning_rate": 3.5026605886481736e-08,
+      "logits/chosen": -1.4267765283584595,
+      "logits/rejected": -1.4384936094284058,
+      "logps/chosen": -40.923892974853516,
+      "logps/rejected": -66.135009765625,
+      "loss": 0.0146,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4209558963775635,
+      "rewards/margins": 7.333900451660156,
+      "rewards/rejected": -8.75485610961914,
+      "step": 569
+    },
+    {
+      "epoch": 6.745562130177515,
+      "grad_norm": 4.91012737015821,
+      "learning_rate": 3.436573395162179e-08,
+      "logits/chosen": -1.4396589994430542,
+      "logits/rejected": -1.3906140327453613,
+      "logps/chosen": -34.09233856201172,
+      "logps/rejected": -70.2711181640625,
+      "loss": 0.0231,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1198707818984985,
+      "rewards/margins": 8.739965438842773,
+      "rewards/rejected": -9.85983657836914,
+      "step": 570
+    },
+    {
+      "epoch": 6.757396449704142,
+      "grad_norm": 3.6007019387475703,
+      "learning_rate": 3.371069569758511e-08,
+      "logits/chosen": -1.251147747039795,
+      "logits/rejected": -1.1890754699707031,
+      "logps/chosen": -53.093597412109375,
+      "logps/rejected": -83.10395050048828,
+      "loss": 0.0152,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4066200256347656,
+      "rewards/margins": 8.366990089416504,
+      "rewards/rejected": -9.773611068725586,
+      "step": 571
+    },
+    {
+      "epoch": 6.769230769230769,
+      "grad_norm": 3.8258738774626986,
+      "learning_rate": 3.306150884550732e-08,
+      "logits/chosen": -1.2099252939224243,
+      "logits/rejected": -1.2655442953109741,
+      "logps/chosen": -43.43336486816406,
+      "logps/rejected": -58.43470764160156,
+      "loss": 0.0164,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.2070910930633545,
+      "rewards/margins": 6.868325710296631,
+      "rewards/rejected": -9.075416564941406,
+      "step": 572
+    },
+    {
+      "epoch": 6.781065088757396,
+      "grad_norm": 4.510756393854719,
+      "learning_rate": 3.241819095822288e-08,
+      "logits/chosen": -1.2797162532806396,
+      "logits/rejected": -1.3578741550445557,
+      "logps/chosen": -37.647857666015625,
+      "logps/rejected": -63.263267517089844,
+      "loss": 0.0177,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.05109795928001404,
+      "rewards/margins": 7.741544246673584,
+      "rewards/rejected": -7.690445899963379,
+      "step": 573
+    },
+    {
+      "epoch": 6.792899408284024,
+      "grad_norm": 3.963518873210086,
+      "learning_rate": 3.17807594397895e-08,
+      "logits/chosen": -1.1429615020751953,
+      "logits/rejected": -1.1946382522583008,
+      "logps/chosen": -34.58401870727539,
+      "logps/rejected": -63.32178497314453,
+      "loss": 0.0157,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.431036114692688,
+      "rewards/margins": 7.827291488647461,
+      "rewards/rejected": -9.25832748413086,
+      "step": 574
+    },
+    {
+      "epoch": 6.804733727810651,
+      "grad_norm": 3.969471879010277,
+      "learning_rate": 3.114923153501747e-08,
+      "logits/chosen": -1.2164993286132812,
+      "logits/rejected": -1.1609983444213867,
+      "logps/chosen": -31.950668334960938,
+      "logps/rejected": -61.592769622802734,
+      "loss": 0.0163,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.7955553531646729,
+      "rewards/margins": 7.445010185241699,
+      "rewards/rejected": -9.24056625366211,
+      "step": 575
+    },
+    {
+      "epoch": 6.816568047337278,
+      "grad_norm": 4.7113289076048,
+      "learning_rate": 3.052362432900332e-08,
+      "logits/chosen": -1.1342840194702148,
+      "logits/rejected": -1.0492384433746338,
+      "logps/chosen": -32.85274124145508,
+      "logps/rejected": -72.11370086669922,
+      "loss": 0.0195,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.3644051551818848,
+      "rewards/margins": 8.635282516479492,
+      "rewards/rejected": -9.999687194824219,
+      "step": 576
+    },
+    {
+      "epoch": 6.828402366863905,
+      "grad_norm": 3.7717255825498675,
+      "learning_rate": 2.990395474666724e-08,
+      "logits/chosen": -1.0524818897247314,
+      "logits/rejected": -0.9995134472846985,
+      "logps/chosen": -40.310935974121094,
+      "logps/rejected": -60.963748931884766,
+      "loss": 0.0161,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.0798089504241943,
+      "rewards/margins": 6.669445991516113,
+      "rewards/rejected": -8.74925422668457,
+      "step": 577
+    },
+    {
+      "epoch": 6.840236686390533,
+      "grad_norm": 4.294637735523631,
+      "learning_rate": 2.9290239552295538e-08,
+      "logits/chosen": -1.2513582706451416,
+      "logits/rejected": -1.1278815269470215,
+      "logps/chosen": -37.120304107666016,
+      "logps/rejected": -60.05486297607422,
+      "loss": 0.017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5432335734367371,
+      "rewards/margins": 6.656371593475342,
+      "rewards/rejected": -7.1996049880981445,
+      "step": 578
+    },
+    {
+      "epoch": 6.85207100591716,
+      "grad_norm": 3.417783997535037,
+      "learning_rate": 2.8682495349086816e-08,
+      "logits/chosen": -1.1197659969329834,
+      "logits/rejected": -1.3288072347640991,
+      "logps/chosen": -47.85517883300781,
+      "logps/rejected": -61.156795501708984,
+      "loss": 0.0143,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.6630420684814453,
+      "rewards/margins": 7.849754810333252,
+      "rewards/rejected": -9.512797355651855,
+      "step": 579
+    },
+    {
+      "epoch": 6.8639053254437865,
+      "grad_norm": 3.9888601265802626,
+      "learning_rate": 2.8080738578703052e-08,
+      "logits/chosen": -1.3530460596084595,
+      "logits/rejected": -1.4569690227508545,
+      "logps/chosen": -54.98475646972656,
+      "logps/rejected": -76.0159912109375,
+      "loss": 0.0173,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.483710765838623,
+      "rewards/margins": 9.008731842041016,
+      "rewards/rejected": -10.49244213104248,
+      "step": 580
+    },
+    {
+      "epoch": 6.875739644970414,
+      "grad_norm": 2.9654411597287273,
+      "learning_rate": 2.748498552082465e-08,
+      "logits/chosen": -1.2621766328811646,
+      "logits/rejected": -1.3557482957839966,
+      "logps/chosen": -41.09724044799805,
+      "logps/rejected": -64.43111419677734,
+      "loss": 0.0105,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.5274691581726074,
+      "rewards/margins": 7.362679481506348,
+      "rewards/rejected": -8.890148162841797,
+      "step": 581
+    },
+    {
+      "epoch": 6.887573964497041,
+      "grad_norm": 3.6343192832198272,
+      "learning_rate": 2.6895252292709974e-08,
+      "logits/chosen": -1.2823108434677124,
+      "logits/rejected": -1.2685920000076294,
+      "logps/chosen": -47.48131561279297,
+      "logps/rejected": -65.60801696777344,
+      "loss": 0.0168,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.3462042808532715,
+      "rewards/margins": 6.321001052856445,
+      "rewards/rejected": -7.667204856872559,
+      "step": 582
+    },
+    {
+      "epoch": 6.899408284023669,
+      "grad_norm": 4.413434221844061,
+      "learning_rate": 2.631155484875952e-08,
+      "logits/chosen": -1.2258765697479248,
+      "logits/rejected": -1.1181480884552002,
+      "logps/chosen": -43.612274169921875,
+      "logps/rejected": -72.54850769042969,
+      "loss": 0.0163,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.8452584743499756,
+      "rewards/margins": 8.232542991638184,
+      "rewards/rejected": -10.077801704406738,
+      "step": 583
+    },
+    {
+      "epoch": 6.911242603550296,
+      "grad_norm": 3.1918002353766037,
+      "learning_rate": 2.5733908980083984e-08,
+      "logits/chosen": -1.3073086738586426,
+      "logits/rejected": -1.2347233295440674,
+      "logps/chosen": -40.125244140625,
+      "logps/rejected": -69.67649841308594,
+      "loss": 0.0126,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.3976001739501953,
+      "rewards/margins": 8.126445770263672,
+      "rewards/rejected": -10.524045944213867,
+      "step": 584
+    },
+    {
+      "epoch": 6.923076923076923,
+      "grad_norm": 3.672509775762771,
+      "learning_rate": 2.5162330314077385e-08,
+      "logits/chosen": -1.2911275625228882,
+      "logits/rejected": -1.332797646522522,
+      "logps/chosen": -60.97373580932617,
+      "logps/rejected": -87.76260375976562,
+      "loss": 0.0129,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -4.335407733917236,
+      "rewards/margins": 8.518815994262695,
+      "rewards/rejected": -12.854223251342773,
+      "step": 585
+    },
+    {
+      "epoch": 6.93491124260355,
+      "grad_norm": 2.4576523514031123,
+      "learning_rate": 2.4596834313994037e-08,
+      "logits/chosen": -1.215404748916626,
+      "logits/rejected": -1.20293128490448,
+      "logps/chosen": -31.431991577148438,
+      "logps/rejected": -50.589115142822266,
+      "loss": 0.0103,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9661941528320312,
+      "rewards/margins": 6.143428325653076,
+      "rewards/rejected": -7.109622955322266,
+      "step": 586
+    },
+    {
+      "epoch": 6.946745562130177,
+      "grad_norm": 4.1809106849701685,
+      "learning_rate": 2.403743627853039e-08,
+      "logits/chosen": -1.1825206279754639,
+      "logits/rejected": -1.193249225616455,
+      "logps/chosen": -36.14801788330078,
+      "logps/rejected": -66.2503662109375,
+      "loss": 0.0191,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1976075172424316,
+      "rewards/margins": 9.384969711303711,
+      "rewards/rejected": -10.582576751708984,
+      "step": 587
+    },
+    {
+      "epoch": 6.958579881656805,
+      "grad_norm": 4.981046398430429,
+      "learning_rate": 2.3484151341411018e-08,
+      "logits/chosen": -1.0915954113006592,
+      "logits/rejected": -1.0409646034240723,
+      "logps/chosen": -41.00914001464844,
+      "logps/rejected": -60.438446044921875,
+      "loss": 0.0215,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.23959925770759583,
+      "rewards/margins": 7.114398956298828,
+      "rewards/rejected": -7.353998184204102,
+      "step": 588
+    },
+    {
+      "epoch": 6.970414201183432,
+      "grad_norm": 3.7291093865283065,
+      "learning_rate": 2.2936994470979188e-08,
+      "logits/chosen": -1.3922624588012695,
+      "logits/rejected": -1.4559788703918457,
+      "logps/chosen": -56.968406677246094,
+      "logps/rejected": -65.14087677001953,
+      "loss": 0.0173,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4613820314407349,
+      "rewards/margins": 7.349767684936523,
+      "rewards/rejected": -8.811149597167969,
+      "step": 589
+    },
+    {
+      "epoch": 6.982248520710059,
+      "grad_norm": 4.305952133916051,
+      "learning_rate": 2.23959804697921e-08,
+      "logits/chosen": -1.133141040802002,
+      "logits/rejected": -1.262980341911316,
+      "logps/chosen": -45.40880584716797,
+      "logps/rejected": -58.749446868896484,
+      "loss": 0.0174,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.105184316635132,
+      "rewards/margins": 7.048664093017578,
+      "rewards/rejected": -9.153848648071289,
+      "step": 590
+    },
+    {
+      "epoch": 6.994082840236686,
+      "grad_norm": 4.189520921835817,
+      "learning_rate": 2.1861123974220158e-08,
+      "logits/chosen": -1.2279367446899414,
+      "logits/rejected": -1.281379222869873,
+      "logps/chosen": -45.95500183105469,
+      "logps/rejected": -78.72282409667969,
+      "loss": 0.0147,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.1575565338134766,
+      "rewards/margins": 9.946080207824707,
+      "rewards/rejected": -12.103636741638184,
+      "step": 591
+    },
+    {
+      "epoch": 7.005917159763314,
+      "grad_norm": 3.8418221234553425,
+      "learning_rate": 2.1332439454051277e-08,
+      "logits/chosen": -1.03047513961792,
+      "logits/rejected": -0.9760541915893555,
+      "logps/chosen": -35.22906494140625,
+      "logps/rejected": -65.05894470214844,
+      "loss": 0.0177,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0646643340587616,
+      "rewards/margins": 7.141489028930664,
+      "rewards/rejected": -7.206153869628906,
+      "step": 592
+    },
+    {
+      "epoch": 7.017751479289941,
+      "grad_norm": 4.37902481619465,
+      "learning_rate": 2.080994121209914e-08,
+      "logits/chosen": -1.2298972606658936,
+      "logits/rejected": -1.2462295293807983,
+      "logps/chosen": -52.52454376220703,
+      "logps/rejected": -73.28079223632812,
+      "loss": 0.0196,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.6756205558776855,
+      "rewards/margins": 7.350424766540527,
+      "rewards/rejected": -10.026044845581055,
+      "step": 593
+    },
+    {
+      "epoch": 7.029585798816568,
+      "grad_norm": 3.9841674822113564,
+      "learning_rate": 2.029364338381656e-08,
+      "logits/chosen": -1.2008388042449951,
+      "logits/rejected": -1.2781215906143188,
+      "logps/chosen": -46.399505615234375,
+      "logps/rejected": -59.717586517333984,
+      "loss": 0.0172,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.2308552265167236,
+      "rewards/margins": 6.461019515991211,
+      "rewards/rejected": -8.691874504089355,
+      "step": 594
+    },
+    {
+      "epoch": 7.041420118343195,
+      "grad_norm": 3.5149491748374957,
+      "learning_rate": 1.9783559936912773e-08,
+      "logits/chosen": -1.4331488609313965,
+      "logits/rejected": -1.3519785404205322,
+      "logps/chosen": -44.29254150390625,
+      "logps/rejected": -78.45376586914062,
+      "loss": 0.0137,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.327043056488037,
+      "rewards/margins": 8.162921905517578,
+      "rewards/rejected": -9.489965438842773,
+      "step": 595
+    },
+    {
+      "epoch": 7.053254437869822,
+      "grad_norm": 3.509610689232701,
+      "learning_rate": 1.9279704670975726e-08,
+      "logits/chosen": -1.0991450548171997,
+      "logits/rejected": -1.106555700302124,
+      "logps/chosen": -48.36073303222656,
+      "logps/rejected": -64.15289306640625,
+      "loss": 0.0135,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.800105631351471,
+      "rewards/margins": 6.824061393737793,
+      "rewards/rejected": -7.624166965484619,
+      "step": 596
+    },
+    {
+      "epoch": 7.06508875739645,
+      "grad_norm": 4.256443303454753,
+      "learning_rate": 1.8782091217098728e-08,
+      "logits/chosen": -1.4569830894470215,
+      "logits/rejected": -1.4492859840393066,
+      "logps/chosen": -38.11000061035156,
+      "logps/rejected": -70.66535949707031,
+      "loss": 0.0201,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.615638017654419,
+      "rewards/margins": 8.343268394470215,
+      "rewards/rejected": -10.958906173706055,
+      "step": 597
+    },
+    {
+      "epoch": 7.076923076923077,
+      "grad_norm": 3.1992906689262,
+      "learning_rate": 1.829073303751172e-08,
+      "logits/chosen": -1.296358585357666,
+      "logits/rejected": -1.2894511222839355,
+      "logps/chosen": -49.65617370605469,
+      "logps/rejected": -63.472599029541016,
+      "loss": 0.0142,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -3.235658884048462,
+      "rewards/margins": 6.7473063468933105,
+      "rewards/rejected": -9.982964515686035,
+      "step": 598
+    },
+    {
+      "epoch": 7.088757396449704,
+      "grad_norm": 3.941759552775281,
+      "learning_rate": 1.780564342521698e-08,
+      "logits/chosen": -1.2821180820465088,
+      "logits/rejected": -1.2152577638626099,
+      "logps/chosen": -43.84495544433594,
+      "logps/rejected": -70.52769470214844,
+      "loss": 0.0169,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.0584795475006104,
+      "rewards/margins": 7.640247344970703,
+      "rewards/rejected": -8.69872760772705,
+      "step": 599
+    },
+    {
+      "epoch": 7.100591715976331,
+      "grad_norm": 3.553599235582575,
+      "learning_rate": 1.732683550362954e-08,
+      "logits/chosen": -1.1523908376693726,
+      "logits/rejected": -1.2564313411712646,
+      "logps/chosen": -35.25387954711914,
+      "logps/rejected": -55.94652557373047,
+      "loss": 0.0169,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.38266000151634216,
+      "rewards/margins": 6.977762222290039,
+      "rewards/rejected": -7.360422134399414,
+      "step": 600
+    },
+    {
+      "epoch": 7.112426035502959,
+      "grad_norm": 3.8288450725810717,
+      "learning_rate": 1.6854322226222102e-08,
+      "logits/chosen": -1.2984129190444946,
+      "logits/rejected": -1.2461819648742676,
+      "logps/chosen": -46.678077697753906,
+      "logps/rejected": -75.3173828125,
+      "loss": 0.0145,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.282292604446411,
+      "rewards/margins": 8.546947479248047,
+      "rewards/rejected": -10.829240798950195,
+      "step": 601
+    },
+    {
+      "epoch": 7.124260355029586,
+      "grad_norm": 4.219862245925515,
+      "learning_rate": 1.6388116376174765e-08,
+      "logits/chosen": -1.2928462028503418,
+      "logits/rejected": -1.1129931211471558,
+      "logps/chosen": -40.22349548339844,
+      "logps/rejected": -70.40786743164062,
+      "loss": 0.0185,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.8234593868255615,
+      "rewards/margins": 8.468381881713867,
+      "rewards/rejected": -10.291842460632324,
+      "step": 602
+    },
+    {
+      "epoch": 7.136094674556213,
+      "grad_norm": 3.8475070615621014,
+      "learning_rate": 1.5928230566028932e-08,
+      "logits/chosen": -1.4136502742767334,
+      "logits/rejected": -1.2762510776519775,
+      "logps/chosen": -35.088523864746094,
+      "logps/rejected": -61.09199523925781,
+      "loss": 0.0145,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4712510108947754,
+      "rewards/margins": 7.1925201416015625,
+      "rewards/rejected": -8.663771629333496,
+      "step": 603
+    },
+    {
+      "epoch": 7.14792899408284,
+      "grad_norm": 3.206852988789503,
+      "learning_rate": 1.5474677237346468e-08,
+      "logits/chosen": -1.3350969552993774,
+      "logits/rejected": -1.2451387643814087,
+      "logps/chosen": -28.593502044677734,
+      "logps/rejected": -62.2099494934082,
+      "loss": 0.0137,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.741662859916687,
+      "rewards/margins": 8.64741325378418,
+      "rewards/rejected": -9.389076232910156,
+      "step": 604
+    },
+    {
+      "epoch": 7.159763313609467,
+      "grad_norm": 3.844656802369703,
+      "learning_rate": 1.5027468660372604e-08,
+      "logits/chosen": -1.3713629245758057,
+      "logits/rejected": -1.378098964691162,
+      "logps/chosen": -47.545536041259766,
+      "logps/rejected": -69.47958374023438,
+      "loss": 0.0172,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.9896124601364136,
+      "rewards/margins": 8.07425308227539,
+      "rewards/rejected": -10.063865661621094,
+      "step": 605
+    },
+    {
+      "epoch": 7.171597633136095,
+      "grad_norm": 3.0241225907385165,
+      "learning_rate": 1.4586616933704527e-08,
+      "logits/chosen": -1.2971570491790771,
+      "logits/rejected": -1.245847225189209,
+      "logps/chosen": -38.302677154541016,
+      "logps/rejected": -68.13219451904297,
+      "loss": 0.0139,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4747118949890137,
+      "rewards/margins": 8.88740062713623,
+      "rewards/rejected": -10.362112998962402,
+      "step": 606
+    },
+    {
+      "epoch": 7.183431952662722,
+      "grad_norm": 5.654903481243723,
+      "learning_rate": 1.4152133983963643e-08,
+      "logits/chosen": -1.3624151945114136,
+      "logits/rejected": -1.289166808128357,
+      "logps/chosen": -36.01699447631836,
+      "logps/rejected": -65.6912841796875,
+      "loss": 0.0258,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.6957451105117798,
+      "rewards/margins": 7.106748580932617,
+      "rewards/rejected": -8.80249309539795,
+      "step": 607
+    },
+    {
+      "epoch": 7.195266272189349,
+      "grad_norm": 2.9794480072644394,
+      "learning_rate": 1.372403156547311e-08,
+      "logits/chosen": -1.2158567905426025,
+      "logits/rejected": -1.2747105360031128,
+      "logps/chosen": -34.01419448852539,
+      "logps/rejected": -57.90458297729492,
+      "loss": 0.0134,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9422094821929932,
+      "rewards/margins": 7.069450378417969,
+      "rewards/rejected": -8.011659622192383,
+      "step": 608
+    },
+    {
+      "epoch": 7.207100591715976,
+      "grad_norm": 4.261973188621215,
+      "learning_rate": 1.330232125993988e-08,
+      "logits/chosen": -1.3627076148986816,
+      "logits/rejected": -1.3639800548553467,
+      "logps/chosen": -48.03813552856445,
+      "logps/rejected": -64.04691314697266,
+      "loss": 0.0188,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.8824987411499023,
+      "rewards/margins": 7.479591369628906,
+      "rewards/rejected": -10.362090110778809,
+      "step": 609
+    },
+    {
+      "epoch": 7.218934911242604,
+      "grad_norm": 4.803739657888739,
+      "learning_rate": 1.2887014476141212e-08,
+      "logits/chosen": -1.4838051795959473,
+      "logits/rejected": -1.3311843872070312,
+      "logps/chosen": -43.892982482910156,
+      "logps/rejected": -81.17485046386719,
+      "loss": 0.0189,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.9665614366531372,
+      "rewards/margins": 9.852771759033203,
+      "rewards/rejected": -11.81933307647705,
+      "step": 610
+    },
+    {
+      "epoch": 7.230769230769231,
+      "grad_norm": 4.0756670658237475,
+      "learning_rate": 1.2478122449616212e-08,
+      "logits/chosen": -1.1747201681137085,
+      "logits/rejected": -1.2890151739120483,
+      "logps/chosen": -56.25837707519531,
+      "logps/rejected": -65.45867156982422,
+      "loss": 0.0164,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.2632644176483154,
+      "rewards/margins": 7.053145885467529,
+      "rewards/rejected": -9.316410064697266,
+      "step": 611
+    },
+    {
+      "epoch": 7.242603550295858,
+      "grad_norm": 4.38053938987994,
+      "learning_rate": 1.2075656242361732e-08,
+      "logits/chosen": -1.0797568559646606,
+      "logits/rejected": -1.0762195587158203,
+      "logps/chosen": -33.57958221435547,
+      "logps/rejected": -71.72637176513672,
+      "loss": 0.0159,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.2206077575683594,
+      "rewards/margins": 10.526464462280273,
+      "rewards/rejected": -12.74707317352295,
+      "step": 612
+    },
+    {
+      "epoch": 7.254437869822485,
+      "grad_norm": 2.925185904249556,
+      "learning_rate": 1.16796267425332e-08,
+      "logits/chosen": -1.4559069871902466,
+      "logits/rejected": -1.3925589323043823,
+      "logps/chosen": -40.79030990600586,
+      "logps/rejected": -67.69854736328125,
+      "loss": 0.0118,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.6704390048980713,
+      "rewards/margins": 7.342378616333008,
+      "rewards/rejected": -10.0128173828125,
+      "step": 613
+    },
+    {
+      "epoch": 7.266272189349112,
+      "grad_norm": 4.199453917170064,
+      "learning_rate": 1.1290044664149873e-08,
+      "logits/chosen": -1.323655128479004,
+      "logits/rejected": -1.2902326583862305,
+      "logps/chosen": -43.997676849365234,
+      "logps/rejected": -70.29308319091797,
+      "loss": 0.0186,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.0053741931915283,
+      "rewards/margins": 7.789745330810547,
+      "rewards/rejected": -9.795119285583496,
+      "step": 614
+    },
+    {
+      "epoch": 7.27810650887574,
+      "grad_norm": 2.9644130977906675,
+      "learning_rate": 1.0906920546805253e-08,
+      "logits/chosen": -1.331533670425415,
+      "logits/rejected": -1.318424940109253,
+      "logps/chosen": -41.30125045776367,
+      "logps/rejected": -69.17133331298828,
+      "loss": 0.0117,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.8695695400238037,
+      "rewards/margins": 8.263007164001465,
+      "rewards/rejected": -10.132576942443848,
+      "step": 615
+    },
+    {
+      "epoch": 7.289940828402367,
+      "grad_norm": 2.9111429661875294,
+      "learning_rate": 1.0530264755381824e-08,
+      "logits/chosen": -1.5518194437026978,
+      "logits/rejected": -1.2643704414367676,
+      "logps/chosen": -25.555744171142578,
+      "logps/rejected": -70.7457275390625,
+      "loss": 0.0111,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.22286051511764526,
+      "rewards/margins": 10.125096321105957,
+      "rewards/rejected": -10.347957611083984,
+      "step": 616
+    },
+    {
+      "epoch": 7.3017751479289945,
+      "grad_norm": 3.222903401246695,
+      "learning_rate": 1.0160087479770513e-08,
+      "logits/chosen": -1.147964358329773,
+      "logits/rejected": -1.1484017372131348,
+      "logps/chosen": -36.42656326293945,
+      "logps/rejected": -54.97199249267578,
+      "loss": 0.0144,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2413222789764404,
+      "rewards/margins": 6.4268999099731445,
+      "rewards/rejected": -7.668221950531006,
+      "step": 617
+    },
+    {
+      "epoch": 7.313609467455621,
+      "grad_norm": 4.567760348222854,
+      "learning_rate": 9.796398734595284e-09,
+      "logits/chosen": -1.261488437652588,
+      "logits/rejected": -1.1865301132202148,
+      "logps/chosen": -33.513763427734375,
+      "logps/rejected": -80.79141235351562,
+      "loss": 0.0239,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4354597330093384,
+      "rewards/margins": 9.321052551269531,
+      "rewards/rejected": -10.756511688232422,
+      "step": 618
+    },
+    {
+      "epoch": 7.325443786982248,
+      "grad_norm": 3.241502517680726,
+      "learning_rate": 9.439208358941907e-09,
+      "logits/chosen": -1.1972136497497559,
+      "logits/rejected": -1.074997901916504,
+      "logps/chosen": -47.66037368774414,
+      "logps/rejected": -74.0272445678711,
+      "loss": 0.0139,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.2272040843963623,
+      "rewards/margins": 8.553009033203125,
+      "rewards/rejected": -10.78021240234375,
+      "step": 619
+    },
+    {
+      "epoch": 7.337278106508876,
+      "grad_norm": 5.389252767817157,
+      "learning_rate": 9.088526016092141e-09,
+      "logits/chosen": -1.5381629467010498,
+      "logits/rejected": -1.3760014772415161,
+      "logps/chosen": -40.557369232177734,
+      "logps/rejected": -73.04681396484375,
+      "loss": 0.0224,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -3.1969547271728516,
+      "rewards/margins": 8.717408180236816,
+      "rewards/rejected": -11.914361953735352,
+      "step": 620
+    },
+    {
+      "epoch": 7.349112426035503,
+      "grad_norm": 4.492355419112391,
+      "learning_rate": 8.744361193261912e-09,
+      "logits/chosen": -1.2798972129821777,
+      "logits/rejected": -1.2866759300231934,
+      "logps/chosen": -46.820777893066406,
+      "logps/rejected": -71.25297546386719,
+      "loss": 0.0208,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.2585588693618774,
+      "rewards/margins": 8.543848037719727,
+      "rewards/rejected": -9.802406311035156,
+      "step": 621
+    },
+    {
+      "epoch": 7.3609467455621305,
+      "grad_norm": 2.590234218369342,
+      "learning_rate": 8.40672320134489e-09,
+      "logits/chosen": -1.3868498802185059,
+      "logits/rejected": -1.3360377550125122,
+      "logps/chosen": -32.945518493652344,
+      "logps/rejected": -53.556705474853516,
+      "loss": 0.011,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.3330631256103516,
+      "rewards/margins": 6.077226638793945,
+      "rewards/rejected": -8.410289764404297,
+      "step": 622
+    },
+    {
+      "epoch": 7.372781065088757,
+      "grad_norm": 4.999566857886415,
+      "learning_rate": 8.075621174660625e-09,
+      "logits/chosen": -1.1354734897613525,
+      "logits/rejected": -1.1073994636535645,
+      "logps/chosen": -46.984012603759766,
+      "logps/rejected": -67.13949584960938,
+      "loss": 0.0205,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.1461362838745117,
+      "rewards/margins": 7.404544353485107,
+      "rewards/rejected": -9.550680160522461,
+      "step": 623
+    },
+    {
+      "epoch": 7.384615384615385,
+      "grad_norm": 4.442848421246316,
+      "learning_rate": 7.751064070707247e-09,
+      "logits/chosen": -0.9358320236206055,
+      "logits/rejected": -1.0718661546707153,
+      "logps/chosen": -57.682579040527344,
+      "logps/rejected": -67.44542694091797,
+      "loss": 0.0198,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.3646697998046875,
+      "rewards/margins": 7.940317153930664,
+      "rewards/rejected": -9.304986953735352,
+      "step": 624
+    },
+    {
+      "epoch": 7.396449704142012,
+      "grad_norm": 3.6393760430903463,
+      "learning_rate": 7.4330606699193055e-09,
+      "logits/chosen": -1.3721582889556885,
+      "logits/rejected": -1.194779634475708,
+      "logps/chosen": -41.03765869140625,
+      "logps/rejected": -67.66683959960938,
+      "loss": 0.0137,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.100832939147949,
+      "rewards/margins": 7.313269138336182,
+      "rewards/rejected": -9.414101600646973,
+      "step": 625
+    },
+    {
+      "epoch": 7.408284023668639,
+      "grad_norm": 2.8173995600402995,
+      "learning_rate": 7.12161957543006e-09,
+      "logits/chosen": -1.1632230281829834,
+      "logits/rejected": -1.1450843811035156,
+      "logps/chosen": -45.309226989746094,
+      "logps/rejected": -67.70977783203125,
+      "loss": 0.0096,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.5633859634399414,
+      "rewards/margins": 7.859521865844727,
+      "rewards/rejected": -9.422907829284668,
+      "step": 626
+    },
+    {
+      "epoch": 7.420118343195266,
+      "grad_norm": 3.6516772499433054,
+      "learning_rate": 6.816749212839007e-09,
+      "logits/chosen": -1.2808470726013184,
+      "logits/rejected": -1.3078573942184448,
+      "logps/chosen": -44.84081268310547,
+      "logps/rejected": -70.39693450927734,
+      "loss": 0.017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.452430009841919,
+      "rewards/margins": 7.464086532592773,
+      "rewards/rejected": -9.91651725769043,
+      "step": 627
+    },
+    {
+      "epoch": 7.431952662721893,
+      "grad_norm": 3.889072407283795,
+      "learning_rate": 6.518457829983559e-09,
+      "logits/chosen": -1.185568928718567,
+      "logits/rejected": -1.052988052368164,
+      "logps/chosen": -41.724830627441406,
+      "logps/rejected": -69.43457794189453,
+      "loss": 0.0125,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4893982410430908,
+      "rewards/margins": 6.81561803817749,
+      "rewards/rejected": -8.30501651763916,
+      "step": 628
+    },
+    {
+      "epoch": 7.443786982248521,
+      "grad_norm": 2.887200349157875,
+      "learning_rate": 6.226753496716253e-09,
+      "logits/chosen": -1.2950429916381836,
+      "logits/rejected": -1.252742052078247,
+      "logps/chosen": -30.59502410888672,
+      "logps/rejected": -50.878868103027344,
+      "loss": 0.0114,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0032743215560913086,
+      "rewards/margins": 6.188243865966797,
+      "rewards/rejected": -6.1915178298950195,
+      "step": 629
+    },
+    {
+      "epoch": 7.455621301775148,
+      "grad_norm": 4.244222893379881,
+      "learning_rate": 5.9416441046862555e-09,
+      "logits/chosen": -1.301514983177185,
+      "logits/rejected": -1.3513267040252686,
+      "logps/chosen": -39.38859558105469,
+      "logps/rejected": -67.55226135253906,
+      "loss": 0.017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.7557300329208374,
+      "rewards/margins": 9.34812068939209,
+      "rewards/rejected": -11.103851318359375,
+      "step": 630
+    },
+    {
+      "epoch": 7.4674556213017755,
+      "grad_norm": 4.290461165134473,
+      "learning_rate": 5.663137367125898e-09,
+      "logits/chosen": -1.3903230428695679,
+      "logits/rejected": -1.3901610374450684,
+      "logps/chosen": -46.33057403564453,
+      "logps/rejected": -58.841392517089844,
+      "loss": 0.0177,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.6279226541519165,
+      "rewards/margins": 7.242958068847656,
+      "rewards/rejected": -8.870880126953125,
+      "step": 631
+    },
+    {
+      "epoch": 7.479289940828402,
+      "grad_norm": 3.439301636458819,
+      "learning_rate": 5.3912408186420064e-09,
+      "logits/chosen": -1.1362035274505615,
+      "logits/rejected": -1.0982844829559326,
+      "logps/chosen": -37.11709213256836,
+      "logps/rejected": -61.59524154663086,
+      "loss": 0.0152,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.149167537689209,
+      "rewards/margins": 7.633286952972412,
+      "rewards/rejected": -8.782454490661621,
+      "step": 632
+    },
+    {
+      "epoch": 7.491124260355029,
+      "grad_norm": 4.470095918148292,
+      "learning_rate": 5.12596181501207e-09,
+      "logits/chosen": -1.2651134729385376,
+      "logits/rejected": -1.3249282836914062,
+      "logps/chosen": -30.522804260253906,
+      "logps/rejected": -51.84233856201172,
+      "loss": 0.02,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.16683726012706757,
+      "rewards/margins": 7.101442813873291,
+      "rewards/rejected": -7.268280029296875,
+      "step": 633
+    },
+    {
+      "epoch": 7.502958579881657,
+      "grad_norm": 3.3611637574552025,
+      "learning_rate": 4.867307532985227e-09,
+      "logits/chosen": -1.364112377166748,
+      "logits/rejected": -1.2576634883880615,
+      "logps/chosen": -49.3748664855957,
+      "logps/rejected": -83.98435974121094,
+      "loss": 0.0129,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.741516590118408,
+      "rewards/margins": 9.956401824951172,
+      "rewards/rejected": -12.697917938232422,
+      "step": 634
+    },
+    {
+      "epoch": 7.514792899408284,
+      "grad_norm": 2.9574639791748036,
+      "learning_rate": 4.615284970088173e-09,
+      "logits/chosen": -1.1360864639282227,
+      "logits/rejected": -1.1513174772262573,
+      "logps/chosen": -36.475341796875,
+      "logps/rejected": -70.95317077636719,
+      "loss": 0.0151,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.0532495975494385,
+      "rewards/margins": 8.06146240234375,
+      "rewards/rejected": -10.114712715148926,
+      "step": 635
+    },
+    {
+      "epoch": 7.5266272189349115,
+      "grad_norm": 3.4730054907543497,
+      "learning_rate": 4.369900944435734e-09,
+      "logits/chosen": -1.369525671005249,
+      "logits/rejected": -1.2865912914276123,
+      "logps/chosen": -40.689971923828125,
+      "logps/rejected": -69.06320190429688,
+      "loss": 0.012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.378922700881958,
+      "rewards/margins": 8.550222396850586,
+      "rewards/rejected": -10.929145812988281,
+      "step": 636
+    },
+    {
+      "epoch": 7.538461538461538,
+      "grad_norm": 4.223323554636038,
+      "learning_rate": 4.131162094546531e-09,
+      "logits/chosen": -1.3048131465911865,
+      "logits/rejected": -1.3768178224563599,
+      "logps/chosen": -57.49602508544922,
+      "logps/rejected": -68.12883758544922,
+      "loss": 0.0152,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -4.1675872802734375,
+      "rewards/margins": 7.353907585144043,
+      "rewards/rejected": -11.521493911743164,
+      "step": 637
+    },
+    {
+      "epoch": 7.550295857988166,
+      "grad_norm": 3.593337678884347,
+      "learning_rate": 3.899074879163244e-09,
+      "logits/chosen": -0.9969456195831299,
+      "logits/rejected": -1.053671956062317,
+      "logps/chosen": -44.34745788574219,
+      "logps/rejected": -60.61125183105469,
+      "loss": 0.0148,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.0102906227111816,
+      "rewards/margins": 7.186006546020508,
+      "rewards/rejected": -8.196297645568848,
+      "step": 638
+    },
+    {
+      "epoch": 7.562130177514793,
+      "grad_norm": 4.313259960588259,
+      "learning_rate": 3.6736455770781104e-09,
+      "logits/chosen": -1.0693228244781494,
+      "logits/rejected": -1.02896249294281,
+      "logps/chosen": -38.95392608642578,
+      "logps/rejected": -63.7398796081543,
+      "loss": 0.0165,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.6491146087646484,
+      "rewards/margins": 5.766778945922852,
+      "rewards/rejected": -7.4158935546875,
+      "step": 639
+    },
+    {
+      "epoch": 7.57396449704142,
+      "grad_norm": 3.606040906516179,
+      "learning_rate": 3.4548802869627804e-09,
+      "logits/chosen": -1.2556226253509521,
+      "logits/rejected": -1.1845530271530151,
+      "logps/chosen": -42.52489471435547,
+      "logps/rejected": -69.6248550415039,
+      "loss": 0.0147,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -3.0042614936828613,
+      "rewards/margins": 8.077693939208984,
+      "rewards/rejected": -11.081954956054688,
+      "step": 640
+    },
+    {
+      "epoch": 7.585798816568047,
+      "grad_norm": 3.105559539498645,
+      "learning_rate": 3.2427849272035067e-09,
+      "logits/chosen": -0.9435504674911499,
+      "logits/rejected": -0.9759422540664673,
+      "logps/chosen": -40.1533203125,
+      "logps/rejected": -66.1624526977539,
+      "loss": 0.0137,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.300183653831482,
+      "rewards/margins": 7.85737943649292,
+      "rewards/rejected": -9.157563209533691,
+      "step": 641
+    },
+    {
+      "epoch": 7.597633136094674,
+      "grad_norm": 3.558021048910989,
+      "learning_rate": 3.037365235741024e-09,
+      "logits/chosen": -1.1166036128997803,
+      "logits/rejected": -1.038915753364563,
+      "logps/chosen": -46.17367935180664,
+      "logps/rejected": -77.80493927001953,
+      "loss": 0.0141,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.5460665225982666,
+      "rewards/margins": 9.366143226623535,
+      "rewards/rejected": -11.912210464477539,
+      "step": 642
+    },
+    {
+      "epoch": 7.609467455621302,
+      "grad_norm": 3.2180872359588033,
+      "learning_rate": 2.8386267699152256e-09,
+      "logits/chosen": -1.32170832157135,
+      "logits/rejected": -1.1445355415344238,
+      "logps/chosen": -33.306297302246094,
+      "logps/rejected": -64.81905364990234,
+      "loss": 0.0115,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.5187739133834839,
+      "rewards/margins": 7.3708953857421875,
+      "rewards/rejected": -8.889669418334961,
+      "step": 643
+    },
+    {
+      "epoch": 7.621301775147929,
+      "grad_norm": 4.516310865033977,
+      "learning_rate": 2.6465749063149245e-09,
+      "logits/chosen": -1.3887925148010254,
+      "logits/rejected": -1.5802987813949585,
+      "logps/chosen": -41.122772216796875,
+      "logps/rejected": -60.960174560546875,
+      "loss": 0.0193,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.43830376863479614,
+      "rewards/margins": 8.353026390075684,
+      "rewards/rejected": -8.79133129119873,
+      "step": 644
+    },
+    {
+      "epoch": 7.633136094674557,
+      "grad_norm": 3.522662435537469,
+      "learning_rate": 2.461214840632331e-09,
+      "logits/chosen": -1.2272520065307617,
+      "logits/rejected": -1.2148579359054565,
+      "logps/chosen": -40.47775650024414,
+      "logps/rejected": -66.96976470947266,
+      "loss": 0.0126,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9865707159042358,
+      "rewards/margins": 8.534340858459473,
+      "rewards/rejected": -9.520912170410156,
+      "step": 645
+    },
+    {
+      "epoch": 7.644970414201183,
+      "grad_norm": 5.08495729529155,
+      "learning_rate": 2.282551587522441e-09,
+      "logits/chosen": -1.2286673784255981,
+      "logits/rejected": -1.187401294708252,
+      "logps/chosen": -38.408416748046875,
+      "logps/rejected": -66.8298110961914,
+      "loss": 0.017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.147416591644287,
+      "rewards/margins": 8.060477256774902,
+      "rewards/rejected": -10.207894325256348,
+      "step": 646
+    },
+    {
+      "epoch": 7.65680473372781,
+      "grad_norm": 3.2465540149343206,
+      "learning_rate": 2.1105899804675363e-09,
+      "logits/chosen": -1.167961597442627,
+      "logits/rejected": -1.1866670846939087,
+      "logps/chosen": -45.849952697753906,
+      "logps/rejected": -69.32218170166016,
+      "loss": 0.0147,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.9047467708587646,
+      "rewards/margins": 8.320653915405273,
+      "rewards/rejected": -10.225400924682617,
+      "step": 647
+    },
+    {
+      "epoch": 7.668639053254438,
+      "grad_norm": 3.0376394347131352,
+      "learning_rate": 1.9453346716462316e-09,
+      "logits/chosen": -1.45213782787323,
+      "logits/rejected": -1.4141486883163452,
+      "logps/chosen": -36.55447006225586,
+      "logps/rejected": -63.271644592285156,
+      "loss": 0.012,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9351499080657959,
+      "rewards/margins": 8.255223274230957,
+      "rewards/rejected": -9.190373420715332,
+      "step": 648
+    },
+    {
+      "epoch": 7.680473372781065,
+      "grad_norm": 4.323080858658896,
+      "learning_rate": 1.7867901318077695e-09,
+      "logits/chosen": -1.2777527570724487,
+      "logits/rejected": -1.1896119117736816,
+      "logps/chosen": -50.66539764404297,
+      "logps/rejected": -77.70809936523438,
+      "loss": 0.0179,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -3.193788766860962,
+      "rewards/margins": 8.507574081420898,
+      "rewards/rejected": -11.701362609863281,
+      "step": 649
+    },
+    {
+      "epoch": 7.6923076923076925,
+      "grad_norm": 5.437515225024147,
+      "learning_rate": 1.6349606501509794e-09,
+      "logits/chosen": -1.4479446411132812,
+      "logits/rejected": -1.4918639659881592,
+      "logps/chosen": -37.68550109863281,
+      "logps/rejected": -60.039100646972656,
+      "loss": 0.0225,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.733797550201416,
+      "rewards/margins": 7.636280059814453,
+      "rewards/rejected": -9.370077133178711,
+      "step": 650
+    },
+    {
+      "epoch": 7.704142011834319,
+      "grad_norm": 4.271376085946802,
+      "learning_rate": 1.489850334208259e-09,
+      "logits/chosen": -1.4708189964294434,
+      "logits/rejected": -1.3335071802139282,
+      "logps/chosen": -36.9536247253418,
+      "logps/rejected": -68.65011596679688,
+      "loss": 0.0198,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.618829369544983,
+      "rewards/margins": 7.648519515991211,
+      "rewards/rejected": -9.267349243164062,
+      "step": 651
+    },
+    {
+      "epoch": 7.715976331360947,
+      "grad_norm": 4.326704771148402,
+      "learning_rate": 1.351463109734441e-09,
+      "logits/chosen": -1.3178719282150269,
+      "logits/rejected": -1.2290101051330566,
+      "logps/chosen": -39.71061325073242,
+      "logps/rejected": -64.45901489257812,
+      "loss": 0.0151,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.014697790145874,
+      "rewards/margins": 7.242136478424072,
+      "rewards/rejected": -8.256834030151367,
+      "step": 652
+    },
+    {
+      "epoch": 7.727810650887574,
+      "grad_norm": 3.7356892625464018,
+      "learning_rate": 1.2198027206006822e-09,
+      "logits/chosen": -1.1409059762954712,
+      "logits/rejected": -1.035496711730957,
+      "logps/chosen": -44.18361282348633,
+      "logps/rejected": -72.97080993652344,
+      "loss": 0.0142,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.7735768556594849,
+      "rewards/margins": 8.308616638183594,
+      "rewards/rejected": -10.082193374633789,
+      "step": 653
+    },
+    {
+      "epoch": 7.739644970414201,
+      "grad_norm": 3.888887440172467,
+      "learning_rate": 1.0948727286930192e-09,
+      "logits/chosen": -1.0706766843795776,
+      "logits/rejected": -1.0967326164245605,
+      "logps/chosen": -52.38780212402344,
+      "logps/rejected": -74.34490966796875,
+      "loss": 0.0158,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.6245856285095215,
+      "rewards/margins": 8.349431037902832,
+      "rewards/rejected": -10.974016189575195,
+      "step": 654
+    },
+    {
+      "epoch": 7.7514792899408285,
+      "grad_norm": 4.910196712545756,
+      "learning_rate": 9.766765138160827e-10,
+      "logits/chosen": -1.3771216869354248,
+      "logits/rejected": -1.3673535585403442,
+      "logps/chosen": -26.542402267456055,
+      "logps/rejected": -53.72931671142578,
+      "loss": 0.0224,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.7858462929725647,
+      "rewards/margins": 8.258723258972168,
+      "rewards/rejected": -9.044569969177246,
+      "step": 655
+    },
+    {
+      "epoch": 7.763313609467455,
+      "grad_norm": 3.6303736143642094,
+      "learning_rate": 8.652172736017816e-10,
+      "logits/chosen": -1.2556291818618774,
+      "logits/rejected": -1.3194972276687622,
+      "logps/chosen": -45.683570861816406,
+      "logps/rejected": -59.34172821044922,
+      "loss": 0.0157,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.3988208770751953,
+      "rewards/margins": 7.259479522705078,
+      "rewards/rejected": -8.658300399780273,
+      "step": 656
+    },
+    {
+      "epoch": 7.775147928994083,
+      "grad_norm": 4.139059122264169,
+      "learning_rate": 7.604980234225122e-10,
+      "logits/chosen": -1.1913983821868896,
+      "logits/rejected": -1.2692341804504395,
+      "logps/chosen": -34.41987228393555,
+      "logps/rejected": -62.04487609863281,
+      "loss": 0.0172,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.5674678087234497,
+      "rewards/margins": 9.807378768920898,
+      "rewards/rejected": -10.374847412109375,
+      "step": 657
+    },
+    {
+      "epoch": 7.78698224852071,
+      "grad_norm": 3.618597489407487,
+      "learning_rate": 6.625215963098896e-10,
+      "logits/chosen": -1.019304871559143,
+      "logits/rejected": -1.0009970664978027,
+      "logps/chosen": -41.301448822021484,
+      "logps/rejected": -56.97692108154297,
+      "loss": 0.0147,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1815185546875,
+      "rewards/margins": 5.260707855224609,
+      "rewards/rejected": -6.442225933074951,
+      "step": 658
+    },
+    {
+      "epoch": 7.798816568047338,
+      "grad_norm": 3.336765309826322,
+      "learning_rate": 5.712906428778919e-10,
+      "logits/chosen": -1.4606183767318726,
+      "logits/rejected": -1.3886492252349854,
+      "logps/chosen": -27.533708572387695,
+      "logps/rejected": -57.605125427246094,
+      "loss": 0.0118,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.1746628284454346,
+      "rewards/margins": 7.080971717834473,
+      "rewards/rejected": -8.255634307861328,
+      "step": 659
+    },
+    {
+      "epoch": 7.810650887573964,
+      "grad_norm": 3.5093648749750477,
+      "learning_rate": 4.868076312512515e-10,
+      "logits/chosen": -1.306321144104004,
+      "logits/rejected": -1.302777886390686,
+      "logps/chosen": -35.620086669921875,
+      "logps/rejected": -84.12001037597656,
+      "loss": 0.0144,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.335205078125,
+      "rewards/margins": 9.759173393249512,
+      "rewards/rejected": -11.094379425048828,
+      "step": 660
+    },
+    {
+      "epoch": 7.822485207100591,
+      "grad_norm": 4.227789623534498,
+      "learning_rate": 4.090748469986471e-10,
+      "logits/chosen": -1.2036335468292236,
+      "logits/rejected": -1.2865405082702637,
+      "logps/chosen": -35.319541931152344,
+      "logps/rejected": -55.186180114746094,
+      "loss": 0.018,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.08307170867919922,
+      "rewards/margins": 6.8585710525512695,
+      "rewards/rejected": -6.77549934387207,
+      "step": 661
+    },
+    {
+      "epoch": 7.834319526627219,
+      "grad_norm": 3.6041816520598307,
+      "learning_rate": 3.3809439307086463e-10,
+      "logits/chosen": -1.1416850090026855,
+      "logits/rejected": -1.2989619970321655,
+      "logps/chosen": -46.875675201416016,
+      "logps/rejected": -77.26806640625,
+      "loss": 0.0178,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.14378023147583,
+      "rewards/margins": 8.901089668273926,
+      "rewards/rejected": -11.044870376586914,
+      "step": 662
+    },
+    {
+      "epoch": 7.846153846153846,
+      "grad_norm": 4.065861079364059,
+      "learning_rate": 2.7386818974395323e-10,
+      "logits/chosen": -1.4301998615264893,
+      "logits/rejected": -1.3408135175704956,
+      "logps/chosen": -44.26204299926758,
+      "logps/rejected": -67.56184387207031,
+      "loss": 0.0172,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.9295194149017334,
+      "rewards/margins": 6.951984882354736,
+      "rewards/rejected": -9.88150405883789,
+      "step": 663
+    },
+    {
+      "epoch": 7.8579881656804735,
+      "grad_norm": 3.4612690772571497,
+      "learning_rate": 2.1639797456723952e-10,
+      "logits/chosen": -1.22926664352417,
+      "logits/rejected": -1.2551913261413574,
+      "logps/chosen": -35.541709899902344,
+      "logps/rejected": -57.685508728027344,
+      "loss": 0.0141,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.7379496693611145,
+      "rewards/margins": 7.399726867675781,
+      "rewards/rejected": -8.137676239013672,
+      "step": 664
+    },
+    {
+      "epoch": 7.8698224852071,
+      "grad_norm": 3.9801474006233795,
+      "learning_rate": 1.6568530231628185e-10,
+      "logits/chosen": -1.4825738668441772,
+      "logits/rejected": -1.4905058145523071,
+      "logps/chosen": -37.85133361816406,
+      "logps/rejected": -63.87050247192383,
+      "loss": 0.0166,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.4308565855026245,
+      "rewards/margins": 8.55210018157959,
+      "rewards/rejected": -9.982955932617188,
+      "step": 665
+    },
+    {
+      "epoch": 7.881656804733728,
+      "grad_norm": 4.104029377641207,
+      "learning_rate": 1.21731544950876e-10,
+      "logits/chosen": -1.226098656654358,
+      "logits/rejected": -1.371010422706604,
+      "logps/chosen": -48.73766326904297,
+      "logps/rejected": -67.00294494628906,
+      "loss": 0.0171,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.1735711097717285,
+      "rewards/margins": 7.302567481994629,
+      "rewards/rejected": -9.476139068603516,
+      "step": 666
+    },
+    {
+      "epoch": 7.893491124260355,
+      "grad_norm": 4.0448161239485145,
+      "learning_rate": 8.453789157794599e-11,
+      "logits/chosen": -1.1803443431854248,
+      "logits/rejected": -1.2014808654785156,
+      "logps/chosen": -38.218624114990234,
+      "logps/rejected": -63.20378112792969,
+      "loss": 0.0165,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.791195273399353,
+      "rewards/margins": 8.358329772949219,
+      "rewards/rejected": -9.14952564239502,
+      "step": 667
+    },
+    {
+      "epoch": 7.905325443786982,
+      "grad_norm": 4.422193721862127,
+      "learning_rate": 5.4105348419264394e-11,
+      "logits/chosen": -1.3956677913665771,
+      "logits/rejected": -1.1959235668182373,
+      "logps/chosen": -37.42644119262695,
+      "logps/rejected": -67.5661392211914,
+      "loss": 0.0177,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.9582293629646301,
+      "rewards/margins": 8.263663291931152,
+      "rewards/rejected": -9.221893310546875,
+      "step": 668
+    },
+    {
+      "epoch": 7.9171597633136095,
+      "grad_norm": 2.7605966616715523,
+      "learning_rate": 3.043473878436287e-11,
+      "logits/chosen": -1.191653847694397,
+      "logits/rejected": -1.153673768043518,
+      "logps/chosen": -44.5345573425293,
+      "logps/rejected": -70.56991577148438,
+      "loss": 0.0105,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.7923838496208191,
+      "rewards/margins": 8.033748626708984,
+      "rewards/rejected": -8.826131820678711,
+      "step": 669
+    },
+    {
+      "epoch": 7.928994082840236,
+      "grad_norm": 3.739721658753547,
+      "learning_rate": 1.3526703048216682e-11,
+      "logits/chosen": -1.5901668071746826,
+      "logits/rejected": -1.452099323272705,
+      "logps/chosen": -38.17021942138672,
+      "logps/rejected": -63.99696350097656,
+      "loss": 0.0157,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.6206248998641968,
+      "rewards/margins": 6.980554580688477,
+      "rewards/rejected": -8.601179122924805,
+      "step": 670
+    },
+    {
+      "epoch": 7.940828402366864,
+      "grad_norm": 4.73168147418833,
+      "learning_rate": 3.3816986338142117e-12,
+      "logits/chosen": -1.312976598739624,
+      "logits/rejected": -1.323945164680481,
+      "logps/chosen": -33.24012756347656,
+      "logps/rejected": -62.29106140136719,
+      "loss": 0.0186,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -1.5721168518066406,
+      "rewards/margins": 8.698356628417969,
+      "rewards/rejected": -10.27047348022461,
+      "step": 671
+    },
+    {
+      "epoch": 7.952662721893491,
+      "grad_norm": 4.043153951778064,
+      "learning_rate": 0.0,
+      "logits/chosen": -1.2264971733093262,
+      "logits/rejected": -1.2341418266296387,
+      "logps/chosen": -45.756378173828125,
+      "logps/rejected": -61.860496520996094,
+      "loss": 0.0137,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -2.606259822845459,
+      "rewards/margins": 6.51239013671875,
+      "rewards/rejected": -9.118650436401367,
+      "step": 672
+    },
+    {
+      "epoch": 7.952662721893491,
+      "step": 672,
+      "total_flos": 0.0,
+      "train_loss": 0.13648563410000256,
+      "train_runtime": 6735.2566,
+      "train_samples_per_second": 12.822,
+      "train_steps_per_second": 0.1
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 672,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 8,
+  "save_steps": 300,
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}