diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,2940 @@
+{
+  "best_metric": 1.5154520273208618,
+  "best_model_checkpoint": "saves/Falcon-7B-Instruct/lora/orpo/checkpoint-1500",
+  "epoch": 2.997999555456768,
+  "eval_steps": 500,
+  "global_step": 1686,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.017781729273171815,
+      "grad_norm": 0.45023536682128906,
+      "learning_rate": 4.9995745934141085e-06,
+      "logits/chosen": -14.31452751159668,
+      "logits/rejected": -14.272933959960938,
+      "logps/chosen": -1.777596116065979,
+      "logps/rejected": -1.814857840538025,
+      "loss": 1.8528,
+      "odds_ratio_loss": 0.7518970966339111,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": -0.17775960266590118,
+      "rewards/margins": 0.0037261671386659145,
+      "rewards/rejected": -0.18148578703403473,
+      "sft_loss": 1.777596116065979,
+      "step": 10
+    },
+    {
+      "epoch": 0.03556345854634363,
+      "grad_norm": 0.6821511387825012,
+      "learning_rate": 4.9982812903243405e-06,
+      "logits/chosen": -14.213617324829102,
+      "logits/rejected": -14.412919998168945,
+      "logps/chosen": -1.9183998107910156,
+      "logps/rejected": -1.8259010314941406,
+      "loss": 2.0025,
+      "odds_ratio_loss": 0.8412569761276245,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": -0.19183996319770813,
+      "rewards/margins": -0.009249850176274776,
+      "rewards/rejected": -0.18259011209011078,
+      "sft_loss": 1.9183998107910156,
+      "step": 20
+    },
+    {
+      "epoch": 0.05334518781951545,
+      "grad_norm": 0.50360107421875,
+      "learning_rate": 4.996120496405222e-06,
+      "logits/chosen": -14.275195121765137,
+      "logits/rejected": -14.341901779174805,
+      "logps/chosen": -1.8644087314605713,
+      "logps/rejected": -2.0387845039367676,
+      "loss": 1.9359,
+      "odds_ratio_loss": 0.7153545022010803,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.1864408552646637,
+      "rewards/margins": 0.01743762008845806,
+      "rewards/rejected": -0.2038784772157669,
+      "sft_loss": 1.8644087314605713,
+      "step": 30
+    },
+    {
+      "epoch": 0.07112691709268726,
+      "grad_norm": 0.6971050500869751,
+      "learning_rate": 4.99309296196014e-06,
+      "logits/chosen": -14.182516098022461,
+      "logits/rejected": -14.20283317565918,
+      "logps/chosen": -1.9314234256744385,
+      "logps/rejected": -1.899929404258728,
+      "loss": 2.0128,
+      "odds_ratio_loss": 0.8141088485717773,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": -0.1931423395872116,
+      "rewards/margins": -0.003149367868900299,
+      "rewards/rejected": -0.1899929791688919,
+      "sft_loss": 1.9314234256744385,
+      "step": 40
+    },
+    {
+      "epoch": 0.08890864636585907,
+      "grad_norm": 0.5635890960693359,
+      "learning_rate": 4.989199738255166e-06,
+      "logits/chosen": -14.374763488769531,
+      "logits/rejected": -14.155324935913086,
+      "logps/chosen": -1.9009517431259155,
+      "logps/rejected": -1.9379255771636963,
+      "loss": 1.9795,
+      "odds_ratio_loss": 0.7854829430580139,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.19009515643119812,
+      "rewards/margins": 0.003697408363223076,
+      "rewards/rejected": -0.19379255175590515,
+      "sft_loss": 1.9009517431259155,
+      "step": 50
+    },
+    {
+      "epoch": 0.1066903756390309,
+      "grad_norm": 0.6436208486557007,
+      "learning_rate": 4.984442177154031e-06,
+      "logits/chosen": -14.250883102416992,
+      "logits/rejected": -14.289509773254395,
+      "logps/chosen": -1.9730733633041382,
+      "logps/rejected": -2.040274143218994,
+      "loss": 2.0526,
+      "odds_ratio_loss": 0.7955271601676941,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": -0.19730734825134277,
+      "rewards/margins": 0.006720039062201977,
+      "rewards/rejected": -0.20402738451957703,
+      "sft_loss": 1.9730733633041382,
+      "step": 60
+    },
+    {
+      "epoch": 0.12447210491220272,
+      "grad_norm": 0.4920930862426758,
+      "learning_rate": 4.978821930648704e-06,
+      "logits/chosen": -14.044062614440918,
+      "logits/rejected": -14.116564750671387,
+      "logps/chosen": -1.9218193292617798,
+      "logps/rejected": -1.7758527994155884,
+      "loss": 2.0142,
+      "odds_ratio_loss": 0.9241151809692383,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.19218194484710693,
+      "rewards/margins": -0.014596650376915932,
+      "rewards/rejected": -0.17758527398109436,
+      "sft_loss": 1.9218193292617798,
+      "step": 70
+    },
+    {
+      "epoch": 0.14225383418537452,
+      "grad_norm": 0.6744620203971863,
+      "learning_rate": 4.97234095028576e-06,
+      "logits/chosen": -14.337008476257324,
+      "logits/rejected": -14.242892265319824,
+      "logps/chosen": -1.873708963394165,
+      "logps/rejected": -1.8873860836029053,
+      "loss": 1.9505,
+      "odds_ratio_loss": 0.767748236656189,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.1873709261417389,
+      "rewards/margins": 0.0013677121605724096,
+      "rewards/rejected": -0.1887386292219162,
+      "sft_loss": 1.873708963394165,
+      "step": 80
+    },
+    {
+      "epoch": 0.16003556345854633,
+      "grad_norm": 0.48053959012031555,
+      "learning_rate": 4.965001486488743e-06,
+      "logits/chosen": -14.241889953613281,
+      "logits/rejected": -14.200053215026855,
+      "logps/chosen": -1.7481162548065186,
+      "logps/rejected": -1.7742578983306885,
+      "loss": 1.8239,
+      "odds_ratio_loss": 0.7575067281723022,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.17481163144111633,
+      "rewards/margins": 0.0026141509879380465,
+      "rewards/rejected": -0.17742577195167542,
+      "sft_loss": 1.7481162548065186,
+      "step": 90
+    },
+    {
+      "epoch": 0.17781729273171815,
+      "grad_norm": 0.7781735062599182,
+      "learning_rate": 4.956806087776732e-06,
+      "logits/chosen": -14.596258163452148,
+      "logits/rejected": -14.529101371765137,
+      "logps/chosen": -1.8169043064117432,
+      "logps/rejected": -1.9338127374649048,
+      "loss": 1.8891,
+      "odds_ratio_loss": 0.7220322489738464,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.18169045448303223,
+      "rewards/margins": 0.01169085968285799,
+      "rewards/rejected": -0.19338129460811615,
+      "sft_loss": 1.8169043064117432,
+      "step": 100
+    },
+    {
+      "epoch": 0.19559902200489,
+      "grad_norm": 0.845944881439209,
+      "learning_rate": 4.947757599879411e-06,
+      "logits/chosen": -14.283439636230469,
+      "logits/rejected": -14.439828872680664,
+      "logps/chosen": -1.7649319171905518,
+      "logps/rejected": -1.82681405544281,
+      "loss": 1.8394,
+      "odds_ratio_loss": 0.7447811961174011,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.17649319767951965,
+      "rewards/margins": 0.0061882128939032555,
+      "rewards/rejected": -0.1826813966035843,
+      "sft_loss": 1.7649319171905518,
+      "step": 110
+    },
+    {
+      "epoch": 0.2133807512780618,
+      "grad_norm": 0.6829086542129517,
+      "learning_rate": 4.937859164748931e-06,
+      "logits/chosen": -14.171781539916992,
+      "logits/rejected": -14.2664155960083,
+      "logps/chosen": -1.65048348903656,
+      "logps/rejected": -1.6755653619766235,
+      "loss": 1.7267,
+      "odds_ratio_loss": 0.7621053457260132,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.16504836082458496,
+      "rewards/margins": 0.00250818463973701,
+      "rewards/rejected": -0.1675565242767334,
+      "sft_loss": 1.65048348903656,
+      "step": 120
+    },
+    {
+      "epoch": 0.23116248055123362,
+      "grad_norm": 0.6720818877220154,
+      "learning_rate": 4.92711421946891e-06,
+      "logits/chosen": -14.323086738586426,
+      "logits/rejected": -13.87572956085205,
+      "logps/chosen": -1.6958109140396118,
+      "logps/rejected": -1.8111820220947266,
+      "loss": 1.7688,
+      "odds_ratio_loss": 0.7296444177627563,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.1695810854434967,
+      "rewards/margins": 0.011537122540175915,
+      "rewards/rejected": -0.1811182051897049,
+      "sft_loss": 1.6958109140396118,
+      "step": 130
+    },
+    {
+      "epoch": 0.24894420982440543,
+      "grad_norm": 1.665626049041748,
+      "learning_rate": 4.915526495060961e-06,
+      "logits/chosen": -14.461613655090332,
+      "logits/rejected": -14.22163200378418,
+      "logps/chosen": -1.670013666152954,
+      "logps/rejected": -1.7781444787979126,
+      "loss": 1.7433,
+      "odds_ratio_loss": 0.7326729893684387,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.1670013815164566,
+      "rewards/margins": 0.010813087224960327,
+      "rewards/rejected": -0.17781445384025574,
+      "sft_loss": 1.670013666152954,
+      "step": 140
+    },
+    {
+      "epoch": 0.26672593909757725,
+      "grad_norm": 1.3471460342407227,
+      "learning_rate": 4.903100015189153e-06,
+      "logits/chosen": -14.236448287963867,
+      "logits/rejected": -14.436059951782227,
+      "logps/chosen": -1.6695849895477295,
+      "logps/rejected": -1.7570854425430298,
+      "loss": 1.7435,
+      "odds_ratio_loss": 0.7395648956298828,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.1669585108757019,
+      "rewards/margins": 0.008750038221478462,
+      "rewards/rejected": -0.17570854723453522,
+      "sft_loss": 1.6695849895477295,
+      "step": 150
+    },
+    {
+      "epoch": 0.28450766837074903,
+      "grad_norm": 1.0652554035186768,
+      "learning_rate": 4.889839094762848e-06,
+      "logits/chosen": -14.326433181762695,
+      "logits/rejected": -14.2631196975708,
+      "logps/chosen": -1.6757389307022095,
+      "logps/rejected": -1.8023579120635986,
+      "loss": 1.7489,
+      "odds_ratio_loss": 0.7316839098930359,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.16757391393184662,
+      "rewards/margins": 0.012661868706345558,
+      "rewards/rejected": -0.18023577332496643,
+      "sft_loss": 1.6757389307022095,
+      "step": 160
+    },
+    {
+      "epoch": 0.3022893976439209,
+      "grad_norm": 1.0970226526260376,
+      "learning_rate": 4.875748338438416e-06,
+      "logits/chosen": -14.249468803405762,
+      "logits/rejected": -14.319056510925293,
+      "logps/chosen": -1.6604044437408447,
+      "logps/rejected": -1.6869754791259766,
+      "loss": 1.7344,
+      "odds_ratio_loss": 0.7400213479995728,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.16604045033454895,
+      "rewards/margins": 0.0026570975314825773,
+      "rewards/rejected": -0.1686975508928299,
+      "sft_loss": 1.6604044437408447,
+      "step": 170
+    },
+    {
+      "epoch": 0.32007112691709266,
+      "grad_norm": 0.7075946927070618,
+      "learning_rate": 4.8608326390203386e-06,
+      "logits/chosen": -14.197771072387695,
+      "logits/rejected": -14.143452644348145,
+      "logps/chosen": -1.6201465129852295,
+      "logps/rejected": -1.7580602169036865,
+      "loss": 1.6907,
+      "odds_ratio_loss": 0.7053945660591125,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.1620146632194519,
+      "rewards/margins": 0.013791357167065144,
+      "rewards/rejected": -0.17580604553222656,
+      "sft_loss": 1.6201465129852295,
+      "step": 180
+    },
+    {
+      "epoch": 0.3378528561902645,
+      "grad_norm": 1.0393530130386353,
+      "learning_rate": 4.845097175762251e-06,
+      "logits/chosen": -14.362152099609375,
+      "logits/rejected": -14.374476432800293,
+      "logps/chosen": -1.6125462055206299,
+      "logps/rejected": -1.6287224292755127,
+      "loss": 1.6892,
+      "odds_ratio_loss": 0.7666895985603333,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": -0.1612546145915985,
+      "rewards/margins": 0.001617613947018981,
+      "rewards/rejected": -0.16287222504615784,
+      "sft_loss": 1.6125462055206299,
+      "step": 190
+    },
+    {
+      "epoch": 0.3556345854634363,
+      "grad_norm": 1.0453855991363525,
+      "learning_rate": 4.8285474125685286e-06,
+      "logits/chosen": -14.311877250671387,
+      "logits/rejected": -14.24933910369873,
+      "logps/chosen": -1.6674257516860962,
+      "logps/rejected": -1.6954532861709595,
+      "loss": 1.7433,
+      "odds_ratio_loss": 0.7591363191604614,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.16674259305000305,
+      "rewards/margins": 0.0028027337975800037,
+      "rewards/rejected": -0.16954532265663147,
+      "sft_loss": 1.6674257516860962,
+      "step": 200
+    },
+    {
+      "epoch": 0.37341631473660813,
+      "grad_norm": 1.7740540504455566,
+      "learning_rate": 4.811189096097025e-06,
+      "logits/chosen": -14.10380744934082,
+      "logits/rejected": -14.06958293914795,
+      "logps/chosen": -1.6661155223846436,
+      "logps/rejected": -1.7266288995742798,
+      "loss": 1.7425,
+      "odds_ratio_loss": 0.7635276913642883,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.16661155223846436,
+      "rewards/margins": 0.006051325239241123,
+      "rewards/rejected": -0.1726628839969635,
+      "sft_loss": 1.6661155223846436,
+      "step": 210
+    },
+    {
+      "epoch": 0.39119804400978,
+      "grad_norm": 1.2109721899032593,
+      "learning_rate": 4.793028253763633e-06,
+      "logits/chosen": -14.37977123260498,
+      "logits/rejected": -14.255584716796875,
+      "logps/chosen": -1.5491920709609985,
+      "logps/rejected": -1.6557328701019287,
+      "loss": 1.6259,
+      "odds_ratio_loss": 0.7670000791549683,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.15491920709609985,
+      "rewards/margins": 0.010654089972376823,
+      "rewards/rejected": -0.16557331383228302,
+      "sft_loss": 1.5491920709609985,
+      "step": 220
+    },
+    {
+      "epoch": 0.40897977328295176,
+      "grad_norm": 1.2553755044937134,
+      "learning_rate": 4.774071191649352e-06,
+      "logits/chosen": -14.052825927734375,
+      "logits/rejected": -14.052751541137695,
+      "logps/chosen": -1.540856122970581,
+      "logps/rejected": -1.742560625076294,
+      "loss": 1.6084,
+      "odds_ratio_loss": 0.6751853227615356,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.15408562123775482,
+      "rewards/margins": 0.0201704278588295,
+      "rewards/rejected": -0.17425604164600372,
+      "sft_loss": 1.540856122970581,
+      "step": 230
+    },
+    {
+      "epoch": 0.4267615025561236,
+      "grad_norm": 1.4113616943359375,
+      "learning_rate": 4.7543244923105975e-06,
+      "logits/chosen": -14.20154094696045,
+      "logits/rejected": -14.3230619430542,
+      "logps/chosen": -1.6533008813858032,
+      "logps/rejected": -1.6262308359146118,
+      "loss": 1.7354,
+      "odds_ratio_loss": 0.8208959698677063,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.16533009707927704,
+      "rewards/margins": -0.002707002917304635,
+      "rewards/rejected": -0.1626230925321579,
+      "sft_loss": 1.6533008813858032,
+      "step": 240
+    },
+    {
+      "epoch": 0.4445432318292954,
+      "grad_norm": 0.9963915348052979,
+      "learning_rate": 4.733795012493506e-06,
+      "logits/chosen": -14.148083686828613,
+      "logits/rejected": -14.310300827026367,
+      "logps/chosen": -1.6552881002426147,
+      "logps/rejected": -1.6675243377685547,
+      "loss": 1.7326,
+      "odds_ratio_loss": 0.7732909321784973,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.165528804063797,
+      "rewards/margins": 0.0012236315524205565,
+      "rewards/rejected": -0.16675245761871338,
+      "sft_loss": 1.6552881002426147,
+      "step": 250
+    },
+    {
+      "epoch": 0.46232496110246724,
+      "grad_norm": 0.9358872175216675,
+      "learning_rate": 4.712489880753035e-06,
+      "logits/chosen": -14.420260429382324,
+      "logits/rejected": -14.414996147155762,
+      "logps/chosen": -1.4923756122589111,
+      "logps/rejected": -1.5968248844146729,
+      "loss": 1.5619,
+      "odds_ratio_loss": 0.6949405670166016,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.14923755824565887,
+      "rewards/margins": 0.010444932617247105,
+      "rewards/rejected": -0.1596824824810028,
+      "sft_loss": 1.4923756122589111,
+      "step": 260
+    },
+    {
+      "epoch": 0.480106690375639,
+      "grad_norm": 2.065678596496582,
+      "learning_rate": 4.690416494977673e-06,
+      "logits/chosen": -14.560025215148926,
+      "logits/rejected": -14.5877103805542,
+      "logps/chosen": -1.5790516138076782,
+      "logps/rejected": -1.7484729290008545,
+      "loss": 1.6496,
+      "odds_ratio_loss": 0.7055513858795166,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.15790514647960663,
+      "rewards/margins": 0.016942132264375687,
+      "rewards/rejected": -0.1748472899198532,
+      "sft_loss": 1.5790516138076782,
+      "step": 270
+    },
+    {
+      "epoch": 0.49788841964881086,
+      "grad_norm": 3.234992265701294,
+      "learning_rate": 4.667582519820639e-06,
+      "logits/chosen": -14.247453689575195,
+      "logits/rejected": -14.422063827514648,
+      "logps/chosen": -1.5711301565170288,
+      "logps/rejected": -1.6209745407104492,
+      "loss": 1.6445,
+      "odds_ratio_loss": 0.7338452339172363,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.1571130007505417,
+      "rewards/margins": 0.0049844542518258095,
+      "rewards/rejected": -0.16209746897220612,
+      "sft_loss": 1.5711301565170288,
+      "step": 280
+    },
+    {
+      "epoch": 0.5156701489219827,
+      "grad_norm": 1.1059269905090332,
+      "learning_rate": 4.643995884038443e-06,
+      "logits/chosen": -14.227750778198242,
+      "logits/rejected": -14.276082038879395,
+      "logps/chosen": -1.6161178350448608,
+      "logps/rejected": -1.7087091207504272,
+      "loss": 1.6894,
+      "odds_ratio_loss": 0.7330858111381531,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.16161179542541504,
+      "rewards/margins": 0.009259124286472797,
+      "rewards/rejected": -0.17087092995643616,
+      "sft_loss": 1.6161178350448608,
+      "step": 290
+    },
+    {
+      "epoch": 0.5334518781951545,
+      "grad_norm": 2.079979658126831,
+      "learning_rate": 4.6196647777377475e-06,
+      "logits/chosen": -14.415349960327148,
+      "logits/rejected": -14.31702995300293,
+      "logps/chosen": -1.5473922491073608,
+      "logps/rejected": -1.5794051885604858,
+      "loss": 1.6226,
+      "odds_ratio_loss": 0.752013087272644,
+      "rewards/accuracies": 0.4312500059604645,
+      "rewards/chosen": -0.15473923087120056,
+      "rewards/margins": 0.0032012953888624907,
+      "rewards/rejected": -0.15794052183628082,
+      "sft_loss": 1.5473922491073608,
+      "step": 300
+    },
+    {
+      "epoch": 0.5512336074683263,
+      "grad_norm": 0.9750655889511108,
+      "learning_rate": 4.59459764953147e-06,
+      "logits/chosen": -14.490264892578125,
+      "logits/rejected": -14.242512702941895,
+      "logps/chosen": -1.6246612071990967,
+      "logps/rejected": -1.6596574783325195,
+      "loss": 1.6988,
+      "odds_ratio_loss": 0.7414273619651794,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.1624661237001419,
+      "rewards/margins": 0.00349963316693902,
+      "rewards/rejected": -0.1659657508134842,
+      "sft_loss": 1.6246612071990967,
+      "step": 310
+    },
+    {
+      "epoch": 0.5690153367414981,
+      "grad_norm": 1.211684226989746,
+      "learning_rate": 4.568803203605133e-06,
+      "logits/chosen": -14.534784317016602,
+      "logits/rejected": -14.392961502075195,
+      "logps/chosen": -1.6170380115509033,
+      "logps/rejected": -1.6397918462753296,
+      "loss": 1.6966,
+      "odds_ratio_loss": 0.7957952618598938,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.16170379519462585,
+      "rewards/margins": 0.0022753949742764235,
+      "rewards/rejected": -0.1639791876077652,
+      "sft_loss": 1.6170380115509033,
+      "step": 320
+    },
+    {
+      "epoch": 0.58679706601467,
+      "grad_norm": 2.359046459197998,
+      "learning_rate": 4.542290396694462e-06,
+      "logits/chosen": -14.300097465515137,
+      "logits/rejected": -14.317700386047363,
+      "logps/chosen": -1.5104106664657593,
+      "logps/rejected": -1.5948156118392944,
+      "loss": 1.5853,
+      "odds_ratio_loss": 0.7493588328361511,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -0.15104106068611145,
+      "rewards/margins": 0.008440487086772919,
+      "rewards/rejected": -0.15948157012462616,
+      "sft_loss": 1.5104106664657593,
+      "step": 330
+    },
+    {
+      "epoch": 0.6045787952878418,
+      "grad_norm": 1.9586892127990723,
+      "learning_rate": 4.515068434975298e-06,
+      "logits/chosen": -14.25054931640625,
+      "logits/rejected": -14.31701946258545,
+      "logps/chosen": -1.544409990310669,
+      "logps/rejected": -1.6858068704605103,
+      "loss": 1.6155,
+      "odds_ratio_loss": 0.7111681699752808,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.1544409841299057,
+      "rewards/margins": 0.014139704406261444,
+      "rewards/rejected": -0.16858068108558655,
+      "sft_loss": 1.544409990310669,
+      "step": 340
+    },
+    {
+      "epoch": 0.6223605245610135,
+      "grad_norm": 0.8323342800140381,
+      "learning_rate": 4.487146770866887e-06,
+      "logits/chosen": -14.46008586883545,
+      "logits/rejected": -14.50438404083252,
+      "logps/chosen": -1.6048237085342407,
+      "logps/rejected": -1.6058448553085327,
+      "loss": 1.6804,
+      "odds_ratio_loss": 0.7562613487243652,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": -0.16048237681388855,
+      "rewards/margins": 0.00010212887718807906,
+      "rewards/rejected": -0.16058449447155,
+      "sft_loss": 1.6048237085342407,
+      "step": 350
+    },
+    {
+      "epoch": 0.6401422538341853,
+      "grad_norm": 1.1129921674728394,
+      "learning_rate": 4.458535099749666e-06,
+      "logits/chosen": -14.248858451843262,
+      "logits/rejected": -14.238241195678711,
+      "logps/chosen": -1.6117630004882812,
+      "logps/rejected": -1.6199924945831299,
+      "loss": 1.6917,
+      "odds_ratio_loss": 0.799778938293457,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": -0.16117632389068604,
+      "rewards/margins": 0.0008229434606619179,
+      "rewards/rejected": -0.16199925541877747,
+      "sft_loss": 1.6117630004882812,
+      "step": 360
+    },
+    {
+      "epoch": 0.6579239831073572,
+      "grad_norm": 0.7189633250236511,
+      "learning_rate": 4.429243356598694e-06,
+      "logits/chosen": -14.405240058898926,
+      "logits/rejected": -14.393125534057617,
+      "logps/chosen": -1.485335111618042,
+      "logps/rejected": -1.6254221200942993,
+      "loss": 1.5568,
+      "odds_ratio_loss": 0.7148580551147461,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.14853352308273315,
+      "rewards/margins": 0.014008693397045135,
+      "rewards/rejected": -0.1625421941280365,
+      "sft_loss": 1.485335111618042,
+      "step": 370
+    },
+    {
+      "epoch": 0.675705712380529,
+      "grad_norm": 1.066667914390564,
+      "learning_rate": 4.399281712533875e-06,
+      "logits/chosen": -14.525976181030273,
+      "logits/rejected": -14.461471557617188,
+      "logps/chosen": -1.4982891082763672,
+      "logps/rejected": -1.5271437168121338,
+      "loss": 1.5745,
+      "odds_ratio_loss": 0.7622246146202087,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.14982891082763672,
+      "rewards/margins": 0.00288546085357666,
+      "rewards/rejected": -0.15271437168121338,
+      "sft_loss": 1.4982891082763672,
+      "step": 380
+    },
+    {
+      "epoch": 0.6934874416537008,
+      "grad_norm": 1.138702392578125,
+      "learning_rate": 4.368660571288192e-06,
+      "logits/chosen": -14.477781295776367,
+      "logits/rejected": -14.516647338867188,
+      "logps/chosen": -1.5524108409881592,
+      "logps/rejected": -1.5919525623321533,
+      "loss": 1.6311,
+      "odds_ratio_loss": 0.7873157262802124,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.15524108707904816,
+      "rewards/margins": 0.003954165615141392,
+      "rewards/rejected": -0.15919525921344757,
+      "sft_loss": 1.5524108409881592,
+      "step": 390
+    },
+    {
+      "epoch": 0.7112691709268726,
+      "grad_norm": 0.8798184394836426,
+      "learning_rate": 4.337390565595163e-06,
+      "logits/chosen": -14.244585037231445,
+      "logits/rejected": -14.454017639160156,
+      "logps/chosen": -1.5560581684112549,
+      "logps/rejected": -1.6040303707122803,
+      "loss": 1.6304,
+      "odds_ratio_loss": 0.743826150894165,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.15560582280158997,
+      "rewards/margins": 0.004797201603651047,
+      "rewards/rejected": -0.1604030430316925,
+      "sft_loss": 1.5560581684112549,
+      "step": 400
+    },
+    {
+      "epoch": 0.7290509002000445,
+      "grad_norm": 1.082557201385498,
+      "learning_rate": 4.305482553496786e-06,
+      "logits/chosen": -14.316876411437988,
+      "logits/rejected": -14.236166000366211,
+      "logps/chosen": -1.5271369218826294,
+      "logps/rejected": -1.5710500478744507,
+      "loss": 1.6033,
+      "odds_ratio_loss": 0.7612074613571167,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.15271370112895966,
+      "rewards/margins": 0.004391324706375599,
+      "rewards/rejected": -0.15710501372814178,
+      "sft_loss": 1.5271369218826294,
+      "step": 410
+    },
+    {
+      "epoch": 0.7468326294732163,
+      "grad_norm": 1.7793493270874023,
+      "learning_rate": 4.272947614573244e-06,
+      "logits/chosen": -14.312704086303711,
+      "logits/rejected": -14.391843795776367,
+      "logps/chosen": -1.6422202587127686,
+      "logps/rejected": -1.6885766983032227,
+      "loss": 1.7183,
+      "odds_ratio_loss": 0.761288046836853,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.16422203183174133,
+      "rewards/margins": 0.004635652992874384,
+      "rewards/rejected": -0.16885769367218018,
+      "sft_loss": 1.6422202587127686,
+      "step": 420
+    },
+    {
+      "epoch": 0.7646143587463881,
+      "grad_norm": 0.7179546356201172,
+      "learning_rate": 4.23979704609569e-06,
+      "logits/chosen": -14.418655395507812,
+      "logits/rejected": -14.451852798461914,
+      "logps/chosen": -1.5165865421295166,
+      "logps/rejected": -1.5794544219970703,
+      "loss": 1.5868,
+      "odds_ratio_loss": 0.7019873857498169,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.15165865421295166,
+      "rewards/margins": 0.006286793854087591,
+      "rewards/rejected": -0.157945454120636,
+      "sft_loss": 1.5165865421295166,
+      "step": 430
+    },
+    {
+      "epoch": 0.78239608801956,
+      "grad_norm": 1.1082905530929565,
+      "learning_rate": 4.206042359103435e-06,
+      "logits/chosen": -14.335638046264648,
+      "logits/rejected": -14.379010200500488,
+      "logps/chosen": -1.6033432483673096,
+      "logps/rejected": -1.648553490638733,
+      "loss": 1.6799,
+      "odds_ratio_loss": 0.7652724981307983,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.16033433377742767,
+      "rewards/margins": 0.004520997405052185,
+      "rewards/rejected": -0.16485533118247986,
+      "sft_loss": 1.6033432483673096,
+      "step": 440
+    },
+    {
+      "epoch": 0.8001778172927317,
+      "grad_norm": 1.853089690208435,
+      "learning_rate": 4.17169527440691e-06,
+      "logits/chosen": -14.423266410827637,
+      "logits/rejected": -14.40173625946045,
+      "logps/chosen": -1.62055242061615,
+      "logps/rejected": -1.6249053478240967,
+      "loss": 1.6973,
+      "odds_ratio_loss": 0.7678921222686768,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.16205522418022156,
+      "rewards/margins": 0.00043528637615963817,
+      "rewards/rejected": -0.16249051690101624,
+      "sft_loss": 1.62055242061615,
+      "step": 450
+    },
+    {
+      "epoch": 0.8179595465659035,
+      "grad_norm": 1.930782675743103,
+      "learning_rate": 4.136767718517797e-06,
+      "logits/chosen": -14.467844009399414,
+      "logits/rejected": -14.397687911987305,
+      "logps/chosen": -1.423508882522583,
+      "logps/rejected": -1.5464013814926147,
+      "loss": 1.4918,
+      "odds_ratio_loss": 0.6831967830657959,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.14235089719295502,
+      "rewards/margins": 0.012289242818951607,
+      "rewards/rejected": -0.15464013814926147,
+      "sft_loss": 1.423508882522583,
+      "step": 460
+    },
+    {
+      "epoch": 0.8357412758390753,
+      "grad_norm": 4.023218631744385,
+      "learning_rate": 4.1012718195077196e-06,
+      "logits/chosen": -14.457601547241211,
+      "logits/rejected": -14.645208358764648,
+      "logps/chosen": -1.5619157552719116,
+      "logps/rejected": -1.5855516195297241,
+      "loss": 1.6365,
+      "odds_ratio_loss": 0.7463001608848572,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": -0.15619158744812012,
+      "rewards/margins": 0.0023635749239474535,
+      "rewards/rejected": -0.15855516493320465,
+      "sft_loss": 1.5619157552719116,
+      "step": 470
+    },
+    {
+      "epoch": 0.8535230051122472,
+      "grad_norm": 3.4996113777160645,
+      "learning_rate": 4.065219902796953e-06,
+      "logits/chosen": -14.268010139465332,
+      "logits/rejected": -14.2593994140625,
+      "logps/chosen": -1.537496566772461,
+      "logps/rejected": -1.6044343709945679,
+      "loss": 1.6152,
+      "odds_ratio_loss": 0.7775283455848694,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.15374964475631714,
+      "rewards/margins": 0.006693786941468716,
+      "rewards/rejected": -0.16044344007968903,
+      "sft_loss": 1.537496566772461,
+      "step": 480
+    },
+    {
+      "epoch": 0.871304734385419,
+      "grad_norm": 0.903472900390625,
+      "learning_rate": 4.028624486874608e-06,
+      "logits/chosen": -14.208300590515137,
+      "logits/rejected": -14.410181045532227,
+      "logps/chosen": -1.4613759517669678,
+      "logps/rejected": -1.5640151500701904,
+      "loss": 1.5343,
+      "odds_ratio_loss": 0.728947103023529,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.14613759517669678,
+      "rewards/margins": 0.01026391051709652,
+      "rewards/rejected": -0.15640152990818024,
+      "sft_loss": 1.4613759517669678,
+      "step": 490
+    },
+    {
+      "epoch": 0.8890864636585908,
+      "grad_norm": 0.9934778809547424,
+      "learning_rate": 3.99149827895177e-06,
+      "logits/chosen": -14.429773330688477,
+      "logits/rejected": -14.360036849975586,
+      "logps/chosen": -1.5578300952911377,
+      "logps/rejected": -1.5907199382781982,
+      "loss": 1.6309,
+      "odds_ratio_loss": 0.7302489280700684,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.155783012509346,
+      "rewards/margins": 0.0032889836002141237,
+      "rewards/rejected": -0.15907198190689087,
+      "sft_loss": 1.5578300952911377,
+      "step": 500
+    },
+    {
+      "epoch": 0.8890864636585908,
+      "eval_logits/chosen": -14.421330451965332,
+      "eval_logits/rejected": -14.49679946899414,
+      "eval_logps/chosen": -1.5096371173858643,
+      "eval_logps/rejected": -1.598750114440918,
+      "eval_loss": 1.5815595388412476,
+      "eval_odds_ratio_loss": 0.7192248106002808,
+      "eval_rewards/accuracies": 0.49399998784065247,
+      "eval_rewards/chosen": -0.150963693857193,
+      "eval_rewards/margins": 0.008911306038498878,
+      "eval_rewards/rejected": -0.1598750203847885,
+      "eval_runtime": 203.6844,
+      "eval_samples_per_second": 4.91,
+      "eval_sft_loss": 1.5096371173858643,
+      "eval_steps_per_second": 2.455,
+      "step": 500
+    },
+    {
+      "epoch": 0.9068681929317626,
+      "grad_norm": 1.555288314819336,
+      "learning_rate": 3.953854170549114e-06,
+      "logits/chosen": -14.514457702636719,
+      "logits/rejected": -14.522333145141602,
+      "logps/chosen": -1.5636659860610962,
+      "logps/rejected": -1.554619550704956,
+      "loss": 1.6424,
+      "odds_ratio_loss": 0.7876118421554565,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.15636660158634186,
+      "rewards/margins": -0.0009046423947438598,
+      "rewards/rejected": -0.15546198189258575,
+      "sft_loss": 1.5636659860610962,
+      "step": 510
+    },
+    {
+      "epoch": 0.9246499222049345,
+      "grad_norm": 1.688568353652954,
+      "learning_rate": 3.91570523302051e-06,
+      "logits/chosen": -14.493148803710938,
+      "logits/rejected": -14.430615425109863,
+      "logps/chosen": -1.4217069149017334,
+      "logps/rejected": -1.5474450588226318,
+      "loss": 1.4923,
+      "odds_ratio_loss": 0.7059618830680847,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.14217069745063782,
+      "rewards/margins": 0.012573805637657642,
+      "rewards/rejected": -0.15474450588226318,
+      "sft_loss": 1.4217069149017334,
+      "step": 520
+    },
+    {
+      "epoch": 0.9424316514781063,
+      "grad_norm": 1.1463229656219482,
+      "learning_rate": 3.8770647130141996e-06,
+      "logits/chosen": -14.525823593139648,
+      "logits/rejected": -14.399996757507324,
+      "logps/chosen": -1.4726465940475464,
+      "logps/rejected": -1.5658392906188965,
+      "loss": 1.5466,
+      "odds_ratio_loss": 0.7396677136421204,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.14726465940475464,
+      "rewards/margins": 0.009319271892309189,
+      "rewards/rejected": -0.15658393502235413,
+      "sft_loss": 1.4726465940475464,
+      "step": 530
+    },
+    {
+      "epoch": 0.960213380751278,
+      "grad_norm": 2.26594877243042,
+      "learning_rate": 3.837946027873086e-06,
+      "logits/chosen": -14.543024063110352,
+      "logits/rejected": -14.401150703430176,
+      "logps/chosen": -1.5846188068389893,
+      "logps/rejected": -1.6536743640899658,
+      "loss": 1.661,
+      "odds_ratio_loss": 0.7640754580497742,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.15846188366413116,
+      "rewards/margins": 0.006905559450387955,
+      "rewards/rejected": -0.16536743938922882,
+      "sft_loss": 1.5846188068389893,
+      "step": 540
+    },
+    {
+      "epoch": 0.9779951100244498,
+      "grad_norm": 1.5690348148345947,
+      "learning_rate": 3.7983627609757713e-06,
+      "logits/chosen": -14.507429122924805,
+      "logits/rejected": -14.452433586120605,
+      "logps/chosen": -1.5472663640975952,
+      "logps/rejected": -1.544019103050232,
+      "loss": 1.6249,
+      "odds_ratio_loss": 0.7762556076049805,
+      "rewards/accuracies": 0.4312500059604645,
+      "rewards/chosen": -0.15472662448883057,
+      "rewards/margins": -0.0003247327113058418,
+      "rewards/rejected": -0.15440191328525543,
+      "sft_loss": 1.5472663640975952,
+      "step": 550
+    },
+    {
+      "epoch": 0.9957768392976217,
+      "grad_norm": 1.6722140312194824,
+      "learning_rate": 3.758328657019924e-06,
+      "logits/chosen": -14.339811325073242,
+      "logits/rejected": -14.203923225402832,
+      "logps/chosen": -1.4747674465179443,
+      "logps/rejected": -1.5587810277938843,
+      "loss": 1.5469,
+      "odds_ratio_loss": 0.7211123704910278,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.14747674763202667,
+      "rewards/margins": 0.008401353843510151,
+      "rewards/rejected": -0.15587811172008514,
+      "sft_loss": 1.4747674465179443,
+      "step": 560
+    },
+    {
+      "epoch": 1.0135585685707935,
+      "grad_norm": 4.022447109222412,
+      "learning_rate": 3.717857617249642e-06,
+      "logits/chosen": -14.2714204788208,
+      "logits/rejected": -14.344494819641113,
+      "logps/chosen": -1.5280239582061768,
+      "logps/rejected": -1.6387897729873657,
+      "loss": 1.6007,
+      "odds_ratio_loss": 0.72718346118927,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.15280239284038544,
+      "rewards/margins": 0.011076575145125389,
+      "rewards/rejected": -0.16387899219989777,
+      "sft_loss": 1.5280239582061768,
+      "step": 570
+    },
+    {
+      "epoch": 1.0313402978439654,
+      "grad_norm": 0.8425918817520142,
+      "learning_rate": 3.6769636946284543e-06,
+      "logits/chosen": -14.346611022949219,
+      "logits/rejected": -14.158876419067383,
+      "logps/chosen": -1.3988748788833618,
+      "logps/rejected": -1.519100308418274,
+      "loss": 1.4702,
+      "odds_ratio_loss": 0.7135868072509766,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.1398874968290329,
+      "rewards/margins": 0.012022523209452629,
+      "rewards/rejected": -0.15191002190113068,
+      "sft_loss": 1.3988748788833618,
+      "step": 580
+    },
+    {
+      "epoch": 1.049122027117137,
+      "grad_norm": 1.2780736684799194,
+      "learning_rate": 3.6356610889596355e-06,
+      "logits/chosen": -14.491503715515137,
+      "logits/rejected": -14.515363693237305,
+      "logps/chosen": -1.567704200744629,
+      "logps/rejected": -1.5939210653305054,
+      "loss": 1.6438,
+      "odds_ratio_loss": 0.7608035206794739,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.15677042305469513,
+      "rewards/margins": 0.002621681662276387,
+      "rewards/rejected": -0.1593921184539795,
+      "sft_loss": 1.567704200744629,
+      "step": 590
+    },
+    {
+      "epoch": 1.066903756390309,
+      "grad_norm": 1.1325277090072632,
+      "learning_rate": 3.593964141955541e-06,
+      "logits/chosen": -14.486841201782227,
+      "logits/rejected": -14.375473022460938,
+      "logps/chosen": -1.4968254566192627,
+      "logps/rejected": -1.535290241241455,
+      "loss": 1.5708,
+      "odds_ratio_loss": 0.7399110198020935,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.14968255162239075,
+      "rewards/margins": 0.0038464791141450405,
+      "rewards/rejected": -0.15352903306484222,
+      "sft_loss": 1.4968254566192627,
+      "step": 600
+    },
+    {
+      "epoch": 1.0846854856634809,
+      "grad_norm": 1.0757452249526978,
+      "learning_rate": 3.5518873322576573e-06,
+      "logits/chosen": -14.257904052734375,
+      "logits/rejected": -14.527667045593262,
+      "logps/chosen": -1.5230721235275269,
+      "logps/rejected": -1.5504690408706665,
+      "loss": 1.6005,
+      "odds_ratio_loss": 0.774411678314209,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.1523071974515915,
+      "rewards/margins": 0.0027397016528993845,
+      "rewards/rejected": -0.15504691004753113,
+      "sft_loss": 1.5230721235275269,
+      "step": 610
+    },
+    {
+      "epoch": 1.1024672149366526,
+      "grad_norm": 0.802010715007782,
+      "learning_rate": 3.5094452704091143e-06,
+      "logits/chosen": -14.364301681518555,
+      "logits/rejected": -14.379095077514648,
+      "logps/chosen": -1.4637863636016846,
+      "logps/rejected": -1.551966667175293,
+      "loss": 1.5353,
+      "odds_ratio_loss": 0.7155525088310242,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.14637863636016846,
+      "rewards/margins": 0.008818031288683414,
+      "rewards/rejected": -0.1551966667175293,
+      "sft_loss": 1.4637863636016846,
+      "step": 620
+    },
+    {
+      "epoch": 1.1202489442098245,
+      "grad_norm": 1.9857678413391113,
+      "learning_rate": 3.46665269378139e-06,
+      "logits/chosen": -14.460027694702148,
+      "logits/rejected": -14.280843734741211,
+      "logps/chosen": -1.57863187789917,
+      "logps/rejected": -1.5787197351455688,
+      "loss": 1.6576,
+      "odds_ratio_loss": 0.7900050282478333,
+      "rewards/accuracies": 0.4312500059604645,
+      "rewards/chosen": -0.15786318480968475,
+      "rewards/margins": 8.772313776717056e-06,
+      "rewards/rejected": -0.15787196159362793,
+      "sft_loss": 1.57863187789917,
+      "step": 630
+    },
+    {
+      "epoch": 1.1380306734829961,
+      "grad_norm": 3.4590837955474854,
+      "learning_rate": 3.4235244614569794e-06,
+      "logits/chosen": -14.505206108093262,
+      "logits/rejected": -14.5104398727417,
+      "logps/chosen": -1.5605857372283936,
+      "logps/rejected": -1.5010731220245361,
+      "loss": 1.6419,
+      "odds_ratio_loss": 0.8131183385848999,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.15605857968330383,
+      "rewards/margins": -0.0059512583538889885,
+      "rewards/rejected": -0.15010732412338257,
+      "sft_loss": 1.5605857372283936,
+      "step": 640
+    },
+    {
+      "epoch": 1.155812402756168,
+      "grad_norm": 0.7217416763305664,
+      "learning_rate": 3.3800755490698008e-06,
+      "logits/chosen": -14.571185111999512,
+      "logits/rejected": -14.486068725585938,
+      "logps/chosen": -1.416325569152832,
+      "logps/rejected": -1.6571706533432007,
+      "loss": 1.4832,
+      "odds_ratio_loss": 0.6685255765914917,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.1416325569152832,
+      "rewards/margins": 0.024084512144327164,
+      "rewards/rejected": -0.16571708023548126,
+      "sft_loss": 1.416325569152832,
+      "step": 650
+    },
+    {
+      "epoch": 1.17359413202934,
+      "grad_norm": 1.5739328861236572,
+      "learning_rate": 3.3363210436051287e-06,
+      "logits/chosen": -14.533714294433594,
+      "logits/rejected": -14.47203540802002,
+      "logps/chosen": -1.5552722215652466,
+      "logps/rejected": -1.6581999063491821,
+      "loss": 1.6292,
+      "odds_ratio_loss": 0.7395690083503723,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.1555272340774536,
+      "rewards/margins": 0.010292761959135532,
+      "rewards/rejected": -0.16581998765468597,
+      "sft_loss": 1.5552722215652466,
+      "step": 660
+    },
+    {
+      "epoch": 1.1913758613025116,
+      "grad_norm": 1.1496367454528809,
+      "learning_rate": 3.292276138160867e-06,
+      "logits/chosen": -14.616167068481445,
+      "logits/rejected": -14.592849731445312,
+      "logps/chosen": -1.4817497730255127,
+      "logps/rejected": -1.5020571947097778,
+      "loss": 1.5589,
+      "odds_ratio_loss": 0.7710050940513611,
+      "rewards/accuracies": 0.41874998807907104,
+      "rewards/chosen": -0.1481749713420868,
+      "rewards/margins": 0.002030743286013603,
+      "rewards/rejected": -0.15020573139190674,
+      "sft_loss": 1.4817497730255127,
+      "step": 670
+    },
+    {
+      "epoch": 1.2091575905756835,
+      "grad_norm": 0.9039623737335205,
+      "learning_rate": 3.2479561266719694e-06,
+      "logits/chosen": -14.403009414672852,
+      "logits/rejected": -14.395463943481445,
+      "logps/chosen": -1.536474347114563,
+      "logps/rejected": -1.5868223905563354,
+      "loss": 1.6131,
+      "odds_ratio_loss": 0.7661021947860718,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.15364743769168854,
+      "rewards/margins": 0.005034810863435268,
+      "rewards/rejected": -0.15868225693702698,
+      "sft_loss": 1.536474347114563,
+      "step": 680
+    },
+    {
+      "epoch": 1.2269393198488552,
+      "grad_norm": 1.4711978435516357,
+      "learning_rate": 3.2033763985998533e-06,
+      "logits/chosen": -14.493423461914062,
+      "logits/rejected": -14.387487411499023,
+      "logps/chosen": -1.4371049404144287,
+      "logps/rejected": -1.6720597743988037,
+      "loss": 1.5038,
+      "odds_ratio_loss": 0.6674162149429321,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.14371049404144287,
+      "rewards/margins": 0.023495487868785858,
+      "rewards/rejected": -0.16720597445964813,
+      "sft_loss": 1.4371049404144287,
+      "step": 690
+    },
+    {
+      "epoch": 1.244721049122027,
+      "grad_norm": 1.8284207582473755,
+      "learning_rate": 3.1585524335886335e-06,
+      "logits/chosen": -14.462885856628418,
+      "logits/rejected": -14.4248628616333,
+      "logps/chosen": -1.4241148233413696,
+      "logps/rejected": -1.5204662084579468,
+      "loss": 1.4937,
+      "odds_ratio_loss": 0.695872962474823,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.1424115002155304,
+      "rewards/margins": 0.00963515229523182,
+      "rewards/rejected": -0.15204663574695587,
+      "sft_loss": 1.4241148233413696,
+      "step": 700
+    },
+    {
+      "epoch": 1.262502778395199,
+      "grad_norm": 1.8534456491470337,
+      "learning_rate": 3.1134997960900536e-06,
+      "logits/chosen": -14.414436340332031,
+      "logits/rejected": -14.282020568847656,
+      "logps/chosen": -1.4227323532104492,
+      "logps/rejected": -1.592142105102539,
+      "loss": 1.4906,
+      "odds_ratio_loss": 0.678249180316925,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.14227323234081268,
+      "rewards/margins": 0.01694098487496376,
+      "rewards/rejected": -0.15921422839164734,
+      "sft_loss": 1.4227323532104492,
+      "step": 710
+    },
+    {
+      "epoch": 1.2802845076683709,
+      "grad_norm": 1.5054043531417847,
+      "learning_rate": 3.0682341299589583e-06,
+      "logits/chosen": -14.277090072631836,
+      "logits/rejected": -14.398614883422852,
+      "logps/chosen": -1.473953127861023,
+      "logps/rejected": -1.4768346548080444,
+      "loss": 1.5498,
+      "odds_ratio_loss": 0.7589610815048218,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.1473953276872635,
+      "rewards/margins": 0.00028814561665058136,
+      "rewards/rejected": -0.14768347144126892,
+      "sft_loss": 1.473953127861023,
+      "step": 720
+    },
+    {
+      "epoch": 1.2980662369415426,
+      "grad_norm": 2.268519878387451,
+      "learning_rate": 3.022771153021201e-06,
+      "logits/chosen": -14.384374618530273,
+      "logits/rejected": -14.452374458312988,
+      "logps/chosen": -1.4778945446014404,
+      "logps/rejected": -1.574272871017456,
+      "loss": 1.5509,
+      "odds_ratio_loss": 0.7302027940750122,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.14778944849967957,
+      "rewards/margins": 0.009637835435569286,
+      "rewards/rejected": -0.15742729604244232,
+      "sft_loss": 1.4778945446014404,
+      "step": 730
+    },
+    {
+      "epoch": 1.3158479662147144,
+      "grad_norm": 1.292470097541809,
+      "learning_rate": 2.9771266516158625e-06,
+      "logits/chosen": -14.381269454956055,
+      "logits/rejected": -14.515772819519043,
+      "logps/chosen": -1.467395544052124,
+      "logps/rejected": -1.5885987281799316,
+      "loss": 1.5376,
+      "odds_ratio_loss": 0.702446699142456,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.14673955738544464,
+      "rewards/margins": 0.01212033536285162,
+      "rewards/rejected": -0.15885989367961884,
+      "sft_loss": 1.467395544052124,
+      "step": 740
+    },
+    {
+      "epoch": 1.3336296954878861,
+      "grad_norm": 1.0139861106872559,
+      "learning_rate": 2.9313164751136802e-06,
+      "logits/chosen": -14.319999694824219,
+      "logits/rejected": -14.376035690307617,
+      "logps/chosen": -1.4767507314682007,
+      "logps/rejected": -1.5156667232513428,
+      "loss": 1.5505,
+      "odds_ratio_loss": 0.7372480630874634,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.1476750671863556,
+      "rewards/margins": 0.0038915909826755524,
+      "rewards/rejected": -0.15156666934490204,
+      "sft_loss": 1.4767507314682007,
+      "step": 750
+    },
+    {
+      "epoch": 1.351411424761058,
+      "grad_norm": 1.8870171308517456,
+      "learning_rate": 2.8853565304135956e-06,
+      "logits/chosen": -14.622869491577148,
+      "logits/rejected": -14.402894973754883,
+      "logps/chosen": -1.464207410812378,
+      "logps/rejected": -1.509709358215332,
+      "loss": 1.5409,
+      "odds_ratio_loss": 0.7670952081680298,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.14642071723937988,
+      "rewards/margins": 0.004550204146653414,
+      "rewards/rejected": -0.1509709358215332,
+      "sft_loss": 1.464207410812378,
+      "step": 760
+    },
+    {
+      "epoch": 1.36919315403423,
+      "grad_norm": 2.0388317108154297,
+      "learning_rate": 2.839262776419313e-06,
+      "logits/chosen": -14.462852478027344,
+      "logits/rejected": -14.366838455200195,
+      "logps/chosen": -1.4720194339752197,
+      "logps/rejected": -1.659558892250061,
+      "loss": 1.5416,
+      "odds_ratio_loss": 0.6956937313079834,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.14720192551612854,
+      "rewards/margins": 0.01875394582748413,
+      "rewards/rejected": -0.16595587134361267,
+      "sft_loss": 1.4720194339752197,
+      "step": 770
+    },
+    {
+      "epoch": 1.3869748833074016,
+      "grad_norm": 1.2085295915603638,
+      "learning_rate": 2.793051218497817e-06,
+      "logits/chosen": -14.64861011505127,
+      "logits/rejected": -14.640996932983398,
+      "logps/chosen": -1.4684925079345703,
+      "logps/rejected": -1.4641777276992798,
+      "loss": 1.5454,
+      "odds_ratio_loss": 0.7690992951393127,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.14684924483299255,
+      "rewards/margins": -0.00043147840187884867,
+      "rewards/rejected": -0.1464177817106247,
+      "sft_loss": 1.4684925079345703,
+      "step": 780
+    },
+    {
+      "epoch": 1.4047566125805735,
+      "grad_norm": 0.9894825220108032,
+      "learning_rate": 2.7467379029217437e-06,
+      "logits/chosen": -14.430456161499023,
+      "logits/rejected": -14.312626838684082,
+      "logps/chosen": -1.4456686973571777,
+      "logps/rejected": -1.5765860080718994,
+      "loss": 1.517,
+      "odds_ratio_loss": 0.7131879925727844,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.1445668637752533,
+      "rewards/margins": 0.013091735541820526,
+      "rewards/rejected": -0.15765860676765442,
+      "sft_loss": 1.4456686973571777,
+      "step": 790
+    },
+    {
+      "epoch": 1.4225383418537452,
+      "grad_norm": 0.9283164143562317,
+      "learning_rate": 2.7003389112975546e-06,
+      "logits/chosen": -14.672605514526367,
+      "logits/rejected": -14.80772590637207,
+      "logps/chosen": -1.5466537475585938,
+      "logps/rejected": -1.5707231760025024,
+      "loss": 1.6221,
+      "odds_ratio_loss": 0.7544839382171631,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -0.15466538071632385,
+      "rewards/margins": 0.002406922634691,
+      "rewards/rejected": -0.1570723056793213,
+      "sft_loss": 1.5466537475585938,
+      "step": 800
+    },
+    {
+      "epoch": 1.440320071126917,
+      "grad_norm": 2.0001485347747803,
+      "learning_rate": 2.653870354981437e-06,
+      "logits/chosen": -14.508201599121094,
+      "logits/rejected": -14.300331115722656,
+      "logps/chosen": -1.343572735786438,
+      "logps/rejected": -1.4564281702041626,
+      "loss": 1.4137,
+      "odds_ratio_loss": 0.7009164690971375,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.1343572735786438,
+      "rewards/margins": 0.011285548098385334,
+      "rewards/rejected": -0.14564281702041626,
+      "sft_loss": 1.343572735786438,
+      "step": 810
+    },
+    {
+      "epoch": 1.458101800400089,
+      "grad_norm": 2.214451313018799,
+      "learning_rate": 2.6073483694848777e-06,
+      "logits/chosen": -14.371310234069824,
+      "logits/rejected": -14.641912460327148,
+      "logps/chosen": -1.459166169166565,
+      "logps/rejected": -1.5504926443099976,
+      "loss": 1.5337,
+      "odds_ratio_loss": 0.7451664805412292,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.14591661095619202,
+      "rewards/margins": 0.009132652543485165,
+      "rewards/rejected": -0.15504927933216095,
+      "sft_loss": 1.459166169166565,
+      "step": 820
+    },
+    {
+      "epoch": 1.4758835296732609,
+      "grad_norm": 0.7959926724433899,
+      "learning_rate": 2.560789108871847e-06,
+      "logits/chosen": -14.411550521850586,
+      "logits/rejected": -14.393269538879395,
+      "logps/chosen": -1.436858892440796,
+      "logps/rejected": -1.588494896888733,
+      "loss": 1.5064,
+      "odds_ratio_loss": 0.6953193545341492,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.14368589222431183,
+      "rewards/margins": 0.015163603238761425,
+      "rewards/rejected": -0.15884950757026672,
+      "sft_loss": 1.436858892440796,
+      "step": 830
+    },
+    {
+      "epoch": 1.4936652589464325,
+      "grad_norm": 2.3933050632476807,
+      "learning_rate": 2.514208740149544e-06,
+      "logits/chosen": -14.369955062866211,
+      "logits/rejected": -14.41723346710205,
+      "logps/chosen": -1.4793574810028076,
+      "logps/rejected": -1.566274881362915,
+      "loss": 1.5523,
+      "odds_ratio_loss": 0.729671061038971,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.14793574810028076,
+      "rewards/margins": 0.008691729977726936,
+      "rewards/rejected": -0.15662747621536255,
+      "sft_loss": 1.4793574810028076,
+      "step": 840
+    },
+    {
+      "epoch": 1.5114469882196042,
+      "grad_norm": 1.3297358751296997,
+      "learning_rate": 2.46762343765464e-06,
+      "logits/chosen": -14.52283000946045,
+      "logits/rejected": -14.487058639526367,
+      "logps/chosen": -1.4763939380645752,
+      "logps/rejected": -1.6392600536346436,
+      "loss": 1.5463,
+      "odds_ratio_loss": 0.6985622644424438,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.14763939380645752,
+      "rewards/margins": 0.01628662459552288,
+      "rewards/rejected": -0.16392602026462555,
+      "sft_loss": 1.4763939380645752,
+      "step": 850
+    },
+    {
+      "epoch": 1.5292287174927761,
+      "grad_norm": 1.4351073503494263,
+      "learning_rate": 2.4210493774369903e-06,
+      "logits/chosen": -14.422691345214844,
+      "logits/rejected": -14.469161987304688,
+      "logps/chosen": -1.5580244064331055,
+      "logps/rejected": -1.5905354022979736,
+      "loss": 1.6351,
+      "odds_ratio_loss": 0.770281970500946,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.1558024138212204,
+      "rewards/margins": 0.003251106943935156,
+      "rewards/rejected": -0.15905353426933289,
+      "sft_loss": 1.5580244064331055,
+      "step": 860
+    },
+    {
+      "epoch": 1.547010446765948,
+      "grad_norm": 0.898558497428894,
+      "learning_rate": 2.374502731642732e-06,
+      "logits/chosen": -14.498028755187988,
+      "logits/rejected": -14.491365432739258,
+      "logps/chosen": -1.4510562419891357,
+      "logps/rejected": -1.4965808391571045,
+      "loss": 1.5247,
+      "odds_ratio_loss": 0.7362778782844543,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.14510561525821686,
+      "rewards/margins": 0.004552468657493591,
+      "rewards/rejected": -0.14965808391571045,
+      "sft_loss": 1.4510562419891357,
+      "step": 870
+    },
+    {
+      "epoch": 1.56479217603912,
+      "grad_norm": 1.0833823680877686,
+      "learning_rate": 2.3279996628987556e-06,
+      "logits/chosen": -14.427210807800293,
+      "logits/rejected": -14.47814655303955,
+      "logps/chosen": -1.4597257375717163,
+      "logps/rejected": -1.4845510721206665,
+      "loss": 1.5356,
+      "odds_ratio_loss": 0.7589424252510071,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.14597256481647491,
+      "rewards/margins": 0.002482546027749777,
+      "rewards/rejected": -0.14845511317253113,
+      "sft_loss": 1.4597257375717163,
+      "step": 880
+    },
+    {
+      "epoch": 1.5825739053122916,
+      "grad_norm": 1.345390796661377,
+      "learning_rate": 2.281556318700474e-06,
+      "logits/chosen": -14.579635620117188,
+      "logits/rejected": -14.662760734558105,
+      "logps/chosen": -1.435423493385315,
+      "logps/rejected": -1.4268932342529297,
+      "loss": 1.5144,
+      "odds_ratio_loss": 0.789936900138855,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.1435423642396927,
+      "rewards/margins": -0.0008530290797352791,
+      "rewards/rejected": -0.14268933236598969,
+      "sft_loss": 1.435423493385315,
+      "step": 890
+    },
+    {
+      "epoch": 1.6003556345854635,
+      "grad_norm": 2.053659439086914,
+      "learning_rate": 2.2351888258048408e-06,
+      "logits/chosen": -14.430631637573242,
+      "logits/rejected": -14.57084846496582,
+      "logps/chosen": -1.4910775423049927,
+      "logps/rejected": -1.588700532913208,
+      "loss": 1.5647,
+      "odds_ratio_loss": 0.7359753251075745,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.14910776913166046,
+      "rewards/margins": 0.009762286208570004,
+      "rewards/rejected": -0.15887002646923065,
+      "sft_loss": 1.4910775423049927,
+      "step": 900
+    },
+    {
+      "epoch": 1.6181373638586352,
+      "grad_norm": 1.0991109609603882,
+      "learning_rate": 2.188913284630584e-06,
+      "logits/chosen": -14.372177124023438,
+      "logits/rejected": -14.45659065246582,
+      "logps/chosen": -1.5502886772155762,
+      "logps/rejected": -1.5803656578063965,
+      "loss": 1.6264,
+      "odds_ratio_loss": 0.7611321210861206,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.15502886474132538,
+      "rewards/margins": 0.0030077022965997458,
+      "rewards/rejected": -0.15803657472133636,
+      "sft_loss": 1.5502886772155762,
+      "step": 910
+    },
+    {
+      "epoch": 1.635919093131807,
+      "grad_norm": 4.411732196807861,
+      "learning_rate": 2.1427457636675652e-06,
+      "logits/chosen": -14.42602825164795,
+      "logits/rejected": -14.588623046875,
+      "logps/chosen": -1.5284509658813477,
+      "logps/rejected": -1.539262056350708,
+      "loss": 1.609,
+      "odds_ratio_loss": 0.8054282069206238,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -0.152845099568367,
+      "rewards/margins": 0.0010811155661940575,
+      "rewards/rejected": -0.15392622351646423,
+      "sft_loss": 1.5284509658813477,
+      "step": 920
+    },
+    {
+      "epoch": 1.653700822404979,
+      "grad_norm": 1.1233824491500854,
+      "learning_rate": 2.096702293897247e-06,
+      "logits/chosen": -14.348466873168945,
+      "logits/rejected": -14.271936416625977,
+      "logps/chosen": -1.4055891036987305,
+      "logps/rejected": -1.5611364841461182,
+      "loss": 1.4753,
+      "odds_ratio_loss": 0.6970332860946655,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.1405588984489441,
+      "rewards/margins": 0.015554727986454964,
+      "rewards/rejected": -0.1561136394739151,
+      "sft_loss": 1.4055891036987305,
+      "step": 930
+    },
+    {
+      "epoch": 1.6714825516781509,
+      "grad_norm": 1.966296672821045,
+      "learning_rate": 2.0507988632261672e-06,
+      "logits/chosen": -14.399055480957031,
+      "logits/rejected": -14.409767150878906,
+      "logps/chosen": -1.4748440980911255,
+      "logps/rejected": -1.4950190782546997,
+      "loss": 1.5523,
+      "odds_ratio_loss": 0.7747657895088196,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.1474844217300415,
+      "rewards/margins": 0.002017489168792963,
+      "rewards/rejected": -0.14950190484523773,
+      "sft_loss": 1.4748440980911255,
+      "step": 940
+    },
+    {
+      "epoch": 1.6892642809513225,
+      "grad_norm": 2.740964651107788,
+      "learning_rate": 2.005051410934382e-06,
+      "logits/chosen": -14.488139152526855,
+      "logits/rejected": -14.520662307739258,
+      "logps/chosen": -1.5530728101730347,
+      "logps/rejected": -1.6438829898834229,
+      "loss": 1.6259,
+      "odds_ratio_loss": 0.7283372282981873,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.15530726313591003,
+      "rewards/margins": 0.009081020019948483,
+      "rewards/rejected": -0.16438829898834229,
+      "sft_loss": 1.5530728101730347,
+      "step": 950
+    },
+    {
+      "epoch": 1.7070460102244942,
+      "grad_norm": 2.387444496154785,
+      "learning_rate": 1.9594758221407843e-06,
+      "logits/chosen": -14.521173477172852,
+      "logits/rejected": -14.390649795532227,
+      "logps/chosen": -1.3837093114852905,
+      "logps/rejected": -1.492570161819458,
+      "loss": 1.453,
+      "odds_ratio_loss": 0.6927343606948853,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.13837094604969025,
+      "rewards/margins": 0.010886076837778091,
+      "rewards/rejected": -0.14925701916217804,
+      "sft_loss": 1.3837093114852905,
+      "step": 960
+    },
+    {
+      "epoch": 1.724827739497666,
+      "grad_norm": 1.347974181175232,
+      "learning_rate": 1.9140879222872408e-06,
+      "logits/chosen": -14.385538101196289,
+      "logits/rejected": -14.44627857208252,
+      "logps/chosen": -1.4018323421478271,
+      "logps/rejected": -1.4524750709533691,
+      "loss": 1.4772,
+      "odds_ratio_loss": 0.7537996172904968,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.140183225274086,
+      "rewards/margins": 0.0050642904825508595,
+      "rewards/rejected": -0.14524750411510468,
+      "sft_loss": 1.4018323421478271,
+      "step": 970
+    },
+    {
+      "epoch": 1.742609468770838,
+      "grad_norm": 0.7524885535240173,
+      "learning_rate": 1.8689034716434346e-06,
+      "logits/chosen": -14.513898849487305,
+      "logits/rejected": -14.415435791015625,
+      "logps/chosen": -1.4909882545471191,
+      "logps/rejected": -1.4778468608856201,
+      "loss": 1.5681,
+      "odds_ratio_loss": 0.771571159362793,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -0.14909882843494415,
+      "rewards/margins": -0.0013141353847458959,
+      "rewards/rejected": -0.14778469502925873,
+      "sft_loss": 1.4909882545471191,
+      "step": 980
+    },
+    {
+      "epoch": 1.76039119804401,
+      "grad_norm": 1.0324140787124634,
+      "learning_rate": 1.8239381598343576e-06,
+      "logits/chosen": -14.462181091308594,
+      "logits/rejected": -14.475656509399414,
+      "logps/chosen": -1.411113977432251,
+      "logps/rejected": -1.4429736137390137,
+      "loss": 1.4872,
+      "odds_ratio_loss": 0.7608811259269714,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.14111140370368958,
+      "rewards/margins": 0.003185967681929469,
+      "rewards/rejected": -0.14429736137390137,
+      "sft_loss": 1.411113977432251,
+      "step": 990
+    },
+    {
+      "epoch": 1.7781729273171816,
+      "grad_norm": 1.5470359325408936,
+      "learning_rate": 1.779207600392312e-06,
+      "logits/chosen": -14.624302864074707,
+      "logits/rejected": -14.673869132995605,
+      "logps/chosen": -1.4659007787704468,
+      "logps/rejected": -1.5123026371002197,
+      "loss": 1.5401,
+      "odds_ratio_loss": 0.7417243123054504,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.14659008383750916,
+      "rewards/margins": 0.004640174098312855,
+      "rewards/rejected": -0.15123026072978973,
+      "sft_loss": 1.4659007787704468,
+      "step": 1000
+    },
+    {
+      "epoch": 1.7781729273171816,
+      "eval_logits/chosen": -14.472068786621094,
+      "eval_logits/rejected": -14.548614501953125,
+      "eval_logps/chosen": -1.4554640054702759,
+      "eval_logps/rejected": -1.5491538047790527,
+      "eval_loss": 1.5269325971603394,
+      "eval_odds_ratio_loss": 0.7146860361099243,
+      "eval_rewards/accuracies": 0.5019999742507935,
+      "eval_rewards/chosen": -0.14554640650749207,
+      "eval_rewards/margins": 0.00936897937208414,
+      "eval_rewards/rejected": -0.15491539239883423,
+      "eval_runtime": 408.1752,
+      "eval_samples_per_second": 2.45,
+      "eval_sft_loss": 1.4554640054702759,
+      "eval_steps_per_second": 1.225,
+      "step": 1000
+    },
+    {
+      "epoch": 1.7959546565903532,
+      "grad_norm": 2.230626106262207,
+      "learning_rate": 1.7347273253353552e-06,
+      "logits/chosen": -14.449197769165039,
+      "logits/rejected": -14.371235847473145,
+      "logps/chosen": -1.4797186851501465,
+      "logps/rejected": -1.5376956462860107,
+      "loss": 1.5549,
+      "odds_ratio_loss": 0.7520232796669006,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -0.14797186851501465,
+      "rewards/margins": 0.005797683726996183,
+      "rewards/rejected": -0.15376953780651093,
+      "sft_loss": 1.4797186851501465,
+      "step": 1010
+    },
+    {
+      "epoch": 1.8137363858635251,
+      "grad_norm": 3.5348658561706543,
+      "learning_rate": 1.690512779774029e-06,
+      "logits/chosen": -14.508018493652344,
+      "logits/rejected": -14.555743217468262,
+      "logps/chosen": -1.4630930423736572,
+      "logps/rejected": -1.5662147998809814,
+      "loss": 1.5345,
+      "odds_ratio_loss": 0.714568018913269,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.14630930125713348,
+      "rewards/margins": 0.010312167927622795,
+      "rewards/rejected": -0.15662148594856262,
+      "sft_loss": 1.4630930423736572,
+      "step": 1020
+    },
+    {
+      "epoch": 1.831518115136697,
+      "grad_norm": 1.3148037195205688,
+      "learning_rate": 1.6465793165482838e-06,
+      "logits/chosen": -14.601341247558594,
+      "logits/rejected": -14.604815483093262,
+      "logps/chosen": -1.3863935470581055,
+      "logps/rejected": -1.5016463994979858,
+      "loss": 1.4553,
+      "odds_ratio_loss": 0.6889203786849976,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.13863936066627502,
+      "rewards/margins": 0.011525283567607403,
+      "rewards/rejected": -0.1501646339893341,
+      "sft_loss": 1.3863935470581055,
+      "step": 1030
+    },
+    {
+      "epoch": 1.849299844409869,
+      "grad_norm": 1.944263219833374,
+      "learning_rate": 1.6029421908964305e-06,
+      "logits/chosen": -14.364709854125977,
+      "logits/rejected": -14.301609992980957,
+      "logps/chosen": -1.444108247756958,
+      "logps/rejected": -1.735345482826233,
+      "loss": 1.5112,
+      "odds_ratio_loss": 0.6704747080802917,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.1444108486175537,
+      "rewards/margins": 0.029123712331056595,
+      "rewards/rejected": -0.1735345423221588,
+      "sft_loss": 1.444108247756958,
+      "step": 1040
+    },
+    {
+      "epoch": 1.8670815736830408,
+      "grad_norm": 3.4116580486297607,
+      "learning_rate": 1.559616555157985e-06,
+      "logits/chosen": -14.63707160949707,
+      "logits/rejected": -14.487701416015625,
+      "logps/chosen": -1.4554407596588135,
+      "logps/rejected": -1.532787561416626,
+      "loss": 1.529,
+      "odds_ratio_loss": 0.7356154322624207,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.14554408192634583,
+      "rewards/margins": 0.007734687067568302,
+      "rewards/rejected": -0.15327878296375275,
+      "sft_loss": 1.4554407596588135,
+      "step": 1050
+    },
+    {
+      "epoch": 1.8848633029562125,
+      "grad_norm": 1.2246363162994385,
+      "learning_rate": 1.516617453512252e-06,
+      "logits/chosen": -14.424253463745117,
+      "logits/rejected": -14.5010347366333,
+      "logps/chosen": -1.5207793712615967,
+      "logps/rejected": -1.531063437461853,
+      "loss": 1.5985,
+      "odds_ratio_loss": 0.7770354151725769,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.15207794308662415,
+      "rewards/margins": 0.0010283945593982935,
+      "rewards/rejected": -0.15310634672641754,
+      "sft_loss": 1.5207793712615967,
+      "step": 1060
+    },
+    {
+      "epoch": 1.9026450322293842,
+      "grad_norm": 2.65510630607605,
+      "learning_rate": 1.473959816754449e-06,
+      "logits/chosen": -14.302286148071289,
+      "logits/rejected": -14.37928581237793,
+      "logps/chosen": -1.3779585361480713,
+      "logps/rejected": -1.444608211517334,
+      "loss": 1.4506,
+      "odds_ratio_loss": 0.7259268164634705,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.1377958357334137,
+      "rewards/margins": 0.00666497927159071,
+      "rewards/rejected": -0.14446081221103668,
+      "sft_loss": 1.3779585361480713,
+      "step": 1070
+    },
+    {
+      "epoch": 1.920426761502556,
+      "grad_norm": 1.5242667198181152,
+      "learning_rate": 1.4316584571112213e-06,
+      "logits/chosen": -14.791394233703613,
+      "logits/rejected": -14.659652709960938,
+      "logps/chosen": -1.451185941696167,
+      "logps/rejected": -1.5176652669906616,
+      "loss": 1.5255,
+      "odds_ratio_loss": 0.7434892058372498,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.1451185941696167,
+      "rewards/margins": 0.0066479267552495,
+      "rewards/rejected": -0.15176650881767273,
+      "sft_loss": 1.451185941696167,
+      "step": 1080
+    },
+    {
+      "epoch": 1.938208490775728,
+      "grad_norm": 1.3730899095535278,
+      "learning_rate": 1.389728063097306e-06,
+      "logits/chosen": -14.725341796875,
+      "logits/rejected": -14.647857666015625,
+      "logps/chosen": -1.4444434642791748,
+      "logps/rejected": -1.6104562282562256,
+      "loss": 1.5134,
+      "odds_ratio_loss": 0.6893559098243713,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.14444434642791748,
+      "rewards/margins": 0.016601284965872765,
+      "rewards/rejected": -0.161045640707016,
+      "sft_loss": 1.4444434642791748,
+      "step": 1090
+    },
+    {
+      "epoch": 1.9559902200488999,
+      "grad_norm": 0.9227787852287292,
+      "learning_rate": 1.348183194415179e-06,
+      "logits/chosen": -14.495756149291992,
+      "logits/rejected": -14.327432632446289,
+      "logps/chosen": -1.419993281364441,
+      "logps/rejected": -1.5734179019927979,
+      "loss": 1.4884,
+      "odds_ratio_loss": 0.6841065883636475,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.14199933409690857,
+      "rewards/margins": 0.0153424721211195,
+      "rewards/rejected": -0.15734180808067322,
+      "sft_loss": 1.419993281364441,
+      "step": 1100
+    },
+    {
+      "epoch": 1.9737719493220716,
+      "grad_norm": 0.9474197626113892,
+      "learning_rate": 1.3070382768994015e-06,
+      "logits/chosen": -14.650873184204102,
+      "logits/rejected": -14.58227825164795,
+      "logps/chosen": -1.4364961385726929,
+      "logps/rejected": -1.5456112623214722,
+      "loss": 1.5057,
+      "odds_ratio_loss": 0.6923903226852417,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.1436496078968048,
+      "rewards/margins": 0.010911517776548862,
+      "rewards/rejected": -0.1545611321926117,
+      "sft_loss": 1.4364961385726929,
+      "step": 1110
+    },
+    {
+      "epoch": 1.9915536785952432,
+      "grad_norm": 1.042833924293518,
+      "learning_rate": 1.2663075975074746e-06,
+      "logits/chosen": -14.496850967407227,
+      "logits/rejected": -14.485113143920898,
+      "logps/chosen": -1.4912959337234497,
+      "logps/rejected": -1.6449800729751587,
+      "loss": 1.5613,
+      "odds_ratio_loss": 0.699638307094574,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.14912959933280945,
+      "rewards/margins": 0.015368418768048286,
+      "rewards/rejected": -0.1644980013370514,
+      "sft_loss": 1.4912959337234497,
+      "step": 1120
+    },
+    {
+      "epoch": 2.009335407868415,
+      "grad_norm": 2.740922689437866,
+      "learning_rate": 1.2260052993589034e-06,
+      "logits/chosen": -14.43859577178955,
+      "logits/rejected": -14.43925952911377,
+      "logps/chosen": -1.535914659500122,
+      "logps/rejected": -1.5225673913955688,
+      "loss": 1.615,
+      "odds_ratio_loss": 0.7913249731063843,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.15359148383140564,
+      "rewards/margins": -0.0013347306521609426,
+      "rewards/rejected": -0.15225675702095032,
+      "sft_loss": 1.535914659500122,
+      "step": 1130
+    },
+    {
+      "epoch": 2.027117137141587,
+      "grad_norm": 0.7980636954307556,
+      "learning_rate": 1.1861453768242099e-06,
+      "logits/chosen": -14.416345596313477,
+      "logits/rejected": -14.40721321105957,
+      "logps/chosen": -1.4432775974273682,
+      "logps/rejected": -1.5133426189422607,
+      "loss": 1.5179,
+      "odds_ratio_loss": 0.7463361620903015,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.14432775974273682,
+      "rewards/margins": 0.007006504572927952,
+      "rewards/rejected": -0.1513342708349228,
+      "sft_loss": 1.4432775974273682,
+      "step": 1140
+    },
+    {
+      "epoch": 2.044898866414759,
+      "grad_norm": 5.74677038192749,
+      "learning_rate": 1.1467416706655982e-06,
+      "logits/chosen": -14.52302074432373,
+      "logits/rejected": -14.662378311157227,
+      "logps/chosen": -1.5389671325683594,
+      "logps/rejected": -1.645978569984436,
+      "loss": 1.6125,
+      "odds_ratio_loss": 0.7352578043937683,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.15389671921730042,
+      "rewards/margins": 0.010701162740588188,
+      "rewards/rejected": -0.16459788382053375,
+      "sft_loss": 1.5389671325683594,
+      "step": 1150
+    },
+    {
+      "epoch": 2.062680595687931,
+      "grad_norm": 0.986960232257843,
+      "learning_rate": 1.1078078632309559e-06,
+      "logits/chosen": -14.417854309082031,
+      "logits/rejected": -14.478330612182617,
+      "logps/chosen": -1.4349641799926758,
+      "logps/rejected": -1.5250955820083618,
+      "loss": 1.5053,
+      "odds_ratio_loss": 0.7031277418136597,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.14349642395973206,
+      "rewards/margins": 0.009013157337903976,
+      "rewards/rejected": -0.15250957012176514,
+      "sft_loss": 1.4349641799926758,
+      "step": 1160
+    },
+    {
+      "epoch": 2.0804623249611023,
+      "grad_norm": 2.110325813293457,
+      "learning_rate": 1.0693574737028627e-06,
+      "logits/chosen": -14.4419584274292,
+      "logits/rejected": -14.437457084655762,
+      "logps/chosen": -1.5028114318847656,
+      "logps/rejected": -1.5467736721038818,
+      "loss": 1.5773,
+      "odds_ratio_loss": 0.7452652454376221,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.1502811312675476,
+      "rewards/margins": 0.004396246280521154,
+      "rewards/rejected": -0.1546773910522461,
+      "sft_loss": 1.5028114318847656,
+      "step": 1170
+    },
+    {
+      "epoch": 2.098244054234274,
+      "grad_norm": 1.9610004425048828,
+      "learning_rate": 1.0314038534042586e-06,
+      "logits/chosen": -14.581426620483398,
+      "logits/rejected": -14.440699577331543,
+      "logps/chosen": -1.3876335620880127,
+      "logps/rejected": -1.5056065320968628,
+      "loss": 1.4589,
+      "odds_ratio_loss": 0.7131034731864929,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.13876333832740784,
+      "rewards/margins": 0.011797326616942883,
+      "rewards/rejected": -0.1505606770515442,
+      "sft_loss": 1.3876335620880127,
+      "step": 1180
+    },
+    {
+      "epoch": 2.116025783507446,
+      "grad_norm": 1.4545583724975586,
+      "learning_rate": 9.939601811623946e-07,
+      "logits/chosen": -14.581433296203613,
+      "logits/rejected": -14.601740837097168,
+      "logps/chosen": -1.422616720199585,
+      "logps/rejected": -1.5079143047332764,
+      "loss": 1.4943,
+      "odds_ratio_loss": 0.7167633771896362,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.14226169884204865,
+      "rewards/margins": 0.00852974783629179,
+      "rewards/rejected": -0.15079143643379211,
+      "sft_loss": 1.422616720199585,
+      "step": 1190
+    },
+    {
+      "epoch": 2.133807512780618,
+      "grad_norm": 1.6818853616714478,
+      "learning_rate": 9.570394587326825e-07,
+      "logits/chosen": -14.631085395812988,
+      "logits/rejected": -14.470014572143555,
+      "logps/chosen": -1.4276001453399658,
+      "logps/rejected": -1.5927342176437378,
+      "loss": 1.4962,
+      "odds_ratio_loss": 0.6859818696975708,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.1427600234746933,
+      "rewards/margins": 0.016513412818312645,
+      "rewards/rejected": -0.1592734307050705,
+      "sft_loss": 1.4276001453399658,
+      "step": 1200
+    },
+    {
+      "epoch": 2.15158924205379,
+      "grad_norm": 0.9193028807640076,
+      "learning_rate": 9.206545062840302e-07,
+      "logits/chosen": -14.681951522827148,
+      "logits/rejected": -14.49401569366455,
+      "logps/chosen": -1.4501049518585205,
+      "logps/rejected": -1.550287127494812,
+      "loss": 1.5204,
+      "odds_ratio_loss": 0.7033563256263733,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.14501050114631653,
+      "rewards/margins": 0.010018205270171165,
+      "rewards/rejected": -0.15502868592739105,
+      "sft_loss": 1.4501049518585205,
+      "step": 1210
+    },
+    {
+      "epoch": 2.1693709713269618,
+      "grad_norm": 0.9245877265930176,
+      "learning_rate": 8.848179579472285e-07,
+      "logits/chosen": -14.563427925109863,
+      "logits/rejected": -14.575236320495605,
+      "logps/chosen": -1.3411179780960083,
+      "logps/rejected": -1.3986274003982544,
+      "loss": 1.4132,
+      "odds_ratio_loss": 0.7204734683036804,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.13411179184913635,
+      "rewards/margins": 0.005750957410782576,
+      "rewards/rejected": -0.13986274600028992,
+      "sft_loss": 1.3411179780960083,
+      "step": 1220
+    },
+    {
+      "epoch": 2.1871527006001332,
+      "grad_norm": 4.244547367095947,
+      "learning_rate": 8.495422574279403e-07,
+      "logits/chosen": -14.301058769226074,
+      "logits/rejected": -14.25413703918457,
+      "logps/chosen": -1.370896339416504,
+      "logps/rejected": -1.5229545831680298,
+      "loss": 1.4426,
+      "odds_ratio_loss": 0.7167730927467346,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.13708963990211487,
+      "rewards/margins": 0.015205820091068745,
+      "rewards/rejected": -0.15229545533657074,
+      "sft_loss": 1.370896339416504,
+      "step": 1230
+    },
+    {
+      "epoch": 2.204934429873305,
+      "grad_norm": 1.5995064973831177,
+      "learning_rate": 8.148396536858063e-07,
+      "logits/chosen": -14.522483825683594,
+      "logits/rejected": -14.575413703918457,
+      "logps/chosen": -1.506316065788269,
+      "logps/rejected": -1.6317975521087646,
+      "loss": 1.578,
+      "odds_ratio_loss": 0.7170661687850952,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.1506316065788269,
+      "rewards/margins": 0.012548169121146202,
+      "rewards/rejected": -0.16317978501319885,
+      "sft_loss": 1.506316065788269,
+      "step": 1240
+    },
+    {
+      "epoch": 2.222716159146477,
+      "grad_norm": 2.302438497543335,
+      "learning_rate": 7.807221966811815e-07,
+      "logits/chosen": -14.514368057250977,
+      "logits/rejected": -14.467686653137207,
+      "logps/chosen": -1.3972229957580566,
+      "logps/rejected": -1.503300666809082,
+      "loss": 1.4695,
+      "odds_ratio_loss": 0.7227479219436646,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.1397223025560379,
+      "rewards/margins": 0.010607777163386345,
+      "rewards/rejected": -0.1503300666809082,
+      "sft_loss": 1.3972229957580566,
+      "step": 1250
+    },
+    {
+      "epoch": 2.240497888419649,
+      "grad_norm": 2.491163492202759,
+      "learning_rate": 7.47201733190962e-07,
+      "logits/chosen": -14.408790588378906,
+      "logits/rejected": -14.414535522460938,
+      "logps/chosen": -1.4305273294448853,
+      "logps/rejected": -1.489025354385376,
+      "loss": 1.5047,
+      "odds_ratio_loss": 0.7419496774673462,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.14305274188518524,
+      "rewards/margins": 0.005849803332239389,
+      "rewards/rejected": -0.1489025503396988,
+      "sft_loss": 1.4305273294448853,
+      "step": 1260
+    },
+    {
+      "epoch": 2.258279617692821,
+      "grad_norm": 1.4214234352111816,
+      "learning_rate": 7.142899026949721e-07,
+      "logits/chosen": -14.524235725402832,
+      "logits/rejected": -14.52795696258545,
+      "logps/chosen": -1.4059574604034424,
+      "logps/rejected": -1.5294010639190674,
+      "loss": 1.4755,
+      "odds_ratio_loss": 0.6952685117721558,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.1405957192182541,
+      "rewards/margins": 0.012344368733465672,
+      "rewards/rejected": -0.15294012427330017,
+      "sft_loss": 1.4059574604034424,
+      "step": 1270
+    },
+    {
+      "epoch": 2.2760613469659923,
+      "grad_norm": 5.792439937591553,
+      "learning_rate": 6.819981333343273e-07,
+      "logits/chosen": -14.282297134399414,
+      "logits/rejected": -14.3455171585083,
+      "logps/chosen": -1.3890929222106934,
+      "logps/rejected": -1.5048874616622925,
+      "loss": 1.4613,
+      "odds_ratio_loss": 0.721687912940979,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.13890929520130157,
+      "rewards/margins": 0.011579466983675957,
+      "rewards/rejected": -0.1504887491464615,
+      "sft_loss": 1.3890929222106934,
+      "step": 1280
+    },
+    {
+      "epoch": 2.293843076239164,
+      "grad_norm": 1.8430945873260498,
+      "learning_rate": 6.503376379431839e-07,
+      "logits/chosen": -14.595362663269043,
+      "logits/rejected": -14.600665092468262,
+      "logps/chosen": -1.5411078929901123,
+      "logps/rejected": -1.4967674016952515,
+      "loss": 1.6191,
+      "odds_ratio_loss": 0.7797070741653442,
+      "rewards/accuracies": 0.41874998807907104,
+      "rewards/chosen": -0.15411078929901123,
+      "rewards/margins": -0.004434044472873211,
+      "rewards/rejected": -0.14967674016952515,
+      "sft_loss": 1.5411078929901123,
+      "step": 1290
+    },
+    {
+      "epoch": 2.311624805512336,
+      "grad_norm": 1.6006221771240234,
+      "learning_rate": 6.193194101552502e-07,
+      "logits/chosen": -14.458605766296387,
+      "logits/rejected": -14.19080638885498,
+      "logps/chosen": -1.4712187051773071,
+      "logps/rejected": -1.5340622663497925,
+      "loss": 1.5413,
+      "odds_ratio_loss": 0.7012200951576233,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.147121861577034,
+      "rewards/margins": 0.006284369621425867,
+      "rewards/rejected": -0.15340623259544373,
+      "sft_loss": 1.4712187051773071,
+      "step": 1300
+    },
+    {
+      "epoch": 2.329406534785508,
+      "grad_norm": 2.382812261581421,
+      "learning_rate": 5.889542205864083e-07,
+      "logits/chosen": -14.490499496459961,
+      "logits/rejected": -14.431979179382324,
+      "logps/chosen": -1.5115288496017456,
+      "logps/rejected": -1.508636474609375,
+      "loss": 1.5893,
+      "odds_ratio_loss": 0.7778019309043884,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.15115289390087128,
+      "rewards/margins": -0.0002892397460527718,
+      "rewards/rejected": -0.1508636474609375,
+      "sft_loss": 1.5115288496017456,
+      "step": 1310
+    },
+    {
+      "epoch": 2.34718826405868,
+      "grad_norm": 1.2705732583999634,
+      "learning_rate": 5.592526130947862e-07,
+      "logits/chosen": -14.570259094238281,
+      "logits/rejected": -14.508715629577637,
+      "logps/chosen": -1.4529051780700684,
+      "logps/rejected": -1.5590078830718994,
+      "loss": 1.5284,
+      "odds_ratio_loss": 0.7552819848060608,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.14529050886631012,
+      "rewards/margins": 0.01061027031391859,
+      "rewards/rejected": -0.15590079128742218,
+      "sft_loss": 1.4529051780700684,
+      "step": 1320
+    },
+    {
+      "epoch": 2.3649699933318518,
+      "grad_norm": 1.9159420728683472,
+      "learning_rate": 5.302249011195507e-07,
+      "logits/chosen": -14.313755989074707,
+      "logits/rejected": -14.381486892700195,
+      "logps/chosen": -1.3798234462738037,
+      "logps/rejected": -1.4174072742462158,
+      "loss": 1.4519,
+      "odds_ratio_loss": 0.7206598520278931,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.1379823386669159,
+      "rewards/margins": 0.0037583820521831512,
+      "rewards/rejected": -0.14174072444438934,
+      "sft_loss": 1.3798234462738037,
+      "step": 1330
+    },
+    {
+      "epoch": 2.382751722605023,
+      "grad_norm": 3.4968583583831787,
+      "learning_rate": 5.018811640997307e-07,
+      "logits/chosen": -14.515790939331055,
+      "logits/rejected": -14.633216857910156,
+      "logps/chosen": -1.471914529800415,
+      "logps/rejected": -1.6871318817138672,
+      "loss": 1.5372,
+      "odds_ratio_loss": 0.6527343392372131,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.14719144999980927,
+      "rewards/margins": 0.02152174338698387,
+      "rewards/rejected": -0.16871318221092224,
+      "sft_loss": 1.471914529800415,
+      "step": 1340
+    },
+    {
+      "epoch": 2.400533451878195,
+      "grad_norm": 1.3940002918243408,
+      "learning_rate": 4.7423124397427105e-07,
+      "logits/chosen": -14.439651489257812,
+      "logits/rejected": -14.579530715942383,
+      "logps/chosen": -1.4648997783660889,
+      "logps/rejected": -1.4503980875015259,
+      "loss": 1.5422,
+      "odds_ratio_loss": 0.773194432258606,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.1464899778366089,
+      "rewards/margins": -0.0014501826371997595,
+      "rewards/rejected": -0.14503981173038483,
+      "sft_loss": 1.4648997783660889,
+      "step": 1350
+    },
+    {
+      "epoch": 2.418315181151367,
+      "grad_norm": 1.0056315660476685,
+      "learning_rate": 4.472847417645787e-07,
+      "logits/chosen": -14.650228500366211,
+      "logits/rejected": -14.57947826385498,
+      "logps/chosen": -1.3967421054840088,
+      "logps/rejected": -1.6349856853485107,
+      "loss": 1.4623,
+      "odds_ratio_loss": 0.6557605266571045,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.13967421650886536,
+      "rewards/margins": 0.023824386298656464,
+      "rewards/rejected": -0.16349859535694122,
+      "sft_loss": 1.3967421054840088,
+      "step": 1360
+    },
+    {
+      "epoch": 2.436096910424539,
+      "grad_norm": 0.7953612804412842,
+      "learning_rate": 4.210510142406993e-07,
+      "logits/chosen": -14.647814750671387,
+      "logits/rejected": -14.501565933227539,
+      "logps/chosen": -1.4687623977661133,
+      "logps/rejected": -1.5674188137054443,
+      "loss": 1.5411,
+      "odds_ratio_loss": 0.723659873008728,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.1468762457370758,
+      "rewards/margins": 0.009865625761449337,
+      "rewards/rejected": -0.15674187242984772,
+      "sft_loss": 1.4687623977661133,
+      "step": 1370
+    },
+    {
+      "epoch": 2.4538786396977104,
+      "grad_norm": 1.0436296463012695,
+      "learning_rate": 3.9553917067232966e-07,
+      "logits/chosen": -14.456913948059082,
+      "logits/rejected": -14.33137321472168,
+      "logps/chosen": -1.451188564300537,
+      "logps/rejected": -1.5067174434661865,
+      "loss": 1.5265,
+      "odds_ratio_loss": 0.7529994249343872,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.1451188623905182,
+      "rewards/margins": 0.005552901886403561,
+      "rewards/rejected": -0.15067176520824432,
+      "sft_loss": 1.451188564300537,
+      "step": 1380
+    },
+    {
+      "epoch": 2.4716603689708823,
+      "grad_norm": 1.177869200706482,
+      "learning_rate": 3.707580696657509e-07,
+      "logits/chosen": -14.569585800170898,
+      "logits/rejected": -14.398585319519043,
+      "logps/chosen": -1.4260759353637695,
+      "logps/rejected": -1.4303407669067383,
+      "loss": 1.5,
+      "odds_ratio_loss": 0.7389937043190002,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.14260759949684143,
+      "rewards/margins": 0.0004264971357770264,
+      "rewards/rejected": -0.14303407073020935,
+      "sft_loss": 1.4260759353637695,
+      "step": 1390
+    },
+    {
+      "epoch": 2.489442098244054,
+      "grad_norm": 1.4175564050674438,
+      "learning_rate": 3.4671631608781815e-07,
+      "logits/chosen": -14.617632865905762,
+      "logits/rejected": -14.494784355163574,
+      "logps/chosen": -1.4826141595840454,
+      "logps/rejected": -1.5348364114761353,
+      "loss": 1.5593,
+      "odds_ratio_loss": 0.7670896649360657,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.1482614129781723,
+      "rewards/margins": 0.005222243722528219,
+      "rewards/rejected": -0.15348365902900696,
+      "sft_loss": 1.4826141595840454,
+      "step": 1400
+    },
+    {
+      "epoch": 2.507223827517226,
+      "grad_norm": 1.6863813400268555,
+      "learning_rate": 3.234222580780405e-07,
+      "logits/chosen": -14.408329963684082,
+      "logits/rejected": -14.414642333984375,
+      "logps/chosen": -1.4218008518218994,
+      "logps/rejected": -1.4535105228424072,
+      "loss": 1.4962,
+      "odds_ratio_loss": 0.7441353797912598,
+      "rewards/accuracies": 0.4312500059604645,
+      "rewards/chosen": -0.14218010008335114,
+      "rewards/margins": 0.0031709708273410797,
+      "rewards/rejected": -0.1453510820865631,
+      "sft_loss": 1.4218008518218994,
+      "step": 1410
+    },
+    {
+      "epoch": 2.525005556790398,
+      "grad_norm": 4.5228376388549805,
+      "learning_rate": 3.0088398414982375e-07,
+      "logits/chosen": -14.276860237121582,
+      "logits/rejected": -14.423869132995605,
+      "logps/chosen": -1.5565452575683594,
+      "logps/rejected": -1.6457360982894897,
+      "loss": 1.6331,
+      "odds_ratio_loss": 0.7656282782554626,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.15565453469753265,
+      "rewards/margins": 0.008919079788029194,
+      "rewards/rejected": -0.16457359492778778,
+      "sft_loss": 1.5565452575683594,
+      "step": 1420
+    },
+    {
+      "epoch": 2.54278728606357,
+      "grad_norm": 6.204384803771973,
+      "learning_rate": 2.7910932038184487e-07,
+      "logits/chosen": -14.392138481140137,
+      "logits/rejected": -14.114950180053711,
+      "logps/chosen": -1.4198369979858398,
+      "logps/rejected": -1.5630581378936768,
+      "loss": 1.4895,
+      "odds_ratio_loss": 0.6968662142753601,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.14198370277881622,
+      "rewards/margins": 0.014322122558951378,
+      "rewards/rejected": -0.15630581974983215,
+      "sft_loss": 1.4198369979858398,
+      "step": 1430
+    },
+    {
+      "epoch": 2.5605690153367417,
+      "grad_norm": 2.048856258392334,
+      "learning_rate": 2.5810582770057325e-07,
+      "logits/chosen": -14.456823348999023,
+      "logits/rejected": -14.554224967956543,
+      "logps/chosen": -1.400504469871521,
+      "logps/rejected": -1.4313546419143677,
+      "loss": 1.4769,
+      "odds_ratio_loss": 0.764117419719696,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.14005044102668762,
+      "rewards/margins": 0.0030850153416395187,
+      "rewards/rejected": -0.1431354582309723,
+      "sft_loss": 1.400504469871521,
+      "step": 1440
+    },
+    {
+      "epoch": 2.578350744609913,
+      "grad_norm": 1.2424274682998657,
+      "learning_rate": 2.3788079925484402e-07,
+      "logits/chosen": -14.654197692871094,
+      "logits/rejected": -14.54980754852295,
+      "logps/chosen": -1.4363069534301758,
+      "logps/rejected": -1.4395811557769775,
+      "loss": 1.5116,
+      "odds_ratio_loss": 0.7526046633720398,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -0.14363068342208862,
+      "rewards/margins": 0.0003274245245847851,
+      "rewards/rejected": -0.14395812153816223,
+      "sft_loss": 1.4363069534301758,
+      "step": 1450
+    },
+    {
+      "epoch": 2.596132473883085,
+      "grad_norm": 1.602452039718628,
+      "learning_rate": 2.1844125788342661e-07,
+      "logits/chosen": -14.462437629699707,
+      "logits/rejected": -14.348231315612793,
+      "logps/chosen": -1.4353057146072388,
+      "logps/rejected": -1.6425654888153076,
+      "loss": 1.5064,
+      "odds_ratio_loss": 0.711114227771759,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.14353057742118835,
+      "rewards/margins": 0.020725984126329422,
+      "rewards/rejected": -0.16425655782222748,
+      "sft_loss": 1.4353057146072388,
+      "step": 1460
+    },
+    {
+      "epoch": 2.613914203156257,
+      "grad_norm": 1.1172051429748535,
+      "learning_rate": 1.9979395367644428e-07,
+      "logits/chosen": -14.612176895141602,
+      "logits/rejected": -14.652105331420898,
+      "logps/chosen": -1.3911590576171875,
+      "logps/rejected": -1.5478867292404175,
+      "loss": 1.4586,
+      "odds_ratio_loss": 0.674477756023407,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.13911592960357666,
+      "rewards/margins": 0.01567276194691658,
+      "rewards/rejected": -0.15478867292404175,
+      "sft_loss": 1.3911590576171875,
+      "step": 1470
+    },
+    {
+      "epoch": 2.631695932429429,
+      "grad_norm": 1.9138628244400024,
+      "learning_rate": 1.81945361631512e-07,
+      "logits/chosen": -14.578519821166992,
+      "logits/rejected": -14.719879150390625,
+      "logps/chosen": -1.458441972732544,
+      "logps/rejected": -1.4730455875396729,
+      "loss": 1.5353,
+      "odds_ratio_loss": 0.7685132622718811,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.14584419131278992,
+      "rewards/margins": 0.0014603782910853624,
+      "rewards/rejected": -0.14730457961559296,
+      "sft_loss": 1.458441972732544,
+      "step": 1480
+    },
+    {
+      "epoch": 2.6494776617026004,
+      "grad_norm": 2.06256103515625,
+      "learning_rate": 1.6490167940538343e-07,
+      "logits/chosen": -14.520217895507812,
+      "logits/rejected": -14.437647819519043,
+      "logps/chosen": -1.4276152849197388,
+      "logps/rejected": -1.4981720447540283,
+      "loss": 1.5003,
+      "odds_ratio_loss": 0.7272015810012817,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.14276152849197388,
+      "rewards/margins": 0.007055687252432108,
+      "rewards/rejected": -0.14981719851493835,
+      "sft_loss": 1.4276152849197388,
+      "step": 1490
+    },
+    {
+      "epoch": 2.6672593909757722,
+      "grad_norm": 1.2633237838745117,
+      "learning_rate": 1.4866882516191339e-07,
+      "logits/chosen": -14.439382553100586,
+      "logits/rejected": -14.5440034866333,
+      "logps/chosen": -1.4182121753692627,
+      "logps/rejected": -1.5234054327011108,
+      "loss": 1.4914,
+      "odds_ratio_loss": 0.7320746779441833,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.1418212354183197,
+      "rewards/margins": 0.01051931269466877,
+      "rewards/rejected": -0.15234054625034332,
+      "sft_loss": 1.4182121753692627,
+      "step": 1500
+    },
+    {
+      "epoch": 2.6672593909757722,
+      "eval_logits/chosen": -14.466456413269043,
+      "eval_logits/rejected": -14.543242454528809,
+      "eval_logps/chosen": -1.44402277469635,
+      "eval_logps/rejected": -1.5389362573623657,
+      "eval_loss": 1.5154520273208618,
+      "eval_odds_ratio_loss": 0.7142924666404724,
+      "eval_rewards/accuracies": 0.5090000033378601,
+      "eval_rewards/chosen": -0.14440228044986725,
+      "eval_rewards/margins": 0.00949135422706604,
+      "eval_rewards/rejected": -0.15389364957809448,
+      "eval_runtime": 411.9433,
+      "eval_samples_per_second": 2.428,
+      "eval_sft_loss": 1.44402277469635,
+      "eval_steps_per_second": 1.214,
+      "step": 1500
+    },
+    {
+      "epoch": 2.685041120248944,
+      "grad_norm": 2.1881158351898193,
+      "learning_rate": 1.3325243551706057e-07,
+      "logits/chosen": -14.318315505981445,
+      "logits/rejected": -14.482050895690918,
+      "logps/chosen": -1.5127537250518799,
+      "logps/rejected": -1.706903100013733,
+      "loss": 1.5833,
+      "odds_ratio_loss": 0.7053884267807007,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.1512753665447235,
+      "rewards/margins": 0.01941494271159172,
+      "rewards/rejected": -0.17069032788276672,
+      "sft_loss": 1.5127537250518799,
+      "step": 1510
+    },
+    {
+      "epoch": 2.702822849522116,
+      "grad_norm": 1.7814643383026123,
+      "learning_rate": 1.1865786358165737e-07,
+      "logits/chosen": -14.374513626098633,
+      "logits/rejected": -14.638038635253906,
+      "logps/chosen": -1.4067102670669556,
+      "logps/rejected": -1.4657062292099,
+      "loss": 1.4794,
+      "odds_ratio_loss": 0.727219820022583,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.1406710147857666,
+      "rewards/margins": 0.005899603012949228,
+      "rewards/rejected": -0.14657063782215118,
+      "sft_loss": 1.4067102670669556,
+      "step": 1520
+    },
+    {
+      "epoch": 2.720604578795288,
+      "grad_norm": 4.55156135559082,
+      "learning_rate": 1.0489017710262311e-07,
+      "logits/chosen": -14.426725387573242,
+      "logits/rejected": -14.426490783691406,
+      "logps/chosen": -1.4883832931518555,
+      "logps/rejected": -1.6930698156356812,
+      "loss": 1.5615,
+      "odds_ratio_loss": 0.731151282787323,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.1488383412361145,
+      "rewards/margins": 0.020468655973672867,
+      "rewards/rejected": -0.16930699348449707,
+      "sft_loss": 1.4883832931518555,
+      "step": 1530
+    },
+    {
+      "epoch": 2.73838630806846,
+      "grad_norm": 1.255363941192627,
+      "learning_rate": 9.195415670326446e-08,
+      "logits/chosen": -14.457158088684082,
+      "logits/rejected": -14.496696472167969,
+      "logps/chosen": -1.4223277568817139,
+      "logps/rejected": -1.5442826747894287,
+      "loss": 1.4941,
+      "odds_ratio_loss": 0.7178906202316284,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.1422327756881714,
+      "rewards/margins": 0.012195492163300514,
+      "rewards/rejected": -0.15442825853824615,
+      "sft_loss": 1.4223277568817139,
+      "step": 1540
+    },
+    {
+      "epoch": 2.7561680373416317,
+      "grad_norm": 1.5739907026290894,
+      "learning_rate": 7.985429422327384e-08,
+      "logits/chosen": -14.502885818481445,
+      "logits/rejected": -14.488322257995605,
+      "logps/chosen": -1.3892616033554077,
+      "logps/rejected": -1.461104154586792,
+      "loss": 1.4636,
+      "odds_ratio_loss": 0.743794322013855,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.138926163315773,
+      "rewards/margins": 0.007184277288615704,
+      "rewards/rejected": -0.1461104303598404,
+      "sft_loss": 1.3892616033554077,
+      "step": 1550
+    },
+    {
+      "epoch": 2.773949766614803,
+      "grad_norm": 1.2010091543197632,
+      "learning_rate": 6.859479115900818e-08,
+      "logits/chosen": -14.52282428741455,
+      "logits/rejected": -14.561497688293457,
+      "logps/chosen": -1.4343992471694946,
+      "logps/rejected": -1.5450398921966553,
+      "loss": 1.5068,
+      "odds_ratio_loss": 0.7239341735839844,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.14343991875648499,
+      "rewards/margins": 0.011064080521464348,
+      "rewards/rejected": -0.15450401604175568,
+      "sft_loss": 1.4343992471694946,
+      "step": 1560
+    },
+    {
+      "epoch": 2.791731495887975,
+      "grad_norm": 4.445690631866455,
+      "learning_rate": 5.817955720457902e-08,
+      "logits/chosen": -14.513537406921387,
+      "logits/rejected": -14.568794250488281,
+      "logps/chosen": -1.4571216106414795,
+      "logps/rejected": -1.440791130065918,
+      "loss": 1.5361,
+      "odds_ratio_loss": 0.7899585962295532,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.14571216702461243,
+      "rewards/margins": -0.0016330400248989463,
+      "rewards/rejected": -0.14407911896705627,
+      "sft_loss": 1.4571216106414795,
+      "step": 1570
+    },
+    {
+      "epoch": 2.809513225161147,
+      "grad_norm": 3.594475030899048,
+      "learning_rate": 4.861220889427199e-08,
+      "logits/chosen": -14.410969734191895,
+      "logits/rejected": -14.302682876586914,
+      "logps/chosen": -1.4560983180999756,
+      "logps/rejected": -1.5099445581436157,
+      "loss": 1.5329,
+      "odds_ratio_loss": 0.7683452367782593,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.14560985565185547,
+      "rewards/margins": 0.005384599789977074,
+      "rewards/rejected": -0.1509944498538971,
+      "sft_loss": 1.4560983180999756,
+      "step": 1580
+    },
+    {
+      "epoch": 2.827294954434319,
+      "grad_norm": 0.9240034818649292,
+      "learning_rate": 3.9896068346758074e-08,
+      "logits/chosen": -14.375950813293457,
+      "logits/rejected": -14.428239822387695,
+      "logps/chosen": -1.4449069499969482,
+      "logps/rejected": -1.507812738418579,
+      "loss": 1.519,
+      "odds_ratio_loss": 0.7404919266700745,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.14449068903923035,
+      "rewards/margins": 0.0062905787490308285,
+      "rewards/rejected": -0.1507812738418579,
+      "sft_loss": 1.4449069499969482,
+      "step": 1590
+    },
+    {
+      "epoch": 2.8450766837074903,
+      "grad_norm": 4.377365589141846,
+      "learning_rate": 3.203416211153832e-08,
+      "logits/chosen": -14.38147258758545,
+      "logits/rejected": -14.620170593261719,
+      "logps/chosen": -1.417458176612854,
+      "logps/rejected": -1.5342447757720947,
+      "loss": 1.491,
+      "odds_ratio_loss": 0.7356201410293579,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.14174583554267883,
+      "rewards/margins": 0.011678656563162804,
+      "rewards/rejected": -0.153424471616745,
+      "sft_loss": 1.417458176612854,
+      "step": 1600
+    },
+    {
+      "epoch": 2.8628584129806622,
+      "grad_norm": 5.88523006439209,
+      "learning_rate": 2.5029220118019393e-08,
+      "logits/chosen": -14.416727066040039,
+      "logits/rejected": -14.47779655456543,
+      "logps/chosen": -1.5760120153427124,
+      "logps/rejected": -1.5707772970199585,
+      "loss": 1.6535,
+      "odds_ratio_loss": 0.774849534034729,
+      "rewards/accuracies": 0.39375001192092896,
+      "rewards/chosen": -0.15760120749473572,
+      "rewards/margins": -0.0005234652198851109,
+      "rewards/rejected": -0.15707774460315704,
+      "sft_loss": 1.5760120153427124,
+      "step": 1610
+    },
+    {
+      "epoch": 2.880640142253834,
+      "grad_norm": 2.9929535388946533,
+      "learning_rate": 1.8883674727586122e-08,
+      "logits/chosen": -14.4614839553833,
+      "logits/rejected": -14.47071647644043,
+      "logps/chosen": -1.3790298700332642,
+      "logps/rejected": -1.5752503871917725,
+      "loss": 1.4464,
+      "odds_ratio_loss": 0.6737684607505798,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.13790300488471985,
+      "rewards/margins": 0.01962204836308956,
+      "rewards/rejected": -0.15752503275871277,
+      "sft_loss": 1.3790298700332642,
+      "step": 1620
+    },
+    {
+      "epoch": 2.898421871527006,
+      "grad_norm": 0.8867095112800598,
+      "learning_rate": 1.3599659889000639e-08,
+      "logits/chosen": -14.817098617553711,
+      "logits/rejected": -14.705721855163574,
+      "logps/chosen": -1.4359638690948486,
+      "logps/rejected": -1.4747111797332764,
+      "loss": 1.5093,
+      "odds_ratio_loss": 0.7331098914146423,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.14359638094902039,
+      "rewards/margins": 0.0038747270591557026,
+      "rewards/rejected": -0.1474711149930954,
+      "sft_loss": 1.4359638690948486,
+      "step": 1630
+    },
+    {
+      "epoch": 2.916203600800178,
+      "grad_norm": 2.0239081382751465,
+      "learning_rate": 9.179010397421528e-09,
+      "logits/chosen": -14.512980461120605,
+      "logits/rejected": -14.645942687988281,
+      "logps/chosen": -1.4728834629058838,
+      "logps/rejected": -1.5710668563842773,
+      "loss": 1.5439,
+      "odds_ratio_loss": 0.709987998008728,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.14728835225105286,
+      "rewards/margins": 0.00981833878904581,
+      "rewards/rejected": -0.1571066826581955,
+      "sft_loss": 1.4728834629058838,
+      "step": 1640
+    },
+    {
+      "epoch": 2.93398533007335,
+      "grad_norm": 3.0479774475097656,
+      "learning_rate": 5.623261257296509e-09,
+      "logits/chosen": -14.522771835327148,
+      "logits/rejected": -14.621217727661133,
+      "logps/chosen": -1.3921829462051392,
+      "logps/rejected": -1.5190517902374268,
+      "loss": 1.461,
+      "odds_ratio_loss": 0.6878436803817749,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.1392182856798172,
+      "rewards/margins": 0.012686875648796558,
+      "rewards/rejected": -0.15190517902374268,
+      "sft_loss": 1.3921829462051392,
+      "step": 1650
+    },
+    {
+      "epoch": 2.9517670593465217,
+      "grad_norm": 1.4828240871429443,
+      "learning_rate": 2.933647149357122e-09,
+      "logits/chosen": -14.36890697479248,
+      "logits/rejected": -14.4072847366333,
+      "logps/chosen": -1.3632400035858154,
+      "logps/rejected": -1.4689347743988037,
+      "loss": 1.4361,
+      "odds_ratio_loss": 0.7281399369239807,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.13632400333881378,
+      "rewards/margins": 0.010569497011601925,
+      "rewards/rejected": -0.1468934863805771,
+      "sft_loss": 1.3632400035858154,
+      "step": 1660
+    },
+    {
+      "epoch": 2.969548788619693,
+      "grad_norm": 1.2580466270446777,
+      "learning_rate": 1.1111020018930717e-09,
+      "logits/chosen": -14.664459228515625,
+      "logits/rejected": -14.4978666305542,
+      "logps/chosen": -1.4255958795547485,
+      "logps/rejected": -1.4593638181686401,
+      "loss": 1.5006,
+      "odds_ratio_loss": 0.7499974966049194,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": -0.14255960285663605,
+      "rewards/margins": 0.0033767870627343655,
+      "rewards/rejected": -0.14593638479709625,
+      "sft_loss": 1.4255958795547485,
+      "step": 1670
+    },
+    {
+      "epoch": 2.987330517892865,
+      "grad_norm": 1.022547960281372,
+      "learning_rate": 1.5625866646051813e-10,
+      "logits/chosen": -14.368474960327148,
+      "logits/rejected": -14.423803329467773,
+      "logps/chosen": -1.413266897201538,
+      "logps/rejected": -1.573032021522522,
+      "loss": 1.4799,
+      "odds_ratio_loss": 0.6666654348373413,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.1413266807794571,
+      "rewards/margins": 0.015976523980498314,
+      "rewards/rejected": -0.15730319917201996,
+      "sft_loss": 1.413266897201538,
+      "step": 1680
+    },
+    {
+      "epoch": 2.997999555456768,
+      "step": 1686,
+      "total_flos": 1.8091810238164992e+18,
+      "train_loss": 1.5885293396059446,
+      "train_runtime": 25020.7826,
+      "train_samples_per_second": 1.079,
+      "train_steps_per_second": 0.067
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1686,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "total_flos": 1.8091810238164992e+18,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}