diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0010465724751439038,
-      "grad_norm": 13.051470496546411,
+      "grad_norm": 13.050625530699566,
       "learning_rate": 5.208333333333333e-09,
       "logits/chosen": -2.8506698608398438,
       "logits/rejected": -2.73948335647583,
@@ -25,1581 +25,1581 @@
     },
     {
       "epoch": 0.010465724751439037,
-      "grad_norm": 9.930470236754596,
+      "grad_norm": 9.920541756105786,
       "learning_rate": 5.208333333333333e-08,
-      "logits/chosen": -2.473865270614624,
-      "logits/rejected": -2.444345474243164,
-      "logps/chosen": -267.0189208984375,
-      "logps/rejected": -241.91099548339844,
-      "loss": 0.6931,
-      "rewards/accuracies": 0.3402777910232544,
-      "rewards/chosen": -0.0007909854757599533,
-      "rewards/margins": -0.0007218287792056799,
-      "rewards/rejected": -6.915684207342565e-05,
+      "logits/chosen": -2.4737415313720703,
+      "logits/rejected": -2.4442765712738037,
+      "logps/chosen": -266.9542236328125,
+      "logps/rejected": -241.89979553222656,
+      "loss": 0.693,
+      "rewards/accuracies": 0.3819444477558136,
+      "rewards/chosen": -0.00014399096835404634,
+      "rewards/margins": -0.00018663256196305156,
+      "rewards/rejected": 4.2641553591238335e-05,
       "step": 10
     },
     {
       "epoch": 0.020931449502878074,
-      "grad_norm": 11.896085597664555,
+      "grad_norm": 11.877212722194296,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -2.4787793159484863,
-      "logits/rejected": -2.4331250190734863,
-      "logps/chosen": -290.9656677246094,
-      "logps/rejected": -289.4096984863281,
+      "logits/chosen": -2.4783947467803955,
+      "logits/rejected": -2.432711601257324,
+      "logps/chosen": -290.97509765625,
+      "logps/rejected": -289.4117126464844,
       "loss": 0.6931,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": 0.00036076834658160806,
-      "rewards/margins": 0.0002623483305796981,
-      "rewards/rejected": 9.842005238169804e-05,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0002665543870534748,
+      "rewards/margins": 0.0001886741374619305,
+      "rewards/rejected": 7.788027141941711e-05,
       "step": 20
     },
     {
       "epoch": 0.03139717425431711,
-      "grad_norm": 11.466941852693491,
+      "grad_norm": 11.429814607169217,
       "learning_rate": 1.5624999999999999e-07,
-      "logits/chosen": -2.5793910026550293,
-      "logits/rejected": -2.5509467124938965,
-      "logps/chosen": -280.1932067871094,
-      "logps/rejected": -281.7767639160156,
+      "logits/chosen": -2.579545259475708,
+      "logits/rejected": -2.551072359085083,
+      "logps/chosen": -280.2004699707031,
+      "logps/rejected": -281.75982666015625,
       "loss": 0.6928,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.0013170629972591996,
-      "rewards/margins": 0.0012757038930431008,
-      "rewards/rejected": 4.135915150982328e-05,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0012443868909031153,
+      "rewards/margins": 0.0010337729472666979,
+      "rewards/rejected": 0.00021061387087684125,
       "step": 30
     },
     {
       "epoch": 0.04186289900575615,
-      "grad_norm": 11.971540688024561,
+      "grad_norm": 12.001629084710268,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -2.5403754711151123,
-      "logits/rejected": -2.4496283531188965,
-      "logps/chosen": -298.2412414550781,
-      "logps/rejected": -286.4029235839844,
-      "loss": 0.6918,
+      "logits/chosen": -2.54026198387146,
+      "logits/rejected": -2.449479103088379,
+      "logps/chosen": -298.2272644042969,
+      "logps/rejected": -286.3971252441406,
+      "loss": 0.692,
       "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.002237908076494932,
-      "rewards/margins": 0.0025104572996497154,
-      "rewards/rejected": -0.0002725489321164787,
+      "rewards/chosen": 0.002377925906330347,
+      "rewards/margins": 0.002592656761407852,
+      "rewards/rejected": -0.00021473097149282694,
       "step": 40
     },
     {
       "epoch": 0.052328623757195186,
-      "grad_norm": 10.953203550036346,
+      "grad_norm": 10.912228829257876,
       "learning_rate": 2.604166666666667e-07,
-      "logits/chosen": -2.5587046146392822,
-      "logits/rejected": -2.463972568511963,
-      "logps/chosen": -274.0966796875,
-      "logps/rejected": -240.55062866210938,
+      "logits/chosen": -2.558757781982422,
+      "logits/rejected": -2.463988780975342,
+      "logps/chosen": -274.10211181640625,
+      "logps/rejected": -240.5286865234375,
       "loss": 0.6898,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.0028838147409260273,
-      "rewards/margins": 0.005623813718557358,
-      "rewards/rejected": -0.0027399989776313305,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.002829686040058732,
+      "rewards/margins": 0.005349998362362385,
+      "rewards/rejected": -0.002520312089473009,
       "step": 50
     },
     {
       "epoch": 0.06279434850863422,
-      "grad_norm": 10.943227101823018,
+      "grad_norm": 11.002827911052439,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -2.5307021141052246,
-      "logits/rejected": -2.4873709678649902,
-      "logps/chosen": -310.4714660644531,
-      "logps/rejected": -283.3081970214844,
-      "loss": 0.6858,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 0.007902054116129875,
-      "rewards/margins": 0.013244956731796265,
-      "rewards/rejected": -0.005342902150005102,
+      "logits/chosen": -2.531254768371582,
+      "logits/rejected": -2.487887144088745,
+      "logps/chosen": -310.49102783203125,
+      "logps/rejected": -283.2752990722656,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.007706484757363796,
+      "rewards/margins": 0.012720311060547829,
+      "rewards/rejected": -0.005013825837522745,
       "step": 60
     },
     {
       "epoch": 0.07326007326007326,
-      "grad_norm": 10.581002478128156,
+      "grad_norm": 10.866381858570492,
       "learning_rate": 3.645833333333333e-07,
-      "logits/chosen": -2.411393642425537,
-      "logits/rejected": -2.4127840995788574,
-      "logps/chosen": -272.46405029296875,
-      "logps/rejected": -267.72686767578125,
+      "logits/chosen": -2.412121295928955,
+      "logits/rejected": -2.413834810256958,
+      "logps/chosen": -272.3429870605469,
+      "logps/rejected": -267.58709716796875,
       "loss": 0.681,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.011364299803972244,
-      "rewards/margins": 0.030219241976737976,
-      "rewards/rejected": -0.04158353805541992,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.0101536950096488,
+      "rewards/margins": 0.030031735077500343,
+      "rewards/rejected": -0.04018542915582657,
       "step": 70
     },
     {
       "epoch": 0.0837257980115123,
-      "grad_norm": 11.42733355645212,
+      "grad_norm": 11.09456859720252,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.481160879135132,
-      "logits/rejected": -2.397334575653076,
-      "logps/chosen": -288.3390197753906,
-      "logps/rejected": -264.55322265625,
-      "loss": 0.6712,
+      "logits/chosen": -2.48347806930542,
+      "logits/rejected": -2.3994758129119873,
+      "logps/chosen": -288.26800537109375,
+      "logps/rejected": -264.4568176269531,
+      "loss": 0.6713,
       "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.03238832578063011,
-      "rewards/margins": 0.04954018443822861,
-      "rewards/rejected": -0.08192850649356842,
+      "rewards/chosen": -0.031678371131420135,
+      "rewards/margins": 0.04928610473871231,
+      "rewards/rejected": -0.08096449077129364,
       "step": 80
     },
     {
       "epoch": 0.09419152276295134,
-      "grad_norm": 11.210988509809624,
+      "grad_norm": 11.047168505910133,
       "learning_rate": 4.6874999999999996e-07,
-      "logits/chosen": -2.407602310180664,
-      "logits/rejected": -2.3880608081817627,
-      "logps/chosen": -270.7894592285156,
-      "logps/rejected": -281.30572509765625,
-      "loss": 0.6685,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.05281522870063782,
-      "rewards/margins": 0.03494281694293022,
-      "rewards/rejected": -0.08775804191827774,
+      "logits/chosen": -2.410482406616211,
+      "logits/rejected": -2.390921115875244,
+      "logps/chosen": -270.5683898925781,
+      "logps/rejected": -281.0517883300781,
+      "loss": 0.6686,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.05060455948114395,
+      "rewards/margins": 0.03461429476737976,
+      "rewards/rejected": -0.08521886169910431,
       "step": 90
     },
     {
       "epoch": 0.10465724751439037,
-      "grad_norm": 11.737180826993443,
+      "grad_norm": 11.672590333361306,
       "learning_rate": 4.999732492681437e-07,
-      "logits/chosen": -2.4048094749450684,
-      "logits/rejected": -2.339949607849121,
-      "logps/chosen": -313.2823486328125,
-      "logps/rejected": -298.7939453125,
-      "loss": 0.6509,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.03915167227387428,
-      "rewards/margins": 0.094016894698143,
-      "rewards/rejected": -0.1331685483455658,
+      "logits/chosen": -2.4052681922912598,
+      "logits/rejected": -2.339871883392334,
+      "logps/chosen": -312.91473388671875,
+      "logps/rejected": -298.09039306640625,
+      "loss": 0.652,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.035475730895996094,
+      "rewards/margins": 0.0906570553779602,
+      "rewards/rejected": -0.1261327713727951,
       "step": 100
     },
     {
       "epoch": 0.10465724751439037,
-      "eval_logits/chosen": -2.4569716453552246,
-      "eval_logits/rejected": -2.39499831199646,
-      "eval_logps/chosen": -294.89752197265625,
-      "eval_logps/rejected": -280.17822265625,
-      "eval_loss": 0.6506768465042114,
-      "eval_rewards/accuracies": 0.6865079402923584,
-      "eval_rewards/chosen": -0.03169262409210205,
-      "eval_rewards/margins": 0.10856584459543228,
-      "eval_rewards/rejected": -0.14025846123695374,
-      "eval_runtime": 205.869,
-      "eval_samples_per_second": 9.715,
+      "eval_logits/chosen": -2.454148530960083,
+      "eval_logits/rejected": -2.3909990787506104,
+      "eval_logps/chosen": -293.7001647949219,
+      "eval_logps/rejected": -278.65301513671875,
+      "eval_loss": 0.6515971422195435,
+      "eval_rewards/accuracies": 0.6904761791229248,
+      "eval_rewards/chosen": -0.01971943862736225,
+      "eval_rewards/margins": 0.10528726875782013,
+      "eval_rewards/rejected": -0.12500669062137604,
+      "eval_runtime": 205.9488,
+      "eval_samples_per_second": 9.711,
       "eval_steps_per_second": 0.306,
       "step": 100
     },
     {
       "epoch": 0.1151229722658294,
-      "grad_norm": 18.957823734967118,
+      "grad_norm": 15.062003030385734,
       "learning_rate": 4.996723692767926e-07,
-      "logits/chosen": -2.443211793899536,
-      "logits/rejected": -2.3995585441589355,
-      "logps/chosen": -254.23428344726562,
-      "logps/rejected": -261.7113952636719,
-      "loss": 0.645,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.07062581926584244,
-      "rewards/margins": 0.11806046962738037,
-      "rewards/rejected": -0.1886862814426422,
+      "logits/chosen": -2.4393675327301025,
+      "logits/rejected": -2.394540786743164,
+      "logps/chosen": -253.6468048095703,
+      "logps/rejected": -260.84027099609375,
+      "loss": 0.6453,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.06475158780813217,
+      "rewards/margins": 0.1152237206697464,
+      "rewards/rejected": -0.17997530102729797,
       "step": 110
     },
     {
       "epoch": 0.12558869701726844,
-      "grad_norm": 211.9091102470587,
+      "grad_norm": 20.5435131882062,
       "learning_rate": 4.990375746213598e-07,
-      "logits/chosen": -2.338498115539551,
-      "logits/rejected": -2.2794461250305176,
-      "logps/chosen": -298.16986083984375,
-      "logps/rejected": -339.7401428222656,
-      "loss": 0.6298,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.2652895748615265,
-      "rewards/margins": 0.1783614456653595,
-      "rewards/rejected": -0.443651020526886,
+      "logits/chosen": -2.333178997039795,
+      "logits/rejected": -2.2723469734191895,
+      "logps/chosen": -297.21044921875,
+      "logps/rejected": -338.71893310546875,
+      "loss": 0.6303,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.25569555163383484,
+      "rewards/margins": 0.1777433454990387,
+      "rewards/rejected": -0.43343886733055115,
       "step": 120
     },
     {
       "epoch": 0.1360544217687075,
-      "grad_norm": 37.07462356301655,
+      "grad_norm": 17.505959054088198,
       "learning_rate": 4.980697142834314e-07,
-      "logits/chosen": -2.244835615158081,
-      "logits/rejected": -2.2257189750671387,
-      "logps/chosen": -322.57122802734375,
-      "logps/rejected": -319.81500244140625,
-      "loss": 0.6229,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.3214291036128998,
-      "rewards/margins": 0.2704463005065918,
-      "rewards/rejected": -0.591875433921814,
+      "logits/chosen": -2.2366394996643066,
+      "logits/rejected": -2.2138123512268066,
+      "logps/chosen": -324.4869079589844,
+      "logps/rejected": -320.10272216796875,
+      "loss": 0.623,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.34058648347854614,
+      "rewards/margins": 0.25416648387908936,
+      "rewards/rejected": -0.5947530269622803,
       "step": 130
     },
     {
       "epoch": 0.14652014652014653,
-      "grad_norm": 18.745612354229714,
+      "grad_norm": 20.654410889137058,
       "learning_rate": 4.967700826904229e-07,
-      "logits/chosen": -2.1823461055755615,
-      "logits/rejected": -2.132317304611206,
-      "logps/chosen": -301.0035705566406,
-      "logps/rejected": -319.9606628417969,
-      "loss": 0.6085,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.06300695240497589,
-      "rewards/margins": 0.2533551752567291,
-      "rewards/rejected": -0.3163621127605438,
+      "logits/chosen": -2.130495071411133,
+      "logits/rejected": -2.0791451930999756,
+      "logps/chosen": -313.0150451660156,
+      "logps/rejected": -338.2718811035156,
+      "loss": 0.5999,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.18312115967273712,
+      "rewards/margins": 0.3163526654243469,
+      "rewards/rejected": -0.49947381019592285,
       "step": 140
     },
     {
       "epoch": 0.15698587127158556,
-      "grad_norm": 20.385737144331163,
+      "grad_norm": 16.089670018723332,
       "learning_rate": 4.951404179843962e-07,
-      "logits/chosen": -2.247217893600464,
-      "logits/rejected": -2.2084200382232666,
-      "logps/chosen": -331.0395812988281,
-      "logps/rejected": -310.9860534667969,
-      "loss": 0.609,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.31488415598869324,
-      "rewards/margins": 0.2636170983314514,
-      "rewards/rejected": -0.578501284122467,
+      "logits/chosen": -2.140552043914795,
+      "logits/rejected": -2.0911858081817627,
+      "logps/chosen": -337.337158203125,
+      "logps/rejected": -320.74053955078125,
+      "loss": 0.6057,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.3778603672981262,
+      "rewards/margins": 0.2981855273246765,
+      "rewards/rejected": -0.6760458946228027,
       "step": 150
     },
     {
       "epoch": 0.1674515960230246,
-      "grad_norm": 17.50514107179683,
+      "grad_norm": 18.730825067265084,
       "learning_rate": 4.931828996974498e-07,
-      "logits/chosen": -2.186821937561035,
-      "logits/rejected": -2.093240261077881,
-      "logps/chosen": -330.5211181640625,
-      "logps/rejected": -320.204345703125,
-      "loss": 0.5859,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.2437433898448944,
-      "rewards/margins": 0.3570416569709778,
-      "rewards/rejected": -0.6007851362228394,
+      "logits/chosen": -2.0323245525360107,
+      "logits/rejected": -1.9224172830581665,
+      "logps/chosen": -333.4700927734375,
+      "logps/rejected": -323.68853759765625,
+      "loss": 0.5855,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.2732333242893219,
+      "rewards/margins": 0.36239343881607056,
+      "rewards/rejected": -0.6356266736984253,
       "step": 160
     },
     {
       "epoch": 0.17791732077446362,
-      "grad_norm": 18.503251999087315,
+      "grad_norm": 19.92022658066836,
       "learning_rate": 4.909001458367866e-07,
-      "logits/chosen": -2.151353597640991,
-      "logits/rejected": -2.0457262992858887,
-      "logps/chosen": -322.26434326171875,
-      "logps/rejected": -320.9549255371094,
-      "loss": 0.5782,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.3654264807701111,
-      "rewards/margins": 0.40083780884742737,
-      "rewards/rejected": -0.7662642598152161,
+      "logits/chosen": -1.9757146835327148,
+      "logits/rejected": -1.856048583984375,
+      "logps/chosen": -338.2750549316406,
+      "logps/rejected": -340.8263854980469,
+      "loss": 0.5792,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.5255335569381714,
+      "rewards/margins": 0.4394455552101135,
+      "rewards/rejected": -0.9649791717529297,
       "step": 170
     },
     {
       "epoch": 0.18838304552590268,
-      "grad_norm": 22.1110027585031,
+      "grad_norm": 18.813658031058317,
       "learning_rate": 4.882952093833627e-07,
-      "logits/chosen": -2.0537924766540527,
-      "logits/rejected": -2.013256549835205,
-      "logps/chosen": -296.3823547363281,
-      "logps/rejected": -343.3452453613281,
-      "loss": 0.5758,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.487266480922699,
-      "rewards/margins": 0.42303138971328735,
-      "rewards/rejected": -0.9102978706359863,
+      "logits/chosen": -1.8651317358016968,
+      "logits/rejected": -1.8147039413452148,
+      "logps/chosen": -310.0968322753906,
+      "logps/rejected": -356.84918212890625,
+      "loss": 0.5733,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.6244109272956848,
+      "rewards/margins": 0.42092594504356384,
+      "rewards/rejected": -1.0453368425369263,
       "step": 180
     },
     {
       "epoch": 0.1988487702773417,
-      "grad_norm": 22.087134238338532,
+      "grad_norm": 18.706911358280287,
       "learning_rate": 4.853715742087946e-07,
-      "logits/chosen": -1.8600575923919678,
-      "logits/rejected": -1.812792420387268,
-      "logps/chosen": -289.62152099609375,
-      "logps/rejected": -342.3699645996094,
-      "loss": 0.5745,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.43460702896118164,
-      "rewards/margins": 0.4210926592350006,
-      "rewards/rejected": -0.8556997179985046,
+      "logits/chosen": -1.6657918691635132,
+      "logits/rejected": -1.6065375804901123,
+      "logps/chosen": -283.023193359375,
+      "logps/rejected": -332.28277587890625,
+      "loss": 0.5699,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.36862367391586304,
+      "rewards/margins": 0.3862038254737854,
+      "rewards/rejected": -0.7548275589942932,
       "step": 190
     },
     {
       "epoch": 0.20931449502878074,
-      "grad_norm": 22.27318395987777,
+      "grad_norm": 24.72226874406459,
       "learning_rate": 4.821331504159906e-07,
-      "logits/chosen": -2.048029661178589,
-      "logits/rejected": -1.8949153423309326,
-      "logps/chosen": -378.06451416015625,
-      "logps/rejected": -363.1112976074219,
-      "loss": 0.5842,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.5441427826881409,
-      "rewards/margins": 0.4527364671230316,
-      "rewards/rejected": -0.9968792796134949,
+      "logits/chosen": -1.787872314453125,
+      "logits/rejected": -1.5908364057540894,
+      "logps/chosen": -387.01885986328125,
+      "logps/rejected": -373.51068115234375,
+      "loss": 0.5817,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.6336859464645386,
+      "rewards/margins": 0.46718692779541016,
+      "rewards/rejected": -1.1008728742599487,
       "step": 200
     },
     {
       "epoch": 0.20931449502878074,
-      "eval_logits/chosen": -1.991257667541504,
-      "eval_logits/rejected": -1.8882709741592407,
-      "eval_logps/chosen": -348.01226806640625,
-      "eval_logps/rejected": -367.80316162109375,
-      "eval_loss": 0.586904764175415,
-      "eval_rewards/accuracies": 0.7182539701461792,
-      "eval_rewards/chosen": -0.5628401637077332,
-      "eval_rewards/margins": 0.45366790890693665,
-      "eval_rewards/rejected": -1.0165081024169922,
-      "eval_runtime": 206.2096,
-      "eval_samples_per_second": 9.699,
-      "eval_steps_per_second": 0.306,
+      "eval_logits/chosen": -1.6837007999420166,
+      "eval_logits/rejected": -1.5444186925888062,
+      "eval_logps/chosen": -376.9992370605469,
+      "eval_logps/rejected": -400.1837463378906,
+      "eval_loss": 0.5832668542861938,
+      "eval_rewards/accuracies": 0.7123016119003296,
+      "eval_rewards/chosen": -0.8527095913887024,
+      "eval_rewards/margins": 0.4876045286655426,
+      "eval_rewards/rejected": -1.3403141498565674,
+      "eval_runtime": 206.2549,
+      "eval_samples_per_second": 9.697,
+      "eval_steps_per_second": 0.305,
       "step": 200
     },
     {
       "epoch": 0.21978021978021978,
-      "grad_norm": 24.077327884786772,
+      "grad_norm": 26.670026802650952,
       "learning_rate": 4.785842691097342e-07,
-      "logits/chosen": -1.94464111328125,
-      "logits/rejected": -1.8161208629608154,
-      "logps/chosen": -353.6456604003906,
-      "logps/rejected": -386.1349792480469,
-      "loss": 0.5866,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.6112793684005737,
-      "rewards/margins": 0.45647066831588745,
-      "rewards/rejected": -1.0677502155303955,
+      "logits/chosen": -1.655531883239746,
+      "logits/rejected": -1.4870684146881104,
+      "logps/chosen": -368.97332763671875,
+      "logps/rejected": -395.6372985839844,
+      "loss": 0.5863,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.7645561099052429,
+      "rewards/margins": 0.39821723103523254,
+      "rewards/rejected": -1.1627732515335083,
       "step": 210
     },
     {
       "epoch": 0.2302459445316588,
-      "grad_norm": 25.256951902015157,
+      "grad_norm": 25.673166847559166,
       "learning_rate": 4.7472967660421603e-07,
-      "logits/chosen": -1.9311796426773071,
-      "logits/rejected": -1.7711641788482666,
-      "logps/chosen": -352.7847595214844,
-      "logps/rejected": -341.3170471191406,
-      "loss": 0.5697,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.564936637878418,
-      "rewards/margins": 0.46136683225631714,
-      "rewards/rejected": -1.0263035297393799,
+      "logits/chosen": -1.6812896728515625,
+      "logits/rejected": -1.4805634021759033,
+      "logps/chosen": -354.3861083984375,
+      "logps/rejected": -341.49957275390625,
+      "loss": 0.5742,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.5809499025344849,
+      "rewards/margins": 0.44717901945114136,
+      "rewards/rejected": -1.0281288623809814,
       "step": 220
     },
     {
       "epoch": 0.24071166928309787,
-      "grad_norm": 25.586478019113667,
+      "grad_norm": 24.43355540456897,
       "learning_rate": 4.705745280752585e-07,
-      "logits/chosen": -1.7587419748306274,
-      "logits/rejected": -1.7059828042984009,
-      "logps/chosen": -347.0582275390625,
-      "logps/rejected": -369.04644775390625,
-      "loss": 0.5672,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.5065922737121582,
-      "rewards/margins": 0.435419499874115,
-      "rewards/rejected": -0.9420117139816284,
+      "logits/chosen": -1.5731475353240967,
+      "logits/rejected": -1.5115373134613037,
+      "logps/chosen": -339.289306640625,
+      "logps/rejected": -354.33526611328125,
+      "loss": 0.5791,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.428903192281723,
+      "rewards/margins": 0.36599647998809814,
+      "rewards/rejected": -0.794899582862854,
       "step": 230
     },
     {
       "epoch": 0.25117739403453687,
-      "grad_norm": 30.28171698877856,
+      "grad_norm": 26.616817677661366,
       "learning_rate": 4.6612438066572555e-07,
-      "logits/chosen": -1.6479380130767822,
-      "logits/rejected": -1.585958480834961,
-      "logps/chosen": -339.24859619140625,
-      "logps/rejected": -380.8953552246094,
-      "loss": 0.5514,
+      "logits/chosen": -1.4116663932800293,
+      "logits/rejected": -1.3324604034423828,
+      "logps/chosen": -328.2694396972656,
+      "logps/rejected": -363.1044921875,
+      "loss": 0.5552,
       "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.6151422262191772,
-      "rewards/margins": 0.5752537846565247,
-      "rewards/rejected": -1.1903960704803467,
+      "rewards/chosen": -0.5053508281707764,
+      "rewards/margins": 0.5071364641189575,
+      "rewards/rejected": -1.0124871730804443,
       "step": 240
     },
     {
       "epoch": 0.2616431187859759,
-      "grad_norm": 27.376050990007407,
+      "grad_norm": 24.322837971374696,
       "learning_rate": 4.6138518605333664e-07,
-      "logits/chosen": -1.6576675176620483,
-      "logits/rejected": -1.5993081331253052,
-      "logps/chosen": -355.48638916015625,
-      "logps/rejected": -390.65313720703125,
-      "loss": 0.5373,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.6634787321090698,
-      "rewards/margins": 0.5651322603225708,
-      "rewards/rejected": -1.228610873222351,
+      "logits/chosen": -1.315580129623413,
+      "logits/rejected": -1.231614351272583,
+      "logps/chosen": -361.7240905761719,
+      "logps/rejected": -399.68170166015625,
+      "loss": 0.5363,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.7258554697036743,
+      "rewards/margins": 0.5930408835411072,
+      "rewards/rejected": -1.3188964128494263,
       "step": 250
     },
     {
       "epoch": 0.272108843537415,
-      "grad_norm": 43.177740519027985,
+      "grad_norm": 23.712464427398043,
       "learning_rate": 4.5636328249082514e-07,
-      "logits/chosen": -1.7310291528701782,
-      "logits/rejected": -1.6111475229263306,
-      "logps/chosen": -385.6434631347656,
-      "logps/rejected": -425.70599365234375,
-      "loss": 0.5521,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.8594309687614441,
-      "rewards/margins": 0.5068761706352234,
-      "rewards/rejected": -1.366307020187378,
+      "logits/chosen": -1.4646880626678467,
+      "logits/rejected": -1.301315426826477,
+      "logps/chosen": -371.54913330078125,
+      "logps/rejected": -411.68341064453125,
+      "loss": 0.5469,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.7184873223304749,
+      "rewards/margins": 0.5075935125350952,
+      "rewards/rejected": -1.2260808944702148,
       "step": 260
     },
     {
       "epoch": 0.282574568288854,
-      "grad_norm": 26.026493875287635,
+      "grad_norm": 26.263289602516945,
       "learning_rate": 4.510653863290871e-07,
-      "logits/chosen": -1.52871572971344,
-      "logits/rejected": -1.4289811849594116,
-      "logps/chosen": -399.4200744628906,
-      "logps/rejected": -410.7467346191406,
-      "loss": 0.5441,
+      "logits/chosen": -1.290191888809204,
+      "logits/rejected": -1.1571375131607056,
+      "logps/chosen": -373.7254333496094,
+      "logps/rejected": -379.77606201171875,
+      "loss": 0.5386,
       "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.8672260046005249,
-      "rewards/margins": 0.6238988637924194,
-      "rewards/rejected": -1.4911251068115234,
+      "rewards/chosen": -0.6102796792984009,
+      "rewards/margins": 0.5711389780044556,
+      "rewards/rejected": -1.181418538093567,
       "step": 270
     },
     {
       "epoch": 0.29304029304029305,
-      "grad_norm": 37.6240820385195,
+      "grad_norm": 26.39990044111727,
       "learning_rate": 4.4549858303465737e-07,
-      "logits/chosen": -1.7061611413955688,
-      "logits/rejected": -1.602936029434204,
-      "logps/chosen": -387.9464111328125,
-      "logps/rejected": -430.4659729003906,
-      "loss": 0.5687,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.8134315609931946,
-      "rewards/margins": 0.6281790733337402,
-      "rewards/rejected": -1.4416106939315796,
+      "logits/chosen": -1.3888221979141235,
+      "logits/rejected": -1.2640901803970337,
+      "logps/chosen": -374.4818115234375,
+      "logps/rejected": -414.4740295410156,
+      "loss": 0.564,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.6787852644920349,
+      "rewards/margins": 0.6029058694839478,
+      "rewards/rejected": -1.2816909551620483,
       "step": 280
     },
     {
       "epoch": 0.3035060177917321,
-      "grad_norm": 30.468189665035492,
+      "grad_norm": 34.560220070957364,
       "learning_rate": 4.396703177135261e-07,
-      "logits/chosen": -1.7203712463378906,
-      "logits/rejected": -1.513145923614502,
-      "logps/chosen": -350.87872314453125,
-      "logps/rejected": -369.96343994140625,
-      "loss": 0.5572,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.6935116648674011,
-      "rewards/margins": 0.5848726034164429,
-      "rewards/rejected": -1.2783840894699097,
+      "logits/chosen": -1.2199509143829346,
+      "logits/rejected": -0.9321581721305847,
+      "logps/chosen": -358.50152587890625,
+      "logps/rejected": -376.865966796875,
+      "loss": 0.5489,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.7697396874427795,
+      "rewards/margins": 0.5776699781417847,
+      "rewards/rejected": -1.3474096059799194,
       "step": 290
     },
     {
       "epoch": 0.3139717425431711,
-      "grad_norm": 25.896063593338788,
+      "grad_norm": 31.906550711331022,
       "learning_rate": 4.335883851539693e-07,
-      "logits/chosen": -1.7010329961776733,
-      "logits/rejected": -1.6187465190887451,
-      "logps/chosen": -348.5817565917969,
-      "logps/rejected": -401.21270751953125,
-      "loss": 0.5518,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.7173490524291992,
-      "rewards/margins": 0.5425899624824524,
-      "rewards/rejected": -1.2599390745162964,
+      "logits/chosen": -1.0410287380218506,
+      "logits/rejected": -0.9161141514778137,
+      "logps/chosen": -363.14996337890625,
+      "logps/rejected": -421.73370361328125,
+      "loss": 0.5434,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.8630313873291016,
+      "rewards/margins": 0.6021174192428589,
+      "rewards/rejected": -1.465148687362671,
       "step": 300
     },
     {
       "epoch": 0.3139717425431711,
-      "eval_logits/chosen": -1.5902302265167236,
-      "eval_logits/rejected": -1.4401179552078247,
-      "eval_logps/chosen": -367.8922424316406,
-      "eval_logps/rejected": -402.97265625,
-      "eval_loss": 0.5573062300682068,
-      "eval_rewards/accuracies": 0.7420634627342224,
-      "eval_rewards/chosen": -0.7616395950317383,
-      "eval_rewards/margins": 0.606563150882721,
-      "eval_rewards/rejected": -1.368202805519104,
-      "eval_runtime": 205.6103,
-      "eval_samples_per_second": 9.727,
-      "eval_steps_per_second": 0.306,
+      "eval_logits/chosen": -0.7917075157165527,
+      "eval_logits/rejected": -0.5465483665466309,
+      "eval_logps/chosen": -387.9329833984375,
+      "eval_logps/rejected": -429.962158203125,
+      "eval_loss": 0.5529596209526062,
+      "eval_rewards/accuracies": 0.7460317611694336,
+      "eval_rewards/chosen": -0.9620471000671387,
+      "eval_rewards/margins": 0.6760506629943848,
+      "eval_rewards/rejected": -1.6380977630615234,
+      "eval_runtime": 206.9827,
+      "eval_samples_per_second": 9.663,
+      "eval_steps_per_second": 0.304,
       "step": 300
     },
     {
       "epoch": 0.32443746729461015,
-      "grad_norm": 23.611287954852504,
+      "grad_norm": 26.6660135399266,
       "learning_rate": 4.272609194017105e-07,
-      "logits/chosen": -1.7664659023284912,
-      "logits/rejected": -1.4679615497589111,
-      "logps/chosen": -360.18707275390625,
-      "logps/rejected": -362.8342590332031,
-      "loss": 0.542,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.678702712059021,
-      "rewards/margins": 0.6288893818855286,
-      "rewards/rejected": -1.3075921535491943,
+      "logits/chosen": -1.0415055751800537,
+      "logits/rejected": -0.5032875537872314,
+      "logps/chosen": -374.76885986328125,
+      "logps/rejected": -379.4770812988281,
+      "loss": 0.5373,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.8245205879211426,
+      "rewards/margins": 0.6495000123977661,
+      "rewards/rejected": -1.4740204811096191,
       "step": 310
     },
     {
       "epoch": 0.3349031920460492,
-      "grad_norm": 24.070079232095797,
+      "grad_norm": 24.17485231271666,
       "learning_rate": 4.2069638288135547e-07,
-      "logits/chosen": -1.6632932424545288,
-      "logits/rejected": -1.5471082925796509,
-      "logps/chosen": -327.7480163574219,
-      "logps/rejected": -383.7098388671875,
-      "loss": 0.558,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.6849709749221802,
-      "rewards/margins": 0.6326489448547363,
-      "rewards/rejected": -1.317620038986206,
+      "logits/chosen": -0.675749659538269,
+      "logits/rejected": -0.4698491096496582,
+      "logps/chosen": -330.9830627441406,
+      "logps/rejected": -390.26177978515625,
+      "loss": 0.5558,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.7173212766647339,
+      "rewards/margins": 0.6658179759979248,
+      "rewards/rejected": -1.3831392526626587,
       "step": 320
     },
     {
       "epoch": 0.3453689167974882,
-      "grad_norm": 38.56102713027139,
+      "grad_norm": 29.466724809508868,
       "learning_rate": 4.139035550786494e-07,
-      "logits/chosen": -1.4653488397598267,
-      "logits/rejected": -1.3664615154266357,
-      "logps/chosen": -347.2395935058594,
-      "logps/rejected": -377.60748291015625,
-      "loss": 0.5465,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.8459569811820984,
-      "rewards/margins": 0.5722988843917847,
-      "rewards/rejected": -1.4182560443878174,
+      "logits/chosen": -0.24337038397789001,
+      "logits/rejected": -0.02021191269159317,
+      "logps/chosen": -352.15679931640625,
+      "logps/rejected": -377.9948425292969,
+      "loss": 0.5515,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.8951289057731628,
+      "rewards/margins": 0.5270006060600281,
+      "rewards/rejected": -1.422129511833191,
       "step": 330
     },
     {
       "epoch": 0.35583464154892724,
-      "grad_norm": 28.031822632702205,
+      "grad_norm": 29.805410198198782,
       "learning_rate": 4.0689152079869306e-07,
-      "logits/chosen": -1.3066160678863525,
-      "logits/rejected": -1.156473159790039,
-      "logps/chosen": -396.61676025390625,
-      "logps/rejected": -416.3009338378906,
-      "loss": 0.574,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.1738026142120361,
-      "rewards/margins": 0.5725993514060974,
-      "rewards/rejected": -1.7464020252227783,
+      "logits/chosen": -0.34342047572135925,
+      "logits/rejected": -0.06829465180635452,
+      "logps/chosen": -365.95794677734375,
+      "logps/rejected": -390.84552001953125,
+      "loss": 0.5632,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.8672142028808594,
+      "rewards/margins": 0.6246346831321716,
+      "rewards/rejected": -1.4918488264083862,
       "step": 340
     },
     {
       "epoch": 0.3663003663003663,
-      "grad_norm": 26.144173653631405,
+      "grad_norm": 28.37056438472245,
       "learning_rate": 3.99669658015821e-07,
-      "logits/chosen": -1.540045976638794,
-      "logits/rejected": -1.4006694555282593,
-      "logps/chosen": -385.5912780761719,
-      "logps/rejected": -410.674072265625,
-      "loss": 0.536,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.8446685075759888,
-      "rewards/margins": 0.569485068321228,
-      "rewards/rejected": -1.4141535758972168,
+      "logits/chosen": -0.5597683191299438,
+      "logits/rejected": -0.32104772329330444,
+      "logps/chosen": -398.7806091308594,
+      "logps/rejected": -420.88330078125,
+      "loss": 0.5319,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.9765618443489075,
+      "rewards/margins": 0.5396836996078491,
+      "rewards/rejected": -1.5162456035614014,
       "step": 350
     },
     {
       "epoch": 0.37676609105180536,
-      "grad_norm": 33.22726534237683,
+      "grad_norm": 31.822767271574797,
       "learning_rate": 3.92247625331392e-07,
-      "logits/chosen": -1.4644501209259033,
-      "logits/rejected": -1.3674944639205933,
-      "logps/chosen": -364.0406188964844,
-      "logps/rejected": -396.0486145019531,
-      "loss": 0.5221,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.7407475709915161,
-      "rewards/margins": 0.5707039833068848,
-      "rewards/rejected": -1.3114515542984009,
+      "logits/chosen": 0.28968971967697144,
+      "logits/rejected": 0.5219839215278625,
+      "logps/chosen": -387.05316162109375,
+      "logps/rejected": -426.6918029785156,
+      "loss": 0.5093,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.9708734750747681,
+      "rewards/margins": 0.6470099687576294,
+      "rewards/rejected": -1.617883324623108,
       "step": 360
     },
     {
       "epoch": 0.3872318158032444,
-      "grad_norm": 22.278385921862558,
+      "grad_norm": 27.78339814036373,
       "learning_rate": 3.846353490562664e-07,
-      "logits/chosen": -1.4632952213287354,
-      "logits/rejected": -1.364483118057251,
-      "logps/chosen": -353.5659484863281,
-      "logps/rejected": -371.3506774902344,
-      "loss": 0.5452,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.8213934898376465,
-      "rewards/margins": 0.4817906320095062,
-      "rewards/rejected": -1.3031842708587646,
+      "logits/chosen": 0.8734736442565918,
+      "logits/rejected": 1.1159971952438354,
+      "logps/chosen": -387.59527587890625,
+      "logps/rejected": -414.43438720703125,
+      "loss": 0.5304,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.161686897277832,
+      "rewards/margins": 0.5723341107368469,
+      "rewards/rejected": -1.7340209484100342,
       "step": 370
     },
     {
       "epoch": 0.3976975405546834,
-      "grad_norm": 26.485819344845584,
+      "grad_norm": 41.70350367774299,
       "learning_rate": 3.768430099352445e-07,
-      "logits/chosen": -1.4255541563034058,
-      "logits/rejected": -1.2725279331207275,
-      "logps/chosen": -331.7218933105469,
-      "logps/rejected": -386.83331298828125,
-      "loss": 0.5351,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.709423840045929,
-      "rewards/margins": 0.6313949823379517,
-      "rewards/rejected": -1.3408187627792358,
+      "logits/chosen": 1.086398959159851,
+      "logits/rejected": 1.3473352193832397,
+      "logps/chosen": -353.5553894042969,
+      "logps/rejected": -417.9972229003906,
+      "loss": 0.5235,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.9277588129043579,
+      "rewards/margins": 0.7246991395950317,
+      "rewards/rejected": -1.6524579524993896,
       "step": 380
     },
     {
       "epoch": 0.40816326530612246,
-      "grad_norm": 26.88634695034012,
+      "grad_norm": 39.51085586749341,
       "learning_rate": 3.6888102953122304e-07,
-      "logits/chosen": -1.4283645153045654,
-      "logits/rejected": -1.313664197921753,
-      "logps/chosen": -346.01446533203125,
-      "logps/rejected": -394.5232849121094,
-      "loss": 0.5411,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.5703771710395813,
-      "rewards/margins": 0.6444854736328125,
-      "rewards/rejected": -1.2148627042770386,
+      "logits/chosen": 1.0017188787460327,
+      "logits/rejected": 1.2839438915252686,
+      "logps/chosen": -368.5979919433594,
+      "logps/rejected": -424.36602783203125,
+      "loss": 0.5384,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.7962121963500977,
+      "rewards/margins": 0.717077910900116,
+      "rewards/rejected": -1.5132901668548584,
       "step": 390
     },
     {
       "epoch": 0.4186289900575615,
-      "grad_norm": 40.41797343152992,
+      "grad_norm": 41.86168586600665,
       "learning_rate": 3.607600562872785e-07,
-      "logits/chosen": -1.3911030292510986,
-      "logits/rejected": -1.366856575012207,
-      "logps/chosen": -342.7760925292969,
-      "logps/rejected": -384.2821960449219,
-      "loss": 0.5596,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.6101195216178894,
-      "rewards/margins": 0.5120452642440796,
-      "rewards/rejected": -1.1221649646759033,
+      "logits/chosen": 1.0247184038162231,
+      "logits/rejected": 1.24605131149292,
+      "logps/chosen": -376.79815673828125,
+      "logps/rejected": -423.98834228515625,
+      "loss": 0.5601,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.9503402709960938,
+      "rewards/margins": 0.5688854455947876,
+      "rewards/rejected": -1.5192257165908813,
       "step": 400
     },
     {
       "epoch": 0.4186289900575615,
-      "eval_logits/chosen": -1.538719892501831,
-      "eval_logits/rejected": -1.3655085563659668,
-      "eval_logps/chosen": -346.5540771484375,
-      "eval_logps/rejected": -384.7071838378906,
-      "eval_loss": 0.5451058745384216,
-      "eval_rewards/accuracies": 0.7579365372657776,
-      "eval_rewards/chosen": -0.5482582449913025,
-      "eval_rewards/margins": 0.6372901797294617,
-      "eval_rewards/rejected": -1.1855483055114746,
-      "eval_runtime": 205.4146,
-      "eval_samples_per_second": 9.736,
-      "eval_steps_per_second": 0.307,
+      "eval_logits/chosen": 0.6505773663520813,
+      "eval_logits/rejected": 1.0674785375595093,
+      "eval_logps/chosen": -375.93438720703125,
+      "eval_logps/rejected": -416.7413635253906,
+      "eval_loss": 0.5357086062431335,
+      "eval_rewards/accuracies": 0.7440476417541504,
+      "eval_rewards/chosen": -0.8420611619949341,
+      "eval_rewards/margins": 0.6638285517692566,
+      "eval_rewards/rejected": -1.505889654159546,
+      "eval_runtime": 206.6953,
+      "eval_samples_per_second": 9.676,
+      "eval_steps_per_second": 0.305,
       "step": 400
     },
     {
       "epoch": 0.4290947148090005,
-      "grad_norm": 21.999057622947905,
+      "grad_norm": 25.812282444686225,
       "learning_rate": 3.5249095128531856e-07,
-      "logits/chosen": -1.391875982284546,
-      "logits/rejected": -1.2355632781982422,
-      "logps/chosen": -341.1160888671875,
-      "logps/rejected": -364.1993713378906,
-      "loss": 0.5513,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.6004370450973511,
-      "rewards/margins": 0.5325397253036499,
-      "rewards/rejected": -1.1329768896102905,
+      "logits/chosen": 1.1224372386932373,
+      "logits/rejected": 1.557915210723877,
+      "logps/chosen": -370.46466064453125,
+      "logps/rejected": -398.7161560058594,
+      "loss": 0.5492,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.8939226269721985,
+      "rewards/margins": 0.5842228531837463,
+      "rewards/rejected": -1.4781454801559448,
       "step": 410
     },
     {
       "epoch": 0.43956043956043955,
-      "grad_norm": 24.300028128821324,
+      "grad_norm": 32.91824929440472,
       "learning_rate": 3.4408477372034736e-07,
-      "logits/chosen": -1.4640319347381592,
-      "logits/rejected": -1.2146576642990112,
-      "logps/chosen": -346.3941345214844,
-      "logps/rejected": -369.1965637207031,
-      "loss": 0.5549,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.5072788596153259,
-      "rewards/margins": 0.5944929122924805,
-      "rewards/rejected": -1.1017718315124512,
+      "logits/chosen": 1.1949050426483154,
+      "logits/rejected": 1.8893373012542725,
+      "logps/chosen": -370.3008728027344,
+      "logps/rejected": -400.72491455078125,
+      "loss": 0.5557,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.7463459372520447,
+      "rewards/margins": 0.6707097887992859,
+      "rewards/rejected": -1.417055606842041,
       "step": 420
     },
     {
       "epoch": 0.4500261643118786,
-      "grad_norm": 22.44936030672002,
+      "grad_norm": 30.820958673441478,
       "learning_rate": 3.3555276610977276e-07,
-      "logits/chosen": -1.2797183990478516,
-      "logits/rejected": -1.1341623067855835,
-      "logps/chosen": -323.0084533691406,
-      "logps/rejected": -373.71795654296875,
-      "loss": 0.5437,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.6086335778236389,
-      "rewards/margins": 0.5621797442436218,
-      "rewards/rejected": -1.1708133220672607,
+      "logits/chosen": 2.197765588760376,
+      "logits/rejected": 2.615821361541748,
+      "logps/chosen": -357.2638244628906,
+      "logps/rejected": -423.7203063964844,
+      "loss": 0.5308,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.9511874318122864,
+      "rewards/margins": 0.7196497917175293,
+      "rewards/rejected": -1.670837163925171,
       "step": 430
     },
     {
       "epoch": 0.4604918890633176,
-      "grad_norm": 58.53891780109035,
+      "grad_norm": 40.555206179922145,
       "learning_rate": 3.269063392575352e-07,
-      "logits/chosen": -1.0216747522354126,
-      "logits/rejected": -0.7915207743644714,
-      "logps/chosen": -390.4550476074219,
-      "logps/rejected": -403.381103515625,
-      "loss": 0.5516,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.9125455617904663,
-      "rewards/margins": 0.5674823522567749,
-      "rewards/rejected": -1.4800279140472412,
+      "logits/chosen": 2.289789915084839,
+      "logits/rejected": 2.8694961071014404,
+      "logps/chosen": -424.05615234375,
+      "logps/rejected": -446.7835998535156,
+      "loss": 0.5367,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.24855637550354,
+      "rewards/margins": 0.6654964685440063,
+      "rewards/rejected": -1.9140527248382568,
       "step": 440
     },
     {
       "epoch": 0.47095761381475665,
-      "grad_norm": 34.41084494366662,
+      "grad_norm": 29.74542911401454,
       "learning_rate": 3.1815705699316964e-07,
-      "logits/chosen": -0.9631792306900024,
-      "logits/rejected": -0.775741457939148,
-      "logps/chosen": -385.7535705566406,
-      "logps/rejected": -470.589111328125,
-      "loss": 0.538,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.9245363473892212,
-      "rewards/margins": 0.9464725255966187,
-      "rewards/rejected": -1.8710088729858398,
+      "logits/chosen": 2.255980968475342,
+      "logits/rejected": 2.68583345413208,
+      "logps/chosen": -406.7576904296875,
+      "logps/rejected": -499.15606689453125,
+      "loss": 0.5368,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.1345776319503784,
+      "rewards/margins": 1.0220998525619507,
+      "rewards/rejected": -2.156677722930908,
       "step": 450
     },
     {
       "epoch": 0.48142333856619574,
-      "grad_norm": 26.318881177937072,
+      "grad_norm": 29.408397209882185,
       "learning_rate": 3.0931662070620794e-07,
-      "logits/chosen": -0.9639442563056946,
-      "logits/rejected": -0.7544153928756714,
-      "logps/chosen": -342.7520751953125,
-      "logps/rejected": -388.74725341796875,
-      "loss": 0.5584,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.773398756980896,
-      "rewards/margins": 0.6263211369514465,
-      "rewards/rejected": -1.3997199535369873,
+      "logits/chosen": 2.3331820964813232,
+      "logits/rejected": 2.7528223991394043,
+      "logps/chosen": -381.4480895996094,
+      "logps/rejected": -433.6028747558594,
+      "loss": 0.5511,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.1603586673736572,
+      "rewards/margins": 0.6879166960716248,
+      "rewards/rejected": -1.8482754230499268,
       "step": 460
     },
     {
       "epoch": 0.49188906331763477,
-      "grad_norm": 42.18792096989277,
+      "grad_norm": 29.458598738113363,
       "learning_rate": 3.003968536966078e-07,
-      "logits/chosen": -1.0974524021148682,
-      "logits/rejected": -0.9117683172225952,
-      "logps/chosen": -347.5935363769531,
-      "logps/rejected": -371.4789123535156,
-      "loss": 0.5098,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.5606697797775269,
-      "rewards/margins": 0.618939995765686,
-      "rewards/rejected": -1.179609775543213,
+      "logits/chosen": 1.9355876445770264,
+      "logits/rejected": 2.3820812702178955,
+      "logps/chosen": -371.4092102050781,
+      "logps/rejected": -399.60906982421875,
+      "loss": 0.4975,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.7988266348838806,
+      "rewards/margins": 0.6620845198631287,
+      "rewards/rejected": -1.4609111547470093,
       "step": 470
     },
     {
       "epoch": 0.5023547880690737,
-      "grad_norm": 26.43002026478988,
+      "grad_norm": 35.38021870536312,
       "learning_rate": 2.9140968536213693e-07,
-      "logits/chosen": -0.8168869018554688,
-      "logits/rejected": -0.45926633477211,
-      "logps/chosen": -346.2986145019531,
-      "logps/rejected": -415.69403076171875,
-      "loss": 0.5151,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.70506352186203,
-      "rewards/margins": 0.8950220942497253,
-      "rewards/rejected": -1.6000858545303345,
+      "logits/chosen": 2.466848373413086,
+      "logits/rejected": 3.151698589324951,
+      "logps/chosen": -376.92083740234375,
+      "logps/rejected": -452.67510986328125,
+      "loss": 0.5106,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.0112861394882202,
+      "rewards/margins": 0.9586098790168762,
+      "rewards/rejected": -1.9698959589004517,
       "step": 480
     },
     {
       "epoch": 0.5128205128205128,
-      "grad_norm": 26.075603530748978,
+      "grad_norm": 35.17308990777428,
       "learning_rate": 2.823671352438608e-07,
-      "logits/chosen": -0.8554686307907104,
-      "logits/rejected": -0.26327481865882874,
-      "logps/chosen": -393.8193664550781,
-      "logps/rejected": -441.4097595214844,
-      "loss": 0.5253,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.7199895977973938,
-      "rewards/margins": 1.007552146911621,
-      "rewards/rejected": -1.7275416851043701,
+      "logits/chosen": 2.032174587249756,
+      "logits/rejected": 3.158629894256592,
+      "logps/chosen": -413.9285583496094,
+      "logps/rejected": -465.38568115234375,
+      "loss": 0.5146,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.9210812449455261,
+      "rewards/margins": 1.046219825744629,
+      "rewards/rejected": -1.9673011302947998,
       "step": 490
     },
     {
       "epoch": 0.5232862375719518,
-      "grad_norm": 46.13803118076279,
+      "grad_norm": 26.43610032807784,
       "learning_rate": 2.73281296951072e-07,
-      "logits/chosen": -0.16928303241729736,
-      "logits/rejected": 0.03523734211921692,
-      "logps/chosen": -358.99627685546875,
-      "logps/rejected": -392.8810119628906,
-      "loss": 0.5352,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.9152957201004028,
-      "rewards/margins": 0.707416296005249,
-      "rewards/rejected": -1.6227118968963623,
+      "logits/chosen": 2.640186071395874,
+      "logits/rejected": 3.0224225521087646,
+      "logps/chosen": -356.60205078125,
+      "logps/rejected": -398.59320068359375,
+      "loss": 0.523,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.8913534879684448,
+      "rewards/margins": 0.7884796857833862,
+      "rewards/rejected": -1.679833173751831,
       "step": 500
     },
     {
       "epoch": 0.5232862375719518,
-      "eval_logits/chosen": -0.40079495310783386,
-      "eval_logits/rejected": -0.09374210983514786,
-      "eval_logps/chosen": -384.1307067871094,
-      "eval_logps/rejected": -433.48065185546875,
-      "eval_loss": 0.5308576822280884,
-      "eval_rewards/accuracies": 0.7440476417541504,
-      "eval_rewards/chosen": -0.9240241050720215,
-      "eval_rewards/margins": 0.7492585182189941,
-      "eval_rewards/rejected": -1.6732827425003052,
-      "eval_runtime": 205.6895,
-      "eval_samples_per_second": 9.723,
-      "eval_steps_per_second": 0.306,
+      "eval_logits/chosen": 2.2498488426208496,
+      "eval_logits/rejected": 2.780888080596924,
+      "eval_logps/chosen": -394.3706359863281,
+      "eval_logps/rejected": -450.094482421875,
+      "eval_loss": 0.5213772058486938,
+      "eval_rewards/accuracies": 0.7599206566810608,
+      "eval_rewards/chosen": -1.0264240503311157,
+      "eval_rewards/margins": 0.8129974603652954,
+      "eval_rewards/rejected": -1.8394215106964111,
+      "eval_runtime": 206.4862,
+      "eval_samples_per_second": 9.686,
+      "eval_steps_per_second": 0.305,
       "step": 500
     },
     {
       "epoch": 0.533751962323391,
-      "grad_norm": 35.03408862664356,
+      "grad_norm": 28.724399791521353,
       "learning_rate": 2.641643219871597e-07,
-      "logits/chosen": -0.4317537248134613,
-      "logits/rejected": -0.18362993001937866,
-      "logps/chosen": -378.89105224609375,
-      "logps/rejected": -445.74951171875,
-      "loss": 0.5042,
+      "logits/chosen": 2.298245906829834,
+      "logits/rejected": 2.758460521697998,
+      "logps/chosen": -393.7535400390625,
+      "logps/rejected": -473.3352966308594,
+      "loss": 0.4929,
       "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.8569504618644714,
-      "rewards/margins": 0.902667224407196,
-      "rewards/rejected": -1.759617805480957,
+      "rewards/chosen": -1.0055750608444214,
+      "rewards/margins": 1.0299009084701538,
+      "rewards/rejected": -2.0354762077331543,
       "step": 510
     },
     {
       "epoch": 0.54421768707483,
-      "grad_norm": 34.20713430220626,
+      "grad_norm": 34.770815041335155,
       "learning_rate": 2.550284034980507e-07,
-      "logits/chosen": 0.1857350617647171,
-      "logits/rejected": 0.4090241491794586,
-      "logps/chosen": -419.68463134765625,
-      "logps/rejected": -458.61297607421875,
-      "loss": 0.5497,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -1.3962366580963135,
-      "rewards/margins": 0.5962225198745728,
-      "rewards/rejected": -1.9924590587615967,
+      "logits/chosen": 2.6805074214935303,
+      "logits/rejected": 2.9783804416656494,
+      "logps/chosen": -444.7898864746094,
+      "logps/rejected": -489.47747802734375,
+      "loss": 0.5437,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6472896337509155,
+      "rewards/margins": 0.653814971446991,
+      "rewards/rejected": -2.30110502243042,
       "step": 520
     },
     {
       "epoch": 0.554683411826269,
-      "grad_norm": 57.42602366755521,
+      "grad_norm": 45.470317006583116,
       "learning_rate": 2.4588575996495794e-07,
-      "logits/chosen": 0.09937302768230438,
-      "logits/rejected": 0.3888925313949585,
-      "logps/chosen": -389.2012023925781,
-      "logps/rejected": -448.75775146484375,
-      "loss": 0.5306,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.1193468570709229,
-      "rewards/margins": 0.7478076219558716,
-      "rewards/rejected": -1.8671547174453735,
+      "logits/chosen": 2.448279857635498,
+      "logits/rejected": 2.818011522293091,
+      "logps/chosen": -400.88739013671875,
+      "logps/rejected": -465.2130432128906,
+      "loss": 0.5235,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.2362090349197388,
+      "rewards/margins": 0.795498251914978,
+      "rewards/rejected": -2.031707286834717,
       "step": 530
     },
     {
       "epoch": 0.565149136577708,
-      "grad_norm": 24.347954290505303,
+      "grad_norm": 27.51988173378998,
       "learning_rate": 2.367486188632446e-07,
-      "logits/chosen": -0.15624569356441498,
-      "logits/rejected": 0.2739402949810028,
-      "logps/chosen": -381.8554382324219,
-      "logps/rejected": -420.7781677246094,
-      "loss": 0.5451,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.8271482586860657,
-      "rewards/margins": 0.7216792702674866,
-      "rewards/rejected": -1.5488274097442627,
+      "logits/chosen": 2.0589711666107178,
+      "logits/rejected": 2.798044443130493,
+      "logps/chosen": -393.33807373046875,
+      "logps/rejected": -435.16839599609375,
+      "loss": 0.5314,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.9419742822647095,
+      "rewards/margins": 0.7507559061050415,
+      "rewards/rejected": -1.692730188369751,
       "step": 540
     },
     {
       "epoch": 0.5756148613291471,
-      "grad_norm": 37.95198659343882,
+      "grad_norm": 35.19191904316796,
       "learning_rate": 2.276292003092593e-07,
-      "logits/chosen": -0.1706000417470932,
-      "logits/rejected": 0.3523309826850891,
-      "logps/chosen": -358.3487854003906,
-      "logps/rejected": -394.71575927734375,
-      "loss": 0.5281,
+      "logits/chosen": 2.2020509243011475,
+      "logits/rejected": 3.076523542404175,
+      "logps/chosen": -378.48150634765625,
+      "logps/rejected": -417.3294982910156,
+      "loss": 0.5218,
       "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.7871420979499817,
-      "rewards/margins": 0.8284309506416321,
-      "rewards/rejected": -1.6155731678009033,
+      "rewards/chosen": -0.9884693026542664,
+      "rewards/margins": 0.8532408475875854,
+      "rewards/rejected": -1.8417103290557861,
       "step": 550
     },
     {
       "epoch": 0.5860805860805861,
-      "grad_norm": 31.617763354602737,
+      "grad_norm": 37.71817738191452,
       "learning_rate": 2.185397007170141e-07,
-      "logits/chosen": 0.26206716895103455,
-      "logits/rejected": 0.7556803822517395,
-      "logps/chosen": -376.7598571777344,
-      "logps/rejected": -428.24951171875,
-      "loss": 0.5501,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.8537102937698364,
-      "rewards/margins": 0.808838963508606,
-      "rewards/rejected": -1.6625492572784424,
+      "logits/chosen": 2.5697808265686035,
+      "logits/rejected": 3.3313114643096924,
+      "logps/chosen": -399.2616271972656,
+      "logps/rejected": -456.85888671875,
+      "loss": 0.5412,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.0787277221679688,
+      "rewards/margins": 0.8699148893356323,
+      "rewards/rejected": -1.9486424922943115,
       "step": 560
     },
     {
       "epoch": 0.5965463108320251,
-      "grad_norm": 34.132968228975315,
+      "grad_norm": 37.430403293642115,
       "learning_rate": 2.094922764865619e-07,
-      "logits/chosen": 0.1466689556837082,
-      "logits/rejected": 0.5085517168045044,
-      "logps/chosen": -406.31854248046875,
-      "logps/rejected": -458.3421936035156,
-      "loss": 0.5302,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.196199655532837,
-      "rewards/margins": 0.6397031545639038,
-      "rewards/rejected": -1.8359029293060303,
+      "logits/chosen": 2.5887045860290527,
+      "logits/rejected": 3.0508625507354736,
+      "logps/chosen": -441.4580078125,
+      "logps/rejected": -495.4439392089844,
+      "loss": 0.5161,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.5475947856903076,
+      "rewards/margins": 0.6593255996704102,
+      "rewards/rejected": -2.2069201469421387,
       "step": 570
     },
     {
       "epoch": 0.6070120355834642,
-      "grad_norm": 32.91671633286688,
+      "grad_norm": 35.56759521621647,
       "learning_rate": 2.0049902774588797e-07,
-      "logits/chosen": 0.3107803463935852,
-      "logits/rejected": 0.6928397417068481,
-      "logps/chosen": -365.2535400390625,
-      "logps/rejected": -391.4024963378906,
-      "loss": 0.54,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.0598257780075073,
-      "rewards/margins": 0.707074761390686,
-      "rewards/rejected": -1.766900658607483,
+      "logits/chosen": 2.469163417816162,
+      "logits/rejected": 3.017552137374878,
+      "logps/chosen": -387.0569763183594,
+      "logps/rejected": -423.3297424316406,
+      "loss": 0.5279,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.2778595685958862,
+      "rewards/margins": 0.8083131909370422,
+      "rewards/rejected": -2.0861728191375732,
       "step": 580
     },
     {
       "epoch": 0.6174777603349032,
-      "grad_norm": 29.0857374634859,
+      "grad_norm": 27.528988989887985,
       "learning_rate": 1.9157198216806238e-07,
-      "logits/chosen": 0.2890316843986511,
-      "logits/rejected": 0.8040902018547058,
-      "logps/chosen": -397.47650146484375,
-      "logps/rejected": -457.5138244628906,
-      "loss": 0.5115,
+      "logits/chosen": 2.1037211418151855,
+      "logits/rejected": 2.916111469268799,
+      "logps/chosen": -424.4861755371094,
+      "logps/rejected": -490.6454162597656,
+      "loss": 0.5016,
       "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.0694525241851807,
-      "rewards/margins": 0.7707843780517578,
-      "rewards/rejected": -1.8402369022369385,
+      "rewards/chosen": -1.3395488262176514,
+      "rewards/margins": 0.8320037722587585,
+      "rewards/rejected": -2.1715526580810547,
       "step": 590
     },
     {
       "epoch": 0.6279434850863422,
-      "grad_norm": 36.666534677506704,
+      "grad_norm": 30.03804365343253,
       "learning_rate": 1.8272307888529274e-07,
-      "logits/chosen": 1.0131226778030396,
-      "logits/rejected": 1.3981071710586548,
-      "logps/chosen": -401.26397705078125,
-      "logps/rejected": -456.7733459472656,
-      "loss": 0.4998,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.118422269821167,
-      "rewards/margins": 0.8214613199234009,
-      "rewards/rejected": -1.9398835897445679,
+      "logits/chosen": 2.898608922958374,
+      "logits/rejected": 3.4447216987609863,
+      "logps/chosen": -423.91363525390625,
+      "logps/rejected": -476.47100830078125,
+      "loss": 0.4939,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.3449193239212036,
+      "rewards/margins": 0.7919400930404663,
+      "rewards/rejected": -2.13685941696167,
       "step": 600
     },
     {
       "epoch": 0.6279434850863422,
-      "eval_logits/chosen": 0.6984339356422424,
-      "eval_logits/rejected": 1.1205775737762451,
-      "eval_logps/chosen": -395.91461181640625,
-      "eval_logps/rejected": -453.32965087890625,
-      "eval_loss": 0.522526741027832,
-      "eval_rewards/accuracies": 0.7400793433189392,
-      "eval_rewards/chosen": -1.0418637990951538,
-      "eval_rewards/margins": 0.8299089670181274,
-      "eval_rewards/rejected": -1.8717727661132812,
-      "eval_runtime": 206.9196,
-      "eval_samples_per_second": 9.666,
+      "eval_logits/chosen": 2.383793592453003,
+      "eval_logits/rejected": 2.9773049354553223,
+      "eval_logps/chosen": -413.4645080566406,
+      "eval_logps/rejected": -471.979736328125,
+      "eval_loss": 0.5187502503395081,
+      "eval_rewards/accuracies": 0.7599206566810608,
+      "eval_rewards/chosen": -1.2173622846603394,
+      "eval_rewards/margins": 0.840911328792572,
+      "eval_rewards/rejected": -2.0582737922668457,
+      "eval_runtime": 206.8983,
+      "eval_samples_per_second": 9.667,
       "eval_steps_per_second": 0.304,
       "step": 600
     },
     {
       "epoch": 0.6384092098377813,
-      "grad_norm": 30.673794835032183,
+      "grad_norm": 32.637049775241344,
       "learning_rate": 1.7396415252139288e-07,
-      "logits/chosen": 0.5988572835922241,
-      "logits/rejected": 1.299567461013794,
-      "logps/chosen": -417.8856506347656,
-      "logps/rejected": -446.78515625,
-      "loss": 0.5258,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.0063849687576294,
-      "rewards/margins": 0.9513689875602722,
-      "rewards/rejected": -1.9577538967132568,
+      "logits/chosen": 2.2484850883483887,
+      "logits/rejected": 3.253376007080078,
+      "logps/chosen": -432.353515625,
+      "logps/rejected": -465.94171142578125,
+      "loss": 0.5206,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.1510635614395142,
+      "rewards/margins": 0.9982560276985168,
+      "rewards/rejected": -2.1493194103240967,
       "step": 610
     },
     {
       "epoch": 0.6488749345892203,
-      "grad_norm": 32.211994842231405,
+      "grad_norm": 34.11849631387884,
       "learning_rate": 1.6530691736402316e-07,
-      "logits/chosen": 0.850233256816864,
-      "logits/rejected": 1.220767617225647,
-      "logps/chosen": -387.00848388671875,
-      "logps/rejected": -435.15289306640625,
-      "loss": 0.4986,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.0632097721099854,
-      "rewards/margins": 0.8516036868095398,
-      "rewards/rejected": -1.9148136377334595,
+      "logits/chosen": 2.6262052059173584,
+      "logits/rejected": 3.1429150104522705,
+      "logps/chosen": -395.3048095703125,
+      "logps/rejected": -444.4956970214844,
+      "loss": 0.4973,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.1461726427078247,
+      "rewards/margins": 0.8620686531066895,
+      "rewards/rejected": -2.0082411766052246,
       "step": 620
     },
     {
       "epoch": 0.6593406593406593,
-      "grad_norm": 38.864669583620696,
+      "grad_norm": 38.951652152964755,
       "learning_rate": 1.5676295169786864e-07,
-      "logits/chosen": 0.809965968132019,
-      "logits/rejected": 1.443335771560669,
-      "logps/chosen": -389.06817626953125,
-      "logps/rejected": -432.38336181640625,
-      "loss": 0.4981,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.0629326105117798,
-      "rewards/margins": 0.8624936938285828,
-      "rewards/rejected": -1.9254264831542969,
+      "logits/chosen": 2.486433744430542,
+      "logits/rejected": 3.3269076347351074,
+      "logps/chosen": -393.5639953613281,
+      "logps/rejected": -434.69805908203125,
+      "loss": 0.5002,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.1078909635543823,
+      "rewards/margins": 0.8406828045845032,
+      "rewards/rejected": -1.9485734701156616,
       "step": 630
     },
     {
       "epoch": 0.6698063840920984,
-      "grad_norm": 34.096975466021526,
+      "grad_norm": 36.483587011604335,
       "learning_rate": 1.483436823197092e-07,
-      "logits/chosen": 1.4999452829360962,
-      "logits/rejected": 1.9908607006072998,
-      "logps/chosen": -405.83526611328125,
-      "logps/rejected": -450.565185546875,
-      "loss": 0.502,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -1.3165745735168457,
-      "rewards/margins": 0.8249285817146301,
-      "rewards/rejected": -2.141503095626831,
+      "logits/chosen": 3.2118659019470215,
+      "logits/rejected": 3.877216339111328,
+      "logps/chosen": -401.62249755859375,
+      "logps/rejected": -449.34637451171875,
+      "loss": 0.4907,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.274446964263916,
+      "rewards/margins": 0.8548680543899536,
+      "rewards/rejected": -2.129315137863159,
       "step": 640
     },
     {
       "epoch": 0.6802721088435374,
-      "grad_norm": 34.771815421003645,
+      "grad_norm": 35.66740450417234,
       "learning_rate": 1.4006036925609243e-07,
-      "logits/chosen": 1.224877953529358,
-      "logits/rejected": 1.449184775352478,
-      "logps/chosen": -389.7561340332031,
-      "logps/rejected": -456.34295654296875,
-      "loss": 0.517,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.1494994163513184,
-      "rewards/margins": 0.6449583768844604,
-      "rewards/rejected": -1.7944577932357788,
+      "logits/chosen": 2.771915912628174,
+      "logits/rejected": 3.053302049636841,
+      "logps/chosen": -380.0518493652344,
+      "logps/rejected": -448.572021484375,
+      "loss": 0.5125,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.0524568557739258,
+      "rewards/margins": 0.6642917394638062,
+      "rewards/rejected": -1.7167484760284424,
       "step": 650
     },
     {
       "epoch": 0.6907378335949764,
-      "grad_norm": 31.45885241937768,
+      "grad_norm": 34.971599333443486,
       "learning_rate": 1.319240907040458e-07,
-      "logits/chosen": 1.1873584985733032,
-      "logits/rejected": 1.763824462890625,
-      "logps/chosen": -402.32806396484375,
-      "logps/rejected": -468.66339111328125,
-      "loss": 0.4886,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.120466709136963,
-      "rewards/margins": 0.9276604652404785,
-      "rewards/rejected": -2.0481271743774414,
+      "logits/chosen": 2.6383063793182373,
+      "logits/rejected": 3.400423765182495,
+      "logps/chosen": -387.20208740234375,
+      "logps/rejected": -454.57000732421875,
+      "loss": 0.487,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.9692069888114929,
+      "rewards/margins": 0.9379868507385254,
+      "rewards/rejected": -1.9071937799453735,
       "step": 660
     },
     {
       "epoch": 0.7012035583464155,
-      "grad_norm": 42.31478816659526,
+      "grad_norm": 40.7843968056207,
       "learning_rate": 1.239457282149695e-07,
-      "logits/chosen": 1.06121826171875,
-      "logits/rejected": 1.486613154411316,
-      "logps/chosen": -420.27947998046875,
-      "logps/rejected": -495.8140563964844,
-      "loss": 0.4935,
+      "logits/chosen": 2.419273614883423,
+      "logits/rejected": 2.95436429977417,
+      "logps/chosen": -410.73583984375,
+      "logps/rejected": -493.061767578125,
+      "loss": 0.4859,
       "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.2196897268295288,
-      "rewards/margins": 0.8256422877311707,
-      "rewards/rejected": -2.045332193374634,
+      "rewards/chosen": -1.1242530345916748,
+      "rewards/margins": 0.8935553431510925,
+      "rewards/rejected": -2.017808437347412,
       "step": 670
     },
     {
       "epoch": 0.7116692830978545,
-      "grad_norm": 46.494592998931914,
+      "grad_norm": 40.49316743258316,
       "learning_rate": 1.1613595214152711e-07,
-      "logits/chosen": 1.2637029886245728,
-      "logits/rejected": 1.7961336374282837,
-      "logps/chosen": -429.7030334472656,
-      "logps/rejected": -490.2943420410156,
-      "loss": 0.5403,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.1113157272338867,
-      "rewards/margins": 0.8543672561645508,
-      "rewards/rejected": -1.9656829833984375,
+      "logits/chosen": 2.7838759422302246,
+      "logits/rejected": 3.4620018005371094,
+      "logps/chosen": -430.22052001953125,
+      "logps/rejected": -497.37286376953125,
+      "loss": 0.5405,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.116490364074707,
+      "rewards/margins": 0.9199777841567993,
+      "rewards/rejected": -2.036468029022217,
       "step": 680
     },
     {
       "epoch": 0.7221350078492935,
-      "grad_norm": 28.22622857193106,
+      "grad_norm": 28.164057249709177,
       "learning_rate": 1.0850520736699362e-07,
-      "logits/chosen": 1.1677778959274292,
-      "logits/rejected": 1.9492908716201782,
-      "logps/chosen": -397.92999267578125,
-      "logps/rejected": -434.52227783203125,
-      "loss": 0.4864,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.1409690380096436,
-      "rewards/margins": 0.8163288235664368,
-      "rewards/rejected": -1.9572980403900146,
+      "logits/chosen": 2.535062074661255,
+      "logits/rejected": 3.562615156173706,
+      "logps/chosen": -399.06634521484375,
+      "logps/rejected": -448.0703125,
+      "loss": 0.4741,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.1523323059082031,
+      "rewards/margins": 0.9404458999633789,
+      "rewards/rejected": -2.092777967453003,
       "step": 690
     },
     {
       "epoch": 0.7326007326007326,
-      "grad_norm": 38.82903353111915,
+      "grad_norm": 35.32971017140881,
       "learning_rate": 1.0106369933615042e-07,
-      "logits/chosen": 1.49738347530365,
-      "logits/rejected": 1.8964507579803467,
-      "logps/chosen": -387.61126708984375,
-      "logps/rejected": -441.9359436035156,
-      "loss": 0.4936,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.250970482826233,
-      "rewards/margins": 0.7425969839096069,
-      "rewards/rejected": -1.9935674667358398,
+      "logits/chosen": 2.851242780685425,
+      "logits/rejected": 3.3012306690216064,
+      "logps/chosen": -387.40582275390625,
+      "logps/rejected": -443.54388427734375,
+      "loss": 0.4934,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.2489162683486938,
+      "rewards/margins": 0.7607301473617554,
+      "rewards/rejected": -2.009646415710449,
       "step": 700
     },
     {
       "epoch": 0.7326007326007326,
-      "eval_logits/chosen": 1.4183756113052368,
-      "eval_logits/rejected": 1.9631398916244507,
-      "eval_logps/chosen": -412.60626220703125,
-      "eval_logps/rejected": -475.4923400878906,
-      "eval_loss": 0.5196009874343872,
-      "eval_rewards/accuracies": 0.7539682388305664,
-      "eval_rewards/chosen": -1.2087799310684204,
-      "eval_rewards/margins": 0.8846201300621033,
-      "eval_rewards/rejected": -2.0933997631073,
-      "eval_runtime": 206.1126,
-      "eval_samples_per_second": 9.703,
-      "eval_steps_per_second": 0.306,
+      "eval_logits/chosen": 2.673452377319336,
+      "eval_logits/rejected": 3.3092777729034424,
+      "eval_logps/chosen": -415.25482177734375,
+      "eval_logps/rejected": -479.7106628417969,
+      "eval_loss": 0.5118083357810974,
+      "eval_rewards/accuracies": 0.7698412537574768,
+      "eval_rewards/chosen": -1.235266089439392,
+      "eval_rewards/margins": 0.900316596031189,
+      "eval_rewards/rejected": -2.135582447052002,
+      "eval_runtime": 206.4499,
+      "eval_samples_per_second": 9.688,
+      "eval_steps_per_second": 0.305,
       "step": 700
     },
     {
       "epoch": 0.7430664573521716,
-      "grad_norm": 41.9438808858937,
+      "grad_norm": 33.82532864442472,
       "learning_rate": 9.382138040640714e-08,
-      "logits/chosen": 1.7710202932357788,
-      "logits/rejected": 2.3763861656188965,
-      "logps/chosen": -406.9272155761719,
-      "logps/rejected": -463.2557067871094,
-      "loss": 0.5672,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.4213149547576904,
-      "rewards/margins": 0.7025940418243408,
-      "rewards/rejected": -2.1239089965820312,
+      "logits/chosen": 3.0481176376342773,
+      "logits/rejected": 3.7348952293395996,
+      "logps/chosen": -400.9176025390625,
+      "logps/rejected": -465.6322326660156,
+      "loss": 0.5519,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.361219048500061,
+      "rewards/margins": 0.7864547371864319,
+      "rewards/rejected": -2.1476736068725586,
       "step": 710
     },
     {
       "epoch": 0.7535321821036107,
-      "grad_norm": 48.08337125318838,
+      "grad_norm": 30.695789285988596,
       "learning_rate": 8.678793653740632e-08,
-      "logits/chosen": 1.6012312173843384,
-      "logits/rejected": 2.216918468475342,
-      "logps/chosen": -419.3685607910156,
-      "logps/rejected": -477.35003662109375,
-      "loss": 0.5009,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.2644264698028564,
-      "rewards/margins": 0.9790066480636597,
-      "rewards/rejected": -2.2434329986572266,
+      "logits/chosen": 2.8603994846343994,
+      "logits/rejected": 3.5425021648406982,
+      "logps/chosen": -408.23785400390625,
+      "logps/rejected": -471.1385192871094,
+      "loss": 0.4848,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.153119444847107,
+      "rewards/margins": 1.028198480606079,
+      "rewards/rejected": -2.1813182830810547,
       "step": 720
     },
     {
       "epoch": 0.7639979068550498,
-      "grad_norm": 40.621592704466096,
+      "grad_norm": 36.30696542372595,
       "learning_rate": 7.997277433690983e-08,
-      "logits/chosen": 1.654722809791565,
-      "logits/rejected": 2.163034677505493,
-      "logps/chosen": -413.20355224609375,
-      "logps/rejected": -446.3681640625,
-      "loss": 0.5238,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -1.399733543395996,
-      "rewards/margins": 0.6443352699279785,
-      "rewards/rejected": -2.0440685749053955,
+      "logits/chosen": 2.892194986343384,
+      "logits/rejected": 3.5528037548065186,
+      "logps/chosen": -413.70111083984375,
+      "logps/rejected": -450.5099182128906,
+      "loss": 0.5201,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.4047088623046875,
+      "rewards/margins": 0.6807772517204285,
+      "rewards/rejected": -2.0854861736297607,
       "step": 730
     },
     {
       "epoch": 0.7744636316064888,
-      "grad_norm": 44.11198771694061,
+      "grad_norm": 34.18486907607281,
       "learning_rate": 7.338500848029602e-08,
-      "logits/chosen": 2.0758748054504395,
-      "logits/rejected": 2.3728959560394287,
-      "logps/chosen": -388.3576965332031,
-      "logps/rejected": -435.0791015625,
-      "loss": 0.5122,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.283231496810913,
-      "rewards/margins": 0.7193259000778198,
-      "rewards/rejected": -2.0025575160980225,
+      "logits/chosen": 3.372150421142578,
+      "logits/rejected": 3.714134931564331,
+      "logps/chosen": -387.54248046875,
+      "logps/rejected": -435.57421875,
+      "loss": 0.5157,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.2750799655914307,
+      "rewards/margins": 0.7324288487434387,
+      "rewards/rejected": -2.0075085163116455,
       "step": 740
     },
     {
       "epoch": 0.7849293563579278,
-      "grad_norm": 39.45060030130625,
+      "grad_norm": 36.701039025278824,
       "learning_rate": 6.70334495204884e-08,
-      "logits/chosen": 1.8493086099624634,
-      "logits/rejected": 2.2490105628967285,
-      "logps/chosen": -393.843994140625,
-      "logps/rejected": -475.54132080078125,
-      "loss": 0.4984,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.3790090084075928,
-      "rewards/margins": 0.8863463401794434,
-      "rewards/rejected": -2.265355348587036,
+      "logits/chosen": 3.0573837757110596,
+      "logits/rejected": 3.5534985065460205,
+      "logps/chosen": -392.74163818359375,
+      "logps/rejected": -474.88836669921875,
+      "loss": 0.496,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.367985725402832,
+      "rewards/margins": 0.8908400535583496,
+      "rewards/rejected": -2.2588257789611816,
       "step": 750
     },
     {
       "epoch": 0.7953950811093669,
-      "grad_norm": 41.13021738331374,
+      "grad_norm": 31.037337423780286,
       "learning_rate": 6.092659210462231e-08,
-      "logits/chosen": 1.9165847301483154,
-      "logits/rejected": 2.292691707611084,
-      "logps/chosen": -383.63818359375,
-      "logps/rejected": -474.6698303222656,
-      "loss": 0.5255,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.3413493633270264,
-      "rewards/margins": 0.8881141543388367,
-      "rewards/rejected": -2.229463577270508,
+      "logits/chosen": 3.2883152961730957,
+      "logits/rejected": 3.7196197509765625,
+      "logps/chosen": -383.4204406738281,
+      "logps/rejected": -471.00799560546875,
+      "loss": 0.522,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.3391718864440918,
+      "rewards/margins": 0.8536735773086548,
+      "rewards/rejected": -2.192845582962036,
       "step": 760
     },
     {
       "epoch": 0.8058608058608059,
-      "grad_norm": 23.227510052162174,
+      "grad_norm": 22.73454656358492,
       "learning_rate": 5.507260361320737e-08,
-      "logits/chosen": 1.4680651426315308,
-      "logits/rejected": 1.5410151481628418,
-      "logps/chosen": -441.0107421875,
-      "logps/rejected": -535.9873657226562,
-      "loss": 0.4871,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.246150255203247,
-      "rewards/margins": 0.7787067294120789,
-      "rewards/rejected": -2.0248570442199707,
+      "logits/chosen": 2.8491129875183105,
+      "logits/rejected": 2.862048625946045,
+      "logps/chosen": -440.95587158203125,
+      "logps/rejected": -535.2119750976562,
+      "loss": 0.4834,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.245601773262024,
+      "rewards/margins": 0.7715011835098267,
+      "rewards/rejected": -2.0171029567718506,
       "step": 770
     },
     {
       "epoch": 0.8163265306122449,
-      "grad_norm": 40.34737389909196,
+      "grad_norm": 40.929399673370185,
       "learning_rate": 4.947931323697982e-08,
-      "logits/chosen": 1.382004976272583,
-      "logits/rejected": 1.967145562171936,
-      "logps/chosen": -438.35150146484375,
-      "logps/rejected": -489.5238342285156,
-      "loss": 0.5137,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.1223596334457397,
-      "rewards/margins": 0.9631366729736328,
-      "rewards/rejected": -2.085496425628662,
+      "logits/chosen": 2.7183022499084473,
+      "logits/rejected": 3.416518449783325,
+      "logps/chosen": -439.4905700683594,
+      "logps/rejected": -486.2300720214844,
+      "loss": 0.5045,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.133750081062317,
+      "rewards/margins": 0.9188081622123718,
+      "rewards/rejected": -2.052558422088623,
       "step": 780
     },
     {
       "epoch": 0.826792255363684,
-      "grad_norm": 40.27301508170132,
+      "grad_norm": 35.79455220387355,
       "learning_rate": 4.415420150605398e-08,
-      "logits/chosen": 1.519224762916565,
-      "logits/rejected": 2.341888904571533,
-      "logps/chosen": -414.23681640625,
-      "logps/rejected": -462.8794860839844,
-      "loss": 0.5392,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.2749950885772705,
-      "rewards/margins": 0.8448735475540161,
-      "rewards/rejected": -2.119868516921997,
+      "logits/chosen": 2.907578706741333,
+      "logits/rejected": 3.9217536449432373,
+      "logps/chosen": -410.330322265625,
+      "logps/rejected": -462.1753845214844,
+      "loss": 0.5317,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.2359302043914795,
+      "rewards/margins": 0.8768974542617798,
+      "rewards/rejected": -2.112827777862549,
       "step": 790
     },
     {
       "epoch": 0.837257980115123,
-      "grad_norm": 45.75902031403285,
+      "grad_norm": 37.70718613231154,
       "learning_rate": 3.9104390285376374e-08,
-      "logits/chosen": 1.8431495428085327,
-      "logits/rejected": 2.1607654094696045,
-      "logps/chosen": -412.042236328125,
-      "logps/rejected": -483.38787841796875,
-      "loss": 0.5125,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.3601198196411133,
-      "rewards/margins": 0.7701460719108582,
-      "rewards/rejected": -2.130265712738037,
+      "logits/chosen": 3.2915122509002686,
+      "logits/rejected": 3.6490883827209473,
+      "logps/chosen": -407.34381103515625,
+      "logps/rejected": -480.871826171875,
+      "loss": 0.4975,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.3131351470947266,
+      "rewards/margins": 0.791969895362854,
+      "rewards/rejected": -2.105104923248291,
       "step": 800
     },
     {
       "epoch": 0.837257980115123,
-      "eval_logits/chosen": 1.3314255475997925,
-      "eval_logits/rejected": 1.873736023902893,
-      "eval_logps/chosen": -410.9924011230469,
-      "eval_logps/rejected": -473.3307189941406,
-      "eval_loss": 0.5170913338661194,
-      "eval_rewards/accuracies": 0.7658730149269104,
-      "eval_rewards/chosen": -1.1926417350769043,
-      "eval_rewards/margins": 0.8791419267654419,
-      "eval_rewards/rejected": -2.0717833042144775,
-      "eval_runtime": 206.6421,
-      "eval_samples_per_second": 9.679,
+      "eval_logits/chosen": 2.7191321849823,
+      "eval_logits/rejected": 3.346607208251953,
+      "eval_logps/chosen": -406.977294921875,
+      "eval_logps/rejected": -468.6864318847656,
+      "eval_loss": 0.5095987915992737,
+      "eval_rewards/accuracies": 0.7678571343421936,
+      "eval_rewards/chosen": -1.152490258216858,
+      "eval_rewards/margins": 0.8728507161140442,
+      "eval_rewards/rejected": -2.025341033935547,
+      "eval_runtime": 206.7923,
+      "eval_samples_per_second": 9.672,
       "eval_steps_per_second": 0.305,
       "step": 800
     },
     {
       "epoch": 0.847723704866562,
-      "grad_norm": 32.21448583582019,
+      "grad_norm": 29.023787850929335,
       "learning_rate": 3.433663324986208e-08,
-      "logits/chosen": 1.3725779056549072,
-      "logits/rejected": 2.195798873901367,
-      "logps/chosen": -422.95086669921875,
-      "logps/rejected": -460.2098083496094,
-      "loss": 0.5186,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.224956750869751,
-      "rewards/margins": 0.8695290684700012,
-      "rewards/rejected": -2.0944857597351074,
+      "logits/chosen": 2.833073377609253,
+      "logits/rejected": 3.8449854850769043,
+      "logps/chosen": -421.5035095214844,
+      "logps/rejected": -458.6492614746094,
+      "loss": 0.5131,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.2104833126068115,
+      "rewards/margins": 0.8683967590332031,
+      "rewards/rejected": -2.0788798332214355,
       "step": 810
     },
     {
       "epoch": 0.858189429618001,
-      "grad_norm": 32.56199170937255,
+      "grad_norm": 31.369371514199123,
       "learning_rate": 2.9857306851953897e-08,
-      "logits/chosen": 1.3711284399032593,
-      "logits/rejected": 1.8220380544662476,
-      "logps/chosen": -428.7027893066406,
-      "logps/rejected": -492.301025390625,
-      "loss": 0.5116,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.1508777141571045,
-      "rewards/margins": 0.8758268356323242,
-      "rewards/rejected": -2.0267043113708496,
+      "logits/chosen": 2.8905856609344482,
+      "logits/rejected": 3.3920249938964844,
+      "logps/chosen": -426.568603515625,
+      "logps/rejected": -487.6808166503906,
+      "loss": 0.514,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.1295357942581177,
+      "rewards/margins": 0.8509663343429565,
+      "rewards/rejected": -1.9805021286010742,
       "step": 820
     },
     {
       "epoch": 0.8686551543694401,
-      "grad_norm": 30.701473639362177,
+      "grad_norm": 32.38184768311142,
       "learning_rate": 2.567240179368185e-08,
-      "logits/chosen": 1.444504976272583,
-      "logits/rejected": 1.7912876605987549,
-      "logps/chosen": -389.9057312011719,
-      "logps/rejected": -459.4881286621094,
-      "loss": 0.5196,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.2401378154754639,
-      "rewards/margins": 0.8090965151786804,
-      "rewards/rejected": -2.04923415184021,
+      "logits/chosen": 2.9815869331359863,
+      "logits/rejected": 3.3694815635681152,
+      "logps/chosen": -392.791015625,
+      "logps/rejected": -463.4598693847656,
+      "loss": 0.5234,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.2689907550811768,
+      "rewards/margins": 0.8199613690376282,
+      "rewards/rejected": -2.08895206451416,
       "step": 830
     },
     {
       "epoch": 0.8791208791208791,
-      "grad_norm": 35.827147674342314,
+      "grad_norm": 29.54787662614835,
       "learning_rate": 2.1787515014630357e-08,
-      "logits/chosen": 1.3566821813583374,
-      "logits/rejected": 2.0197525024414062,
-      "logps/chosen": -408.1112365722656,
-      "logps/rejected": -478.15362548828125,
-      "loss": 0.4883,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.1996033191680908,
-      "rewards/margins": 0.8877967596054077,
-      "rewards/rejected": -2.087400197982788,
+      "logits/chosen": 2.903393030166626,
+      "logits/rejected": 3.6781086921691895,
+      "logps/chosen": -406.471923828125,
+      "logps/rejected": -476.11627197265625,
+      "loss": 0.4784,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.1832104921340942,
+      "rewards/margins": 0.8838159441947937,
+      "rewards/rejected": -2.0670266151428223,
       "step": 840
     },
     {
       "epoch": 0.8895866038723181,
-      "grad_norm": 31.663518232503517,
+      "grad_norm": 29.86964980464965,
       "learning_rate": 1.820784220652766e-08,
-      "logits/chosen": 1.6553951501846313,
-      "logits/rejected": 2.2624051570892334,
-      "logps/chosen": -405.8297424316406,
-      "logps/rejected": -444.5301818847656,
-      "loss": 0.4817,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.1564102172851562,
-      "rewards/margins": 0.9108415842056274,
-      "rewards/rejected": -2.067251682281494,
+      "logits/chosen": 3.334582567214966,
+      "logits/rejected": 4.037064552307129,
+      "logps/chosen": -408.4716491699219,
+      "logps/rejected": -450.2791442871094,
+      "loss": 0.4744,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.1828290224075317,
+      "rewards/margins": 0.9419119954109192,
+      "rewards/rejected": -2.1247410774230957,
       "step": 850
     },
     {
       "epoch": 0.9000523286237572,
-      "grad_norm": 37.26032497580661,
+      "grad_norm": 34.54309480500497,
       "learning_rate": 1.4938170864468636e-08,
-      "logits/chosen": 1.5842053890228271,
-      "logits/rejected": 2.242150068283081,
-      "logps/chosen": -416.7059020996094,
-      "logps/rejected": -471.46600341796875,
-      "loss": 0.4916,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.1967964172363281,
-      "rewards/margins": 0.8268603086471558,
-      "rewards/rejected": -2.0236568450927734,
+      "logits/chosen": 3.1594982147216797,
+      "logits/rejected": 3.9510695934295654,
+      "logps/chosen": -421.34991455078125,
+      "logps/rejected": -475.019775390625,
+      "loss": 0.4958,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.2432363033294678,
+      "rewards/margins": 0.8159576654434204,
+      "rewards/rejected": -2.0591940879821777,
       "step": 860
     },
     {
       "epoch": 0.9105180533751962,
-      "grad_norm": 35.19582912121324,
+      "grad_norm": 32.065838018543445,
       "learning_rate": 1.1982873884064465e-08,
-      "logits/chosen": 1.2853723764419556,
-      "logits/rejected": 1.9532840251922607,
-      "logps/chosen": -392.630615234375,
-      "logps/rejected": -457.26348876953125,
-      "loss": 0.5073,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.140249490737915,
-      "rewards/margins": 0.8805596232414246,
-      "rewards/rejected": -2.0208089351654053,
+      "logits/chosen": 2.7435097694396973,
+      "logits/rejected": 3.531782865524292,
+      "logps/chosen": -395.3089294433594,
+      "logps/rejected": -457.4712829589844,
+      "loss": 0.5109,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1670329570770264,
+      "rewards/margins": 0.8558540344238281,
+      "rewards/rejected": -2.0228867530822754,
       "step": 870
     },
     {
       "epoch": 0.9209837781266352,
-      "grad_norm": 34.02189943600299,
+      "grad_norm": 30.81769554075227,
       "learning_rate": 9.345903713082304e-09,
-      "logits/chosen": 1.409638524055481,
-      "logits/rejected": 1.783506989479065,
-      "logps/chosen": -433.36419677734375,
-      "logps/rejected": -498.5658264160156,
-      "loss": 0.51,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.1791355609893799,
-      "rewards/margins": 0.7773876190185547,
-      "rewards/rejected": -1.9565229415893555,
+      "logits/chosen": 2.844412326812744,
+      "logits/rejected": 3.3182201385498047,
+      "logps/chosen": -432.369873046875,
+      "logps/rejected": -500.48736572265625,
+      "loss": 0.5001,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.1691917181015015,
+      "rewards/margins": 0.8065465688705444,
+      "rewards/rejected": -1.975738286972046,
       "step": 880
     },
     {
       "epoch": 0.9314495028780743,
-      "grad_norm": 101.02453554023373,
+      "grad_norm": 77.64710978911431,
       "learning_rate": 7.030787065396865e-09,
-      "logits/chosen": 1.3969112634658813,
-      "logits/rejected": 1.868054747581482,
-      "logps/chosen": -416.21783447265625,
-      "logps/rejected": -492.66448974609375,
-      "loss": 0.5231,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.1590664386749268,
-      "rewards/margins": 0.812092661857605,
-      "rewards/rejected": -1.9711593389511108,
+      "logits/chosen": 2.76213002204895,
+      "logits/rejected": 3.284177780151367,
+      "logps/chosen": -417.70635986328125,
+      "logps/rejected": -495.9405822753906,
+      "loss": 0.5203,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.1739517450332642,
+      "rewards/margins": 0.8299692869186401,
+      "rewards/rejected": -2.0039210319519043,
       "step": 890
     },
     {
       "epoch": 0.9419152276295133,
-      "grad_norm": 33.05314756036431,
+      "grad_norm": 40.93804688092096,
       "learning_rate": 5.04062020432286e-09,
-      "logits/chosen": 1.3796522617340088,
-      "logits/rejected": 1.653877854347229,
-      "logps/chosen": -404.7085876464844,
-      "logps/rejected": -482.3396911621094,
-      "loss": 0.4976,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.1726219654083252,
-      "rewards/margins": 0.7496272921562195,
-      "rewards/rejected": -1.9222491979599,
+      "logits/chosen": 2.739307403564453,
+      "logits/rejected": 3.0972256660461426,
+      "logps/chosen": -405.97650146484375,
+      "logps/rejected": -486.93511962890625,
+      "loss": 0.4913,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.185300588607788,
+      "rewards/margins": 0.7829028367996216,
+      "rewards/rejected": -1.9682035446166992,
       "step": 900
     },
     {
       "epoch": 0.9419152276295133,
-      "eval_logits/chosen": 1.3479634523391724,
-      "eval_logits/rejected": 1.9026191234588623,
-      "eval_logps/chosen": -408.19427490234375,
-      "eval_logps/rejected": -470.8255310058594,
-      "eval_loss": 0.5151652693748474,
-      "eval_rewards/accuracies": 0.7599206566810608,
-      "eval_rewards/chosen": -1.1646604537963867,
-      "eval_rewards/margins": 0.8820713758468628,
-      "eval_rewards/rejected": -2.046732187271118,
-      "eval_runtime": 206.2937,
-      "eval_samples_per_second": 9.695,
-      "eval_steps_per_second": 0.305,
+      "eval_logits/chosen": 2.7670836448669434,
+      "eval_logits/rejected": 3.4042584896087646,
+      "eval_logps/chosen": -407.51177978515625,
+      "eval_logps/rejected": -470.7423095703125,
+      "eval_loss": 0.5082352757453918,
+      "eval_rewards/accuracies": 0.7638888955116272,
+      "eval_rewards/chosen": -1.1578354835510254,
+      "eval_rewards/margins": 0.8880642056465149,
+      "eval_rewards/rejected": -2.0458993911743164,
+      "eval_runtime": 206.9591,
+      "eval_samples_per_second": 9.664,
+      "eval_steps_per_second": 0.304,
       "step": 900
     },
     {
       "epoch": 0.9523809523809523,
-      "grad_norm": 37.360762359784026,
+      "grad_norm": 38.60236689676376,
       "learning_rate": 3.3780648016376866e-09,
-      "logits/chosen": 1.7493425607681274,
-      "logits/rejected": 2.125293016433716,
-      "logps/chosen": -343.41607666015625,
-      "logps/rejected": -408.11895751953125,
-      "loss": 0.5068,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.1369637250900269,
-      "rewards/margins": 0.7960535883903503,
-      "rewards/rejected": -1.933017373085022,
+      "logits/chosen": 3.2846641540527344,
+      "logits/rejected": 3.6967482566833496,
+      "logps/chosen": -341.8486022949219,
+      "logps/rejected": -413.08203125,
+      "loss": 0.4965,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1212894916534424,
+      "rewards/margins": 0.8613580465316772,
+      "rewards/rejected": -1.9826476573944092,
       "step": 910
     },
     {
       "epoch": 0.9628466771323915,
-      "grad_norm": 34.128880581694396,
+      "grad_norm": 42.33151515365454,
       "learning_rate": 2.0453443778310766e-09,
-      "logits/chosen": 1.4454774856567383,
-      "logits/rejected": 2.106161594390869,
-      "logps/chosen": -403.2652893066406,
-      "logps/rejected": -456.05999755859375,
-      "loss": 0.5096,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.1219370365142822,
-      "rewards/margins": 0.9178807139396667,
-      "rewards/rejected": -2.0398173332214355,
+      "logits/chosen": 2.852766275405884,
+      "logits/rejected": 3.614954710006714,
+      "logps/chosen": -403.8516845703125,
+      "logps/rejected": -453.4590759277344,
+      "loss": 0.5076,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.1278009414672852,
+      "rewards/margins": 0.8860074281692505,
+      "rewards/rejected": -2.013808488845825,
       "step": 920
     },
     {
       "epoch": 0.9733124018838305,
-      "grad_norm": 33.912988919352415,
+      "grad_norm": 35.663283908743544,
       "learning_rate": 1.0442413283435758e-09,
-      "logits/chosen": 1.3229658603668213,
-      "logits/rejected": 2.128333330154419,
-      "logps/chosen": -431.3890075683594,
-      "logps/rejected": -464.32781982421875,
-      "loss": 0.5007,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.0661871433258057,
-      "rewards/margins": 0.8464835286140442,
-      "rewards/rejected": -1.9126704931259155,
+      "logits/chosen": 2.7565677165985107,
+      "logits/rejected": 3.7096633911132812,
+      "logps/chosen": -429.5301208496094,
+      "logps/rejected": -465.82208251953125,
+      "loss": 0.5015,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.0475975275039673,
+      "rewards/margins": 0.8800159692764282,
+      "rewards/rejected": -1.9276134967803955,
       "step": 930
     },
     {
       "epoch": 0.9837781266352695,
-      "grad_norm": 32.48448141098046,
+      "grad_norm": 30.900713639808732,
       "learning_rate": 3.760945397705828e-10,
-      "logits/chosen": 1.3461906909942627,
-      "logits/rejected": 1.898982048034668,
-      "logps/chosen": -381.6216125488281,
-      "logps/rejected": -456.62713623046875,
-      "loss": 0.5001,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.1295430660247803,
-      "rewards/margins": 0.8078519701957703,
-      "rewards/rejected": -1.9373950958251953,
+      "logits/chosen": 2.7506637573242188,
+      "logits/rejected": 3.336564540863037,
+      "logps/chosen": -378.9224853515625,
+      "logps/rejected": -455.04864501953125,
+      "loss": 0.5047,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.102552056312561,
+      "rewards/margins": 0.8190587162971497,
+      "rewards/rejected": -1.9216108322143555,
       "step": 940
     },
     {
       "epoch": 0.9942438513867086,
-      "grad_norm": 33.82160017136217,
+      "grad_norm": 33.94615144806547,
       "learning_rate": 4.17975992204056e-11,
-      "logits/chosen": 1.398242712020874,
-      "logits/rejected": 1.7302888631820679,
-      "logps/chosen": -433.99249267578125,
-      "logps/rejected": -482.1703186035156,
-      "loss": 0.5035,
+      "logits/chosen": 2.811728000640869,
+      "logits/rejected": 3.2094719409942627,
+      "logps/chosen": -430.20428466796875,
+      "logps/rejected": -480.8965759277344,
+      "loss": 0.4993,
       "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.2265104055404663,
-      "rewards/margins": 0.6851701140403748,
-      "rewards/rejected": -1.9116805791854858,
+      "rewards/chosen": -1.1886287927627563,
+      "rewards/margins": 0.7103154063224792,
+      "rewards/rejected": -1.8989441394805908,
       "step": 950
     },
     {
       "epoch": 0.9994767137624281,
       "step": 955,
       "total_flos": 0.0,
-      "train_loss": 0.5508741353818883,
-      "train_runtime": 17482.7224,
-      "train_samples_per_second": 3.497,
-      "train_steps_per_second": 0.055
+      "train_loss": 0.546376849968396,
+      "train_runtime": 17587.6764,
+      "train_samples_per_second": 3.476,
+      "train_steps_per_second": 0.054
     }
   ],
   "logging_steps": 10,