diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -10,4067 +10,4067 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 42.15456008911133,
-      "kl": 0.017818570137023926,
+      "grad_norm": 43.373348236083984,
+      "kl": 0.018903231248259544,
       "learning_rate": 1.592356687898089e-08,
-      "logps/chosen": -285.75128173828125,
-      "logps/rejected": -254.7062530517578,
-      "loss": 0.4999,
-      "rewards/chosen": 0.004669209010899067,
-      "rewards/margins": 0.0025329389609396458,
-      "rewards/rejected": 0.0021362705156207085,
+      "logps/chosen": -299.6726379394531,
+      "logps/rejected": -346.7806396484375,
+      "loss": 0.5001,
+      "rewards/chosen": -0.0007079701172187924,
+      "rewards/margins": -0.001962849870324135,
+      "rewards/rejected": 0.0012548796366900206,
       "step": 5
     },
     {
       "epoch": 0.01,
-      "grad_norm": 44.65619659423828,
-      "kl": 0.05461766570806503,
+      "grad_norm": 45.312686920166016,
+      "kl": 0.05140366405248642,
       "learning_rate": 3.184713375796178e-08,
-      "logps/chosen": -286.11944580078125,
-      "logps/rejected": -276.1832580566406,
-      "loss": 0.496,
-      "rewards/chosen": 0.03117586299777031,
-      "rewards/margins": 0.033454541116952896,
-      "rewards/rejected": -0.00227867579087615,
+      "logps/chosen": -310.0472717285156,
+      "logps/rejected": -311.03424072265625,
+      "loss": 0.4979,
+      "rewards/chosen": 0.019934307783842087,
+      "rewards/margins": 0.010924299247562885,
+      "rewards/rejected": 0.009010007604956627,
       "step": 10
     },
     {
       "epoch": 0.01,
-      "grad_norm": 39.61723327636719,
-      "kl": 0.44255906343460083,
+      "grad_norm": 44.28553771972656,
+      "kl": 0.01275167427957058,
       "learning_rate": 4.777070063694268e-08,
-      "logps/chosen": -269.203125,
-      "logps/rejected": -260.71966552734375,
-      "loss": 0.4849,
-      "rewards/chosen": 0.13460782170295715,
-      "rewards/margins": 0.11511580646038055,
-      "rewards/rejected": 0.0194920115172863,
+      "logps/chosen": -258.36212158203125,
+      "logps/rejected": -348.38824462890625,
+      "loss": 0.4943,
+      "rewards/chosen": 0.04069889709353447,
+      "rewards/margins": 0.04853983223438263,
+      "rewards/rejected": -0.007840934209525585,
       "step": 15
     },
     {
       "epoch": 0.01,
-      "grad_norm": 36.42387771606445,
-      "kl": 0.871076226234436,
+      "grad_norm": 45.054351806640625,
+      "kl": 0.0,
       "learning_rate": 6.369426751592356e-08,
-      "logps/chosen": -244.55203247070312,
-      "logps/rejected": -259.15496826171875,
-      "loss": 0.4573,
-      "rewards/chosen": 0.3324822187423706,
-      "rewards/margins": 0.3418889045715332,
-      "rewards/rejected": -0.00940666627138853,
+      "logps/chosen": -245.97470092773438,
+      "logps/rejected": -350.9342041015625,
+      "loss": 0.4837,
+      "rewards/chosen": 0.07507207244634628,
+      "rewards/margins": 0.13315154612064362,
+      "rewards/rejected": -0.05807947367429733,
       "step": 20
     },
     {
       "epoch": 0.02,
-      "grad_norm": 35.794044494628906,
-      "kl": 0.3753414750099182,
+      "grad_norm": 43.90499496459961,
+      "kl": 0.0,
       "learning_rate": 7.961783439490445e-08,
-      "logps/chosen": -261.01800537109375,
-      "logps/rejected": -271.74786376953125,
-      "loss": 0.4164,
-      "rewards/chosen": 0.48356980085372925,
-      "rewards/margins": 0.7098164558410645,
-      "rewards/rejected": -0.22624659538269043,
+      "logps/chosen": -240.2781524658203,
+      "logps/rejected": -334.9874267578125,
+      "loss": 0.465,
+      "rewards/chosen": 0.16415421664714813,
+      "rewards/margins": 0.27640852332115173,
+      "rewards/rejected": -0.11225433647632599,
       "step": 25
     },
     {
       "epoch": 0.02,
-      "grad_norm": 34.50333786010742,
+      "grad_norm": 42.99668884277344,
       "kl": 0.0,
       "learning_rate": 9.554140127388536e-08,
-      "logps/chosen": -306.0127868652344,
-      "logps/rejected": -244.1356658935547,
-      "loss": 0.3649,
-      "rewards/chosen": 0.746610164642334,
-      "rewards/margins": 1.2027219533920288,
-      "rewards/rejected": -0.4561118483543396,
+      "logps/chosen": -310.78314208984375,
+      "logps/rejected": -325.97113037109375,
+      "loss": 0.4387,
+      "rewards/chosen": 0.21505948901176453,
+      "rewards/margins": 0.4358912408351898,
+      "rewards/rejected": -0.22083178162574768,
       "step": 30
     },
     {
       "epoch": 0.02,
-      "grad_norm": 30.50051498413086,
+      "grad_norm": 42.578617095947266,
       "kl": 0.0,
       "learning_rate": 1.1146496815286624e-07,
-      "logps/chosen": -280.65692138671875,
-      "logps/rejected": -262.2502136230469,
-      "loss": 0.3171,
-      "rewards/chosen": 0.774597704410553,
-      "rewards/margins": 1.728833794593811,
-      "rewards/rejected": -0.9542360305786133,
+      "logps/chosen": -263.4063415527344,
+      "logps/rejected": -312.7994689941406,
+      "loss": 0.4157,
+      "rewards/chosen": 0.25127312541007996,
+      "rewards/margins": 0.7119064331054688,
+      "rewards/rejected": -0.46063321828842163,
       "step": 35
     },
     {
       "epoch": 0.03,
-      "grad_norm": 29.806577682495117,
+      "grad_norm": 38.7848014831543,
       "kl": 0.0,
       "learning_rate": 1.2738853503184713e-07,
-      "logps/chosen": -276.9497375488281,
-      "logps/rejected": -259.6419982910156,
-      "loss": 0.2884,
-      "rewards/chosen": 0.6780191659927368,
-      "rewards/margins": 2.165379047393799,
-      "rewards/rejected": -1.4873597621917725,
+      "logps/chosen": -283.001953125,
+      "logps/rejected": -342.24493408203125,
+      "loss": 0.3764,
+      "rewards/chosen": 0.18532009422779083,
+      "rewards/margins": 1.1129443645477295,
+      "rewards/rejected": -0.9276243448257446,
       "step": 40
     },
     {
       "epoch": 0.03,
-      "grad_norm": 22.508939743041992,
+      "grad_norm": 33.522525787353516,
       "kl": 0.0,
       "learning_rate": 1.43312101910828e-07,
-      "logps/chosen": -253.7576141357422,
-      "logps/rejected": -268.8052062988281,
-      "loss": 0.2596,
-      "rewards/chosen": 1.205094575881958,
-      "rewards/margins": 2.685385227203369,
-      "rewards/rejected": -1.4802907705307007,
+      "logps/chosen": -263.763916015625,
+      "logps/rejected": -356.7685852050781,
+      "loss": 0.3194,
+      "rewards/chosen": 0.22150889039039612,
+      "rewards/margins": 1.852413535118103,
+      "rewards/rejected": -1.6309047937393188,
       "step": 45
     },
     {
       "epoch": 0.03,
-      "grad_norm": 29.29058074951172,
+      "grad_norm": 34.30215072631836,
       "kl": 0.0,
       "learning_rate": 1.592356687898089e-07,
-      "logps/chosen": -284.4584045410156,
-      "logps/rejected": -254.226806640625,
-      "loss": 0.2614,
-      "rewards/chosen": 1.0118796825408936,
-      "rewards/margins": 2.8269615173339844,
-      "rewards/rejected": -1.8150818347930908,
+      "logps/chosen": -305.10076904296875,
+      "logps/rejected": -346.50970458984375,
+      "loss": 0.3089,
+      "rewards/chosen": 0.23655760288238525,
+      "rewards/margins": 2.413478136062622,
+      "rewards/rejected": -2.1769204139709473,
       "step": 50
     },
     {
       "epoch": 0.04,
-      "grad_norm": 29.25771141052246,
+      "grad_norm": 63.321285247802734,
       "kl": 0.0,
       "learning_rate": 1.7515923566878978e-07,
-      "logps/chosen": -266.92254638671875,
-      "logps/rejected": -260.49261474609375,
-      "loss": 0.233,
-      "rewards/chosen": 1.16305673122406,
-      "rewards/margins": 3.4210219383239746,
-      "rewards/rejected": -2.257965326309204,
+      "logps/chosen": -304.7918701171875,
+      "logps/rejected": -375.59271240234375,
+      "loss": 0.289,
+      "rewards/chosen": 0.26213210821151733,
+      "rewards/margins": 2.895750045776367,
+      "rewards/rejected": -2.633617877960205,
       "step": 55
     },
     {
       "epoch": 0.04,
-      "grad_norm": 20.642614364624023,
+      "grad_norm": 31.712352752685547,
       "kl": 0.0,
       "learning_rate": 1.9108280254777072e-07,
-      "logps/chosen": -231.07168579101562,
-      "logps/rejected": -258.5835876464844,
-      "loss": 0.241,
-      "rewards/chosen": 1.3277983665466309,
-      "rewards/margins": 3.532343626022339,
-      "rewards/rejected": -2.204545497894287,
+      "logps/chosen": -261.5682067871094,
+      "logps/rejected": -345.7394104003906,
+      "loss": 0.2843,
+      "rewards/chosen": 0.3677944540977478,
+      "rewards/margins": 2.7658519744873047,
+      "rewards/rejected": -2.398057460784912,
       "step": 60
     },
     {
       "epoch": 0.04,
-      "grad_norm": 23.786853790283203,
+      "grad_norm": 33.49573516845703,
       "kl": 0.0,
       "learning_rate": 2.070063694267516e-07,
-      "logps/chosen": -251.5561981201172,
-      "logps/rejected": -267.85986328125,
-      "loss": 0.2376,
-      "rewards/chosen": 1.2313438653945923,
-      "rewards/margins": 3.837125062942505,
-      "rewards/rejected": -2.605781316757202,
+      "logps/chosen": -300.9778137207031,
+      "logps/rejected": -324.8972473144531,
+      "loss": 0.2645,
+      "rewards/chosen": 0.3324030041694641,
+      "rewards/margins": 3.486532688140869,
+      "rewards/rejected": -3.1541295051574707,
       "step": 65
     },
     {
       "epoch": 0.04,
-      "grad_norm": 24.48412322998047,
+      "grad_norm": 33.059207916259766,
       "kl": 0.0,
       "learning_rate": 2.2292993630573247e-07,
-      "logps/chosen": -244.3961639404297,
-      "logps/rejected": -248.7190704345703,
-      "loss": 0.2304,
-      "rewards/chosen": 1.338683843612671,
-      "rewards/margins": 3.6262855529785156,
-      "rewards/rejected": -2.287601947784424,
+      "logps/chosen": -318.46124267578125,
+      "logps/rejected": -387.02947998046875,
+      "loss": 0.2496,
+      "rewards/chosen": 0.3689573109149933,
+      "rewards/margins": 3.851461887359619,
+      "rewards/rejected": -3.482504367828369,
       "step": 70
     },
     {
       "epoch": 0.05,
-      "grad_norm": 16.654287338256836,
+      "grad_norm": 31.94024658203125,
       "kl": 0.0,
       "learning_rate": 2.388535031847134e-07,
-      "logps/chosen": -261.95819091796875,
-      "logps/rejected": -291.40020751953125,
-      "loss": 0.2003,
-      "rewards/chosen": 1.3882195949554443,
-      "rewards/margins": 4.626662254333496,
-      "rewards/rejected": -3.238443374633789,
+      "logps/chosen": -358.51654052734375,
+      "logps/rejected": -330.6874694824219,
+      "loss": 0.2602,
+      "rewards/chosen": 0.41941699385643005,
+      "rewards/margins": 3.710043430328369,
+      "rewards/rejected": -3.2906270027160645,
       "step": 75
     },
     {
       "epoch": 0.05,
-      "grad_norm": 30.89764976501465,
+      "grad_norm": 34.636436462402344,
       "kl": 0.0,
       "learning_rate": 2.5477707006369425e-07,
-      "logps/chosen": -267.4027099609375,
-      "logps/rejected": -271.3486328125,
-      "loss": 0.1985,
-      "rewards/chosen": 1.4253151416778564,
-      "rewards/margins": 4.816943645477295,
-      "rewards/rejected": -3.3916287422180176,
+      "logps/chosen": -268.0237121582031,
+      "logps/rejected": -380.11676025390625,
+      "loss": 0.2606,
+      "rewards/chosen": 0.410856157541275,
+      "rewards/margins": 4.273175239562988,
+      "rewards/rejected": -3.862318754196167,
       "step": 80
     },
     {
       "epoch": 0.05,
-      "grad_norm": 55.57185363769531,
+      "grad_norm": 29.300586700439453,
       "kl": 0.0,
       "learning_rate": 2.7070063694267513e-07,
-      "logps/chosen": -289.416259765625,
-      "logps/rejected": -282.303955078125,
-      "loss": 0.2075,
-      "rewards/chosen": 1.308215856552124,
-      "rewards/margins": 5.0450119972229,
-      "rewards/rejected": -3.7367959022521973,
+      "logps/chosen": -323.76416015625,
+      "logps/rejected": -359.8213195800781,
+      "loss": 0.2378,
+      "rewards/chosen": 0.46109551191329956,
+      "rewards/margins": 4.482287406921387,
+      "rewards/rejected": -4.021191596984863,
       "step": 85
     },
     {
       "epoch": 0.06,
-      "grad_norm": 34.31930923461914,
+      "grad_norm": 32.00096130371094,
       "kl": 0.0,
       "learning_rate": 2.86624203821656e-07,
-      "logps/chosen": -255.4053192138672,
-      "logps/rejected": -293.8216247558594,
-      "loss": 0.1877,
-      "rewards/chosen": 1.560572862625122,
-      "rewards/margins": 5.238338470458984,
-      "rewards/rejected": -3.6777656078338623,
+      "logps/chosen": -292.0968322753906,
+      "logps/rejected": -414.10980224609375,
+      "loss": 0.2176,
+      "rewards/chosen": 0.5200246572494507,
+      "rewards/margins": 5.136369228363037,
+      "rewards/rejected": -4.616344928741455,
       "step": 90
     },
     {
       "epoch": 0.06,
-      "grad_norm": 22.21489143371582,
+      "grad_norm": 31.802518844604492,
       "kl": 0.0,
       "learning_rate": 3.0254777070063694e-07,
-      "logps/chosen": -286.2869873046875,
-      "logps/rejected": -271.2530212402344,
-      "loss": 0.1985,
-      "rewards/chosen": 1.478058099746704,
-      "rewards/margins": 5.051484107971191,
-      "rewards/rejected": -3.5734260082244873,
+      "logps/chosen": -289.4820556640625,
+      "logps/rejected": -367.87030029296875,
+      "loss": 0.2372,
+      "rewards/chosen": 0.5198050737380981,
+      "rewards/margins": 5.272117614746094,
+      "rewards/rejected": -4.752312183380127,
       "step": 95
     },
     {
       "epoch": 0.06,
-      "grad_norm": 29.9346981048584,
+      "grad_norm": 37.19041061401367,
       "kl": 0.0,
       "learning_rate": 3.184713375796178e-07,
-      "logps/chosen": -245.080810546875,
-      "logps/rejected": -287.15435791015625,
-      "loss": 0.1656,
-      "rewards/chosen": 1.4543142318725586,
-      "rewards/margins": 5.874560356140137,
-      "rewards/rejected": -4.420246124267578,
+      "logps/chosen": -280.93896484375,
+      "logps/rejected": -394.2848205566406,
+      "loss": 0.2338,
+      "rewards/chosen": 0.4976850152015686,
+      "rewards/margins": 5.734658241271973,
+      "rewards/rejected": -5.236973285675049,
       "step": 100
     },
     {
       "epoch": 0.07,
-      "grad_norm": 22.431930541992188,
+      "grad_norm": 26.246625900268555,
       "kl": 0.0,
       "learning_rate": 3.343949044585987e-07,
-      "logps/chosen": -259.4337463378906,
-      "logps/rejected": -309.25164794921875,
-      "loss": 0.1767,
-      "rewards/chosen": 1.4657680988311768,
-      "rewards/margins": 5.859877109527588,
-      "rewards/rejected": -4.39410924911499,
+      "logps/chosen": -315.616943359375,
+      "logps/rejected": -338.24346923828125,
+      "loss": 0.2378,
+      "rewards/chosen": 0.5364874601364136,
+      "rewards/margins": 4.776383399963379,
+      "rewards/rejected": -4.239895820617676,
       "step": 105
     },
     {
       "epoch": 0.07,
-      "grad_norm": 36.517799377441406,
+      "grad_norm": 34.44779586791992,
       "kl": 0.0,
       "learning_rate": 3.5031847133757957e-07,
-      "logps/chosen": -264.08306884765625,
-      "logps/rejected": -295.64788818359375,
-      "loss": 0.1599,
-      "rewards/chosen": 1.291338562965393,
-      "rewards/margins": 5.8188042640686035,
-      "rewards/rejected": -4.5274658203125,
+      "logps/chosen": -253.6793975830078,
+      "logps/rejected": -391.8357238769531,
+      "loss": 0.2379,
+      "rewards/chosen": 0.40439778566360474,
+      "rewards/margins": 5.385909557342529,
+      "rewards/rejected": -4.98151159286499,
       "step": 110
     },
     {
       "epoch": 0.07,
-      "grad_norm": 25.18623924255371,
+      "grad_norm": 45.468196868896484,
       "kl": 0.0,
       "learning_rate": 3.6624203821656045e-07,
-      "logps/chosen": -268.96527099609375,
-      "logps/rejected": -278.4827575683594,
-      "loss": 0.1735,
-      "rewards/chosen": 1.4270073175430298,
-      "rewards/margins": 5.516595363616943,
-      "rewards/rejected": -4.0895891189575195,
+      "logps/chosen": -283.26025390625,
+      "logps/rejected": -404.784423828125,
+      "loss": 0.2448,
+      "rewards/chosen": 0.5039983987808228,
+      "rewards/margins": 5.760153770446777,
+      "rewards/rejected": -5.256155967712402,
       "step": 115
     },
     {
       "epoch": 0.08,
-      "grad_norm": 24.899450302124023,
+      "grad_norm": 29.123733520507812,
       "kl": 0.0,
       "learning_rate": 3.8216560509554143e-07,
-      "logps/chosen": -257.46807861328125,
-      "logps/rejected": -306.6875305175781,
-      "loss": 0.1675,
-      "rewards/chosen": 1.363029956817627,
-      "rewards/margins": 5.9437384605407715,
-      "rewards/rejected": -4.580708980560303,
+      "logps/chosen": -292.062255859375,
+      "logps/rejected": -350.0133056640625,
+      "loss": 0.2361,
+      "rewards/chosen": 0.5945883989334106,
+      "rewards/margins": 5.297600746154785,
+      "rewards/rejected": -4.703011989593506,
       "step": 120
     },
     {
       "epoch": 0.08,
-      "grad_norm": 18.1636962890625,
+      "grad_norm": 31.618877410888672,
       "kl": 0.0,
       "learning_rate": 3.980891719745223e-07,
-      "logps/chosen": -274.8599548339844,
-      "logps/rejected": -316.78057861328125,
-      "loss": 0.1509,
-      "rewards/chosen": 1.4044160842895508,
-      "rewards/margins": 6.668715000152588,
-      "rewards/rejected": -5.264299392700195,
+      "logps/chosen": -304.79302978515625,
+      "logps/rejected": -412.56396484375,
+      "loss": 0.2332,
+      "rewards/chosen": 0.5294879674911499,
+      "rewards/margins": 5.99056339263916,
+      "rewards/rejected": -5.461075782775879,
       "step": 125
     },
     {
       "epoch": 0.08,
-      "grad_norm": 53.83689498901367,
+      "grad_norm": 31.75819969177246,
       "kl": 0.0,
       "learning_rate": 4.140127388535032e-07,
-      "logps/chosen": -316.65460205078125,
-      "logps/rejected": -318.2517395019531,
-      "loss": 0.1667,
-      "rewards/chosen": 1.331627368927002,
-      "rewards/margins": 6.6092848777771,
-      "rewards/rejected": -5.277657508850098,
+      "logps/chosen": -343.9717712402344,
+      "logps/rejected": -413.4139709472656,
+      "loss": 0.2209,
+      "rewards/chosen": 0.6387773752212524,
+      "rewards/margins": 6.253381729125977,
+      "rewards/rejected": -5.6146039962768555,
       "step": 130
     },
     {
       "epoch": 0.09,
-      "grad_norm": 16.44365692138672,
+      "grad_norm": 80.12673950195312,
       "kl": 0.0,
       "learning_rate": 4.2993630573248406e-07,
-      "logps/chosen": -242.02651977539062,
-      "logps/rejected": -283.90106201171875,
-      "loss": 0.1762,
-      "rewards/chosen": 1.6127967834472656,
-      "rewards/margins": 6.091832160949707,
-      "rewards/rejected": -4.479035377502441,
+      "logps/chosen": -276.16864013671875,
+      "logps/rejected": -370.7424621582031,
+      "loss": 0.2331,
+      "rewards/chosen": 0.4045650362968445,
+      "rewards/margins": 5.879421710968018,
+      "rewards/rejected": -5.474856853485107,
       "step": 135
     },
     {
       "epoch": 0.09,
-      "grad_norm": 19.764270782470703,
+      "grad_norm": 30.59686851501465,
       "kl": 0.0,
       "learning_rate": 4.4585987261146494e-07,
-      "logps/chosen": -252.82180786132812,
-      "logps/rejected": -319.73199462890625,
-      "loss": 0.1468,
-      "rewards/chosen": 1.5330384969711304,
-      "rewards/margins": 6.702307224273682,
-      "rewards/rejected": -5.169268608093262,
+      "logps/chosen": -264.196533203125,
+      "logps/rejected": -359.3782043457031,
+      "loss": 0.2489,
+      "rewards/chosen": 0.45963722467422485,
+      "rewards/margins": 4.636446952819824,
+      "rewards/rejected": -4.176809310913086,
       "step": 140
     },
     {
       "epoch": 0.09,
-      "grad_norm": 21.560876846313477,
+      "grad_norm": 28.13033103942871,
       "kl": 0.0,
       "learning_rate": 4.6178343949044587e-07,
-      "logps/chosen": -247.28427124023438,
-      "logps/rejected": -282.38848876953125,
-      "loss": 0.1868,
-      "rewards/chosen": 1.350294828414917,
-      "rewards/margins": 5.6418867111206055,
-      "rewards/rejected": -4.291592597961426,
+      "logps/chosen": -269.31793212890625,
+      "logps/rejected": -414.6089782714844,
+      "loss": 0.2443,
+      "rewards/chosen": 0.4464253783226013,
+      "rewards/margins": 5.700473308563232,
+      "rewards/rejected": -5.2540483474731445,
       "step": 145
     },
     {
       "epoch": 0.1,
-      "grad_norm": 21.553871154785156,
+      "grad_norm": 29.259023666381836,
       "kl": 0.0,
       "learning_rate": 4.777070063694267e-07,
-      "logps/chosen": -286.14312744140625,
-      "logps/rejected": -281.7044372558594,
-      "loss": 0.168,
-      "rewards/chosen": 1.5479779243469238,
-      "rewards/margins": 6.192603588104248,
-      "rewards/rejected": -4.644625663757324,
+      "logps/chosen": -315.52374267578125,
+      "logps/rejected": -365.8375549316406,
+      "loss": 0.2258,
+      "rewards/chosen": 0.6224046349525452,
+      "rewards/margins": 6.201630115509033,
+      "rewards/rejected": -5.579225063323975,
       "step": 150
     },
     {
       "epoch": 0.1,
-      "grad_norm": 24.488561630249023,
+      "grad_norm": 32.15901565551758,
       "kl": 0.0,
       "learning_rate": 4.936305732484076e-07,
-      "logps/chosen": -272.4622497558594,
-      "logps/rejected": -323.91082763671875,
-      "loss": 0.1648,
-      "rewards/chosen": 1.3805065155029297,
-      "rewards/margins": 6.456129550933838,
-      "rewards/rejected": -5.07562255859375,
+      "logps/chosen": -284.3131103515625,
+      "logps/rejected": -374.58770751953125,
+      "loss": 0.2226,
+      "rewards/chosen": 0.5719965100288391,
+      "rewards/margins": 6.714544773101807,
+      "rewards/rejected": -6.142548561096191,
       "step": 155
     },
     {
       "epoch": 0.1,
-      "grad_norm": 21.614229202270508,
+      "grad_norm": 33.57276153564453,
       "kl": 0.0,
       "learning_rate": 4.989331436699858e-07,
-      "logps/chosen": -207.8739776611328,
-      "logps/rejected": -324.73565673828125,
-      "loss": 0.1435,
-      "rewards/chosen": 1.465820074081421,
-      "rewards/margins": 6.953179836273193,
-      "rewards/rejected": -5.487359046936035,
+      "logps/chosen": -291.7167053222656,
+      "logps/rejected": -374.89031982421875,
+      "loss": 0.2333,
+      "rewards/chosen": 0.6062914133071899,
+      "rewards/margins": 6.262972831726074,
+      "rewards/rejected": -5.656682014465332,
       "step": 160
     },
     {
       "epoch": 0.11,
-      "grad_norm": 20.425729751586914,
+      "grad_norm": 27.332050323486328,
       "kl": 0.0,
       "learning_rate": 4.971550497866287e-07,
-      "logps/chosen": -287.088623046875,
-      "logps/rejected": -310.1844787597656,
-      "loss": 0.1568,
-      "rewards/chosen": 1.5799640417099,
-      "rewards/margins": 6.89548397064209,
-      "rewards/rejected": -5.315520286560059,
+      "logps/chosen": -309.1024475097656,
+      "logps/rejected": -402.1785583496094,
+      "loss": 0.2336,
+      "rewards/chosen": 0.5264068841934204,
+      "rewards/margins": 6.772003173828125,
+      "rewards/rejected": -6.245596408843994,
       "step": 165
     },
     {
       "epoch": 0.11,
-      "grad_norm": 21.86062240600586,
+      "grad_norm": 26.317598342895508,
       "kl": 0.0,
       "learning_rate": 4.953769559032717e-07,
-      "logps/chosen": -282.91741943359375,
-      "logps/rejected": -315.76470947265625,
-      "loss": 0.1516,
-      "rewards/chosen": 1.442077875137329,
-      "rewards/margins": 7.074441432952881,
-      "rewards/rejected": -5.632363796234131,
+      "logps/chosen": -277.711669921875,
+      "logps/rejected": -364.07745361328125,
+      "loss": 0.2229,
+      "rewards/chosen": 0.6749172210693359,
+      "rewards/margins": 6.215371131896973,
+      "rewards/rejected": -5.540453910827637,
       "step": 170
     },
     {
       "epoch": 0.11,
-      "grad_norm": 20.31084632873535,
+      "grad_norm": 26.295717239379883,
       "kl": 0.0,
       "learning_rate": 4.935988620199146e-07,
-      "logps/chosen": -274.0014953613281,
-      "logps/rejected": -310.8323059082031,
-      "loss": 0.1477,
-      "rewards/chosen": 1.4551314115524292,
-      "rewards/margins": 6.99865198135376,
-      "rewards/rejected": -5.543520450592041,
+      "logps/chosen": -315.84600830078125,
+      "logps/rejected": -419.64947509765625,
+      "loss": 0.2055,
+      "rewards/chosen": 0.8088045120239258,
+      "rewards/margins": 7.324716091156006,
+      "rewards/rejected": -6.515912055969238,
       "step": 175
     },
     {
       "epoch": 0.12,
-      "grad_norm": 20.067346572875977,
+      "grad_norm": 28.048669815063477,
       "kl": 0.0,
       "learning_rate": 4.918207681365576e-07,
-      "logps/chosen": -274.91705322265625,
-      "logps/rejected": -330.29315185546875,
-      "loss": 0.1491,
-      "rewards/chosen": 1.5545661449432373,
-      "rewards/margins": 7.674098014831543,
-      "rewards/rejected": -6.119531631469727,
+      "logps/chosen": -278.0254821777344,
+      "logps/rejected": -392.7597351074219,
+      "loss": 0.2087,
+      "rewards/chosen": 0.7063338756561279,
+      "rewards/margins": 7.6127119064331055,
+      "rewards/rejected": -6.906378269195557,
       "step": 180
     },
     {
       "epoch": 0.12,
-      "grad_norm": 26.703292846679688,
+      "grad_norm": 29.8698673248291,
       "kl": 0.0,
       "learning_rate": 4.900426742532006e-07,
-      "logps/chosen": -267.2247619628906,
-      "logps/rejected": -308.6265563964844,
-      "loss": 0.142,
-      "rewards/chosen": 1.4495770931243896,
-      "rewards/margins": 7.4614386558532715,
-      "rewards/rejected": -6.011861324310303,
+      "logps/chosen": -272.4722900390625,
+      "logps/rejected": -392.2569580078125,
+      "loss": 0.2483,
+      "rewards/chosen": 0.6673704385757446,
+      "rewards/margins": 6.765078067779541,
+      "rewards/rejected": -6.097708225250244,
       "step": 185
     },
     {
       "epoch": 0.12,
-      "grad_norm": 22.81442642211914,
+      "grad_norm": 27.31839370727539,
       "kl": 0.0,
       "learning_rate": 4.882645803698435e-07,
-      "logps/chosen": -243.8881072998047,
-      "logps/rejected": -318.4960632324219,
-      "loss": 0.1679,
-      "rewards/chosen": 1.2335823774337769,
-      "rewards/margins": 7.564291954040527,
-      "rewards/rejected": -6.330709934234619,
+      "logps/chosen": -295.7112731933594,
+      "logps/rejected": -434.19317626953125,
+      "loss": 0.2083,
+      "rewards/chosen": 0.6688440442085266,
+      "rewards/margins": 7.306863307952881,
+      "rewards/rejected": -6.638019561767578,
       "step": 190
     },
     {
       "epoch": 0.12,
-      "grad_norm": 17.875883102416992,
+      "grad_norm": 35.319053649902344,
       "kl": 0.0,
       "learning_rate": 4.864864864864865e-07,
-      "logps/chosen": -276.0986328125,
-      "logps/rejected": -317.36163330078125,
-      "loss": 0.1442,
-      "rewards/chosen": 1.5666391849517822,
-      "rewards/margins": 7.757128715515137,
-      "rewards/rejected": -6.190489768981934,
+      "logps/chosen": -283.3681945800781,
+      "logps/rejected": -392.4473571777344,
+      "loss": 0.2282,
+      "rewards/chosen": 0.6834143400192261,
+      "rewards/margins": 7.1807074546813965,
+      "rewards/rejected": -6.497292995452881,
       "step": 195
     },
     {
       "epoch": 0.13,
-      "grad_norm": 22.31620216369629,
+      "grad_norm": 28.466360092163086,
       "kl": 0.0,
       "learning_rate": 4.847083926031294e-07,
-      "logps/chosen": -273.843994140625,
-      "logps/rejected": -329.43475341796875,
-      "loss": 0.154,
-      "rewards/chosen": 1.6012461185455322,
-      "rewards/margins": 7.519808292388916,
-      "rewards/rejected": -5.918562412261963,
+      "logps/chosen": -301.3221130371094,
+      "logps/rejected": -402.22845458984375,
+      "loss": 0.2128,
+      "rewards/chosen": 0.8221393823623657,
+      "rewards/margins": 7.278465270996094,
+      "rewards/rejected": -6.456326484680176,
       "step": 200
     },
     {
       "epoch": 0.13,
-      "grad_norm": 19.976451873779297,
+      "grad_norm": 27.28252601623535,
       "kl": 0.0,
       "learning_rate": 4.829302987197724e-07,
-      "logps/chosen": -284.374755859375,
-      "logps/rejected": -326.24371337890625,
-      "loss": 0.1378,
-      "rewards/chosen": 1.5799314975738525,
-      "rewards/margins": 7.78595495223999,
-      "rewards/rejected": -6.206023216247559,
+      "logps/chosen": -306.77642822265625,
+      "logps/rejected": -413.36346435546875,
+      "loss": 0.2073,
+      "rewards/chosen": 0.6411277651786804,
+      "rewards/margins": 7.287237644195557,
+      "rewards/rejected": -6.646109580993652,
       "step": 205
     },
     {
       "epoch": 0.13,
-      "grad_norm": 23.919170379638672,
+      "grad_norm": 32.64134979248047,
       "kl": 0.0,
       "learning_rate": 4.811522048364154e-07,
-      "logps/chosen": -236.8705291748047,
-      "logps/rejected": -316.30413818359375,
-      "loss": 0.1523,
-      "rewards/chosen": 1.5169861316680908,
-      "rewards/margins": 7.513286590576172,
-      "rewards/rejected": -5.99630069732666,
+      "logps/chosen": -291.44818115234375,
+      "logps/rejected": -427.9637756347656,
+      "loss": 0.2164,
+      "rewards/chosen": 0.8150373697280884,
+      "rewards/margins": 7.651273250579834,
+      "rewards/rejected": -6.836236476898193,
       "step": 210
     },
     {
       "epoch": 0.14,
-      "grad_norm": 25.041824340820312,
+      "grad_norm": 23.997915267944336,
       "kl": 0.0,
       "learning_rate": 4.793741109530583e-07,
-      "logps/chosen": -219.86117553710938,
-      "logps/rejected": -331.60418701171875,
-      "loss": 0.1516,
-      "rewards/chosen": 1.5909864902496338,
-      "rewards/margins": 7.9639410972595215,
-      "rewards/rejected": -6.372954845428467,
+      "logps/chosen": -260.71527099609375,
+      "logps/rejected": -391.9830627441406,
+      "loss": 0.2175,
+      "rewards/chosen": 0.8054065704345703,
+      "rewards/margins": 7.5576653480529785,
+      "rewards/rejected": -6.752259254455566,
       "step": 215
     },
     {
       "epoch": 0.14,
-      "grad_norm": 25.45568084716797,
+      "grad_norm": 29.320816040039062,
       "kl": 0.0,
       "learning_rate": 4.775960170697012e-07,
-      "logps/chosen": -271.017333984375,
-      "logps/rejected": -333.2148132324219,
-      "loss": 0.1628,
-      "rewards/chosen": 1.5948388576507568,
-      "rewards/margins": 7.8658246994018555,
-      "rewards/rejected": -6.2709856033325195,
+      "logps/chosen": -268.3311462402344,
+      "logps/rejected": -404.9643249511719,
+      "loss": 0.2104,
+      "rewards/chosen": 0.7653949856758118,
+      "rewards/margins": 7.521485805511475,
+      "rewards/rejected": -6.756091117858887,
       "step": 220
     },
     {
       "epoch": 0.14,
-      "grad_norm": 22.259382247924805,
+      "grad_norm": 27.857921600341797,
       "kl": 0.0,
       "learning_rate": 4.7581792318634425e-07,
-      "logps/chosen": -253.55789184570312,
-      "logps/rejected": -298.8450622558594,
-      "loss": 0.1452,
-      "rewards/chosen": 1.640355110168457,
-      "rewards/margins": 8.043792724609375,
-      "rewards/rejected": -6.403438568115234,
+      "logps/chosen": -290.99407958984375,
+      "logps/rejected": -405.1004943847656,
+      "loss": 0.2164,
+      "rewards/chosen": 0.7016764879226685,
+      "rewards/margins": 7.881253242492676,
+      "rewards/rejected": -7.179576873779297,
       "step": 225
     },
     {
       "epoch": 0.15,
-      "grad_norm": 26.135997772216797,
+      "grad_norm": 26.333301544189453,
       "kl": 0.0,
       "learning_rate": 4.7403982930298717e-07,
-      "logps/chosen": -235.11349487304688,
-      "logps/rejected": -344.72369384765625,
-      "loss": 0.1472,
-      "rewards/chosen": 1.559660792350769,
-      "rewards/margins": 7.8217010498046875,
-      "rewards/rejected": -6.262040615081787,
+      "logps/chosen": -246.4652557373047,
+      "logps/rejected": -408.4164123535156,
+      "loss": 0.2048,
+      "rewards/chosen": 0.7065305709838867,
+      "rewards/margins": 7.9227142333984375,
+      "rewards/rejected": -7.216183662414551,
       "step": 230
     },
     {
       "epoch": 0.15,
-      "grad_norm": 20.143089294433594,
+      "grad_norm": 29.780120849609375,
       "kl": 0.0,
       "learning_rate": 4.7226173541963014e-07,
-      "logps/chosen": -249.42196655273438,
-      "logps/rejected": -277.12554931640625,
-      "loss": 0.1529,
-      "rewards/chosen": 1.6460959911346436,
-      "rewards/margins": 7.348568916320801,
-      "rewards/rejected": -5.702473163604736,
+      "logps/chosen": -300.751953125,
+      "logps/rejected": -409.5931091308594,
+      "loss": 0.2022,
+      "rewards/chosen": 0.8955439329147339,
+      "rewards/margins": 8.069990158081055,
+      "rewards/rejected": -7.174446105957031,
       "step": 235
     },
     {
       "epoch": 0.15,
-      "grad_norm": 23.44059181213379,
+      "grad_norm": 26.441162109375,
       "kl": 0.0,
       "learning_rate": 4.7048364153627306e-07,
-      "logps/chosen": -281.3484802246094,
-      "logps/rejected": -307.25457763671875,
-      "loss": 0.1387,
-      "rewards/chosen": 1.7489140033721924,
-      "rewards/margins": 7.860198974609375,
-      "rewards/rejected": -6.111284255981445,
+      "logps/chosen": -298.4041442871094,
+      "logps/rejected": -396.9103088378906,
+      "loss": 0.2223,
+      "rewards/chosen": 0.9372671842575073,
+      "rewards/margins": 7.630146026611328,
+      "rewards/rejected": -6.692878723144531,
       "step": 240
     },
     {
       "epoch": 0.16,
-      "grad_norm": 23.201915740966797,
+      "grad_norm": 27.794498443603516,
       "kl": 0.0,
       "learning_rate": 4.6870554765291604e-07,
-      "logps/chosen": -272.64031982421875,
-      "logps/rejected": -310.50909423828125,
-      "loss": 0.1356,
-      "rewards/chosen": 1.6725397109985352,
-      "rewards/margins": 8.012718200683594,
-      "rewards/rejected": -6.340178489685059,
+      "logps/chosen": -304.92340087890625,
+      "logps/rejected": -444.83526611328125,
+      "loss": 0.1866,
+      "rewards/chosen": 0.7705713510513306,
+      "rewards/margins": 8.343452453613281,
+      "rewards/rejected": -7.57288122177124,
       "step": 245
     },
     {
       "epoch": 0.16,
-      "grad_norm": 19.45538902282715,
+      "grad_norm": 27.902463912963867,
       "kl": 0.0,
       "learning_rate": 4.66927453769559e-07,
-      "logps/chosen": -272.3523254394531,
-      "logps/rejected": -338.20867919921875,
-      "loss": 0.1246,
-      "rewards/chosen": 1.727900743484497,
-      "rewards/margins": 8.646397590637207,
-      "rewards/rejected": -6.918497562408447,
+      "logps/chosen": -277.5145263671875,
+      "logps/rejected": -426.4693908691406,
+      "loss": 0.2184,
+      "rewards/chosen": 0.719990074634552,
+      "rewards/margins": 8.114697456359863,
+      "rewards/rejected": -7.394706726074219,
       "step": 250
     },
     {
       "epoch": 0.16,
-      "grad_norm": 24.896251678466797,
+      "grad_norm": 33.13041305541992,
       "kl": 0.0,
       "learning_rate": 4.65149359886202e-07,
-      "logps/chosen": -269.20794677734375,
-      "logps/rejected": -328.73486328125,
-      "loss": 0.1281,
-      "rewards/chosen": 1.6262487173080444,
-      "rewards/margins": 9.223333358764648,
-      "rewards/rejected": -7.597084999084473,
+      "logps/chosen": -333.77032470703125,
+      "logps/rejected": -404.6194152832031,
+      "loss": 0.2038,
+      "rewards/chosen": 0.7432358860969543,
+      "rewards/margins": 8.134955406188965,
+      "rewards/rejected": -7.391718864440918,
       "step": 255
     },
     {
       "epoch": 0.17,
-      "grad_norm": 20.800025939941406,
+      "grad_norm": 29.542213439941406,
       "kl": 0.0,
       "learning_rate": 4.633712660028449e-07,
-      "logps/chosen": -270.3649597167969,
-      "logps/rejected": -330.872314453125,
-      "loss": 0.1279,
-      "rewards/chosen": 1.5803369283676147,
-      "rewards/margins": 9.708114624023438,
-      "rewards/rejected": -8.127778053283691,
+      "logps/chosen": -286.2034606933594,
+      "logps/rejected": -383.937744140625,
+      "loss": 0.2245,
+      "rewards/chosen": 0.6780925989151001,
+      "rewards/margins": 6.890599727630615,
+      "rewards/rejected": -6.212507724761963,
       "step": 260
     },
     {
       "epoch": 0.17,
-      "grad_norm": 20.315540313720703,
+      "grad_norm": 25.915569305419922,
       "kl": 0.0,
       "learning_rate": 4.615931721194879e-07,
-      "logps/chosen": -277.21807861328125,
-      "logps/rejected": -313.4653625488281,
-      "loss": 0.1461,
-      "rewards/chosen": 1.6411361694335938,
-      "rewards/margins": 7.905195713043213,
-      "rewards/rejected": -6.264059543609619,
+      "logps/chosen": -287.2189025878906,
+      "logps/rejected": -431.18115234375,
+      "loss": 0.2253,
+      "rewards/chosen": 0.7109718918800354,
+      "rewards/margins": 7.668128967285156,
+      "rewards/rejected": -6.957158088684082,
       "step": 265
     },
     {
       "epoch": 0.17,
-      "grad_norm": 19.68859100341797,
+      "grad_norm": 28.604249954223633,
       "kl": 0.0,
       "learning_rate": 4.5981507823613085e-07,
-      "logps/chosen": -275.69696044921875,
-      "logps/rejected": -295.79400634765625,
-      "loss": 0.1547,
-      "rewards/chosen": 1.6596572399139404,
-      "rewards/margins": 7.9982404708862305,
-      "rewards/rejected": -6.338583469390869,
+      "logps/chosen": -249.9962158203125,
+      "logps/rejected": -427.6434631347656,
+      "loss": 0.1922,
+      "rewards/chosen": 0.9254335165023804,
+      "rewards/margins": 8.525291442871094,
+      "rewards/rejected": -7.599857330322266,
       "step": 270
     },
     {
       "epoch": 0.18,
-      "grad_norm": 27.070371627807617,
+      "grad_norm": 32.13933563232422,
       "kl": 0.0,
       "learning_rate": 4.580369843527738e-07,
-      "logps/chosen": -272.72735595703125,
-      "logps/rejected": -310.302734375,
-      "loss": 0.1673,
-      "rewards/chosen": 1.2246049642562866,
-      "rewards/margins": 8.44649600982666,
-      "rewards/rejected": -7.221890449523926,
+      "logps/chosen": -250.48397827148438,
+      "logps/rejected": -437.86541748046875,
+      "loss": 0.2104,
+      "rewards/chosen": 0.916563868522644,
+      "rewards/margins": 8.427824020385742,
+      "rewards/rejected": -7.511260986328125,
       "step": 275
     },
     {
       "epoch": 0.18,
-      "grad_norm": 15.166617393493652,
+      "grad_norm": 24.797935485839844,
       "kl": 0.0,
       "learning_rate": 4.562588904694168e-07,
-      "logps/chosen": -267.9317932128906,
-      "logps/rejected": -316.60479736328125,
-      "loss": 0.1304,
-      "rewards/chosen": 1.7944562435150146,
-      "rewards/margins": 8.752721786499023,
-      "rewards/rejected": -6.958265781402588,
+      "logps/chosen": -289.09368896484375,
+      "logps/rejected": -382.2262268066406,
+      "loss": 0.2292,
+      "rewards/chosen": 0.8693010210990906,
+      "rewards/margins": 8.239991188049316,
+      "rewards/rejected": -7.37069034576416,
       "step": 280
     },
     {
       "epoch": 0.18,
-      "grad_norm": 27.143291473388672,
+      "grad_norm": 29.941953659057617,
       "kl": 0.0,
       "learning_rate": 4.544807965860597e-07,
-      "logps/chosen": -257.7388916015625,
-      "logps/rejected": -308.5174560546875,
-      "loss": 0.1421,
-      "rewards/chosen": 1.6366589069366455,
-      "rewards/margins": 8.678224563598633,
-      "rewards/rejected": -7.04156494140625,
+      "logps/chosen": -286.82781982421875,
+      "logps/rejected": -399.9955139160156,
+      "loss": 0.2097,
+      "rewards/chosen": 1.1082446575164795,
+      "rewards/margins": 8.612449645996094,
+      "rewards/rejected": -7.504205226898193,
       "step": 285
     },
     {
       "epoch": 0.19,
-      "grad_norm": 22.1202392578125,
+      "grad_norm": 28.574798583984375,
       "kl": 0.0,
       "learning_rate": 4.5270270270270264e-07,
-      "logps/chosen": -266.8901062011719,
-      "logps/rejected": -322.6683654785156,
-      "loss": 0.1447,
-      "rewards/chosen": 1.787255883216858,
-      "rewards/margins": 9.206243515014648,
-      "rewards/rejected": -7.418987274169922,
+      "logps/chosen": -296.76751708984375,
+      "logps/rejected": -426.483154296875,
+      "loss": 0.201,
+      "rewards/chosen": 1.1846811771392822,
+      "rewards/margins": 8.457054138183594,
+      "rewards/rejected": -7.272373199462891,
       "step": 290
     },
     {
       "epoch": 0.19,
-      "grad_norm": 24.7782039642334,
+      "grad_norm": 24.285078048706055,
       "kl": 0.0,
       "learning_rate": 4.509246088193456e-07,
-      "logps/chosen": -274.30438232421875,
-      "logps/rejected": -342.98541259765625,
-      "loss": 0.1421,
-      "rewards/chosen": 1.776602029800415,
-      "rewards/margins": 9.053030967712402,
-      "rewards/rejected": -7.276429176330566,
+      "logps/chosen": -253.2152099609375,
+      "logps/rejected": -407.7524719238281,
+      "loss": 0.2124,
+      "rewards/chosen": 0.8104816675186157,
+      "rewards/margins": 8.143354415893555,
+      "rewards/rejected": -7.332873344421387,
       "step": 295
     },
     {
       "epoch": 0.19,
-      "grad_norm": 23.54313087463379,
+      "grad_norm": 28.82489585876465,
       "kl": 0.0,
       "learning_rate": 4.491465149359886e-07,
-      "logps/chosen": -215.9270477294922,
-      "logps/rejected": -303.5273742675781,
-      "loss": 0.1438,
-      "rewards/chosen": 1.868506669998169,
-      "rewards/margins": 9.129117965698242,
-      "rewards/rejected": -7.260611534118652,
+      "logps/chosen": -282.62725830078125,
+      "logps/rejected": -399.4122009277344,
+      "loss": 0.2058,
+      "rewards/chosen": 0.829262375831604,
+      "rewards/margins": 8.509421348571777,
+      "rewards/rejected": -7.680159568786621,
       "step": 300
     },
     {
       "epoch": 0.2,
-      "grad_norm": 17.70969009399414,
+      "grad_norm": 26.383644104003906,
       "kl": 0.0,
       "learning_rate": 4.4736842105263156e-07,
-      "logps/chosen": -240.4123992919922,
-      "logps/rejected": -336.1444396972656,
-      "loss": 0.1403,
-      "rewards/chosen": 1.7039588689804077,
-      "rewards/margins": 9.00100326538086,
-      "rewards/rejected": -7.297044277191162,
+      "logps/chosen": -253.8015899658203,
+      "logps/rejected": -423.59454345703125,
+      "loss": 0.1916,
+      "rewards/chosen": 0.9279552698135376,
+      "rewards/margins": 8.754355430603027,
+      "rewards/rejected": -7.8264007568359375,
       "step": 305
     },
     {
       "epoch": 0.2,
-      "grad_norm": 19.908315658569336,
+      "grad_norm": 30.41068458557129,
       "kl": 0.0,
       "learning_rate": 4.4559032716927454e-07,
-      "logps/chosen": -212.1704864501953,
-      "logps/rejected": -340.3092041015625,
-      "loss": 0.1338,
-      "rewards/chosen": 1.6462271213531494,
-      "rewards/margins": 8.726274490356445,
-      "rewards/rejected": -7.0800461769104,
+      "logps/chosen": -306.1068420410156,
+      "logps/rejected": -404.15777587890625,
+      "loss": 0.1763,
+      "rewards/chosen": 1.2404173612594604,
+      "rewards/margins": 8.863131523132324,
+      "rewards/rejected": -7.622714042663574,
       "step": 310
     },
     {
       "epoch": 0.2,
-      "grad_norm": 22.928499221801758,
+      "grad_norm": 27.128496170043945,
       "kl": 0.0,
       "learning_rate": 4.438122332859175e-07,
-      "logps/chosen": -243.7815704345703,
-      "logps/rejected": -324.60772705078125,
-      "loss": 0.1323,
-      "rewards/chosen": 1.6998401880264282,
-      "rewards/margins": 8.809240341186523,
-      "rewards/rejected": -7.109400749206543,
+      "logps/chosen": -288.37420654296875,
+      "logps/rejected": -434.837158203125,
+      "loss": 0.2103,
+      "rewards/chosen": 0.9712983965873718,
+      "rewards/margins": 8.523191452026367,
+      "rewards/rejected": -7.551893711090088,
       "step": 315
     },
     {
       "epoch": 0.2,
-      "grad_norm": 20.08190155029297,
+      "grad_norm": 24.96108627319336,
       "kl": 0.0,
       "learning_rate": 4.420341394025605e-07,
-      "logps/chosen": -263.37725830078125,
-      "logps/rejected": -317.4496154785156,
-      "loss": 0.1443,
-      "rewards/chosen": 1.706053376197815,
-      "rewards/margins": 8.708757400512695,
-      "rewards/rejected": -7.002703666687012,
+      "logps/chosen": -225.9550018310547,
+      "logps/rejected": -397.37725830078125,
+      "loss": 0.1946,
+      "rewards/chosen": 0.9277496337890625,
+      "rewards/margins": 8.359576225280762,
+      "rewards/rejected": -7.431826591491699,
       "step": 320
     },
     {
       "epoch": 0.21,
-      "grad_norm": 25.728233337402344,
+      "grad_norm": 27.4498348236084,
       "kl": 0.0,
       "learning_rate": 4.4025604551920335e-07,
-      "logps/chosen": -282.44989013671875,
-      "logps/rejected": -325.9697265625,
-      "loss": 0.1351,
-      "rewards/chosen": 1.6976518630981445,
-      "rewards/margins": 9.016552925109863,
-      "rewards/rejected": -7.318901062011719,
+      "logps/chosen": -326.1064453125,
+      "logps/rejected": -457.33917236328125,
+      "loss": 0.1974,
+      "rewards/chosen": 0.9113430976867676,
+      "rewards/margins": 9.018231391906738,
+      "rewards/rejected": -8.106887817382812,
       "step": 325
     },
     {
       "epoch": 0.21,
-      "grad_norm": 22.311450958251953,
+      "grad_norm": 26.263656616210938,
       "kl": 0.0,
       "learning_rate": 4.384779516358463e-07,
-      "logps/chosen": -260.39447021484375,
-      "logps/rejected": -309.620849609375,
-      "loss": 0.1311,
-      "rewards/chosen": 1.8077850341796875,
-      "rewards/margins": 9.406209945678711,
-      "rewards/rejected": -7.598425388336182,
+      "logps/chosen": -264.4283752441406,
+      "logps/rejected": -424.5361328125,
+      "loss": 0.2007,
+      "rewards/chosen": 0.9189130663871765,
+      "rewards/margins": 8.478607177734375,
+      "rewards/rejected": -7.559694766998291,
       "step": 330
     },
     {
       "epoch": 0.21,
-      "grad_norm": 23.20633316040039,
+      "grad_norm": 32.865238189697266,
       "kl": 0.0,
       "learning_rate": 4.366998577524893e-07,
-      "logps/chosen": -302.47509765625,
-      "logps/rejected": -318.767578125,
-      "loss": 0.128,
-      "rewards/chosen": 1.7027689218521118,
-      "rewards/margins": 10.0870943069458,
-      "rewards/rejected": -8.38432502746582,
+      "logps/chosen": -321.8217468261719,
+      "logps/rejected": -422.74346923828125,
+      "loss": 0.1828,
+      "rewards/chosen": 1.213265299797058,
+      "rewards/margins": 9.324636459350586,
+      "rewards/rejected": -8.111371994018555,
       "step": 335
     },
     {
       "epoch": 0.22,
-      "grad_norm": 25.405933380126953,
+      "grad_norm": 25.276630401611328,
       "kl": 0.0,
       "learning_rate": 4.3492176386913227e-07,
-      "logps/chosen": -238.94827270507812,
-      "logps/rejected": -326.12786865234375,
-      "loss": 0.1505,
-      "rewards/chosen": 1.7603483200073242,
-      "rewards/margins": 9.71554183959961,
-      "rewards/rejected": -7.955193519592285,
+      "logps/chosen": -286.50762939453125,
+      "logps/rejected": -416.09246826171875,
+      "loss": 0.1918,
+      "rewards/chosen": 1.121449589729309,
+      "rewards/margins": 8.465108871459961,
+      "rewards/rejected": -7.343659400939941,
       "step": 340
     },
     {
       "epoch": 0.22,
-      "grad_norm": 22.41493034362793,
+      "grad_norm": 33.83981704711914,
       "kl": 0.0,
       "learning_rate": 4.3314366998577524e-07,
-      "logps/chosen": -253.826416015625,
-      "logps/rejected": -330.7270812988281,
-      "loss": 0.1413,
-      "rewards/chosen": 1.8580402135849,
-      "rewards/margins": 9.722744941711426,
-      "rewards/rejected": -7.8647050857543945,
+      "logps/chosen": -310.34197998046875,
+      "logps/rejected": -378.6121826171875,
+      "loss": 0.2018,
+      "rewards/chosen": 1.2548576593399048,
+      "rewards/margins": 8.125253677368164,
+      "rewards/rejected": -6.870396614074707,
       "step": 345
     },
     {
       "epoch": 0.22,
-      "grad_norm": 26.315628051757812,
+      "grad_norm": 33.48633575439453,
       "kl": 0.0,
       "learning_rate": 4.313655761024182e-07,
-      "logps/chosen": -286.4155578613281,
-      "logps/rejected": -342.62652587890625,
-      "loss": 0.1422,
-      "rewards/chosen": 1.7754218578338623,
-      "rewards/margins": 10.7272367477417,
-      "rewards/rejected": -8.951814651489258,
+      "logps/chosen": -281.07489013671875,
+      "logps/rejected": -429.463134765625,
+      "loss": 0.193,
+      "rewards/chosen": 1.1550464630126953,
+      "rewards/margins": 9.02354907989502,
+      "rewards/rejected": -7.868502616882324,
       "step": 350
     },
     {
       "epoch": 0.23,
-      "grad_norm": 22.07290267944336,
+      "grad_norm": 24.945789337158203,
       "kl": 0.0,
       "learning_rate": 4.2958748221906114e-07,
-      "logps/chosen": -239.4450225830078,
-      "logps/rejected": -310.7589111328125,
-      "loss": 0.1108,
-      "rewards/chosen": 1.9980005025863647,
-      "rewards/margins": 10.0792818069458,
-      "rewards/rejected": -8.081281661987305,
+      "logps/chosen": -272.5113830566406,
+      "logps/rejected": -378.4676513671875,
+      "loss": 0.1983,
+      "rewards/chosen": 1.260040044784546,
+      "rewards/margins": 8.773920059204102,
+      "rewards/rejected": -7.513879299163818,
       "step": 355
     },
     {
       "epoch": 0.23,
-      "grad_norm": 22.56456756591797,
+      "grad_norm": 27.782686233520508,
       "kl": 0.0,
       "learning_rate": 4.278093883357041e-07,
-      "logps/chosen": -269.43560791015625,
-      "logps/rejected": -328.5326843261719,
-      "loss": 0.1524,
-      "rewards/chosen": 1.747078537940979,
-      "rewards/margins": 9.907144546508789,
-      "rewards/rejected": -8.160065650939941,
+      "logps/chosen": -265.1444396972656,
+      "logps/rejected": -415.40264892578125,
+      "loss": 0.1671,
+      "rewards/chosen": 1.351818323135376,
+      "rewards/margins": 9.423505783081055,
+      "rewards/rejected": -8.071688652038574,
       "step": 360
     },
     {
       "epoch": 0.23,
-      "grad_norm": 24.159225463867188,
+      "grad_norm": 25.29680824279785,
       "kl": 0.0,
       "learning_rate": 4.260312944523471e-07,
-      "logps/chosen": -241.72286987304688,
-      "logps/rejected": -353.99493408203125,
-      "loss": 0.1185,
-      "rewards/chosen": 2.0929551124572754,
-      "rewards/margins": 10.861230850219727,
-      "rewards/rejected": -8.768275260925293,
+      "logps/chosen": -320.04193115234375,
+      "logps/rejected": -462.45574951171875,
+      "loss": 0.1719,
+      "rewards/chosen": 1.2879645824432373,
+      "rewards/margins": 9.490580558776855,
+      "rewards/rejected": -8.202616691589355,
       "step": 365
     },
     {
       "epoch": 0.24,
-      "grad_norm": 21.119632720947266,
+      "grad_norm": 24.49195671081543,
       "kl": 0.0,
       "learning_rate": 4.2425320056899e-07,
-      "logps/chosen": -263.73370361328125,
-      "logps/rejected": -343.69866943359375,
-      "loss": 0.1164,
-      "rewards/chosen": 1.9714298248291016,
-      "rewards/margins": 10.647806167602539,
-      "rewards/rejected": -8.676377296447754,
+      "logps/chosen": -289.80413818359375,
+      "logps/rejected": -404.28094482421875,
+      "loss": 0.1993,
+      "rewards/chosen": 1.0154571533203125,
+      "rewards/margins": 8.819131851196289,
+      "rewards/rejected": -7.803675174713135,
       "step": 370
     },
     {
       "epoch": 0.24,
-      "grad_norm": 23.59329605102539,
+      "grad_norm": 26.57324981689453,
       "kl": 0.0,
       "learning_rate": 4.22475106685633e-07,
-      "logps/chosen": -244.0771026611328,
-      "logps/rejected": -348.35693359375,
-      "loss": 0.1218,
-      "rewards/chosen": 1.8124616146087646,
-      "rewards/margins": 10.569868087768555,
-      "rewards/rejected": -8.757406234741211,
+      "logps/chosen": -249.9967041015625,
+      "logps/rejected": -436.7953186035156,
+      "loss": 0.1902,
+      "rewards/chosen": 1.0482652187347412,
+      "rewards/margins": 9.115781784057617,
+      "rewards/rejected": -8.067517280578613,
       "step": 375
     },
     {
       "epoch": 0.24,
-      "grad_norm": 23.84940528869629,
+      "grad_norm": 28.738710403442383,
       "kl": 0.0,
       "learning_rate": 4.2069701280227595e-07,
-      "logps/chosen": -241.4117889404297,
-      "logps/rejected": -336.4566345214844,
-      "loss": 0.1294,
-      "rewards/chosen": 1.8767503499984741,
-      "rewards/margins": 10.489054679870605,
-      "rewards/rejected": -8.612303733825684,
+      "logps/chosen": -241.308349609375,
+      "logps/rejected": -425.63970947265625,
+      "loss": 0.2145,
+      "rewards/chosen": 0.9155824780464172,
+      "rewards/margins": 8.491414070129395,
+      "rewards/rejected": -7.575830936431885,
       "step": 380
     },
     {
       "epoch": 0.25,
-      "grad_norm": 16.005657196044922,
+      "grad_norm": 26.93291664123535,
       "kl": 0.0,
       "learning_rate": 4.189189189189189e-07,
-      "logps/chosen": -307.21600341796875,
-      "logps/rejected": -341.0569152832031,
-      "loss": 0.1154,
-      "rewards/chosen": 1.901063323020935,
-      "rewards/margins": 10.400434494018555,
-      "rewards/rejected": -8.499369621276855,
+      "logps/chosen": -298.3195495605469,
+      "logps/rejected": -429.3912048339844,
+      "loss": 0.1832,
+      "rewards/chosen": 1.0986849069595337,
+      "rewards/margins": 9.5584716796875,
+      "rewards/rejected": -8.459787368774414,
       "step": 385
     },
     {
       "epoch": 0.25,
-      "grad_norm": 16.274534225463867,
+      "grad_norm": 28.49407386779785,
       "kl": 0.0,
       "learning_rate": 4.1714082503556185e-07,
-      "logps/chosen": -253.8114471435547,
-      "logps/rejected": -333.0570983886719,
-      "loss": 0.1327,
-      "rewards/chosen": 1.955413818359375,
-      "rewards/margins": 10.895428657531738,
-      "rewards/rejected": -8.940014839172363,
+      "logps/chosen": -289.1352844238281,
+      "logps/rejected": -421.0469665527344,
+      "loss": 0.1782,
+      "rewards/chosen": 0.9938220977783203,
+      "rewards/margins": 9.06061840057373,
+      "rewards/rejected": -8.066795349121094,
       "step": 390
     },
     {
       "epoch": 0.25,
-      "grad_norm": 20.72968101501465,
+      "grad_norm": 23.859844207763672,
       "kl": 0.0,
       "learning_rate": 4.153627311522048e-07,
-      "logps/chosen": -261.703857421875,
-      "logps/rejected": -339.4162902832031,
-      "loss": 0.1308,
-      "rewards/chosen": 1.7927815914154053,
-      "rewards/margins": 10.807384490966797,
-      "rewards/rejected": -9.014602661132812,
+      "logps/chosen": -259.1231689453125,
+      "logps/rejected": -452.96746826171875,
+      "loss": 0.1904,
+      "rewards/chosen": 0.8238496780395508,
+      "rewards/margins": 9.638608932495117,
+      "rewards/rejected": -8.814759254455566,
       "step": 395
     },
     {
       "epoch": 0.26,
-      "grad_norm": 21.173845291137695,
+      "grad_norm": 32.167110443115234,
       "kl": 0.0,
       "learning_rate": 4.135846372688478e-07,
-      "logps/chosen": -260.5010681152344,
-      "logps/rejected": -359.41851806640625,
-      "loss": 0.1131,
-      "rewards/chosen": 1.8654544353485107,
-      "rewards/margins": 11.833639144897461,
-      "rewards/rejected": -9.968184471130371,
+      "logps/chosen": -295.9509582519531,
+      "logps/rejected": -448.93377685546875,
+      "loss": 0.1753,
+      "rewards/chosen": 0.963021457195282,
+      "rewards/margins": 9.281633377075195,
+      "rewards/rejected": -8.318610191345215,
       "step": 400
     },
     {
       "epoch": 0.26,
-      "grad_norm": 17.008028030395508,
+      "grad_norm": 27.581504821777344,
       "kl": 0.0,
       "learning_rate": 4.1180654338549077e-07,
-      "logps/chosen": -234.20315551757812,
-      "logps/rejected": -342.5408020019531,
-      "loss": 0.1224,
-      "rewards/chosen": 2.086550235748291,
-      "rewards/margins": 10.627596855163574,
-      "rewards/rejected": -8.541048049926758,
+      "logps/chosen": -281.5043029785156,
+      "logps/rejected": -388.4980163574219,
+      "loss": 0.2066,
+      "rewards/chosen": 1.0012767314910889,
+      "rewards/margins": 8.807787895202637,
+      "rewards/rejected": -7.806510925292969,
       "step": 405
     },
     {
       "epoch": 0.26,
-      "grad_norm": 19.24855613708496,
+      "grad_norm": 29.895170211791992,
       "kl": 0.0,
       "learning_rate": 4.100284495021337e-07,
-      "logps/chosen": -282.03106689453125,
-      "logps/rejected": -347.0184631347656,
-      "loss": 0.1212,
-      "rewards/chosen": 1.9099693298339844,
-      "rewards/margins": 10.274763107299805,
-      "rewards/rejected": -8.36479377746582,
+      "logps/chosen": -287.0378723144531,
+      "logps/rejected": -427.29864501953125,
+      "loss": 0.2007,
+      "rewards/chosen": 0.8726502656936646,
+      "rewards/margins": 9.106569290161133,
+      "rewards/rejected": -8.233919143676758,
       "step": 410
     },
     {
       "epoch": 0.27,
-      "grad_norm": 19.914691925048828,
+      "grad_norm": 24.92208480834961,
       "kl": 0.0,
       "learning_rate": 4.082503556187766e-07,
-      "logps/chosen": -278.811767578125,
-      "logps/rejected": -348.43096923828125,
-      "loss": 0.1209,
-      "rewards/chosen": 1.7562357187271118,
-      "rewards/margins": 11.304061889648438,
-      "rewards/rejected": -9.547826766967773,
+      "logps/chosen": -268.7417297363281,
+      "logps/rejected": -411.2549743652344,
+      "loss": 0.1975,
+      "rewards/chosen": 0.902958869934082,
+      "rewards/margins": 8.691009521484375,
+      "rewards/rejected": -7.788050174713135,
       "step": 415
     },
     {
       "epoch": 0.27,
-      "grad_norm": 32.35190963745117,
+      "grad_norm": 30.231552124023438,
       "kl": 0.0,
       "learning_rate": 4.064722617354196e-07,
-      "logps/chosen": -272.9192199707031,
-      "logps/rejected": -339.4073181152344,
-      "loss": 0.1005,
-      "rewards/chosen": 2.1069161891937256,
-      "rewards/margins": 11.689797401428223,
-      "rewards/rejected": -9.582880973815918,
+      "logps/chosen": -286.7518310546875,
+      "logps/rejected": -451.4786071777344,
+      "loss": 0.1867,
+      "rewards/chosen": 1.2976207733154297,
+      "rewards/margins": 9.737428665161133,
+      "rewards/rejected": -8.43980884552002,
       "step": 420
     },
     {
       "epoch": 0.27,
-      "grad_norm": 16.27385902404785,
+      "grad_norm": 24.8756046295166,
       "kl": 0.0,
       "learning_rate": 4.0469416785206256e-07,
-      "logps/chosen": -233.26083374023438,
-      "logps/rejected": -329.8385314941406,
-      "loss": 0.1135,
-      "rewards/chosen": 2.1503446102142334,
-      "rewards/margins": 10.886509895324707,
-      "rewards/rejected": -8.736165046691895,
+      "logps/chosen": -282.55706787109375,
+      "logps/rejected": -433.0348205566406,
+      "loss": 0.1846,
+      "rewards/chosen": 1.047466516494751,
+      "rewards/margins": 9.786076545715332,
+      "rewards/rejected": -8.738609313964844,
       "step": 425
     },
     {
       "epoch": 0.28,
-      "grad_norm": 22.10724449157715,
+      "grad_norm": 29.7546329498291,
       "kl": 0.0,
       "learning_rate": 4.0291607396870553e-07,
-      "logps/chosen": -267.9533996582031,
-      "logps/rejected": -359.1152038574219,
-      "loss": 0.1197,
-      "rewards/chosen": 1.8767788410186768,
-      "rewards/margins": 11.441519737243652,
-      "rewards/rejected": -9.564741134643555,
+      "logps/chosen": -285.54852294921875,
+      "logps/rejected": -450.5826110839844,
+      "loss": 0.1708,
+      "rewards/chosen": 1.2604820728302002,
+      "rewards/margins": 9.911845207214355,
+      "rewards/rejected": -8.651362419128418,
       "step": 430
     },
     {
       "epoch": 0.28,
-      "grad_norm": 29.721330642700195,
+      "grad_norm": 26.89280891418457,
       "kl": 0.0,
       "learning_rate": 4.011379800853485e-07,
-      "logps/chosen": -268.9942932128906,
-      "logps/rejected": -377.97308349609375,
-      "loss": 0.1061,
-      "rewards/chosen": 2.016845703125,
-      "rewards/margins": 11.474628448486328,
-      "rewards/rejected": -9.457781791687012,
+      "logps/chosen": -274.69195556640625,
+      "logps/rejected": -415.19989013671875,
+      "loss": 0.1939,
+      "rewards/chosen": 1.1895002126693726,
+      "rewards/margins": 9.175015449523926,
+      "rewards/rejected": -7.985515594482422,
       "step": 435
     },
     {
       "epoch": 0.28,
-      "grad_norm": 24.95069122314453,
+      "grad_norm": 49.50102996826172,
       "kl": 0.0,
       "learning_rate": 3.993598862019915e-07,
-      "logps/chosen": -279.73944091796875,
-      "logps/rejected": -370.55474853515625,
-      "loss": 0.1045,
-      "rewards/chosen": 2.340444803237915,
-      "rewards/margins": 11.733491897583008,
-      "rewards/rejected": -9.393046379089355,
+      "logps/chosen": -257.03985595703125,
+      "logps/rejected": -437.6024475097656,
+      "loss": 0.179,
+      "rewards/chosen": 1.1160862445831299,
+      "rewards/margins": 10.291946411132812,
+      "rewards/rejected": -9.175861358642578,
       "step": 440
     },
     {
       "epoch": 0.28,
-      "grad_norm": 26.99854278564453,
+      "grad_norm": 23.507465362548828,
       "kl": 0.0,
       "learning_rate": 3.975817923186344e-07,
-      "logps/chosen": -245.24417114257812,
-      "logps/rejected": -336.7696533203125,
-      "loss": 0.1234,
-      "rewards/chosen": 2.2276217937469482,
-      "rewards/margins": 10.63569164276123,
-      "rewards/rejected": -8.40807056427002,
+      "logps/chosen": -244.9132537841797,
+      "logps/rejected": -417.92010498046875,
+      "loss": 0.1943,
+      "rewards/chosen": 0.9916771054267883,
+      "rewards/margins": 8.110875129699707,
+      "rewards/rejected": -7.119197845458984,
       "step": 445
     },
     {
       "epoch": 0.29,
-      "grad_norm": 22.343507766723633,
+      "grad_norm": 23.709091186523438,
       "kl": 0.0,
       "learning_rate": 3.9580369843527737e-07,
-      "logps/chosen": -250.68197631835938,
-      "logps/rejected": -364.1970520019531,
-      "loss": 0.1342,
-      "rewards/chosen": 2.1794769763946533,
-      "rewards/margins": 10.827537536621094,
-      "rewards/rejected": -8.64806079864502,
+      "logps/chosen": -266.93243408203125,
+      "logps/rejected": -474.86004638671875,
+      "loss": 0.1754,
+      "rewards/chosen": 1.332706093788147,
+      "rewards/margins": 9.872304916381836,
+      "rewards/rejected": -8.53959846496582,
       "step": 450
     },
     {
       "epoch": 0.29,
-      "grad_norm": 16.532583236694336,
+      "grad_norm": 28.132789611816406,
       "kl": 0.0,
       "learning_rate": 3.940256045519203e-07,
-      "logps/chosen": -246.8504638671875,
-      "logps/rejected": -336.2493896484375,
-      "loss": 0.1151,
-      "rewards/chosen": 1.9471546411514282,
-      "rewards/margins": 11.059728622436523,
-      "rewards/rejected": -9.112573623657227,
+      "logps/chosen": -269.0207824707031,
+      "logps/rejected": -441.16259765625,
+      "loss": 0.1764,
+      "rewards/chosen": 1.136040449142456,
+      "rewards/margins": 9.543194770812988,
+      "rewards/rejected": -8.407154083251953,
       "step": 455
     },
     {
       "epoch": 0.29,
-      "grad_norm": 19.48154067993164,
+      "grad_norm": 27.44135284423828,
       "kl": 0.0,
       "learning_rate": 3.9224751066856327e-07,
-      "logps/chosen": -246.47671508789062,
-      "logps/rejected": -328.21038818359375,
-      "loss": 0.1156,
-      "rewards/chosen": 2.022249221801758,
-      "rewards/margins": 11.125974655151367,
-      "rewards/rejected": -9.103724479675293,
+      "logps/chosen": -261.8495178222656,
+      "logps/rejected": -416.13372802734375,
+      "loss": 0.2019,
+      "rewards/chosen": 1.0264383554458618,
+      "rewards/margins": 9.626348495483398,
+      "rewards/rejected": -8.599910736083984,
       "step": 460
     },
     {
       "epoch": 0.3,
-      "grad_norm": 17.954296112060547,
+      "grad_norm": 34.613067626953125,
       "kl": 0.0,
       "learning_rate": 3.9046941678520624e-07,
-      "logps/chosen": -254.69808959960938,
-      "logps/rejected": -347.45758056640625,
-      "loss": 0.1254,
-      "rewards/chosen": 2.0911641120910645,
-      "rewards/margins": 11.680787086486816,
-      "rewards/rejected": -9.58962345123291,
+      "logps/chosen": -305.896240234375,
+      "logps/rejected": -436.7113342285156,
+      "loss": 0.2045,
+      "rewards/chosen": 1.227920651435852,
+      "rewards/margins": 9.572298049926758,
+      "rewards/rejected": -8.344377517700195,
       "step": 465
     },
     {
       "epoch": 0.3,
-      "grad_norm": 21.16987419128418,
+      "grad_norm": 31.94422721862793,
       "kl": 0.0,
       "learning_rate": 3.886913229018492e-07,
-      "logps/chosen": -226.58358764648438,
-      "logps/rejected": -334.48138427734375,
-      "loss": 0.1437,
-      "rewards/chosen": 1.8157398700714111,
-      "rewards/margins": 10.399962425231934,
-      "rewards/rejected": -8.584221839904785,
+      "logps/chosen": -275.5303955078125,
+      "logps/rejected": -421.15020751953125,
+      "loss": 0.2019,
+      "rewards/chosen": 1.0201870203018188,
+      "rewards/margins": 9.202049255371094,
+      "rewards/rejected": -8.181862831115723,
       "step": 470
     },
     {
       "epoch": 0.3,
-      "grad_norm": 18.404508590698242,
+      "grad_norm": 24.941471099853516,
       "kl": 0.0,
       "learning_rate": 3.8691322901849213e-07,
-      "logps/chosen": -263.36395263671875,
-      "logps/rejected": -337.2552795410156,
-      "loss": 0.1078,
-      "rewards/chosen": 2.1132290363311768,
-      "rewards/margins": 11.090356826782227,
-      "rewards/rejected": -8.977127075195312,
+      "logps/chosen": -309.7427978515625,
+      "logps/rejected": -456.80987548828125,
+      "loss": 0.1867,
+      "rewards/chosen": 1.4211246967315674,
+      "rewards/margins": 9.695344924926758,
+      "rewards/rejected": -8.274219512939453,
       "step": 475
     },
     {
       "epoch": 0.31,
-      "grad_norm": 18.355857849121094,
+      "grad_norm": 25.9896240234375,
       "kl": 0.0,
       "learning_rate": 3.851351351351351e-07,
-      "logps/chosen": -256.04412841796875,
-      "logps/rejected": -356.82073974609375,
-      "loss": 0.1145,
-      "rewards/chosen": 1.9654195308685303,
-      "rewards/margins": 11.159137725830078,
-      "rewards/rejected": -9.193717956542969,
+      "logps/chosen": -285.4349365234375,
+      "logps/rejected": -400.69219970703125,
+      "loss": 0.1817,
+      "rewards/chosen": 1.505924105644226,
+      "rewards/margins": 8.47026538848877,
+      "rewards/rejected": -6.964341640472412,
       "step": 480
     },
     {
       "epoch": 0.31,
-      "grad_norm": 16.297401428222656,
+      "grad_norm": 29.482158660888672,
       "kl": 0.0,
       "learning_rate": 3.833570412517781e-07,
-      "logps/chosen": -330.457275390625,
-      "logps/rejected": -367.7776794433594,
-      "loss": 0.113,
-      "rewards/chosen": 1.8794314861297607,
-      "rewards/margins": 11.851155281066895,
-      "rewards/rejected": -9.971722602844238,
+      "logps/chosen": -262.8547058105469,
+      "logps/rejected": -404.0810546875,
+      "loss": 0.1838,
+      "rewards/chosen": 1.2003206014633179,
+      "rewards/margins": 8.89756965637207,
+      "rewards/rejected": -7.697249412536621,
       "step": 485
     },
     {
       "epoch": 0.31,
-      "grad_norm": 22.856266021728516,
+      "grad_norm": 24.927303314208984,
       "kl": 0.0,
       "learning_rate": 3.8157894736842105e-07,
-      "logps/chosen": -254.3659210205078,
-      "logps/rejected": -343.2192687988281,
-      "loss": 0.1138,
-      "rewards/chosen": 2.034006118774414,
-      "rewards/margins": 11.903759002685547,
-      "rewards/rejected": -9.869752883911133,
+      "logps/chosen": -279.3279724121094,
+      "logps/rejected": -426.81646728515625,
+      "loss": 0.1816,
+      "rewards/chosen": 1.2700178623199463,
+      "rewards/margins": 9.352417945861816,
+      "rewards/rejected": -8.082399368286133,
       "step": 490
     },
     {
       "epoch": 0.32,
-      "grad_norm": 21.443723678588867,
+      "grad_norm": 19.47848892211914,
       "kl": 0.0,
       "learning_rate": 3.7980085348506403e-07,
-      "logps/chosen": -278.9993591308594,
-      "logps/rejected": -333.3291320800781,
-      "loss": 0.1287,
-      "rewards/chosen": 1.906224012374878,
-      "rewards/margins": 10.766765594482422,
-      "rewards/rejected": -8.860541343688965,
+      "logps/chosen": -254.8472137451172,
+      "logps/rejected": -410.69219970703125,
+      "loss": 0.1617,
+      "rewards/chosen": 1.251443862915039,
+      "rewards/margins": 9.37706184387207,
+      "rewards/rejected": -8.125617980957031,
       "step": 495
     },
     {
       "epoch": 0.32,
-      "grad_norm": 24.97509002685547,
+      "grad_norm": 27.02752685546875,
       "kl": 0.0,
       "learning_rate": 3.7802275960170695e-07,
-      "logps/chosen": -238.44290161132812,
-      "logps/rejected": -338.4499206542969,
-      "loss": 0.1142,
-      "rewards/chosen": 2.103523015975952,
-      "rewards/margins": 11.162189483642578,
-      "rewards/rejected": -9.058666229248047,
+      "logps/chosen": -326.91485595703125,
+      "logps/rejected": -427.81121826171875,
+      "loss": 0.1869,
+      "rewards/chosen": 1.2399226427078247,
+      "rewards/margins": 9.168660163879395,
+      "rewards/rejected": -7.928736686706543,
       "step": 500
     },
     {
       "epoch": 0.32,
-      "grad_norm": 22.563114166259766,
+      "grad_norm": 23.25274658203125,
       "kl": 0.0,
       "learning_rate": 3.7624466571834987e-07,
-      "logps/chosen": -238.7085723876953,
-      "logps/rejected": -352.23663330078125,
-      "loss": 0.1396,
-      "rewards/chosen": 1.8689464330673218,
-      "rewards/margins": 10.945247650146484,
-      "rewards/rejected": -9.076301574707031,
+      "logps/chosen": -263.9060363769531,
+      "logps/rejected": -486.6805114746094,
+      "loss": 0.1867,
+      "rewards/chosen": 1.059422254562378,
+      "rewards/margins": 9.522220611572266,
+      "rewards/rejected": -8.462798118591309,
       "step": 505
     },
     {
       "epoch": 0.33,
-      "grad_norm": 17.017030715942383,
+      "grad_norm": 28.608474731445312,
       "kl": 0.0,
       "learning_rate": 3.7446657183499284e-07,
-      "logps/chosen": -241.0718231201172,
-      "logps/rejected": -340.98968505859375,
-      "loss": 0.1197,
-      "rewards/chosen": 1.9388542175292969,
-      "rewards/margins": 11.241914749145508,
-      "rewards/rejected": -9.303060531616211,
+      "logps/chosen": -294.48590087890625,
+      "logps/rejected": -468.9127502441406,
+      "loss": 0.1785,
+      "rewards/chosen": 1.3862662315368652,
+      "rewards/margins": 10.08003044128418,
+      "rewards/rejected": -8.693764686584473,
       "step": 510
     },
     {
       "epoch": 0.33,
-      "grad_norm": 16.066030502319336,
+      "grad_norm": 28.52597999572754,
       "kl": 0.0,
       "learning_rate": 3.726884779516358e-07,
-      "logps/chosen": -224.85086059570312,
-      "logps/rejected": -349.51800537109375,
-      "loss": 0.1094,
-      "rewards/chosen": 2.1027920246124268,
-      "rewards/margins": 11.039840698242188,
-      "rewards/rejected": -8.937047958374023,
+      "logps/chosen": -303.76116943359375,
+      "logps/rejected": -385.89471435546875,
+      "loss": 0.1849,
+      "rewards/chosen": 0.8810665011405945,
+      "rewards/margins": 8.036872863769531,
+      "rewards/rejected": -7.155806064605713,
       "step": 515
     },
     {
       "epoch": 0.33,
-      "grad_norm": 17.983535766601562,
+      "grad_norm": 26.567203521728516,
       "kl": 0.0,
       "learning_rate": 3.709103840682788e-07,
-      "logps/chosen": -269.41693115234375,
-      "logps/rejected": -347.45477294921875,
-      "loss": 0.1234,
-      "rewards/chosen": 2.2909629344940186,
-      "rewards/margins": 11.061802864074707,
-      "rewards/rejected": -8.770838737487793,
+      "logps/chosen": -300.5899963378906,
+      "logps/rejected": -455.53887939453125,
+      "loss": 0.1752,
+      "rewards/chosen": 1.6550683975219727,
+      "rewards/margins": 9.88080883026123,
+      "rewards/rejected": -8.225740432739258,
       "step": 520
     },
     {
       "epoch": 0.34,
-      "grad_norm": 32.42300796508789,
+      "grad_norm": 26.40464973449707,
       "kl": 0.0,
       "learning_rate": 3.6913229018492176e-07,
-      "logps/chosen": -274.3470764160156,
-      "logps/rejected": -335.10345458984375,
-      "loss": 0.1186,
-      "rewards/chosen": 2.1239845752716064,
-      "rewards/margins": 11.3440523147583,
-      "rewards/rejected": -9.220067977905273,
+      "logps/chosen": -291.1101989746094,
+      "logps/rejected": -429.8409118652344,
+      "loss": 0.1738,
+      "rewards/chosen": 1.6012027263641357,
+      "rewards/margins": 10.346236228942871,
+      "rewards/rejected": -8.745033264160156,
       "step": 525
     },
     {
       "epoch": 0.34,
-      "grad_norm": 16.601402282714844,
+      "grad_norm": 25.405059814453125,
       "kl": 0.0,
       "learning_rate": 3.6735419630156474e-07,
-      "logps/chosen": -232.43392944335938,
-      "logps/rejected": -354.2206726074219,
-      "loss": 0.1218,
-      "rewards/chosen": 2.117833137512207,
-      "rewards/margins": 11.192573547363281,
-      "rewards/rejected": -9.07474136352539,
+      "logps/chosen": -277.75,
+      "logps/rejected": -444.23687744140625,
+      "loss": 0.1632,
+      "rewards/chosen": 1.4320881366729736,
+      "rewards/margins": 10.243904113769531,
+      "rewards/rejected": -8.811816215515137,
       "step": 530
     },
     {
       "epoch": 0.34,
-      "grad_norm": 24.04266357421875,
+      "grad_norm": 26.102277755737305,
       "kl": 0.0,
       "learning_rate": 3.655761024182077e-07,
-      "logps/chosen": -277.41241455078125,
-      "logps/rejected": -334.04815673828125,
-      "loss": 0.1025,
-      "rewards/chosen": 2.3604273796081543,
-      "rewards/margins": 11.987722396850586,
-      "rewards/rejected": -9.627294540405273,
+      "logps/chosen": -275.7671813964844,
+      "logps/rejected": -451.432861328125,
+      "loss": 0.1818,
+      "rewards/chosen": 1.5062631368637085,
+      "rewards/margins": 9.949407577514648,
+      "rewards/rejected": -8.443144798278809,
       "step": 535
     },
     {
       "epoch": 0.35,
-      "grad_norm": 15.607158660888672,
+      "grad_norm": 27.81848907470703,
       "kl": 0.0,
       "learning_rate": 3.637980085348506e-07,
-      "logps/chosen": -286.7027282714844,
-      "logps/rejected": -319.68896484375,
-      "loss": 0.1124,
-      "rewards/chosen": 2.4556756019592285,
-      "rewards/margins": 11.582249641418457,
-      "rewards/rejected": -9.126574516296387,
+      "logps/chosen": -285.43463134765625,
+      "logps/rejected": -456.70257568359375,
+      "loss": 0.1667,
+      "rewards/chosen": 1.1070717573165894,
+      "rewards/margins": 10.036450386047363,
+      "rewards/rejected": -8.9293794631958,
       "step": 540
     },
     {
       "epoch": 0.35,
-      "grad_norm": 22.26024055480957,
+      "grad_norm": 26.733715057373047,
       "kl": 0.0,
       "learning_rate": 3.6201991465149355e-07,
-      "logps/chosen": -239.4503173828125,
-      "logps/rejected": -345.2344970703125,
-      "loss": 0.112,
-      "rewards/chosen": 1.9915698766708374,
-      "rewards/margins": 11.853796005249023,
-      "rewards/rejected": -9.862226486206055,
+      "logps/chosen": -275.365478515625,
+      "logps/rejected": -447.39874267578125,
+      "loss": 0.1584,
+      "rewards/chosen": 1.669955849647522,
+      "rewards/margins": 9.940733909606934,
+      "rewards/rejected": -8.27077865600586,
       "step": 545
     },
     {
       "epoch": 0.35,
-      "grad_norm": 29.981088638305664,
+      "grad_norm": 28.002460479736328,
       "kl": 0.0,
       "learning_rate": 3.602418207681365e-07,
-      "logps/chosen": -241.8583984375,
-      "logps/rejected": -365.6163024902344,
-      "loss": 0.1207,
-      "rewards/chosen": 1.9029747247695923,
-      "rewards/margins": 11.538789749145508,
-      "rewards/rejected": -9.635814666748047,
+      "logps/chosen": -275.7063903808594,
+      "logps/rejected": -432.69097900390625,
+      "loss": 0.1961,
+      "rewards/chosen": 1.1493828296661377,
+      "rewards/margins": 9.39341926574707,
+      "rewards/rejected": -8.244035720825195,
       "step": 550
     },
     {
       "epoch": 0.36,
-      "grad_norm": 17.248044967651367,
+      "grad_norm": 25.290157318115234,
       "kl": 0.0,
       "learning_rate": 3.584637268847795e-07,
-      "logps/chosen": -255.35635375976562,
-      "logps/rejected": -348.6636962890625,
-      "loss": 0.1267,
-      "rewards/chosen": 2.0199761390686035,
-      "rewards/margins": 11.458740234375,
-      "rewards/rejected": -9.438763618469238,
+      "logps/chosen": -276.50872802734375,
+      "logps/rejected": -427.5359802246094,
+      "loss": 0.1825,
+      "rewards/chosen": 1.6107838153839111,
+      "rewards/margins": 10.058174133300781,
+      "rewards/rejected": -8.447389602661133,
       "step": 555
     },
     {
       "epoch": 0.36,
-      "grad_norm": 20.46364974975586,
+      "grad_norm": 27.16476058959961,
       "kl": 0.0,
       "learning_rate": 3.5668563300142247e-07,
-      "logps/chosen": -271.98846435546875,
-      "logps/rejected": -319.4902038574219,
-      "loss": 0.1209,
-      "rewards/chosen": 2.140854597091675,
-      "rewards/margins": 11.506881713867188,
-      "rewards/rejected": -9.36602783203125,
+      "logps/chosen": -259.827880859375,
+      "logps/rejected": -409.2623596191406,
+      "loss": 0.1808,
+      "rewards/chosen": 1.3736449480056763,
+      "rewards/margins": 9.484289169311523,
+      "rewards/rejected": -8.110644340515137,
       "step": 560
     },
     {
       "epoch": 0.36,
-      "grad_norm": 13.751622200012207,
+      "grad_norm": 28.099790573120117,
       "kl": 0.0,
       "learning_rate": 3.5490753911806545e-07,
-      "logps/chosen": -249.31838989257812,
-      "logps/rejected": -363.65313720703125,
-      "loss": 0.1066,
-      "rewards/chosen": 2.302562713623047,
-      "rewards/margins": 12.324169158935547,
-      "rewards/rejected": -10.0216064453125,
+      "logps/chosen": -279.4620666503906,
+      "logps/rejected": -404.58563232421875,
+      "loss": 0.185,
+      "rewards/chosen": 1.319478988647461,
+      "rewards/margins": 9.3961181640625,
+      "rewards/rejected": -8.076639175415039,
       "step": 565
     },
     {
       "epoch": 0.36,
-      "grad_norm": 23.35749053955078,
+      "grad_norm": 28.673524856567383,
       "kl": 0.0,
       "learning_rate": 3.5312944523470837e-07,
-      "logps/chosen": -242.62533569335938,
-      "logps/rejected": -326.2989807128906,
-      "loss": 0.1213,
-      "rewards/chosen": 2.164386749267578,
-      "rewards/margins": 11.125585556030273,
-      "rewards/rejected": -8.961198806762695,
+      "logps/chosen": -283.595703125,
+      "logps/rejected": -403.426025390625,
+      "loss": 0.1838,
+      "rewards/chosen": 1.2756729125976562,
+      "rewards/margins": 9.318367958068848,
+      "rewards/rejected": -8.042695999145508,
       "step": 570
     },
     {
       "epoch": 0.37,
-      "grad_norm": 22.92085075378418,
+      "grad_norm": 24.276857376098633,
       "kl": 0.0,
       "learning_rate": 3.5135135135135134e-07,
-      "logps/chosen": -215.1080322265625,
-      "logps/rejected": -343.92315673828125,
-      "loss": 0.1096,
-      "rewards/chosen": 2.084261417388916,
-      "rewards/margins": 12.25261402130127,
-      "rewards/rejected": -10.168352127075195,
+      "logps/chosen": -278.02337646484375,
+      "logps/rejected": -390.92987060546875,
+      "loss": 0.189,
+      "rewards/chosen": 1.1147047281265259,
+      "rewards/margins": 9.350360870361328,
+      "rewards/rejected": -8.235654830932617,
       "step": 575
     },
     {
       "epoch": 0.37,
-      "grad_norm": 19.450267791748047,
+      "grad_norm": 24.185720443725586,
       "kl": 0.0,
       "learning_rate": 3.495732574679943e-07,
-      "logps/chosen": -217.7822723388672,
-      "logps/rejected": -329.91375732421875,
-      "loss": 0.1294,
-      "rewards/chosen": 2.3254265785217285,
-      "rewards/margins": 11.483414649963379,
-      "rewards/rejected": -9.157987594604492,
+      "logps/chosen": -317.0126953125,
+      "logps/rejected": -441.64324951171875,
+      "loss": 0.1709,
+      "rewards/chosen": 1.2199360132217407,
+      "rewards/margins": 10.402612686157227,
+      "rewards/rejected": -9.182676315307617,
       "step": 580
     },
     {
       "epoch": 0.37,
-      "grad_norm": 18.078920364379883,
+      "grad_norm": 21.31283187866211,
       "kl": 0.0,
       "learning_rate": 3.4779516358463724e-07,
-      "logps/chosen": -253.4072723388672,
-      "logps/rejected": -344.491455078125,
-      "loss": 0.0927,
-      "rewards/chosen": 2.3925743103027344,
-      "rewards/margins": 12.241676330566406,
-      "rewards/rejected": -9.849101066589355,
+      "logps/chosen": -299.08355712890625,
+      "logps/rejected": -408.64312744140625,
+      "loss": 0.1669,
+      "rewards/chosen": 1.33742356300354,
+      "rewards/margins": 9.711603164672852,
+      "rewards/rejected": -8.374178886413574,
       "step": 585
     },
     {
       "epoch": 0.38,
-      "grad_norm": 17.867704391479492,
+      "grad_norm": 27.55930519104004,
       "kl": 0.0,
       "learning_rate": 3.460170697012802e-07,
-      "logps/chosen": -253.86178588867188,
-      "logps/rejected": -367.8619689941406,
-      "loss": 0.1152,
-      "rewards/chosen": 2.281512975692749,
-      "rewards/margins": 12.078498840332031,
-      "rewards/rejected": -9.79698657989502,
+      "logps/chosen": -358.379150390625,
+      "logps/rejected": -390.5495910644531,
+      "loss": 0.1711,
+      "rewards/chosen": 1.5247728824615479,
+      "rewards/margins": 10.199918746948242,
+      "rewards/rejected": -8.675146102905273,
       "step": 590
     },
     {
       "epoch": 0.38,
-      "grad_norm": 19.863170623779297,
+      "grad_norm": 22.836402893066406,
       "kl": 0.0,
       "learning_rate": 3.442389758179232e-07,
-      "logps/chosen": -289.83990478515625,
-      "logps/rejected": -346.80987548828125,
-      "loss": 0.1098,
-      "rewards/chosen": 2.1612110137939453,
-      "rewards/margins": 12.505953788757324,
-      "rewards/rejected": -10.344742774963379,
+      "logps/chosen": -265.90155029296875,
+      "logps/rejected": -407.1119079589844,
+      "loss": 0.1813,
+      "rewards/chosen": 1.2516599893569946,
+      "rewards/margins": 9.78925895690918,
+      "rewards/rejected": -8.53760051727295,
       "step": 595
     },
     {
       "epoch": 0.38,
-      "grad_norm": 14.594950675964355,
+      "grad_norm": 22.36507225036621,
       "kl": 0.0,
       "learning_rate": 3.424608819345661e-07,
-      "logps/chosen": -199.16708374023438,
-      "logps/rejected": -352.14447021484375,
-      "loss": 0.0929,
-      "rewards/chosen": 1.9940685033798218,
-      "rewards/margins": 12.916508674621582,
-      "rewards/rejected": -10.922439575195312,
+      "logps/chosen": -272.3090515136719,
+      "logps/rejected": -486.1949157714844,
+      "loss": 0.1542,
+      "rewards/chosen": 1.5906411409378052,
+      "rewards/margins": 10.901646614074707,
+      "rewards/rejected": -9.311005592346191,
       "step": 600
     },
     {
       "epoch": 0.39,
-      "grad_norm": 20.205915451049805,
+      "grad_norm": 25.3875732421875,
       "kl": 0.0,
       "learning_rate": 3.406827880512091e-07,
-      "logps/chosen": -251.85546875,
-      "logps/rejected": -347.35113525390625,
-      "loss": 0.1137,
-      "rewards/chosen": 2.3019704818725586,
-      "rewards/margins": 12.349814414978027,
-      "rewards/rejected": -10.047843933105469,
+      "logps/chosen": -263.0488586425781,
+      "logps/rejected": -443.8939514160156,
+      "loss": 0.1639,
+      "rewards/chosen": 1.4088127613067627,
+      "rewards/margins": 10.464536666870117,
+      "rewards/rejected": -9.05572509765625,
       "step": 605
     },
     {
       "epoch": 0.39,
-      "grad_norm": 21.62274742126465,
+      "grad_norm": 49.29353713989258,
       "kl": 0.0,
       "learning_rate": 3.3890469416785205e-07,
-      "logps/chosen": -241.4209442138672,
-      "logps/rejected": -366.311767578125,
-      "loss": 0.104,
-      "rewards/chosen": 2.3162879943847656,
-      "rewards/margins": 11.884176254272461,
-      "rewards/rejected": -9.567889213562012,
+      "logps/chosen": -277.53131103515625,
+      "logps/rejected": -402.04229736328125,
+      "loss": 0.1994,
+      "rewards/chosen": 1.2445075511932373,
+      "rewards/margins": 9.684932708740234,
+      "rewards/rejected": -8.440424919128418,
       "step": 610
     },
     {
       "epoch": 0.39,
-      "grad_norm": 18.622377395629883,
+      "grad_norm": 21.34065055847168,
       "kl": 0.0,
       "learning_rate": 3.37126600284495e-07,
-      "logps/chosen": -270.2699279785156,
-      "logps/rejected": -338.7982482910156,
-      "loss": 0.1127,
-      "rewards/chosen": 2.2165682315826416,
-      "rewards/margins": 11.826861381530762,
-      "rewards/rejected": -9.6102933883667,
+      "logps/chosen": -231.4390106201172,
+      "logps/rejected": -402.5743103027344,
+      "loss": 0.1749,
+      "rewards/chosen": 1.636435866355896,
+      "rewards/margins": 9.378325462341309,
+      "rewards/rejected": -7.741889953613281,
       "step": 615
     },
     {
       "epoch": 0.4,
-      "grad_norm": 19.930374145507812,
+      "grad_norm": 28.961084365844727,
       "kl": 0.0,
       "learning_rate": 3.35348506401138e-07,
-      "logps/chosen": -265.74432373046875,
-      "logps/rejected": -366.4137268066406,
-      "loss": 0.1235,
-      "rewards/chosen": 2.140533924102783,
-      "rewards/margins": 12.077839851379395,
-      "rewards/rejected": -9.937305450439453,
+      "logps/chosen": -255.66268920898438,
+      "logps/rejected": -418.0624084472656,
+      "loss": 0.1836,
+      "rewards/chosen": 1.2639700174331665,
+      "rewards/margins": 9.351852416992188,
+      "rewards/rejected": -8.087881088256836,
       "step": 620
     },
     {
       "epoch": 0.4,
-      "grad_norm": 20.78235626220703,
+      "grad_norm": 28.00955581665039,
       "kl": 0.0,
       "learning_rate": 3.335704125177809e-07,
-      "logps/chosen": -251.783935546875,
-      "logps/rejected": -321.7081298828125,
-      "loss": 0.1251,
-      "rewards/chosen": 2.4168410301208496,
-      "rewards/margins": 11.567142486572266,
-      "rewards/rejected": -9.150300979614258,
+      "logps/chosen": -257.3089599609375,
+      "logps/rejected": -375.06072998046875,
+      "loss": 0.1711,
+      "rewards/chosen": 1.5304003953933716,
+      "rewards/margins": 9.457714080810547,
+      "rewards/rejected": -7.927313804626465,
       "step": 625
     },
     {
       "epoch": 0.4,
-      "grad_norm": 18.487886428833008,
+      "grad_norm": 27.666671752929688,
       "kl": 0.0,
       "learning_rate": 3.3179231863442384e-07,
-      "logps/chosen": -244.60250854492188,
-      "logps/rejected": -386.168212890625,
-      "loss": 0.0922,
-      "rewards/chosen": 2.371992588043213,
-      "rewards/margins": 13.357122421264648,
-      "rewards/rejected": -10.985128402709961,
+      "logps/chosen": -310.2152404785156,
+      "logps/rejected": -440.0098571777344,
+      "loss": 0.1814,
+      "rewards/chosen": 1.2127114534378052,
+      "rewards/margins": 10.335000991821289,
+      "rewards/rejected": -9.122289657592773,
       "step": 630
     },
     {
       "epoch": 0.41,
-      "grad_norm": 16.354122161865234,
+      "grad_norm": 22.16933822631836,
       "kl": 0.0,
       "learning_rate": 3.300142247510668e-07,
-      "logps/chosen": -252.8701171875,
-      "logps/rejected": -334.31744384765625,
-      "loss": 0.1003,
-      "rewards/chosen": 2.335937023162842,
-      "rewards/margins": 12.5576753616333,
-      "rewards/rejected": -10.2217378616333,
+      "logps/chosen": -260.0948181152344,
+      "logps/rejected": -431.2792053222656,
+      "loss": 0.1578,
+      "rewards/chosen": 1.282593011856079,
+      "rewards/margins": 9.947371482849121,
+      "rewards/rejected": -8.664778709411621,
       "step": 635
     },
     {
       "epoch": 0.41,
-      "grad_norm": 16.126724243164062,
+      "grad_norm": 25.740550994873047,
       "kl": 0.0,
       "learning_rate": 3.282361308677098e-07,
-      "logps/chosen": -240.2421417236328,
-      "logps/rejected": -346.31414794921875,
-      "loss": 0.1069,
-      "rewards/chosen": 2.2422432899475098,
-      "rewards/margins": 11.99687671661377,
-      "rewards/rejected": -9.754633903503418,
+      "logps/chosen": -293.82623291015625,
+      "logps/rejected": -439.79083251953125,
+      "loss": 0.1757,
+      "rewards/chosen": 1.2234735488891602,
+      "rewards/margins": 10.45723819732666,
+      "rewards/rejected": -9.233762741088867,
       "step": 640
     },
     {
       "epoch": 0.41,
-      "grad_norm": 21.066267013549805,
+      "grad_norm": 23.11900520324707,
       "kl": 0.0,
       "learning_rate": 3.2645803698435276e-07,
-      "logps/chosen": -264.81964111328125,
-      "logps/rejected": -338.95379638671875,
-      "loss": 0.1126,
-      "rewards/chosen": 2.3289685249328613,
-      "rewards/margins": 11.437704086303711,
-      "rewards/rejected": -9.108736038208008,
+      "logps/chosen": -284.302490234375,
+      "logps/rejected": -444.50164794921875,
+      "loss": 0.1794,
+      "rewards/chosen": 1.5410137176513672,
+      "rewards/margins": 10.157942771911621,
+      "rewards/rejected": -8.616928100585938,
       "step": 645
     },
     {
       "epoch": 0.42,
-      "grad_norm": 14.892672538757324,
+      "grad_norm": 25.599075317382812,
       "kl": 0.0,
       "learning_rate": 3.2467994310099573e-07,
-      "logps/chosen": -240.17236328125,
-      "logps/rejected": -355.543212890625,
-      "loss": 0.0884,
-      "rewards/chosen": 2.1047472953796387,
-      "rewards/margins": 12.319875717163086,
-      "rewards/rejected": -10.215127944946289,
+      "logps/chosen": -297.5779724121094,
+      "logps/rejected": -416.24603271484375,
+      "loss": 0.188,
+      "rewards/chosen": 1.241750955581665,
+      "rewards/margins": 10.071483612060547,
+      "rewards/rejected": -8.829732894897461,
       "step": 650
     },
     {
       "epoch": 0.42,
-      "grad_norm": 22.722187042236328,
+      "grad_norm": 32.84114074707031,
       "kl": 0.0,
       "learning_rate": 3.229018492176387e-07,
-      "logps/chosen": -235.84341430664062,
-      "logps/rejected": -363.74041748046875,
-      "loss": 0.0958,
-      "rewards/chosen": 2.262721300125122,
-      "rewards/margins": 12.947868347167969,
-      "rewards/rejected": -10.685147285461426,
+      "logps/chosen": -295.51666259765625,
+      "logps/rejected": -381.6005859375,
+      "loss": 0.1772,
+      "rewards/chosen": 1.4009755849838257,
+      "rewards/margins": 10.052778244018555,
+      "rewards/rejected": -8.651803016662598,
       "step": 655
     },
     {
       "epoch": 0.42,
-      "grad_norm": 16.061914443969727,
+      "grad_norm": 25.808273315429688,
       "kl": 0.0,
       "learning_rate": 3.211237553342817e-07,
-      "logps/chosen": -232.480224609375,
-      "logps/rejected": -363.93792724609375,
-      "loss": 0.0947,
-      "rewards/chosen": 2.245720863342285,
-      "rewards/margins": 13.019895553588867,
-      "rewards/rejected": -10.774174690246582,
+      "logps/chosen": -252.7512969970703,
+      "logps/rejected": -456.89727783203125,
+      "loss": 0.1781,
+      "rewards/chosen": 1.603581190109253,
+      "rewards/margins": 11.127111434936523,
+      "rewards/rejected": -9.523530960083008,
       "step": 660
     },
     {
       "epoch": 0.43,
-      "grad_norm": 17.95708656311035,
+      "grad_norm": 22.859973907470703,
       "kl": 0.0,
       "learning_rate": 3.193456614509246e-07,
-      "logps/chosen": -266.2756652832031,
-      "logps/rejected": -349.5681457519531,
-      "loss": 0.0981,
-      "rewards/chosen": 2.0552451610565186,
-      "rewards/margins": 12.47476863861084,
-      "rewards/rejected": -10.419523239135742,
+      "logps/chosen": -257.5951232910156,
+      "logps/rejected": -429.0784606933594,
+      "loss": 0.1679,
+      "rewards/chosen": 1.4137523174285889,
+      "rewards/margins": 10.75071907043457,
+      "rewards/rejected": -9.336965560913086,
       "step": 665
     },
     {
       "epoch": 0.43,
-      "grad_norm": 22.32909393310547,
+      "grad_norm": 20.03971290588379,
       "kl": 0.0,
       "learning_rate": 3.175675675675675e-07,
-      "logps/chosen": -262.5782165527344,
-      "logps/rejected": -350.5921936035156,
-      "loss": 0.1155,
-      "rewards/chosen": 2.324476718902588,
-      "rewards/margins": 12.327436447143555,
-      "rewards/rejected": -10.002958297729492,
+      "logps/chosen": -275.7456359863281,
+      "logps/rejected": -432.35675048828125,
+      "loss": 0.166,
+      "rewards/chosen": 1.5532034635543823,
+      "rewards/margins": 10.516412734985352,
+      "rewards/rejected": -8.963208198547363,
       "step": 670
     },
     {
       "epoch": 0.43,
-      "grad_norm": 14.884044647216797,
+      "grad_norm": 25.613548278808594,
       "kl": 0.0,
       "learning_rate": 3.157894736842105e-07,
-      "logps/chosen": -257.66094970703125,
-      "logps/rejected": -344.09869384765625,
-      "loss": 0.096,
-      "rewards/chosen": 2.2759206295013428,
-      "rewards/margins": 13.264117240905762,
-      "rewards/rejected": -10.988197326660156,
+      "logps/chosen": -259.3094482421875,
+      "logps/rejected": -428.52545166015625,
+      "loss": 0.1715,
+      "rewards/chosen": 1.5239208936691284,
+      "rewards/margins": 10.248991012573242,
+      "rewards/rejected": -8.72507095336914,
       "step": 675
     },
     {
       "epoch": 0.44,
-      "grad_norm": 18.041053771972656,
+      "grad_norm": 24.860313415527344,
       "kl": 0.0,
       "learning_rate": 3.1401137980085347e-07,
-      "logps/chosen": -270.93646240234375,
-      "logps/rejected": -360.5249938964844,
-      "loss": 0.0958,
-      "rewards/chosen": 2.366556167602539,
-      "rewards/margins": 13.1105318069458,
-      "rewards/rejected": -10.743974685668945,
+      "logps/chosen": -253.43408203125,
+      "logps/rejected": -421.2649841308594,
+      "loss": 0.1661,
+      "rewards/chosen": 1.7669661045074463,
+      "rewards/margins": 10.6715726852417,
+      "rewards/rejected": -8.904605865478516,
       "step": 680
     },
     {
       "epoch": 0.44,
-      "grad_norm": 17.0706729888916,
+      "grad_norm": 20.709943771362305,
       "kl": 0.0,
       "learning_rate": 3.1223328591749644e-07,
-      "logps/chosen": -214.73690795898438,
-      "logps/rejected": -342.7330017089844,
-      "loss": 0.1107,
-      "rewards/chosen": 2.152832508087158,
-      "rewards/margins": 12.126806259155273,
-      "rewards/rejected": -9.973973274230957,
+      "logps/chosen": -309.0604248046875,
+      "logps/rejected": -446.6376953125,
+      "loss": 0.1677,
+      "rewards/chosen": 1.6336666345596313,
+      "rewards/margins": 10.273514747619629,
+      "rewards/rejected": -8.639848709106445,
       "step": 685
     },
     {
       "epoch": 0.44,
-      "grad_norm": 25.26626968383789,
+      "grad_norm": 24.48786163330078,
       "kl": 0.0,
       "learning_rate": 3.104551920341394e-07,
-      "logps/chosen": -293.3183898925781,
-      "logps/rejected": -356.9164123535156,
-      "loss": 0.0973,
-      "rewards/chosen": 2.3354926109313965,
-      "rewards/margins": 13.199078559875488,
-      "rewards/rejected": -10.863585472106934,
+      "logps/chosen": -258.40936279296875,
+      "logps/rejected": -425.4024963378906,
+      "loss": 0.1901,
+      "rewards/chosen": 1.3902490139007568,
+      "rewards/margins": 10.214468002319336,
+      "rewards/rejected": -8.824217796325684,
       "step": 690
     },
     {
       "epoch": 0.44,
-      "grad_norm": 28.42453384399414,
+      "grad_norm": 23.349231719970703,
       "kl": 0.0,
       "learning_rate": 3.0867709815078234e-07,
-      "logps/chosen": -285.2272644042969,
-      "logps/rejected": -367.3414001464844,
-      "loss": 0.1091,
-      "rewards/chosen": 2.12477707862854,
-      "rewards/margins": 13.724832534790039,
-      "rewards/rejected": -11.600054740905762,
+      "logps/chosen": -283.63848876953125,
+      "logps/rejected": -453.16619873046875,
+      "loss": 0.1636,
+      "rewards/chosen": 1.5939074754714966,
+      "rewards/margins": 10.390283584594727,
+      "rewards/rejected": -8.79637622833252,
       "step": 695
     },
     {
       "epoch": 0.45,
-      "grad_norm": 17.899433135986328,
+      "grad_norm": 19.494522094726562,
       "kl": 0.0,
       "learning_rate": 3.068990042674253e-07,
-      "logps/chosen": -262.5505065917969,
-      "logps/rejected": -371.45025634765625,
-      "loss": 0.1083,
-      "rewards/chosen": 2.060147762298584,
-      "rewards/margins": 12.760570526123047,
-      "rewards/rejected": -10.700422286987305,
+      "logps/chosen": -273.1427917480469,
+      "logps/rejected": -421.3829650878906,
+      "loss": 0.1513,
+      "rewards/chosen": 1.827331304550171,
+      "rewards/margins": 10.396958351135254,
+      "rewards/rejected": -8.569626808166504,
       "step": 700
     },
     {
       "epoch": 0.45,
-      "grad_norm": 19.646583557128906,
+      "grad_norm": 23.5523624420166,
       "kl": 0.0,
       "learning_rate": 3.051209103840683e-07,
-      "logps/chosen": -274.9856262207031,
-      "logps/rejected": -360.3489685058594,
-      "loss": 0.1105,
-      "rewards/chosen": 2.4847919940948486,
-      "rewards/margins": 13.168169975280762,
-      "rewards/rejected": -10.683378219604492,
+      "logps/chosen": -246.75692749023438,
+      "logps/rejected": -434.19219970703125,
+      "loss": 0.1616,
+      "rewards/chosen": 1.8926893472671509,
+      "rewards/margins": 10.59654712677002,
+      "rewards/rejected": -8.703857421875,
       "step": 705
     },
     {
       "epoch": 0.45,
-      "grad_norm": 14.362700462341309,
+      "grad_norm": 25.01667022705078,
       "kl": 0.0,
       "learning_rate": 3.033428165007112e-07,
-      "logps/chosen": -253.5523681640625,
-      "logps/rejected": -336.803466796875,
-      "loss": 0.1024,
-      "rewards/chosen": 2.3810598850250244,
-      "rewards/margins": 12.508206367492676,
-      "rewards/rejected": -10.127145767211914,
+      "logps/chosen": -266.95745849609375,
+      "logps/rejected": -419.81640625,
+      "loss": 0.1713,
+      "rewards/chosen": 1.3775568008422852,
+      "rewards/margins": 9.493722915649414,
+      "rewards/rejected": -8.116167068481445,
       "step": 710
     },
     {
       "epoch": 0.46,
-      "grad_norm": 22.93763542175293,
+      "grad_norm": 23.61768341064453,
       "kl": 0.0,
       "learning_rate": 3.015647226173542e-07,
-      "logps/chosen": -229.1265869140625,
-      "logps/rejected": -354.0566101074219,
-      "loss": 0.1135,
-      "rewards/chosen": 2.1574866771698,
-      "rewards/margins": 12.193730354309082,
-      "rewards/rejected": -10.036243438720703,
+      "logps/chosen": -261.8187561035156,
+      "logps/rejected": -449.9925842285156,
+      "loss": 0.15,
+      "rewards/chosen": 1.6472257375717163,
+      "rewards/margins": 10.508228302001953,
+      "rewards/rejected": -8.861001968383789,
       "step": 715
     },
     {
       "epoch": 0.46,
-      "grad_norm": 22.314695358276367,
+      "grad_norm": 22.56501007080078,
       "kl": 0.0,
       "learning_rate": 2.9978662873399715e-07,
-      "logps/chosen": -274.2774353027344,
-      "logps/rejected": -405.2040100097656,
-      "loss": 0.0953,
-      "rewards/chosen": 2.191157817840576,
-      "rewards/margins": 13.628583908081055,
-      "rewards/rejected": -11.43742561340332,
+      "logps/chosen": -332.85504150390625,
+      "logps/rejected": -378.87347412109375,
+      "loss": 0.1669,
+      "rewards/chosen": 1.8122440576553345,
+      "rewards/margins": 10.017203330993652,
+      "rewards/rejected": -8.20495891571045,
       "step": 720
     },
     {
       "epoch": 0.46,
-      "grad_norm": 20.03668975830078,
+      "grad_norm": 20.667505264282227,
       "kl": 0.0,
       "learning_rate": 2.9800853485064007e-07,
-      "logps/chosen": -250.0018768310547,
-      "logps/rejected": -340.919677734375,
-      "loss": 0.1022,
-      "rewards/chosen": 2.4375367164611816,
-      "rewards/margins": 12.8548583984375,
-      "rewards/rejected": -10.417322158813477,
+      "logps/chosen": -276.8742980957031,
+      "logps/rejected": -414.7447204589844,
+      "loss": 0.1585,
+      "rewards/chosen": 1.9667861461639404,
+      "rewards/margins": 10.267745971679688,
+      "rewards/rejected": -8.300959587097168,
       "step": 725
     },
     {
       "epoch": 0.47,
-      "grad_norm": 20.21332359313965,
+      "grad_norm": 24.693187713623047,
       "kl": 0.0,
       "learning_rate": 2.9623044096728305e-07,
-      "logps/chosen": -262.5707092285156,
-      "logps/rejected": -341.51556396484375,
-      "loss": 0.113,
-      "rewards/chosen": 2.2380969524383545,
-      "rewards/margins": 12.677546501159668,
-      "rewards/rejected": -10.439449310302734,
+      "logps/chosen": -274.4808044433594,
+      "logps/rejected": -429.78558349609375,
+      "loss": 0.1546,
+      "rewards/chosen": 1.594257116317749,
+      "rewards/margins": 9.889884948730469,
+      "rewards/rejected": -8.29562759399414,
       "step": 730
     },
     {
       "epoch": 0.47,
-      "grad_norm": 18.686241149902344,
+      "grad_norm": 30.14167022705078,
       "kl": 0.0,
       "learning_rate": 2.94452347083926e-07,
-      "logps/chosen": -259.5292053222656,
-      "logps/rejected": -340.3607482910156,
-      "loss": 0.1188,
-      "rewards/chosen": 2.1912286281585693,
-      "rewards/margins": 12.15953540802002,
-      "rewards/rejected": -9.968307495117188,
+      "logps/chosen": -272.5168762207031,
+      "logps/rejected": -415.374755859375,
+      "loss": 0.1694,
+      "rewards/chosen": 1.5294649600982666,
+      "rewards/margins": 10.039915084838867,
+      "rewards/rejected": -8.51045036315918,
       "step": 735
     },
     {
       "epoch": 0.47,
-      "grad_norm": 19.728899002075195,
+      "grad_norm": 22.99307632446289,
       "kl": 0.0,
       "learning_rate": 2.92674253200569e-07,
-      "logps/chosen": -249.8451385498047,
-      "logps/rejected": -381.3360290527344,
-      "loss": 0.0967,
-      "rewards/chosen": 2.2508890628814697,
-      "rewards/margins": 13.840009689331055,
-      "rewards/rejected": -11.589120864868164,
+      "logps/chosen": -246.6560821533203,
+      "logps/rejected": -418.26104736328125,
+      "loss": 0.1584,
+      "rewards/chosen": 1.566481113433838,
+      "rewards/margins": 9.754480361938477,
+      "rewards/rejected": -8.18799877166748,
       "step": 740
     },
     {
       "epoch": 0.48,
-      "grad_norm": 17.338115692138672,
+      "grad_norm": 19.63002586364746,
       "kl": 0.0,
       "learning_rate": 2.9089615931721197e-07,
-      "logps/chosen": -268.9515380859375,
-      "logps/rejected": -368.5078125,
-      "loss": 0.1015,
-      "rewards/chosen": 2.28879976272583,
-      "rewards/margins": 13.396432876586914,
-      "rewards/rejected": -11.107633590698242,
+      "logps/chosen": -289.92266845703125,
+      "logps/rejected": -431.01336669921875,
+      "loss": 0.152,
+      "rewards/chosen": 1.7927067279815674,
+      "rewards/margins": 9.82994270324707,
+      "rewards/rejected": -8.037236213684082,
       "step": 745
     },
     {
       "epoch": 0.48,
-      "grad_norm": 19.87563133239746,
+      "grad_norm": 21.843643188476562,
       "kl": 0.0,
       "learning_rate": 2.8911806543385494e-07,
-      "logps/chosen": -293.7141418457031,
-      "logps/rejected": -355.55535888671875,
-      "loss": 0.1071,
-      "rewards/chosen": 2.4517719745635986,
-      "rewards/margins": 13.449313163757324,
-      "rewards/rejected": -10.997541427612305,
+      "logps/chosen": -272.4095764160156,
+      "logps/rejected": -384.18939208984375,
+      "loss": 0.1658,
+      "rewards/chosen": 1.6324083805084229,
+      "rewards/margins": 9.954187393188477,
+      "rewards/rejected": -8.321779251098633,
       "step": 750
     },
     {
       "epoch": 0.48,
-      "grad_norm": 19.807565689086914,
+      "grad_norm": 27.172988891601562,
       "kl": 0.0,
       "learning_rate": 2.873399715504978e-07,
-      "logps/chosen": -248.6322784423828,
-      "logps/rejected": -353.8228759765625,
-      "loss": 0.1088,
-      "rewards/chosen": 2.2985501289367676,
-      "rewards/margins": 12.722993850708008,
-      "rewards/rejected": -10.424444198608398,
+      "logps/chosen": -260.12127685546875,
+      "logps/rejected": -388.8544616699219,
+      "loss": 0.163,
+      "rewards/chosen": 1.7627140283584595,
+      "rewards/margins": 10.121767044067383,
+      "rewards/rejected": -8.359052658081055,
       "step": 755
     },
     {
       "epoch": 0.49,
-      "grad_norm": 14.098715782165527,
+      "grad_norm": 23.648014068603516,
       "kl": 0.0,
       "learning_rate": 2.855618776671408e-07,
-      "logps/chosen": -240.1453857421875,
-      "logps/rejected": -330.6163635253906,
-      "loss": 0.0891,
-      "rewards/chosen": 2.550762414932251,
-      "rewards/margins": 12.813929557800293,
-      "rewards/rejected": -10.263166427612305,
+      "logps/chosen": -262.92486572265625,
+      "logps/rejected": -410.4188537597656,
+      "loss": 0.1686,
+      "rewards/chosen": 1.493422508239746,
+      "rewards/margins": 9.909089088439941,
+      "rewards/rejected": -8.415667533874512,
       "step": 760
     },
     {
       "epoch": 0.49,
-      "grad_norm": 16.266538619995117,
+      "grad_norm": 23.27745819091797,
       "kl": 0.0,
       "learning_rate": 2.8378378378378376e-07,
-      "logps/chosen": -236.1558380126953,
-      "logps/rejected": -360.686767578125,
-      "loss": 0.0973,
-      "rewards/chosen": 2.2286760807037354,
-      "rewards/margins": 13.117490768432617,
-      "rewards/rejected": -10.888814926147461,
+      "logps/chosen": -288.63421630859375,
+      "logps/rejected": -406.29974365234375,
+      "loss": 0.1504,
+      "rewards/chosen": 1.8163105249404907,
+      "rewards/margins": 10.033960342407227,
+      "rewards/rejected": -8.217649459838867,
       "step": 765
     },
     {
       "epoch": 0.49,
-      "grad_norm": 16.414714813232422,
+      "grad_norm": 22.61689567565918,
       "kl": 0.0,
       "learning_rate": 2.8200568990042673e-07,
-      "logps/chosen": -283.36944580078125,
-      "logps/rejected": -368.7194519042969,
-      "loss": 0.0967,
-      "rewards/chosen": 2.4076812267303467,
-      "rewards/margins": 12.956690788269043,
-      "rewards/rejected": -10.549009323120117,
+      "logps/chosen": -246.67105102539062,
+      "logps/rejected": -425.2874450683594,
+      "loss": 0.1575,
+      "rewards/chosen": 1.4416098594665527,
+      "rewards/margins": 10.138015747070312,
+      "rewards/rejected": -8.696405410766602,
       "step": 770
     },
     {
       "epoch": 0.5,
-      "grad_norm": 18.088735580444336,
+      "grad_norm": 20.569795608520508,
       "kl": 0.0,
       "learning_rate": 2.802275960170697e-07,
-      "logps/chosen": -302.5070495605469,
-      "logps/rejected": -348.53289794921875,
-      "loss": 0.1047,
-      "rewards/chosen": 2.4432990550994873,
-      "rewards/margins": 12.935566902160645,
-      "rewards/rejected": -10.492268562316895,
+      "logps/chosen": -291.814208984375,
+      "logps/rejected": -399.8885192871094,
+      "loss": 0.1491,
+      "rewards/chosen": 1.771124243736267,
+      "rewards/margins": 9.939413070678711,
+      "rewards/rejected": -8.168289184570312,
       "step": 775
     },
     {
       "epoch": 0.5,
-      "grad_norm": 16.778833389282227,
+      "grad_norm": 27.513673782348633,
       "kl": 0.0,
       "learning_rate": 2.784495021337127e-07,
-      "logps/chosen": -243.6858367919922,
-      "logps/rejected": -366.0279235839844,
-      "loss": 0.0852,
-      "rewards/chosen": 2.5511088371276855,
-      "rewards/margins": 13.161443710327148,
-      "rewards/rejected": -10.610334396362305,
+      "logps/chosen": -264.59918212890625,
+      "logps/rejected": -426.15185546875,
+      "loss": 0.1835,
+      "rewards/chosen": 1.4606132507324219,
+      "rewards/margins": 9.935892105102539,
+      "rewards/rejected": -8.475278854370117,
       "step": 780
     },
     {
       "epoch": 0.5,
-      "grad_norm": 17.379335403442383,
+      "grad_norm": 25.520795822143555,
       "kl": 0.0,
       "learning_rate": 2.766714082503556e-07,
-      "logps/chosen": -279.8948669433594,
-      "logps/rejected": -352.0287170410156,
-      "loss": 0.0957,
-      "rewards/chosen": 2.464888334274292,
-      "rewards/margins": 13.103485107421875,
-      "rewards/rejected": -10.638595581054688,
+      "logps/chosen": -298.03466796875,
+      "logps/rejected": -420.57635498046875,
+      "loss": 0.1614,
+      "rewards/chosen": 1.6418640613555908,
+      "rewards/margins": 10.411275863647461,
+      "rewards/rejected": -8.769411087036133,
       "step": 785
     },
     {
       "epoch": 0.51,
-      "grad_norm": 21.737380981445312,
+      "grad_norm": 25.321683883666992,
       "kl": 0.0,
       "learning_rate": 2.7489331436699857e-07,
-      "logps/chosen": -258.93865966796875,
-      "logps/rejected": -345.5055236816406,
-      "loss": 0.1177,
-      "rewards/chosen": 2.079983711242676,
-      "rewards/margins": 12.90088176727295,
-      "rewards/rejected": -10.820898056030273,
+      "logps/chosen": -278.0859069824219,
+      "logps/rejected": -383.1441650390625,
+      "loss": 0.1725,
+      "rewards/chosen": 1.6416689157485962,
+      "rewards/margins": 9.834056854248047,
+      "rewards/rejected": -8.192388534545898,
       "step": 790
     },
     {
       "epoch": 0.51,
-      "grad_norm": 15.227828025817871,
+      "grad_norm": 18.801525115966797,
       "kl": 0.0,
       "learning_rate": 2.7311522048364154e-07,
-      "logps/chosen": -233.96194458007812,
-      "logps/rejected": -370.0096130371094,
-      "loss": 0.1054,
-      "rewards/chosen": 2.696063280105591,
-      "rewards/margins": 13.426877975463867,
-      "rewards/rejected": -10.730813980102539,
+      "logps/chosen": -293.8526306152344,
+      "logps/rejected": -464.01580810546875,
+      "loss": 0.1438,
+      "rewards/chosen": 1.7998111248016357,
+      "rewards/margins": 11.001166343688965,
+      "rewards/rejected": -9.20135498046875,
       "step": 795
     },
     {
       "epoch": 0.51,
-      "grad_norm": 13.938480377197266,
+      "grad_norm": 31.995546340942383,
       "kl": 0.0,
       "learning_rate": 2.7133712660028446e-07,
-      "logps/chosen": -243.730224609375,
-      "logps/rejected": -350.44183349609375,
-      "loss": 0.0976,
-      "rewards/chosen": 2.3282103538513184,
-      "rewards/margins": 13.082319259643555,
-      "rewards/rejected": -10.754108428955078,
+      "logps/chosen": -266.13580322265625,
+      "logps/rejected": -452.4052734375,
+      "loss": 0.1547,
+      "rewards/chosen": 1.85707688331604,
+      "rewards/margins": 10.742141723632812,
+      "rewards/rejected": -8.885065078735352,
       "step": 800
     },
     {
       "epoch": 0.52,
-      "grad_norm": 29.015178680419922,
+      "grad_norm": 28.300655364990234,
       "kl": 0.0,
       "learning_rate": 2.6955903271692744e-07,
-      "logps/chosen": -216.66488647460938,
-      "logps/rejected": -368.9279479980469,
-      "loss": 0.1045,
-      "rewards/chosen": 2.2219786643981934,
-      "rewards/margins": 13.255975723266602,
-      "rewards/rejected": -11.033994674682617,
+      "logps/chosen": -276.36151123046875,
+      "logps/rejected": -424.6527404785156,
+      "loss": 0.1654,
+      "rewards/chosen": 1.4731123447418213,
+      "rewards/margins": 10.520303726196289,
+      "rewards/rejected": -9.04719066619873,
       "step": 805
     },
     {
       "epoch": 0.52,
-      "grad_norm": 12.759780883789062,
+      "grad_norm": 21.696815490722656,
       "kl": 0.0,
       "learning_rate": 2.677809388335704e-07,
-      "logps/chosen": -232.0436248779297,
-      "logps/rejected": -358.2416076660156,
-      "loss": 0.0943,
-      "rewards/chosen": 2.235215425491333,
-      "rewards/margins": 12.901985168457031,
-      "rewards/rejected": -10.666769027709961,
+      "logps/chosen": -248.7739715576172,
+      "logps/rejected": -468.64617919921875,
+      "loss": 0.14,
+      "rewards/chosen": 1.3544524908065796,
+      "rewards/margins": 10.655045509338379,
+      "rewards/rejected": -9.300593376159668,
       "step": 810
     },
     {
       "epoch": 0.52,
-      "grad_norm": 19.0715389251709,
+      "grad_norm": 26.037267684936523,
       "kl": 0.0,
       "learning_rate": 2.6600284495021333e-07,
-      "logps/chosen": -245.0729217529297,
-      "logps/rejected": -379.9324645996094,
-      "loss": 0.1173,
-      "rewards/chosen": 2.423140287399292,
-      "rewards/margins": 13.03075885772705,
-      "rewards/rejected": -10.60761833190918,
+      "logps/chosen": -278.1371765136719,
+      "logps/rejected": -395.54315185546875,
+      "loss": 0.1708,
+      "rewards/chosen": 1.6430387496948242,
+      "rewards/margins": 9.990675926208496,
+      "rewards/rejected": -8.347638130187988,
       "step": 815
     },
     {
       "epoch": 0.52,
-      "grad_norm": 15.53042984008789,
+      "grad_norm": 27.351716995239258,
       "kl": 0.0,
       "learning_rate": 2.642247510668563e-07,
-      "logps/chosen": -261.9339294433594,
-      "logps/rejected": -327.780517578125,
-      "loss": 0.0903,
-      "rewards/chosen": 2.4564433097839355,
-      "rewards/margins": 13.401847839355469,
-      "rewards/rejected": -10.945404052734375,
+      "logps/chosen": -296.94207763671875,
+      "logps/rejected": -418.74951171875,
+      "loss": 0.159,
+      "rewards/chosen": 1.9864873886108398,
+      "rewards/margins": 11.01016902923584,
+      "rewards/rejected": -9.023681640625,
       "step": 820
     },
     {
       "epoch": 0.53,
-      "grad_norm": 26.67053985595703,
+      "grad_norm": 27.478801727294922,
       "kl": 0.0,
       "learning_rate": 2.624466571834993e-07,
-      "logps/chosen": -248.34487915039062,
-      "logps/rejected": -345.79083251953125,
-      "loss": 0.1071,
-      "rewards/chosen": 2.359476089477539,
-      "rewards/margins": 13.113537788391113,
-      "rewards/rejected": -10.754061698913574,
+      "logps/chosen": -288.36370849609375,
+      "logps/rejected": -431.9353942871094,
+      "loss": 0.1483,
+      "rewards/chosen": 2.1340115070343018,
+      "rewards/margins": 11.531509399414062,
+      "rewards/rejected": -9.397497177124023,
       "step": 825
     },
     {
       "epoch": 0.53,
-      "grad_norm": 22.24759292602539,
+      "grad_norm": 30.95049285888672,
       "kl": 0.0,
       "learning_rate": 2.6066856330014225e-07,
-      "logps/chosen": -262.4079895019531,
-      "logps/rejected": -364.2686462402344,
-      "loss": 0.1076,
-      "rewards/chosen": 2.37235689163208,
-      "rewards/margins": 13.127764701843262,
-      "rewards/rejected": -10.75540828704834,
+      "logps/chosen": -252.1579132080078,
+      "logps/rejected": -424.7654724121094,
+      "loss": 0.1578,
+      "rewards/chosen": 1.2540996074676514,
+      "rewards/margins": 10.03384780883789,
+      "rewards/rejected": -8.77974796295166,
       "step": 830
     },
     {
       "epoch": 0.53,
-      "grad_norm": 17.638994216918945,
+      "grad_norm": 25.74098777770996,
       "kl": 0.0,
       "learning_rate": 2.5889046941678523e-07,
-      "logps/chosen": -273.74432373046875,
-      "logps/rejected": -383.0593566894531,
-      "loss": 0.0984,
-      "rewards/chosen": 2.275177478790283,
-      "rewards/margins": 14.033803939819336,
-      "rewards/rejected": -11.758626937866211,
+      "logps/chosen": -260.67315673828125,
+      "logps/rejected": -398.57952880859375,
+      "loss": 0.1554,
+      "rewards/chosen": 1.9767462015151978,
+      "rewards/margins": 10.994711875915527,
+      "rewards/rejected": -9.017965316772461,
       "step": 835
     },
     {
       "epoch": 0.54,
-      "grad_norm": 15.510226249694824,
+      "grad_norm": 24.75625991821289,
       "kl": 0.0,
       "learning_rate": 2.5711237553342815e-07,
-      "logps/chosen": -268.3088073730469,
-      "logps/rejected": -372.16876220703125,
-      "loss": 0.075,
-      "rewards/chosen": 2.7448737621307373,
-      "rewards/margins": 14.005487442016602,
-      "rewards/rejected": -11.260614395141602,
+      "logps/chosen": -297.8096923828125,
+      "logps/rejected": -454.16265869140625,
+      "loss": 0.1601,
+      "rewards/chosen": 1.8933601379394531,
+      "rewards/margins": 10.956032752990723,
+      "rewards/rejected": -9.062673568725586,
       "step": 840
     },
     {
       "epoch": 0.54,
-      "grad_norm": 18.002506256103516,
+      "grad_norm": 19.37633514404297,
       "kl": 0.0,
       "learning_rate": 2.5533428165007107e-07,
-      "logps/chosen": -214.6236572265625,
-      "logps/rejected": -379.697265625,
-      "loss": 0.1156,
-      "rewards/chosen": 2.090531349182129,
-      "rewards/margins": 14.037800788879395,
-      "rewards/rejected": -11.947270393371582,
+      "logps/chosen": -246.20755004882812,
+      "logps/rejected": -417.218994140625,
+      "loss": 0.1625,
+      "rewards/chosen": 1.7149181365966797,
+      "rewards/margins": 10.08183479309082,
+      "rewards/rejected": -8.366915702819824,
       "step": 845
     },
     {
       "epoch": 0.54,
-      "grad_norm": 18.8648681640625,
+      "grad_norm": 22.729887008666992,
       "kl": 0.0,
       "learning_rate": 2.5355618776671404e-07,
-      "logps/chosen": -230.46810913085938,
-      "logps/rejected": -345.8827209472656,
-      "loss": 0.1028,
-      "rewards/chosen": 2.3623595237731934,
-      "rewards/margins": 13.34724235534668,
-      "rewards/rejected": -10.984883308410645,
+      "logps/chosen": -295.47833251953125,
+      "logps/rejected": -383.05230712890625,
+      "loss": 0.1556,
+      "rewards/chosen": 1.9083483219146729,
+      "rewards/margins": 10.142326354980469,
+      "rewards/rejected": -8.233978271484375,
       "step": 850
     },
     {
       "epoch": 0.55,
-      "grad_norm": 15.514137268066406,
+      "grad_norm": 20.790363311767578,
       "kl": 0.0,
       "learning_rate": 2.51778093883357e-07,
-      "logps/chosen": -224.1614227294922,
-      "logps/rejected": -365.08477783203125,
-      "loss": 0.0985,
-      "rewards/chosen": 2.2946102619171143,
-      "rewards/margins": 14.019018173217773,
-      "rewards/rejected": -11.724408149719238,
+      "logps/chosen": -252.0298309326172,
+      "logps/rejected": -439.210205078125,
+      "loss": 0.1419,
+      "rewards/chosen": 1.6935409307479858,
+      "rewards/margins": 10.923540115356445,
+      "rewards/rejected": -9.229999542236328,
       "step": 855
     },
     {
       "epoch": 0.55,
-      "grad_norm": 17.679288864135742,
+      "grad_norm": 20.501516342163086,
       "kl": 0.0,
       "learning_rate": 2.5e-07,
-      "logps/chosen": -231.95565795898438,
-      "logps/rejected": -349.056640625,
-      "loss": 0.0966,
-      "rewards/chosen": 2.3342556953430176,
-      "rewards/margins": 13.233372688293457,
-      "rewards/rejected": -10.899115562438965,
+      "logps/chosen": -305.12030029296875,
+      "logps/rejected": -444.84796142578125,
+      "loss": 0.1369,
+      "rewards/chosen": 1.9432222843170166,
+      "rewards/margins": 11.106147766113281,
+      "rewards/rejected": -9.162925720214844,
       "step": 860
     },
     {
       "epoch": 0.55,
-      "grad_norm": 19.04916000366211,
+      "grad_norm": 18.870508193969727,
       "kl": 0.0,
       "learning_rate": 2.4822190611664296e-07,
-      "logps/chosen": -233.5281982421875,
-      "logps/rejected": -374.4831848144531,
-      "loss": 0.0922,
-      "rewards/chosen": 2.58280873298645,
-      "rewards/margins": 13.661170959472656,
-      "rewards/rejected": -11.078360557556152,
+      "logps/chosen": -276.5811462402344,
+      "logps/rejected": -418.9603576660156,
+      "loss": 0.155,
+      "rewards/chosen": 1.7197465896606445,
+      "rewards/margins": 10.493929862976074,
+      "rewards/rejected": -8.774185180664062,
       "step": 865
     },
     {
       "epoch": 0.56,
-      "grad_norm": 16.834497451782227,
+      "grad_norm": 29.126863479614258,
       "kl": 0.0,
       "learning_rate": 2.4644381223328594e-07,
-      "logps/chosen": -236.75106811523438,
-      "logps/rejected": -353.3365478515625,
-      "loss": 0.096,
-      "rewards/chosen": 2.7401375770568848,
-      "rewards/margins": 13.776086807250977,
-      "rewards/rejected": -11.035948753356934,
+      "logps/chosen": -265.5671691894531,
+      "logps/rejected": -456.1929626464844,
+      "loss": 0.1591,
+      "rewards/chosen": 1.5650779008865356,
+      "rewards/margins": 10.798846244812012,
+      "rewards/rejected": -9.233770370483398,
       "step": 870
     },
     {
       "epoch": 0.56,
-      "grad_norm": 24.789772033691406,
+      "grad_norm": 24.848697662353516,
       "kl": 0.0,
       "learning_rate": 2.4466571834992886e-07,
-      "logps/chosen": -267.3653869628906,
-      "logps/rejected": -370.1556091308594,
-      "loss": 0.0949,
-      "rewards/chosen": 2.2669379711151123,
-      "rewards/margins": 13.626495361328125,
-      "rewards/rejected": -11.35955810546875,
+      "logps/chosen": -254.2496337890625,
+      "logps/rejected": -432.71038818359375,
+      "loss": 0.1563,
+      "rewards/chosen": 1.713888168334961,
+      "rewards/margins": 10.464970588684082,
+      "rewards/rejected": -8.751081466674805,
       "step": 875
     },
     {
       "epoch": 0.56,
-      "grad_norm": 12.911144256591797,
+      "grad_norm": 25.702816009521484,
       "kl": 0.0,
       "learning_rate": 2.4288762446657183e-07,
-      "logps/chosen": -250.23556518554688,
-      "logps/rejected": -363.0490417480469,
-      "loss": 0.0717,
-      "rewards/chosen": 2.6297707557678223,
-      "rewards/margins": 15.319549560546875,
-      "rewards/rejected": -12.689778327941895,
+      "logps/chosen": -275.37030029296875,
+      "logps/rejected": -427.5972595214844,
+      "loss": 0.1629,
+      "rewards/chosen": 1.6576436758041382,
+      "rewards/margins": 10.351664543151855,
+      "rewards/rejected": -8.69402027130127,
       "step": 880
     },
     {
       "epoch": 0.57,
-      "grad_norm": 18.159881591796875,
+      "grad_norm": 22.683273315429688,
       "kl": 0.0,
       "learning_rate": 2.411095305832148e-07,
-      "logps/chosen": -276.918701171875,
-      "logps/rejected": -351.54132080078125,
-      "loss": 0.1062,
-      "rewards/chosen": 2.5637714862823486,
-      "rewards/margins": 13.2720365524292,
-      "rewards/rejected": -10.708267211914062,
+      "logps/chosen": -282.93426513671875,
+      "logps/rejected": -441.66064453125,
+      "loss": 0.1249,
+      "rewards/chosen": 1.9189395904541016,
+      "rewards/margins": 11.167387008666992,
+      "rewards/rejected": -9.248448371887207,
       "step": 885
     },
     {
       "epoch": 0.57,
-      "grad_norm": 16.24278450012207,
+      "grad_norm": 24.9903564453125,
       "kl": 0.0,
       "learning_rate": 2.393314366998578e-07,
-      "logps/chosen": -264.356689453125,
-      "logps/rejected": -345.72064208984375,
-      "loss": 0.1061,
-      "rewards/chosen": 2.57415509223938,
-      "rewards/margins": 13.096124649047852,
-      "rewards/rejected": -10.521968841552734,
+      "logps/chosen": -294.500732421875,
+      "logps/rejected": -424.95391845703125,
+      "loss": 0.1396,
+      "rewards/chosen": 1.8850313425064087,
+      "rewards/margins": 10.923759460449219,
+      "rewards/rejected": -9.038727760314941,
       "step": 890
     },
     {
       "epoch": 0.57,
-      "grad_norm": 15.353395462036133,
+      "grad_norm": 22.959312438964844,
       "kl": 0.0,
       "learning_rate": 2.375533428165007e-07,
-      "logps/chosen": -245.558349609375,
-      "logps/rejected": -346.11151123046875,
-      "loss": 0.1098,
-      "rewards/chosen": 2.0979220867156982,
-      "rewards/margins": 13.061151504516602,
-      "rewards/rejected": -10.963228225708008,
+      "logps/chosen": -276.75775146484375,
+      "logps/rejected": -445.98052978515625,
+      "loss": 0.1612,
+      "rewards/chosen": 1.7431560754776,
+      "rewards/margins": 10.924168586730957,
+      "rewards/rejected": -9.181012153625488,
       "step": 895
     },
     {
       "epoch": 0.58,
-      "grad_norm": 15.291166305541992,
+      "grad_norm": 21.280902862548828,
       "kl": 0.0,
       "learning_rate": 2.3577524893314365e-07,
-      "logps/chosen": -268.0149841308594,
-      "logps/rejected": -359.99676513671875,
-      "loss": 0.0935,
-      "rewards/chosen": 2.660017251968384,
-      "rewards/margins": 13.785112380981445,
-      "rewards/rejected": -11.125094413757324,
+      "logps/chosen": -266.2899475097656,
+      "logps/rejected": -419.53594970703125,
+      "loss": 0.1366,
+      "rewards/chosen": 2.116002321243286,
+      "rewards/margins": 10.943870544433594,
+      "rewards/rejected": -8.82786750793457,
       "step": 900
     },
     {
       "epoch": 0.58,
-      "grad_norm": 27.338308334350586,
+      "grad_norm": 23.134008407592773,
       "kl": 0.0,
       "learning_rate": 2.3399715504978662e-07,
-      "logps/chosen": -223.98062133789062,
-      "logps/rejected": -382.53924560546875,
-      "loss": 0.0733,
-      "rewards/chosen": 2.573464870452881,
-      "rewards/margins": 14.070528030395508,
-      "rewards/rejected": -11.497062683105469,
+      "logps/chosen": -295.0397033691406,
+      "logps/rejected": -433.302978515625,
+      "loss": 0.1485,
+      "rewards/chosen": 1.6045787334442139,
+      "rewards/margins": 10.714460372924805,
+      "rewards/rejected": -9.109882354736328,
       "step": 905
     },
     {
       "epoch": 0.58,
-      "grad_norm": 28.126638412475586,
+      "grad_norm": 21.066099166870117,
       "kl": 0.0,
       "learning_rate": 2.322190611664296e-07,
-      "logps/chosen": -225.90432739257812,
-      "logps/rejected": -369.765380859375,
-      "loss": 0.1068,
-      "rewards/chosen": 2.4907584190368652,
-      "rewards/margins": 13.410183906555176,
-      "rewards/rejected": -10.919425964355469,
+      "logps/chosen": -242.4779815673828,
+      "logps/rejected": -413.7901306152344,
+      "loss": 0.1463,
+      "rewards/chosen": 1.6984901428222656,
+      "rewards/margins": 10.702180862426758,
+      "rewards/rejected": -9.003690719604492,
       "step": 910
     },
     {
       "epoch": 0.59,
-      "grad_norm": 23.488555908203125,
+      "grad_norm": 23.61625099182129,
       "kl": 0.0,
       "learning_rate": 2.304409672830725e-07,
-      "logps/chosen": -264.053466796875,
-      "logps/rejected": -347.4461364746094,
-      "loss": 0.0978,
-      "rewards/chosen": 2.3625271320343018,
-      "rewards/margins": 13.296697616577148,
-      "rewards/rejected": -10.934170722961426,
+      "logps/chosen": -270.14599609375,
+      "logps/rejected": -437.8699645996094,
+      "loss": 0.1589,
+      "rewards/chosen": 1.7722479104995728,
+      "rewards/margins": 10.022378921508789,
+      "rewards/rejected": -8.250131607055664,
       "step": 915
     },
     {
       "epoch": 0.59,
-      "grad_norm": 16.538496017456055,
+      "grad_norm": 22.70294952392578,
       "kl": 0.0,
       "learning_rate": 2.2866287339971549e-07,
-      "logps/chosen": -305.2149353027344,
-      "logps/rejected": -387.3183898925781,
-      "loss": 0.0895,
-      "rewards/chosen": 2.51884126663208,
-      "rewards/margins": 14.519546508789062,
-      "rewards/rejected": -12.000704765319824,
+      "logps/chosen": -285.99383544921875,
+      "logps/rejected": -425.30511474609375,
+      "loss": 0.1263,
+      "rewards/chosen": 2.163151502609253,
+      "rewards/margins": 10.922938346862793,
+      "rewards/rejected": -8.759786605834961,
       "step": 920
     },
     {
       "epoch": 0.59,
-      "grad_norm": 14.570239067077637,
+      "grad_norm": 20.896127700805664,
       "kl": 0.0,
       "learning_rate": 2.2688477951635846e-07,
-      "logps/chosen": -288.55859375,
-      "logps/rejected": -360.0328063964844,
-      "loss": 0.0891,
-      "rewards/chosen": 2.6060962677001953,
-      "rewards/margins": 13.63697338104248,
-      "rewards/rejected": -11.030878067016602,
+      "logps/chosen": -257.12646484375,
+      "logps/rejected": -474.15789794921875,
+      "loss": 0.1402,
+      "rewards/chosen": 1.7868337631225586,
+      "rewards/margins": 10.79575252532959,
+      "rewards/rejected": -9.008918762207031,
       "step": 925
     },
     {
       "epoch": 0.6,
-      "grad_norm": 14.252922058105469,
+      "grad_norm": 20.20517921447754,
       "kl": 0.0,
       "learning_rate": 2.251066856330014e-07,
-      "logps/chosen": -237.12985229492188,
-      "logps/rejected": -375.7847900390625,
-      "loss": 0.0968,
-      "rewards/chosen": 2.277179718017578,
-      "rewards/margins": 13.618906021118164,
-      "rewards/rejected": -11.341727256774902,
+      "logps/chosen": -281.36016845703125,
+      "logps/rejected": -409.21807861328125,
+      "loss": 0.1405,
+      "rewards/chosen": 1.9572327136993408,
+      "rewards/margins": 10.755359649658203,
+      "rewards/rejected": -8.798127174377441,
       "step": 930
     },
     {
       "epoch": 0.6,
-      "grad_norm": 14.224340438842773,
+      "grad_norm": 24.792766571044922,
       "kl": 0.0,
       "learning_rate": 2.2332859174964438e-07,
-      "logps/chosen": -247.994384765625,
-      "logps/rejected": -384.1309509277344,
-      "loss": 0.0921,
-      "rewards/chosen": 2.5065815448760986,
-      "rewards/margins": 14.67822551727295,
-      "rewards/rejected": -12.17164421081543,
+      "logps/chosen": -274.1449890136719,
+      "logps/rejected": -440.4224548339844,
+      "loss": 0.1612,
+      "rewards/chosen": 1.4629716873168945,
+      "rewards/margins": 10.279823303222656,
+      "rewards/rejected": -8.816852569580078,
       "step": 935
     },
     {
       "epoch": 0.6,
-      "grad_norm": 15.533319473266602,
+      "grad_norm": 25.996437072753906,
       "kl": 0.0,
       "learning_rate": 2.2155049786628733e-07,
-      "logps/chosen": -217.72982788085938,
-      "logps/rejected": -364.0068054199219,
-      "loss": 0.1044,
-      "rewards/chosen": 2.4342358112335205,
-      "rewards/margins": 13.825854301452637,
-      "rewards/rejected": -11.391618728637695,
+      "logps/chosen": -258.3365783691406,
+      "logps/rejected": -455.66571044921875,
+      "loss": 0.1593,
+      "rewards/chosen": 1.8357813358306885,
+      "rewards/margins": 11.063421249389648,
+      "rewards/rejected": -9.227640151977539,
       "step": 940
     },
     {
       "epoch": 0.6,
-      "grad_norm": 12.951379776000977,
+      "grad_norm": 21.876049041748047,
       "kl": 0.0,
       "learning_rate": 2.1977240398293027e-07,
-      "logps/chosen": -216.9298858642578,
-      "logps/rejected": -391.88916015625,
-      "loss": 0.0959,
-      "rewards/chosen": 2.2248167991638184,
-      "rewards/margins": 14.502099990844727,
-      "rewards/rejected": -12.27728271484375,
+      "logps/chosen": -254.3147430419922,
+      "logps/rejected": -438.74005126953125,
+      "loss": 0.1293,
+      "rewards/chosen": 1.6060841083526611,
+      "rewards/margins": 10.991594314575195,
+      "rewards/rejected": -9.38550853729248,
       "step": 945
     },
     {
       "epoch": 0.61,
-      "grad_norm": 12.726017951965332,
+      "grad_norm": 27.55910301208496,
       "kl": 0.0,
       "learning_rate": 2.1799431009957325e-07,
-      "logps/chosen": -250.4014129638672,
-      "logps/rejected": -367.2666931152344,
-      "loss": 0.099,
-      "rewards/chosen": 2.6326889991760254,
-      "rewards/margins": 14.203544616699219,
-      "rewards/rejected": -11.570856094360352,
+      "logps/chosen": -275.89837646484375,
+      "logps/rejected": -442.4541931152344,
+      "loss": 0.1545,
+      "rewards/chosen": 1.7877737283706665,
+      "rewards/margins": 11.597868919372559,
+      "rewards/rejected": -9.81009578704834,
       "step": 950
     },
     {
       "epoch": 0.61,
-      "grad_norm": 24.116321563720703,
+      "grad_norm": 19.6149959564209,
       "kl": 0.0,
       "learning_rate": 2.1621621621621622e-07,
-      "logps/chosen": -256.0219421386719,
-      "logps/rejected": -359.89410400390625,
-      "loss": 0.1114,
-      "rewards/chosen": 2.291504383087158,
-      "rewards/margins": 13.204824447631836,
-      "rewards/rejected": -10.913320541381836,
+      "logps/chosen": -327.98602294921875,
+      "logps/rejected": -448.33856201171875,
+      "loss": 0.0995,
+      "rewards/chosen": 2.8413889408111572,
+      "rewards/margins": 11.999773025512695,
+      "rewards/rejected": -9.158384323120117,
       "step": 955
     },
     {
       "epoch": 0.61,
-      "grad_norm": 21.19695472717285,
+      "grad_norm": 20.440847396850586,
       "kl": 0.0,
       "learning_rate": 2.1443812233285914e-07,
-      "logps/chosen": -253.9010009765625,
-      "logps/rejected": -348.592041015625,
-      "loss": 0.0887,
-      "rewards/chosen": 2.41105580329895,
-      "rewards/margins": 13.58563232421875,
-      "rewards/rejected": -11.174577713012695,
+      "logps/chosen": -273.84930419921875,
+      "logps/rejected": -407.75018310546875,
+      "loss": 0.1382,
+      "rewards/chosen": 2.0720973014831543,
+      "rewards/margins": 10.910283088684082,
+      "rewards/rejected": -8.83818531036377,
       "step": 960
     },
     {
       "epoch": 0.62,
-      "grad_norm": 19.317626953125,
+      "grad_norm": 24.151046752929688,
       "kl": 0.0,
       "learning_rate": 2.1266002844950212e-07,
-      "logps/chosen": -245.3134765625,
-      "logps/rejected": -380.17327880859375,
-      "loss": 0.0998,
-      "rewards/chosen": 2.4610061645507812,
-      "rewards/margins": 13.953335762023926,
-      "rewards/rejected": -11.492330551147461,
+      "logps/chosen": -284.7245788574219,
+      "logps/rejected": -433.08197021484375,
+      "loss": 0.1285,
+      "rewards/chosen": 2.2533042430877686,
+      "rewards/margins": 11.679128646850586,
+      "rewards/rejected": -9.425825119018555,
       "step": 965
     },
     {
       "epoch": 0.62,
-      "grad_norm": 22.688295364379883,
+      "grad_norm": 24.470195770263672,
       "kl": 0.0,
       "learning_rate": 2.108819345661451e-07,
-      "logps/chosen": -278.2052917480469,
-      "logps/rejected": -357.1881408691406,
-      "loss": 0.1063,
-      "rewards/chosen": 2.404897928237915,
-      "rewards/margins": 13.51880931854248,
-      "rewards/rejected": -11.113912582397461,
+      "logps/chosen": -320.61468505859375,
+      "logps/rejected": -442.22869873046875,
+      "loss": 0.1429,
+      "rewards/chosen": 1.9059406518936157,
+      "rewards/margins": 11.600639343261719,
+      "rewards/rejected": -9.69469928741455,
       "step": 970
     },
     {
       "epoch": 0.62,
-      "grad_norm": 14.956331253051758,
+      "grad_norm": 26.81120491027832,
       "kl": 0.0,
       "learning_rate": 2.0910384068278806e-07,
-      "logps/chosen": -274.7865295410156,
-      "logps/rejected": -348.69024658203125,
-      "loss": 0.1178,
-      "rewards/chosen": 2.560181140899658,
-      "rewards/margins": 13.09942626953125,
-      "rewards/rejected": -10.539244651794434,
+      "logps/chosen": -257.6952209472656,
+      "logps/rejected": -392.0545959472656,
+      "loss": 0.1483,
+      "rewards/chosen": 1.6184053421020508,
+      "rewards/margins": 10.500303268432617,
+      "rewards/rejected": -8.881898880004883,
       "step": 975
     },
     {
       "epoch": 0.63,
-      "grad_norm": 14.130335807800293,
+      "grad_norm": 19.685026168823242,
       "kl": 0.0,
       "learning_rate": 2.0732574679943098e-07,
-      "logps/chosen": -241.42257690429688,
-      "logps/rejected": -374.1187438964844,
-      "loss": 0.0728,
-      "rewards/chosen": 2.970536470413208,
-      "rewards/margins": 14.709304809570312,
-      "rewards/rejected": -11.738768577575684,
+      "logps/chosen": -305.773681640625,
+      "logps/rejected": -393.2521057128906,
+      "loss": 0.1456,
+      "rewards/chosen": 2.0023820400238037,
+      "rewards/margins": 11.047486305236816,
+      "rewards/rejected": -9.045104026794434,
       "step": 980
     },
     {
       "epoch": 0.63,
-      "grad_norm": 20.794334411621094,
+      "grad_norm": 24.304468154907227,
       "kl": 0.0,
       "learning_rate": 2.0554765291607396e-07,
-      "logps/chosen": -230.2430419921875,
-      "logps/rejected": -380.54608154296875,
-      "loss": 0.09,
-      "rewards/chosen": 2.673881769180298,
-      "rewards/margins": 14.4403076171875,
-      "rewards/rejected": -11.766425132751465,
+      "logps/chosen": -266.98773193359375,
+      "logps/rejected": -409.9839782714844,
+      "loss": 0.152,
+      "rewards/chosen": 1.678897500038147,
+      "rewards/margins": 10.779475212097168,
+      "rewards/rejected": -9.100576400756836,
       "step": 985
     },
     {
       "epoch": 0.63,
-      "grad_norm": 16.297340393066406,
+      "grad_norm": 23.530902862548828,
       "kl": 0.0,
       "learning_rate": 2.0376955903271693e-07,
-      "logps/chosen": -264.10137939453125,
-      "logps/rejected": -372.11346435546875,
-      "loss": 0.0896,
-      "rewards/chosen": 2.7595813274383545,
-      "rewards/margins": 13.790555953979492,
-      "rewards/rejected": -11.030974388122559,
+      "logps/chosen": -268.0154724121094,
+      "logps/rejected": -446.49041748046875,
+      "loss": 0.1365,
+      "rewards/chosen": 1.426351547241211,
+      "rewards/margins": 10.987870216369629,
+      "rewards/rejected": -9.561517715454102,
       "step": 990
     },
     {
       "epoch": 0.64,
-      "grad_norm": 20.304162979125977,
+      "grad_norm": 25.060619354248047,
       "kl": 0.0,
       "learning_rate": 2.0199146514935988e-07,
-      "logps/chosen": -243.2149200439453,
-      "logps/rejected": -342.080322265625,
-      "loss": 0.1091,
-      "rewards/chosen": 2.3006443977355957,
-      "rewards/margins": 13.069559097290039,
-      "rewards/rejected": -10.768914222717285,
+      "logps/chosen": -304.2224426269531,
+      "logps/rejected": -436.77734375,
+      "loss": 0.1559,
+      "rewards/chosen": 1.8927513360977173,
+      "rewards/margins": 10.877464294433594,
+      "rewards/rejected": -8.984713554382324,
       "step": 995
     },
     {
       "epoch": 0.64,
-      "grad_norm": 19.80646514892578,
+      "grad_norm": 26.11311149597168,
       "kl": 0.0,
       "learning_rate": 2.0021337126600283e-07,
-      "logps/chosen": -251.78524780273438,
-      "logps/rejected": -360.4002380371094,
-      "loss": 0.088,
-      "rewards/chosen": 2.7110652923583984,
-      "rewards/margins": 13.96354866027832,
-      "rewards/rejected": -11.252483367919922,
+      "logps/chosen": -263.1839599609375,
+      "logps/rejected": -427.57110595703125,
+      "loss": 0.1546,
+      "rewards/chosen": 1.825537919998169,
+      "rewards/margins": 10.585159301757812,
+      "rewards/rejected": -8.759620666503906,
       "step": 1000
     },
     {
       "epoch": 0.64,
-      "grad_norm": 19.04472541809082,
+      "grad_norm": 21.942873001098633,
       "kl": 0.0,
       "learning_rate": 1.984352773826458e-07,
-      "logps/chosen": -227.35873413085938,
-      "logps/rejected": -363.24713134765625,
-      "loss": 0.0952,
-      "rewards/chosen": 2.7294869422912598,
-      "rewards/margins": 14.287317276000977,
-      "rewards/rejected": -11.557830810546875,
+      "logps/chosen": -278.3871765136719,
+      "logps/rejected": -397.9619140625,
+      "loss": 0.1374,
+      "rewards/chosen": 1.7600288391113281,
+      "rewards/margins": 10.372520446777344,
+      "rewards/rejected": -8.612492561340332,
       "step": 1005
     },
     {
       "epoch": 0.65,
-      "grad_norm": 14.0511474609375,
+      "grad_norm": 21.323123931884766,
       "kl": 0.0,
       "learning_rate": 1.9665718349928875e-07,
-      "logps/chosen": -248.2740478515625,
-      "logps/rejected": -359.29534912109375,
-      "loss": 0.0783,
-      "rewards/chosen": 2.7295162677764893,
-      "rewards/margins": 14.418438911437988,
-      "rewards/rejected": -11.688921928405762,
+      "logps/chosen": -238.81234741210938,
+      "logps/rejected": -444.0513610839844,
+      "loss": 0.1392,
+      "rewards/chosen": 1.7811119556427002,
+      "rewards/margins": 10.868303298950195,
+      "rewards/rejected": -9.087190628051758,
       "step": 1010
     },
     {
       "epoch": 0.65,
-      "grad_norm": 21.88315773010254,
+      "grad_norm": 23.634305953979492,
       "kl": 0.0,
       "learning_rate": 1.9487908961593172e-07,
-      "logps/chosen": -228.99447631835938,
-      "logps/rejected": -387.5201721191406,
-      "loss": 0.0771,
-      "rewards/chosen": 2.3790504932403564,
-      "rewards/margins": 13.84051513671875,
-      "rewards/rejected": -11.461464881896973,
+      "logps/chosen": -254.67422485351562,
+      "logps/rejected": -439.06842041015625,
+      "loss": 0.1281,
+      "rewards/chosen": 1.955566167831421,
+      "rewards/margins": 11.299899101257324,
+      "rewards/rejected": -9.344332695007324,
       "step": 1015
     },
     {
       "epoch": 0.65,
-      "grad_norm": 15.932575225830078,
+      "grad_norm": 23.24001121520996,
       "kl": 0.0,
       "learning_rate": 1.931009957325747e-07,
-      "logps/chosen": -278.6766357421875,
-      "logps/rejected": -372.94793701171875,
-      "loss": 0.0826,
-      "rewards/chosen": 2.363306760787964,
-      "rewards/margins": 14.281936645507812,
-      "rewards/rejected": -11.918628692626953,
+      "logps/chosen": -261.807861328125,
+      "logps/rejected": -439.89434814453125,
+      "loss": 0.1331,
+      "rewards/chosen": 1.958783745765686,
+      "rewards/margins": 10.877193450927734,
+      "rewards/rejected": -8.91840934753418,
       "step": 1020
     },
     {
       "epoch": 0.66,
-      "grad_norm": 11.954800605773926,
+      "grad_norm": 21.924964904785156,
       "kl": 0.0,
       "learning_rate": 1.9132290184921761e-07,
-      "logps/chosen": -223.6442108154297,
-      "logps/rejected": -371.5616149902344,
-      "loss": 0.0961,
-      "rewards/chosen": 2.332761526107788,
-      "rewards/margins": 13.71965217590332,
-      "rewards/rejected": -11.386890411376953,
+      "logps/chosen": -232.3147735595703,
+      "logps/rejected": -428.94415283203125,
+      "loss": 0.1509,
+      "rewards/chosen": 1.5595568418502808,
+      "rewards/margins": 10.47365951538086,
+      "rewards/rejected": -8.914102554321289,
       "step": 1025
     },
     {
       "epoch": 0.66,
-      "grad_norm": 14.154836654663086,
+      "grad_norm": 22.37010383605957,
       "kl": 0.0,
       "learning_rate": 1.895448079658606e-07,
-      "logps/chosen": -223.6517791748047,
-      "logps/rejected": -386.1749267578125,
-      "loss": 0.0908,
-      "rewards/chosen": 2.7226128578186035,
-      "rewards/margins": 14.438260078430176,
-      "rewards/rejected": -11.71564769744873,
+      "logps/chosen": -310.47308349609375,
+      "logps/rejected": -423.1930236816406,
+      "loss": 0.1239,
+      "rewards/chosen": 2.3751728534698486,
+      "rewards/margins": 11.746635437011719,
+      "rewards/rejected": -9.371461868286133,
       "step": 1030
     },
     {
       "epoch": 0.66,
-      "grad_norm": 18.134370803833008,
+      "grad_norm": 18.277446746826172,
       "kl": 0.0,
       "learning_rate": 1.8776671408250356e-07,
-      "logps/chosen": -260.0550842285156,
-      "logps/rejected": -346.24127197265625,
-      "loss": 0.1007,
-      "rewards/chosen": 2.5339343547821045,
-      "rewards/margins": 13.723528861999512,
-      "rewards/rejected": -11.189595222473145,
+      "logps/chosen": -238.3054656982422,
+      "logps/rejected": -409.2549743652344,
+      "loss": 0.1534,
+      "rewards/chosen": 2.0489933490753174,
+      "rewards/margins": 10.762951850891113,
+      "rewards/rejected": -8.713958740234375,
       "step": 1035
     },
     {
       "epoch": 0.67,
-      "grad_norm": 15.416353225708008,
+      "grad_norm": 23.205121994018555,
       "kl": 0.0,
       "learning_rate": 1.859886201991465e-07,
-      "logps/chosen": -247.7535400390625,
-      "logps/rejected": -371.42034912109375,
-      "loss": 0.0912,
-      "rewards/chosen": 2.3352103233337402,
-      "rewards/margins": 13.56823444366455,
-      "rewards/rejected": -11.233022689819336,
+      "logps/chosen": -312.9981994628906,
+      "logps/rejected": -412.560791015625,
+      "loss": 0.1551,
+      "rewards/chosen": 1.8553714752197266,
+      "rewards/margins": 10.774530410766602,
+      "rewards/rejected": -8.919158935546875,
       "step": 1040
     },
     {
       "epoch": 0.67,
-      "grad_norm": 18.23054313659668,
+      "grad_norm": 22.926227569580078,
       "kl": 0.0,
       "learning_rate": 1.8421052631578946e-07,
-      "logps/chosen": -213.8344268798828,
-      "logps/rejected": -369.88702392578125,
-      "loss": 0.0758,
-      "rewards/chosen": 2.6615304946899414,
-      "rewards/margins": 14.028742790222168,
-      "rewards/rejected": -11.367212295532227,
+      "logps/chosen": -326.86187744140625,
+      "logps/rejected": -425.36798095703125,
+      "loss": 0.1256,
+      "rewards/chosen": 2.2971606254577637,
+      "rewards/margins": 11.508152961730957,
+      "rewards/rejected": -9.210992813110352,
       "step": 1045
     },
     {
       "epoch": 0.67,
-      "grad_norm": 18.42432975769043,
+      "grad_norm": 19.493038177490234,
       "kl": 0.0,
       "learning_rate": 1.8243243243243243e-07,
-      "logps/chosen": -238.5988006591797,
-      "logps/rejected": -392.305419921875,
-      "loss": 0.0965,
-      "rewards/chosen": 2.5309016704559326,
-      "rewards/margins": 14.56842041015625,
-      "rewards/rejected": -12.037518501281738,
+      "logps/chosen": -293.5108337402344,
+      "logps/rejected": -433.5061950683594,
+      "loss": 0.1287,
+      "rewards/chosen": 1.9984190464019775,
+      "rewards/margins": 11.403292655944824,
+      "rewards/rejected": -9.40487289428711,
       "step": 1050
     },
     {
       "epoch": 0.67,
-      "grad_norm": 15.395295143127441,
+      "grad_norm": 20.409494400024414,
       "kl": 0.0,
       "learning_rate": 1.8065433854907538e-07,
-      "logps/chosen": -242.9210968017578,
-      "logps/rejected": -375.423828125,
-      "loss": 0.0985,
-      "rewards/chosen": 2.644166946411133,
-      "rewards/margins": 13.905471801757812,
-      "rewards/rejected": -11.26130485534668,
+      "logps/chosen": -254.408935546875,
+      "logps/rejected": -463.6639099121094,
+      "loss": 0.1377,
+      "rewards/chosen": 2.059924602508545,
+      "rewards/margins": 11.853521347045898,
+      "rewards/rejected": -9.793596267700195,
       "step": 1055
     },
     {
       "epoch": 0.68,
-      "grad_norm": 16.343006134033203,
+      "grad_norm": 24.266460418701172,
       "kl": 0.0,
       "learning_rate": 1.7887624466571835e-07,
-      "logps/chosen": -198.0528106689453,
-      "logps/rejected": -354.1863098144531,
-      "loss": 0.092,
-      "rewards/chosen": 2.4538371562957764,
-      "rewards/margins": 12.678072929382324,
-      "rewards/rejected": -10.224235534667969,
+      "logps/chosen": -272.4609375,
+      "logps/rejected": -444.1348571777344,
+      "loss": 0.1345,
+      "rewards/chosen": 1.9829723834991455,
+      "rewards/margins": 11.607297897338867,
+      "rewards/rejected": -9.6243257522583,
       "step": 1060
     },
     {
       "epoch": 0.68,
-      "grad_norm": 19.673669815063477,
+      "grad_norm": 22.047306060791016,
       "kl": 0.0,
       "learning_rate": 1.770981507823613e-07,
-      "logps/chosen": -241.1757049560547,
-      "logps/rejected": -339.1280822753906,
-      "loss": 0.088,
-      "rewards/chosen": 2.550891876220703,
-      "rewards/margins": 13.831698417663574,
-      "rewards/rejected": -11.280807495117188,
+      "logps/chosen": -298.2177429199219,
+      "logps/rejected": -434.88232421875,
+      "loss": 0.1425,
+      "rewards/chosen": 1.9892578125,
+      "rewards/margins": 11.173238754272461,
+      "rewards/rejected": -9.183981895446777,
       "step": 1065
     },
     {
       "epoch": 0.68,
-      "grad_norm": 15.624777793884277,
+      "grad_norm": 17.649913787841797,
       "kl": 0.0,
       "learning_rate": 1.7532005689900424e-07,
-      "logps/chosen": -247.8914031982422,
-      "logps/rejected": -364.5996398925781,
-      "loss": 0.1038,
-      "rewards/chosen": 2.5451152324676514,
-      "rewards/margins": 13.162847518920898,
-      "rewards/rejected": -10.617732048034668,
+      "logps/chosen": -275.7359619140625,
+      "logps/rejected": -409.9220275878906,
+      "loss": 0.1268,
+      "rewards/chosen": 1.9714463949203491,
+      "rewards/margins": 10.847058296203613,
+      "rewards/rejected": -8.875612258911133,
       "step": 1070
     },
     {
       "epoch": 0.69,
-      "grad_norm": 16.0270938873291,
+      "grad_norm": 21.967309951782227,
       "kl": 0.0,
       "learning_rate": 1.7354196301564722e-07,
-      "logps/chosen": -271.2415466308594,
-      "logps/rejected": -348.6362609863281,
-      "loss": 0.0832,
-      "rewards/chosen": 2.6901183128356934,
-      "rewards/margins": 13.692281723022461,
-      "rewards/rejected": -11.002164840698242,
+      "logps/chosen": -281.526123046875,
+      "logps/rejected": -468.4922790527344,
+      "loss": 0.1484,
+      "rewards/chosen": 1.7273063659667969,
+      "rewards/margins": 11.125950813293457,
+      "rewards/rejected": -9.39864444732666,
       "step": 1075
     },
     {
       "epoch": 0.69,
-      "grad_norm": 16.198070526123047,
+      "grad_norm": 24.07299041748047,
       "kl": 0.0,
       "learning_rate": 1.717638691322902e-07,
-      "logps/chosen": -228.4770965576172,
-      "logps/rejected": -371.384521484375,
-      "loss": 0.0879,
-      "rewards/chosen": 2.317864179611206,
-      "rewards/margins": 14.147501945495605,
-      "rewards/rejected": -11.82963752746582,
+      "logps/chosen": -288.5100402832031,
+      "logps/rejected": -438.12579345703125,
+      "loss": 0.1292,
+      "rewards/chosen": 2.2782421112060547,
+      "rewards/margins": 11.389139175415039,
+      "rewards/rejected": -9.110896110534668,
       "step": 1080
     },
     {
       "epoch": 0.69,
-      "grad_norm": 24.463367462158203,
+      "grad_norm": 24.698833465576172,
       "kl": 0.0,
       "learning_rate": 1.6998577524893314e-07,
-      "logps/chosen": -271.0051574707031,
-      "logps/rejected": -403.8445129394531,
-      "loss": 0.0817,
-      "rewards/chosen": 2.82243275642395,
-      "rewards/margins": 14.68195629119873,
-      "rewards/rejected": -11.859524726867676,
+      "logps/chosen": -262.21307373046875,
+      "logps/rejected": -432.73095703125,
+      "loss": 0.1417,
+      "rewards/chosen": 1.5888358354568481,
+      "rewards/margins": 11.009684562683105,
+      "rewards/rejected": -9.42085075378418,
       "step": 1085
     },
     {
       "epoch": 0.7,
-      "grad_norm": 21.81173324584961,
+      "grad_norm": 19.981353759765625,
       "kl": 0.0,
       "learning_rate": 1.6820768136557609e-07,
-      "logps/chosen": -209.64572143554688,
-      "logps/rejected": -360.5612487792969,
-      "loss": 0.0849,
-      "rewards/chosen": 2.4915404319763184,
-      "rewards/margins": 13.439886093139648,
-      "rewards/rejected": -10.948348045349121,
+      "logps/chosen": -248.55294799804688,
+      "logps/rejected": -417.9847717285156,
+      "loss": 0.142,
+      "rewards/chosen": 2.168029308319092,
+      "rewards/margins": 11.213571548461914,
+      "rewards/rejected": -9.04554271697998,
       "step": 1090
     },
     {
       "epoch": 0.7,
-      "grad_norm": 20.325061798095703,
+      "grad_norm": 25.718555450439453,
       "kl": 0.0,
       "learning_rate": 1.6642958748221906e-07,
-      "logps/chosen": -244.2580108642578,
-      "logps/rejected": -348.1650390625,
-      "loss": 0.0804,
-      "rewards/chosen": 2.744297742843628,
-      "rewards/margins": 13.572134017944336,
-      "rewards/rejected": -10.827836990356445,
+      "logps/chosen": -310.0647888183594,
+      "logps/rejected": -412.52001953125,
+      "loss": 0.1359,
+      "rewards/chosen": 2.2913904190063477,
+      "rewards/margins": 11.509147644042969,
+      "rewards/rejected": -9.217756271362305,
       "step": 1095
     },
     {
       "epoch": 0.7,
-      "grad_norm": 24.380229949951172,
+      "grad_norm": 20.378263473510742,
       "kl": 0.0,
       "learning_rate": 1.64651493598862e-07,
-      "logps/chosen": -253.1858673095703,
-      "logps/rejected": -348.04144287109375,
-      "loss": 0.0846,
-      "rewards/chosen": 2.656752824783325,
-      "rewards/margins": 13.392297744750977,
-      "rewards/rejected": -10.735544204711914,
+      "logps/chosen": -271.46343994140625,
+      "logps/rejected": -405.5977478027344,
+      "loss": 0.1325,
+      "rewards/chosen": 2.2153549194335938,
+      "rewards/margins": 11.555242538452148,
+      "rewards/rejected": -9.339888572692871,
       "step": 1100
     },
     {
       "epoch": 0.71,
-      "grad_norm": 12.965998649597168,
+      "grad_norm": 23.4803466796875,
       "kl": 0.0,
       "learning_rate": 1.6287339971550498e-07,
-      "logps/chosen": -262.8606262207031,
-      "logps/rejected": -356.92242431640625,
-      "loss": 0.0783,
-      "rewards/chosen": 2.8926401138305664,
-      "rewards/margins": 14.1354341506958,
-      "rewards/rejected": -11.242793083190918,
+      "logps/chosen": -292.6392822265625,
+      "logps/rejected": -450.56121826171875,
+      "loss": 0.1272,
+      "rewards/chosen": 1.7447988986968994,
+      "rewards/margins": 11.052331924438477,
+      "rewards/rejected": -9.307534217834473,
       "step": 1105
     },
     {
       "epoch": 0.71,
-      "grad_norm": 16.383338928222656,
+      "grad_norm": 21.62887954711914,
       "kl": 0.0,
       "learning_rate": 1.6109530583214793e-07,
-      "logps/chosen": -300.94384765625,
-      "logps/rejected": -358.71588134765625,
-      "loss": 0.0951,
-      "rewards/chosen": 2.539355993270874,
-      "rewards/margins": 13.336338996887207,
-      "rewards/rejected": -10.79698371887207,
+      "logps/chosen": -255.4695281982422,
+      "logps/rejected": -440.318115234375,
+      "loss": 0.1474,
+      "rewards/chosen": 2.0870609283447266,
+      "rewards/margins": 11.327590942382812,
+      "rewards/rejected": -9.240530014038086,
       "step": 1110
     },
     {
       "epoch": 0.71,
-      "grad_norm": 15.756513595581055,
+      "grad_norm": 20.0389461517334,
       "kl": 0.0,
       "learning_rate": 1.5931721194879087e-07,
-      "logps/chosen": -261.84637451171875,
-      "logps/rejected": -374.3198547363281,
-      "loss": 0.0791,
-      "rewards/chosen": 2.755915880203247,
-      "rewards/margins": 14.211552619934082,
-      "rewards/rejected": -11.455635070800781,
+      "logps/chosen": -288.50909423828125,
+      "logps/rejected": -440.1390075683594,
+      "loss": 0.1267,
+      "rewards/chosen": 2.4988930225372314,
+      "rewards/margins": 12.02427864074707,
+      "rewards/rejected": -9.525384902954102,
       "step": 1115
     },
     {
       "epoch": 0.72,
-      "grad_norm": 18.087120056152344,
+      "grad_norm": 27.30327606201172,
       "kl": 0.0,
       "learning_rate": 1.5753911806543385e-07,
-      "logps/chosen": -274.73370361328125,
-      "logps/rejected": -326.76123046875,
-      "loss": 0.0935,
-      "rewards/chosen": 2.786874294281006,
-      "rewards/margins": 12.87488079071045,
-      "rewards/rejected": -10.088006973266602,
+      "logps/chosen": -315.9127197265625,
+      "logps/rejected": -439.2584533691406,
+      "loss": 0.1366,
+      "rewards/chosen": 2.387932300567627,
+      "rewards/margins": 11.112966537475586,
+      "rewards/rejected": -8.725034713745117,
       "step": 1120
     },
     {
       "epoch": 0.72,
-      "grad_norm": 15.65069580078125,
+      "grad_norm": 22.370424270629883,
       "kl": 0.0,
       "learning_rate": 1.5576102418207682e-07,
-      "logps/chosen": -235.80062866210938,
-      "logps/rejected": -340.6771545410156,
-      "loss": 0.1005,
-      "rewards/chosen": 2.691729784011841,
-      "rewards/margins": 13.200857162475586,
-      "rewards/rejected": -10.509127616882324,
+      "logps/chosen": -296.82879638671875,
+      "logps/rejected": -427.3119201660156,
+      "loss": 0.1312,
+      "rewards/chosen": 2.379429817199707,
+      "rewards/margins": 11.889947891235352,
+      "rewards/rejected": -9.510516166687012,
       "step": 1125
     },
     {
       "epoch": 0.72,
-      "grad_norm": 25.325843811035156,
+      "grad_norm": 22.60016441345215,
       "kl": 0.0,
       "learning_rate": 1.5398293029871974e-07,
-      "logps/chosen": -244.3973388671875,
-      "logps/rejected": -343.04412841796875,
-      "loss": 0.0883,
-      "rewards/chosen": 2.485719680786133,
-      "rewards/margins": 13.161664962768555,
-      "rewards/rejected": -10.675946235656738,
+      "logps/chosen": -284.07672119140625,
+      "logps/rejected": -433.29278564453125,
+      "loss": 0.1373,
+      "rewards/chosen": 1.8526710271835327,
+      "rewards/margins": 10.757834434509277,
+      "rewards/rejected": -8.905162811279297,
       "step": 1130
     },
     {
       "epoch": 0.73,
-      "grad_norm": 17.044322967529297,
+      "grad_norm": 21.129493713378906,
       "kl": 0.0,
       "learning_rate": 1.5220483641536272e-07,
-      "logps/chosen": -244.70559692382812,
-      "logps/rejected": -358.86920166015625,
-      "loss": 0.0987,
-      "rewards/chosen": 2.5949511528015137,
-      "rewards/margins": 13.485272407531738,
-      "rewards/rejected": -10.890320777893066,
+      "logps/chosen": -293.85552978515625,
+      "logps/rejected": -427.25091552734375,
+      "loss": 0.1265,
+      "rewards/chosen": 2.3632965087890625,
+      "rewards/margins": 11.906625747680664,
+      "rewards/rejected": -9.543329238891602,
       "step": 1135
     },
     {
       "epoch": 0.73,
-      "grad_norm": 27.13024139404297,
+      "grad_norm": 19.694997787475586,
       "kl": 0.0,
       "learning_rate": 1.504267425320057e-07,
-      "logps/chosen": -247.91641235351562,
-      "logps/rejected": -343.4930419921875,
-      "loss": 0.0899,
-      "rewards/chosen": 2.4175989627838135,
-      "rewards/margins": 13.832735061645508,
-      "rewards/rejected": -11.415135383605957,
+      "logps/chosen": -303.8923645019531,
+      "logps/rejected": -465.65948486328125,
+      "loss": 0.1168,
+      "rewards/chosen": 2.5519447326660156,
+      "rewards/margins": 12.214879989624023,
+      "rewards/rejected": -9.662935256958008,
       "step": 1140
     },
     {
       "epoch": 0.73,
-      "grad_norm": 14.320551872253418,
+      "grad_norm": 22.003957748413086,
       "kl": 0.0,
       "learning_rate": 1.4864864864864866e-07,
-      "logps/chosen": -246.96670532226562,
-      "logps/rejected": -370.0877990722656,
-      "loss": 0.0758,
-      "rewards/chosen": 2.691598653793335,
-      "rewards/margins": 13.812780380249023,
-      "rewards/rejected": -11.121182441711426,
+      "logps/chosen": -275.05450439453125,
+      "logps/rejected": -431.39459228515625,
+      "loss": 0.1346,
+      "rewards/chosen": 1.9555597305297852,
+      "rewards/margins": 11.127113342285156,
+      "rewards/rejected": -9.171553611755371,
       "step": 1145
     },
     {
       "epoch": 0.74,
-      "grad_norm": 16.582712173461914,
+      "grad_norm": 22.918169021606445,
       "kl": 0.0,
       "learning_rate": 1.4687055476529158e-07,
-      "logps/chosen": -224.07785034179688,
-      "logps/rejected": -349.31927490234375,
-      "loss": 0.082,
-      "rewards/chosen": 2.6934258937835693,
-      "rewards/margins": 13.236696243286133,
-      "rewards/rejected": -10.543269157409668,
+      "logps/chosen": -302.3306579589844,
+      "logps/rejected": -442.83282470703125,
+      "loss": 0.1249,
+      "rewards/chosen": 2.11725115776062,
+      "rewards/margins": 10.936277389526367,
+      "rewards/rejected": -8.819025039672852,
       "step": 1150
     },
     {
       "epoch": 0.74,
-      "grad_norm": 17.989919662475586,
+      "grad_norm": 21.72097396850586,
       "kl": 0.0,
       "learning_rate": 1.4509246088193456e-07,
-      "logps/chosen": -243.8923797607422,
-      "logps/rejected": -367.2593688964844,
-      "loss": 0.0846,
-      "rewards/chosen": 2.7338271141052246,
-      "rewards/margins": 14.203822135925293,
-      "rewards/rejected": -11.469995498657227,
+      "logps/chosen": -287.4019470214844,
+      "logps/rejected": -401.7447814941406,
+      "loss": 0.1338,
+      "rewards/chosen": 2.2235841751098633,
+      "rewards/margins": 10.831747055053711,
+      "rewards/rejected": -8.608162879943848,
       "step": 1155
     },
     {
       "epoch": 0.74,
-      "grad_norm": 17.582975387573242,
+      "grad_norm": 22.042200088500977,
       "kl": 0.0,
       "learning_rate": 1.4331436699857753e-07,
-      "logps/chosen": -227.1583709716797,
-      "logps/rejected": -378.5196228027344,
-      "loss": 0.1011,
-      "rewards/chosen": 2.531038284301758,
-      "rewards/margins": 13.578967094421387,
-      "rewards/rejected": -11.047929763793945,
+      "logps/chosen": -278.91546630859375,
+      "logps/rejected": -414.25885009765625,
+      "loss": 0.1356,
+      "rewards/chosen": 2.200042724609375,
+      "rewards/margins": 11.185142517089844,
+      "rewards/rejected": -8.985098838806152,
       "step": 1160
     },
     {
       "epoch": 0.75,
-      "grad_norm": 8.821680068969727,
+      "grad_norm": 17.934968948364258,
       "kl": 0.0,
       "learning_rate": 1.4153627311522048e-07,
-      "logps/chosen": -277.82684326171875,
-      "logps/rejected": -364.6512145996094,
-      "loss": 0.0742,
-      "rewards/chosen": 2.9555764198303223,
-      "rewards/margins": 14.14686393737793,
-      "rewards/rejected": -11.19128704071045,
+      "logps/chosen": -308.2718200683594,
+      "logps/rejected": -443.8597717285156,
+      "loss": 0.1129,
+      "rewards/chosen": 2.3466713428497314,
+      "rewards/margins": 11.657171249389648,
+      "rewards/rejected": -9.31049919128418,
       "step": 1165
     },
     {
       "epoch": 0.75,
-      "grad_norm": 15.817394256591797,
+      "grad_norm": 24.983243942260742,
       "kl": 0.0,
       "learning_rate": 1.3975817923186345e-07,
-      "logps/chosen": -271.718017578125,
-      "logps/rejected": -352.4165954589844,
-      "loss": 0.0848,
-      "rewards/chosen": 2.5184245109558105,
-      "rewards/margins": 13.12096881866455,
-      "rewards/rejected": -10.602544784545898,
+      "logps/chosen": -260.9937438964844,
+      "logps/rejected": -389.94403076171875,
+      "loss": 0.1388,
+      "rewards/chosen": 2.287144184112549,
+      "rewards/margins": 10.213197708129883,
+      "rewards/rejected": -7.926053524017334,
       "step": 1170
     },
     {
       "epoch": 0.75,
-      "grad_norm": 18.2441463470459,
+      "grad_norm": 23.24259376525879,
       "kl": 0.0,
       "learning_rate": 1.379800853485064e-07,
-      "logps/chosen": -232.19467163085938,
-      "logps/rejected": -379.3485412597656,
-      "loss": 0.0909,
-      "rewards/chosen": 2.5335030555725098,
-      "rewards/margins": 14.489652633666992,
-      "rewards/rejected": -11.956149101257324,
+      "logps/chosen": -270.7221374511719,
+      "logps/rejected": -430.4684143066406,
+      "loss": 0.1291,
+      "rewards/chosen": 2.354933977127075,
+      "rewards/margins": 11.62955379486084,
+      "rewards/rejected": -9.274620056152344,
       "step": 1175
     },
     {
       "epoch": 0.75,
-      "grad_norm": 30.594867706298828,
+      "grad_norm": 21.11787986755371,
       "kl": 0.0,
       "learning_rate": 1.3620199146514935e-07,
-      "logps/chosen": -265.6886291503906,
-      "logps/rejected": -361.285400390625,
-      "loss": 0.0964,
-      "rewards/chosen": 2.450854778289795,
-      "rewards/margins": 13.442533493041992,
-      "rewards/rejected": -10.991681098937988,
+      "logps/chosen": -304.3623962402344,
+      "logps/rejected": -417.33929443359375,
+      "loss": 0.1336,
+      "rewards/chosen": 2.0863852500915527,
+      "rewards/margins": 11.270166397094727,
+      "rewards/rejected": -9.183780670166016,
       "step": 1180
     },
     {
       "epoch": 0.76,
-      "grad_norm": 12.528332710266113,
+      "grad_norm": 23.130449295043945,
       "kl": 0.0,
       "learning_rate": 1.3442389758179232e-07,
-      "logps/chosen": -288.8167724609375,
-      "logps/rejected": -352.4346618652344,
-      "loss": 0.0741,
-      "rewards/chosen": 2.7889742851257324,
-      "rewards/margins": 14.034700393676758,
-      "rewards/rejected": -11.245725631713867,
+      "logps/chosen": -289.29156494140625,
+      "logps/rejected": -438.5927734375,
+      "loss": 0.1334,
+      "rewards/chosen": 1.9771530628204346,
+      "rewards/margins": 11.332016944885254,
+      "rewards/rejected": -9.354864120483398,
       "step": 1185
     },
     {
       "epoch": 0.76,
-      "grad_norm": 13.565973281860352,
+      "grad_norm": 21.25185775756836,
       "kl": 0.0,
       "learning_rate": 1.326458036984353e-07,
-      "logps/chosen": -263.8364562988281,
-      "logps/rejected": -346.26019287109375,
-      "loss": 0.0843,
-      "rewards/chosen": 2.638777256011963,
-      "rewards/margins": 13.351663589477539,
-      "rewards/rejected": -10.712886810302734,
+      "logps/chosen": -246.78213500976562,
+      "logps/rejected": -435.413818359375,
+      "loss": 0.1494,
+      "rewards/chosen": 1.6873725652694702,
+      "rewards/margins": 10.664718627929688,
+      "rewards/rejected": -8.977346420288086,
       "step": 1190
     },
     {
       "epoch": 0.76,
-      "grad_norm": 21.124759674072266,
+      "grad_norm": 16.545639038085938,
       "kl": 0.0,
       "learning_rate": 1.3086770981507821e-07,
-      "logps/chosen": -245.15576171875,
-      "logps/rejected": -372.0450134277344,
-      "loss": 0.0898,
-      "rewards/chosen": 2.657459259033203,
-      "rewards/margins": 14.276753425598145,
-      "rewards/rejected": -11.619293212890625,
+      "logps/chosen": -270.4247741699219,
+      "logps/rejected": -447.81787109375,
+      "loss": 0.1282,
+      "rewards/chosen": 2.250525951385498,
+      "rewards/margins": 11.644302368164062,
+      "rewards/rejected": -9.393776893615723,
       "step": 1195
     },
     {
       "epoch": 0.77,
-      "grad_norm": 15.396940231323242,
+      "grad_norm": 18.8116512298584,
       "kl": 0.0,
       "learning_rate": 1.290896159317212e-07,
-      "logps/chosen": -247.9556121826172,
-      "logps/rejected": -371.3470764160156,
-      "loss": 0.0828,
-      "rewards/chosen": 2.4249672889709473,
-      "rewards/margins": 14.25316333770752,
-      "rewards/rejected": -11.828195571899414,
+      "logps/chosen": -315.2632751464844,
+      "logps/rejected": -431.31005859375,
+      "loss": 0.1056,
+      "rewards/chosen": 2.4115493297576904,
+      "rewards/margins": 11.539453506469727,
+      "rewards/rejected": -9.127903938293457,
       "step": 1200
     },
     {
       "epoch": 0.77,
-      "grad_norm": 14.409557342529297,
+      "grad_norm": 22.677959442138672,
       "kl": 0.0,
       "learning_rate": 1.2731152204836416e-07,
-      "logps/chosen": -252.07241821289062,
-      "logps/rejected": -361.1800842285156,
-      "loss": 0.085,
-      "rewards/chosen": 2.680985689163208,
-      "rewards/margins": 14.30175495147705,
-      "rewards/rejected": -11.620769500732422,
+      "logps/chosen": -250.3134307861328,
+      "logps/rejected": -441.03948974609375,
+      "loss": 0.1227,
+      "rewards/chosen": 2.1084024906158447,
+      "rewards/margins": 11.651433944702148,
+      "rewards/rejected": -9.543031692504883,
       "step": 1205
     },
     {
       "epoch": 0.77,
-      "grad_norm": 13.202485084533691,
+      "grad_norm": 21.911409378051758,
       "kl": 0.0,
       "learning_rate": 1.255334281650071e-07,
-      "logps/chosen": -237.88565063476562,
-      "logps/rejected": -350.7021179199219,
-      "loss": 0.0874,
-      "rewards/chosen": 2.6545567512512207,
-      "rewards/margins": 13.95142936706543,
-      "rewards/rejected": -11.29687213897705,
+      "logps/chosen": -237.79061889648438,
+      "logps/rejected": -430.192138671875,
+      "loss": 0.1288,
+      "rewards/chosen": 2.0766196250915527,
+      "rewards/margins": 11.207183837890625,
+      "rewards/rejected": -9.13056468963623,
       "step": 1210
     },
     {
       "epoch": 0.78,
-      "grad_norm": 28.25654411315918,
+      "grad_norm": 21.959409713745117,
       "kl": 0.0,
       "learning_rate": 1.2375533428165005e-07,
-      "logps/chosen": -282.26031494140625,
-      "logps/rejected": -377.5342712402344,
-      "loss": 0.0771,
-      "rewards/chosen": 2.6503818035125732,
-      "rewards/margins": 14.911274909973145,
-      "rewards/rejected": -12.260892868041992,
+      "logps/chosen": -308.5229187011719,
+      "logps/rejected": -417.3135681152344,
+      "loss": 0.1202,
+      "rewards/chosen": 2.373558759689331,
+      "rewards/margins": 11.75732135772705,
+      "rewards/rejected": -9.383761405944824,
       "step": 1215
     },
     {
       "epoch": 0.78,
-      "grad_norm": 20.115049362182617,
+      "grad_norm": 20.8580265045166,
       "kl": 0.0,
       "learning_rate": 1.2197724039829303e-07,
-      "logps/chosen": -234.886962890625,
-      "logps/rejected": -347.6160583496094,
-      "loss": 0.0879,
-      "rewards/chosen": 2.644916296005249,
-      "rewards/margins": 14.264989852905273,
-      "rewards/rejected": -11.620074272155762,
+      "logps/chosen": -243.6355438232422,
+      "logps/rejected": -430.52801513671875,
+      "loss": 0.1265,
+      "rewards/chosen": 2.0868420600891113,
+      "rewards/margins": 11.455188751220703,
+      "rewards/rejected": -9.36834716796875,
       "step": 1220
     },
     {
       "epoch": 0.78,
-      "grad_norm": 17.797800064086914,
+      "grad_norm": 17.877643585205078,
       "kl": 0.0,
       "learning_rate": 1.2019914651493598e-07,
-      "logps/chosen": -246.7455291748047,
-      "logps/rejected": -392.910400390625,
-      "loss": 0.0791,
-      "rewards/chosen": 2.706545352935791,
-      "rewards/margins": 14.843210220336914,
-      "rewards/rejected": -12.136663436889648,
+      "logps/chosen": -261.396484375,
+      "logps/rejected": -397.22882080078125,
+      "loss": 0.1269,
+      "rewards/chosen": 2.2846105098724365,
+      "rewards/margins": 11.026288986206055,
+      "rewards/rejected": -8.741678237915039,
       "step": 1225
     },
     {
       "epoch": 0.79,
-      "grad_norm": 12.662943840026855,
+      "grad_norm": 20.07047462463379,
       "kl": 0.0,
       "learning_rate": 1.1842105263157894e-07,
-      "logps/chosen": -225.7235870361328,
-      "logps/rejected": -402.0716247558594,
-      "loss": 0.0925,
-      "rewards/chosen": 2.4518020153045654,
-      "rewards/margins": 14.207118034362793,
-      "rewards/rejected": -11.755315780639648,
+      "logps/chosen": -252.1815643310547,
+      "logps/rejected": -434.57977294921875,
+      "loss": 0.1224,
+      "rewards/chosen": 2.228175640106201,
+      "rewards/margins": 11.402864456176758,
+      "rewards/rejected": -9.174688339233398,
       "step": 1230
     },
     {
       "epoch": 0.79,
-      "grad_norm": 16.056779861450195,
+      "grad_norm": 22.779888153076172,
       "kl": 0.0,
       "learning_rate": 1.166429587482219e-07,
-      "logps/chosen": -215.66708374023438,
-      "logps/rejected": -375.41717529296875,
-      "loss": 0.0844,
-      "rewards/chosen": 2.730950355529785,
-      "rewards/margins": 14.23332691192627,
-      "rewards/rejected": -11.502375602722168,
+      "logps/chosen": -310.24267578125,
+      "logps/rejected": -429.3841857910156,
+      "loss": 0.132,
+      "rewards/chosen": 2.069182872772217,
+      "rewards/margins": 11.373079299926758,
+      "rewards/rejected": -9.3038969039917,
       "step": 1235
     },
     {
       "epoch": 0.79,
-      "grad_norm": 16.644628524780273,
+      "grad_norm": 20.758487701416016,
       "kl": 0.0,
       "learning_rate": 1.1486486486486487e-07,
-      "logps/chosen": -266.43878173828125,
-      "logps/rejected": -358.8226318359375,
-      "loss": 0.0835,
-      "rewards/chosen": 2.870779514312744,
-      "rewards/margins": 14.435707092285156,
-      "rewards/rejected": -11.56492805480957,
+      "logps/chosen": -254.3895263671875,
+      "logps/rejected": -465.3462829589844,
+      "loss": 0.1237,
+      "rewards/chosen": 2.068655252456665,
+      "rewards/margins": 11.601139068603516,
+      "rewards/rejected": -9.532483100891113,
       "step": 1240
     },
     {
       "epoch": 0.8,
-      "grad_norm": 8.560157775878906,
+      "grad_norm": 19.828561782836914,
       "kl": 0.0,
       "learning_rate": 1.1308677098150782e-07,
-      "logps/chosen": -238.55319213867188,
-      "logps/rejected": -390.98358154296875,
-      "loss": 0.0804,
-      "rewards/chosen": 2.759446620941162,
-      "rewards/margins": 14.843562126159668,
-      "rewards/rejected": -12.084115028381348,
+      "logps/chosen": -282.51617431640625,
+      "logps/rejected": -437.31103515625,
+      "loss": 0.1296,
+      "rewards/chosen": 2.1852059364318848,
+      "rewards/margins": 11.828689575195312,
+      "rewards/rejected": -9.64348316192627,
       "step": 1245
     },
     {
       "epoch": 0.8,
-      "grad_norm": 16.91299819946289,
+      "grad_norm": 19.10045623779297,
       "kl": 0.0,
       "learning_rate": 1.1130867709815078e-07,
-      "logps/chosen": -234.5216064453125,
-      "logps/rejected": -338.272705078125,
-      "loss": 0.093,
-      "rewards/chosen": 2.591742992401123,
-      "rewards/margins": 14.04884147644043,
-      "rewards/rejected": -11.457098007202148,
+      "logps/chosen": -290.7330017089844,
+      "logps/rejected": -437.0845642089844,
+      "loss": 0.1163,
+      "rewards/chosen": 2.1793453693389893,
+      "rewards/margins": 11.713521003723145,
+      "rewards/rejected": -9.534174919128418,
       "step": 1250
     },
     {
       "epoch": 0.8,
-      "grad_norm": 13.1578369140625,
+      "grad_norm": 22.69818878173828,
       "kl": 0.0,
       "learning_rate": 1.0953058321479374e-07,
-      "logps/chosen": -275.4759826660156,
-      "logps/rejected": -379.9882507324219,
-      "loss": 0.0692,
-      "rewards/chosen": 2.984293222427368,
-      "rewards/margins": 15.574743270874023,
-      "rewards/rejected": -12.59045124053955,
+      "logps/chosen": -288.8890075683594,
+      "logps/rejected": -411.38629150390625,
+      "loss": 0.1308,
+      "rewards/chosen": 2.0378472805023193,
+      "rewards/margins": 10.46517276763916,
+      "rewards/rejected": -8.427325248718262,
       "step": 1255
     },
     {
       "epoch": 0.81,
-      "grad_norm": 17.815027236938477,
+      "grad_norm": 21.247556686401367,
       "kl": 0.0,
       "learning_rate": 1.077524893314367e-07,
-      "logps/chosen": -267.3926696777344,
-      "logps/rejected": -353.8580017089844,
-      "loss": 0.0725,
-      "rewards/chosen": 2.5867929458618164,
-      "rewards/margins": 14.201835632324219,
-      "rewards/rejected": -11.615041732788086,
+      "logps/chosen": -283.88751220703125,
+      "logps/rejected": -429.2752990722656,
+      "loss": 0.119,
+      "rewards/chosen": 2.1524832248687744,
+      "rewards/margins": 11.539883613586426,
+      "rewards/rejected": -9.387399673461914,
       "step": 1260
     },
     {
       "epoch": 0.81,
-      "grad_norm": 11.758225440979004,
+      "grad_norm": 21.65992546081543,
       "kl": 0.0,
       "learning_rate": 1.0597439544807964e-07,
-      "logps/chosen": -249.6155242919922,
-      "logps/rejected": -364.9371337890625,
-      "loss": 0.0769,
-      "rewards/chosen": 2.7065796852111816,
-      "rewards/margins": 14.05157470703125,
-      "rewards/rejected": -11.344995498657227,
+      "logps/chosen": -278.073486328125,
+      "logps/rejected": -448.2752990722656,
+      "loss": 0.1171,
+      "rewards/chosen": 2.266263484954834,
+      "rewards/margins": 12.238935470581055,
+      "rewards/rejected": -9.972672462463379,
       "step": 1265
     },
     {
       "epoch": 0.81,
-      "grad_norm": 16.8466739654541,
+      "grad_norm": 19.32549285888672,
       "kl": 0.0,
       "learning_rate": 1.0419630156472262e-07,
-      "logps/chosen": -238.05673217773438,
-      "logps/rejected": -373.03546142578125,
-      "loss": 0.0748,
-      "rewards/chosen": 2.9012789726257324,
-      "rewards/margins": 14.46104621887207,
-      "rewards/rejected": -11.55976676940918,
+      "logps/chosen": -303.88616943359375,
+      "logps/rejected": -435.61810302734375,
+      "loss": 0.1139,
+      "rewards/chosen": 2.234591007232666,
+      "rewards/margins": 11.52080249786377,
+      "rewards/rejected": -9.286211013793945,
       "step": 1270
     },
     {
       "epoch": 0.82,
-      "grad_norm": 14.22836685180664,
+      "grad_norm": 20.137916564941406,
       "kl": 0.0,
       "learning_rate": 1.0241820768136557e-07,
-      "logps/chosen": -224.74887084960938,
-      "logps/rejected": -332.44415283203125,
-      "loss": 0.084,
-      "rewards/chosen": 2.65814208984375,
-      "rewards/margins": 13.293182373046875,
-      "rewards/rejected": -10.635040283203125,
+      "logps/chosen": -281.22686767578125,
+      "logps/rejected": -411.6228942871094,
+      "loss": 0.1232,
+      "rewards/chosen": 2.257664203643799,
+      "rewards/margins": 11.2540864944458,
+      "rewards/rejected": -8.996421813964844,
       "step": 1275
     },
     {
       "epoch": 0.82,
-      "grad_norm": 11.591891288757324,
+      "grad_norm": 15.928502082824707,
       "kl": 0.0,
       "learning_rate": 1.0064011379800854e-07,
-      "logps/chosen": -280.44427490234375,
-      "logps/rejected": -365.83392333984375,
-      "loss": 0.0801,
-      "rewards/chosen": 2.5924744606018066,
-      "rewards/margins": 14.31025505065918,
-      "rewards/rejected": -11.717779159545898,
+      "logps/chosen": -316.901611328125,
+      "logps/rejected": -428.92822265625,
+      "loss": 0.1213,
+      "rewards/chosen": 2.580500841140747,
+      "rewards/margins": 11.729310989379883,
+      "rewards/rejected": -9.148810386657715,
       "step": 1280
     },
     {
       "epoch": 0.82,
-      "grad_norm": 14.504942893981934,
+      "grad_norm": 24.323083877563477,
       "kl": 0.0,
       "learning_rate": 9.886201991465149e-08,
-      "logps/chosen": -277.6451416015625,
-      "logps/rejected": -350.4718322753906,
-      "loss": 0.0661,
-      "rewards/chosen": 2.9364330768585205,
-      "rewards/margins": 14.681096076965332,
-      "rewards/rejected": -11.744662284851074,
+      "logps/chosen": -261.3116149902344,
+      "logps/rejected": -456.38201904296875,
+      "loss": 0.1232,
+      "rewards/chosen": 1.9357669353485107,
+      "rewards/margins": 11.504796981811523,
+      "rewards/rejected": -9.569029808044434,
       "step": 1285
     },
     {
       "epoch": 0.83,
-      "grad_norm": 16.065649032592773,
+      "grad_norm": 19.80019760131836,
       "kl": 0.0,
       "learning_rate": 9.708392603129445e-08,
-      "logps/chosen": -262.8613586425781,
-      "logps/rejected": -391.71649169921875,
-      "loss": 0.0723,
-      "rewards/chosen": 2.7925260066986084,
-      "rewards/margins": 15.433810234069824,
-      "rewards/rejected": -12.641283988952637,
+      "logps/chosen": -286.7679748535156,
+      "logps/rejected": -409.57220458984375,
+      "loss": 0.1177,
+      "rewards/chosen": 1.9313856363296509,
+      "rewards/margins": 11.41652774810791,
+      "rewards/rejected": -9.48514175415039,
       "step": 1290
     },
     {
       "epoch": 0.83,
-      "grad_norm": 21.038543701171875,
+      "grad_norm": 15.446332931518555,
       "kl": 0.0,
       "learning_rate": 9.530583214793741e-08,
-      "logps/chosen": -262.80584716796875,
-      "logps/rejected": -380.09625244140625,
-      "loss": 0.0801,
-      "rewards/chosen": 2.761460781097412,
-      "rewards/margins": 15.412538528442383,
-      "rewards/rejected": -12.651077270507812,
+      "logps/chosen": -256.6762390136719,
+      "logps/rejected": -445.1065368652344,
+      "loss": 0.1037,
+      "rewards/chosen": 2.22512149810791,
+      "rewards/margins": 12.185127258300781,
+      "rewards/rejected": -9.960004806518555,
       "step": 1295
     },
     {
       "epoch": 0.83,
-      "grad_norm": 18.872102737426758,
+      "grad_norm": 17.317325592041016,
       "kl": 0.0,
       "learning_rate": 9.352773826458037e-08,
-      "logps/chosen": -285.86712646484375,
-      "logps/rejected": -342.8421325683594,
-      "loss": 0.0819,
-      "rewards/chosen": 2.6993701457977295,
-      "rewards/margins": 14.52336311340332,
-      "rewards/rejected": -11.823991775512695,
+      "logps/chosen": -248.8364715576172,
+      "logps/rejected": -417.3561096191406,
+      "loss": 0.1064,
+      "rewards/chosen": 2.178281307220459,
+      "rewards/margins": 12.139802932739258,
+      "rewards/rejected": -9.96152114868164,
       "step": 1300
     },
     {
       "epoch": 0.83,
-      "grad_norm": 15.029848098754883,
+      "grad_norm": 16.94424057006836,
       "kl": 0.0,
       "learning_rate": 9.174964438122331e-08,
-      "logps/chosen": -245.3412322998047,
-      "logps/rejected": -356.58172607421875,
-      "loss": 0.094,
-      "rewards/chosen": 2.818472385406494,
-      "rewards/margins": 14.51873779296875,
-      "rewards/rejected": -11.700265884399414,
+      "logps/chosen": -253.8694305419922,
+      "logps/rejected": -388.809326171875,
+      "loss": 0.1238,
+      "rewards/chosen": 1.9573352336883545,
+      "rewards/margins": 11.183786392211914,
+      "rewards/rejected": -9.22645092010498,
       "step": 1305
     },
     {
       "epoch": 0.84,
-      "grad_norm": 17.082958221435547,
+      "grad_norm": 19.46892738342285,
       "kl": 0.0,
       "learning_rate": 8.997155049786629e-08,
-      "logps/chosen": -286.61383056640625,
-      "logps/rejected": -395.6304626464844,
-      "loss": 0.0716,
-      "rewards/chosen": 2.8008289337158203,
-      "rewards/margins": 15.597787857055664,
-      "rewards/rejected": -12.796960830688477,
+      "logps/chosen": -299.6037902832031,
+      "logps/rejected": -408.68701171875,
+      "loss": 0.1313,
+      "rewards/chosen": 2.082526683807373,
+      "rewards/margins": 11.771112442016602,
+      "rewards/rejected": -9.688586235046387,
       "step": 1310
     },
     {
       "epoch": 0.84,
-      "grad_norm": 22.768911361694336,
+      "grad_norm": 21.77025604248047,
       "kl": 0.0,
       "learning_rate": 8.819345661450925e-08,
-      "logps/chosen": -250.6887664794922,
-      "logps/rejected": -358.8474426269531,
-      "loss": 0.0929,
-      "rewards/chosen": 2.7452361583709717,
-      "rewards/margins": 14.326945304870605,
-      "rewards/rejected": -11.581708908081055,
+      "logps/chosen": -276.5013122558594,
+      "logps/rejected": -462.1937561035156,
+      "loss": 0.1368,
+      "rewards/chosen": 1.8290278911590576,
+      "rewards/margins": 11.848515510559082,
+      "rewards/rejected": -10.019487380981445,
       "step": 1315
     },
     {
       "epoch": 0.84,
-      "grad_norm": 21.478580474853516,
+      "grad_norm": 20.317564010620117,
       "kl": 0.0,
       "learning_rate": 8.64153627311522e-08,
-      "logps/chosen": -267.79974365234375,
-      "logps/rejected": -369.04815673828125,
-      "loss": 0.0838,
-      "rewards/chosen": 2.744828224182129,
-      "rewards/margins": 14.584070205688477,
-      "rewards/rejected": -11.839241027832031,
+      "logps/chosen": -275.123046875,
+      "logps/rejected": -400.65899658203125,
+      "loss": 0.1178,
+      "rewards/chosen": 2.3277885913848877,
+      "rewards/margins": 11.247326850891113,
+      "rewards/rejected": -8.919538497924805,
       "step": 1320
     },
     {
       "epoch": 0.85,
-      "grad_norm": 12.477672576904297,
+      "grad_norm": 18.717248916625977,
       "kl": 0.0,
       "learning_rate": 8.463726884779517e-08,
-      "logps/chosen": -239.75399780273438,
-      "logps/rejected": -383.63812255859375,
-      "loss": 0.0833,
-      "rewards/chosen": 2.6762642860412598,
-      "rewards/margins": 14.272786140441895,
-      "rewards/rejected": -11.596521377563477,
+      "logps/chosen": -261.8305358886719,
+      "logps/rejected": -415.6966857910156,
+      "loss": 0.1082,
+      "rewards/chosen": 2.135937452316284,
+      "rewards/margins": 11.28874683380127,
+      "rewards/rejected": -9.152809143066406,
       "step": 1325
     },
     {
       "epoch": 0.85,
-      "grad_norm": 19.415245056152344,
+      "grad_norm": 21.309633255004883,
       "kl": 0.0,
       "learning_rate": 8.285917496443812e-08,
-      "logps/chosen": -258.97406005859375,
-      "logps/rejected": -372.96173095703125,
-      "loss": 0.0874,
-      "rewards/chosen": 2.7613649368286133,
-      "rewards/margins": 14.08574390411377,
-      "rewards/rejected": -11.324378967285156,
+      "logps/chosen": -231.76626586914062,
+      "logps/rejected": -448.5263671875,
+      "loss": 0.1118,
+      "rewards/chosen": 2.206472873687744,
+      "rewards/margins": 12.001043319702148,
+      "rewards/rejected": -9.794569969177246,
       "step": 1330
     },
     {
       "epoch": 0.85,
-      "grad_norm": 13.463936805725098,
+      "grad_norm": 19.966053009033203,
       "kl": 0.0,
       "learning_rate": 8.108108108108108e-08,
-      "logps/chosen": -228.6225128173828,
-      "logps/rejected": -344.24688720703125,
-      "loss": 0.0777,
-      "rewards/chosen": 2.942438840866089,
-      "rewards/margins": 14.587237358093262,
-      "rewards/rejected": -11.644798278808594,
+      "logps/chosen": -287.36712646484375,
+      "logps/rejected": -427.26898193359375,
+      "loss": 0.1209,
+      "rewards/chosen": 2.056323528289795,
+      "rewards/margins": 10.724042892456055,
+      "rewards/rejected": -8.667718887329102,
       "step": 1335
     },
     {
       "epoch": 0.86,
-      "grad_norm": 13.836956977844238,
+      "grad_norm": 22.85637855529785,
       "kl": 0.0,
       "learning_rate": 7.930298719772404e-08,
-      "logps/chosen": -247.45742797851562,
-      "logps/rejected": -371.2969055175781,
-      "loss": 0.0721,
-      "rewards/chosen": 3.017321825027466,
-      "rewards/margins": 14.967196464538574,
-      "rewards/rejected": -11.949874877929688,
+      "logps/chosen": -253.1752471923828,
+      "logps/rejected": -414.54827880859375,
+      "loss": 0.1332,
+      "rewards/chosen": 1.9090397357940674,
+      "rewards/margins": 11.734771728515625,
+      "rewards/rejected": -9.825732231140137,
       "step": 1340
     },
     {
       "epoch": 0.86,
-      "grad_norm": 16.551321029663086,
+      "grad_norm": 17.524154663085938,
       "kl": 0.0,
       "learning_rate": 7.7524893314367e-08,
-      "logps/chosen": -230.99227905273438,
-      "logps/rejected": -384.6443786621094,
-      "loss": 0.0871,
-      "rewards/chosen": 2.6670827865600586,
-      "rewards/margins": 14.841870307922363,
-      "rewards/rejected": -12.174787521362305,
+      "logps/chosen": -271.58856201171875,
+      "logps/rejected": -431.1787109375,
+      "loss": 0.1257,
+      "rewards/chosen": 2.0929927825927734,
+      "rewards/margins": 11.595911026000977,
+      "rewards/rejected": -9.502918243408203,
       "step": 1345
     },
     {
       "epoch": 0.86,
-      "grad_norm": 15.87748908996582,
+      "grad_norm": 20.388002395629883,
       "kl": 0.0,
       "learning_rate": 7.574679943100994e-08,
-      "logps/chosen": -266.7972717285156,
-      "logps/rejected": -335.5400390625,
-      "loss": 0.0976,
-      "rewards/chosen": 2.702658176422119,
-      "rewards/margins": 13.561399459838867,
-      "rewards/rejected": -10.858741760253906,
+      "logps/chosen": -294.2145080566406,
+      "logps/rejected": -427.0972595214844,
+      "loss": 0.132,
+      "rewards/chosen": 2.2338738441467285,
+      "rewards/margins": 11.665974617004395,
+      "rewards/rejected": -9.432100296020508,
       "step": 1350
     },
     {
       "epoch": 0.87,
-      "grad_norm": 17.662395477294922,
+      "grad_norm": 27.609825134277344,
       "kl": 0.0,
       "learning_rate": 7.396870554765292e-08,
-      "logps/chosen": -269.71026611328125,
-      "logps/rejected": -383.00628662109375,
-      "loss": 0.0921,
-      "rewards/chosen": 2.5814826488494873,
-      "rewards/margins": 14.34886646270752,
-      "rewards/rejected": -11.76738452911377,
+      "logps/chosen": -277.4027404785156,
+      "logps/rejected": -441.5484924316406,
+      "loss": 0.1205,
+      "rewards/chosen": 1.9443111419677734,
+      "rewards/margins": 11.26426887512207,
+      "rewards/rejected": -9.319957733154297,
       "step": 1355
     },
     {
       "epoch": 0.87,
-      "grad_norm": 16.09382438659668,
+      "grad_norm": 21.430208206176758,
       "kl": 0.0,
       "learning_rate": 7.219061166429587e-08,
-      "logps/chosen": -269.3471984863281,
-      "logps/rejected": -375.32208251953125,
-      "loss": 0.0886,
-      "rewards/chosen": 2.8077282905578613,
-      "rewards/margins": 14.510396003723145,
-      "rewards/rejected": -11.702667236328125,
+      "logps/chosen": -232.32846069335938,
+      "logps/rejected": -424.41668701171875,
+      "loss": 0.143,
+      "rewards/chosen": 2.0610389709472656,
+      "rewards/margins": 10.865748405456543,
+      "rewards/rejected": -8.804708480834961,
       "step": 1360
     },
     {
       "epoch": 0.87,
-      "grad_norm": 13.27104377746582,
+      "grad_norm": 17.94335174560547,
       "kl": 0.0,
       "learning_rate": 7.041251778093883e-08,
-      "logps/chosen": -273.42950439453125,
-      "logps/rejected": -371.14837646484375,
-      "loss": 0.0685,
-      "rewards/chosen": 2.944091558456421,
-      "rewards/margins": 15.448992729187012,
-      "rewards/rejected": -12.504900932312012,
+      "logps/chosen": -246.70504760742188,
+      "logps/rejected": -452.76953125,
+      "loss": 0.0997,
+      "rewards/chosen": 2.304919481277466,
+      "rewards/margins": 11.7770414352417,
+      "rewards/rejected": -9.472122192382812,
       "step": 1365
     },
     {
       "epoch": 0.88,
-      "grad_norm": 12.18476390838623,
+      "grad_norm": 22.950284957885742,
       "kl": 0.0,
       "learning_rate": 6.863442389758179e-08,
-      "logps/chosen": -256.47161865234375,
-      "logps/rejected": -384.1866760253906,
-      "loss": 0.0742,
-      "rewards/chosen": 2.7861623764038086,
-      "rewards/margins": 15.34550952911377,
-      "rewards/rejected": -12.559347152709961,
+      "logps/chosen": -267.05242919921875,
+      "logps/rejected": -420.9137268066406,
+      "loss": 0.1369,
+      "rewards/chosen": 1.899043083190918,
+      "rewards/margins": 11.120938301086426,
+      "rewards/rejected": -9.221895217895508,
       "step": 1370
     },
     {
       "epoch": 0.88,
-      "grad_norm": 23.00941276550293,
+      "grad_norm": 18.250551223754883,
       "kl": 0.0,
       "learning_rate": 6.685633001422475e-08,
-      "logps/chosen": -242.3636474609375,
-      "logps/rejected": -404.00848388671875,
-      "loss": 0.0747,
-      "rewards/chosen": 2.5074126720428467,
-      "rewards/margins": 14.919418334960938,
-      "rewards/rejected": -12.412006378173828,
+      "logps/chosen": -231.0555877685547,
+      "logps/rejected": -449.4051818847656,
+      "loss": 0.13,
+      "rewards/chosen": 2.14901065826416,
+      "rewards/margins": 11.469152450561523,
+      "rewards/rejected": -9.320141792297363,
       "step": 1375
     },
     {
       "epoch": 0.88,
-      "grad_norm": 10.992572784423828,
+      "grad_norm": 20.363611221313477,
       "kl": 0.0,
       "learning_rate": 6.507823613086771e-08,
-      "logps/chosen": -248.75967407226562,
-      "logps/rejected": -368.59063720703125,
-      "loss": 0.0867,
-      "rewards/chosen": 2.401717185974121,
-      "rewards/margins": 14.726252555847168,
-      "rewards/rejected": -12.32453441619873,
+      "logps/chosen": -278.7294006347656,
+      "logps/rejected": -418.37030029296875,
+      "loss": 0.1272,
+      "rewards/chosen": 2.073495626449585,
+      "rewards/margins": 10.949335098266602,
+      "rewards/rejected": -8.875838279724121,
       "step": 1380
     },
     {
       "epoch": 0.89,
-      "grad_norm": 11.035951614379883,
+      "grad_norm": 18.06746482849121,
       "kl": 0.0,
       "learning_rate": 6.330014224751067e-08,
-      "logps/chosen": -240.3590850830078,
-      "logps/rejected": -347.6131896972656,
-      "loss": 0.0921,
-      "rewards/chosen": 2.680974245071411,
-      "rewards/margins": 14.051698684692383,
-      "rewards/rejected": -11.370722770690918,
+      "logps/chosen": -283.6380920410156,
+      "logps/rejected": -453.05987548828125,
+      "loss": 0.1107,
+      "rewards/chosen": 2.3998241424560547,
+      "rewards/margins": 12.352110862731934,
+      "rewards/rejected": -9.952287673950195,
       "step": 1385
     },
     {
       "epoch": 0.89,
-      "grad_norm": 21.619415283203125,
+      "grad_norm": 20.415699005126953,
       "kl": 0.0,
       "learning_rate": 6.152204836415363e-08,
-      "logps/chosen": -240.7654266357422,
-      "logps/rejected": -381.09307861328125,
-      "loss": 0.087,
-      "rewards/chosen": 2.646531105041504,
-      "rewards/margins": 14.633687019348145,
-      "rewards/rejected": -11.987154960632324,
+      "logps/chosen": -305.8134765625,
+      "logps/rejected": -397.2032775878906,
+      "loss": 0.1191,
+      "rewards/chosen": 2.6986749172210693,
+      "rewards/margins": 11.584226608276367,
+      "rewards/rejected": -8.885551452636719,
       "step": 1390
     },
     {
       "epoch": 0.89,
-      "grad_norm": 20.996952056884766,
+      "grad_norm": 20.92513084411621,
       "kl": 0.0,
       "learning_rate": 5.974395448079659e-08,
-      "logps/chosen": -228.12826538085938,
-      "logps/rejected": -373.8050231933594,
-      "loss": 0.0903,
-      "rewards/chosen": 2.5932085514068604,
-      "rewards/margins": 14.422918319702148,
-      "rewards/rejected": -11.829710960388184,
+      "logps/chosen": -295.1770935058594,
+      "logps/rejected": -471.5146484375,
+      "loss": 0.1234,
+      "rewards/chosen": 1.9177255630493164,
+      "rewards/margins": 11.333423614501953,
+      "rewards/rejected": -9.415698051452637,
       "step": 1395
     },
     {
       "epoch": 0.9,
-      "grad_norm": 12.941699981689453,
+      "grad_norm": 21.794998168945312,
       "kl": 0.0,
       "learning_rate": 5.796586059743954e-08,
-      "logps/chosen": -266.34759521484375,
-      "logps/rejected": -377.67572021484375,
-      "loss": 0.0733,
-      "rewards/chosen": 2.6969380378723145,
-      "rewards/margins": 14.922724723815918,
-      "rewards/rejected": -12.225786209106445,
+      "logps/chosen": -271.8743591308594,
+      "logps/rejected": -505.2965393066406,
+      "loss": 0.1268,
+      "rewards/chosen": 1.8860387802124023,
+      "rewards/margins": 12.0774507522583,
+      "rewards/rejected": -10.191411972045898,
       "step": 1400
     },
     {
       "epoch": 0.9,
-      "grad_norm": 18.563539505004883,
+      "grad_norm": 21.202049255371094,
       "kl": 0.0,
       "learning_rate": 5.61877667140825e-08,
-      "logps/chosen": -233.05337524414062,
-      "logps/rejected": -351.65478515625,
-      "loss": 0.0819,
-      "rewards/chosen": 2.4380977153778076,
-      "rewards/margins": 14.38970947265625,
-      "rewards/rejected": -11.951611518859863,
+      "logps/chosen": -277.30548095703125,
+      "logps/rejected": -465.62744140625,
+      "loss": 0.1104,
+      "rewards/chosen": 2.4026882648468018,
+      "rewards/margins": 12.06915283203125,
+      "rewards/rejected": -9.666464805603027,
       "step": 1405
     },
     {
       "epoch": 0.9,
-      "grad_norm": 14.77774715423584,
+      "grad_norm": 23.27330780029297,
       "kl": 0.0,
       "learning_rate": 5.4409672830725456e-08,
-      "logps/chosen": -272.6517028808594,
-      "logps/rejected": -359.89013671875,
-      "loss": 0.0762,
-      "rewards/chosen": 2.888676404953003,
-      "rewards/margins": 14.460576057434082,
-      "rewards/rejected": -11.571900367736816,
+      "logps/chosen": -276.54815673828125,
+      "logps/rejected": -438.12005615234375,
+      "loss": 0.1114,
+      "rewards/chosen": 2.2622342109680176,
+      "rewards/margins": 11.430865287780762,
+      "rewards/rejected": -9.168631553649902,
       "step": 1410
     },
     {
       "epoch": 0.91,
-      "grad_norm": 15.590229988098145,
+      "grad_norm": 22.490863800048828,
       "kl": 0.0,
       "learning_rate": 5.2631578947368416e-08,
-      "logps/chosen": -256.3451843261719,
-      "logps/rejected": -375.3144226074219,
-      "loss": 0.0762,
-      "rewards/chosen": 2.667102336883545,
-      "rewards/margins": 15.10230827331543,
-      "rewards/rejected": -12.435206413269043,
+      "logps/chosen": -275.0463562011719,
+      "logps/rejected": -461.47552490234375,
+      "loss": 0.1348,
+      "rewards/chosen": 2.0476670265197754,
+      "rewards/margins": 12.236173629760742,
+      "rewards/rejected": -10.188508033752441,
       "step": 1415
     },
     {
       "epoch": 0.91,
-      "grad_norm": 10.8274507522583,
+      "grad_norm": 22.146873474121094,
       "kl": 0.0,
       "learning_rate": 5.0853485064011376e-08,
-      "logps/chosen": -268.83636474609375,
-      "logps/rejected": -372.27838134765625,
-      "loss": 0.0648,
-      "rewards/chosen": 2.9176650047302246,
-      "rewards/margins": 14.840046882629395,
-      "rewards/rejected": -11.922381401062012,
+      "logps/chosen": -285.8138122558594,
+      "logps/rejected": -442.80169677734375,
+      "loss": 0.114,
+      "rewards/chosen": 2.3625457286834717,
+      "rewards/margins": 11.950872421264648,
+      "rewards/rejected": -9.588325500488281,
       "step": 1420
     },
     {
       "epoch": 0.91,
-      "grad_norm": 15.774340629577637,
+      "grad_norm": 18.684457778930664,
       "kl": 0.0,
       "learning_rate": 4.9075391180654337e-08,
-      "logps/chosen": -260.6099853515625,
-      "logps/rejected": -358.04241943359375,
-      "loss": 0.0585,
-      "rewards/chosen": 2.9233345985412598,
-      "rewards/margins": 14.653867721557617,
-      "rewards/rejected": -11.730535507202148,
+      "logps/chosen": -281.9924011230469,
+      "logps/rejected": -444.9959411621094,
+      "loss": 0.1139,
+      "rewards/chosen": 2.2367939949035645,
+      "rewards/margins": 11.815010070800781,
+      "rewards/rejected": -9.578217506408691,
       "step": 1425
     },
     {
       "epoch": 0.91,
-      "grad_norm": 15.554561614990234,
+      "grad_norm": 24.193016052246094,
       "kl": 0.0,
       "learning_rate": 4.72972972972973e-08,
-      "logps/chosen": -238.23739624023438,
-      "logps/rejected": -386.2373962402344,
-      "loss": 0.0833,
-      "rewards/chosen": 2.658268451690674,
-      "rewards/margins": 14.15376091003418,
-      "rewards/rejected": -11.49549388885498,
+      "logps/chosen": -319.1490783691406,
+      "logps/rejected": -415.7984924316406,
+      "loss": 0.1288,
+      "rewards/chosen": 2.3501713275909424,
+      "rewards/margins": 11.565084457397461,
+      "rewards/rejected": -9.214912414550781,
       "step": 1430
     },
     {
       "epoch": 0.92,
-      "grad_norm": 17.29914093017578,
+      "grad_norm": 20.57202911376953,
       "kl": 0.0,
       "learning_rate": 4.551920341394026e-08,
-      "logps/chosen": -248.63204956054688,
-      "logps/rejected": -371.215087890625,
-      "loss": 0.0872,
-      "rewards/chosen": 2.9267373085021973,
-      "rewards/margins": 14.192481994628906,
-      "rewards/rejected": -11.265745162963867,
+      "logps/chosen": -267.69696044921875,
+      "logps/rejected": -447.53302001953125,
+      "loss": 0.139,
+      "rewards/chosen": 2.0053253173828125,
+      "rewards/margins": 11.46650218963623,
+      "rewards/rejected": -9.461176872253418,
       "step": 1435
     },
     {
       "epoch": 0.92,
-      "grad_norm": 14.301885604858398,
+      "grad_norm": 20.329166412353516,
       "kl": 0.0,
       "learning_rate": 4.374110953058322e-08,
-      "logps/chosen": -240.0107421875,
-      "logps/rejected": -355.2042541503906,
-      "loss": 0.0846,
-      "rewards/chosen": 2.4140007495880127,
-      "rewards/margins": 13.864062309265137,
-      "rewards/rejected": -11.450060844421387,
+      "logps/chosen": -264.3152160644531,
+      "logps/rejected": -474.037353515625,
+      "loss": 0.117,
+      "rewards/chosen": 2.193683624267578,
+      "rewards/margins": 12.409022331237793,
+      "rewards/rejected": -10.215338706970215,
       "step": 1440
     },
     {
       "epoch": 0.92,
-      "grad_norm": 13.32541275024414,
+      "grad_norm": 22.352367401123047,
       "kl": 0.0,
       "learning_rate": 4.196301564722617e-08,
-      "logps/chosen": -247.22152709960938,
-      "logps/rejected": -360.07110595703125,
-      "loss": 0.0659,
-      "rewards/chosen": 2.7749335765838623,
-      "rewards/margins": 14.867749214172363,
-      "rewards/rejected": -12.092815399169922,
+      "logps/chosen": -286.8226623535156,
+      "logps/rejected": -478.8553771972656,
+      "loss": 0.1177,
+      "rewards/chosen": 1.9940738677978516,
+      "rewards/margins": 12.453526496887207,
+      "rewards/rejected": -10.459451675415039,
       "step": 1445
     },
     {
       "epoch": 0.93,
-      "grad_norm": 12.356550216674805,
+      "grad_norm": 18.939308166503906,
       "kl": 0.0,
       "learning_rate": 4.018492176386913e-08,
-      "logps/chosen": -256.69683837890625,
-      "logps/rejected": -363.08489990234375,
-      "loss": 0.0799,
-      "rewards/chosen": 2.8861594200134277,
-      "rewards/margins": 14.505203247070312,
-      "rewards/rejected": -11.619044303894043,
+      "logps/chosen": -276.21746826171875,
+      "logps/rejected": -413.55389404296875,
+      "loss": 0.1328,
+      "rewards/chosen": 2.231210708618164,
+      "rewards/margins": 11.543118476867676,
+      "rewards/rejected": -9.311908721923828,
       "step": 1450
     },
     {
       "epoch": 0.93,
-      "grad_norm": 14.969185829162598,
+      "grad_norm": 18.116722106933594,
       "kl": 0.0,
       "learning_rate": 3.840682788051209e-08,
-      "logps/chosen": -254.7665252685547,
-      "logps/rejected": -354.30841064453125,
-      "loss": 0.0757,
-      "rewards/chosen": 2.8421378135681152,
-      "rewards/margins": 14.373845100402832,
-      "rewards/rejected": -11.531707763671875,
+      "logps/chosen": -240.20028686523438,
+      "logps/rejected": -416.8455505371094,
+      "loss": 0.1142,
+      "rewards/chosen": 2.345201253890991,
+      "rewards/margins": 11.189725875854492,
+      "rewards/rejected": -8.844523429870605,
       "step": 1455
     },
     {
       "epoch": 0.93,
-      "grad_norm": 17.030691146850586,
+      "grad_norm": 25.406269073486328,
       "kl": 0.0,
       "learning_rate": 3.6628733997155046e-08,
-      "logps/chosen": -263.8084411621094,
-      "logps/rejected": -395.4825439453125,
-      "loss": 0.0759,
-      "rewards/chosen": 2.9121432304382324,
-      "rewards/margins": 15.505640983581543,
-      "rewards/rejected": -12.593496322631836,
+      "logps/chosen": -254.66464233398438,
+      "logps/rejected": -418.0938415527344,
+      "loss": 0.1186,
+      "rewards/chosen": 2.0774874687194824,
+      "rewards/margins": 11.201638221740723,
+      "rewards/rejected": -9.124151229858398,
       "step": 1460
     },
     {
       "epoch": 0.94,
-      "grad_norm": 12.211559295654297,
+      "grad_norm": 18.636796951293945,
       "kl": 0.0,
       "learning_rate": 3.4850640113798006e-08,
-      "logps/chosen": -236.26431274414062,
-      "logps/rejected": -357.1573791503906,
-      "loss": 0.0755,
-      "rewards/chosen": 2.843881130218506,
-      "rewards/margins": 14.507527351379395,
-      "rewards/rejected": -11.66364574432373,
+      "logps/chosen": -275.15472412109375,
+      "logps/rejected": -442.4652404785156,
+      "loss": 0.1126,
+      "rewards/chosen": 2.310152053833008,
+      "rewards/margins": 12.194239616394043,
+      "rewards/rejected": -9.884087562561035,
       "step": 1465
     },
     {
       "epoch": 0.94,
-      "grad_norm": 17.574806213378906,
+      "grad_norm": 27.727458953857422,
       "kl": 0.0,
       "learning_rate": 3.3072546230440967e-08,
-      "logps/chosen": -285.59942626953125,
-      "logps/rejected": -368.08245849609375,
-      "loss": 0.0676,
-      "rewards/chosen": 2.877531051635742,
-      "rewards/margins": 14.671765327453613,
-      "rewards/rejected": -11.794233322143555,
+      "logps/chosen": -282.36627197265625,
+      "logps/rejected": -417.9161071777344,
+      "loss": 0.1169,
+      "rewards/chosen": 2.358586549758911,
+      "rewards/margins": 11.822304725646973,
+      "rewards/rejected": -9.463717460632324,
       "step": 1470
     },
     {
       "epoch": 0.94,
-      "grad_norm": 13.167614936828613,
+      "grad_norm": 18.799943923950195,
       "kl": 0.0,
       "learning_rate": 3.129445234708392e-08,
-      "logps/chosen": -240.7215576171875,
-      "logps/rejected": -382.0995788574219,
-      "loss": 0.0651,
-      "rewards/chosen": 2.778978109359741,
-      "rewards/margins": 14.833264350891113,
-      "rewards/rejected": -12.054286003112793,
+      "logps/chosen": -282.34100341796875,
+      "logps/rejected": -400.6058044433594,
+      "loss": 0.1164,
+      "rewards/chosen": 2.1689610481262207,
+      "rewards/margins": 11.757662773132324,
+      "rewards/rejected": -9.588701248168945,
       "step": 1475
     },
     {
       "epoch": 0.95,
-      "grad_norm": 19.462112426757812,
+      "grad_norm": 18.94513702392578,
       "kl": 0.0,
       "learning_rate": 2.9516358463726884e-08,
-      "logps/chosen": -243.0684356689453,
-      "logps/rejected": -366.4331359863281,
-      "loss": 0.0802,
-      "rewards/chosen": 2.8256452083587646,
-      "rewards/margins": 15.236946105957031,
-      "rewards/rejected": -12.411300659179688,
+      "logps/chosen": -265.2249755859375,
+      "logps/rejected": -420.23870849609375,
+      "loss": 0.1276,
+      "rewards/chosen": 2.1199347972869873,
+      "rewards/margins": 11.105647087097168,
+      "rewards/rejected": -8.985712051391602,
       "step": 1480
     },
     {
       "epoch": 0.95,
-      "grad_norm": 20.093833923339844,
+      "grad_norm": 17.380094528198242,
       "kl": 0.0,
       "learning_rate": 2.7738264580369844e-08,
-      "logps/chosen": -209.17770385742188,
-      "logps/rejected": -359.9656677246094,
-      "loss": 0.0839,
-      "rewards/chosen": 2.901711940765381,
-      "rewards/margins": 14.679159164428711,
-      "rewards/rejected": -11.777449607849121,
+      "logps/chosen": -291.02392578125,
+      "logps/rejected": -407.71038818359375,
+      "loss": 0.1115,
+      "rewards/chosen": 2.1979808807373047,
+      "rewards/margins": 11.310641288757324,
+      "rewards/rejected": -9.11266040802002,
       "step": 1485
     },
     {
       "epoch": 0.95,
-      "grad_norm": 13.270203590393066,
+      "grad_norm": 19.2188720703125,
       "kl": 0.0,
       "learning_rate": 2.59601706970128e-08,
-      "logps/chosen": -272.44610595703125,
-      "logps/rejected": -357.05975341796875,
-      "loss": 0.0868,
-      "rewards/chosen": 2.6972391605377197,
-      "rewards/margins": 15.144004821777344,
-      "rewards/rejected": -12.446764945983887,
+      "logps/chosen": -280.51495361328125,
+      "logps/rejected": -436.34893798828125,
+      "loss": 0.1182,
+      "rewards/chosen": 2.2310471534729004,
+      "rewards/margins": 10.99431324005127,
+      "rewards/rejected": -8.763265609741211,
       "step": 1490
     },
     {
       "epoch": 0.96,
-      "grad_norm": 14.922286987304688,
+      "grad_norm": 23.651437759399414,
       "kl": 0.0,
       "learning_rate": 2.418207681365576e-08,
-      "logps/chosen": -238.18191528320312,
-      "logps/rejected": -367.4979553222656,
-      "loss": 0.0824,
-      "rewards/chosen": 2.5710983276367188,
-      "rewards/margins": 14.130340576171875,
-      "rewards/rejected": -11.559242248535156,
+      "logps/chosen": -260.20220947265625,
+      "logps/rejected": -447.58489990234375,
+      "loss": 0.1224,
+      "rewards/chosen": 1.9462487697601318,
+      "rewards/margins": 11.421347618103027,
+      "rewards/rejected": -9.475099563598633,
       "step": 1495
     },
     {
       "epoch": 0.96,
-      "grad_norm": 21.186229705810547,
+      "grad_norm": 24.50865936279297,
       "kl": 0.0,
       "learning_rate": 2.240398293029872e-08,
-      "logps/chosen": -240.6565399169922,
-      "logps/rejected": -387.65936279296875,
-      "loss": 0.0783,
-      "rewards/chosen": 2.6365768909454346,
-      "rewards/margins": 15.131543159484863,
-      "rewards/rejected": -12.494966506958008,
+      "logps/chosen": -273.6228942871094,
+      "logps/rejected": -388.49114990234375,
+      "loss": 0.1235,
+      "rewards/chosen": 2.335383415222168,
+      "rewards/margins": 10.98925495147705,
+      "rewards/rejected": -8.653871536254883,
       "step": 1500
     },
     {
       "epoch": 0.96,
-      "grad_norm": 14.05397891998291,
+      "grad_norm": 17.999431610107422,
       "kl": 0.0,
       "learning_rate": 2.0625889046941676e-08,
-      "logps/chosen": -307.04583740234375,
-      "logps/rejected": -361.1058654785156,
-      "loss": 0.0903,
-      "rewards/chosen": 2.804368257522583,
-      "rewards/margins": 14.683168411254883,
-      "rewards/rejected": -11.878799438476562,
+      "logps/chosen": -287.52899169921875,
+      "logps/rejected": -439.962646484375,
+      "loss": 0.1142,
+      "rewards/chosen": 2.2732601165771484,
+      "rewards/margins": 11.966636657714844,
+      "rewards/rejected": -9.693376541137695,
       "step": 1505
     },
     {
       "epoch": 0.97,
-      "grad_norm": 10.290091514587402,
+      "grad_norm": 22.014801025390625,
       "kl": 0.0,
       "learning_rate": 1.8847795163584636e-08,
-      "logps/chosen": -280.72406005859375,
-      "logps/rejected": -393.7200622558594,
-      "loss": 0.0643,
-      "rewards/chosen": 2.76891827583313,
-      "rewards/margins": 15.010998725891113,
-      "rewards/rejected": -12.24207878112793,
+      "logps/chosen": -259.125,
+      "logps/rejected": -452.64752197265625,
+      "loss": 0.1162,
+      "rewards/chosen": 2.1289539337158203,
+      "rewards/margins": 11.959836959838867,
+      "rewards/rejected": -9.830883026123047,
       "step": 1510
     },
     {
       "epoch": 0.97,
-      "grad_norm": 13.187542915344238,
+      "grad_norm": 24.860143661499023,
       "kl": 0.0,
       "learning_rate": 1.7069701280227596e-08,
-      "logps/chosen": -250.7726287841797,
-      "logps/rejected": -378.99993896484375,
-      "loss": 0.0697,
-      "rewards/chosen": 2.8748562335968018,
-      "rewards/margins": 15.209932327270508,
-      "rewards/rejected": -12.335077285766602,
+      "logps/chosen": -233.20724487304688,
+      "logps/rejected": -443.26153564453125,
+      "loss": 0.1203,
+      "rewards/chosen": 1.8372972011566162,
+      "rewards/margins": 11.322072982788086,
+      "rewards/rejected": -9.484776496887207,
       "step": 1515
     },
     {
       "epoch": 0.97,
-      "grad_norm": 13.64420223236084,
+      "grad_norm": 16.33979034423828,
       "kl": 0.0,
       "learning_rate": 1.5291607396870554e-08,
-      "logps/chosen": -253.7127227783203,
-      "logps/rejected": -367.41558837890625,
-      "loss": 0.0824,
-      "rewards/chosen": 3.031802177429199,
-      "rewards/margins": 14.762832641601562,
-      "rewards/rejected": -11.73102855682373,
+      "logps/chosen": -242.3277587890625,
+      "logps/rejected": -427.3155822753906,
+      "loss": 0.1275,
+      "rewards/chosen": 2.065825939178467,
+      "rewards/margins": 11.441205978393555,
+      "rewards/rejected": -9.375380516052246,
       "step": 1520
     },
     {
       "epoch": 0.98,
-      "grad_norm": 10.866116523742676,
+      "grad_norm": 22.828136444091797,
       "kl": 0.0,
       "learning_rate": 1.3513513513513514e-08,
-      "logps/chosen": -250.94400024414062,
-      "logps/rejected": -381.5174865722656,
-      "loss": 0.0793,
-      "rewards/chosen": 2.8337435722351074,
-      "rewards/margins": 15.124834060668945,
-      "rewards/rejected": -12.29109001159668,
+      "logps/chosen": -296.04815673828125,
+      "logps/rejected": -441.4925842285156,
+      "loss": 0.1139,
+      "rewards/chosen": 2.2007675170898438,
+      "rewards/margins": 11.669252395629883,
+      "rewards/rejected": -9.468484878540039,
       "step": 1525
     },
     {
       "epoch": 0.98,
-      "grad_norm": 11.604872703552246,
+      "grad_norm": 24.322763442993164,
       "kl": 0.0,
       "learning_rate": 1.1735419630156473e-08,
-      "logps/chosen": -233.5682830810547,
-      "logps/rejected": -377.261962890625,
-      "loss": 0.0687,
-      "rewards/chosen": 3.119915008544922,
-      "rewards/margins": 15.134057998657227,
-      "rewards/rejected": -12.014142990112305,
+      "logps/chosen": -313.05377197265625,
+      "logps/rejected": -380.57952880859375,
+      "loss": 0.1281,
+      "rewards/chosen": 2.0147297382354736,
+      "rewards/margins": 10.899151802062988,
+      "rewards/rejected": -8.884422302246094,
       "step": 1530
     },
     {
       "epoch": 0.98,
-      "grad_norm": 20.006731033325195,
+      "grad_norm": 15.227581977844238,
       "kl": 0.0,
       "learning_rate": 9.95732574679943e-09,
-      "logps/chosen": -257.07525634765625,
-      "logps/rejected": -346.5194396972656,
-      "loss": 0.0907,
-      "rewards/chosen": 2.676340103149414,
-      "rewards/margins": 14.137725830078125,
-      "rewards/rejected": -11.461384773254395,
+      "logps/chosen": -231.4268798828125,
+      "logps/rejected": -411.5580139160156,
+      "loss": 0.1128,
+      "rewards/chosen": 2.0990829467773438,
+      "rewards/margins": 11.027290344238281,
+      "rewards/rejected": -8.928207397460938,
       "step": 1535
     },
     {
       "epoch": 0.99,
-      "grad_norm": 17.203874588012695,
+      "grad_norm": 21.88052749633789,
       "kl": 0.0,
       "learning_rate": 8.179231863442388e-09,
-      "logps/chosen": -226.61843872070312,
-      "logps/rejected": -381.11029052734375,
-      "loss": 0.0683,
-      "rewards/chosen": 2.940342903137207,
-      "rewards/margins": 15.23707103729248,
-      "rewards/rejected": -12.296728134155273,
+      "logps/chosen": -233.326416015625,
+      "logps/rejected": -443.15234375,
+      "loss": 0.1306,
+      "rewards/chosen": 1.9721084833145142,
+      "rewards/margins": 11.35097885131836,
+      "rewards/rejected": -9.378870010375977,
       "step": 1540
     },
     {
       "epoch": 0.99,
-      "grad_norm": 14.477093696594238,
+      "grad_norm": 17.996585845947266,
       "kl": 0.0,
       "learning_rate": 6.401137980085348e-09,
-      "logps/chosen": -240.14157104492188,
-      "logps/rejected": -388.9137268066406,
-      "loss": 0.0658,
-      "rewards/chosen": 2.875831127166748,
-      "rewards/margins": 15.294588088989258,
-      "rewards/rejected": -12.418756484985352,
+      "logps/chosen": -264.0694274902344,
+      "logps/rejected": -443.869384765625,
+      "loss": 0.1047,
+      "rewards/chosen": 2.1066718101501465,
+      "rewards/margins": 11.750232696533203,
+      "rewards/rejected": -9.643560409545898,
       "step": 1545
     },
     {
       "epoch": 0.99,
-      "grad_norm": 10.444808959960938,
+      "grad_norm": 14.86182689666748,
       "kl": 0.0,
       "learning_rate": 4.623044096728307e-09,
-      "logps/chosen": -273.6938781738281,
-      "logps/rejected": -361.58782958984375,
-      "loss": 0.0592,
-      "rewards/chosen": 3.0903258323669434,
-      "rewards/margins": 15.089597702026367,
-      "rewards/rejected": -11.999273300170898,
+      "logps/chosen": -302.3523864746094,
+      "logps/rejected": -436.1973571777344,
+      "loss": 0.1027,
+      "rewards/chosen": 2.3409790992736816,
+      "rewards/margins": 12.13941764831543,
+      "rewards/rejected": -9.79843807220459,
       "step": 1550
     },
     {
       "epoch": 0.99,
-      "grad_norm": 13.01125717163086,
+      "grad_norm": 20.058828353881836,
       "kl": 0.0,
       "learning_rate": 2.844950213371266e-09,
-      "logps/chosen": -263.2748107910156,
-      "logps/rejected": -382.49481201171875,
-      "loss": 0.0724,
-      "rewards/chosen": 2.801457166671753,
-      "rewards/margins": 14.89887809753418,
-      "rewards/rejected": -12.097421646118164,
+      "logps/chosen": -264.7875061035156,
+      "logps/rejected": -401.29241943359375,
+      "loss": 0.118,
+      "rewards/chosen": 2.1413722038269043,
+      "rewards/margins": 11.171432495117188,
+      "rewards/rejected": -9.030058860778809,
       "step": 1555
     },
     {
       "epoch": 1.0,
-      "grad_norm": 13.397442817687988,
+      "grad_norm": 14.352324485778809,
       "kl": 0.0,
       "learning_rate": 1.0668563300142248e-09,
-      "logps/chosen": -225.07894897460938,
-      "logps/rejected": -353.08856201171875,
-      "loss": 0.0824,
-      "rewards/chosen": 2.47590970993042,
-      "rewards/margins": 14.8174409866333,
-      "rewards/rejected": -12.341531753540039,
+      "logps/chosen": -248.8438262939453,
+      "logps/rejected": -420.352783203125,
+      "loss": 0.1116,
+      "rewards/chosen": 2.209176540374756,
+      "rewards/margins": 12.117198944091797,
+      "rewards/rejected": -9.9080228805542,
       "step": 1560
     },
     {
       "epoch": 1.0,
       "step": 1563,
       "total_flos": 0.0,
-      "train_loss": 0.11715243643320149,
-      "train_runtime": 11089.6574,
-      "train_samples_per_second": 9.017,
+      "train_loss": 0.1710020688315347,
+      "train_runtime": 11106.5263,
+      "train_samples_per_second": 9.004,
       "train_steps_per_second": 0.141
     }
   ],