Model save

README.md

@@ -2,15 +2,9 @@
 license: apache-2.0
 base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
-- trl
-- dpo
-- generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: zephyr-7b-dpo-full
   results: []
@@ -21,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.2784
-- Rewards/chosen: 31.9054
-- Rewards/rejected: 27.5130
-- Rewards/accuracies: 0.5742
-- Rewards/margins: 4.3924
-- Logps/rejected: -235.1542
-- Logps/chosen: -230.6881
-- Logits/rejected: -2.6646
-- Logits/chosen: -2.6896
+- Loss: 0.4917
+- Rewards/chosen: 21.2230
+- Rewards/rejected: 6.3692
+- Rewards/accuracies: 0.7344
+- Rewards/margins: 14.8539
+- Logps/rejected: -256.2980
+- Logps/chosen: -241.3705
+- Logits/rejected: -2.7037
+- Logits/chosen: -2.7315
 
 ## Model description
 
@@ -68,10 +62,10 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.3163        | 0.21  | 100  | 0.3129          | 27.2147        | 24.2237          | 0.5820             | 2.9910          | -238.4435      | -235.3788    | -2.6701         | -2.6969       |
-| 0.2918        | 0.42  | 200  | 0.2923          | 30.3522        | 26.4568          | 0.5820             | 3.8954          | -236.2104      | -232.2412    | -2.6826         | -2.7085       |
-| 0.286         | 0.63  | 300  | 0.2921          | 31.4394        | 27.2385          | 0.5820             | 4.2009          | -235.4286      | -231.1540    | -2.6717         | -2.6957       |
-| 0.2819        | 0.84  | 400  | 0.2788          | 31.8547        | 27.4761          | 0.5781             | 4.3786          | -235.1911      | -230.7387    | -2.6619         | -2.6869       |
+| 0.5187        | 0.21  | 100  | 0.5296          | 19.0644        | 9.0310           | 0.7227             | 10.0334         | -253.6362      | -243.5290    | -2.7384         | -2.7638       |
+| 0.508         | 0.42  | 200  | 0.5006          | 20.6504        | 7.0237           | 0.7266             | 13.6267         | -255.6435      | -241.9431    | -2.7569         | -2.7826       |
+| 0.4808        | 0.63  | 300  | 0.4966          | 20.8183        | 6.9540           | 0.7227             | 13.8643         | -255.7132      | -241.7751    | -2.7115         | -2.7378       |
+| 0.4835        | 0.84  | 400  | 0.4917          | 21.2230        | 6.3692           | 0.7344             | 14.8539         | -256.2980      | -241.3705    | -2.7037         | -2.7315       |
 
 
 ### Framework versions

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.689567804336548,
-    "eval_logits/rejected": -2.664588451385498,
-    "eval_logps/chosen": -230.68809509277344,
-    "eval_logps/rejected": -235.1542205810547,
-    "eval_loss": 0.27835845947265625,
-    "eval_rewards/accuracies": 0.57421875,
-    "eval_rewards/chosen": 31.905372619628906,
-    "eval_rewards/margins": 4.392405033111572,
-    "eval_rewards/rejected": 27.51296615600586,
-    "eval_runtime": 96.4814,
-    "eval_samples": 2000,
-    "eval_samples_per_second": 20.729,
-    "eval_steps_per_second": 0.332,
-    "train_loss": 0.31381568898715734,
-    "train_runtime": 7749.4814,
+    "train_loss": 0.5147256711536871,
+    "train_runtime": 7551.132,
     "train_samples": 61134,
-    "train_samples_per_second": 7.889,
-    "train_steps_per_second": 0.062
+    "train_samples_per_second": 8.096,
+    "train_steps_per_second": 0.063
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aeaa284c42a7973b56bb02672fa19944707a1761c29070843c6f9f62dd72e4fe
+oid sha256:cc366785bb090b61844d3334a997422ad3d771ce8be204a936bc25dbbdca80c7
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5fe9c0ae7edc563803fced44889ee13e00331e230ffe85265ea71dd173dd7599
+oid sha256:300478fcbb7886772c950cffb296e8aa0f8100dee2533cd0f29f38b9e71b79e0
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4eb8c28dbe053ab22cc46c14d5c8cb9815cb514a15461a39bb7903e6e94dd96a
+oid sha256:d61731df17ec5159cd05d998bcdbadb621478295c6553a07ec6f60275d8223e4
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.31381568898715734,
-    "train_runtime": 7749.4814,
+    "train_loss": 0.5147256711536871,
+    "train_runtime": 7551.132,
     "train_samples": 61134,
-    "train_samples_per_second": 7.889,
-    "train_steps_per_second": 0.062
+    "train_samples_per_second": 8.096,
+    "train_steps_per_second": 0.063
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 74.50819179863889,
+      "grad_norm": 45.900680687932606,
       "learning_rate": 1.0416666666666666e-08,
       "logits/chosen": -2.7660439014434814,
       "logits/rejected": -2.717564582824707,
@@ -25,781 +25,781 @@
     },
     {
       "epoch": 0.02,
-      "grad_norm": 71.5827858042053,
+      "grad_norm": 42.914729941865076,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -2.592801809310913,
-      "logits/rejected": -2.5633366107940674,
-      "logps/chosen": -264.5331726074219,
-      "logps/rejected": -251.33367919921875,
-      "loss": 0.6884,
+      "logits/chosen": -2.592682361602783,
+      "logits/rejected": -2.5630006790161133,
+      "logps/chosen": -264.6473388671875,
+      "logps/rejected": -251.43508911132812,
+      "loss": 0.6923,
       "rewards/accuracies": 0.4444444477558136,
-      "rewards/chosen": 0.2647041380405426,
-      "rewards/margins": 0.0454571396112442,
-      "rewards/rejected": 0.2192470282316208,
+      "rewards/chosen": 0.15051230788230896,
+      "rewards/margins": 0.03262672945857048,
+      "rewards/rejected": 0.11788560450077057,
       "step": 10
     },
     {
       "epoch": 0.04,
-      "grad_norm": 33.37630632393394,
+      "grad_norm": 38.452837409916924,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -2.6635663509368896,
-      "logits/rejected": -2.6177525520324707,
-      "logps/chosen": -275.1928405761719,
-      "logps/rejected": -290.4365539550781,
-      "loss": 0.5763,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 6.3604888916015625,
-      "rewards/margins": -0.009852093644440174,
-      "rewards/rejected": 6.370340824127197,
+      "logits/chosen": -2.6565070152282715,
+      "logits/rejected": -2.609382390975952,
+      "logps/chosen": -278.57049560546875,
+      "logps/rejected": -293.88580322265625,
+      "loss": 0.675,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 2.982862949371338,
+      "rewards/margins": 0.061783939599990845,
+      "rewards/rejected": 2.92107892036438,
       "step": 20
     },
     {
       "epoch": 0.06,
-      "grad_norm": 22.1278736890366,
+      "grad_norm": 33.8019211383058,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -2.7272486686706543,
-      "logits/rejected": -2.667067527770996,
-      "logps/chosen": -285.1613464355469,
-      "logps/rejected": -249.3108367919922,
-      "loss": 0.4416,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": 15.510467529296875,
-      "rewards/margins": 0.8711569905281067,
-      "rewards/rejected": 14.639310836791992,
+      "logits/chosen": -2.682241916656494,
+      "logits/rejected": -2.6154792308807373,
+      "logps/chosen": -288.4586181640625,
+      "logps/rejected": -252.54623413085938,
+      "loss": 0.6263,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 12.213181495666504,
+      "rewards/margins": 0.8092762231826782,
+      "rewards/rejected": 11.403905868530273,
       "step": 30
     },
     {
       "epoch": 0.08,
-      "grad_norm": 17.071895487907064,
+      "grad_norm": 31.31645871670535,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.6888694763183594,
-      "logits/rejected": -2.6701016426086426,
-      "logps/chosen": -247.84716796875,
-      "logps/rejected": -227.38131713867188,
-      "loss": 0.3982,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": 19.278215408325195,
-      "rewards/margins": 2.267552137374878,
-      "rewards/rejected": 17.010662078857422,
+      "logits/chosen": -2.6231982707977295,
+      "logits/rejected": -2.59993052482605,
+      "logps/chosen": -250.78829956054688,
+      "logps/rejected": -231.2698974609375,
+      "loss": 0.5948,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": 16.337059020996094,
+      "rewards/margins": 3.214967727661133,
+      "rewards/rejected": 13.122090339660645,
       "step": 40
     },
     {
       "epoch": 0.1,
-      "grad_norm": 14.78162706214556,
+      "grad_norm": 28.69978279384899,
       "learning_rate": 4.999733114418725e-07,
-      "logits/chosen": -2.659508466720581,
-      "logits/rejected": -2.6249804496765137,
-      "logps/chosen": -259.9454650878906,
-      "logps/rejected": -272.14227294921875,
-      "loss": 0.3676,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 19.786420822143555,
-      "rewards/margins": -0.8553922772407532,
-      "rewards/rejected": 20.64181137084961,
+      "logits/chosen": -2.62522554397583,
+      "logits/rejected": -2.5902233123779297,
+      "logps/chosen": -264.43536376953125,
+      "logps/rejected": -280.06646728515625,
+      "loss": 0.5678,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 15.29652214050293,
+      "rewards/margins": 2.578859806060791,
+      "rewards/rejected": 12.717663764953613,
       "step": 50
     },
     {
       "epoch": 0.13,
-      "grad_norm": 14.285832773490087,
+      "grad_norm": 31.624733594392392,
       "learning_rate": 4.990398100856366e-07,
-      "logits/chosen": -2.6977388858795166,
-      "logits/rejected": -2.654181957244873,
-      "logps/chosen": -247.1780242919922,
-      "logps/rejected": -275.7373962402344,
-      "loss": 0.3521,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 24.428516387939453,
-      "rewards/margins": 2.0845706462860107,
-      "rewards/rejected": 22.343944549560547,
+      "logits/chosen": -2.714506149291992,
+      "logits/rejected": -2.663816213607788,
+      "logps/chosen": -253.0142364501953,
+      "logps/rejected": -284.220703125,
+      "loss": 0.5655,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 18.592296600341797,
+      "rewards/margins": 4.731665134429932,
+      "rewards/rejected": 13.860631942749023,
       "step": 60
     },
     {
       "epoch": 0.15,
-      "grad_norm": 14.416469937136577,
+      "grad_norm": 31.114783845109457,
       "learning_rate": 4.967775735898179e-07,
-      "logits/chosen": -2.6118428707122803,
-      "logits/rejected": -2.625479221343994,
-      "logps/chosen": -239.4540252685547,
-      "logps/rejected": -232.90463256835938,
-      "loss": 0.3304,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 26.162424087524414,
-      "rewards/margins": 2.349818706512451,
-      "rewards/rejected": 23.812606811523438,
+      "logits/chosen": -2.659493923187256,
+      "logits/rejected": -2.6703927516937256,
+      "logps/chosen": -245.8984375,
+      "logps/rejected": -243.549560546875,
+      "loss": 0.5399,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 19.718013763427734,
+      "rewards/margins": 6.550329685211182,
+      "rewards/rejected": 13.167686462402344,
       "step": 70
     },
     {
       "epoch": 0.17,
-      "grad_norm": 15.840881084472352,
+      "grad_norm": 41.80005183977875,
       "learning_rate": 4.931986719649298e-07,
-      "logits/chosen": -2.7612788677215576,
-      "logits/rejected": -2.7243030071258545,
-      "logps/chosen": -295.0336608886719,
-      "logps/rejected": -240.8730010986328,
-      "loss": 0.3248,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 27.784252166748047,
-      "rewards/margins": 4.598628997802734,
-      "rewards/rejected": 23.185623168945312,
+      "logits/chosen": -2.816267490386963,
+      "logits/rejected": -2.7780513763427734,
+      "logps/chosen": -303.81060791015625,
+      "logps/rejected": -252.63510131835938,
+      "loss": 0.5406,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 19.00731658935547,
+      "rewards/margins": 7.583803653717041,
+      "rewards/rejected": 11.42351245880127,
       "step": 80
     },
     {
       "epoch": 0.19,
-      "grad_norm": 13.661268677283298,
+      "grad_norm": 29.109112028609335,
       "learning_rate": 4.883222001996351e-07,
-      "logits/chosen": -2.6661014556884766,
-      "logits/rejected": -2.645249128341675,
-      "logps/chosen": -231.57553100585938,
-      "logps/rejected": -228.09091186523438,
-      "loss": 0.3223,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 27.535770416259766,
-      "rewards/margins": 3.228619337081909,
-      "rewards/rejected": 24.30714988708496,
+      "logits/chosen": -2.7375311851501465,
+      "logits/rejected": -2.7109663486480713,
+      "logps/chosen": -239.97073364257812,
+      "logps/rejected": -243.8080596923828,
+      "loss": 0.5261,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 19.140594482421875,
+      "rewards/margins": 10.550561904907227,
+      "rewards/rejected": 8.590032577514648,
       "step": 90
     },
     {
       "epoch": 0.21,
-      "grad_norm": 11.61288143003843,
+      "grad_norm": 27.121615894160126,
       "learning_rate": 4.821741763807186e-07,
-      "logits/chosen": -2.6386702060699463,
-      "logits/rejected": -2.6339759826660156,
-      "logps/chosen": -233.39047241210938,
-      "logps/rejected": -232.5922393798828,
-      "loss": 0.3163,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 26.968032836914062,
-      "rewards/margins": 2.5318057537078857,
-      "rewards/rejected": 24.436227798461914,
+      "logits/chosen": -2.7122435569763184,
+      "logits/rejected": -2.710378646850586,
+      "logps/chosen": -242.0121307373047,
+      "logps/rejected": -249.0485382080078,
+      "loss": 0.5187,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 18.346372604370117,
+      "rewards/margins": 10.366473197937012,
+      "rewards/rejected": 7.979898929595947,
       "step": 100
     },
     {
       "epoch": 0.21,
-      "eval_logits/chosen": -2.6968541145324707,
-      "eval_logits/rejected": -2.670072555541992,
-      "eval_logps/chosen": -235.37875366210938,
-      "eval_logps/rejected": -238.44345092773438,
-      "eval_loss": 0.31289389729499817,
-      "eval_rewards/accuracies": 0.58203125,
-      "eval_rewards/chosen": 27.21471405029297,
-      "eval_rewards/margins": 2.99098801612854,
-      "eval_rewards/rejected": 24.223726272583008,
-      "eval_runtime": 96.735,
-      "eval_samples_per_second": 20.675,
-      "eval_steps_per_second": 0.331,
+      "eval_logits/chosen": -2.7638356685638428,
+      "eval_logits/rejected": -2.7384395599365234,
+      "eval_logps/chosen": -243.52903747558594,
+      "eval_logps/rejected": -253.63619995117188,
+      "eval_loss": 0.5296456813812256,
+      "eval_rewards/accuracies": 0.72265625,
+      "eval_rewards/chosen": 19.06442642211914,
+      "eval_rewards/margins": 10.033439636230469,
+      "eval_rewards/rejected": 9.030986785888672,
+      "eval_runtime": 97.082,
+      "eval_samples_per_second": 20.601,
+      "eval_steps_per_second": 0.33,
       "step": 100
     },
     {
       "epoch": 0.23,
-      "grad_norm": 11.688620320219954,
+      "grad_norm": 34.19751869568068,
       "learning_rate": 4.747874028753375e-07,
-      "logits/chosen": -2.7125041484832764,
-      "logits/rejected": -2.6624934673309326,
-      "logps/chosen": -276.029052734375,
-      "logps/rejected": -234.1141815185547,
-      "loss": 0.3136,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 28.551036834716797,
-      "rewards/margins": 4.829342842102051,
-      "rewards/rejected": 23.72169303894043,
+      "logits/chosen": -2.7722010612487793,
+      "logits/rejected": -2.725037097930908,
+      "logps/chosen": -285.4516906738281,
+      "logps/rejected": -248.623046875,
+      "loss": 0.5339,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 19.128398895263672,
+      "rewards/margins": 9.91553783416748,
+      "rewards/rejected": 9.212862014770508,
       "step": 110
     },
     {
       "epoch": 0.25,
-      "grad_norm": 14.849649400244427,
+      "grad_norm": 31.322721824556634,
       "learning_rate": 4.662012913161997e-07,
-      "logits/chosen": -2.6516470909118652,
-      "logits/rejected": -2.647688865661621,
-      "logps/chosen": -253.4019317626953,
-      "logps/rejected": -234.5045623779297,
-      "loss": 0.3065,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": 27.765233993530273,
-      "rewards/margins": 2.4132068157196045,
-      "rewards/rejected": 25.352027893066406,
+      "logits/chosen": -2.70879864692688,
+      "logits/rejected": -2.7064578533172607,
+      "logps/chosen": -262.66522216796875,
+      "logps/rejected": -249.9930419921875,
+      "loss": 0.5035,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 18.501953125,
+      "rewards/margins": 8.638445854187012,
+      "rewards/rejected": 9.863507270812988,
       "step": 120
     },
     {
       "epoch": 0.27,
-      "grad_norm": 12.095477452171375,
+      "grad_norm": 29.14222659930629,
       "learning_rate": 4.5646165232345103e-07,
-      "logits/chosen": -2.679412364959717,
-      "logits/rejected": -2.6742541790008545,
-      "logps/chosen": -249.6054229736328,
-      "logps/rejected": -241.8912811279297,
-      "loss": 0.2993,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": 32.39772415161133,
-      "rewards/margins": 5.853152275085449,
-      "rewards/rejected": 26.544570922851562,
+      "logits/chosen": -2.7430055141448975,
+      "logits/rejected": -2.7424817085266113,
+      "logps/chosen": -258.7245788574219,
+      "logps/rejected": -257.83563232421875,
+      "loss": 0.5095,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 23.278545379638672,
+      "rewards/margins": 12.678305625915527,
+      "rewards/rejected": 10.600237846374512,
       "step": 130
     },
     {
       "epoch": 0.29,
-      "grad_norm": 13.237989201417717,
+      "grad_norm": 31.79008837992398,
       "learning_rate": 4.456204510851956e-07,
-      "logits/chosen": -2.7010607719421387,
-      "logits/rejected": -2.689103603363037,
-      "logps/chosen": -284.6669921875,
-      "logps/rejected": -270.44970703125,
-      "loss": 0.3016,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 31.298425674438477,
-      "rewards/margins": 1.071274995803833,
-      "rewards/rejected": 30.22715187072754,
+      "logits/chosen": -2.768493175506592,
+      "logits/rejected": -2.759028911590576,
+      "logps/chosen": -295.2568359375,
+      "logps/rejected": -290.87103271484375,
+      "loss": 0.5009,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 20.70852279663086,
+      "rewards/margins": 10.902701377868652,
+      "rewards/rejected": 9.80582332611084,
       "step": 140
     },
     {
       "epoch": 0.31,
-      "grad_norm": 11.533759549255185,
+      "grad_norm": 30.12291595525387,
       "learning_rate": 4.337355301007335e-07,
-      "logits/chosen": -2.6910300254821777,
-      "logits/rejected": -2.6623480319976807,
-      "logps/chosen": -251.215576171875,
-      "logps/rejected": -248.98348999023438,
-      "loss": 0.2985,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": 32.008628845214844,
-      "rewards/margins": 4.783123970031738,
-      "rewards/rejected": 27.225509643554688,
+      "logits/chosen": -2.757780075073242,
+      "logits/rejected": -2.7286086082458496,
+      "logps/chosen": -264.67327880859375,
+      "logps/rejected": -267.2817077636719,
+      "loss": 0.5095,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 18.550952911376953,
+      "rewards/margins": 9.623664855957031,
+      "rewards/rejected": 8.927289009094238,
       "step": 150
     },
     {
       "epoch": 0.33,
-      "grad_norm": 13.117822478323479,
+      "grad_norm": 36.61347768159374,
       "learning_rate": 4.2087030056579986e-07,
-      "logits/chosen": -2.721895217895508,
-      "logits/rejected": -2.675842523574829,
-      "logps/chosen": -242.4053192138672,
-      "logps/rejected": -230.8060302734375,
-      "loss": 0.3009,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 30.662723541259766,
-      "rewards/margins": 4.044883728027344,
-      "rewards/rejected": 26.61783790588379,
+      "logits/chosen": -2.8047165870666504,
+      "logits/rejected": -2.7731175422668457,
+      "logps/chosen": -251.7733612060547,
+      "logps/rejected": -248.71923828125,
+      "loss": 0.5234,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 21.294713973999023,
+      "rewards/margins": 12.590093612670898,
+      "rewards/rejected": 8.704621315002441,
       "step": 160
     },
     {
       "epoch": 0.36,
-      "grad_norm": 11.340151801902158,
+      "grad_norm": 35.61122377710651,
       "learning_rate": 4.070934040463998e-07,
-      "logits/chosen": -2.670436382293701,
-      "logits/rejected": -2.632450819015503,
-      "logps/chosen": -220.5222625732422,
-      "logps/rejected": -204.80908203125,
-      "loss": 0.2938,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": 28.81294822692871,
-      "rewards/margins": 2.497253179550171,
-      "rewards/rejected": 26.31569480895996,
+      "logits/chosen": -2.7708938121795654,
+      "logits/rejected": -2.7386412620544434,
+      "logps/chosen": -230.99172973632812,
+      "logps/rejected": -221.28964233398438,
+      "loss": 0.5068,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 18.34353256225586,
+      "rewards/margins": 8.508430480957031,
+      "rewards/rejected": 9.835103988647461,
       "step": 170
     },
     {
       "epoch": 0.38,
-      "grad_norm": 11.477634324684333,
+      "grad_norm": 38.25232870150566,
       "learning_rate": 3.9247834624635404e-07,
-      "logits/chosen": -2.646768093109131,
-      "logits/rejected": -2.6306955814361572,
-      "logps/chosen": -225.45016479492188,
-      "logps/rejected": -200.42015075683594,
-      "loss": 0.2914,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 30.908817291259766,
-      "rewards/margins": 3.7578415870666504,
-      "rewards/rejected": 27.150976181030273,
+      "logits/chosen": -2.73944091796875,
+      "logits/rejected": -2.7309627532958984,
+      "logps/chosen": -237.39566040039062,
+      "logps/rejected": -219.0083770751953,
+      "loss": 0.4872,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 18.96334457397461,
+      "rewards/margins": 10.400626182556152,
+      "rewards/rejected": 8.562715530395508,
       "step": 180
     },
     {
       "epoch": 0.4,
-      "grad_norm": 13.566633133843082,
+      "grad_norm": 33.97866980743485,
       "learning_rate": 3.7710310482256523e-07,
-      "logits/chosen": -2.679771900177002,
-      "logits/rejected": -2.6499440670013428,
-      "logps/chosen": -241.45156860351562,
-      "logps/rejected": -231.2630615234375,
-      "loss": 0.2963,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": 29.79128646850586,
-      "rewards/margins": 1.2995483875274658,
-      "rewards/rejected": 28.49173927307129,
+      "logits/chosen": -2.7516446113586426,
+      "logits/rejected": -2.7251124382019043,
+      "logps/chosen": -251.7504425048828,
+      "logps/rejected": -249.9357147216797,
+      "loss": 0.5079,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 19.492395401000977,
+      "rewards/margins": 9.673317909240723,
+      "rewards/rejected": 9.819077491760254,
       "step": 190
     },
     {
       "epoch": 0.42,
-      "grad_norm": 16.736011308973627,
+      "grad_norm": 134.1578718379133,
       "learning_rate": 3.610497133404795e-07,
-      "logits/chosen": -2.630007028579712,
-      "logits/rejected": -2.6183559894561768,
-      "logps/chosen": -230.09048461914062,
-      "logps/rejected": -223.8180694580078,
-      "loss": 0.2918,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": 29.806177139282227,
-      "rewards/margins": 1.575269341468811,
-      "rewards/rejected": 28.230907440185547,
+      "logits/chosen": -2.702623128890991,
+      "logits/rejected": -2.6973912715911865,
+      "logps/chosen": -240.9090118408203,
+      "logps/rejected": -245.4873046875,
+      "loss": 0.508,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 18.98764419555664,
+      "rewards/margins": 12.425970077514648,
+      "rewards/rejected": 6.561669826507568,
       "step": 200
     },
     {
       "epoch": 0.42,
-      "eval_logits/chosen": -2.708475112915039,
-      "eval_logits/rejected": -2.682575225830078,
-      "eval_logps/chosen": -232.24124145507812,
-      "eval_logps/rejected": -236.21038818359375,
-      "eval_loss": 0.29230329394340515,
-      "eval_rewards/accuracies": 0.58203125,
-      "eval_rewards/chosen": 30.35222816467285,
-      "eval_rewards/margins": 3.8954334259033203,
-      "eval_rewards/rejected": 26.45679473876953,
-      "eval_runtime": 96.829,
-      "eval_samples_per_second": 20.655,
-      "eval_steps_per_second": 0.33,
+      "eval_logits/chosen": -2.782581090927124,
+      "eval_logits/rejected": -2.756884813308716,
+      "eval_logps/chosen": -241.94308471679688,
+      "eval_logps/rejected": -255.6435089111328,
+      "eval_loss": 0.5005597472190857,
+      "eval_rewards/accuracies": 0.7265625,
+      "eval_rewards/chosen": 20.650381088256836,
+      "eval_rewards/margins": 13.626703262329102,
+      "eval_rewards/rejected": 7.02367639541626,
+      "eval_runtime": 96.3977,
+      "eval_samples_per_second": 20.747,
+      "eval_steps_per_second": 0.332,
       "step": 200
     },
     {
       "epoch": 0.44,
-      "grad_norm": 11.417465496451523,
+      "grad_norm": 29.796012700680272,
       "learning_rate": 3.4440382358952115e-07,
-      "logits/chosen": -2.6330389976501465,
-      "logits/rejected": -2.6055209636688232,
-      "logps/chosen": -257.6673889160156,
-      "logps/rejected": -225.943359375,
-      "loss": 0.2902,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": 32.02475357055664,
-      "rewards/margins": 6.720486640930176,
-      "rewards/rejected": 25.304264068603516,
+      "logits/chosen": -2.7070722579956055,
+      "logits/rejected": -2.683690071105957,
+      "logps/chosen": -269.78045654296875,
+      "logps/rejected": -245.2332763671875,
+      "loss": 0.5103,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 19.911705017089844,
+      "rewards/margins": 13.897372245788574,
+      "rewards/rejected": 6.0143327713012695,
       "step": 210
     },
     {
       "epoch": 0.46,
-      "grad_norm": 12.04727391696027,
+      "grad_norm": 30.457889800742976,
       "learning_rate": 3.272542485937368e-07,
-      "logits/chosen": -2.5957412719726562,
-      "logits/rejected": -2.5795822143554688,
-      "logps/chosen": -233.29476928710938,
-      "logps/rejected": -217.3531951904297,
-      "loss": 0.2919,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 32.082313537597656,
-      "rewards/margins": 3.7717392444610596,
-      "rewards/rejected": 28.310577392578125,
+      "logits/chosen": -2.6613078117370605,
+      "logits/rejected": -2.6541290283203125,
+      "logps/chosen": -245.76773071289062,
+      "logps/rejected": -238.1407470703125,
+      "loss": 0.5084,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 19.60938835144043,
+      "rewards/margins": 12.086370468139648,
+      "rewards/rejected": 7.523016452789307,
       "step": 220
     },
     {
       "epoch": 0.48,
-      "grad_norm": 11.505656123665526,
+      "grad_norm": 32.04472046652176,
       "learning_rate": 3.096924887558854e-07,
-      "logits/chosen": -2.6124305725097656,
-      "logits/rejected": -2.5944228172302246,
-      "logps/chosen": -217.5354461669922,
-      "logps/rejected": -220.5460205078125,
-      "loss": 0.3047,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 31.32999038696289,
-      "rewards/margins": 4.138183116912842,
-      "rewards/rejected": 27.19180679321289,
+      "logits/chosen": -2.682553768157959,
+      "logits/rejected": -2.6649653911590576,
+      "logps/chosen": -228.4560089111328,
+      "logps/rejected": -240.5241241455078,
+      "loss": 0.5079,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 20.409427642822266,
+      "rewards/margins": 13.195713996887207,
+      "rewards/rejected": 7.2137131690979,
       "step": 230
     },
     {
       "epoch": 0.5,
-      "grad_norm": 11.083392566284138,
+      "grad_norm": 33.84679720475086,
       "learning_rate": 2.9181224366319943e-07,
-      "logits/chosen": -2.660727024078369,
-      "logits/rejected": -2.6385245323181152,
-      "logps/chosen": -232.0665740966797,
-      "logps/rejected": -219.62210083007812,
-      "loss": 0.2834,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 31.633642196655273,
-      "rewards/margins": 2.1873562335968018,
-      "rewards/rejected": 29.446285247802734,
+      "logits/chosen": -2.7205722332000732,
+      "logits/rejected": -2.701112985610962,
+      "logps/chosen": -243.3650665283203,
+      "logps/rejected": -238.9823760986328,
+      "loss": 0.4938,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 20.33513069152832,
+      "rewards/margins": 10.249124526977539,
+      "rewards/rejected": 10.086007118225098,
       "step": 240
     },
     {
       "epoch": 0.52,
-      "grad_norm": 11.463127161742676,
+      "grad_norm": 31.858527336933598,
       "learning_rate": 2.7370891215954565e-07,
-      "logits/chosen": -2.6206917762756348,
-      "logits/rejected": -2.576387405395508,
-      "logps/chosen": -264.06439208984375,
-      "logps/rejected": -229.7786865234375,
-      "loss": 0.2818,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 34.12608337402344,
-      "rewards/margins": 4.382205009460449,
-      "rewards/rejected": 29.743881225585938,
+      "logits/chosen": -2.6577446460723877,
+      "logits/rejected": -2.6170592308044434,
+      "logps/chosen": -275.0615234375,
+      "logps/rejected": -250.7952880859375,
+      "loss": 0.486,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 23.128969192504883,
+      "rewards/margins": 14.401697158813477,
+      "rewards/rejected": 8.727272987365723,
       "step": 250
     },
     {
       "epoch": 0.54,
-      "grad_norm": 10.661524920447267,
+      "grad_norm": 33.909893139050666,
       "learning_rate": 2.55479083351317e-07,
-      "logits/chosen": -2.6774675846099854,
-      "logits/rejected": -2.668527364730835,
-      "logps/chosen": -260.33514404296875,
-      "logps/rejected": -225.80810546875,
-      "loss": 0.2858,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": 33.976402282714844,
-      "rewards/margins": 5.804098606109619,
-      "rewards/rejected": 28.17230224609375,
+      "logits/chosen": -2.7196850776672363,
+      "logits/rejected": -2.708618640899658,
+      "logps/chosen": -272.7483825683594,
+      "logps/rejected": -246.864013671875,
+      "loss": 0.4929,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 21.56315803527832,
+      "rewards/margins": 14.446769714355469,
+      "rewards/rejected": 7.116389274597168,
       "step": 260
     },
     {
       "epoch": 0.56,
-      "grad_norm": 11.916616915089687,
+      "grad_norm": 32.106664429608195,
       "learning_rate": 2.3722002126275822e-07,
-      "logits/chosen": -2.6731224060058594,
-      "logits/rejected": -2.6551766395568848,
-      "logps/chosen": -245.6435089111328,
-      "logps/rejected": -228.1649932861328,
-      "loss": 0.2808,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 31.35245704650879,
-      "rewards/margins": 1.8731645345687866,
-      "rewards/rejected": 29.479290008544922,
+      "logits/chosen": -2.7111871242523193,
+      "logits/rejected": -2.69014310836792,
+      "logps/chosen": -257.98822021484375,
+      "logps/rejected": -246.855712890625,
+      "loss": 0.5003,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": 19.00775718688965,
+      "rewards/margins": 8.219191551208496,
+      "rewards/rejected": 10.788566589355469,
       "step": 270
     },
     {
       "epoch": 0.59,
-      "grad_norm": 11.982078860289866,
+      "grad_norm": 36.46282639590835,
       "learning_rate": 2.19029145890313e-07,
-      "logits/chosen": -2.6452529430389404,
-      "logits/rejected": -2.6127915382385254,
-      "logps/chosen": -229.02554321289062,
-      "logps/rejected": -215.188720703125,
-      "loss": 0.2835,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 32.651554107666016,
-      "rewards/margins": 5.653929233551025,
-      "rewards/rejected": 26.99761962890625,
+      "logits/chosen": -2.6879024505615234,
+      "logits/rejected": -2.6575160026550293,
+      "logps/chosen": -241.15469360351562,
+      "logps/rejected": -236.71749877929688,
+      "loss": 0.5092,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 20.52240753173828,
+      "rewards/margins": 15.053570747375488,
+      "rewards/rejected": 5.468836784362793,
       "step": 280
     },
     {
       "epoch": 0.61,
-      "grad_norm": 11.17239233559609,
+      "grad_norm": 32.938385675504726,
       "learning_rate": 2.0100351342479216e-07,
-      "logits/chosen": -2.675553321838379,
-      "logits/rejected": -2.662069082260132,
-      "logps/chosen": -219.8170928955078,
-      "logps/rejected": -211.7806396484375,
-      "loss": 0.2849,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": 31.27024269104004,
-      "rewards/margins": 1.0949894189834595,
-      "rewards/rejected": 30.175247192382812,
+      "logits/chosen": -2.730700969696045,
+      "logits/rejected": -2.7169148921966553,
+      "logps/chosen": -231.1471710205078,
+      "logps/rejected": -233.15213012695312,
+      "loss": 0.4989,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 19.9401798248291,
+      "rewards/margins": 11.136396408081055,
+      "rewards/rejected": 8.803783416748047,
       "step": 290
     },
     {
       "epoch": 0.63,
-      "grad_norm": 9.847053265544167,
+      "grad_norm": 40.46462898598811,
       "learning_rate": 1.8323929841460178e-07,
-      "logits/chosen": -2.65397572517395,
-      "logits/rejected": -2.6134414672851562,
-      "logps/chosen": -268.84588623046875,
-      "logps/rejected": -232.80752563476562,
-      "loss": 0.286,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 32.15021514892578,
-      "rewards/margins": 4.852233409881592,
-      "rewards/rejected": 27.297988891601562,
+      "logits/chosen": -2.699219226837158,
+      "logits/rejected": -2.6627821922302246,
+      "logps/chosen": -282.5068054199219,
+      "logps/rejected": -253.93002319335938,
+      "loss": 0.4808,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 18.48929214477539,
+      "rewards/margins": 12.313825607299805,
+      "rewards/rejected": 6.175467491149902,
       "step": 300
     },
     {
       "epoch": 0.63,
-      "eval_logits/chosen": -2.695726156234741,
-      "eval_logits/rejected": -2.6716713905334473,
-      "eval_logps/chosen": -231.15402221679688,
-      "eval_logps/rejected": -235.42864990234375,
-      "eval_loss": 0.29209351539611816,
-      "eval_rewards/accuracies": 0.58203125,
-      "eval_rewards/chosen": 31.439437866210938,
-      "eval_rewards/margins": 4.200903415679932,
-      "eval_rewards/rejected": 27.238534927368164,
-      "eval_runtime": 96.789,
-      "eval_samples_per_second": 20.664,
-      "eval_steps_per_second": 0.331,
+      "eval_logits/chosen": -2.737816095352173,
+      "eval_logits/rejected": -2.7115261554718018,
+      "eval_logps/chosen": -241.775146484375,
+      "eval_logps/rejected": -255.71316528320312,
+      "eval_loss": 0.4966064989566803,
+      "eval_rewards/accuracies": 0.72265625,
+      "eval_rewards/chosen": 20.818317413330078,
+      "eval_rewards/margins": 13.864299774169922,
+      "eval_rewards/rejected": 6.954016208648682,
+      "eval_runtime": 96.5223,
+      "eval_samples_per_second": 20.721,
+      "eval_steps_per_second": 0.332,
       "step": 300
     },
     {
       "epoch": 0.65,
-      "grad_norm": 11.299461074514115,
+      "grad_norm": 34.221593161714,
       "learning_rate": 1.6583128063291573e-07,
-      "logits/chosen": -2.6087942123413086,
-      "logits/rejected": -2.607959270477295,
-      "logps/chosen": -263.2939758300781,
-      "logps/rejected": -229.5752716064453,
-      "loss": 0.2804,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": 32.041908264160156,
-      "rewards/margins": 3.100654363632202,
-      "rewards/rejected": 28.941247940063477,
+      "logits/chosen": -2.650965452194214,
+      "logits/rejected": -2.6548705101013184,
+      "logps/chosen": -275.9816589355469,
+      "logps/rejected": -252.0779571533203,
+      "loss": 0.4758,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 19.354223251342773,
+      "rewards/margins": 12.915657043457031,
+      "rewards/rejected": 6.438567161560059,
       "step": 310
     },
     {
       "epoch": 0.67,
-      "grad_norm": 11.979925902064297,
+      "grad_norm": 33.001844259909745,
       "learning_rate": 1.488723393865766e-07,
-      "logits/chosen": -2.652468204498291,
-      "logits/rejected": -2.6433398723602295,
-      "logps/chosen": -260.83233642578125,
-      "logps/rejected": -216.2664337158203,
-      "loss": 0.2788,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 32.8377571105957,
-      "rewards/margins": 4.280916213989258,
-      "rewards/rejected": 28.556838989257812,
+      "logits/chosen": -2.6921844482421875,
+      "logits/rejected": -2.691749095916748,
+      "logps/chosen": -272.9283142089844,
+      "logps/rejected": -237.22213745117188,
+      "loss": 0.4701,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 20.741744995117188,
+      "rewards/margins": 13.140623092651367,
+      "rewards/rejected": 7.6011223793029785,
       "step": 320
     },
     {
       "epoch": 0.69,
-      "grad_norm": 10.289416601586245,
+      "grad_norm": 29.49781014497435,
       "learning_rate": 1.3245295796480788e-07,
-      "logits/chosen": -2.678496837615967,
-      "logits/rejected": -2.634920835494995,
-      "logps/chosen": -229.55624389648438,
-      "logps/rejected": -231.64407348632812,
-      "loss": 0.2812,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": 32.6539306640625,
-      "rewards/margins": 4.799349784851074,
-      "rewards/rejected": 27.854583740234375,
+      "logits/chosen": -2.7120227813720703,
+      "logits/rejected": -2.677337169647217,
+      "logps/chosen": -243.09988403320312,
+      "logps/rejected": -251.96121215820312,
+      "loss": 0.4909,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 19.110301971435547,
+      "rewards/margins": 11.57287883758545,
+      "rewards/rejected": 7.537426948547363,
       "step": 330
     },
     {
       "epoch": 0.71,
-      "grad_norm": 12.940304501019066,
+      "grad_norm": 41.44497406324907,
       "learning_rate": 1.1666074087171627e-07,
-      "logits/chosen": -2.687782049179077,
-      "logits/rejected": -2.6474757194519043,
-      "logps/chosen": -258.529541015625,
-      "logps/rejected": -247.69125366210938,
-      "loss": 0.2752,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": 31.464908599853516,
-      "rewards/margins": -0.8856052160263062,
-      "rewards/rejected": 32.35051727294922,
+      "logits/chosen": -2.723417043685913,
+      "logits/rejected": -2.691898822784424,
+      "logps/chosen": -269.8048095703125,
+      "logps/rejected": -273.60491943359375,
+      "loss": 0.4884,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 20.189619064331055,
+      "rewards/margins": 13.752766609191895,
+      "rewards/rejected": 6.43685245513916,
       "step": 340
     },
     {
       "epoch": 0.73,
-      "grad_norm": 13.446019747621028,
+      "grad_norm": 34.6350412976571,
       "learning_rate": 1.0157994641835734e-07,
-      "logits/chosen": -2.6681811809539795,
-      "logits/rejected": -2.6358139514923096,
-      "logps/chosen": -227.58425903320312,
-      "logps/rejected": -212.9467010498047,
-      "loss": 0.2866,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": 30.626983642578125,
-      "rewards/margins": 2.8648905754089355,
-      "rewards/rejected": 27.7620906829834,
+      "logits/chosen": -2.700378179550171,
+      "logits/rejected": -2.6705689430236816,
+      "logps/chosen": -238.4877166748047,
+      "logps/rejected": -235.2138214111328,
+      "loss": 0.4772,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 19.723506927490234,
+      "rewards/margins": 14.228517532348633,
+      "rewards/rejected": 5.494990825653076,
       "step": 350
     },
     {
       "epoch": 0.75,
-      "grad_norm": 10.212615361555141,
+      "grad_norm": 42.44685611987456,
       "learning_rate": 8.729103716819111e-08,
-      "logits/chosen": -2.691338300704956,
-      "logits/rejected": -2.6329030990600586,
-      "logps/chosen": -269.2547302246094,
-      "logps/rejected": -233.14053344726562,
-      "loss": 0.2785,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 33.437278747558594,
-      "rewards/margins": 5.27285623550415,
-      "rewards/rejected": 28.1644287109375,
+      "logits/chosen": -2.732093334197998,
+      "logits/rejected": -2.685842990875244,
+      "logps/chosen": -282.49151611328125,
+      "logps/rejected": -255.60482788085938,
+      "loss": 0.5041,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 20.200531005859375,
+      "rewards/margins": 14.500404357910156,
+      "rewards/rejected": 5.700125694274902,
       "step": 360
     },
     {
       "epoch": 0.77,
-      "grad_norm": 12.701608094493194,
+      "grad_norm": 33.10027222498208,
       "learning_rate": 7.387025063449081e-08,
-      "logits/chosen": -2.6507325172424316,
-      "logits/rejected": -2.6226696968078613,
-      "logps/chosen": -243.0960693359375,
-      "logps/rejected": -207.664794921875,
-      "loss": 0.2854,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": 32.23695373535156,
-      "rewards/margins": 1.6676933765411377,
-      "rewards/rejected": 30.569263458251953,
+      "logits/chosen": -2.69976544380188,
+      "logits/rejected": -2.6724953651428223,
+      "logps/chosen": -256.20074462890625,
+      "logps/rejected": -229.0190887451172,
+      "loss": 0.5081,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 19.13229751586914,
+      "rewards/margins": 9.917332649230957,
+      "rewards/rejected": 9.2149658203125,
       "step": 370
     },
     {
       "epoch": 0.79,
-      "grad_norm": 11.004484883830752,
+      "grad_norm": 30.0028500984578,
       "learning_rate": 6.138919252022435e-08,
-      "logits/chosen": -2.592874526977539,
-      "logits/rejected": -2.5939741134643555,
-      "logps/chosen": -206.689697265625,
-      "logps/rejected": -228.67898559570312,
-      "loss": 0.2774,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 32.79497146606445,
-      "rewards/margins": 2.7575299739837646,
-      "rewards/rejected": 30.037445068359375,
+      "logits/chosen": -2.6289491653442383,
+      "logits/rejected": -2.6343891620635986,
+      "logps/chosen": -221.06399536132812,
+      "logps/rejected": -254.60464477539062,
+      "loss": 0.4847,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 18.420679092407227,
+      "rewards/margins": 14.3089017868042,
+      "rewards/rejected": 4.111776828765869,
       "step": 380
     },
     {
       "epoch": 0.82,
-      "grad_norm": 12.608909298282311,
+      "grad_norm": 41.20295613426392,
       "learning_rate": 4.991445467064689e-08,
-      "logits/chosen": -2.6360385417938232,
-      "logits/rejected": -2.6261894702911377,
-      "logps/chosen": -270.9910888671875,
-      "logps/rejected": -252.8332977294922,
-      "loss": 0.276,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 34.645816802978516,
-      "rewards/margins": 3.508648633956909,
-      "rewards/rejected": 31.137165069580078,
+      "logits/chosen": -2.673283338546753,
+      "logits/rejected": -2.668842315673828,
+      "logps/chosen": -285.0795593261719,
+      "logps/rejected": -274.74163818359375,
+      "loss": 0.4871,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 20.55733871459961,
+      "rewards/margins": 11.32852840423584,
+      "rewards/rejected": 9.228808403015137,
       "step": 390
     },
     {
       "epoch": 0.84,
-      "grad_norm": 11.347134923103408,
+      "grad_norm": 32.56833882197615,
       "learning_rate": 3.9507259776993954e-08,
-      "logits/chosen": -2.632523775100708,
-      "logits/rejected": -2.594832181930542,
-      "logps/chosen": -236.8807830810547,
-      "logps/rejected": -237.6399688720703,
-      "loss": 0.2819,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 33.6544189453125,
-      "rewards/margins": 4.281933784484863,
-      "rewards/rejected": 29.372488021850586,
+      "logits/chosen": -2.676893949508667,
+      "logits/rejected": -2.6494295597076416,
+      "logps/chosen": -250.10324096679688,
+      "logps/rejected": -261.3594055175781,
+      "loss": 0.4835,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 20.431964874267578,
+      "rewards/margins": 14.77888011932373,
+      "rewards/rejected": 5.653082370758057,
       "step": 400
     },
     {
       "epoch": 0.84,
-      "eval_logits/chosen": -2.6868975162506104,
-      "eval_logits/rejected": -2.66192626953125,
-      "eval_logps/chosen": -230.7387237548828,
-      "eval_logps/rejected": -235.19105529785156,
-      "eval_loss": 0.2787904143333435,
-      "eval_rewards/accuracies": 0.578125,
-      "eval_rewards/chosen": 31.854747772216797,
-      "eval_rewards/margins": 4.3786234855651855,
-      "eval_rewards/rejected": 27.476125717163086,
-      "eval_runtime": 96.6885,
-      "eval_samples_per_second": 20.685,
-      "eval_steps_per_second": 0.331,
+      "eval_logits/chosen": -2.731470823287964,
+      "eval_logits/rejected": -2.703744649887085,
+      "eval_logps/chosen": -241.37046813964844,
+      "eval_logps/rejected": -256.2980041503906,
+      "eval_loss": 0.4916878044605255,
+      "eval_rewards/accuracies": 0.734375,
+      "eval_rewards/chosen": 21.223026275634766,
+      "eval_rewards/margins": 14.853860855102539,
+      "eval_rewards/rejected": 6.369164943695068,
+      "eval_runtime": 96.4786,
+      "eval_samples_per_second": 20.73,
+      "eval_steps_per_second": 0.332,
       "step": 400
     },
     {
       "epoch": 0.86,
-      "grad_norm": 12.175943173191595,
+      "grad_norm": 33.3145791190938,
       "learning_rate": 3.022313472693447e-08,
-      "logits/chosen": -2.6695199012756348,
-      "logits/rejected": -2.626798152923584,
-      "logps/chosen": -263.4989318847656,
-      "logps/rejected": -240.9721221923828,
-      "loss": 0.2806,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 35.418556213378906,
-      "rewards/margins": 7.573515892028809,
-      "rewards/rejected": 27.845043182373047,
+      "logits/chosen": -2.7175421714782715,
+      "logits/rejected": -2.6865715980529785,
+      "logps/chosen": -276.17120361328125,
+      "logps/rejected": -262.1912841796875,
+      "loss": 0.4963,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 22.746294021606445,
+      "rewards/margins": 16.12038803100586,
+      "rewards/rejected": 6.625903129577637,
       "step": 410
     },
     {
       "epoch": 0.88,
-      "grad_norm": 11.7624491150407,
+      "grad_norm": 38.26789970081439,
       "learning_rate": 2.2111614344599684e-08,
-      "logits/chosen": -2.6308817863464355,
-      "logits/rejected": -2.620222568511963,
-      "logps/chosen": -264.280517578125,
-      "logps/rejected": -247.2097625732422,
-      "loss": 0.2882,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 32.79326248168945,
-      "rewards/margins": 5.5407843589782715,
-      "rewards/rejected": 27.252477645874023,
+      "logits/chosen": -2.6725165843963623,
+      "logits/rejected": -2.6631102561950684,
+      "logps/chosen": -278.31524658203125,
+      "logps/rejected": -268.63787841796875,
+      "loss": 0.497,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 18.758575439453125,
+      "rewards/margins": 12.9342041015625,
+      "rewards/rejected": 5.824368953704834,
       "step": 420
     },
     {
       "epoch": 0.9,
-      "grad_norm": 11.16296113559481,
+      "grad_norm": 29.576447372480786,
       "learning_rate": 1.521597710086439e-08,
-      "logits/chosen": -2.577580213546753,
-      "logits/rejected": -2.5429909229278564,
-      "logps/chosen": -248.5481719970703,
-      "logps/rejected": -228.4681396484375,
-      "loss": 0.2851,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": 30.489971160888672,
-      "rewards/margins": 1.1781085729599,
-      "rewards/rejected": 29.311859130859375,
+      "logits/chosen": -2.622992515563965,
+      "logits/rejected": -2.5860610008239746,
+      "logps/chosen": -260.331787109375,
+      "logps/rejected": -252.45510864257812,
+      "loss": 0.4834,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 18.70633316040039,
+      "rewards/margins": 13.38142204284668,
+      "rewards/rejected": 5.3249101638793945,
       "step": 430
     },
     {
       "epoch": 0.92,
-      "grad_norm": 10.453636294498436,
+      "grad_norm": 31.865322800676886,
       "learning_rate": 9.57301420397924e-09,
-      "logits/chosen": -2.654780864715576,
-      "logits/rejected": -2.619481086730957,
-      "logps/chosen": -251.1508026123047,
-      "logps/rejected": -240.0060272216797,
-      "loss": 0.2805,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": 33.18633270263672,
-      "rewards/margins": 4.080627918243408,
-      "rewards/rejected": 29.1057071685791,
+      "logits/chosen": -2.696772813796997,
+      "logits/rejected": -2.6611225605010986,
+      "logps/chosen": -262.6088562011719,
+      "logps/rejected": -258.1268005371094,
+      "loss": 0.485,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 21.72829246520996,
+      "rewards/margins": 10.743375778198242,
+      "rewards/rejected": 10.984918594360352,
       "step": 440
     },
     {
       "epoch": 0.94,
-      "grad_norm": 10.779162534358996,
+      "grad_norm": 29.684061550729,
       "learning_rate": 5.212833302556258e-09,
-      "logits/chosen": -2.598240375518799,
-      "logits/rejected": -2.6028037071228027,
-      "logps/chosen": -259.9753112792969,
-      "logps/rejected": -276.95166015625,
-      "loss": 0.2836,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 33.70884323120117,
-      "rewards/margins": 3.5860488414764404,
-      "rewards/rejected": 30.122793197631836,
+      "logits/chosen": -2.6331818103790283,
+      "logits/rejected": -2.646833658218384,
+      "logps/chosen": -273.6771240234375,
+      "logps/rejected": -299.3607482910156,
+      "loss": 0.4824,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 20.00701904296875,
+      "rewards/margins": 12.293313980102539,
+      "rewards/rejected": 7.7137041091918945,
       "step": 450
     },
     {
       "epoch": 0.96,
-      "grad_norm": 12.07874608208951,
+      "grad_norm": 35.062338944611746,
       "learning_rate": 2.158697848236607e-09,
-      "logits/chosen": -2.6384501457214355,
-      "logits/rejected": -2.618943452835083,
-      "logps/chosen": -240.47885131835938,
-      "logps/rejected": -213.6422882080078,
-      "loss": 0.2815,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 31.485698699951172,
-      "rewards/margins": 2.44018292427063,
-      "rewards/rejected": 29.045513153076172,
+      "logits/chosen": -2.686453342437744,
+      "logits/rejected": -2.6655733585357666,
+      "logps/chosen": -252.51998901367188,
+      "logps/rejected": -234.3843231201172,
+      "loss": 0.4908,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 19.44460105895996,
+      "rewards/margins": 11.141111373901367,
+      "rewards/rejected": 8.303489685058594,
       "step": 460
     },
     {
       "epoch": 0.98,
-      "grad_norm": 11.390948919388384,
+      "grad_norm": 32.367340987762965,
       "learning_rate": 4.269029751107489e-10,
-      "logits/chosen": -2.6327641010284424,
-      "logits/rejected": -2.6079437732696533,
-      "logps/chosen": -245.8006591796875,
-      "logps/rejected": -253.76730346679688,
-      "loss": 0.2778,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": 32.898033142089844,
-      "rewards/margins": 4.314266204833984,
-      "rewards/rejected": 28.58376121520996,
+      "logits/chosen": -2.6681036949157715,
+      "logits/rejected": -2.640652656555176,
+      "logps/chosen": -258.857177734375,
+      "logps/rejected": -274.1594543457031,
+      "loss": 0.4653,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 19.84146499633789,
+      "rewards/margins": 11.649839401245117,
+      "rewards/rejected": 8.191625595092773,
       "step": 470
     },
     {
       "epoch": 1.0,
       "step": 478,
       "total_flos": 0.0,
-      "train_loss": 0.31381568898715734,
-      "train_runtime": 7749.4814,
-      "train_samples_per_second": 7.889,
-      "train_steps_per_second": 0.062
+      "train_loss": 0.5147256711536871,
+      "train_runtime": 7551.132,
+      "train_samples_per_second": 8.096,
+      "train_steps_per_second": 0.063
     }
   ],
   "logging_steps": 10,