Model save

README.md

@@ -2,15 +2,9 @@
 license: apache-2.0
 base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
-- trl
-- dpo
-- generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: zephyr-7b-dpo-full
   results: []
@@ -21,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.2130
-- Rewards/chosen: -2.1752
-- Rewards/rejected: -6.3268
-- Rewards/accuracies: 0.8984
-- Rewards/margins: 4.1516
-- Logps/rejected: -812.6320
-- Logps/chosen: -474.9558
-- Logits/rejected: 1.0727
-- Logits/chosen: -0.3077
+- Loss: 0.2103
+- Rewards/chosen: 4.5125
+- Rewards/rejected: -5.2808
+- Rewards/accuracies: 0.9258
+- Rewards/margins: 9.7933
+- Logps/rejected: -190.5157
+- Logps/chosen: -248.4084
+- Logits/rejected: -2.4965
+- Logits/chosen: -2.5123
 
 ## Model description
 
@@ -68,10 +62,10 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.3617        | 0.21  | 100  | 0.3445          | -1.1381        | -2.9522          | 0.8633             | 1.8140          | -475.1705      | -371.2483    | -1.3192         | -1.5665       |
-| 0.2941        | 0.42  | 200  | 0.2595          | -1.5303        | -4.5965          | 0.8711             | 3.0662          | -639.6045      | -410.4631    | -0.2909         | -1.0255       |
-| 0.259         | 0.63  | 300  | 0.2187          | -2.2257        | -6.1115          | 0.8945             | 3.8858          | -791.1059      | -480.0016    | 1.2573          | -0.0803       |
-| 0.2268        | 0.84  | 400  | 0.2144          | -2.2032        | -6.3258          | 0.8984             | 4.1226          | -812.5331      | -477.7561    | 1.2277          | -0.1965       |
+| 0.3315        | 0.21  | 100  | 0.2975          | 3.4373         | -3.8279          | 0.9023             | 7.2653          | -187.6101      | -250.5587    | -2.5158         | -2.5301       |
+| 0.2909        | 0.42  | 200  | 0.2754          | 4.8618         | -4.1200          | 0.9180             | 8.9818          | -188.1942      | -247.7097    | -2.5310         | -2.5459       |
+| 0.6445        | 0.63  | 300  | 0.2245          | 4.2165         | -5.3713          | 0.9102             | 9.5878          | -190.6968      | -249.0004    | -2.4915         | -2.5059       |
+| 0.2653        | 0.84  | 400  | 0.2103          | 4.5125         | -5.2808          | 0.9258             | 9.7933          | -190.5157      | -248.4084    | -2.4965         | -2.5123       |
 
 
 ### Framework versions

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -0.30772945284843445,
-    "eval_logits/rejected": 1.072676658630371,
-    "eval_logps/chosen": -474.955810546875,
-    "eval_logps/rejected": -812.6320190429688,
-    "eval_loss": 0.21300281584262848,
-    "eval_rewards/accuracies": 0.8984375,
-    "eval_rewards/chosen": -2.175225019454956,
-    "eval_rewards/margins": 4.151553153991699,
-    "eval_rewards/rejected": -6.326778411865234,
-    "eval_runtime": 97.1627,
-    "eval_samples": 2000,
-    "eval_samples_per_second": 20.584,
-    "eval_steps_per_second": 0.329,
-    "train_loss": 0.3183088973476298,
-    "train_runtime": 7617.3811,
+    "train_loss": 0.4018711235732713,
+    "train_runtime": 7633.33,
     "train_samples": 61135,
-    "train_samples_per_second": 8.026,
+    "train_samples_per_second": 8.009,
     "train_steps_per_second": 0.063
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:53ac6df0d03218239512bb011a349cc1a6665ac951834fb76560fada358e9db4
+oid sha256:dfb8492185343c9d55349b5a527c87db359b2d402335a698c56b9e370a925913
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db33c501b889745f07777b00b2c0fd1fd8665c2ff7a19cee38258a0e09a03e03
+oid sha256:ba393e9a2d11b34d07ac9d99ef5e04d768742661ce3b2481b1f60cc95ca2f01b
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b223406133e79f5c8c003c669f1770e612e12d740a68558422c1308ec1945f1
+oid sha256:5d94e5203e43f9e579be385f67b02371d9511906fc38a8db6ea9093ff97e8bee
 size 4540516344

runs/May14_20-23-03_n136-100-194/events.out.tfevents.1715689406.n136-100-194.2851926.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d08383880dffba823bfba035bab3b5b5d47c0ae4af47983d874a57d41ad405c0
-size 35906
+oid sha256:5b21f361a2f1b9226a3d464b4ca00300094d3406bd4598a1725bc34ccf713d40
+size 41076

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.3183088973476298,
-    "train_runtime": 7617.3811,
+    "train_loss": 0.4018711235732713,
+    "train_runtime": 7633.33,
     "train_samples": 61135,
-    "train_samples_per_second": 8.026,
+    "train_samples_per_second": 8.009,
     "train_steps_per_second": 0.063
 }

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 11.649026188613895,
+      "grad_norm": 579.7744748341399,
       "learning_rate": 1.0416666666666666e-08,
       "logits/chosen": -2.408252239227295,
       "logits/rejected": -2.408294677734375,
       "logps/chosen": -208.4792022705078,
       "logps/rejected": -178.0951690673828,
-      "loss": 0.693,
+      "loss": 0.69,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,780 +25,780 @@
     },
     {
       "epoch": 0.02,
-      "grad_norm": 10.900219473552317,
+      "grad_norm": 535.1486260819535,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -2.5447659492492676,
-      "logits/rejected": -2.538891315460205,
-      "logps/chosen": -261.517333984375,
-      "logps/rejected": -166.39056396484375,
-      "loss": 0.693,
-      "rewards/accuracies": 0.4027777910232544,
-      "rewards/chosen": 0.00011150226055178791,
-      "rewards/margins": 5.223603511694819e-05,
-      "rewards/rejected": 5.9266225434839725e-05,
+      "logits/chosen": -2.544614553451538,
+      "logits/rejected": -2.538756847381592,
+      "logps/chosen": -261.5119323730469,
+      "logps/rejected": -166.40280151367188,
+      "loss": 0.7009,
+      "rewards/accuracies": 0.4652777910232544,
+      "rewards/chosen": 0.008291814476251602,
+      "rewards/margins": 0.011442840099334717,
+      "rewards/rejected": -0.003151026088744402,
       "step": 10
     },
     {
       "epoch": 0.04,
-      "grad_norm": 11.586256170716794,
+      "grad_norm": 384.83945719674614,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -2.5193417072296143,
-      "logits/rejected": -2.526468276977539,
-      "logps/chosen": -252.56442260742188,
-      "logps/rejected": -178.0738525390625,
-      "loss": 0.69,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.0030069700442254543,
-      "rewards/margins": 0.006722055375576019,
-      "rewards/rejected": -0.003715085331350565,
+      "logits/chosen": -2.5195038318634033,
+      "logits/rejected": -2.5265002250671387,
+      "logps/chosen": -252.65768432617188,
+      "logps/rejected": -178.087158203125,
+      "loss": 0.5876,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.10371126234531403,
+      "rewards/margins": 0.296107679605484,
+      "rewards/rejected": -0.1923964023590088,
       "step": 20
     },
     {
       "epoch": 0.06,
-      "grad_norm": 37.65056149520632,
+      "grad_norm": 202.13874393663576,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -2.4743459224700928,
-      "logits/rejected": -2.4606471061706543,
-      "logps/chosen": -240.6038818359375,
-      "logps/rejected": -181.88919067382812,
-      "loss": 0.6755,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": 0.008357289247214794,
-      "rewards/margins": 0.03687674552202225,
-      "rewards/rejected": -0.028519460931420326,
+      "logits/chosen": -2.4858596324920654,
+      "logits/rejected": -2.47269606590271,
+      "logps/chosen": -240.64242553710938,
+      "logps/rejected": -181.0841522216797,
+      "loss": 0.3862,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 0.3985957205295563,
+      "rewards/margins": 1.4220483303070068,
+      "rewards/rejected": -1.023452639579773,
       "step": 30
     },
     {
       "epoch": 0.08,
-      "grad_norm": 11.450052464046935,
+      "grad_norm": 142.19955934070376,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.3846631050109863,
-      "logits/rejected": -2.3543992042541504,
-      "logps/chosen": -280.9198913574219,
-      "logps/rejected": -211.95913696289062,
-      "loss": 0.6346,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.06864231824874878,
-      "rewards/margins": 0.12311089038848877,
-      "rewards/rejected": -0.05446857959032059,
+      "logits/chosen": -2.4594264030456543,
+      "logits/rejected": -2.4308407306671143,
+      "logps/chosen": -283.8061828613281,
+      "logps/rejected": -208.5924835205078,
+      "loss": 0.3332,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 1.9889726638793945,
+      "rewards/margins": 3.029081344604492,
+      "rewards/rejected": -1.0401084423065186,
       "step": 40
     },
     {
       "epoch": 0.1,
-      "grad_norm": 13.049787778502237,
+      "grad_norm": 713.6549875009309,
       "learning_rate": 4.999733114418725e-07,
-      "logits/chosen": -2.256699323654175,
-      "logits/rejected": -2.269990921020508,
-      "logps/chosen": -250.5127716064453,
-      "logps/rejected": -210.73342895507812,
-      "loss": 0.5816,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": 0.005404283292591572,
-      "rewards/margins": 0.28600800037384033,
-      "rewards/rejected": -0.2806037366390228,
+      "logits/chosen": -2.442108631134033,
+      "logits/rejected": -2.46040678024292,
+      "logps/chosen": -246.4988555908203,
+      "logps/rejected": -187.14138793945312,
+      "loss": 0.4454,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": 2.2771620750427246,
+      "rewards/margins": 4.51132345199585,
+      "rewards/rejected": -2.234161853790283,
       "step": 50
     },
     {
       "epoch": 0.13,
-      "grad_norm": 23.40921657971245,
+      "grad_norm": 206.28301377298783,
       "learning_rate": 4.990398100856366e-07,
-      "logits/chosen": -2.1533896923065186,
-      "logits/rejected": -2.1007308959960938,
-      "logps/chosen": -282.63739013671875,
-      "logps/rejected": -243.5396270751953,
-      "loss": 0.5257,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -0.08040512353181839,
-      "rewards/margins": 0.459780216217041,
-      "rewards/rejected": -0.540185272693634,
+      "logits/chosen": -2.4876925945281982,
+      "logits/rejected": -2.449016571044922,
+      "logps/chosen": -271.1671142578125,
+      "logps/rejected": -196.48915100097656,
+      "loss": 0.282,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 1.714887261390686,
+      "rewards/margins": 5.19891881942749,
+      "rewards/rejected": -3.4840316772460938,
       "step": 60
     },
     {
       "epoch": 0.15,
-      "grad_norm": 18.316826712401,
+      "grad_norm": 133.99852927659347,
       "learning_rate": 4.967775735898179e-07,
-      "logits/chosen": -2.008669137954712,
-      "logits/rejected": -2.016932249069214,
-      "logps/chosen": -298.5509338378906,
-      "logps/rejected": -281.0208435058594,
-      "loss": 0.4813,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -0.18154878914356232,
-      "rewards/margins": 0.7048229575157166,
-      "rewards/rejected": -0.8863717317581177,
+      "logits/chosen": -2.4287192821502686,
+      "logits/rejected": -2.4621310234069824,
+      "logps/chosen": -274.3385925292969,
+      "logps/rejected": -198.64663696289062,
+      "loss": 0.3118,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 3.02872371673584,
+      "rewards/margins": 6.160218238830566,
+      "rewards/rejected": -3.1314942836761475,
       "step": 70
     },
     {
       "epoch": 0.17,
-      "grad_norm": 17.640568045589177,
+      "grad_norm": 274.89332519675594,
       "learning_rate": 4.931986719649298e-07,
-      "logits/chosen": -1.7213737964630127,
-      "logits/rejected": -1.639634370803833,
-      "logps/chosen": -321.4610900878906,
-      "logps/rejected": -327.48675537109375,
-      "loss": 0.4345,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -0.5062848925590515,
-      "rewards/margins": 0.9567643404006958,
-      "rewards/rejected": -1.4630491733551025,
+      "logits/chosen": -2.42030668258667,
+      "logits/rejected": -2.423424243927002,
+      "logps/chosen": -263.48272705078125,
+      "logps/rejected": -186.29991149902344,
+      "loss": 0.3017,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 3.6749210357666016,
+      "rewards/margins": 6.233966827392578,
+      "rewards/rejected": -2.559046506881714,
       "step": 80
     },
     {
       "epoch": 0.19,
-      "grad_norm": 29.13663512751633,
+      "grad_norm": 202.07196994453352,
       "learning_rate": 4.883222001996351e-07,
-      "logits/chosen": -1.2982016801834106,
-      "logits/rejected": -1.1025583744049072,
-      "logps/chosen": -372.08209228515625,
-      "logps/rejected": -424.2044372558594,
-      "loss": 0.4001,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.9675124287605286,
-      "rewards/margins": 1.4019938707351685,
-      "rewards/rejected": -2.369506359100342,
+      "logits/chosen": -2.4479732513427734,
+      "logits/rejected": -2.4088387489318848,
+      "logps/chosen": -267.5174255371094,
+      "logps/rejected": -193.828125,
+      "loss": 0.3603,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 3.906665802001953,
+      "rewards/margins": 7.1938323974609375,
+      "rewards/rejected": -3.2871665954589844,
       "step": 90
     },
     {
       "epoch": 0.21,
-      "grad_norm": 20.753196262748457,
+      "grad_norm": 260.799637557766,
       "learning_rate": 4.821741763807186e-07,
-      "logits/chosen": -1.386314034461975,
-      "logits/rejected": -1.0822856426239014,
-      "logps/chosen": -367.71942138671875,
-      "logps/rejected": -453.9052734375,
-      "loss": 0.3617,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -1.3193985223770142,
-      "rewards/margins": 1.4559285640716553,
-      "rewards/rejected": -2.77532696723938,
+      "logits/chosen": -2.4610652923583984,
+      "logits/rejected": -2.454207181930542,
+      "logps/chosen": -231.1355743408203,
+      "logps/rejected": -183.55123901367188,
+      "loss": 0.3315,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 2.3220086097717285,
+      "rewards/margins": 5.911337852478027,
+      "rewards/rejected": -3.589329242706299,
       "step": 100
     },
     {
       "epoch": 0.21,
-      "eval_logits/chosen": -1.5665340423583984,
-      "eval_logits/rejected": -1.3192166090011597,
-      "eval_logps/chosen": -371.2482604980469,
-      "eval_logps/rejected": -475.1705017089844,
-      "eval_loss": 0.34448927640914917,
-      "eval_rewards/accuracies": 0.86328125,
-      "eval_rewards/chosen": -1.138149380683899,
-      "eval_rewards/margins": 1.8140134811401367,
-      "eval_rewards/rejected": -2.952162981033325,
-      "eval_runtime": 97.3413,
-      "eval_samples_per_second": 20.546,
-      "eval_steps_per_second": 0.329,
+      "eval_logits/chosen": -2.5300614833831787,
+      "eval_logits/rejected": -2.515779495239258,
+      "eval_logps/chosen": -250.55865478515625,
+      "eval_logps/rejected": -187.6100616455078,
+      "eval_loss": 0.297485888004303,
+      "eval_rewards/accuracies": 0.90234375,
+      "eval_rewards/chosen": 3.437309980392456,
+      "eval_rewards/margins": 7.265252113342285,
+      "eval_rewards/rejected": -3.82794189453125,
+      "eval_runtime": 97.5532,
+      "eval_samples_per_second": 20.502,
+      "eval_steps_per_second": 0.328,
       "step": 100
     },
     {
       "epoch": 0.23,
-      "grad_norm": 18.97759709132058,
+      "grad_norm": 164.61060726491735,
       "learning_rate": 4.747874028753375e-07,
-      "logits/chosen": -1.1811072826385498,
-      "logits/rejected": -0.979759693145752,
-      "logps/chosen": -376.93353271484375,
-      "logps/rejected": -487.2027282714844,
-      "loss": 0.3614,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.3289892673492432,
-      "rewards/margins": 1.6757100820541382,
-      "rewards/rejected": -3.004699230194092,
+      "logits/chosen": -2.36721134185791,
+      "logits/rejected": -2.398892879486084,
+      "logps/chosen": -238.1865692138672,
+      "logps/rejected": -193.205810546875,
+      "loss": 0.3602,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 2.924015522003174,
+      "rewards/margins": 6.160517692565918,
+      "rewards/rejected": -3.2365028858184814,
       "step": 110
     },
     {
       "epoch": 0.25,
-      "grad_norm": 15.72680069184703,
+      "grad_norm": 394.19842542709944,
       "learning_rate": 4.662012913161997e-07,
-      "logits/chosen": -0.8997787237167358,
-      "logits/rejected": -0.6056855916976929,
-      "logps/chosen": -377.9947509765625,
-      "logps/rejected": -490.7288513183594,
-      "loss": 0.3357,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.3285906314849854,
-      "rewards/margins": 1.7274820804595947,
-      "rewards/rejected": -3.056072473526001,
+      "logits/chosen": -2.4796204566955566,
+      "logits/rejected": -2.462428331375122,
+      "logps/chosen": -237.8124237060547,
+      "logps/rejected": -191.6023712158203,
+      "loss": 0.2323,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 3.661634922027588,
+      "rewards/margins": 6.902056694030762,
+      "rewards/rejected": -3.240421772003174,
       "step": 120
     },
     {
       "epoch": 0.27,
-      "grad_norm": 22.07740307319267,
+      "grad_norm": 409.3218341794219,
       "learning_rate": 4.5646165232345103e-07,
-      "logits/chosen": -0.7402850389480591,
-      "logits/rejected": -0.2074509561061859,
-      "logps/chosen": -404.84442138671875,
-      "logps/rejected": -538.0674438476562,
-      "loss": 0.3035,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": -1.3628482818603516,
-      "rewards/margins": 2.222651720046997,
-      "rewards/rejected": -3.5854995250701904,
+      "logits/chosen": -2.4416961669921875,
+      "logits/rejected": -2.473330497741699,
+      "logps/chosen": -259.2460632324219,
+      "logps/rejected": -185.64683532714844,
+      "loss": 0.3435,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 4.6567606925964355,
+      "rewards/margins": 7.721456050872803,
+      "rewards/rejected": -3.064694881439209,
       "step": 130
     },
     {
       "epoch": 0.29,
-      "grad_norm": 23.838607690547235,
+      "grad_norm": 328.4389039905761,
       "learning_rate": 4.456204510851956e-07,
-      "logits/chosen": -0.677836537361145,
-      "logits/rejected": -0.11348800361156464,
-      "logps/chosen": -422.9839782714844,
-      "logps/rejected": -556.4301147460938,
-      "loss": 0.3117,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.539533257484436,
-      "rewards/margins": 2.1120970249176025,
-      "rewards/rejected": -3.651630401611328,
+      "logits/chosen": -2.326636791229248,
+      "logits/rejected": -2.3331127166748047,
+      "logps/chosen": -261.88006591796875,
+      "logps/rejected": -197.84036254882812,
+      "loss": 0.2942,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 3.5752804279327393,
+      "rewards/margins": 6.8619065284729,
+      "rewards/rejected": -3.2866263389587402,
       "step": 140
     },
     {
       "epoch": 0.31,
-      "grad_norm": 27.05325199994427,
+      "grad_norm": 664.6717326413732,
       "learning_rate": 4.337355301007335e-07,
-      "logits/chosen": -0.3853974938392639,
-      "logits/rejected": 0.20319394767284393,
-      "logps/chosen": -417.6180725097656,
-      "logps/rejected": -582.4807739257812,
-      "loss": 0.3038,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -1.7072023153305054,
-      "rewards/margins": 2.395305871963501,
-      "rewards/rejected": -4.102508068084717,
+      "logits/chosen": -2.4599616527557373,
+      "logits/rejected": -2.4934587478637695,
+      "logps/chosen": -239.1993865966797,
+      "logps/rejected": -179.16249084472656,
+      "loss": 0.4201,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 3.849210262298584,
+      "rewards/margins": 7.315499782562256,
+      "rewards/rejected": -3.46628999710083,
       "step": 150
     },
     {
       "epoch": 0.33,
-      "grad_norm": 17.832180625180946,
+      "grad_norm": 389.31781863733255,
       "learning_rate": 4.2087030056579986e-07,
-      "logits/chosen": -0.3905831575393677,
-      "logits/rejected": 0.32560938596725464,
-      "logps/chosen": -401.91839599609375,
-      "logps/rejected": -541.798583984375,
-      "loss": 0.291,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -1.3554723262786865,
-      "rewards/margins": 2.1516823768615723,
-      "rewards/rejected": -3.507154941558838,
+      "logits/chosen": -2.4196219444274902,
+      "logits/rejected": -2.3700897693634033,
+      "logps/chosen": -260.79736328125,
+      "logps/rejected": -198.62042236328125,
+      "loss": 0.4699,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 2.786912441253662,
+      "rewards/margins": 6.555578708648682,
+      "rewards/rejected": -3.768665313720703,
       "step": 160
     },
     {
       "epoch": 0.36,
-      "grad_norm": 33.740386423043056,
+      "grad_norm": 450.98983026990686,
       "learning_rate": 4.070934040463998e-07,
-      "logits/chosen": -0.7972911596298218,
-      "logits/rejected": 0.013139176182448864,
-      "logps/chosen": -431.9474182128906,
-      "logps/rejected": -614.335205078125,
-      "loss": 0.2818,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -1.7542915344238281,
-      "rewards/margins": 2.622352361679077,
-      "rewards/rejected": -4.376644134521484,
+      "logits/chosen": -2.5323286056518555,
+      "logits/rejected": -2.534219741821289,
+      "logps/chosen": -251.9270782470703,
+      "logps/rejected": -188.5828094482422,
+      "loss": 0.3003,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 2.2955946922302246,
+      "rewards/margins": 8.251626968383789,
+      "rewards/rejected": -5.956032752990723,
       "step": 170
     },
     {
       "epoch": 0.38,
-      "grad_norm": 36.32706683293822,
+      "grad_norm": 222.8025881040315,
       "learning_rate": 3.9247834624635404e-07,
-      "logits/chosen": -0.6585810780525208,
-      "logits/rejected": 0.35586631298065186,
-      "logps/chosen": -425.5533752441406,
-      "logps/rejected": -627.4122924804688,
-      "loss": 0.2821,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -1.5755112171173096,
-      "rewards/margins": 2.9501824378967285,
-      "rewards/rejected": -4.525693416595459,
+      "logits/chosen": -2.4943268299102783,
+      "logits/rejected": -2.4797425270080566,
+      "logps/chosen": -261.6493835449219,
+      "logps/rejected": -186.13827514648438,
+      "loss": 0.3893,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 3.1764323711395264,
+      "rewards/margins": 8.824117660522461,
+      "rewards/rejected": -5.6476850509643555,
       "step": 180
     },
     {
       "epoch": 0.4,
-      "grad_norm": 17.303305712606004,
+      "grad_norm": 170.38903025294675,
       "learning_rate": 3.7710310482256523e-07,
-      "logits/chosen": 0.05398033186793327,
-      "logits/rejected": 0.8622593879699707,
-      "logps/chosen": -430.6288146972656,
-      "logps/rejected": -650.1162109375,
-      "loss": 0.2857,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -1.835845947265625,
-      "rewards/margins": 2.724783420562744,
-      "rewards/rejected": -4.560629844665527,
+      "logits/chosen": -2.4573581218719482,
+      "logits/rejected": -2.4471302032470703,
+      "logps/chosen": -242.3362274169922,
+      "logps/rejected": -206.1191864013672,
+      "loss": 0.2873,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 2.354017496109009,
+      "rewards/margins": 8.386995315551758,
+      "rewards/rejected": -6.0329766273498535,
       "step": 190
     },
     {
       "epoch": 0.42,
-      "grad_norm": 25.750253047861463,
+      "grad_norm": 233.638489796621,
       "learning_rate": 3.610497133404795e-07,
-      "logits/chosen": -0.71821129322052,
-      "logits/rejected": 0.2892279028892517,
-      "logps/chosen": -424.61773681640625,
-      "logps/rejected": -588.4847412109375,
-      "loss": 0.2941,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": -1.6481313705444336,
-      "rewards/margins": 2.587430477142334,
-      "rewards/rejected": -4.235561847686768,
+      "logits/chosen": -2.4837393760681152,
+      "logits/rejected": -2.4798505306243896,
+      "logps/chosen": -252.32040405273438,
+      "logps/rejected": -173.71713256835938,
+      "loss": 0.2909,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 3.7421042919158936,
+      "rewards/margins": 8.136415481567383,
+      "rewards/rejected": -4.394310474395752,
       "step": 200
     },
     {
       "epoch": 0.42,
-      "eval_logits/chosen": -1.0254615545272827,
-      "eval_logits/rejected": -0.29090192914009094,
-      "eval_logps/chosen": -410.46307373046875,
-      "eval_logps/rejected": -639.6044921875,
-      "eval_loss": 0.2594895660877228,
-      "eval_rewards/accuracies": 0.87109375,
-      "eval_rewards/chosen": -1.5302979946136475,
-      "eval_rewards/margins": 3.0662055015563965,
-      "eval_rewards/rejected": -4.596503734588623,
-      "eval_runtime": 97.4272,
-      "eval_samples_per_second": 20.528,
-      "eval_steps_per_second": 0.328,
+      "eval_logits/chosen": -2.5459139347076416,
+      "eval_logits/rejected": -2.5309653282165527,
+      "eval_logps/chosen": -247.70968627929688,
+      "eval_logps/rejected": -188.19419860839844,
+      "eval_loss": 0.2753676474094391,
+      "eval_rewards/accuracies": 0.91796875,
+      "eval_rewards/chosen": 4.861801624298096,
+      "eval_rewards/margins": 8.981813430786133,
+      "eval_rewards/rejected": -4.120011806488037,
+      "eval_runtime": 97.2888,
+      "eval_samples_per_second": 20.557,
+      "eval_steps_per_second": 0.329,
       "step": 200
     },
     {
       "epoch": 0.44,
-      "grad_norm": 22.833636886609728,
+      "grad_norm": 337.30781066112155,
       "learning_rate": 3.4440382358952115e-07,
-      "logits/chosen": -0.23435378074645996,
-      "logits/rejected": 0.7262285351753235,
-      "logps/chosen": -413.39495849609375,
-      "logps/rejected": -610.934326171875,
-      "loss": 0.2652,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.7757046222686768,
-      "rewards/margins": 2.6218485832214355,
-      "rewards/rejected": -4.397553443908691,
+      "logits/chosen": -2.4054043292999268,
+      "logits/rejected": -2.334240436553955,
+      "logps/chosen": -229.5664825439453,
+      "logps/rejected": -177.37989807128906,
+      "loss": 0.3982,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 3.1289913654327393,
+      "rewards/margins": 6.229460716247559,
+      "rewards/rejected": -3.100468873977661,
       "step": 210
     },
     {
       "epoch": 0.46,
-      "grad_norm": 19.35747892667393,
+      "grad_norm": 196.23824528381684,
       "learning_rate": 3.272542485937368e-07,
-      "logits/chosen": -0.03855214640498161,
-      "logits/rejected": 1.1163934469223022,
-      "logps/chosen": -422.90655517578125,
-      "logps/rejected": -638.5581665039062,
-      "loss": 0.2649,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -1.665327787399292,
-      "rewards/margins": 2.9846928119659424,
-      "rewards/rejected": -4.650020122528076,
+      "logits/chosen": -2.4668684005737305,
+      "logits/rejected": -2.4853501319885254,
+      "logps/chosen": -247.2740478515625,
+      "logps/rejected": -182.8220977783203,
+      "loss": 0.564,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 4.549849510192871,
+      "rewards/margins": 9.18285083770752,
+      "rewards/rejected": -4.633000373840332,
       "step": 220
     },
     {
       "epoch": 0.48,
-      "grad_norm": 21.48535535733013,
+      "grad_norm": 188.59878410714853,
       "learning_rate": 3.096924887558854e-07,
-      "logits/chosen": 0.25134754180908203,
-      "logits/rejected": 1.690708875656128,
-      "logps/chosen": -465.64324951171875,
-      "logps/rejected": -718.0191650390625,
-      "loss": 0.2576,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -1.8850457668304443,
-      "rewards/margins": 3.3224689960479736,
-      "rewards/rejected": -5.20751428604126,
+      "logits/chosen": -2.4467105865478516,
+      "logits/rejected": -2.394397735595703,
+      "logps/chosen": -269.22100830078125,
+      "logps/rejected": -207.54287719726562,
+      "loss": 0.61,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 3.9587960243225098,
+      "rewards/margins": 9.096400260925293,
+      "rewards/rejected": -5.137604713439941,
       "step": 230
     },
     {
       "epoch": 0.5,
-      "grad_norm": 21.422183287654935,
+      "grad_norm": 248.56418253631352,
       "learning_rate": 2.9181224366319943e-07,
-      "logits/chosen": 0.8468208312988281,
-      "logits/rejected": 2.1126351356506348,
-      "logps/chosen": -494.220703125,
-      "logps/rejected": -765.5153198242188,
-      "loss": 0.2597,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -2.599130153656006,
-      "rewards/margins": 3.3152058124542236,
-      "rewards/rejected": -5.914336204528809,
+      "logits/chosen": -2.438028335571289,
+      "logits/rejected": -2.4430088996887207,
+      "logps/chosen": -228.5868377685547,
+      "logps/rejected": -184.39010620117188,
+      "loss": 0.2929,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 2.860379934310913,
+      "rewards/margins": 8.014602661132812,
+      "rewards/rejected": -5.15422248840332,
       "step": 240
     },
     {
       "epoch": 0.52,
-      "grad_norm": 20.426996979966383,
+      "grad_norm": 236.10656708079867,
       "learning_rate": 2.7370891215954565e-07,
-      "logits/chosen": -0.12486964464187622,
-      "logits/rejected": 1.0274794101715088,
-      "logps/chosen": -427.1630859375,
-      "logps/rejected": -634.4889526367188,
-      "loss": 0.2648,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": -1.721609115600586,
-      "rewards/margins": 2.7955689430236816,
-      "rewards/rejected": -4.517177581787109,
+      "logits/chosen": -2.468085527420044,
+      "logits/rejected": -2.471389055252075,
+      "logps/chosen": -245.74221801757812,
+      "logps/rejected": -189.75096130371094,
+      "loss": 0.2793,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 4.629996299743652,
+      "rewards/margins": 8.119891166687012,
+      "rewards/rejected": -3.489894151687622,
       "step": 250
     },
     {
       "epoch": 0.54,
-      "grad_norm": 25.70991400557029,
+      "grad_norm": 536.510359732396,
       "learning_rate": 2.55479083351317e-07,
-      "logits/chosen": -0.1112385243177414,
-      "logits/rejected": 1.0832737684249878,
-      "logps/chosen": -449.45709228515625,
-      "logps/rejected": -671.4852294921875,
-      "loss": 0.2494,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -1.8507035970687866,
-      "rewards/margins": 3.2304184436798096,
-      "rewards/rejected": -5.081121921539307,
+      "logits/chosen": -2.4861550331115723,
+      "logits/rejected": -2.4899330139160156,
+      "logps/chosen": -254.806396484375,
+      "logps/rejected": -172.7798614501953,
+      "loss": 0.343,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 4.790152549743652,
+      "rewards/margins": 9.49361515045166,
+      "rewards/rejected": -4.703463077545166,
       "step": 260
     },
     {
       "epoch": 0.56,
-      "grad_norm": 26.743751356983804,
+      "grad_norm": 203.6794572887938,
       "learning_rate": 2.3722002126275822e-07,
-      "logits/chosen": -0.3570239245891571,
-      "logits/rejected": 0.8737660646438599,
-      "logps/chosen": -427.79931640625,
-      "logps/rejected": -680.0591430664062,
-      "loss": 0.2351,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -1.8343093395233154,
-      "rewards/margins": 3.1806130409240723,
-      "rewards/rejected": -5.014922142028809,
+      "logits/chosen": -2.456101179122925,
+      "logits/rejected": -2.407947063446045,
+      "logps/chosen": -236.4749755859375,
+      "logps/rejected": -187.09112548828125,
+      "loss": 0.2939,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 3.9466958045959473,
+      "rewards/margins": 8.208832740783691,
+      "rewards/rejected": -4.262135982513428,
       "step": 270
     },
     {
       "epoch": 0.59,
-      "grad_norm": 20.437919430365636,
+      "grad_norm": 394.1475095150393,
       "learning_rate": 2.19029145890313e-07,
-      "logits/chosen": -0.14857754111289978,
-      "logits/rejected": 0.7723418474197388,
-      "logps/chosen": -476.52099609375,
-      "logps/rejected": -727.02734375,
-      "loss": 0.2542,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -2.1171650886535645,
-      "rewards/margins": 3.235008716583252,
-      "rewards/rejected": -5.352174282073975,
+      "logits/chosen": -2.343445301055908,
+      "logits/rejected": -2.3995354175567627,
+      "logps/chosen": -257.370361328125,
+      "logps/rejected": -201.5979461669922,
+      "loss": 0.3501,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 3.717083692550659,
+      "rewards/margins": 8.61104965209961,
+      "rewards/rejected": -4.893965721130371,
       "step": 280
     },
     {
       "epoch": 0.61,
-      "grad_norm": 18.45732619902188,
+      "grad_norm": 206.90087071186883,
       "learning_rate": 2.0100351342479216e-07,
-      "logits/chosen": -0.04501671344041824,
-      "logits/rejected": 0.9899295568466187,
-      "logps/chosen": -446.81451416015625,
-      "logps/rejected": -742.0169677734375,
-      "loss": 0.2265,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -2.055858850479126,
-      "rewards/margins": 3.6457011699676514,
-      "rewards/rejected": -5.701560020446777,
+      "logits/chosen": -2.4418747425079346,
+      "logits/rejected": -2.428316593170166,
+      "logps/chosen": -234.2547607421875,
+      "logps/rejected": -182.3361053466797,
+      "loss": 0.3955,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 3.4869391918182373,
+      "rewards/margins": 8.724508285522461,
+      "rewards/rejected": -5.2375688552856445,
       "step": 290
     },
     {
       "epoch": 0.63,
-      "grad_norm": 18.065200464485685,
+      "grad_norm": 232.39029956497873,
       "learning_rate": 1.8323929841460178e-07,
-      "logits/chosen": -0.25825151801109314,
-      "logits/rejected": 1.0551276206970215,
-      "logps/chosen": -482.5679626464844,
-      "logps/rejected": -777.6588134765625,
-      "loss": 0.259,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": -2.0962753295898438,
-      "rewards/margins": 3.666637897491455,
-      "rewards/rejected": -5.762913227081299,
+      "logits/chosen": -2.434248208999634,
+      "logits/rejected": -2.357980251312256,
+      "logps/chosen": -265.85308837890625,
+      "logps/rejected": -211.96859741210938,
+      "loss": 0.6445,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 3.5436806678771973,
+      "rewards/margins": 8.844223976135254,
+      "rewards/rejected": -5.300544738769531,
       "step": 300
     },
     {
       "epoch": 0.63,
-      "eval_logits/chosen": -0.08032596111297607,
-      "eval_logits/rejected": 1.257252812385559,
-      "eval_logps/chosen": -480.0015563964844,
-      "eval_logps/rejected": -791.1058959960938,
-      "eval_loss": 0.21872717142105103,
-      "eval_rewards/accuracies": 0.89453125,
-      "eval_rewards/chosen": -2.2256827354431152,
-      "eval_rewards/margins": 3.8858344554901123,
-      "eval_rewards/rejected": -6.111516952514648,
-      "eval_runtime": 97.4838,
-      "eval_samples_per_second": 20.516,
-      "eval_steps_per_second": 0.328,
+      "eval_logits/chosen": -2.5059330463409424,
+      "eval_logits/rejected": -2.491495132446289,
+      "eval_logps/chosen": -249.0003662109375,
+      "eval_logps/rejected": -190.69680786132812,
+      "eval_loss": 0.22447091341018677,
+      "eval_rewards/accuracies": 0.91015625,
+      "eval_rewards/chosen": 4.216455936431885,
+      "eval_rewards/margins": 9.587770462036133,
+      "eval_rewards/rejected": -5.371314525604248,
+      "eval_runtime": 97.2649,
+      "eval_samples_per_second": 20.562,
+      "eval_steps_per_second": 0.329,
       "step": 300
     },
     {
       "epoch": 0.65,
-      "grad_norm": 52.672419540257096,
+      "grad_norm": 216.5700858394941,
       "learning_rate": 1.6583128063291573e-07,
-      "logits/chosen": -0.3141949772834778,
-      "logits/rejected": 0.9004285931587219,
-      "logps/chosen": -475.15936279296875,
-      "logps/rejected": -737.10009765625,
-      "loss": 0.2596,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -2.111830949783325,
-      "rewards/margins": 3.3903605937957764,
-      "rewards/rejected": -5.502191543579102,
+      "logits/chosen": -2.4202470779418945,
+      "logits/rejected": -2.42097806930542,
+      "logps/chosen": -255.98904418945312,
+      "logps/rejected": -196.6630401611328,
+      "loss": 0.3789,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 3.9936203956604004,
+      "rewards/margins": 8.884687423706055,
+      "rewards/rejected": -4.891066551208496,
       "step": 310
     },
     {
       "epoch": 0.67,
-      "grad_norm": 22.65476006635999,
+      "grad_norm": 232.75719945601446,
       "learning_rate": 1.488723393865766e-07,
-      "logits/chosen": -0.16484542191028595,
-      "logits/rejected": 1.090689778327942,
-      "logps/chosen": -455.7721252441406,
-      "logps/rejected": -757.0924072265625,
-      "loss": 0.2429,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": -2.08683180809021,
-      "rewards/margins": 3.6439883708953857,
-      "rewards/rejected": -5.730820178985596,
+      "logits/chosen": -2.493706464767456,
+      "logits/rejected": -2.461465358734131,
+      "logps/chosen": -238.67599487304688,
+      "logps/rejected": -194.2203826904297,
+      "loss": 0.3515,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 4.206465244293213,
+      "rewards/margins": 9.311426162719727,
+      "rewards/rejected": -5.104961395263672,
       "step": 320
     },
     {
       "epoch": 0.69,
-      "grad_norm": 19.80833079454062,
+      "grad_norm": 281.239048485679,
       "learning_rate": 1.3245295796480788e-07,
-      "logits/chosen": -0.27468693256378174,
-      "logits/rejected": 1.061683177947998,
-      "logps/chosen": -481.45501708984375,
-      "logps/rejected": -751.7589721679688,
-      "loss": 0.2279,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -2.233105421066284,
-      "rewards/margins": 3.570781707763672,
-      "rewards/rejected": -5.803887367248535,
+      "logits/chosen": -2.4040043354034424,
+      "logits/rejected": -2.4380762577056885,
+      "logps/chosen": -247.7274169921875,
+      "logps/rejected": -179.15911865234375,
+      "loss": 0.4043,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 5.208529472351074,
+      "rewards/margins": 9.102964401245117,
+      "rewards/rejected": -3.8944339752197266,
       "step": 330
     },
     {
       "epoch": 0.71,
-      "grad_norm": 24.881750772146,
+      "grad_norm": 183.00963279480595,
       "learning_rate": 1.1666074087171627e-07,
-      "logits/chosen": -0.03138185292482376,
-      "logits/rejected": 1.0509425401687622,
-      "logps/chosen": -470.1802673339844,
-      "logps/rejected": -690.557373046875,
-      "loss": 0.237,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -2.2647526264190674,
-      "rewards/margins": 3.035360097885132,
-      "rewards/rejected": -5.300112724304199,
+      "logits/chosen": -2.3740360736846924,
+      "logits/rejected": -2.4260330200195312,
+      "logps/chosen": -235.00357055664062,
+      "logps/rejected": -166.9297637939453,
+      "loss": 0.2751,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 4.350711822509766,
+      "rewards/margins": 7.54253625869751,
+      "rewards/rejected": -3.1918249130249023,
       "step": 340
     },
     {
       "epoch": 0.73,
-      "grad_norm": 19.587851322719366,
+      "grad_norm": 258.2328231454923,
       "learning_rate": 1.0157994641835734e-07,
-      "logits/chosen": 0.006643450353294611,
-      "logits/rejected": 1.0511295795440674,
-      "logps/chosen": -478.51055908203125,
-      "logps/rejected": -806.5072631835938,
-      "loss": 0.2256,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -2.348606586456299,
-      "rewards/margins": 3.6702816486358643,
-      "rewards/rejected": -6.018888473510742,
+      "logits/chosen": -2.4078361988067627,
+      "logits/rejected": -2.4156928062438965,
+      "logps/chosen": -234.7695770263672,
+      "logps/rejected": -214.1270294189453,
+      "loss": 0.3005,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 4.440161228179932,
+      "rewards/margins": 9.19445514678955,
+      "rewards/rejected": -4.754293918609619,
       "step": 350
     },
     {
       "epoch": 0.75,
-      "grad_norm": 22.679114876876472,
+      "grad_norm": 260.2428729882813,
       "learning_rate": 8.729103716819111e-08,
-      "logits/chosen": -0.024983350187540054,
-      "logits/rejected": 1.071063756942749,
-      "logps/chosen": -459.17669677734375,
-      "logps/rejected": -731.5552368164062,
-      "loss": 0.2495,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -2.0910515785217285,
-      "rewards/margins": 3.42362642288208,
-      "rewards/rejected": -5.514677047729492,
+      "logits/chosen": -2.372528076171875,
+      "logits/rejected": -2.3916471004486084,
+      "logps/chosen": -241.23934936523438,
+      "logps/rejected": -187.92953491210938,
+      "loss": 0.2667,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 4.416121482849121,
+      "rewards/margins": 8.337173461914062,
+      "rewards/rejected": -3.921051502227783,
       "step": 360
     },
     {
       "epoch": 0.77,
-      "grad_norm": 24.62831310204124,
+      "grad_norm": 316.09399625633,
       "learning_rate": 7.387025063449081e-08,
-      "logits/chosen": -0.07435999810695648,
-      "logits/rejected": 1.1522341966629028,
-      "logps/chosen": -457.25128173828125,
-      "logps/rejected": -728.47412109375,
-      "loss": 0.234,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": -2.266648292541504,
-      "rewards/margins": 3.339157819747925,
-      "rewards/rejected": -5.60580587387085,
+      "logits/chosen": -2.4834017753601074,
+      "logits/rejected": -2.428774118423462,
+      "logps/chosen": -223.4984893798828,
+      "logps/rejected": -176.99790954589844,
+      "loss": 0.3594,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 3.544004440307617,
+      "rewards/margins": 8.096233367919922,
+      "rewards/rejected": -4.552228927612305,
       "step": 370
     },
     {
       "epoch": 0.79,
-      "grad_norm": 18.56178014954704,
+      "grad_norm": 229.93737644129538,
       "learning_rate": 6.138919252022435e-08,
-      "logits/chosen": -0.3609544634819031,
-      "logits/rejected": 0.9311397671699524,
-      "logps/chosen": -469.6280212402344,
-      "logps/rejected": -802.5978393554688,
-      "loss": 0.2329,
+      "logits/chosen": -2.4690134525299072,
+      "logits/rejected": -2.4799323081970215,
+      "logps/chosen": -249.5747528076172,
+      "logps/rejected": -198.73275756835938,
+      "loss": 0.3121,
       "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": -2.1230244636535645,
-      "rewards/margins": 4.024425506591797,
-      "rewards/rejected": -6.147449016571045,
+      "rewards/chosen": 3.8754029273986816,
+      "rewards/margins": 9.315340042114258,
+      "rewards/rejected": -5.439937591552734,
       "step": 380
     },
     {
       "epoch": 0.82,
-      "grad_norm": 32.926290807333665,
+      "grad_norm": 178.09073107787978,
       "learning_rate": 4.991445467064689e-08,
-      "logits/chosen": -0.012720714323222637,
-      "logits/rejected": 1.3005478382110596,
-      "logps/chosen": -489.65802001953125,
-      "logps/rejected": -779.3419799804688,
-      "loss": 0.2265,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -2.43365740776062,
-      "rewards/margins": 3.560826539993286,
-      "rewards/rejected": -5.994483947753906,
+      "logits/chosen": -2.463972568511963,
+      "logits/rejected": -2.4395322799682617,
+      "logps/chosen": -239.1776123046875,
+      "logps/rejected": -189.71803283691406,
+      "loss": 0.222,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 3.557317018508911,
+      "rewards/margins": 8.469549179077148,
+      "rewards/rejected": -4.912230968475342,
       "step": 390
     },
     {
       "epoch": 0.84,
-      "grad_norm": 30.149198169987162,
+      "grad_norm": 211.56309254204024,
       "learning_rate": 3.9507259776993954e-08,
-      "logits/chosen": 0.26025086641311646,
-      "logits/rejected": 1.3578455448150635,
-      "logps/chosen": -470.298095703125,
-      "logps/rejected": -760.2213134765625,
-      "loss": 0.2268,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -2.2305097579956055,
-      "rewards/margins": 3.6479294300079346,
-      "rewards/rejected": -5.878438472747803,
+      "logits/chosen": -2.3866400718688965,
+      "logits/rejected": -2.4389476776123047,
+      "logps/chosen": -239.34860229492188,
+      "logps/rejected": -181.77676391601562,
+      "loss": 0.2653,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 3.949272632598877,
+      "rewards/margins": 8.648920059204102,
+      "rewards/rejected": -4.699648857116699,
       "step": 400
     },
     {
       "epoch": 0.84,
-      "eval_logits/chosen": -0.19649244844913483,
-      "eval_logits/rejected": 1.227736234664917,
-      "eval_logps/chosen": -477.7560729980469,
-      "eval_logps/rejected": -812.5330810546875,
-      "eval_loss": 0.2144031673669815,
-      "eval_rewards/accuracies": 0.8984375,
-      "eval_rewards/chosen": -2.2032277584075928,
-      "eval_rewards/margins": 4.122560977935791,
-      "eval_rewards/rejected": -6.325788974761963,
-      "eval_runtime": 97.384,
-      "eval_samples_per_second": 20.537,
+      "eval_logits/chosen": -2.512333631515503,
+      "eval_logits/rejected": -2.4965250492095947,
+      "eval_logps/chosen": -248.40835571289062,
+      "eval_logps/rejected": -190.5157470703125,
+      "eval_loss": 0.21030600368976593,
+      "eval_rewards/accuracies": 0.92578125,
+      "eval_rewards/chosen": 4.512471675872803,
+      "eval_rewards/margins": 9.793259620666504,
+      "eval_rewards/rejected": -5.280787944793701,
+      "eval_runtime": 97.2069,
+      "eval_samples_per_second": 20.575,
       "eval_steps_per_second": 0.329,
       "step": 400
     },
     {
       "epoch": 0.86,
-      "grad_norm": 15.686531185394536,
+      "grad_norm": 337.4045735200527,
       "learning_rate": 3.022313472693447e-08,
-      "logits/chosen": -0.13663654029369354,
-      "logits/rejected": 1.1233164072036743,
-      "logps/chosen": -492.34588623046875,
-      "logps/rejected": -787.8416137695312,
-      "loss": 0.2073,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": -2.1595890522003174,
-      "rewards/margins": 3.8674533367156982,
-      "rewards/rejected": -6.027042388916016,
+      "logits/chosen": -2.431990146636963,
+      "logits/rejected": -2.4296875,
+      "logps/chosen": -269.318603515625,
+      "logps/rejected": -194.96946716308594,
+      "loss": 0.6111,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 3.5342013835906982,
+      "rewards/margins": 8.450251579284668,
+      "rewards/rejected": -4.916050910949707,
       "step": 410
     },
     {
       "epoch": 0.88,
-      "grad_norm": 25.36773289746514,
+      "grad_norm": 403.24647604289254,
       "learning_rate": 2.2111614344599684e-08,
-      "logits/chosen": 0.07713554799556732,
-      "logits/rejected": 1.1611943244934082,
-      "logps/chosen": -468.55523681640625,
-      "logps/rejected": -737.3845825195312,
-      "loss": 0.2294,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -2.246133804321289,
-      "rewards/margins": 3.3159337043762207,
-      "rewards/rejected": -5.562067985534668,
+      "logits/chosen": -2.39760684967041,
+      "logits/rejected": -2.376661777496338,
+      "logps/chosen": -237.7588653564453,
+      "logps/rejected": -190.60279846191406,
+      "loss": 0.3801,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 3.0914576053619385,
+      "rewards/margins": 7.803999423980713,
+      "rewards/rejected": -4.712540626525879,
       "step": 420
     },
     {
       "epoch": 0.9,
-      "grad_norm": 23.64735863277161,
+      "grad_norm": 208.90618387532385,
       "learning_rate": 1.521597710086439e-08,
-      "logits/chosen": -0.11199776083230972,
-      "logits/rejected": 1.2544041872024536,
-      "logps/chosen": -487.00787353515625,
-      "logps/rejected": -748.0345458984375,
-      "loss": 0.2283,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -2.158412218093872,
-      "rewards/margins": 3.493891954421997,
-      "rewards/rejected": -5.652304649353027,
+      "logits/chosen": -2.4121103286743164,
+      "logits/rejected": -2.3859496116638184,
+      "logps/chosen": -263.43292236328125,
+      "logps/rejected": -191.71656799316406,
+      "loss": 0.3449,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 3.8668575286865234,
+      "rewards/margins": 8.32303524017334,
+      "rewards/rejected": -4.456177234649658,
       "step": 430
     },
     {
       "epoch": 0.92,
-      "grad_norm": 32.07046590554996,
+      "grad_norm": 221.6697110810539,
       "learning_rate": 9.57301420397924e-09,
-      "logits/chosen": -0.2779064476490021,
-      "logits/rejected": 0.990594744682312,
-      "logps/chosen": -476.5415954589844,
-      "logps/rejected": -816.4949340820312,
-      "loss": 0.2335,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -2.1302719116210938,
-      "rewards/margins": 4.236142158508301,
-      "rewards/rejected": -6.366414546966553,
+      "logits/chosen": -2.5127618312835693,
+      "logits/rejected": -2.4956202507019043,
+      "logps/chosen": -254.6200408935547,
+      "logps/rejected": -190.78817749023438,
+      "loss": 0.3624,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 4.447186470031738,
+      "rewards/margins": 9.914505004882812,
+      "rewards/rejected": -5.467317581176758,
       "step": 440
     },
     {
       "epoch": 0.94,
-      "grad_norm": 29.966664104398923,
+      "grad_norm": 304.27875154546507,
       "learning_rate": 5.212833302556258e-09,
-      "logits/chosen": -0.2724097967147827,
-      "logits/rejected": 1.02344810962677,
-      "logps/chosen": -477.5227966308594,
-      "logps/rejected": -801.6556396484375,
-      "loss": 0.235,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -2.263077735900879,
-      "rewards/margins": 3.993912935256958,
-      "rewards/rejected": -6.256990432739258,
+      "logits/chosen": -2.5326547622680664,
+      "logits/rejected": -2.4754910469055176,
+      "logps/chosen": -242.94784545898438,
+      "logps/rejected": -187.10147094726562,
+      "loss": 0.2887,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 4.133586406707764,
+      "rewards/margins": 9.706035614013672,
+      "rewards/rejected": -5.572449207305908,
       "step": 450
     },
     {
       "epoch": 0.96,
-      "grad_norm": 54.30402743121173,
+      "grad_norm": 480.62571484354635,
       "learning_rate": 2.158697848236607e-09,
-      "logits/chosen": -0.18335244059562683,
-      "logits/rejected": 0.9830573201179504,
-      "logps/chosen": -479.820556640625,
-      "logps/rejected": -772.7644653320312,
-      "loss": 0.2201,
+      "logits/chosen": -2.419916868209839,
+      "logits/rejected": -2.4316189289093018,
+      "logps/chosen": -255.69723510742188,
+      "logps/rejected": -203.59130859375,
+      "loss": 0.4414,
       "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": -2.1699745655059814,
-      "rewards/margins": 3.6132774353027344,
-      "rewards/rejected": -5.783252239227295,
+      "rewards/chosen": 3.5629382133483887,
+      "rewards/margins": 8.139005661010742,
+      "rewards/rejected": -4.576067924499512,
       "step": 460
     },
     {
       "epoch": 0.98,
-      "grad_norm": 24.80298516842009,
+      "grad_norm": 258.3454560672873,
       "learning_rate": 4.269029751107489e-10,
-      "logits/chosen": -0.0621672198176384,
-      "logits/rejected": 1.1333558559417725,
-      "logps/chosen": -449.02386474609375,
-      "logps/rejected": -767.277099609375,
-      "loss": 0.2269,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -2.1144473552703857,
-      "rewards/margins": 3.668858289718628,
-      "rewards/rejected": -5.783305644989014,
+      "logits/chosen": -2.416884660720825,
+      "logits/rejected": -2.3896336555480957,
+      "logps/chosen": -230.7972412109375,
+      "logps/rejected": -200.25830078125,
+      "loss": 0.2781,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 3.390946865081787,
+      "rewards/margins": 9.046823501586914,
+      "rewards/rejected": -5.655877113342285,
       "step": 470
     },
     {
       "epoch": 1.0,
       "step": 478,
       "total_flos": 0.0,
-      "train_loss": 0.3183088973476298,
-      "train_runtime": 7617.3811,
-      "train_samples_per_second": 8.026,
+      "train_loss": 0.4018711235732713,
+      "train_runtime": 7633.33,
+      "train_samples_per_second": 8.009,
       "train_steps_per_second": 0.063
     }
   ],