Model save

README.md

@@ -1,4 +1,6 @@
 ---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
 - trl
 - dpo
@@ -13,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model was trained from scratch on the None dataset.
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.1592
-- Rewards/chosen: 8.1420
-- Rewards/rejected: -16.5137
-- Rewards/accuracies: 0.9219
-- Rewards/margins: 24.6558
-- Logps/rejected: -126.2682
-- Logps/chosen: -121.4882
-- Logits/rejected: -1.8539
-- Logits/chosen: -1.8512
+- Loss: 0.2144
+- Rewards/chosen: -2.2032
+- Rewards/rejected: -6.3258
+- Rewards/accuracies: 0.8984
+- Rewards/margins: 4.1226
+- Logps/rejected: -812.5331
+- Logps/chosen: -477.7561
+- Logits/rejected: 1.2277
+- Logits/chosen: -0.1965
 
 ## Model description
 
@@ -60,10 +62,10 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.1483        | 0.21  | 100  | 0.1321          | 6.9750         | -14.0592         | 0.9336             | 21.0342         | -121.3591      | -123.8222    | -1.7705         | -1.7813       |
-| 0.2084        | 0.42  | 200  | 0.1510          | 8.5839         | -14.8837         | 0.9375             | 23.4677         | -123.0082      | -120.6044    | -1.8622         | -1.8630       |
-| 0.2072        | 0.63  | 300  | 0.1658          | 7.7715         | -16.5062         | 0.9219             | 24.2778         | -126.2532      | -122.2292    | -1.8451         | -1.8438       |
-| 0.1918        | 0.84  | 400  | 0.1592          | 8.1420         | -16.5137         | 0.9219             | 24.6558         | -126.2682      | -121.4882    | -1.8539         | -1.8512       |
+| 0.3617        | 0.21  | 100  | 0.3445          | -1.1381        | -2.9522          | 0.8633             | 1.8140          | -475.1705      | -371.2483    | -1.3192         | -1.5665       |
+| 0.2941        | 0.42  | 200  | 0.2595          | -1.5303        | -4.5965          | 0.8711             | 3.0662          | -639.6045      | -410.4631    | -0.2909         | -1.0255       |
+| 0.259         | 0.63  | 300  | 0.2187          | -2.2257        | -6.1115          | 0.8945             | 3.8858          | -791.1059      | -480.0016    | 1.2573          | -0.0803       |
+| 0.2268        | 0.84  | 400  | 0.2144          | -2.2032        | -6.3258          | 0.8984             | 4.1226          | -812.5331      | -477.7561    | 1.2277          | -0.1965       |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.19963366765856244,
-    "train_runtime": 7635.6428,
+    "train_loss": 0.3183088973476298,
+    "train_runtime": 7617.3811,
     "train_samples": 61135,
-    "train_samples_per_second": 8.007,
+    "train_samples_per_second": 8.026,
     "train_steps_per_second": 0.063
 }

generation_config.json

@@ -1,6 +1,6 @@
 {
   "_from_model_config": true,
   "bos_token_id": 1,
-  "eos_token_id": 32000,
+  "eos_token_id": 2,
   "transformers_version": "4.38.2"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:500e19fac2c01478bc3db38253938d7d32fe3a63f7f11d0a98fa9dff084a904e
-size 4943178720
+oid sha256:53ac6df0d03218239512bb011a349cc1a6665ac951834fb76560fada358e9db4
+size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f98248bb1cf0b17228033770296f1cc3c11b5cb0eaaa4555e57ebcd8e5b29fd
+oid sha256:db33c501b889745f07777b00b2c0fd1fd8665c2ff7a19cee38258a0e09a03e03
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c3c37844ea3f77105b46b9f89e25d911069f6a73e947fd25ea4d94229b5ec676
-size 4540532728
+oid sha256:3b223406133e79f5c8c003c669f1770e612e12d740a68558422c1308ec1945f1
+size 4540516344

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 14483496960
+    "total_size": 14483464192
   },
   "weight_map": {
     "lm_head.weight": "model-00003-of-00003.safetensors",

runs/May14_18-00-17_n136-100-194/events.out.tfevents.1715680842.n136-100-194.2788262.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd7946f0195ba72815d74593c02f703f01e8129d7b556b53ec9b63a312996123
-size 35910
+oid sha256:f0da92e3ef05224c708975a5403f5d29b215358ec58f2d606fdb0281b18037bf
+size 41080

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.19963366765856244,
-    "train_runtime": 7635.6428,
+    "train_loss": 0.3183088973476298,
+    "train_runtime": 7617.3811,
     "train_samples": 61135,
-    "train_samples_per_second": 8.007,
+    "train_samples_per_second": 8.026,
     "train_steps_per_second": 0.063
 }

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 2193.4714669768246,
+      "grad_norm": 11.649026188613895,
       "learning_rate": 1.0416666666666666e-08,
-      "logits/chosen": -1.689455509185791,
-      "logits/rejected": -1.4794573783874512,
-      "logps/chosen": -126.21005249023438,
-      "logps/rejected": -98.13133239746094,
-      "loss": 0.6931,
+      "logits/chosen": -2.408252239227295,
+      "logits/rejected": -2.408294677734375,
+      "logps/chosen": -208.4792022705078,
+      "logps/rejected": -178.0951690673828,
+      "loss": 0.693,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,780 +25,780 @@
     },
     {
       "epoch": 0.02,
-      "grad_norm": 1601.7083484310278,
+      "grad_norm": 10.900219473552317,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -1.7075374126434326,
-      "logits/rejected": -1.610562801361084,
-      "logps/chosen": -139.61968994140625,
-      "logps/rejected": -91.38065338134766,
-      "loss": 0.6802,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.051607053726911545,
-      "rewards/margins": 0.09388630837202072,
-      "rewards/rejected": -0.042279258370399475,
+      "logits/chosen": -2.5447659492492676,
+      "logits/rejected": -2.538891315460205,
+      "logps/chosen": -261.517333984375,
+      "logps/rejected": -166.39056396484375,
+      "loss": 0.693,
+      "rewards/accuracies": 0.4027777910232544,
+      "rewards/chosen": 0.00011150226055178791,
+      "rewards/margins": 5.223603511694819e-05,
+      "rewards/rejected": 5.9266225434839725e-05,
       "step": 10
     },
     {
       "epoch": 0.04,
-      "grad_norm": 542.7830677289153,
+      "grad_norm": 11.586256170716794,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -1.639533281326294,
-      "logits/rejected": -1.6490185260772705,
-      "logps/chosen": -130.57711791992188,
-      "logps/rejected": -94.0321044921875,
-      "loss": 0.4027,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 1.0487556457519531,
-      "rewards/margins": 1.4229849576950073,
-      "rewards/rejected": -0.3742292523384094,
+      "logits/chosen": -2.5193417072296143,
+      "logits/rejected": -2.526468276977539,
+      "logps/chosen": -252.56442260742188,
+      "logps/rejected": -178.0738525390625,
+      "loss": 0.69,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.0030069700442254543,
+      "rewards/margins": 0.006722055375576019,
+      "rewards/rejected": -0.003715085331350565,
       "step": 20
     },
     {
       "epoch": 0.06,
-      "grad_norm": 556.5878829215873,
+      "grad_norm": 37.65056149520632,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -1.6960318088531494,
-      "logits/rejected": -1.6311280727386475,
-      "logps/chosen": -131.44830322265625,
-      "logps/rejected": -105.1076431274414,
-      "loss": 0.2285,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 3.3746559619903564,
-      "rewards/margins": 5.053637981414795,
-      "rewards/rejected": -1.6789820194244385,
+      "logits/chosen": -2.4743459224700928,
+      "logits/rejected": -2.4606471061706543,
+      "logps/chosen": -240.6038818359375,
+      "logps/rejected": -181.88919067382812,
+      "loss": 0.6755,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.008357289247214794,
+      "rewards/margins": 0.03687674552202225,
+      "rewards/rejected": -0.028519460931420326,
       "step": 30
     },
     {
       "epoch": 0.08,
-      "grad_norm": 453.8019698930014,
+      "grad_norm": 11.450052464046935,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -1.6398160457611084,
-      "logits/rejected": -1.5611240863800049,
-      "logps/chosen": -142.69259643554688,
-      "logps/rejected": -108.96995544433594,
-      "loss": 0.1824,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 4.514187335968018,
-      "rewards/margins": 9.713580131530762,
-      "rewards/rejected": -5.199393272399902,
+      "logits/chosen": -2.3846631050109863,
+      "logits/rejected": -2.3543992042541504,
+      "logps/chosen": -280.9198913574219,
+      "logps/rejected": -211.95913696289062,
+      "loss": 0.6346,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.06864231824874878,
+      "rewards/margins": 0.12311089038848877,
+      "rewards/rejected": -0.05446857959032059,
       "step": 40
     },
     {
       "epoch": 0.1,
-      "grad_norm": 749.5022801510694,
+      "grad_norm": 13.049787778502237,
       "learning_rate": 4.999733114418725e-07,
-      "logits/chosen": -1.6066747903823853,
-      "logits/rejected": -1.6321827173233032,
-      "logps/chosen": -126.60260009765625,
-      "logps/rejected": -116.45572662353516,
-      "loss": 0.1818,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 4.235908508300781,
-      "rewards/margins": 12.905720710754395,
-      "rewards/rejected": -8.66981029510498,
+      "logits/chosen": -2.256699323654175,
+      "logits/rejected": -2.269990921020508,
+      "logps/chosen": -250.5127716064453,
+      "logps/rejected": -210.73342895507812,
+      "loss": 0.5816,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.005404283292591572,
+      "rewards/margins": 0.28600800037384033,
+      "rewards/rejected": -0.2806037366390228,
       "step": 50
     },
     {
       "epoch": 0.13,
-      "grad_norm": 373.1578806081686,
+      "grad_norm": 23.40921657971245,
       "learning_rate": 4.990398100856366e-07,
-      "logits/chosen": -1.713249921798706,
-      "logits/rejected": -1.6583478450775146,
-      "logps/chosen": -141.51318359375,
-      "logps/rejected": -125.83363342285156,
-      "loss": 0.171,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 4.743514537811279,
-      "rewards/margins": 16.781068801879883,
-      "rewards/rejected": -12.037551879882812,
+      "logits/chosen": -2.1533896923065186,
+      "logits/rejected": -2.1007308959960938,
+      "logps/chosen": -282.63739013671875,
+      "logps/rejected": -243.5396270751953,
+      "loss": 0.5257,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.08040512353181839,
+      "rewards/margins": 0.459780216217041,
+      "rewards/rejected": -0.540185272693634,
       "step": 60
     },
     {
       "epoch": 0.15,
-      "grad_norm": 362.49470680909604,
+      "grad_norm": 18.316826712401,
       "learning_rate": 4.967775735898179e-07,
-      "logits/chosen": -1.6701009273529053,
-      "logits/rejected": -1.698505163192749,
-      "logps/chosen": -134.6632537841797,
-      "logps/rejected": -126.83392333984375,
-      "loss": 0.1747,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 5.217758655548096,
-      "rewards/margins": 19.652034759521484,
-      "rewards/rejected": -14.434274673461914,
+      "logits/chosen": -2.008669137954712,
+      "logits/rejected": -2.016932249069214,
+      "logps/chosen": -298.5509338378906,
+      "logps/rejected": -281.0208435058594,
+      "loss": 0.4813,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -0.18154878914356232,
+      "rewards/margins": 0.7048229575157166,
+      "rewards/rejected": -0.8863717317581177,
       "step": 70
     },
     {
       "epoch": 0.17,
-      "grad_norm": 361.796137554066,
+      "grad_norm": 17.640568045589177,
       "learning_rate": 4.931986719649298e-07,
-      "logits/chosen": -1.6567662954330444,
-      "logits/rejected": -1.6445653438568115,
-      "logps/chosen": -128.80026245117188,
-      "logps/rejected": -119.8623046875,
-      "loss": 0.1674,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 5.829800128936768,
-      "rewards/margins": 18.897483825683594,
-      "rewards/rejected": -13.0676851272583,
+      "logits/chosen": -1.7213737964630127,
+      "logits/rejected": -1.639634370803833,
+      "logps/chosen": -321.4610900878906,
+      "logps/rejected": -327.48675537109375,
+      "loss": 0.4345,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.5062848925590515,
+      "rewards/margins": 0.9567643404006958,
+      "rewards/rejected": -1.4630491733551025,
       "step": 80
     },
     {
       "epoch": 0.19,
-      "grad_norm": 567.4822004779005,
+      "grad_norm": 29.13663512751633,
       "learning_rate": 4.883222001996351e-07,
-      "logits/chosen": -1.689691185951233,
-      "logits/rejected": -1.7095468044281006,
-      "logps/chosen": -136.06716918945312,
-      "logps/rejected": -131.6322784423828,
-      "loss": 0.1532,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 6.571531772613525,
-      "rewards/margins": 21.44127655029297,
-      "rewards/rejected": -14.869746208190918,
+      "logits/chosen": -1.2982016801834106,
+      "logits/rejected": -1.1025583744049072,
+      "logps/chosen": -372.08209228515625,
+      "logps/rejected": -424.2044372558594,
+      "loss": 0.4001,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.9675124287605286,
+      "rewards/margins": 1.4019938707351685,
+      "rewards/rejected": -2.369506359100342,
       "step": 90
     },
     {
       "epoch": 0.21,
-      "grad_norm": 313.1047143857541,
+      "grad_norm": 20.753196262748457,
       "learning_rate": 4.821741763807186e-07,
-      "logits/chosen": -1.7361831665039062,
-      "logits/rejected": -1.719515085220337,
-      "logps/chosen": -116.3790054321289,
-      "logps/rejected": -117.08357238769531,
-      "loss": 0.1483,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 5.799386501312256,
-      "rewards/margins": 19.013914108276367,
-      "rewards/rejected": -13.21452808380127,
+      "logits/chosen": -1.386314034461975,
+      "logits/rejected": -1.0822856426239014,
+      "logps/chosen": -367.71942138671875,
+      "logps/rejected": -453.9052734375,
+      "loss": 0.3617,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.3193985223770142,
+      "rewards/margins": 1.4559285640716553,
+      "rewards/rejected": -2.77532696723938,
       "step": 100
     },
     {
       "epoch": 0.21,
-      "eval_logits/chosen": -1.7813222408294678,
-      "eval_logits/rejected": -1.7704529762268066,
-      "eval_logps/chosen": -123.8221664428711,
-      "eval_logps/rejected": -121.35905456542969,
-      "eval_loss": 0.13205325603485107,
-      "eval_rewards/accuracies": 0.93359375,
-      "eval_rewards/chosen": 6.975039482116699,
-      "eval_rewards/margins": 21.034198760986328,
-      "eval_rewards/rejected": -14.059160232543945,
-      "eval_runtime": 97.7776,
-      "eval_samples_per_second": 20.455,
-      "eval_steps_per_second": 0.327,
+      "eval_logits/chosen": -1.5665340423583984,
+      "eval_logits/rejected": -1.3192166090011597,
+      "eval_logps/chosen": -371.2482604980469,
+      "eval_logps/rejected": -475.1705017089844,
+      "eval_loss": 0.34448927640914917,
+      "eval_rewards/accuracies": 0.86328125,
+      "eval_rewards/chosen": -1.138149380683899,
+      "eval_rewards/margins": 1.8140134811401367,
+      "eval_rewards/rejected": -2.952162981033325,
+      "eval_runtime": 97.3413,
+      "eval_samples_per_second": 20.546,
+      "eval_steps_per_second": 0.329,
       "step": 100
     },
     {
       "epoch": 0.23,
-      "grad_norm": 297.03412259661775,
+      "grad_norm": 18.97759709132058,
       "learning_rate": 4.747874028753375e-07,
-      "logits/chosen": -1.6531705856323242,
-      "logits/rejected": -1.7122666835784912,
-      "logps/chosen": -120.41741943359375,
-      "logps/rejected": -124.6469497680664,
-      "loss": 0.1606,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 5.814640998840332,
-      "rewards/margins": 17.050968170166016,
-      "rewards/rejected": -11.236326217651367,
+      "logits/chosen": -1.1811072826385498,
+      "logits/rejected": -0.979759693145752,
+      "logps/chosen": -376.93353271484375,
+      "logps/rejected": -487.2027282714844,
+      "loss": 0.3614,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.3289892673492432,
+      "rewards/margins": 1.6757100820541382,
+      "rewards/rejected": -3.004699230194092,
       "step": 110
     },
     {
       "epoch": 0.25,
-      "grad_norm": 308.6589257383676,
+      "grad_norm": 15.72680069184703,
       "learning_rate": 4.662012913161997e-07,
-      "logits/chosen": -1.7084993124008179,
-      "logits/rejected": -1.699831247329712,
-      "logps/chosen": -118.6161880493164,
-      "logps/rejected": -125.55290222167969,
-      "loss": 0.1532,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 7.1110429763793945,
-      "rewards/margins": 19.216121673583984,
-      "rewards/rejected": -12.105077743530273,
+      "logits/chosen": -0.8997787237167358,
+      "logits/rejected": -0.6056855916976929,
+      "logps/chosen": -377.9947509765625,
+      "logps/rejected": -490.7288513183594,
+      "loss": 0.3357,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -1.3285906314849854,
+      "rewards/margins": 1.7274820804595947,
+      "rewards/rejected": -3.056072473526001,
       "step": 120
     },
     {
       "epoch": 0.27,
-      "grad_norm": 215.2839633523937,
+      "grad_norm": 22.07740307319267,
       "learning_rate": 4.5646165232345103e-07,
-      "logits/chosen": -1.6880385875701904,
-      "logits/rejected": -1.708009123802185,
-      "logps/chosen": -124.7245101928711,
-      "logps/rejected": -121.17189025878906,
-      "loss": 0.1588,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 7.9950103759765625,
-      "rewards/margins": 21.937570571899414,
-      "rewards/rejected": -13.942561149597168,
+      "logits/chosen": -0.7402850389480591,
+      "logits/rejected": -0.2074509561061859,
+      "logps/chosen": -404.84442138671875,
+      "logps/rejected": -538.0674438476562,
+      "loss": 0.3035,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.3628482818603516,
+      "rewards/margins": 2.222651720046997,
+      "rewards/rejected": -3.5854995250701904,
       "step": 130
     },
     {
       "epoch": 0.29,
-      "grad_norm": 353.0707817255302,
+      "grad_norm": 23.838607690547235,
       "learning_rate": 4.456204510851956e-07,
-      "logits/chosen": -1.6008964776992798,
-      "logits/rejected": -1.578404188156128,
-      "logps/chosen": -122.19905853271484,
-      "logps/rejected": -118.03678131103516,
-      "loss": 0.1592,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 7.6850481033325195,
-      "rewards/margins": 20.170154571533203,
-      "rewards/rejected": -12.485105514526367,
+      "logits/chosen": -0.677836537361145,
+      "logits/rejected": -0.11348800361156464,
+      "logps/chosen": -422.9839782714844,
+      "logps/rejected": -556.4301147460938,
+      "loss": 0.3117,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.539533257484436,
+      "rewards/margins": 2.1120970249176025,
+      "rewards/rejected": -3.651630401611328,
       "step": 140
     },
     {
       "epoch": 0.31,
-      "grad_norm": 387.84134751485317,
+      "grad_norm": 27.05325199994427,
       "learning_rate": 4.337355301007335e-07,
-      "logits/chosen": -1.73464035987854,
-      "logits/rejected": -1.7599799633026123,
-      "logps/chosen": -120.38832092285156,
-      "logps/rejected": -115.9221420288086,
-      "loss": 0.1376,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 7.665247917175293,
-      "rewards/margins": 20.418846130371094,
-      "rewards/rejected": -12.753596305847168,
+      "logits/chosen": -0.3853974938392639,
+      "logits/rejected": 0.20319394767284393,
+      "logps/chosen": -417.6180725097656,
+      "logps/rejected": -582.4807739257812,
+      "loss": 0.3038,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -1.7072023153305054,
+      "rewards/margins": 2.395305871963501,
+      "rewards/rejected": -4.102508068084717,
       "step": 150
     },
     {
       "epoch": 0.33,
-      "grad_norm": 695.9762417445575,
+      "grad_norm": 17.832180625180946,
       "learning_rate": 4.2087030056579986e-07,
-      "logits/chosen": -1.6107909679412842,
-      "logits/rejected": -1.5863116979599,
-      "logps/chosen": -127.7414779663086,
-      "logps/rejected": -122.50279235839844,
-      "loss": 0.1702,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 7.1121721267700195,
-      "rewards/margins": 18.89084243774414,
-      "rewards/rejected": -11.778670310974121,
+      "logits/chosen": -0.3905831575393677,
+      "logits/rejected": 0.32560938596725464,
+      "logps/chosen": -401.91839599609375,
+      "logps/rejected": -541.798583984375,
+      "loss": 0.291,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -1.3554723262786865,
+      "rewards/margins": 2.1516823768615723,
+      "rewards/rejected": -3.507154941558838,
       "step": 160
     },
     {
       "epoch": 0.36,
-      "grad_norm": 501.7172276328359,
+      "grad_norm": 33.740386423043056,
       "learning_rate": 4.070934040463998e-07,
-      "logits/chosen": -1.7882769107818604,
-      "logits/rejected": -1.7692320346832275,
-      "logps/chosen": -122.8949966430664,
-      "logps/rejected": -123.34075927734375,
-      "loss": 0.1729,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 8.123270034790039,
-      "rewards/margins": 21.957849502563477,
-      "rewards/rejected": -13.834577560424805,
+      "logits/chosen": -0.7972911596298218,
+      "logits/rejected": 0.013139176182448864,
+      "logps/chosen": -431.9474182128906,
+      "logps/rejected": -614.335205078125,
+      "loss": 0.2818,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -1.7542915344238281,
+      "rewards/margins": 2.622352361679077,
+      "rewards/rejected": -4.376644134521484,
       "step": 170
     },
     {
       "epoch": 0.38,
-      "grad_norm": 517.7468532784978,
+      "grad_norm": 36.32706683293822,
       "learning_rate": 3.9247834624635404e-07,
-      "logits/chosen": -1.7249009609222412,
-      "logits/rejected": -1.7209745645523071,
-      "logps/chosen": -126.01353454589844,
-      "logps/rejected": -118.68614196777344,
-      "loss": 0.1857,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 8.569825172424316,
-      "rewards/margins": 22.478046417236328,
-      "rewards/rejected": -13.908220291137695,
+      "logits/chosen": -0.6585810780525208,
+      "logits/rejected": 0.35586631298065186,
+      "logps/chosen": -425.5533752441406,
+      "logps/rejected": -627.4122924804688,
+      "loss": 0.2821,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -1.5755112171173096,
+      "rewards/margins": 2.9501824378967285,
+      "rewards/rejected": -4.525693416595459,
       "step": 180
     },
     {
       "epoch": 0.4,
-      "grad_norm": 407.33922019202464,
+      "grad_norm": 17.303305712606004,
       "learning_rate": 3.7710310482256523e-07,
-      "logits/chosen": -1.7666248083114624,
-      "logits/rejected": -1.7464529275894165,
-      "logps/chosen": -114.92606353759766,
-      "logps/rejected": -131.9868927001953,
-      "loss": 0.188,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 8.436745643615723,
-      "rewards/margins": 23.382564544677734,
-      "rewards/rejected": -14.945820808410645,
+      "logits/chosen": 0.05398033186793327,
+      "logits/rejected": 0.8622593879699707,
+      "logps/chosen": -430.6288146972656,
+      "logps/rejected": -650.1162109375,
+      "loss": 0.2857,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -1.835845947265625,
+      "rewards/margins": 2.724783420562744,
+      "rewards/rejected": -4.560629844665527,
       "step": 190
     },
     {
       "epoch": 0.42,
-      "grad_norm": 470.1346443625949,
+      "grad_norm": 25.750253047861463,
       "learning_rate": 3.610497133404795e-07,
-      "logits/chosen": -1.750284194946289,
-      "logits/rejected": -1.7601133584976196,
-      "logps/chosen": -120.80158996582031,
-      "logps/rejected": -115.9892807006836,
-      "loss": 0.2084,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 9.167404174804688,
-      "rewards/margins": 23.158000946044922,
-      "rewards/rejected": -13.990595817565918,
+      "logits/chosen": -0.71821129322052,
+      "logits/rejected": 0.2892279028892517,
+      "logps/chosen": -424.61773681640625,
+      "logps/rejected": -588.4847412109375,
+      "loss": 0.2941,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -1.6481313705444336,
+      "rewards/margins": 2.587430477142334,
+      "rewards/rejected": -4.235561847686768,
       "step": 200
     },
     {
       "epoch": 0.42,
-      "eval_logits/chosen": -1.8629834651947021,
-      "eval_logits/rejected": -1.8622163534164429,
-      "eval_logps/chosen": -120.60441589355469,
-      "eval_logps/rejected": -123.00822448730469,
-      "eval_loss": 0.15096141397953033,
-      "eval_rewards/accuracies": 0.9375,
-      "eval_rewards/chosen": 8.583921432495117,
-      "eval_rewards/margins": 23.467666625976562,
-      "eval_rewards/rejected": -14.883745193481445,
-      "eval_runtime": 97.7447,
-      "eval_samples_per_second": 20.461,
-      "eval_steps_per_second": 0.327,
+      "eval_logits/chosen": -1.0254615545272827,
+      "eval_logits/rejected": -0.29090192914009094,
+      "eval_logps/chosen": -410.46307373046875,
+      "eval_logps/rejected": -639.6044921875,
+      "eval_loss": 0.2594895660877228,
+      "eval_rewards/accuracies": 0.87109375,
+      "eval_rewards/chosen": -1.5302979946136475,
+      "eval_rewards/margins": 3.0662055015563965,
+      "eval_rewards/rejected": -4.596503734588623,
+      "eval_runtime": 97.4272,
+      "eval_samples_per_second": 20.528,
+      "eval_steps_per_second": 0.328,
       "step": 200
     },
     {
       "epoch": 0.44,
-      "grad_norm": 460.33381421539644,
+      "grad_norm": 22.833636886609728,
       "learning_rate": 3.4440382358952115e-07,
-      "logits/chosen": -1.7009090185165405,
-      "logits/rejected": -1.6389843225479126,
-      "logps/chosen": -119.08918762207031,
-      "logps/rejected": -112.73565673828125,
-      "loss": 0.1747,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 6.750785827636719,
-      "rewards/margins": 18.317689895629883,
-      "rewards/rejected": -11.566904067993164,
+      "logits/chosen": -0.23435378074645996,
+      "logits/rejected": 0.7262285351753235,
+      "logps/chosen": -413.39495849609375,
+      "logps/rejected": -610.934326171875,
+      "loss": 0.2652,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -1.7757046222686768,
+      "rewards/margins": 2.6218485832214355,
+      "rewards/rejected": -4.397553443908691,
       "step": 210
     },
     {
       "epoch": 0.46,
-      "grad_norm": 293.2833966309726,
+      "grad_norm": 19.35747892667393,
       "learning_rate": 3.272542485937368e-07,
-      "logits/chosen": -1.8287239074707031,
-      "logits/rejected": -1.8750501871109009,
-      "logps/chosen": -120.6048583984375,
-      "logps/rejected": -113.45402526855469,
-      "loss": 0.1905,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 7.93521785736084,
-      "rewards/margins": 20.89790916442871,
-      "rewards/rejected": -12.962692260742188,
+      "logits/chosen": -0.03855214640498161,
+      "logits/rejected": 1.1163934469223022,
+      "logps/chosen": -422.90655517578125,
+      "logps/rejected": -638.5581665039062,
+      "loss": 0.2649,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -1.665327787399292,
+      "rewards/margins": 2.9846928119659424,
+      "rewards/rejected": -4.650020122528076,
       "step": 220
     },
     {
       "epoch": 0.48,
-      "grad_norm": 186.44723799481827,
+      "grad_norm": 21.48535535733013,
       "learning_rate": 3.096924887558854e-07,
-      "logits/chosen": -1.7752487659454346,
-      "logits/rejected": -1.728193998336792,
-      "logps/chosen": -132.70101928710938,
-      "logps/rejected": -132.2133331298828,
-      "loss": 0.2334,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 7.50391149520874,
-      "rewards/margins": 23.704010009765625,
-      "rewards/rejected": -16.200098037719727,
+      "logits/chosen": 0.25134754180908203,
+      "logits/rejected": 1.690708875656128,
+      "logps/chosen": -465.64324951171875,
+      "logps/rejected": -718.0191650390625,
+      "loss": 0.2576,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -1.8850457668304443,
+      "rewards/margins": 3.3224689960479736,
+      "rewards/rejected": -5.20751428604126,
       "step": 230
     },
     {
       "epoch": 0.5,
-      "grad_norm": 492.6328669399213,
+      "grad_norm": 21.422183287654935,
       "learning_rate": 2.9181224366319943e-07,
-      "logits/chosen": -1.7902244329452515,
-      "logits/rejected": -1.8099979162216187,
-      "logps/chosen": -119.7879409790039,
-      "logps/rejected": -124.16999816894531,
-      "loss": 0.2101,
+      "logits/chosen": 0.8468208312988281,
+      "logits/rejected": 2.1126351356506348,
+      "logps/chosen": -494.220703125,
+      "logps/rejected": -765.5153198242188,
+      "loss": 0.2597,
       "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 6.068596363067627,
-      "rewards/margins": 19.338857650756836,
-      "rewards/rejected": -13.270261764526367,
+      "rewards/chosen": -2.599130153656006,
+      "rewards/margins": 3.3152058124542236,
+      "rewards/rejected": -5.914336204528809,
       "step": 240
     },
     {
       "epoch": 0.52,
-      "grad_norm": 497.9717087136447,
+      "grad_norm": 20.426996979966383,
       "learning_rate": 2.7370891215954565e-07,
-      "logits/chosen": -1.7690420150756836,
-      "logits/rejected": -1.8053967952728271,
-      "logps/chosen": -123.1581039428711,
-      "logps/rejected": -126.19282531738281,
-      "loss": 0.232,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 7.128526210784912,
-      "rewards/margins": 21.73324966430664,
-      "rewards/rejected": -14.604723930358887,
+      "logits/chosen": -0.12486964464187622,
+      "logits/rejected": 1.0274794101715088,
+      "logps/chosen": -427.1630859375,
+      "logps/rejected": -634.4889526367188,
+      "loss": 0.2648,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -1.721609115600586,
+      "rewards/margins": 2.7955689430236816,
+      "rewards/rejected": -4.517177581787109,
       "step": 250
     },
     {
       "epoch": 0.54,
-      "grad_norm": 219.15496679415526,
+      "grad_norm": 25.70991400557029,
       "learning_rate": 2.55479083351317e-07,
-      "logits/chosen": -1.8088661432266235,
-      "logits/rejected": -1.830862045288086,
-      "logps/chosen": -126.8784408569336,
-      "logps/rejected": -114.4140396118164,
-      "loss": 0.172,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 8.474285125732422,
-      "rewards/margins": 23.2431583404541,
-      "rewards/rejected": -14.768875122070312,
+      "logits/chosen": -0.1112385243177414,
+      "logits/rejected": 1.0832737684249878,
+      "logps/chosen": -449.45709228515625,
+      "logps/rejected": -671.4852294921875,
+      "loss": 0.2494,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.8507035970687866,
+      "rewards/margins": 3.2304184436798096,
+      "rewards/rejected": -5.081121921539307,
       "step": 260
     },
     {
       "epoch": 0.56,
-      "grad_norm": 339.5863335988586,
+      "grad_norm": 26.743751356983804,
       "learning_rate": 2.3722002126275822e-07,
-      "logits/chosen": -1.8175914287567139,
-      "logits/rejected": -1.7589060068130493,
-      "logps/chosen": -117.0767593383789,
-      "logps/rejected": -118.7080078125,
-      "loss": 0.1731,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 7.490860939025879,
-      "rewards/margins": 21.22795295715332,
-      "rewards/rejected": -13.737092971801758,
+      "logits/chosen": -0.3570239245891571,
+      "logits/rejected": 0.8737660646438599,
+      "logps/chosen": -427.79931640625,
+      "logps/rejected": -680.0591430664062,
+      "loss": 0.2351,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -1.8343093395233154,
+      "rewards/margins": 3.1806130409240723,
+      "rewards/rejected": -5.014922142028809,
       "step": 270
     },
     {
       "epoch": 0.59,
-      "grad_norm": 442.5487563129449,
+      "grad_norm": 20.437919430365636,
       "learning_rate": 2.19029145890313e-07,
-      "logits/chosen": -1.678240418434143,
-      "logits/rejected": -1.765446424484253,
-      "logps/chosen": -123.12864685058594,
-      "logps/rejected": -131.90528869628906,
-      "loss": 0.2242,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 6.964009761810303,
-      "rewards/margins": 22.241891860961914,
-      "rewards/rejected": -15.27788257598877,
+      "logits/chosen": -0.14857754111289978,
+      "logits/rejected": 0.7723418474197388,
+      "logps/chosen": -476.52099609375,
+      "logps/rejected": -727.02734375,
+      "loss": 0.2542,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -2.1171650886535645,
+      "rewards/margins": 3.235008716583252,
+      "rewards/rejected": -5.352174282073975,
       "step": 280
     },
     {
       "epoch": 0.61,
-      "grad_norm": 445.7460142068266,
+      "grad_norm": 18.45732619902188,
       "learning_rate": 2.0100351342479216e-07,
-      "logits/chosen": -1.771707534790039,
-      "logits/rejected": -1.7493677139282227,
-      "logps/chosen": -114.3892593383789,
-      "logps/rejected": -119.188720703125,
-      "loss": 0.2179,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 6.886797904968262,
-      "rewards/margins": 21.92293930053711,
-      "rewards/rejected": -15.036142349243164,
+      "logits/chosen": -0.04501671344041824,
+      "logits/rejected": 0.9899295568466187,
+      "logps/chosen": -446.81451416015625,
+      "logps/rejected": -742.0169677734375,
+      "loss": 0.2265,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -2.055858850479126,
+      "rewards/margins": 3.6457011699676514,
+      "rewards/rejected": -5.701560020446777,
       "step": 290
     },
     {
       "epoch": 0.63,
-      "grad_norm": 347.93449539220404,
+      "grad_norm": 18.065200464485685,
       "learning_rate": 1.8323929841460178e-07,
-      "logits/chosen": -1.7396576404571533,
-      "logits/rejected": -1.6877906322479248,
-      "logps/chosen": -132.26669311523438,
-      "logps/rejected": -135.92764282226562,
-      "loss": 0.2072,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 7.086742401123047,
-      "rewards/margins": 21.930522918701172,
-      "rewards/rejected": -14.843778610229492,
+      "logits/chosen": -0.25825151801109314,
+      "logits/rejected": 1.0551276206970215,
+      "logps/chosen": -482.5679626464844,
+      "logps/rejected": -777.6588134765625,
+      "loss": 0.259,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -2.0962753295898438,
+      "rewards/margins": 3.666637897491455,
+      "rewards/rejected": -5.762913227081299,
       "step": 300
     },
     {
       "epoch": 0.63,
-      "eval_logits/chosen": -1.84382963180542,
-      "eval_logits/rejected": -1.8451404571533203,
-      "eval_logps/chosen": -122.22917175292969,
-      "eval_logps/rejected": -126.25321960449219,
-      "eval_loss": 0.16582313179969788,
-      "eval_rewards/accuracies": 0.921875,
-      "eval_rewards/chosen": 7.771534442901611,
-      "eval_rewards/margins": 24.27777671813965,
-      "eval_rewards/rejected": -16.506242752075195,
-      "eval_runtime": 97.6572,
-      "eval_samples_per_second": 20.48,
+      "eval_logits/chosen": -0.08032596111297607,
+      "eval_logits/rejected": 1.257252812385559,
+      "eval_logps/chosen": -480.0015563964844,
+      "eval_logps/rejected": -791.1058959960938,
+      "eval_loss": 0.21872717142105103,
+      "eval_rewards/accuracies": 0.89453125,
+      "eval_rewards/chosen": -2.2256827354431152,
+      "eval_rewards/margins": 3.8858344554901123,
+      "eval_rewards/rejected": -6.111516952514648,
+      "eval_runtime": 97.4838,
+      "eval_samples_per_second": 20.516,
       "eval_steps_per_second": 0.328,
       "step": 300
     },
     {
       "epoch": 0.65,
-      "grad_norm": 522.5571001977761,
+      "grad_norm": 52.672419540257096,
       "learning_rate": 1.6583128063291573e-07,
-      "logits/chosen": -1.7435277700424194,
-      "logits/rejected": -1.7735698223114014,
-      "logps/chosen": -123.75146484375,
-      "logps/rejected": -126.75709533691406,
-      "loss": 0.2065,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 7.7931108474731445,
-      "rewards/margins": 21.902111053466797,
-      "rewards/rejected": -14.109001159667969,
+      "logits/chosen": -0.3141949772834778,
+      "logits/rejected": 0.9004285931587219,
+      "logps/chosen": -475.15936279296875,
+      "logps/rejected": -737.10009765625,
+      "loss": 0.2596,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -2.111830949783325,
+      "rewards/margins": 3.3903605937957764,
+      "rewards/rejected": -5.502191543579102,
       "step": 310
     },
     {
       "epoch": 0.67,
-      "grad_norm": 355.22236619382966,
+      "grad_norm": 22.65476006635999,
       "learning_rate": 1.488723393865766e-07,
-      "logits/chosen": -1.7776333093643188,
-      "logits/rejected": -1.7483892440795898,
-      "logps/chosen": -113.23478698730469,
-      "logps/rejected": -126.3783950805664,
-      "loss": 0.1913,
+      "logits/chosen": -0.16484542191028595,
+      "logits/rejected": 1.090689778327942,
+      "logps/chosen": -455.7721252441406,
+      "logps/rejected": -757.0924072265625,
+      "loss": 0.2429,
       "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 8.781044006347656,
-      "rewards/margins": 23.842554092407227,
-      "rewards/rejected": -15.061508178710938,
+      "rewards/chosen": -2.08683180809021,
+      "rewards/margins": 3.6439883708953857,
+      "rewards/rejected": -5.730820178985596,
       "step": 320
     },
     {
       "epoch": 0.69,
-      "grad_norm": 322.18484805793145,
+      "grad_norm": 19.80833079454062,
       "learning_rate": 1.3245295796480788e-07,
-      "logits/chosen": -1.7006194591522217,
-      "logits/rejected": -1.7839641571044922,
-      "logps/chosen": -123.49153137207031,
-      "logps/rejected": -121.6719741821289,
-      "loss": 0.2011,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 7.345705986022949,
-      "rewards/margins": 21.307662963867188,
-      "rewards/rejected": -13.961955070495605,
+      "logits/chosen": -0.27468693256378174,
+      "logits/rejected": 1.061683177947998,
+      "logps/chosen": -481.45501708984375,
+      "logps/rejected": -751.7589721679688,
+      "loss": 0.2279,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -2.233105421066284,
+      "rewards/margins": 3.570781707763672,
+      "rewards/rejected": -5.803887367248535,
       "step": 330
     },
     {
       "epoch": 0.71,
-      "grad_norm": 281.7232240729927,
+      "grad_norm": 24.881750772146,
       "learning_rate": 1.1666074087171627e-07,
-      "logits/chosen": -1.6561883687973022,
-      "logits/rejected": -1.7283906936645508,
-      "logps/chosen": -117.68770599365234,
-      "logps/rejected": -112.509765625,
-      "loss": 0.2456,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 6.401297569274902,
-      "rewards/margins": 18.83791732788086,
-      "rewards/rejected": -12.43661880493164,
+      "logits/chosen": -0.03138185292482376,
+      "logits/rejected": 1.0509425401687622,
+      "logps/chosen": -470.1802673339844,
+      "logps/rejected": -690.557373046875,
+      "loss": 0.237,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -2.2647526264190674,
+      "rewards/margins": 3.035360097885132,
+      "rewards/rejected": -5.300112724304199,
       "step": 340
     },
     {
       "epoch": 0.73,
-      "grad_norm": 736.7599817555671,
+      "grad_norm": 19.587851322719366,
       "learning_rate": 1.0157994641835734e-07,
-      "logits/chosen": -1.6912691593170166,
-      "logits/rejected": -1.7067359685897827,
-      "logps/chosen": -117.70341491699219,
-      "logps/rejected": -132.61856079101562,
-      "loss": 0.1648,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 6.1907782554626465,
-      "rewards/margins": 21.720661163330078,
-      "rewards/rejected": -15.529881477355957,
+      "logits/chosen": 0.006643450353294611,
+      "logits/rejected": 1.0511295795440674,
+      "logps/chosen": -478.51055908203125,
+      "logps/rejected": -806.5072631835938,
+      "loss": 0.2256,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -2.348606586456299,
+      "rewards/margins": 3.6702816486358643,
+      "rewards/rejected": -6.018888473510742,
       "step": 350
     },
     {
       "epoch": 0.75,
-      "grad_norm": 471.30491456250144,
+      "grad_norm": 22.679114876876472,
       "learning_rate": 8.729103716819111e-08,
-      "logits/chosen": -1.6851584911346436,
-      "logits/rejected": -1.7095420360565186,
-      "logps/chosen": -124.0709228515625,
-      "logps/rejected": -127.85160827636719,
-      "loss": 0.1683,
+      "logits/chosen": -0.024983350187540054,
+      "logits/rejected": 1.071063756942749,
+      "logps/chosen": -459.17669677734375,
+      "logps/rejected": -731.5552368164062,
+      "loss": 0.2495,
       "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 7.039712429046631,
-      "rewards/margins": 21.306758880615234,
-      "rewards/rejected": -14.267045974731445,
+      "rewards/chosen": -2.0910515785217285,
+      "rewards/margins": 3.42362642288208,
+      "rewards/rejected": -5.514677047729492,
       "step": 360
     },
     {
       "epoch": 0.77,
-      "grad_norm": 673.9872436763492,
+      "grad_norm": 24.62831310204124,
       "learning_rate": 7.387025063449081e-08,
-      "logits/chosen": -1.7928199768066406,
-      "logits/rejected": -1.7365026473999023,
-      "logps/chosen": -113.66243743896484,
-      "logps/rejected": -117.55149841308594,
-      "loss": 0.1713,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 6.36365270614624,
-      "rewards/margins": 18.882360458374023,
-      "rewards/rejected": -12.518709182739258,
+      "logits/chosen": -0.07435999810695648,
+      "logits/rejected": 1.1522341966629028,
+      "logps/chosen": -457.25128173828125,
+      "logps/rejected": -728.47412109375,
+      "loss": 0.234,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -2.266648292541504,
+      "rewards/margins": 3.339157819747925,
+      "rewards/rejected": -5.60580587387085,
       "step": 370
     },
     {
       "epoch": 0.79,
-      "grad_norm": 424.179983945121,
+      "grad_norm": 18.56178014954704,
       "learning_rate": 6.138919252022435e-08,
-      "logits/chosen": -1.8132708072662354,
-      "logits/rejected": -1.8341960906982422,
-      "logps/chosen": -121.0456771850586,
-      "logps/rejected": -131.6220703125,
-      "loss": 0.1821,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 7.8040289878845215,
-      "rewards/margins": 24.01057243347168,
-      "rewards/rejected": -16.20654296875,
+      "logits/chosen": -0.3609544634819031,
+      "logits/rejected": 0.9311397671699524,
+      "logps/chosen": -469.6280212402344,
+      "logps/rejected": -802.5978393554688,
+      "loss": 0.2329,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -2.1230244636535645,
+      "rewards/margins": 4.024425506591797,
+      "rewards/rejected": -6.147449016571045,
       "step": 380
     },
     {
       "epoch": 0.82,
-      "grad_norm": 480.45370869608473,
+      "grad_norm": 32.926290807333665,
       "learning_rate": 4.991445467064689e-08,
-      "logits/chosen": -1.704850196838379,
-      "logits/rejected": -1.7101322412490845,
-      "logps/chosen": -116.9380874633789,
-      "logps/rejected": -119.96983337402344,
-      "loss": 0.1493,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 7.778868198394775,
-      "rewards/margins": 22.611705780029297,
-      "rewards/rejected": -14.832839965820312,
+      "logits/chosen": -0.012720714323222637,
+      "logits/rejected": 1.3005478382110596,
+      "logps/chosen": -489.65802001953125,
+      "logps/rejected": -779.3419799804688,
+      "loss": 0.2265,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -2.43365740776062,
+      "rewards/margins": 3.560826539993286,
+      "rewards/rejected": -5.994483947753906,
       "step": 390
     },
     {
       "epoch": 0.84,
-      "grad_norm": 500.3403823746712,
+      "grad_norm": 30.149198169987162,
       "learning_rate": 3.9507259776993954e-08,
-      "logits/chosen": -1.6802362203598022,
-      "logits/rejected": -1.7585910558700562,
-      "logps/chosen": -119.65745544433594,
-      "logps/rejected": -120.54292297363281,
-      "loss": 0.1918,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 6.5520124435424805,
-      "rewards/margins": 21.033418655395508,
-      "rewards/rejected": -14.481407165527344,
+      "logits/chosen": 0.26025086641311646,
+      "logits/rejected": 1.3578455448150635,
+      "logps/chosen": -470.298095703125,
+      "logps/rejected": -760.2213134765625,
+      "loss": 0.2268,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -2.2305097579956055,
+      "rewards/margins": 3.6479294300079346,
+      "rewards/rejected": -5.878438472747803,
       "step": 400
     },
     {
       "epoch": 0.84,
-      "eval_logits/chosen": -1.8512133359909058,
-      "eval_logits/rejected": -1.8539310693740845,
-      "eval_logps/chosen": -121.4881591796875,
-      "eval_logps/rejected": -126.26817321777344,
-      "eval_loss": 0.1591520756483078,
-      "eval_rewards/accuracies": 0.921875,
-      "eval_rewards/chosen": 8.14204216003418,
-      "eval_rewards/margins": 24.65576171875,
-      "eval_rewards/rejected": -16.51371955871582,
-      "eval_runtime": 97.6311,
-      "eval_samples_per_second": 20.485,
-      "eval_steps_per_second": 0.328,
+      "eval_logits/chosen": -0.19649244844913483,
+      "eval_logits/rejected": 1.227736234664917,
+      "eval_logps/chosen": -477.7560729980469,
+      "eval_logps/rejected": -812.5330810546875,
+      "eval_loss": 0.2144031673669815,
+      "eval_rewards/accuracies": 0.8984375,
+      "eval_rewards/chosen": -2.2032277584075928,
+      "eval_rewards/margins": 4.122560977935791,
+      "eval_rewards/rejected": -6.325788974761963,
+      "eval_runtime": 97.384,
+      "eval_samples_per_second": 20.537,
+      "eval_steps_per_second": 0.329,
       "step": 400
     },
     {
       "epoch": 0.86,
-      "grad_norm": 464.7237726717384,
+      "grad_norm": 15.686531185394536,
       "learning_rate": 3.022313472693447e-08,
-      "logits/chosen": -1.7741676568984985,
-      "logits/rejected": -1.8052194118499756,
-      "logps/chosen": -130.3295135498047,
-      "logps/rejected": -124.33296203613281,
-      "loss": 0.1851,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 8.492754936218262,
-      "rewards/margins": 24.606307983398438,
-      "rewards/rejected": -16.113550186157227,
+      "logits/chosen": -0.13663654029369354,
+      "logits/rejected": 1.1233164072036743,
+      "logps/chosen": -492.34588623046875,
+      "logps/rejected": -787.8416137695312,
+      "loss": 0.2073,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -2.1595890522003174,
+      "rewards/margins": 3.8674533367156982,
+      "rewards/rejected": -6.027042388916016,
       "step": 410
     },
     {
       "epoch": 0.88,
-      "grad_norm": 372.69082796807044,
+      "grad_norm": 25.36773289746514,
       "learning_rate": 2.2111614344599684e-08,
-      "logits/chosen": -1.7444789409637451,
-      "logits/rejected": -1.7450637817382812,
-      "logps/chosen": -115.13069915771484,
-      "logps/rejected": -119.38905334472656,
-      "loss": 0.1709,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 6.589479923248291,
-      "rewards/margins": 20.951730728149414,
-      "rewards/rejected": -14.362251281738281,
+      "logits/chosen": 0.07713554799556732,
+      "logits/rejected": 1.1611943244934082,
+      "logps/chosen": -468.55523681640625,
+      "logps/rejected": -737.3845825195312,
+      "loss": 0.2294,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -2.246133804321289,
+      "rewards/margins": 3.3159337043762207,
+      "rewards/rejected": -5.562067985534668,
       "step": 420
     },
     {
       "epoch": 0.9,
-      "grad_norm": 487.96100980731563,
+      "grad_norm": 23.64735863277161,
       "learning_rate": 1.521597710086439e-08,
-      "logits/chosen": -1.7001349925994873,
-      "logits/rejected": -1.7239036560058594,
-      "logps/chosen": -131.1899871826172,
-      "logps/rejected": -119.99398040771484,
-      "loss": 0.189,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 7.4596357345581055,
-      "rewards/margins": 20.71387481689453,
-      "rewards/rejected": -13.254237174987793,
+      "logits/chosen": -0.11199776083230972,
+      "logits/rejected": 1.2544041872024536,
+      "logps/chosen": -487.00787353515625,
+      "logps/rejected": -748.0345458984375,
+      "loss": 0.2283,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.158412218093872,
+      "rewards/margins": 3.493891954421997,
+      "rewards/rejected": -5.652304649353027,
       "step": 430
     },
     {
       "epoch": 0.92,
-      "grad_norm": 322.7631079024311,
+      "grad_norm": 32.07046590554996,
       "learning_rate": 9.57301420397924e-09,
-      "logits/chosen": -1.8264925479888916,
-      "logits/rejected": -1.8369083404541016,
-      "logps/chosen": -122.54289245605469,
-      "logps/rejected": -125.7545394897461,
-      "loss": 0.1707,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 8.215115547180176,
-      "rewards/margins": 24.390819549560547,
-      "rewards/rejected": -16.175704956054688,
+      "logits/chosen": -0.2779064476490021,
+      "logits/rejected": 0.990594744682312,
+      "logps/chosen": -476.5415954589844,
+      "logps/rejected": -816.4949340820312,
+      "loss": 0.2335,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -2.1302719116210938,
+      "rewards/margins": 4.236142158508301,
+      "rewards/rejected": -6.366414546966553,
       "step": 440
     },
     {
       "epoch": 0.94,
-      "grad_norm": 358.10893603221916,
+      "grad_norm": 29.966664104398923,
       "learning_rate": 5.212833302556258e-09,
-      "logits/chosen": -1.836982011795044,
-      "logits/rejected": -1.800283432006836,
-      "logps/chosen": -118.0952377319336,
-      "logps/rejected": -121.89033508300781,
-      "loss": 0.2021,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 8.605842590332031,
-      "rewards/margins": 25.365093231201172,
-      "rewards/rejected": -16.759248733520508,
+      "logits/chosen": -0.2724097967147827,
+      "logits/rejected": 1.02344810962677,
+      "logps/chosen": -477.5227966308594,
+      "logps/rejected": -801.6556396484375,
+      "loss": 0.235,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -2.263077735900879,
+      "rewards/margins": 3.993912935256958,
+      "rewards/rejected": -6.256990432739258,
       "step": 450
     },
     {
       "epoch": 0.96,
-      "grad_norm": 246.32063500635945,
+      "grad_norm": 54.30402743121173,
       "learning_rate": 2.158697848236607e-09,
-      "logits/chosen": -1.7269080877304077,
-      "logits/rejected": -1.749202013015747,
-      "logps/chosen": -122.84416198730469,
-      "logps/rejected": -125.47819519042969,
-      "loss": 0.1556,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 8.254826545715332,
-      "rewards/margins": 23.179615020751953,
-      "rewards/rejected": -14.92479133605957,
+      "logits/chosen": -0.18335244059562683,
+      "logits/rejected": 0.9830573201179504,
+      "logps/chosen": -479.820556640625,
+      "logps/rejected": -772.7644653320312,
+      "loss": 0.2201,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -2.1699745655059814,
+      "rewards/margins": 3.6132774353027344,
+      "rewards/rejected": -5.783252239227295,
       "step": 460
     },
     {
       "epoch": 0.98,
-      "grad_norm": 311.90558289785304,
+      "grad_norm": 24.80298516842009,
       "learning_rate": 4.269029751107489e-10,
-      "logits/chosen": -1.7059637308120728,
-      "logits/rejected": -1.7222188711166382,
-      "logps/chosen": -116.1396255493164,
-      "logps/rejected": -135.0008544921875,
-      "loss": 0.1821,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 6.786002159118652,
-      "rewards/margins": 22.398935317993164,
-      "rewards/rejected": -15.612930297851562,
+      "logits/chosen": -0.0621672198176384,
+      "logits/rejected": 1.1333558559417725,
+      "logps/chosen": -449.02386474609375,
+      "logps/rejected": -767.277099609375,
+      "loss": 0.2269,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -2.1144473552703857,
+      "rewards/margins": 3.668858289718628,
+      "rewards/rejected": -5.783305644989014,
       "step": 470
     },
     {
       "epoch": 1.0,
       "step": 478,
       "total_flos": 0.0,
-      "train_loss": 0.19963366765856244,
-      "train_runtime": 7635.6428,
-      "train_samples_per_second": 8.007,
+      "train_loss": 0.3183088973476298,
+      "train_runtime": 7617.3811,
+      "train_samples_per_second": 8.026,
       "train_steps_per_second": 0.063
     }
   ],