Model save

README.md

@@ -4,6 +4,7 @@ base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
 - trl
 - dpo
+- alignment-handbook
 - generated_from_trainer
 model-index:
 - name: zephyr-7b-dpo-full-gpt-reward-scale-05
@@ -17,15 +18,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5099
-- Rewards/chosen: -2.9128
-- Rewards/rejected: -4.2868
-- Rewards/accuracies: 0.7328
-- Rewards/margins: 1.3740
-- Logps/rejected: -674.3233
-- Logps/chosen: -575.2830
-- Logits/rejected: 2.9146
-- Logits/chosen: 0.8634
+- Loss: 0.5238
+- Rewards/chosen: -1.1890
+- Rewards/rejected: -2.1821
+- Rewards/accuracies: 0.7241
+- Rewards/margins: 0.9930
+- Logps/rejected: -463.8542
+- Logps/chosen: -402.9079
+- Logits/rejected: 3.3069
+- Logits/chosen: 1.9855
 
 ## Model description
 
@@ -62,14 +63,14 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.6688        | 0.1147 | 50   | 0.6561          | -0.0264        | -0.1297          | 0.6767             | 0.1033          | -258.6208      | -286.6466    | -2.5069         | -2.6066       |
-| 0.5822        | 0.2294 | 100  | 0.5813          | -0.9069        | -1.4729          | 0.6724             | 0.5660          | -392.9425      | -374.6986    | 0.4795          | -0.0936       |
-| 0.5512        | 0.3440 | 150  | 0.5534          | -0.9841        | -1.8057          | 0.7284             | 0.8216          | -426.2166      | -382.4128    | 2.9841          | 1.9098        |
-| 0.5364        | 0.4587 | 200  | 0.5368          | -1.4185        | -2.3616          | 0.7328             | 0.9430          | -481.8062      | -425.8600    | 3.8766          | 2.4303        |
-| 0.5308        | 0.5734 | 250  | 0.5235          | -2.3552        | -3.5698          | 0.7284             | 1.2145          | -602.6266      | -519.5305    | 4.2409          | 2.4548        |
-| 0.532         | 0.6881 | 300  | 0.5117          | -2.5133        | -3.7138          | 0.7198             | 1.2004          | -617.0262      | -535.3408    | 3.0154          | 1.1080        |
-| 0.5064        | 0.8028 | 350  | 0.5117          | -2.6361        | -3.9100          | 0.7241             | 1.2739          | -636.6490      | -547.6198    | 2.6854          | 0.7127        |
-| 0.5105        | 0.9174 | 400  | 0.5099          | -2.9128        | -4.2868          | 0.7328             | 1.3740          | -674.3233      | -575.2830    | 2.9146          | 0.8634        |
+| 0.6687        | 0.1147 | 50   | 0.6560          | -0.0264        | -0.1298          | 0.6724             | 0.1034          | -258.6246      | -286.6438    | -2.5075         | -2.6072       |
+| 0.581         | 0.2294 | 100  | 0.5764          | -0.7311        | -1.3172          | 0.7155             | 0.5861          | -377.3666      | -357.1160    | 0.6340          | 0.0270        |
+| 0.558         | 0.3440 | 150  | 0.5510          | -1.2031        | -1.9696          | 0.7241             | 0.7665          | -442.6071      | -404.3199    | 3.0036          | 2.0828        |
+| 0.5346        | 0.4587 | 200  | 0.5381          | -1.1677        | -2.0355          | 0.7112             | 0.8679          | -449.2019      | -400.7711    | 2.7759          | 1.7577        |
+| 0.5391        | 0.5734 | 250  | 0.5333          | -1.0858        | -1.9666          | 0.7198             | 0.8807          | -442.3041      | -392.5903    | 2.9561          | 1.8167        |
+| 0.5479        | 0.6881 | 300  | 0.5265          | -1.0463        | -1.9706          | 0.7069             | 0.9243          | -442.7093      | -388.6379    | 3.2239          | 2.0026        |
+| 0.5232        | 0.8028 | 350  | 0.5262          | -1.3359        | -2.3191          | 0.7241             | 0.9832          | -477.5577      | -417.5966    | 3.6066          | 2.3484        |
+| 0.5267        | 0.9174 | 400  | 0.5238          | -1.1890        | -2.1821          | 0.7241             | 0.9930          | -463.8542      | -402.9079    | 3.3069          | 1.9855        |
 
 
 ### Framework versions

all_results.json

@@ -1,9 +1,22 @@
 {
     "epoch": 1.0,
+    "eval_logits/chosen": 0.9167268872261047,
+    "eval_logits/rejected": 2.964365243911743,
+    "eval_logps/chosen": -579.6903076171875,
+    "eval_logps/rejected": -678.6295776367188,
+    "eval_loss": 0.5096524953842163,
+    "eval_rewards/accuracies": 0.7284482717514038,
+    "eval_rewards/chosen": -2.956843376159668,
+    "eval_rewards/margins": 1.3729774951934814,
+    "eval_rewards/rejected": -4.3298211097717285,
+    "eval_runtime": 91.8301,
+    "eval_samples": 1818,
+    "eval_samples_per_second": 19.797,
+    "eval_steps_per_second": 0.316,
     "total_flos": 0.0,
-    "train_loss": 0.5604413999330013,
-    "train_runtime": 11415.5934,
+    "train_loss": 0.5659637576943144,
+    "train_runtime": 11398.0027,
     "train_samples": 55758,
-    "train_samples_per_second": 4.884,
+    "train_samples_per_second": 4.892,
     "train_steps_per_second": 0.038
 }

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": 0.9167268872261047,
+    "eval_logits/rejected": 2.964365243911743,
+    "eval_logps/chosen": -579.6903076171875,
+    "eval_logps/rejected": -678.6295776367188,
+    "eval_loss": 0.5096524953842163,
+    "eval_rewards/accuracies": 0.7284482717514038,
+    "eval_rewards/chosen": -2.956843376159668,
+    "eval_rewards/margins": 1.3729774951934814,
+    "eval_rewards/rejected": -4.3298211097717285,
+    "eval_runtime": 91.8301,
+    "eval_samples": 1818,
+    "eval_samples_per_second": 19.797,
+    "eval_steps_per_second": 0.316
+}

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:94471c8f16f3ef1e863bc713de005404f0b4e00ebd6a1fd9f11057b64399e6d4
+oid sha256:a30539688b6bace3e527da9c2325f0b02708498d49bbd03cc3cb31fe89ad1d30
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa02efacc8b4a11ea44b76e1523eb0dab68e840cefd5584f389e07cf4339aef2
+oid sha256:71be5f61b23a0c39e97d24c838f1f720f09b436e0ce27d6c8d5d93ddb21b085c
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3304ab6e41e497d269a92e459f7fbe3e0c40021d7284db90ce72ada43ee0ab73
+oid sha256:c5ef3a7c1e95efc67fb5751d976ca4e06caeceb0258db56196b431129e7b286a
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.5604413999330013,
-    "train_runtime": 11415.5934,
+    "train_loss": 0.5659637576943144,
+    "train_runtime": 11398.0027,
     "train_samples": 55758,
-    "train_samples_per_second": 4.884,
+    "train_samples_per_second": 4.892,
     "train_steps_per_second": 0.038
 }

trainer_state.json

@@ -10,784 +10,784 @@
   "log_history": [
     {
       "epoch": 0.022935779816513763,
-      "grad_norm": 5.353972534143438,
+      "grad_norm": 5.356178331285126,
       "learning_rate": 1.1363636363636363e-07,
-      "logits/chosen": -2.6582446098327637,
-      "logits/rejected": -2.612395763397217,
-      "logps/chosen": -310.3081359863281,
-      "logps/rejected": -241.6246337890625,
+      "logits/chosen": -2.6583542823791504,
+      "logits/rejected": -2.612396240234375,
+      "logps/chosen": -310.2690124511719,
+      "logps/rejected": -241.6248321533203,
       "loss": 0.6932,
-      "rewards/accuracies": 0.3499999940395355,
-      "rewards/chosen": -0.00043685571290552616,
-      "rewards/margins": -0.0005496515659615397,
-      "rewards/rejected": 0.0001127958094002679,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": -4.61353047285229e-05,
+      "rewards/margins": -0.00015705036639701575,
+      "rewards/rejected": 0.00011091506894445047,
       "step": 10
     },
     {
       "epoch": 0.045871559633027525,
-      "grad_norm": 6.431385284276218,
+      "grad_norm": 6.4233925318831595,
       "learning_rate": 2.2727272727272726e-07,
-      "logits/chosen": -2.690976142883301,
-      "logits/rejected": -2.615501880645752,
-      "logps/chosen": -293.55859375,
-      "logps/rejected": -265.65789794921875,
+      "logits/chosen": -2.691195011138916,
+      "logits/rejected": -2.6153342723846436,
+      "logps/chosen": -293.5455627441406,
+      "logps/rejected": -265.6838684082031,
       "loss": 0.6924,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.001354431384243071,
-      "rewards/margins": 0.0023786118254065514,
-      "rewards/rejected": -0.0010241802083328366,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": 0.001484546228311956,
+      "rewards/margins": 0.002768759150058031,
+      "rewards/rejected": -0.0012842128053307533,
       "step": 20
     },
     {
       "epoch": 0.06880733944954129,
-      "grad_norm": 5.140938328988767,
+      "grad_norm": 5.149124678509347,
       "learning_rate": 3.4090909090909085e-07,
-      "logits/chosen": -2.6976418495178223,
-      "logits/rejected": -2.6304168701171875,
-      "logps/chosen": -277.8341064453125,
-      "logps/rejected": -297.1772155761719,
+      "logits/chosen": -2.6977083683013916,
+      "logits/rejected": -2.63045072555542,
+      "logps/chosen": -277.82159423828125,
+      "logps/rejected": -297.18646240234375,
       "loss": 0.6892,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.004078004974871874,
-      "rewards/margins": 0.009664928540587425,
-      "rewards/rejected": -0.005586923565715551,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.004203228745609522,
+      "rewards/margins": 0.009881972335278988,
+      "rewards/rejected": -0.005678744055330753,
       "step": 30
     },
     {
       "epoch": 0.09174311926605505,
-      "grad_norm": 5.971632655809275,
+      "grad_norm": 6.002207032235101,
       "learning_rate": 4.545454545454545e-07,
-      "logits/chosen": -2.616170883178711,
-      "logits/rejected": -2.5451369285583496,
-      "logps/chosen": -283.9632568359375,
-      "logps/rejected": -259.82861328125,
+      "logits/chosen": -2.616579294204712,
+      "logits/rejected": -2.5455870628356934,
+      "logps/chosen": -283.92156982421875,
+      "logps/rejected": -259.82562255859375,
       "loss": 0.6798,
       "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.03654901683330536,
-      "rewards/margins": 0.045721281319856644,
-      "rewards/rejected": -0.00917226541787386,
+      "rewards/chosen": 0.036965593695640564,
+      "rewards/margins": 0.04610789567232132,
+      "rewards/rejected": -0.009142300114035606,
       "step": 40
     },
     {
       "epoch": 0.11467889908256881,
-      "grad_norm": 5.916656852320022,
+      "grad_norm": 5.926817590245787,
       "learning_rate": 4.997110275491701e-07,
-      "logits/chosen": -2.5970985889434814,
-      "logits/rejected": -2.5133914947509766,
-      "logps/chosen": -285.24835205078125,
-      "logps/rejected": -247.303466796875,
-      "loss": 0.6688,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.006145569030195475,
-      "rewards/margins": 0.0578111931681633,
-      "rewards/rejected": -0.06395676732063293,
+      "logits/chosen": -2.596590518951416,
+      "logits/rejected": -2.512640953063965,
+      "logps/chosen": -285.3323669433594,
+      "logps/rejected": -247.4479522705078,
+      "loss": 0.6687,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.006985962390899658,
+      "rewards/margins": 0.058415599167346954,
+      "rewards/rejected": -0.06540156155824661,
       "step": 50
     },
     {
       "epoch": 0.11467889908256881,
-      "eval_logits/chosen": -2.6066324710845947,
-      "eval_logits/rejected": -2.506901979446411,
-      "eval_logps/chosen": -286.6465759277344,
-      "eval_logps/rejected": -258.62078857421875,
-      "eval_loss": 0.6561177968978882,
-      "eval_rewards/accuracies": 0.6767241358757019,
-      "eval_rewards/chosen": -0.02640603668987751,
-      "eval_rewards/margins": 0.10332722216844559,
-      "eval_rewards/rejected": -0.12973324954509735,
-      "eval_runtime": 91.0244,
-      "eval_samples_per_second": 19.973,
-      "eval_steps_per_second": 0.319,
+      "eval_logits/chosen": -2.607215166091919,
+      "eval_logits/rejected": -2.5074896812438965,
+      "eval_logps/chosen": -286.6437683105469,
+      "eval_logps/rejected": -258.6246032714844,
+      "eval_loss": 0.6559526920318604,
+      "eval_rewards/accuracies": 0.6724137663841248,
+      "eval_rewards/chosen": -0.026378028094768524,
+      "eval_rewards/margins": 0.10339301824569702,
+      "eval_rewards/rejected": -0.12977103888988495,
+      "eval_runtime": 92.1507,
+      "eval_samples_per_second": 19.729,
+      "eval_steps_per_second": 0.315,
       "step": 50
     },
     {
       "epoch": 0.13761467889908258,
-      "grad_norm": 7.499634288772489,
+      "grad_norm": 7.494952728753531,
       "learning_rate": 4.979475034558115e-07,
-      "logits/chosen": -2.582371234893799,
-      "logits/rejected": -2.5081627368927,
-      "logps/chosen": -292.10491943359375,
-      "logps/rejected": -282.31195068359375,
+      "logits/chosen": -2.582334518432617,
+      "logits/rejected": -2.508467197418213,
+      "logps/chosen": -292.1842346191406,
+      "logps/rejected": -282.423583984375,
       "loss": 0.6423,
       "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.11516664922237396,
-      "rewards/margins": 0.19041100144386292,
-      "rewards/rejected": -0.30557766556739807,
+      "rewards/chosen": -0.11595962941646576,
+      "rewards/margins": 0.1907343566417694,
+      "rewards/rejected": -0.306693971157074,
       "step": 60
     },
     {
       "epoch": 0.16055045871559634,
-      "grad_norm": 17.30037068758165,
+      "grad_norm": 18.148816686471342,
       "learning_rate": 4.945923025551788e-07,
-      "logits/chosen": -2.4502875804901123,
-      "logits/rejected": -2.3790054321289062,
-      "logps/chosen": -298.32244873046875,
-      "logps/rejected": -273.11859130859375,
-      "loss": 0.6397,
+      "logits/chosen": -2.459238052368164,
+      "logits/rejected": -2.3897058963775635,
+      "logps/chosen": -298.2831115722656,
+      "logps/rejected": -273.2386474609375,
+      "loss": 0.6393,
       "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.2997247576713562,
-      "rewards/margins": 0.23786215484142303,
-      "rewards/rejected": -0.5375869870185852,
+      "rewards/chosen": -0.29933103919029236,
+      "rewards/margins": 0.23945657908916473,
+      "rewards/rejected": -0.5387876629829407,
       "step": 70
     },
     {
       "epoch": 0.1834862385321101,
-      "grad_norm": 11.228813057299567,
+      "grad_norm": 12.734144337443169,
       "learning_rate": 4.896669632591651e-07,
-      "logits/chosen": -2.5100908279418945,
-      "logits/rejected": -2.4027259349823,
-      "logps/chosen": -306.67510986328125,
-      "logps/rejected": -322.7925720214844,
-      "loss": 0.6257,
+      "logits/chosen": -2.5085086822509766,
+      "logits/rejected": -2.3976407051086426,
+      "logps/chosen": -305.76031494140625,
+      "logps/rejected": -321.8554992675781,
+      "loss": 0.6235,
       "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.36647987365722656,
-      "rewards/margins": 0.28450754284858704,
-      "rewards/rejected": -0.6509873867034912,
+      "rewards/chosen": -0.3573322296142578,
+      "rewards/margins": 0.28428393602371216,
+      "rewards/rejected": -0.6416162252426147,
       "step": 80
     },
     {
       "epoch": 0.20642201834862386,
-      "grad_norm": 15.58920411413326,
+      "grad_norm": 14.039079346644037,
       "learning_rate": 4.832031033425662e-07,
-      "logits/chosen": -1.5505931377410889,
-      "logits/rejected": -1.3694034814834595,
-      "logps/chosen": -357.6716613769531,
-      "logps/rejected": -372.05133056640625,
-      "loss": 0.5967,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.5348917245864868,
-      "rewards/margins": 0.4341323971748352,
-      "rewards/rejected": -0.9690243005752563,
+      "logits/chosen": -1.4997788667678833,
+      "logits/rejected": -1.313194990158081,
+      "logps/chosen": -348.44805908203125,
+      "logps/rejected": -361.76226806640625,
+      "loss": 0.5956,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.44265589118003845,
+      "rewards/margins": 0.4234777092933655,
+      "rewards/rejected": -0.8661335706710815,
       "step": 90
     },
     {
       "epoch": 0.22935779816513763,
-      "grad_norm": 13.006583087547677,
+      "grad_norm": 13.29279140070498,
       "learning_rate": 4.752422169756047e-07,
-      "logits/chosen": -0.7837198972702026,
-      "logits/rejected": -0.35428792238235474,
-      "logps/chosen": -326.9918518066406,
-      "logps/rejected": -346.36737060546875,
-      "loss": 0.5822,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.4772585332393646,
-      "rewards/margins": 0.4704399108886719,
-      "rewards/rejected": -0.9476984143257141,
+      "logits/chosen": -0.19194559752941132,
+      "logits/rejected": 0.2622618079185486,
+      "logps/chosen": -339.16339111328125,
+      "logps/rejected": -359.37176513671875,
+      "loss": 0.581,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.598974347114563,
+      "rewards/margins": 0.4787676930427551,
+      "rewards/rejected": -1.0777419805526733,
       "step": 100
     },
     {
       "epoch": 0.22935779816513763,
-      "eval_logits/chosen": -0.09357786923646927,
-      "eval_logits/rejected": 0.4795497953891754,
-      "eval_logps/chosen": -374.6986083984375,
-      "eval_logps/rejected": -392.9424743652344,
-      "eval_loss": 0.5813368558883667,
-      "eval_rewards/accuracies": 0.6724137663841248,
-      "eval_rewards/chosen": -0.9069267511367798,
-      "eval_rewards/margins": 0.5660232305526733,
-      "eval_rewards/rejected": -1.4729499816894531,
-      "eval_runtime": 91.4662,
-      "eval_samples_per_second": 19.876,
-      "eval_steps_per_second": 0.317,
+      "eval_logits/chosen": 0.026995467022061348,
+      "eval_logits/rejected": 0.6340460777282715,
+      "eval_logps/chosen": -357.115966796875,
+      "eval_logps/rejected": -377.3665771484375,
+      "eval_loss": 0.5763944387435913,
+      "eval_rewards/accuracies": 0.7155172228813171,
+      "eval_rewards/chosen": -0.7311002016067505,
+      "eval_rewards/margins": 0.5860908627510071,
+      "eval_rewards/rejected": -1.3171910047531128,
+      "eval_runtime": 91.0093,
+      "eval_samples_per_second": 19.976,
+      "eval_steps_per_second": 0.319,
       "step": 100
     },
     {
       "epoch": 0.25229357798165136,
-      "grad_norm": 14.7655268267239,
+      "grad_norm": 27.36521925016087,
       "learning_rate": 4.658354083558188e-07,
-      "logits/chosen": -0.23613190650939941,
-      "logits/rejected": 0.2948758006095886,
-      "logps/chosen": -371.15667724609375,
-      "logps/rejected": -427.76885986328125,
-      "loss": 0.5606,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.7391853332519531,
-      "rewards/margins": 0.7208150625228882,
-      "rewards/rejected": -1.4600005149841309,
+      "logits/chosen": -0.14074298739433289,
+      "logits/rejected": 0.41164666414260864,
+      "logps/chosen": -359.0007019042969,
+      "logps/rejected": -422.62353515625,
+      "loss": 0.5561,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.6176259517669678,
+      "rewards/margins": 0.7909212708473206,
+      "rewards/rejected": -1.4085471630096436,
       "step": 110
     },
     {
       "epoch": 0.27522935779816515,
-      "grad_norm": 28.232913631245626,
+      "grad_norm": 18.22825267425928,
       "learning_rate": 4.550430636492389e-07,
-      "logits/chosen": 0.3172193467617035,
-      "logits/rejected": 1.228100299835205,
-      "logps/chosen": -412.1929626464844,
-      "logps/rejected": -428.08056640625,
-      "loss": 0.5789,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.0178120136260986,
-      "rewards/margins": 0.6625908613204956,
-      "rewards/rejected": -1.6804027557373047,
+      "logits/chosen": 0.28136759996414185,
+      "logits/rejected": 1.2520945072174072,
+      "logps/chosen": -414.25665283203125,
+      "logps/rejected": -428.6090393066406,
+      "loss": 0.5788,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.0384491682052612,
+      "rewards/margins": 0.647238552570343,
+      "rewards/rejected": -1.6856876611709595,
       "step": 120
     },
     {
       "epoch": 0.2981651376146789,
-      "grad_norm": 23.106046920597972,
+      "grad_norm": 18.72996488177851,
       "learning_rate": 4.429344633468004e-07,
-      "logits/chosen": 1.296276330947876,
-      "logits/rejected": 2.0952706336975098,
-      "logps/chosen": -377.18572998046875,
-      "logps/rejected": -435.5022888183594,
-      "loss": 0.5712,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.8953048586845398,
-      "rewards/margins": 0.8917394876480103,
-      "rewards/rejected": -1.7870445251464844,
+      "logits/chosen": 1.1580041646957397,
+      "logits/rejected": 1.9673328399658203,
+      "logps/chosen": -384.8316650390625,
+      "logps/rejected": -440.20672607421875,
+      "loss": 0.5744,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.9717643857002258,
+      "rewards/margins": 0.8623247146606445,
+      "rewards/rejected": -1.8340890407562256,
       "step": 130
     },
     {
       "epoch": 0.3211009174311927,
-      "grad_norm": 21.3617509080007,
+      "grad_norm": 18.77533851044078,
       "learning_rate": 4.2958733752443187e-07,
-      "logits/chosen": 1.0354318618774414,
-      "logits/rejected": 2.103768825531006,
-      "logps/chosen": -374.42938232421875,
-      "logps/rejected": -408.32342529296875,
-      "loss": 0.5477,
+      "logits/chosen": 0.9655276536941528,
+      "logits/rejected": 1.986130952835083,
+      "logps/chosen": -377.4757995605469,
+      "logps/rejected": -408.6956481933594,
+      "loss": 0.553,
       "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.9035611152648926,
-      "rewards/margins": 0.7404158115386963,
-      "rewards/rejected": -1.643977165222168,
+      "rewards/chosen": -0.9340255856513977,
+      "rewards/margins": 0.7136737704277039,
+      "rewards/rejected": -1.6476993560791016,
       "step": 140
     },
     {
       "epoch": 0.3440366972477064,
-      "grad_norm": 21.53708307235743,
+      "grad_norm": 22.441752676286086,
       "learning_rate": 4.150873668617898e-07,
-      "logits/chosen": 0.8976553678512573,
-      "logits/rejected": 2.0599629878997803,
-      "logps/chosen": -370.0615234375,
-      "logps/rejected": -411.820068359375,
-      "loss": 0.5512,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.7934576869010925,
-      "rewards/margins": 0.716931939125061,
-      "rewards/rejected": -1.5103896856307983,
+      "logits/chosen": 1.651755928993225,
+      "logits/rejected": 2.6961984634399414,
+      "logps/chosen": -394.5315856933594,
+      "logps/rejected": -437.6512756347656,
+      "loss": 0.558,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.0381582975387573,
+      "rewards/margins": 0.7305435538291931,
+      "rewards/rejected": -1.7687019109725952,
       "step": 150
     },
     {
       "epoch": 0.3440366972477064,
-      "eval_logits/chosen": 1.9097994565963745,
-      "eval_logits/rejected": 2.9840593338012695,
-      "eval_logps/chosen": -382.4128112792969,
-      "eval_logps/rejected": -426.216552734375,
-      "eval_loss": 0.5533820390701294,
-      "eval_rewards/accuracies": 0.7284482717514038,
-      "eval_rewards/chosen": -0.9840683937072754,
-      "eval_rewards/margins": 0.8216219544410706,
-      "eval_rewards/rejected": -1.8056902885437012,
-      "eval_runtime": 91.5586,
-      "eval_samples_per_second": 19.856,
-      "eval_steps_per_second": 0.317,
+      "eval_logits/chosen": 2.0827815532684326,
+      "eval_logits/rejected": 3.0035645961761475,
+      "eval_logps/chosen": -404.3199157714844,
+      "eval_logps/rejected": -442.60711669921875,
+      "eval_loss": 0.5509841442108154,
+      "eval_rewards/accuracies": 0.7241379022598267,
+      "eval_rewards/chosen": -1.203139305114746,
+      "eval_rewards/margins": 0.7664569616317749,
+      "eval_rewards/rejected": -1.9695963859558105,
+      "eval_runtime": 90.3932,
+      "eval_samples_per_second": 20.112,
+      "eval_steps_per_second": 0.321,
       "step": 150
     },
     {
       "epoch": 0.3669724770642202,
-      "grad_norm": 17.634177990437703,
+      "grad_norm": 24.238500011603442,
       "learning_rate": 3.9952763262280397e-07,
-      "logits/chosen": 1.8957267999649048,
-      "logits/rejected": 2.8357367515563965,
-      "logps/chosen": -408.04705810546875,
-      "logps/rejected": -450.3290100097656,
-      "loss": 0.5609,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.1164201498031616,
-      "rewards/margins": 0.8357731103897095,
-      "rewards/rejected": -1.952193021774292,
+      "logits/chosen": 1.6490274667739868,
+      "logits/rejected": 2.5100581645965576,
+      "logps/chosen": -409.46240234375,
+      "logps/rejected": -448.33001708984375,
+      "loss": 0.557,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1305733919143677,
+      "rewards/margins": 0.8016298413276672,
+      "rewards/rejected": -1.9322032928466797,
       "step": 160
     },
     {
       "epoch": 0.38990825688073394,
-      "grad_norm": 24.6798606158854,
+      "grad_norm": 29.076032215796957,
       "learning_rate": 3.8300801912883414e-07,
-      "logits/chosen": 1.3192155361175537,
-      "logits/rejected": 2.3843648433685303,
-      "logps/chosen": -356.1672058105469,
-      "logps/rejected": -387.1358947753906,
-      "loss": 0.5414,
+      "logits/chosen": 1.5585577487945557,
+      "logits/rejected": 2.380032777786255,
+      "logps/chosen": -372.0144958496094,
+      "logps/rejected": -400.96905517578125,
+      "loss": 0.5388,
       "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.9024198651313782,
-      "rewards/margins": 0.7546060681343079,
-      "rewards/rejected": -1.6570260524749756,
+      "rewards/chosen": -1.0608928203582764,
+      "rewards/margins": 0.7344645261764526,
+      "rewards/rejected": -1.795357346534729,
       "step": 170
     },
     {
       "epoch": 0.41284403669724773,
-      "grad_norm": 22.218831723434445,
+      "grad_norm": 23.777603972721764,
       "learning_rate": 3.6563457256020884e-07,
-      "logits/chosen": 1.3455697298049927,
-      "logits/rejected": 2.5438590049743652,
-      "logps/chosen": -351.62774658203125,
-      "logps/rejected": -430.138427734375,
-      "loss": 0.5396,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.9148648381233215,
-      "rewards/margins": 0.9510505795478821,
-      "rewards/rejected": -1.8659156560897827,
+      "logits/chosen": 1.052141785621643,
+      "logits/rejected": 1.8935604095458984,
+      "logps/chosen": -356.8204650878906,
+      "logps/rejected": -432.20001220703125,
+      "loss": 0.5439,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.9667918086051941,
+      "rewards/margins": 0.9197394251823425,
+      "rewards/rejected": -1.886531114578247,
       "step": 180
     },
     {
       "epoch": 0.43577981651376146,
-      "grad_norm": 25.297690497973328,
+      "grad_norm": 20.231853124698564,
       "learning_rate": 3.475188202022617e-07,
-      "logits/chosen": 1.7387921810150146,
-      "logits/rejected": 2.998396396636963,
-      "logps/chosen": -333.116455078125,
-      "logps/rejected": -437.8639221191406,
-      "loss": 0.5399,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.7444877624511719,
-      "rewards/margins": 1.007294774055481,
-      "rewards/rejected": -1.7517824172973633,
+      "logits/chosen": 1.569053292274475,
+      "logits/rejected": 2.5012192726135254,
+      "logps/chosen": -349.7216491699219,
+      "logps/rejected": -458.28955078125,
+      "loss": 0.5442,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.9105401039123535,
+      "rewards/margins": 1.0454990863800049,
+      "rewards/rejected": -1.9560391902923584,
       "step": 190
     },
     {
       "epoch": 0.45871559633027525,
-      "grad_norm": 25.947440207471573,
+      "grad_norm": 20.18742592623794,
       "learning_rate": 3.287770545059052e-07,
-      "logits/chosen": 1.9480648040771484,
-      "logits/rejected": 2.9033870697021484,
-      "logps/chosen": -380.21185302734375,
-      "logps/rejected": -424.3128967285156,
-      "loss": 0.5364,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.1435054540634155,
-      "rewards/margins": 0.736918032169342,
-      "rewards/rejected": -1.8804235458374023,
+      "logits/chosen": 2.6468214988708496,
+      "logits/rejected": 3.313246965408325,
+      "logps/chosen": -413.1968688964844,
+      "logps/rejected": -454.881591796875,
+      "loss": 0.5346,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.473356008529663,
+      "rewards/margins": 0.712754487991333,
+      "rewards/rejected": -2.186110258102417,
       "step": 200
     },
     {
       "epoch": 0.45871559633027525,
-      "eval_logits/chosen": 2.4303267002105713,
-      "eval_logits/rejected": 3.876626491546631,
-      "eval_logps/chosen": -425.8599853515625,
-      "eval_logps/rejected": -481.8061828613281,
-      "eval_loss": 0.5367683172225952,
-      "eval_rewards/accuracies": 0.732758641242981,
-      "eval_rewards/chosen": -1.4185398817062378,
-      "eval_rewards/margins": 0.9430465698242188,
-      "eval_rewards/rejected": -2.361586570739746,
-      "eval_runtime": 91.0832,
-      "eval_samples_per_second": 19.96,
-      "eval_steps_per_second": 0.318,
+      "eval_logits/chosen": 1.7577229738235474,
+      "eval_logits/rejected": 2.7758734226226807,
+      "eval_logps/chosen": -400.7710876464844,
+      "eval_logps/rejected": -449.201904296875,
+      "eval_loss": 0.5381261706352234,
+      "eval_rewards/accuracies": 0.7112069129943848,
+      "eval_rewards/chosen": -1.1676514148712158,
+      "eval_rewards/margins": 0.8678924441337585,
+      "eval_rewards/rejected": -2.03554368019104,
+      "eval_runtime": 90.283,
+      "eval_samples_per_second": 20.137,
+      "eval_steps_per_second": 0.321,
       "step": 200
     },
     {
       "epoch": 0.481651376146789,
-      "grad_norm": 19.563478098052215,
+      "grad_norm": 21.096800994630236,
       "learning_rate": 3.0952958655864954e-07,
-      "logits/chosen": 3.3911328315734863,
-      "logits/rejected": 4.194566249847412,
-      "logps/chosen": -451.25238037109375,
-      "logps/rejected": -539.4793701171875,
-      "loss": 0.5321,
+      "logits/chosen": 2.1683189868927,
+      "logits/rejected": 2.6720829010009766,
+      "logps/chosen": -401.7050476074219,
+      "logps/rejected": -487.34161376953125,
+      "loss": 0.5345,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.744564414024353,
-      "rewards/margins": 0.8036998510360718,
-      "rewards/rejected": -2.548264503479004,
+      "rewards/chosen": -1.2490909099578857,
+      "rewards/margins": 0.7777953743934631,
+      "rewards/rejected": -2.026886463165283,
       "step": 210
     },
     {
       "epoch": 0.5045871559633027,
-      "grad_norm": 24.94888206530693,
+      "grad_norm": 35.955511790614246,
       "learning_rate": 2.898999737583448e-07,
-      "logits/chosen": 2.1577422618865967,
-      "logits/rejected": 3.670943021774292,
-      "logps/chosen": -426.3487854003906,
-      "logps/rejected": -508.80780029296875,
-      "loss": 0.5365,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.5799643993377686,
-      "rewards/margins": 0.9677878618240356,
-      "rewards/rejected": -2.5477521419525146,
+      "logits/chosen": 1.9502754211425781,
+      "logits/rejected": 2.887373447418213,
+      "logps/chosen": -407.0714111328125,
+      "logps/rejected": -475.75860595703125,
+      "loss": 0.5405,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.3871901035308838,
+      "rewards/margins": 0.8300696611404419,
+      "rewards/rejected": -2.2172598838806152,
       "step": 220
     },
     {
       "epoch": 0.5275229357798165,
-      "grad_norm": 23.826139773858404,
+      "grad_norm": 21.81682834473053,
       "learning_rate": 2.7001422664752333e-07,
-      "logits/chosen": 0.8777297735214233,
-      "logits/rejected": 2.3443570137023926,
-      "logps/chosen": -384.8174743652344,
-      "logps/rejected": -467.87298583984375,
-      "loss": 0.5416,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.1064906120300293,
-      "rewards/margins": 1.0363706350326538,
-      "rewards/rejected": -2.1428613662719727,
+      "logits/chosen": 2.0954604148864746,
+      "logits/rejected": 3.134028673171997,
+      "logps/chosen": -393.80865478515625,
+      "logps/rejected": -481.6973571777344,
+      "loss": 0.535,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.1964021921157837,
+      "rewards/margins": 1.084702968597412,
+      "rewards/rejected": -2.281104803085327,
       "step": 230
     },
     {
       "epoch": 0.5504587155963303,
-      "grad_norm": 18.150746967508407,
+      "grad_norm": 20.331534801215742,
       "learning_rate": 2.5e-07,
-      "logits/chosen": 2.9278922080993652,
-      "logits/rejected": 3.511791706085205,
-      "logps/chosen": -450.8653259277344,
-      "logps/rejected": -547.5108642578125,
-      "loss": 0.5638,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -1.810927391052246,
-      "rewards/margins": 1.0066088438034058,
-      "rewards/rejected": -2.8175363540649414,
+      "logits/chosen": 2.4693617820739746,
+      "logits/rejected": 2.7029402256011963,
+      "logps/chosen": -397.209716796875,
+      "logps/rejected": -480.30621337890625,
+      "loss": 0.5634,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.274371862411499,
+      "rewards/margins": 0.8711179494857788,
+      "rewards/rejected": -2.1454896926879883,
       "step": 240
     },
     {
       "epoch": 0.573394495412844,
-      "grad_norm": 21.73286552315769,
+      "grad_norm": 21.16814139127329,
       "learning_rate": 2.2998577335247667e-07,
-      "logits/chosen": 2.6097209453582764,
-      "logits/rejected": 3.895547389984131,
-      "logps/chosen": -484.319091796875,
-      "logps/rejected": -561.3115234375,
-      "loss": 0.5308,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -2.1941843032836914,
-      "rewards/margins": 0.9737985730171204,
-      "rewards/rejected": -3.167982578277588,
+      "logits/chosen": 2.334216356277466,
+      "logits/rejected": 3.1122984886169434,
+      "logps/chosen": -399.35968017578125,
+      "logps/rejected": -462.42877197265625,
+      "loss": 0.5391,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.344590425491333,
+      "rewards/margins": 0.8345645070075989,
+      "rewards/rejected": -2.179154872894287,
       "step": 250
     },
     {
       "epoch": 0.573394495412844,
-      "eval_logits/chosen": 2.454780101776123,
-      "eval_logits/rejected": 4.2408928871154785,
-      "eval_logps/chosen": -519.5304565429688,
-      "eval_logps/rejected": -602.6265869140625,
-      "eval_loss": 0.5234553217887878,
-      "eval_rewards/accuracies": 0.7284482717514038,
-      "eval_rewards/chosen": -2.3552448749542236,
-      "eval_rewards/margins": 1.2145458459854126,
-      "eval_rewards/rejected": -3.5697906017303467,
-      "eval_runtime": 91.7528,
-      "eval_samples_per_second": 19.814,
-      "eval_steps_per_second": 0.316,
+      "eval_logits/chosen": 1.8166545629501343,
+      "eval_logits/rejected": 2.9561386108398438,
+      "eval_logps/chosen": -392.5903015136719,
+      "eval_logps/rejected": -442.3040771484375,
+      "eval_loss": 0.5333030819892883,
+      "eval_rewards/accuracies": 0.7198275923728943,
+      "eval_rewards/chosen": -1.0858436822891235,
+      "eval_rewards/margins": 0.8807222843170166,
+      "eval_rewards/rejected": -1.9665659666061401,
+      "eval_runtime": 91.6089,
+      "eval_samples_per_second": 19.845,
+      "eval_steps_per_second": 0.317,
       "step": 250
     },
     {
       "epoch": 0.5963302752293578,
-      "grad_norm": 29.190042715796082,
+      "grad_norm": 24.05630881187602,
       "learning_rate": 2.1010002624165524e-07,
-      "logits/chosen": 2.5975215435028076,
-      "logits/rejected": 4.360453128814697,
-      "logps/chosen": -554.3533935546875,
-      "logps/rejected": -655.7716064453125,
-      "loss": 0.5307,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -2.5185165405273438,
-      "rewards/margins": 1.447409987449646,
-      "rewards/rejected": -3.9659264087677,
+      "logits/chosen": 2.180393934249878,
+      "logits/rejected": 3.2447829246520996,
+      "logps/chosen": -416.7367248535156,
+      "logps/rejected": -477.38671875,
+      "loss": 0.5431,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.1423505544662476,
+      "rewards/margins": 1.0397279262542725,
+      "rewards/rejected": -2.1820783615112305,
       "step": 260
     },
     {
       "epoch": 0.6192660550458715,
-      "grad_norm": 20.081922974931803,
+      "grad_norm": 16.426211814362816,
       "learning_rate": 1.9047041344135043e-07,
-      "logits/chosen": 2.110996723175049,
-      "logits/rejected": 3.4121272563934326,
-      "logps/chosen": -542.6881103515625,
-      "logps/rejected": -613.2174682617188,
-      "loss": 0.5514,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -2.664341449737549,
-      "rewards/margins": 0.9934176206588745,
-      "rewards/rejected": -3.657759189605713,
+      "logits/chosen": 2.4754998683929443,
+      "logits/rejected": 3.3202342987060547,
+      "logps/chosen": -418.9905700683594,
+      "logps/rejected": -466.9713439941406,
+      "loss": 0.5554,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.4273664951324463,
+      "rewards/margins": 0.7679312229156494,
+      "rewards/rejected": -2.1952977180480957,
       "step": 270
     },
     {
       "epoch": 0.6422018348623854,
-      "grad_norm": 20.797156741141926,
+      "grad_norm": 25.36799111369545,
       "learning_rate": 1.7122294549409482e-07,
-      "logits/chosen": 1.95541250705719,
-      "logits/rejected": 3.574702024459839,
-      "logps/chosen": -541.0426025390625,
-      "logps/rejected": -644.5146484375,
-      "loss": 0.522,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -2.637174129486084,
-      "rewards/margins": 1.117336630821228,
-      "rewards/rejected": -3.7545104026794434,
+      "logits/chosen": 2.9461216926574707,
+      "logits/rejected": 3.8612606525421143,
+      "logps/chosen": -443.60198974609375,
+      "logps/rejected": -535.1948852539062,
+      "loss": 0.5313,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.662767767906189,
+      "rewards/margins": 0.998543918132782,
+      "rewards/rejected": -2.6613118648529053,
       "step": 280
     },
     {
       "epoch": 0.6651376146788991,
-      "grad_norm": 22.250484161252675,
+      "grad_norm": 15.931208067906516,
       "learning_rate": 1.524811797977383e-07,
-      "logits/chosen": 1.9210926294326782,
-      "logits/rejected": 3.2735812664031982,
-      "logps/chosen": -576.992431640625,
-      "logps/rejected": -663.1363525390625,
-      "loss": 0.5191,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -2.8690013885498047,
-      "rewards/margins": 1.0208569765090942,
-      "rewards/rejected": -3.8898582458496094,
+      "logits/chosen": 2.2281603813171387,
+      "logits/rejected": 3.0743608474731445,
+      "logps/chosen": -415.99908447265625,
+      "logps/rejected": -480.72003173828125,
+      "loss": 0.5279,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.2590678930282593,
+      "rewards/margins": 0.8066269159317017,
+      "rewards/rejected": -2.065694808959961,
       "step": 290
     },
     {
       "epoch": 0.6880733944954128,
-      "grad_norm": 24.39105290704078,
+      "grad_norm": 18.614598999130695,
       "learning_rate": 1.3436542743979125e-07,
-      "logits/chosen": 1.646095871925354,
-      "logits/rejected": 3.443913221359253,
-      "logps/chosen": -558.287353515625,
-      "logps/rejected": -657.3651123046875,
-      "loss": 0.532,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -2.7306606769561768,
-      "rewards/margins": 1.3434104919433594,
-      "rewards/rejected": -4.074070930480957,
+      "logits/chosen": 2.0644378662109375,
+      "logits/rejected": 3.2977874279022217,
+      "logps/chosen": -393.56756591796875,
+      "logps/rejected": -459.68646240234375,
+      "loss": 0.5479,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.0834629535675049,
+      "rewards/margins": 1.0138219594955444,
+      "rewards/rejected": -2.097285032272339,
       "step": 300
     },
     {
       "epoch": 0.6880733944954128,
-      "eval_logits/chosen": 1.1080348491668701,
-      "eval_logits/rejected": 3.015399694442749,
-      "eval_logps/chosen": -535.3407592773438,
-      "eval_logps/rejected": -617.0262451171875,
-      "eval_loss": 0.5116756558418274,
-      "eval_rewards/accuracies": 0.7198275923728943,
-      "eval_rewards/chosen": -2.513347864151001,
-      "eval_rewards/margins": 1.200439691543579,
-      "eval_rewards/rejected": -3.713787794113159,
-      "eval_runtime": 91.1655,
-      "eval_samples_per_second": 19.942,
-      "eval_steps_per_second": 0.318,
+      "eval_logits/chosen": 2.0026185512542725,
+      "eval_logits/rejected": 3.223935604095459,
+      "eval_logps/chosen": -388.63787841796875,
+      "eval_logps/rejected": -442.7093200683594,
+      "eval_loss": 0.5265418291091919,
+      "eval_rewards/accuracies": 0.7068965435028076,
+      "eval_rewards/chosen": -1.0463188886642456,
+      "eval_rewards/margins": 0.9242996573448181,
+      "eval_rewards/rejected": -1.970618486404419,
+      "eval_runtime": 90.447,
+      "eval_samples_per_second": 20.1,
+      "eval_steps_per_second": 0.321,
       "step": 300
     },
     {
       "epoch": 0.7110091743119266,
-      "grad_norm": 27.989896858900266,
+      "grad_norm": 25.782071483124422,
       "learning_rate": 1.1699198087116588e-07,
-      "logits/chosen": 2.1850762367248535,
-      "logits/rejected": 3.5708484649658203,
-      "logps/chosen": -533.7103271484375,
-      "logps/rejected": -638.1661376953125,
-      "loss": 0.54,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -2.7955188751220703,
-      "rewards/margins": 1.163648247718811,
-      "rewards/rejected": -3.959167003631592,
+      "logits/chosen": 2.8770992755889893,
+      "logits/rejected": 3.6848435401916504,
+      "logps/chosen": -387.76580810546875,
+      "logps/rejected": -468.38275146484375,
+      "loss": 0.5499,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.336073875427246,
+      "rewards/margins": 0.9252589344978333,
+      "rewards/rejected": -2.2613327503204346,
       "step": 310
     },
     {
       "epoch": 0.7339449541284404,
-      "grad_norm": 26.423781855718417,
+      "grad_norm": 23.531042495765035,
       "learning_rate": 1.00472367377196e-07,
-      "logits/chosen": 1.8076130151748657,
-      "logits/rejected": 4.071971893310547,
-      "logps/chosen": -614.3458251953125,
-      "logps/rejected": -702.5189819335938,
-      "loss": 0.5138,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -3.0845131874084473,
-      "rewards/margins": 1.4407538175582886,
-      "rewards/rejected": -4.525267601013184,
+      "logits/chosen": 2.587601900100708,
+      "logits/rejected": 3.9543087482452393,
+      "logps/chosen": -440.2958984375,
+      "logps/rejected": -498.0613708496094,
+      "loss": 0.5302,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.3440136909484863,
+      "rewards/margins": 1.1366775035858154,
+      "rewards/rejected": -2.4806911945343018,
       "step": 320
     },
     {
       "epoch": 0.7568807339449541,
-      "grad_norm": 24.484829935546955,
+      "grad_norm": 22.178841978203927,
       "learning_rate": 8.49126331382102e-08,
-      "logits/chosen": 2.0375962257385254,
-      "logits/rejected": 3.5112037658691406,
-      "logps/chosen": -607.512939453125,
-      "logps/rejected": -720.7943115234375,
-      "loss": 0.5187,
+      "logits/chosen": 2.5279412269592285,
+      "logits/rejected": 3.4965198040008545,
+      "logps/chosen": -422.66168212890625,
+      "logps/rejected": -501.438720703125,
+      "loss": 0.5342,
       "rewards/accuracies": 0.6875,
-      "rewards/chosen": -3.2878499031066895,
-      "rewards/margins": 1.2010066509246826,
-      "rewards/rejected": -4.488856315612793,
+      "rewards/chosen": -1.4393374919891357,
+      "rewards/margins": 0.8559640645980835,
+      "rewards/rejected": -2.295301914215088,
       "step": 330
     },
     {
       "epoch": 0.7798165137614679,
-      "grad_norm": 23.41145949202815,
+      "grad_norm": 19.61314237963683,
       "learning_rate": 7.041266247556812e-08,
-      "logits/chosen": 1.657248854637146,
-      "logits/rejected": 3.65797758102417,
-      "logps/chosen": -553.1179809570312,
-      "logps/rejected": -692.9093017578125,
-      "loss": 0.5258,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -2.909062623977661,
-      "rewards/margins": 1.4310705661773682,
-      "rewards/rejected": -4.340132713317871,
+      "logits/chosen": 2.785928726196289,
+      "logits/rejected": 3.915510892868042,
+      "logps/chosen": -388.799072265625,
+      "logps/rejected": -494.65606689453125,
+      "loss": 0.5294,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.265873670578003,
+      "rewards/margins": 1.0917268991470337,
+      "rewards/rejected": -2.357600450515747,
       "step": 340
     },
     {
       "epoch": 0.8027522935779816,
-      "grad_norm": 28.908000369301963,
+      "grad_norm": 22.588827480706584,
       "learning_rate": 5.706553665319955e-08,
-      "logits/chosen": 0.9193560481071472,
-      "logits/rejected": 3.4261555671691895,
-      "logps/chosen": -548.4246215820312,
-      "logps/rejected": -669.2120361328125,
-      "loss": 0.5064,
+      "logits/chosen": 2.3770060539245605,
+      "logits/rejected": 4.068874835968018,
+      "logps/chosen": -419.5255432128906,
+      "logps/rejected": -510.02911376953125,
+      "loss": 0.5232,
       "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.572448968887329,
-      "rewards/margins": 1.67291259765625,
-      "rewards/rejected": -4.245361804962158,
+      "rewards/chosen": -1.2834579944610596,
+      "rewards/margins": 1.3700745105743408,
+      "rewards/rejected": -2.6535322666168213,
       "step": 350
     },
     {
       "epoch": 0.8027522935779816,
-      "eval_logits/chosen": 0.7126501202583313,
-      "eval_logits/rejected": 2.685429096221924,
-      "eval_logps/chosen": -547.6198120117188,
-      "eval_logps/rejected": -636.6490478515625,
-      "eval_loss": 0.5116574168205261,
+      "eval_logits/chosen": 2.348414182662964,
+      "eval_logits/rejected": 3.6065878868103027,
+      "eval_logps/chosen": -417.5965881347656,
+      "eval_logps/rejected": -477.5577392578125,
+      "eval_loss": 0.5262271761894226,
       "eval_rewards/accuracies": 0.7241379022598267,
-      "eval_rewards/chosen": -2.636138677597046,
-      "eval_rewards/margins": 1.2738765478134155,
-      "eval_rewards/rejected": -3.9100148677825928,
-      "eval_runtime": 90.9536,
-      "eval_samples_per_second": 19.988,
-      "eval_steps_per_second": 0.319,
+      "eval_rewards/chosen": -1.3359062671661377,
+      "eval_rewards/margins": 0.9831959009170532,
+      "eval_rewards/rejected": -2.3191022872924805,
+      "eval_runtime": 91.8801,
+      "eval_samples_per_second": 19.787,
+      "eval_steps_per_second": 0.316,
       "step": 350
     },
     {
       "epoch": 0.8256880733944955,
-      "grad_norm": 24.395714355317615,
+      "grad_norm": 22.898724036504742,
       "learning_rate": 4.4956936350761005e-08,
-      "logits/chosen": 0.9300888180732727,
-      "logits/rejected": 2.3748581409454346,
-      "logps/chosen": -543.5307006835938,
-      "logps/rejected": -659.3692016601562,
-      "loss": 0.5084,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -2.6378746032714844,
-      "rewards/margins": 1.2973625659942627,
-      "rewards/rejected": -3.935237169265747,
+      "logits/chosen": 2.4756264686584473,
+      "logits/rejected": 3.231902599334717,
+      "logps/chosen": -419.9034118652344,
+      "logps/rejected": -510.82781982421875,
+      "loss": 0.5254,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.401601791381836,
+      "rewards/margins": 1.0482218265533447,
+      "rewards/rejected": -2.4498236179351807,
       "step": 360
     },
     {
       "epoch": 0.8486238532110092,
-      "grad_norm": 22.09369129566989,
+      "grad_norm": 21.290872916140614,
       "learning_rate": 3.416459164418123e-08,
-      "logits/chosen": 0.2887948155403137,
-      "logits/rejected": 2.4889461994171143,
-      "logps/chosen": -591.7805786132812,
-      "logps/rejected": -667.9407348632812,
-      "loss": 0.5109,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -2.5994677543640137,
-      "rewards/margins": 1.3468105792999268,
-      "rewards/rejected": -3.9462783336639404,
+      "logits/chosen": 1.8261902332305908,
+      "logits/rejected": 3.2766151428222656,
+      "logps/chosen": -459.34906005859375,
+      "logps/rejected": -512.47314453125,
+      "loss": 0.5204,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.2751537561416626,
+      "rewards/margins": 1.1164480447769165,
+      "rewards/rejected": -2.391602039337158,
       "step": 370
     },
     {
       "epoch": 0.8715596330275229,
-      "grad_norm": 26.209345843998328,
+      "grad_norm": 20.41896976274452,
       "learning_rate": 2.475778302439524e-08,
-      "logits/chosen": 0.8550162315368652,
-      "logits/rejected": 3.1779205799102783,
-      "logps/chosen": -580.2351684570312,
-      "logps/rejected": -637.8566284179688,
-      "loss": 0.5075,
+      "logits/chosen": 2.1876559257507324,
+      "logits/rejected": 3.5514347553253174,
+      "logps/chosen": -429.52801513671875,
+      "logps/rejected": -452.6607360839844,
+      "loss": 0.5244,
       "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -2.8695898056030273,
-      "rewards/margins": 1.257644534111023,
-      "rewards/rejected": -4.127234935760498,
+      "rewards/chosen": -1.362518548965454,
+      "rewards/margins": 0.9127564430236816,
+      "rewards/rejected": -2.2752749919891357,
       "step": 380
     },
     {
       "epoch": 0.8944954128440367,
-      "grad_norm": 21.12858235158005,
+      "grad_norm": 20.106111939027084,
       "learning_rate": 1.6796896657433805e-08,
-      "logits/chosen": 0.47364893555641174,
-      "logits/rejected": 2.78879451751709,
-      "logps/chosen": -587.971435546875,
-      "logps/rejected": -724.6598510742188,
-      "loss": 0.5014,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -2.7999932765960693,
-      "rewards/margins": 1.9162429571151733,
-      "rewards/rejected": -4.716236591339111,
+      "logits/chosen": 1.5682854652404785,
+      "logits/rejected": 3.198239803314209,
+      "logps/chosen": -423.41143798828125,
+      "logps/rejected": -513.44140625,
+      "loss": 0.5138,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.1543933153152466,
+      "rewards/margins": 1.4496588706970215,
+      "rewards/rejected": -2.6040520668029785,
       "step": 390
     },
     {
       "epoch": 0.9174311926605505,
-      "grad_norm": 23.688779288096637,
+      "grad_norm": 22.36268387575501,
       "learning_rate": 1.0333036740834855e-08,
-      "logits/chosen": 1.2185232639312744,
-      "logits/rejected": 2.7246413230895996,
-      "logps/chosen": -606.762939453125,
-      "logps/rejected": -717.7750244140625,
-      "loss": 0.5105,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -3.1490612030029297,
-      "rewards/margins": 1.1964399814605713,
-      "rewards/rejected": -4.345500946044922,
+      "logits/chosen": 2.2944397926330566,
+      "logits/rejected": 3.2362308502197266,
+      "logps/chosen": -427.0224609375,
+      "logps/rejected": -509.18438720703125,
+      "loss": 0.5267,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.3516565561294556,
+      "rewards/margins": 0.9079391360282898,
+      "rewards/rejected": -2.2595956325531006,
       "step": 400
     },
     {
       "epoch": 0.9174311926605505,
-      "eval_logits/chosen": 0.863433837890625,
-      "eval_logits/rejected": 2.9146454334259033,
-      "eval_logps/chosen": -575.282958984375,
-      "eval_logps/rejected": -674.3233032226562,
-      "eval_loss": 0.5098804235458374,
-      "eval_rewards/accuracies": 0.732758641242981,
-      "eval_rewards/chosen": -2.9127700328826904,
-      "eval_rewards/margins": 1.3739889860153198,
-      "eval_rewards/rejected": -4.286758899688721,
-      "eval_runtime": 91.1821,
-      "eval_samples_per_second": 19.938,
-      "eval_steps_per_second": 0.318,
+      "eval_logits/chosen": 1.9855237007141113,
+      "eval_logits/rejected": 3.3069264888763428,
+      "eval_logps/chosen": -402.9078674316406,
+      "eval_logps/rejected": -463.85418701171875,
+      "eval_loss": 0.5237594246864319,
+      "eval_rewards/accuracies": 0.7241379022598267,
+      "eval_rewards/chosen": -1.189018964767456,
+      "eval_rewards/margins": 0.9930478930473328,
+      "eval_rewards/rejected": -2.1820664405822754,
+      "eval_runtime": 90.561,
+      "eval_samples_per_second": 20.075,
+      "eval_steps_per_second": 0.32,
       "step": 400
     },
     {
       "epoch": 0.9403669724770642,
-      "grad_norm": 23.71509307695075,
+      "grad_norm": 20.20141424383877,
       "learning_rate": 5.4076974448211685e-09,
-      "logits/chosen": 1.313011884689331,
-      "logits/rejected": 2.788435459136963,
-      "logps/chosen": -609.2062377929688,
-      "logps/rejected": -684.1138305664062,
-      "loss": 0.5379,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -3.2180449962615967,
-      "rewards/margins": 1.0637754201889038,
-      "rewards/rejected": -4.281820297241211,
+      "logits/chosen": 2.3932690620422363,
+      "logits/rejected": 3.2205722332000732,
+      "logps/chosen": -426.5123596191406,
+      "logps/rejected": -476.37139892578125,
+      "loss": 0.5452,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.391105055809021,
+      "rewards/margins": 0.8132905960083008,
+      "rewards/rejected": -2.2043957710266113,
       "step": 410
     },
     {
       "epoch": 0.963302752293578,
-      "grad_norm": 27.57431143958426,
+      "grad_norm": 20.629666257184397,
       "learning_rate": 2.052496544188487e-09,
-      "logits/chosen": 1.1412100791931152,
-      "logits/rejected": 3.4668610095977783,
-      "logps/chosen": -616.3365478515625,
-      "logps/rejected": -685.5581665039062,
-      "loss": 0.5129,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -3.115288496017456,
-      "rewards/margins": 1.404497504234314,
-      "rewards/rejected": -4.5197858810424805,
+      "logits/chosen": 2.141890048980713,
+      "logits/rejected": 3.76823091506958,
+      "logps/chosen": -436.96722412109375,
+      "logps/rejected": -471.711181640625,
+      "loss": 0.5323,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.3215954303741455,
+      "rewards/margins": 1.0597209930419922,
+      "rewards/rejected": -2.381316661834717,
       "step": 420
     },
     {
       "epoch": 0.9862385321100917,
-      "grad_norm": 19.004038331375536,
+      "grad_norm": 17.42236283649955,
       "learning_rate": 2.889724508297886e-10,
-      "logits/chosen": 1.3194568157196045,
-      "logits/rejected": 2.8764185905456543,
-      "logps/chosen": -561.5572509765625,
-      "logps/rejected": -680.3653564453125,
-      "loss": 0.5165,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -2.9964187145233154,
-      "rewards/margins": 1.2785086631774902,
-      "rewards/rejected": -4.274927616119385,
+      "logits/chosen": 2.458095073699951,
+      "logits/rejected": 3.361394166946411,
+      "logps/chosen": -389.62994384765625,
+      "logps/rejected": -474.5247497558594,
+      "loss": 0.5251,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.2771459817886353,
+      "rewards/margins": 0.9393760561943054,
+      "rewards/rejected": -2.216521739959717,
       "step": 430
     },
     {
       "epoch": 1.0,
       "step": 436,
       "total_flos": 0.0,
-      "train_loss": 0.5604413999330013,
-      "train_runtime": 11415.5934,
-      "train_samples_per_second": 4.884,
+      "train_loss": 0.5659637576943144,
+      "train_runtime": 11398.0027,
+      "train_samples_per_second": 4.892,
       "train_steps_per_second": 0.038
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6d1a1283fc740b71c3fbb04baf5f0b79a20c9e4b02b30ac84f6e9343844e7fe6
+oid sha256:aa458f705336ea17545535320d8e0014b23f0668b0759ac5a0227ab2a56cb93f
 size 7544