Model save

README.md

@@ -4,6 +4,7 @@ base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
 - trl
 - dpo
+- alignment-handbook
 - generated_from_trainer
 model-index:
 - name: zephyr-7b-dpo-full-prometheus_consistent-reward-scale-1
@@ -17,15 +18,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5207
-- Rewards/chosen: -1.2231
-- Rewards/rejected: -2.4659
-- Rewards/accuracies: 0.7716
-- Rewards/margins: 1.2428
-- Logps/rejected: -465.6644
-- Logps/chosen: -397.9129
-- Logits/rejected: 0.9533
-- Logits/chosen: -0.4953
+- Loss: 0.5209
+- Rewards/chosen: -1.1123
+- Rewards/rejected: -2.3595
+- Rewards/accuracies: 0.7672
+- Rewards/margins: 1.2472
+- Logps/rejected: -455.0307
+- Logps/chosen: -386.8380
+- Logits/rejected: 3.7050
+- Logits/chosen: 2.1175
 
 ## Model description
 
@@ -62,14 +63,14 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.6662        | 0.1143 | 50   | 0.6533          | -0.0057        | -0.1647          | 0.6853             | 0.1590          | -235.5430      | -276.1736    | -2.4378         | -2.5411       |
-| 0.6064        | 0.2286 | 100  | 0.6013          | -0.6726        | -1.2005          | 0.7414             | 0.5279          | -339.1220      | -342.8641    | -2.3762         | -2.4982       |
-| 0.5361        | 0.3429 | 150  | 0.5524          | -0.9597        | -2.0693          | 0.7629             | 1.1096          | -426.0058      | -371.5730    | 0.1212          | -0.9465       |
-| 0.5408        | 0.4571 | 200  | 0.5365          | -0.9091        | -2.0574          | 0.7543             | 1.1483          | -424.8160      | -366.5098    | 0.2603          | -1.1779       |
-| 0.5444        | 0.5714 | 250  | 0.5319          | -1.0642        | -2.2338          | 0.7672             | 1.1696          | -442.4582      | -382.0276    | 0.2351          | -0.8157       |
-| 0.5195        | 0.6857 | 300  | 0.5224          | -1.0055        | -2.2442          | 0.7802             | 1.2387          | -443.4946      | -376.1569    | 0.5828          | -0.7317       |
-| 0.5294        | 0.8    | 350  | 0.5203          | -1.1639        | -2.4063          | 0.7543             | 1.2424          | -459.7086      | -391.9946    | 0.8069          | -0.5934       |
-| 0.5494        | 0.9143 | 400  | 0.5207          | -1.2231        | -2.4659          | 0.7716             | 1.2428          | -465.6644      | -397.9129    | 0.9533          | -0.4953       |
+| 0.6663        | 0.1143 | 50   | 0.6532          | -0.0060        | -0.1654          | 0.6897             | 0.1594          | -235.6116      | -276.2017    | -2.4378         | -2.5412       |
+| 0.6051        | 0.2286 | 100  | 0.6128          | -0.8115        | -1.2713          | 0.7112             | 0.4599          | -346.2107      | -356.7497    | -2.3746         | -2.4933       |
+| 0.5375        | 0.3429 | 150  | 0.5487          | -0.8314        | -1.9411          | 0.7672             | 1.1097          | -413.1859      | -358.7466    | 1.3727          | 0.2031        |
+| 0.5435        | 0.4571 | 200  | 0.5359          | -0.9895        | -2.0771          | 0.7543             | 1.0876          | -426.7858      | -374.5490    | 2.7076          | 1.3441        |
+| 0.5433        | 0.5714 | 250  | 0.5290          | -0.9003        | -2.0663          | 0.7629             | 1.1660          | -425.7063      | -365.6371    | 2.4121          | 0.8588        |
+| 0.5194        | 0.6857 | 300  | 0.5213          | -0.9607        | -2.2057          | 0.7716             | 1.2450          | -439.6500      | -371.6745    | 3.3117          | 1.7085        |
+| 0.5325        | 0.8    | 350  | 0.5216          | -1.1386        | -2.3763          | 0.7629             | 1.2377          | -456.7085      | -389.4649    | 3.6708          | 2.0842        |
+| 0.5483        | 0.9143 | 400  | 0.5209          | -1.1123        | -2.3595          | 0.7672             | 1.2472          | -455.0307      | -386.8380    | 3.7050          | 2.1175        |
 
 
 ### Framework versions

all_results.json

@@ -1,9 +1,22 @@
 {
     "epoch": 0.9988571428571429,
+    "eval_logits/chosen": -0.5392131209373474,
+    "eval_logits/rejected": 0.8578556776046753,
+    "eval_logps/chosen": -395.783447265625,
+    "eval_logps/rejected": -463.0950012207031,
+    "eval_loss": 0.5202247500419617,
+    "eval_rewards/accuracies": 0.7715517282485962,
+    "eval_rewards/chosen": -1.2017946243286133,
+    "eval_rewards/margins": 1.2383939027786255,
+    "eval_rewards/rejected": -2.440188407897949,
+    "eval_runtime": 91.4513,
+    "eval_samples": 1831,
+    "eval_samples_per_second": 20.022,
+    "eval_steps_per_second": 0.317,
     "total_flos": 0.0,
-    "train_loss": 0.5634412694577494,
-    "train_runtime": 11351.7318,
+    "train_loss": 0.5630035629534339,
+    "train_runtime": 11387.5716,
     "train_samples": 55999,
-    "train_samples_per_second": 4.933,
+    "train_samples_per_second": 4.918,
     "train_steps_per_second": 0.038
 }

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 0.9988571428571429,
+    "eval_logits/chosen": -0.5392131209373474,
+    "eval_logits/rejected": 0.8578556776046753,
+    "eval_logps/chosen": -395.783447265625,
+    "eval_logps/rejected": -463.0950012207031,
+    "eval_loss": 0.5202247500419617,
+    "eval_rewards/accuracies": 0.7715517282485962,
+    "eval_rewards/chosen": -1.2017946243286133,
+    "eval_rewards/margins": 1.2383939027786255,
+    "eval_rewards/rejected": -2.440188407897949,
+    "eval_runtime": 91.4513,
+    "eval_samples": 1831,
+    "eval_samples_per_second": 20.022,
+    "eval_steps_per_second": 0.317
+}

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ad0fbf4d84c6ad8be639b1ad961f5e0067b0a1209980155771d6290df333da23
+oid sha256:f0a8ce97da5127c91a86dc70ed399f0c91aec38f6541f33558e7768f352a7c1c
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b20f14525a8084ec66a3cab4fffaf32fe2343afcaf072255a056873f423c9eb
+oid sha256:c07c837a9dc30d1b62ed3392d568a1efa2e2956c881bee4ac35573a0740d4a06
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bd34bb14714649ff4b2f6e0f4161e406093fab6d585f4d528be315030bc721d9
+oid sha256:0365357c8fc99127f5ae4c330b6991cb1f13cfb273d930921f1bb20e4ce8c500
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9988571428571429,
     "total_flos": 0.0,
-    "train_loss": 0.5634412694577494,
-    "train_runtime": 11351.7318,
+    "train_loss": 0.5630035629534339,
+    "train_runtime": 11387.5716,
     "train_samples": 55999,
-    "train_samples_per_second": 4.933,
+    "train_samples_per_second": 4.918,
     "train_steps_per_second": 0.038
 }

trainer_state.json

@@ -10,784 +10,784 @@
   "log_history": [
     {
       "epoch": 0.022857142857142857,
-      "grad_norm": 6.7934202337025855,
+      "grad_norm": 6.795239341624469,
       "learning_rate": 1.1363636363636363e-07,
-      "logits/chosen": -2.700854778289795,
-      "logits/rejected": -2.6249899864196777,
-      "logps/chosen": -301.2730712890625,
-      "logps/rejected": -281.7767028808594,
+      "logits/chosen": -2.700852632522583,
+      "logits/rejected": -2.6250014305114746,
+      "logps/chosen": -301.27313232421875,
+      "logps/rejected": -281.78619384765625,
       "loss": 0.6931,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.00013105585821904242,
-      "rewards/margins": 0.0004016496823169291,
-      "rewards/rejected": -0.0002705938240978867,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": 0.0001308169448748231,
+      "rewards/margins": 0.0004958957433700562,
+      "rewards/rejected": -0.00036507885670289397,
       "step": 10
     },
     {
       "epoch": 0.045714285714285714,
-      "grad_norm": 5.3206971599426955,
+      "grad_norm": 5.31428372226332,
       "learning_rate": 2.2727272727272726e-07,
-      "logits/chosen": -2.6406970024108887,
-      "logits/rejected": -2.6055245399475098,
-      "logps/chosen": -278.92767333984375,
-      "logps/rejected": -254.66806030273438,
-      "loss": 0.6925,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.0024350073654204607,
-      "rewards/margins": 0.001425002934411168,
-      "rewards/rejected": 0.0010100044310092926,
+      "logits/chosen": -2.6415421962738037,
+      "logits/rejected": -2.606222629547119,
+      "logps/chosen": -278.8970642089844,
+      "logps/rejected": -254.64749145507812,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.0027411712799221277,
+      "rewards/margins": 0.001525188097730279,
+      "rewards/rejected": 0.001215982949361205,
       "step": 20
     },
     {
       "epoch": 0.06857142857142857,
-      "grad_norm": 5.928979160384808,
+      "grad_norm": 5.9664481153189435,
       "learning_rate": 3.4090909090909085e-07,
-      "logits/chosen": -2.63812518119812,
-      "logits/rejected": -2.617269992828369,
-      "logps/chosen": -263.2200622558594,
-      "logps/rejected": -263.3509216308594,
-      "loss": 0.6884,
+      "logits/chosen": -2.638169765472412,
+      "logits/rejected": -2.617159843444824,
+      "logps/chosen": -263.23223876953125,
+      "logps/rejected": -263.40374755859375,
+      "loss": 0.6883,
       "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": 0.014629853889346123,
-      "rewards/margins": 0.00820869579911232,
-      "rewards/rejected": 0.0064211576245725155,
+      "rewards/chosen": 0.014508177526295185,
+      "rewards/margins": 0.00861530750989914,
+      "rewards/rejected": 0.0058928681537508965,
       "step": 30
     },
     {
       "epoch": 0.09142857142857143,
-      "grad_norm": 6.666624768443373,
+      "grad_norm": 6.667336557428276,
       "learning_rate": 4.545454545454545e-07,
-      "logits/chosen": -2.649786949157715,
-      "logits/rejected": -2.586179733276367,
-      "logps/chosen": -290.2237548828125,
-      "logps/rejected": -268.332763671875,
-      "loss": 0.6786,
+      "logits/chosen": -2.648975372314453,
+      "logits/rejected": -2.585244655609131,
+      "logps/chosen": -290.2044372558594,
+      "logps/rejected": -268.3276062011719,
+      "loss": 0.6785,
       "rewards/accuracies": 0.59375,
-      "rewards/chosen": 0.03773483633995056,
-      "rewards/margins": 0.0447491891682148,
-      "rewards/rejected": -0.0070143528282642365,
+      "rewards/chosen": 0.037928324192762375,
+      "rewards/margins": 0.044891245663166046,
+      "rewards/rejected": -0.006962914951145649,
       "step": 40
     },
     {
       "epoch": 0.11428571428571428,
-      "grad_norm": 9.747697215916363,
+      "grad_norm": 9.813117329804816,
       "learning_rate": 4.997124959943201e-07,
-      "logits/chosen": -2.6796483993530273,
-      "logits/rejected": -2.5985970497131348,
-      "logps/chosen": -293.6056213378906,
-      "logps/rejected": -254.311767578125,
-      "loss": 0.6662,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.027046790346503258,
-      "rewards/margins": 0.10152111202478409,
-      "rewards/rejected": -0.07447431981563568,
+      "logits/chosen": -2.6792047023773193,
+      "logits/rejected": -2.5978188514709473,
+      "logps/chosen": -293.65264892578125,
+      "logps/rejected": -254.2649688720703,
+      "loss": 0.6663,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": 0.026576850563287735,
+      "rewards/margins": 0.10058300197124481,
+      "rewards/rejected": -0.07400616258382797,
       "step": 50
     },
     {
       "epoch": 0.11428571428571428,
-      "eval_logits/chosen": -2.541074275970459,
-      "eval_logits/rejected": -2.4378066062927246,
-      "eval_logps/chosen": -276.17364501953125,
-      "eval_logps/rejected": -235.5430145263672,
-      "eval_loss": 0.6532636284828186,
-      "eval_rewards/accuracies": 0.6853448152542114,
-      "eval_rewards/chosen": -0.005697119049727917,
-      "eval_rewards/margins": 0.1589716672897339,
-      "eval_rewards/rejected": -0.16466876864433289,
-      "eval_runtime": 91.4644,
-      "eval_samples_per_second": 20.019,
-      "eval_steps_per_second": 0.317,
+      "eval_logits/chosen": -2.541201591491699,
+      "eval_logits/rejected": -2.4377598762512207,
+      "eval_logps/chosen": -276.20166015625,
+      "eval_logps/rejected": -235.61155700683594,
+      "eval_loss": 0.6532372832298279,
+      "eval_rewards/accuracies": 0.6896551847457886,
+      "eval_rewards/chosen": -0.005977254826575518,
+      "eval_rewards/margins": 0.15937723219394684,
+      "eval_rewards/rejected": -0.16535447537899017,
+      "eval_runtime": 91.1786,
+      "eval_samples_per_second": 20.081,
+      "eval_steps_per_second": 0.318,
       "step": 50
     },
     {
       "epoch": 0.13714285714285715,
-      "grad_norm": 9.11809665360445,
+      "grad_norm": 8.50170881260791,
       "learning_rate": 4.979579212164186e-07,
-      "logits/chosen": -2.579230308532715,
-      "logits/rejected": -2.47153639793396,
-      "logps/chosen": -293.18316650390625,
-      "logps/rejected": -275.13836669921875,
+      "logits/chosen": -2.5797510147094727,
+      "logits/rejected": -2.472832202911377,
+      "logps/chosen": -293.24212646484375,
+      "logps/rejected": -275.13885498046875,
       "loss": 0.646,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.12677232921123505,
-      "rewards/margins": 0.13909009099006653,
-      "rewards/rejected": -0.2658624053001404,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.12736138701438904,
+      "rewards/margins": 0.1385059803724289,
+      "rewards/rejected": -0.2658673822879791,
       "step": 60
     },
     {
       "epoch": 0.16,
-      "grad_norm": 9.033724858243865,
+      "grad_norm": 9.027696167666651,
       "learning_rate": 4.946196886175515e-07,
-      "logits/chosen": -2.587209701538086,
-      "logits/rejected": -2.537858247756958,
-      "logps/chosen": -293.44488525390625,
-      "logps/rejected": -300.06134033203125,
+      "logits/chosen": -2.5882785320281982,
+      "logits/rejected": -2.539330005645752,
+      "logps/chosen": -293.43145751953125,
+      "logps/rejected": -300.1482849121094,
       "loss": 0.6244,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.1766660511493683,
-      "rewards/margins": 0.2276792824268341,
-      "rewards/rejected": -0.4043453335762024,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.17653189599514008,
+      "rewards/margins": 0.22868318855762482,
+      "rewards/rejected": -0.4052151143550873,
       "step": 70
     },
     {
       "epoch": 0.18285714285714286,
-      "grad_norm": 9.3700719469498,
+      "grad_norm": 10.603734895101157,
       "learning_rate": 4.897191188239667e-07,
-      "logits/chosen": -2.622152805328369,
-      "logits/rejected": -2.571810483932495,
-      "logps/chosen": -285.61334228515625,
-      "logps/rejected": -306.8240051269531,
-      "loss": 0.6129,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.1858430951833725,
-      "rewards/margins": 0.3293076157569885,
-      "rewards/rejected": -0.5151506662368774,
+      "logits/chosen": -2.623680591583252,
+      "logits/rejected": -2.5742952823638916,
+      "logps/chosen": -285.3603820800781,
+      "logps/rejected": -306.60211181640625,
+      "loss": 0.6123,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.18331322073936462,
+      "rewards/margins": 0.3296189308166504,
+      "rewards/rejected": -0.5129320621490479,
       "step": 80
     },
     {
       "epoch": 0.2057142857142857,
-      "grad_norm": 12.801415984987877,
+      "grad_norm": 15.25024463895093,
       "learning_rate": 4.832875107981763e-07,
-      "logits/chosen": -2.6656365394592285,
-      "logits/rejected": -2.609879732131958,
-      "logps/chosen": -298.3469543457031,
-      "logps/rejected": -315.2607421875,
-      "loss": 0.6171,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.2206166684627533,
-      "rewards/margins": 0.3765110969543457,
-      "rewards/rejected": -0.5971277952194214,
+      "logits/chosen": -2.6875650882720947,
+      "logits/rejected": -2.6345021724700928,
+      "logps/chosen": -295.8832092285156,
+      "logps/rejected": -313.13983154296875,
+      "loss": 0.6191,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.19597890973091125,
+      "rewards/margins": 0.37993985414505005,
+      "rewards/rejected": -0.5759187340736389,
       "step": 90
     },
     {
       "epoch": 0.22857142857142856,
-      "grad_norm": 13.512709674743487,
+      "grad_norm": 12.34704700331818,
       "learning_rate": 4.753659419387223e-07,
-      "logits/chosen": -2.659526824951172,
-      "logits/rejected": -2.5662050247192383,
-      "logps/chosen": -337.0499267578125,
-      "logps/rejected": -323.8541564941406,
-      "loss": 0.6064,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.5066351294517517,
-      "rewards/margins": 0.39739224314689636,
-      "rewards/rejected": -0.9040273427963257,
+      "logits/chosen": -2.679297685623169,
+      "logits/rejected": -2.5944952964782715,
+      "logps/chosen": -330.1031799316406,
+      "logps/rejected": -318.5290832519531,
+      "loss": 0.6051,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.43716782331466675,
+      "rewards/margins": 0.4136085510253906,
+      "rewards/rejected": -0.8507764935493469,
       "step": 100
     },
     {
       "epoch": 0.22857142857142856,
-      "eval_logits/chosen": -2.4981839656829834,
-      "eval_logits/rejected": -2.3761825561523438,
-      "eval_logps/chosen": -342.8641357421875,
-      "eval_logps/rejected": -339.1220397949219,
-      "eval_loss": 0.601341187953949,
-      "eval_rewards/accuracies": 0.7413793206214905,
-      "eval_rewards/chosen": -0.6726016402244568,
-      "eval_rewards/margins": 0.5278574824333191,
-      "eval_rewards/rejected": -1.2004590034484863,
-      "eval_runtime": 90.7394,
-      "eval_samples_per_second": 20.179,
-      "eval_steps_per_second": 0.32,
+      "eval_logits/chosen": -2.493269920349121,
+      "eval_logits/rejected": -2.3746213912963867,
+      "eval_logps/chosen": -356.7496643066406,
+      "eval_logps/rejected": -346.2107238769531,
+      "eval_loss": 0.612766683101654,
+      "eval_rewards/accuracies": 0.7112069129943848,
+      "eval_rewards/chosen": -0.8114572167396545,
+      "eval_rewards/margins": 0.4598887860774994,
+      "eval_rewards/rejected": -1.2713459730148315,
+      "eval_runtime": 90.1893,
+      "eval_samples_per_second": 20.302,
+      "eval_steps_per_second": 0.322,
       "step": 100
     },
     {
       "epoch": 0.25142857142857145,
-      "grad_norm": 19.006376300807332,
+      "grad_norm": 13.527645485553371,
       "learning_rate": 4.660050057270191e-07,
-      "logits/chosen": -2.255047559738159,
-      "logits/rejected": -2.1392579078674316,
-      "logps/chosen": -385.914794921875,
-      "logps/rejected": -409.34942626953125,
-      "loss": 0.5827,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.6961116194725037,
-      "rewards/margins": 0.4182862639427185,
-      "rewards/rejected": -1.1143978834152222,
+      "logits/chosen": -2.269178628921509,
+      "logits/rejected": -2.168506622314453,
+      "logps/chosen": -387.71820068359375,
+      "logps/rejected": -411.88427734375,
+      "loss": 0.5873,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.7141460180282593,
+      "rewards/margins": 0.4256005883216858,
+      "rewards/rejected": -1.1397466659545898,
       "step": 110
     },
     {
       "epoch": 0.2742857142857143,
-      "grad_norm": 18.203031953893095,
+      "grad_norm": 17.046687026346753,
       "learning_rate": 4.5526448859687144e-07,
-      "logits/chosen": -1.6530084609985352,
-      "logits/rejected": -1.3287270069122314,
-      "logps/chosen": -383.32232666015625,
-      "logps/rejected": -358.1570129394531,
-      "loss": 0.5761,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.7241784334182739,
-      "rewards/margins": 0.5402495861053467,
-      "rewards/rejected": -1.264427900314331,
+      "logits/chosen": -1.293348789215088,
+      "logits/rejected": -0.927165687084198,
+      "logps/chosen": -381.55316162109375,
+      "logps/rejected": -354.0766906738281,
+      "loss": 0.5755,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.7064869403839111,
+      "rewards/margins": 0.5171381831169128,
+      "rewards/rejected": -1.2236251831054688,
       "step": 120
     },
     {
       "epoch": 0.29714285714285715,
-      "grad_norm": 25.870445845824126,
+      "grad_norm": 19.945418074044913,
       "learning_rate": 4.432129880904388e-07,
-      "logits/chosen": -0.8325661420822144,
-      "logits/rejected": -0.3734116554260254,
-      "logps/chosen": -404.2379455566406,
-      "logps/rejected": -406.59088134765625,
-      "loss": 0.5471,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.9908201098442078,
-      "rewards/margins": 0.5703855752944946,
-      "rewards/rejected": -1.5612057447433472,
+      "logits/chosen": -0.14520399272441864,
+      "logits/rejected": 0.31017133593559265,
+      "logps/chosen": -394.09820556640625,
+      "logps/rejected": -395.47674560546875,
+      "loss": 0.5448,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.8894233703613281,
+      "rewards/margins": 0.5606414675712585,
+      "rewards/rejected": -1.4500648975372314,
       "step": 130
     },
     {
       "epoch": 0.32,
-      "grad_norm": 24.51686978994005,
+      "grad_norm": 23.269894199893105,
       "learning_rate": 4.299274747394055e-07,
-      "logits/chosen": -1.1053862571716309,
-      "logits/rejected": -0.7209962606430054,
-      "logps/chosen": -378.8704833984375,
-      "logps/rejected": -408.87982177734375,
-      "loss": 0.5662,
+      "logits/chosen": 0.3922499716281891,
+      "logits/rejected": 0.7626418471336365,
+      "logps/chosen": -402.1969299316406,
+      "logps/rejected": -436.99725341796875,
+      "loss": 0.5611,
       "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.634228527545929,
-      "rewards/margins": 0.735455334186554,
-      "rewards/rejected": -1.3696839809417725,
+      "rewards/chosen": -0.8674923777580261,
+      "rewards/margins": 0.783365786075592,
+      "rewards/rejected": -1.6508581638336182,
       "step": 140
     },
     {
       "epoch": 0.34285714285714286,
-      "grad_norm": 20.058403951755405,
+      "grad_norm": 18.255646968547413,
       "learning_rate": 4.1549280046953653e-07,
-      "logits/chosen": -1.0304594039916992,
-      "logits/rejected": -0.4146367907524109,
-      "logps/chosen": -371.8462829589844,
-      "logps/rejected": -442.0589904785156,
-      "loss": 0.5361,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.848603367805481,
-      "rewards/margins": 0.8089972734451294,
-      "rewards/rejected": -1.6576006412506104,
+      "logits/chosen": -0.056426752358675,
+      "logits/rejected": 0.6437274813652039,
+      "logps/chosen": -360.7496032714844,
+      "logps/rejected": -432.40399169921875,
+      "loss": 0.5375,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.7376368641853333,
+      "rewards/margins": 0.8234134912490845,
+      "rewards/rejected": -1.5610501766204834,
       "step": 150
     },
     {
       "epoch": 0.34285714285714286,
-      "eval_logits/chosen": -0.9465453028678894,
-      "eval_logits/rejected": 0.12122771143913269,
-      "eval_logps/chosen": -371.5729675292969,
-      "eval_logps/rejected": -426.00579833984375,
-      "eval_loss": 0.5524057149887085,
-      "eval_rewards/accuracies": 0.7629310488700867,
-      "eval_rewards/chosen": -0.9596903324127197,
-      "eval_rewards/margins": 1.1096062660217285,
-      "eval_rewards/rejected": -2.069296360015869,
-      "eval_runtime": 90.5989,
-      "eval_samples_per_second": 20.21,
-      "eval_steps_per_second": 0.32,
+      "eval_logits/chosen": 0.20309801399707794,
+      "eval_logits/rejected": 1.3727048635482788,
+      "eval_logps/chosen": -358.7465515136719,
+      "eval_logps/rejected": -413.1859436035156,
+      "eval_loss": 0.5486596822738647,
+      "eval_rewards/accuracies": 0.767241358757019,
+      "eval_rewards/chosen": -0.8314265012741089,
+      "eval_rewards/margins": 1.1096714735031128,
+      "eval_rewards/rejected": -1.9410980939865112,
+      "eval_runtime": 90.1892,
+      "eval_samples_per_second": 20.302,
+      "eval_steps_per_second": 0.322,
       "step": 150
     },
     {
       "epoch": 0.3657142857142857,
-      "grad_norm": 20.887944941915286,
+      "grad_norm": 20.04208599875873,
       "learning_rate": 4.000011566683401e-07,
-      "logits/chosen": -1.1443428993225098,
-      "logits/rejected": -0.4119592607021332,
-      "logps/chosen": -398.99407958984375,
-      "logps/rejected": -444.39227294921875,
-      "loss": 0.5491,
+      "logits/chosen": 0.4694085121154785,
+      "logits/rejected": 1.3121615648269653,
+      "logps/chosen": -412.69488525390625,
+      "logps/rejected": -459.99188232421875,
+      "loss": 0.548,
       "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.9965378642082214,
-      "rewards/margins": 0.8909576535224915,
-      "rewards/rejected": -1.8874953985214233,
+      "rewards/chosen": -1.1335456371307373,
+      "rewards/margins": 0.9099456071853638,
+      "rewards/rejected": -2.0434913635253906,
       "step": 160
     },
     {
       "epoch": 0.38857142857142857,
-      "grad_norm": 26.245115570494136,
+      "grad_norm": 22.516250506361718,
       "learning_rate": 3.8355148537705047e-07,
-      "logits/chosen": -1.1884818077087402,
-      "logits/rejected": -0.4392818808555603,
-      "logps/chosen": -405.96453857421875,
-      "logps/rejected": -431.96875,
-      "loss": 0.545,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.0185964107513428,
-      "rewards/margins": 0.643780529499054,
-      "rewards/rejected": -1.6623769998550415,
+      "logits/chosen": 0.1457391083240509,
+      "logits/rejected": 0.8692816495895386,
+      "logps/chosen": -395.64947509765625,
+      "logps/rejected": -417.6402282714844,
+      "loss": 0.5469,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.9154456257820129,
+      "rewards/margins": 0.6036463379859924,
+      "rewards/rejected": -1.5190918445587158,
       "step": 170
     },
     {
       "epoch": 0.4114285714285714,
-      "grad_norm": 16.97689595709094,
+      "grad_norm": 20.567923807377685,
       "learning_rate": 3.662488473675315e-07,
-      "logits/chosen": -1.2794054746627808,
-      "logits/rejected": -0.2185342013835907,
-      "logps/chosen": -429.20098876953125,
-      "logps/rejected": -465.3219299316406,
-      "loss": 0.554,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.9902114868164062,
-      "rewards/margins": 1.0635021924972534,
-      "rewards/rejected": -2.053713798522949,
+      "logits/chosen": 0.6181103587150574,
+      "logits/rejected": 1.7128187417984009,
+      "logps/chosen": -436.68780517578125,
+      "logps/rejected": -469.717041015625,
+      "loss": 0.5551,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.065079927444458,
+      "rewards/margins": 1.0325844287872314,
+      "rewards/rejected": -2.0976643562316895,
       "step": 180
     },
     {
       "epoch": 0.4342857142857143,
-      "grad_norm": 15.630086574400098,
+      "grad_norm": 20.909674986872478,
       "learning_rate": 3.48203751140067e-07,
-      "logits/chosen": -1.3240091800689697,
-      "logits/rejected": -0.43907278776168823,
-      "logps/chosen": -380.5593566894531,
-      "logps/rejected": -413.9124450683594,
-      "loss": 0.5389,
+      "logits/chosen": 1.2501403093338013,
+      "logits/rejected": 2.2078864574432373,
+      "logps/chosen": -380.656982421875,
+      "logps/rejected": -409.70556640625,
+      "loss": 0.5412,
       "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.043017864227295,
-      "rewards/margins": 0.7511247992515564,
-      "rewards/rejected": -1.794142484664917,
+      "rewards/chosen": -1.0439938306808472,
+      "rewards/margins": 0.7080799341201782,
+      "rewards/rejected": -1.7520736455917358,
       "step": 190
     },
     {
       "epoch": 0.45714285714285713,
-      "grad_norm": 17.980745825993548,
+      "grad_norm": 21.67729434989596,
       "learning_rate": 3.2953144712759537e-07,
-      "logits/chosen": -1.4564930200576782,
-      "logits/rejected": -0.43932801485061646,
-      "logps/chosen": -365.29913330078125,
-      "logps/rejected": -422.6018981933594,
-      "loss": 0.5408,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.9695641398429871,
-      "rewards/margins": 0.9676488041877747,
-      "rewards/rejected": -1.9372127056121826,
+      "logits/chosen": 0.7689538598060608,
+      "logits/rejected": 1.9063518047332764,
+      "logps/chosen": -359.4909362792969,
+      "logps/rejected": -411.184814453125,
+      "loss": 0.5435,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.9114822149276733,
+      "rewards/margins": 0.91156005859375,
+      "rewards/rejected": -1.8230421543121338,
       "step": 200
     },
     {
       "epoch": 0.45714285714285713,
-      "eval_logits/chosen": -1.1779377460479736,
-      "eval_logits/rejected": 0.2602512240409851,
-      "eval_logps/chosen": -366.50982666015625,
-      "eval_logps/rejected": -424.8160095214844,
-      "eval_loss": 0.5365040898323059,
+      "eval_logits/chosen": 1.3441277742385864,
+      "eval_logits/rejected": 2.707573652267456,
+      "eval_logps/chosen": -374.5489501953125,
+      "eval_logps/rejected": -426.7857971191406,
+      "eval_loss": 0.5358834266662598,
       "eval_rewards/accuracies": 0.7543103694915771,
-      "eval_rewards/chosen": -0.9090585112571716,
-      "eval_rewards/margins": 1.148340106010437,
-      "eval_rewards/rejected": -2.057398557662964,
-      "eval_runtime": 92.0351,
-      "eval_samples_per_second": 19.895,
-      "eval_steps_per_second": 0.315,
+      "eval_rewards/chosen": -0.9894503355026245,
+      "eval_rewards/margins": 1.087646245956421,
+      "eval_rewards/rejected": -2.077096462249756,
+      "eval_runtime": 90.1648,
+      "eval_samples_per_second": 20.307,
+      "eval_steps_per_second": 0.322,
       "step": 200
     },
     {
       "epoch": 0.48,
-      "grad_norm": 20.920453502633716,
+      "grad_norm": 19.28945802551147,
       "learning_rate": 3.103511916141658e-07,
-      "logits/chosen": -0.5911210775375366,
-      "logits/rejected": 0.41120481491088867,
-      "logps/chosen": -377.15191650390625,
-      "logps/rejected": -446.093017578125,
-      "loss": 0.5341,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.13698410987854,
-      "rewards/margins": 0.8711468577384949,
-      "rewards/rejected": -2.0081307888031006,
+      "logits/chosen": 1.5224826335906982,
+      "logits/rejected": 2.394577741622925,
+      "logps/chosen": -385.7353210449219,
+      "logps/rejected": -451.604248046875,
+      "loss": 0.5372,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.2228174209594727,
+      "rewards/margins": 0.8404253125190735,
+      "rewards/rejected": -2.0632426738739014,
       "step": 210
     },
     {
       "epoch": 0.5028571428571429,
-      "grad_norm": 22.32799037201365,
+      "grad_norm": 24.47080032118637,
       "learning_rate": 2.9078548506882117e-07,
-      "logits/chosen": -0.42250823974609375,
-      "logits/rejected": 0.5733365416526794,
-      "logps/chosen": -427.79559326171875,
-      "logps/rejected": -463.9886779785156,
-      "loss": 0.5589,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.4272958040237427,
-      "rewards/margins": 0.7507564425468445,
-      "rewards/rejected": -2.1780521869659424,
+      "logits/chosen": 1.5350468158721924,
+      "logits/rejected": 2.541968822479248,
+      "logps/chosen": -425.51287841796875,
+      "logps/rejected": -466.1084899902344,
+      "loss": 0.5604,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.4044690132141113,
+      "rewards/margins": 0.794781506061554,
+      "rewards/rejected": -2.1992506980895996,
       "step": 220
     },
     {
       "epoch": 0.5257142857142857,
-      "grad_norm": 22.22957333517813,
+      "grad_norm": 20.61426463626924,
       "learning_rate": 2.709592897595191e-07,
-      "logits/chosen": -0.6746516823768616,
-      "logits/rejected": 0.0753551721572876,
-      "logps/chosen": -399.5505676269531,
-      "logps/rejected": -437.2046813964844,
-      "loss": 0.5344,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.184914231300354,
-      "rewards/margins": 0.794265627861023,
-      "rewards/rejected": -1.9791797399520874,
+      "logits/chosen": 1.438730001449585,
+      "logits/rejected": 2.638312816619873,
+      "logps/chosen": -390.794189453125,
+      "logps/rejected": -433.10406494140625,
+      "loss": 0.5311,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.0973504781723022,
+      "rewards/margins": 0.8408235311508179,
+      "rewards/rejected": -1.9381740093231201,
       "step": 230
     },
     {
       "epoch": 0.5485714285714286,
-      "grad_norm": 22.46549138618286,
+      "grad_norm": 31.905445593128672,
       "learning_rate": 2.509992316440332e-07,
-      "logits/chosen": -0.9359372854232788,
-      "logits/rejected": 0.04701079800724983,
-      "logps/chosen": -403.68194580078125,
-      "logps/rejected": -491.6949157714844,
-      "loss": 0.5315,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.049721360206604,
-      "rewards/margins": 1.1530095338821411,
-      "rewards/rejected": -2.202730894088745,
+      "logits/chosen": 1.2066385746002197,
+      "logits/rejected": 2.3449177742004395,
+      "logps/chosen": -413.14825439453125,
+      "logps/rejected": -506.625,
+      "loss": 0.5256,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.1443836688995361,
+      "rewards/margins": 1.2076470851898193,
+      "rewards/rejected": -2.3520307540893555,
       "step": 240
     },
     {
       "epoch": 0.5714285714285714,
-      "grad_norm": 18.11548443644794,
+      "grad_norm": 18.97837160736367,
       "learning_rate": 2.3103279163519918e-07,
-      "logits/chosen": -0.7540313601493835,
-      "logits/rejected": -0.2726219594478607,
-      "logps/chosen": -388.8069763183594,
-      "logps/rejected": -476.65692138671875,
-      "loss": 0.5444,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.0818074941635132,
-      "rewards/margins": 0.9827569127082825,
-      "rewards/rejected": -2.0645642280578613,
+      "logits/chosen": 0.9885716438293457,
+      "logits/rejected": 1.7852414846420288,
+      "logps/chosen": -384.52496337890625,
+      "logps/rejected": -472.253662109375,
+      "loss": 0.5433,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.0389870405197144,
+      "rewards/margins": 0.9815452694892883,
+      "rewards/rejected": -2.0205321311950684,
       "step": 250
     },
     {
       "epoch": 0.5714285714285714,
-      "eval_logits/chosen": -0.8157382011413574,
-      "eval_logits/rejected": 0.23507556319236755,
-      "eval_logps/chosen": -382.0275573730469,
-      "eval_logps/rejected": -442.4582214355469,
-      "eval_loss": 0.5319008827209473,
-      "eval_rewards/accuracies": 0.767241358757019,
-      "eval_rewards/chosen": -1.0642356872558594,
-      "eval_rewards/margins": 1.16958487033844,
-      "eval_rewards/rejected": -2.233820676803589,
-      "eval_runtime": 91.9506,
-      "eval_samples_per_second": 19.913,
-      "eval_steps_per_second": 0.315,
+      "eval_logits/chosen": 0.858768880367279,
+      "eval_logits/rejected": 2.412114381790161,
+      "eval_logps/chosen": -365.6370544433594,
+      "eval_logps/rejected": -425.7062683105469,
+      "eval_loss": 0.528998613357544,
+      "eval_rewards/accuracies": 0.7629310488700867,
+      "eval_rewards/chosen": -0.9003310203552246,
+      "eval_rewards/margins": 1.1659703254699707,
+      "eval_rewards/rejected": -2.0663013458251953,
+      "eval_runtime": 90.3653,
+      "eval_samples_per_second": 20.262,
+      "eval_steps_per_second": 0.321,
       "step": 250
     },
     {
       "epoch": 0.5942857142857143,
-      "grad_norm": 23.815191161905986,
+      "grad_norm": 21.94464499251825,
       "learning_rate": 2.1118749140573358e-07,
-      "logits/chosen": -0.47870174050331116,
-      "logits/rejected": -0.004892909433692694,
-      "logps/chosen": -419.97027587890625,
-      "logps/rejected": -487.68896484375,
-      "loss": 0.5427,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.4018356800079346,
-      "rewards/margins": 0.7416240572929382,
-      "rewards/rejected": -2.1434597969055176,
+      "logits/chosen": 1.5066580772399902,
+      "logits/rejected": 2.079137086868286,
+      "logps/chosen": -411.3843688964844,
+      "logps/rejected": -482.978515625,
+      "loss": 0.5408,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.3159770965576172,
+      "rewards/margins": 0.7803784608840942,
+      "rewards/rejected": -2.096355438232422,
       "step": 260
     },
     {
       "epoch": 0.6171428571428571,
-      "grad_norm": 22.07331008404508,
+      "grad_norm": 23.287724561115347,
       "learning_rate": 1.9159007893272703e-07,
-      "logits/chosen": -0.11218120157718658,
-      "logits/rejected": 1.1597168445587158,
-      "logps/chosen": -407.2730712890625,
-      "logps/rejected": -467.1512756347656,
-      "loss": 0.5208,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.3474724292755127,
-      "rewards/margins": 1.0189656019210815,
-      "rewards/rejected": -2.366438150405884,
+      "logits/chosen": 1.869363784790039,
+      "logits/rejected": 3.169628620147705,
+      "logps/chosen": -400.696533203125,
+      "logps/rejected": -456.28155517578125,
+      "loss": 0.517,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.2817071676254272,
+      "rewards/margins": 0.9760338664054871,
+      "rewards/rejected": -2.2577412128448486,
       "step": 270
     },
     {
       "epoch": 0.64,
-      "grad_norm": 26.61085213163748,
+      "grad_norm": 26.1145325639797,
       "learning_rate": 1.7236571898357766e-07,
-      "logits/chosen": -0.20857758820056915,
-      "logits/rejected": 0.7698697447776794,
-      "logps/chosen": -405.8653869628906,
-      "logps/rejected": -496.41900634765625,
-      "loss": 0.5278,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.3282725811004639,
-      "rewards/margins": 1.0048491954803467,
-      "rewards/rejected": -2.3331217765808105,
+      "logits/chosen": 2.085681438446045,
+      "logits/rejected": 2.909884214401245,
+      "logps/chosen": -402.3949890136719,
+      "logps/rejected": -493.7689514160156,
+      "loss": 0.5287,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.2935690879821777,
+      "rewards/margins": 1.0130523443222046,
+      "rewards/rejected": -2.3066213130950928,
       "step": 280
     },
     {
       "epoch": 0.6628571428571428,
-      "grad_norm": 24.592234854455963,
+      "grad_norm": 28.3817297395316,
       "learning_rate": 1.5363719371356882e-07,
-      "logits/chosen": -0.3117966651916504,
-      "logits/rejected": 0.6535851359367371,
-      "logps/chosen": -433.9137268066406,
-      "logps/rejected": -489.1080017089844,
-      "loss": 0.5288,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.3300788402557373,
-      "rewards/margins": 0.9055169820785522,
-      "rewards/rejected": -2.235595703125,
+      "logits/chosen": 1.904044508934021,
+      "logits/rejected": 2.7162575721740723,
+      "logps/chosen": -424.409912109375,
+      "logps/rejected": -482.04913330078125,
+      "loss": 0.5285,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.2350399494171143,
+      "rewards/margins": 0.9299663305282593,
+      "rewards/rejected": -2.165006399154663,
       "step": 290
     },
     {
       "epoch": 0.6857142857142857,
-      "grad_norm": 17.15967829902915,
+      "grad_norm": 17.85129410356221,
       "learning_rate": 1.3552411848071565e-07,
-      "logits/chosen": -0.4298080503940582,
-      "logits/rejected": 0.8657402992248535,
-      "logps/chosen": -430.40216064453125,
-      "logps/rejected": -487.58544921875,
-      "loss": 0.5195,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.2849299907684326,
-      "rewards/margins": 1.0645029544830322,
-      "rewards/rejected": -2.349433183670044,
+      "logits/chosen": 1.697782278060913,
+      "logits/rejected": 3.180041551589966,
+      "logps/chosen": -419.85028076171875,
+      "logps/rejected": -478.419677734375,
+      "loss": 0.5194,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.1794109344482422,
+      "rewards/margins": 1.078364610671997,
+      "rewards/rejected": -2.2577755451202393,
       "step": 300
     },
     {
       "epoch": 0.6857142857142857,
-      "eval_logits/chosen": -0.7317452430725098,
-      "eval_logits/rejected": 0.5828115940093994,
-      "eval_logps/chosen": -376.1568603515625,
-      "eval_logps/rejected": -443.4945983886719,
-      "eval_loss": 0.522352933883667,
-      "eval_rewards/accuracies": 0.7801724076271057,
-      "eval_rewards/chosen": -1.0055286884307861,
-      "eval_rewards/margins": 1.2386558055877686,
-      "eval_rewards/rejected": -2.2441844940185547,
-      "eval_runtime": 90.6437,
-      "eval_samples_per_second": 20.2,
-      "eval_steps_per_second": 0.32,
+      "eval_logits/chosen": 1.7084869146347046,
+      "eval_logits/rejected": 3.311720132827759,
+      "eval_logps/chosen": -371.6744689941406,
+      "eval_logps/rejected": -439.6499938964844,
+      "eval_loss": 0.5213173031806946,
+      "eval_rewards/accuracies": 0.7715517282485962,
+      "eval_rewards/chosen": -0.9607052206993103,
+      "eval_rewards/margins": 1.2450333833694458,
+      "eval_rewards/rejected": -2.2057385444641113,
+      "eval_runtime": 89.9422,
+      "eval_samples_per_second": 20.358,
+      "eval_steps_per_second": 0.322,
       "step": 300
     },
     {
       "epoch": 0.7085714285714285,
-      "grad_norm": 19.670890656220536,
+      "grad_norm": 22.76802438882901,
       "learning_rate": 1.1814217788631473e-07,
-      "logits/chosen": -0.6274710893630981,
-      "logits/rejected": 0.25394701957702637,
-      "logps/chosen": -375.80413818359375,
-      "logps/rejected": -445.390625,
-      "loss": 0.5308,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.1950000524520874,
-      "rewards/margins": 0.8869529962539673,
-      "rewards/rejected": -2.0819530487060547,
+      "logits/chosen": 1.900792121887207,
+      "logits/rejected": 2.7918269634246826,
+      "logps/chosen": -372.843994140625,
+      "logps/rejected": -442.9312438964844,
+      "loss": 0.5285,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.1653985977172852,
+      "rewards/margins": 0.8919604420661926,
+      "rewards/rejected": -2.057358980178833,
       "step": 310
     },
     {
       "epoch": 0.7314285714285714,
-      "grad_norm": 19.334843719063908,
+      "grad_norm": 19.73975657149685,
       "learning_rate": 1.0160238692045331e-07,
-      "logits/chosen": -0.4475599229335785,
-      "logits/rejected": 0.30151477456092834,
-      "logps/chosen": -378.7860107421875,
-      "logps/rejected": -447.75927734375,
-      "loss": 0.5382,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.345036506652832,
-      "rewards/margins": 0.7226740717887878,
-      "rewards/rejected": -2.0677103996276855,
+      "logits/chosen": 2.1896469593048096,
+      "logits/rejected": 2.8715972900390625,
+      "logps/chosen": -380.424560546875,
+      "logps/rejected": -454.2293395996094,
+      "loss": 0.536,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.3614219427108765,
+      "rewards/margins": 0.7709897756576538,
+      "rewards/rejected": -2.132411479949951,
       "step": 320
     },
     {
       "epoch": 0.7542857142857143,
-      "grad_norm": 21.07812644783168,
+      "grad_norm": 29.56922781200817,
       "learning_rate": 8.601038193139438e-08,
-      "logits/chosen": -0.6940028071403503,
-      "logits/rejected": 0.2311718910932541,
-      "logps/chosen": -427.7415466308594,
-      "logps/rejected": -477.0531311035156,
-      "loss": 0.5302,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.2852336168289185,
-      "rewards/margins": 1.0074186325073242,
-      "rewards/rejected": -2.2926523685455322,
+      "logits/chosen": 1.6053155660629272,
+      "logits/rejected": 2.692516565322876,
+      "logps/chosen": -416.57342529296875,
+      "logps/rejected": -465.4991760253906,
+      "loss": 0.5313,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.1735525131225586,
+      "rewards/margins": 1.003560185432434,
+      "rewards/rejected": -2.177112579345703,
       "step": 330
     },
     {
       "epoch": 0.7771428571428571,
-      "grad_norm": 18.64039907134669,
+      "grad_norm": 18.098670935967576,
       "learning_rate": 7.146574594727572e-08,
-      "logits/chosen": -0.3057669997215271,
-      "logits/rejected": 0.44727668166160583,
-      "logps/chosen": -404.18865966796875,
-      "logps/rejected": -483.5855407714844,
-      "loss": 0.5216,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.3823763132095337,
-      "rewards/margins": 1.0332415103912354,
-      "rewards/rejected": -2.4156179428100586,
+      "logits/chosen": 2.0766067504882812,
+      "logits/rejected": 2.8303616046905518,
+      "logps/chosen": -387.4620361328125,
+      "logps/rejected": -468.67718505859375,
+      "loss": 0.5193,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.2151105403900146,
+      "rewards/margins": 1.0514241456985474,
+      "rewards/rejected": -2.2665345668792725,
       "step": 340
     },
     {
       "epoch": 0.8,
-      "grad_norm": 22.5426523083621,
+      "grad_norm": 20.794164513921476,
       "learning_rate": 5.8061372659157306e-08,
-      "logits/chosen": -0.7059189081192017,
-      "logits/rejected": 0.17293286323547363,
-      "logps/chosen": -422.30352783203125,
-      "logps/rejected": -467.66473388671875,
-      "loss": 0.5294,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.325963020324707,
-      "rewards/margins": 0.8077083826065063,
-      "rewards/rejected": -2.133671522140503,
+      "logits/chosen": 1.6319509744644165,
+      "logits/rejected": 2.7972917556762695,
+      "logps/chosen": -412.102783203125,
+      "logps/rejected": -458.27191162109375,
+      "loss": 0.5325,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.2239553928375244,
+      "rewards/margins": 0.8157873153686523,
+      "rewards/rejected": -2.0397427082061768,
       "step": 350
     },
     {
       "epoch": 0.8,
-      "eval_logits/chosen": -0.5933817625045776,
-      "eval_logits/rejected": 0.806897759437561,
-      "eval_logps/chosen": -391.99456787109375,
-      "eval_logps/rejected": -459.7085876464844,
-      "eval_loss": 0.5202531814575195,
-      "eval_rewards/accuracies": 0.7543103694915771,
-      "eval_rewards/chosen": -1.163906455039978,
-      "eval_rewards/margins": 1.2424179315567017,
-      "eval_rewards/rejected": -2.406324625015259,
-      "eval_runtime": 91.3368,
-      "eval_samples_per_second": 20.047,
-      "eval_steps_per_second": 0.318,
+      "eval_logits/chosen": 2.0842368602752686,
+      "eval_logits/rejected": 3.6707816123962402,
+      "eval_logps/chosen": -389.46490478515625,
+      "eval_logps/rejected": -456.7085266113281,
+      "eval_loss": 0.5216463804244995,
+      "eval_rewards/accuracies": 0.7629310488700867,
+      "eval_rewards/chosen": -1.1386092901229858,
+      "eval_rewards/margins": 1.237714409828186,
+      "eval_rewards/rejected": -2.3763234615325928,
+      "eval_runtime": 89.8616,
+      "eval_samples_per_second": 20.376,
+      "eval_steps_per_second": 0.323,
       "step": 350
     },
     {
       "epoch": 0.8228571428571428,
-      "grad_norm": 17.956675630723836,
+      "grad_norm": 18.184259604484346,
       "learning_rate": 4.5882873127531614e-08,
-      "logits/chosen": -0.7362015247344971,
-      "logits/rejected": 0.41978272795677185,
-      "logps/chosen": -408.91522216796875,
-      "logps/rejected": -478.81451416015625,
-      "loss": 0.5169,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.2351018190383911,
-      "rewards/margins": 1.0421779155731201,
-      "rewards/rejected": -2.2772796154022217,
+      "logits/chosen": 1.648209810256958,
+      "logits/rejected": 2.9181623458862305,
+      "logps/chosen": -407.1295166015625,
+      "logps/rejected": -477.27447509765625,
+      "loss": 0.5146,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.217245101928711,
+      "rewards/margins": 1.044634222984314,
+      "rewards/rejected": -2.2618794441223145,
       "step": 360
     },
     {
       "epoch": 0.8457142857142858,
-      "grad_norm": 19.593030414418095,
+      "grad_norm": 19.108285818305696,
       "learning_rate": 3.500802900154412e-08,
-      "logits/chosen": -0.6445934176445007,
-      "logits/rejected": 0.46942490339279175,
-      "logps/chosen": -387.9786071777344,
-      "logps/rejected": -464.98126220703125,
+      "logits/chosen": 1.801898717880249,
+      "logits/rejected": 3.196338176727295,
+      "logps/chosen": -383.25311279296875,
+      "logps/rejected": -463.01727294921875,
       "loss": 0.5188,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.1691720485687256,
-      "rewards/margins": 1.096484899520874,
-      "rewards/rejected": -2.2656567096710205,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.1219167709350586,
+      "rewards/margins": 1.1241002082824707,
+      "rewards/rejected": -2.2460172176361084,
       "step": 370
     },
     {
       "epoch": 0.8685714285714285,
-      "grad_norm": 32.43514788557893,
+      "grad_norm": 23.620382836684982,
       "learning_rate": 2.550629574310309e-08,
-      "logits/chosen": -0.766352117061615,
-      "logits/rejected": 0.4028262197971344,
-      "logps/chosen": -461.1717834472656,
-      "logps/rejected": -484.5889587402344,
-      "loss": 0.5266,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.4311548471450806,
-      "rewards/margins": 0.8407408595085144,
-      "rewards/rejected": -2.2718958854675293,
+      "logits/chosen": 1.4818474054336548,
+      "logits/rejected": 2.90739107131958,
+      "logps/chosen": -453.0061950683594,
+      "logps/rejected": -476.94830322265625,
+      "loss": 0.5263,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.349498987197876,
+      "rewards/margins": 0.8459898233413696,
+      "rewards/rejected": -2.195488691329956,
       "step": 380
     },
     {
       "epoch": 0.8914285714285715,
-      "grad_norm": 24.475035112334364,
+      "grad_norm": 22.14239335519297,
       "learning_rate": 1.7438359028687983e-08,
-      "logits/chosen": -0.6120296716690063,
-      "logits/rejected": 0.2141704559326172,
-      "logps/chosen": -433.58111572265625,
-      "logps/rejected": -508.9109802246094,
-      "loss": 0.5291,
+      "logits/chosen": 1.8351167440414429,
+      "logits/rejected": 2.6260292530059814,
+      "logps/chosen": -425.75128173828125,
+      "logps/rejected": -503.3841857910156,
+      "loss": 0.5275,
       "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.222081184387207,
-      "rewards/margins": 0.9193018674850464,
-      "rewards/rejected": -2.141382932662964,
+      "rewards/chosen": -1.1437828540802002,
+      "rewards/margins": 0.9423319697380066,
+      "rewards/rejected": -2.0861151218414307,
       "step": 390
     },
     {
       "epoch": 0.9142857142857143,
-      "grad_norm": 32.810528214564755,
+      "grad_norm": 32.206706951444914,
       "learning_rate": 1.0855747162029361e-08,
-      "logits/chosen": -0.3861927390098572,
-      "logits/rejected": -0.02623150870203972,
-      "logps/chosen": -416.01171875,
-      "logps/rejected": -484.2184143066406,
-      "loss": 0.5494,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.3727846145629883,
-      "rewards/margins": 0.8036632537841797,
-      "rewards/rejected": -2.176448106765747,
+      "logits/chosen": 2.132110357284546,
+      "logits/rejected": 2.6392226219177246,
+      "logps/chosen": -411.29962158203125,
+      "logps/rejected": -477.0232849121094,
+      "loss": 0.5483,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.3256638050079346,
+      "rewards/margins": 0.7788330316543579,
+      "rewards/rejected": -2.104496955871582,
       "step": 400
     },
     {
       "epoch": 0.9142857142857143,
-      "eval_logits/chosen": -0.495297372341156,
-      "eval_logits/rejected": 0.9533113241195679,
-      "eval_logps/chosen": -397.91290283203125,
-      "eval_logps/rejected": -465.6643981933594,
-      "eval_loss": 0.5207196474075317,
-      "eval_rewards/accuracies": 0.7715517282485962,
-      "eval_rewards/chosen": -1.2230898141860962,
-      "eval_rewards/margins": 1.2427926063537598,
-      "eval_rewards/rejected": -2.4658823013305664,
-      "eval_runtime": 90.0935,
-      "eval_samples_per_second": 20.323,
-      "eval_steps_per_second": 0.322,
+      "eval_logits/chosen": 2.117452621459961,
+      "eval_logits/rejected": 3.7050397396087646,
+      "eval_logps/chosen": -386.83795166015625,
+      "eval_logps/rejected": -455.0307312011719,
+      "eval_loss": 0.520908772945404,
+      "eval_rewards/accuracies": 0.767241358757019,
+      "eval_rewards/chosen": -1.112339973449707,
+      "eval_rewards/margins": 1.2472059726715088,
+      "eval_rewards/rejected": -2.3595457077026367,
+      "eval_runtime": 90.8703,
+      "eval_samples_per_second": 20.15,
+      "eval_steps_per_second": 0.319,
       "step": 400
     },
     {
       "epoch": 0.9371428571428572,
-      "grad_norm": 21.588729517678896,
+      "grad_norm": 21.9038704574243,
       "learning_rate": 5.8005019731033615e-09,
-      "logits/chosen": -0.6237171292304993,
-      "logits/rejected": 0.3366670608520508,
-      "logps/chosen": -432.98870849609375,
-      "logps/rejected": -489.3666076660156,
-      "loss": 0.519,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.448087453842163,
-      "rewards/margins": 0.8482778668403625,
-      "rewards/rejected": -2.296365261077881,
+      "logits/chosen": 1.9021247625350952,
+      "logits/rejected": 2.9709084033966064,
+      "logps/chosen": -423.39990234375,
+      "logps/rejected": -478.46929931640625,
+      "loss": 0.5184,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.3521995544433594,
+      "rewards/margins": 0.8351926803588867,
+      "rewards/rejected": -2.187392234802246,
       "step": 410
     },
     {
       "epoch": 0.96,
-      "grad_norm": 19.892103213679338,
+      "grad_norm": 21.54473906200769,
       "learning_rate": 2.3049103053431886e-09,
-      "logits/chosen": -0.632776141166687,
-      "logits/rejected": 0.6551538705825806,
-      "logps/chosen": -390.06964111328125,
-      "logps/rejected": -465.9344177246094,
-      "loss": 0.5239,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.073215126991272,
-      "rewards/margins": 1.246260404586792,
-      "rewards/rejected": -2.3194754123687744,
+      "logits/chosen": 1.8090896606445312,
+      "logits/rejected": 3.297045946121216,
+      "logps/chosen": -384.42333984375,
+      "logps/rejected": -458.969482421875,
+      "loss": 0.522,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.0167523622512817,
+      "rewards/margins": 1.2330738306045532,
+      "rewards/rejected": -2.249825954437256,
       "step": 420
     },
     {
       "epoch": 0.9828571428571429,
-      "grad_norm": 19.60054815654678,
+      "grad_norm": 22.41955699037185,
       "learning_rate": 3.9129780600541397e-10,
-      "logits/chosen": -0.3492957055568695,
-      "logits/rejected": 0.4392589032649994,
-      "logps/chosen": -409.07745361328125,
-      "logps/rejected": -488.45751953125,
-      "loss": 0.5228,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.2684558629989624,
-      "rewards/margins": 0.9794237017631531,
-      "rewards/rejected": -2.2478795051574707,
+      "logits/chosen": 2.2351975440979004,
+      "logits/rejected": 3.178173065185547,
+      "logps/chosen": -401.39642333984375,
+      "logps/rejected": -481.4127502441406,
+      "loss": 0.5214,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.191645622253418,
+      "rewards/margins": 0.9857856631278992,
+      "rewards/rejected": -2.177431344985962,
       "step": 430
     },
     {
       "epoch": 0.9988571428571429,
       "step": 437,
       "total_flos": 0.0,
-      "train_loss": 0.5634412694577494,
-      "train_runtime": 11351.7318,
-      "train_samples_per_second": 4.933,
+      "train_loss": 0.5630035629534339,
+      "train_runtime": 11387.5716,
+      "train_samples_per_second": 4.918,
       "train_steps_per_second": 0.038
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c0a30a70a7378ce0594a7051256e7737841282b95ced80c146abdeb90b34bb4
+oid sha256:9b630f4639b5ba6f6d9b5599d93dbecdb552035c1f1853a8182828229b0a2cc8
 size 7608