Model save

README.md

@@ -4,6 +4,7 @@ base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
 - trl
 - dpo
+- alignment-handbook
 - generated_from_trainer
 model-index:
 - name: zephyr-7b-dpo-full-prometheus_consistent-reward-scale-01
@@ -17,15 +18,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5422
-- Rewards/chosen: -1.5854
-- Rewards/rejected: -3.0024
-- Rewards/accuracies: 0.7672
-- Rewards/margins: 1.4170
-- Logps/rejected: -519.3195
-- Logps/chosen: -434.1443
-- Logits/rejected: 2.4835
-- Logits/chosen: 0.7325
+- Loss: 0.5426
+- Rewards/chosen: -1.6823
+- Rewards/rejected: -3.0793
+- Rewards/accuracies: 0.7543
+- Rewards/margins: 1.3969
+- Logps/rejected: -527.0015
+- Logps/chosen: -443.8361
+- Logits/rejected: 3.9256
+- Logits/chosen: 2.3196
 
 ## Model description
 
@@ -62,14 +63,14 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.6745        | 0.1143 | 50   | 0.6652          | -0.0219        | -0.1700          | 0.6940             | 0.1481          | -236.0727      | -277.7910    | -2.4379         | -2.5388       |
-| 0.6193        | 0.2286 | 100  | 0.6070          | -0.5960        | -1.2325          | 0.7026             | 0.6364          | -342.3217      | -335.2089    | -2.3205         | -2.4474       |
-| 0.5552        | 0.3429 | 150  | 0.5762          | -1.3758        | -2.5659          | 0.7457             | 1.1901          | -475.6642      | -413.1814    | 1.7135          | 0.6211        |
-| 0.5607        | 0.4571 | 200  | 0.5560          | -1.2277        | -2.4493          | 0.7672             | 1.2216          | -464.0081      | -398.3775    | 0.6492          | -0.7141       |
-| 0.5589        | 0.5714 | 250  | 0.5523          | -1.3250        | -2.6441          | 0.7716             | 1.3191          | -483.4845      | -408.1035    | 1.3227          | -0.2479       |
-| 0.5351        | 0.6857 | 300  | 0.5430          | -1.4547        | -2.8407          | 0.7543             | 1.3859          | -503.1421      | -421.0749    | 1.9525          | 0.2785        |
-| 0.548         | 0.8    | 350  | 0.5426          | -1.5802        | -2.9933          | 0.7716             | 1.4131          | -518.4055      | -433.6277    | 2.3928          | 0.6373        |
-| 0.5602        | 0.9143 | 400  | 0.5422          | -1.5854        | -3.0024          | 0.7672             | 1.4170          | -519.3195      | -434.1443    | 2.4835          | 0.7325        |
+| 0.6745        | 0.1143 | 50   | 0.6652          | -0.0243        | -0.1726          | 0.6853             | 0.1483          | -236.3333      | -278.0297    | -2.4359         | -2.5373       |
+| 0.6243        | 0.2286 | 100  | 0.6148          | -0.4908        | -1.0266          | 0.7026             | 0.5358          | -321.7332      | -324.6850    | -2.3884         | -2.4967       |
+| 0.5613        | 0.3429 | 150  | 0.5828          | -1.5541        | -2.6485          | 0.7241             | 1.0943          | -483.9227      | -431.0166    | 1.8439          | 0.7832        |
+| 0.5618        | 0.4571 | 200  | 0.5594          | -1.3157        | -2.5436          | 0.7457             | 1.2279          | -473.4366      | -407.1749    | 2.6334          | 1.3535        |
+| 0.558         | 0.5714 | 250  | 0.5526          | -1.3313        | -2.5932          | 0.7629             | 1.2619          | -478.3948      | -408.7301    | 3.1078          | 1.5316        |
+| 0.5399        | 0.6857 | 300  | 0.5465          | -1.7354        | -3.0859          | 0.75               | 1.3504          | -527.6615      | -449.1465    | 3.8100          | 2.3192        |
+| 0.5536        | 0.8    | 350  | 0.5458          | -1.8434        | -3.1830          | 0.7672             | 1.3396          | -537.3768      | -459.9458    | 4.1008          | 2.5496        |
+| 0.5612        | 0.9143 | 400  | 0.5426          | -1.6823        | -3.0793          | 0.7543             | 1.3969          | -527.0015      | -443.8361    | 3.9256          | 2.3196        |
 
 
 ### Framework versions

all_results.json

@@ -1,9 +1,22 @@
 {
     "epoch": 0.9988571428571429,
+    "eval_logits/chosen": 0.6892262101173401,
+    "eval_logits/rejected": 2.4422545433044434,
+    "eval_logps/chosen": -432.7330322265625,
+    "eval_logps/rejected": -518.172607421875,
+    "eval_loss": 0.541784942150116,
+    "eval_rewards/accuracies": 0.767241358757019,
+    "eval_rewards/chosen": -1.5712907314300537,
+    "eval_rewards/margins": 1.419674038887024,
+    "eval_rewards/rejected": -2.990964651107788,
+    "eval_runtime": 91.4769,
+    "eval_samples": 1831,
+    "eval_samples_per_second": 20.016,
+    "eval_steps_per_second": 0.317,
     "total_flos": 0.0,
-    "train_loss": 0.5784502373134517,
-    "train_runtime": 11646.632,
+    "train_loss": 0.5805802214336614,
+    "train_runtime": 11351.5205,
     "train_samples": 55999,
-    "train_samples_per_second": 4.808,
+    "train_samples_per_second": 4.933,
     "train_steps_per_second": 0.038
 }

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 0.9988571428571429,
+    "eval_logits/chosen": 0.6892262101173401,
+    "eval_logits/rejected": 2.4422545433044434,
+    "eval_logps/chosen": -432.7330322265625,
+    "eval_logps/rejected": -518.172607421875,
+    "eval_loss": 0.541784942150116,
+    "eval_rewards/accuracies": 0.767241358757019,
+    "eval_rewards/chosen": -1.5712907314300537,
+    "eval_rewards/margins": 1.419674038887024,
+    "eval_rewards/rejected": -2.990964651107788,
+    "eval_runtime": 91.4769,
+    "eval_samples": 1831,
+    "eval_samples_per_second": 20.016,
+    "eval_steps_per_second": 0.317
+}

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0b84947d1b00f891d3a67c309654da672e6fbb3f24e62dc80bc72e8ad2751468
+oid sha256:0d5f78769972eb480f1196b61d1943520e27d12d2a8d6bb4197b2c3d47623163
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6a9def86fa96aad4b6f41ac09ddc862d9e0830107a7501176257ffad79ce85d3
+oid sha256:90a061631375d4c2a597d12cb28aef18b206e177aff25a8b886eacd51a582085
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6857476507a61ef351b76905a0f5bd1f33a9497c44bfdc7131d4c276c9634047
+oid sha256:b4f2c57fcbc5632af1e6981a1d1ad5d486c4c342d9317b8264c6552eae46d1da
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9988571428571429,
     "total_flos": 0.0,
-    "train_loss": 0.5784502373134517,
-    "train_runtime": 11646.632,
+    "train_loss": 0.5805802214336614,
+    "train_runtime": 11351.5205,
     "train_samples": 55999,
-    "train_samples_per_second": 4.808,
+    "train_samples_per_second": 4.933,
     "train_steps_per_second": 0.038
 }

trainer_state.json

@@ -10,784 +10,784 @@
   "log_history": [
     {
       "epoch": 0.022857142857142857,
-      "grad_norm": 5.116205997832364,
+      "grad_norm": 5.131386476065513,
       "learning_rate": 1.1363636363636363e-07,
-      "logits/chosen": -2.700634717941284,
-      "logits/rejected": -2.624845027923584,
-      "logps/chosen": -301.29974365234375,
-      "logps/rejected": -281.7809753417969,
+      "logits/chosen": -2.7012476921081543,
+      "logits/rejected": -2.6254587173461914,
+      "logps/chosen": -301.2864685058594,
+      "logps/rejected": -281.73876953125,
       "loss": 0.6931,
-      "rewards/accuracies": 0.4437499940395355,
-      "rewards/chosen": -0.0001358071604045108,
-      "rewards/margins": 0.00017720976029522717,
-      "rewards/rejected": -0.00031301690614782274,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": -3.3517978863528697e-06,
+      "rewards/margins": -0.00011216916755074635,
+      "rewards/rejected": 0.0001088173157768324,
       "step": 10
     },
     {
       "epoch": 0.045714285714285714,
-      "grad_norm": 4.048845319624005,
+      "grad_norm": 4.052727080189218,
       "learning_rate": 2.2727272727272726e-07,
-      "logits/chosen": -2.6409199237823486,
-      "logits/rejected": -2.605693817138672,
-      "logps/chosen": -278.9129638671875,
-      "logps/rejected": -254.7362060546875,
-      "loss": 0.6926,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": 0.002582464599981904,
-      "rewards/margins": 0.0022538029588758945,
-      "rewards/rejected": 0.0003286615537945181,
+      "logits/chosen": -2.640902042388916,
+      "logits/rejected": -2.6057076454162598,
+      "logps/chosen": -278.9025573730469,
+      "logps/rejected": -254.7306671142578,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.0026862886734306812,
+      "rewards/margins": 0.0023018636275082827,
+      "rewards/rejected": 0.00038442533696070313,
       "step": 20
     },
     {
       "epoch": 0.06857142857142857,
-      "grad_norm": 4.418664852115103,
+      "grad_norm": 4.434624613429657,
       "learning_rate": 3.4090909090909085e-07,
-      "logits/chosen": -2.6376564502716064,
-      "logits/rejected": -2.617002248764038,
-      "logps/chosen": -263.3965148925781,
-      "logps/rejected": -263.52264404296875,
+      "logits/chosen": -2.6378769874572754,
+      "logits/rejected": -2.6170475482940674,
+      "logps/chosen": -263.394287109375,
+      "logps/rejected": -263.47857666015625,
       "loss": 0.6898,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.01286567747592926,
-      "rewards/margins": 0.008161441422998905,
-      "rewards/rejected": 0.004704234655946493,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.012888330034911633,
+      "rewards/margins": 0.007743468042463064,
+      "rewards/rejected": 0.005144862923771143,
       "step": 30
     },
     {
       "epoch": 0.09142857142857143,
-      "grad_norm": 5.048215894637636,
+      "grad_norm": 5.043723997595032,
       "learning_rate": 4.545454545454545e-07,
-      "logits/chosen": -2.647663116455078,
-      "logits/rejected": -2.584965229034424,
-      "logps/chosen": -290.48455810546875,
-      "logps/rejected": -268.39874267578125,
+      "logits/chosen": -2.648149013519287,
+      "logits/rejected": -2.5857646465301514,
+      "logps/chosen": -290.42108154296875,
+      "logps/rejected": -268.315185546875,
       "loss": 0.6832,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.03512703627347946,
-      "rewards/margins": 0.04280107468366623,
-      "rewards/rejected": -0.0076740398071706295,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.03576214984059334,
+      "rewards/margins": 0.042601048946380615,
+      "rewards/rejected": -0.006838902831077576,
       "step": 40
     },
     {
       "epoch": 0.11428571428571428,
-      "grad_norm": 7.712133622084281,
+      "grad_norm": 7.751230449827184,
       "learning_rate": 4.997124959943201e-07,
-      "logits/chosen": -2.6761856079101562,
-      "logits/rejected": -2.5965471267700195,
-      "logps/chosen": -294.49273681640625,
-      "logps/rejected": -254.78549194335938,
+      "logits/chosen": -2.675534725189209,
+      "logits/rejected": -2.5954620838165283,
+      "logps/chosen": -294.5313720703125,
+      "logps/rejected": -254.864013671875,
       "loss": 0.6745,
       "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.018175510689616203,
-      "rewards/margins": 0.09738700091838837,
-      "rewards/rejected": -0.07921148836612701,
+      "rewards/chosen": 0.017789244651794434,
+      "rewards/margins": 0.09778620302677155,
+      "rewards/rejected": -0.07999695837497711,
       "step": 50
     },
     {
       "epoch": 0.11428571428571428,
-      "eval_logits/chosen": -2.5388200283050537,
-      "eval_logits/rejected": -2.4379258155822754,
-      "eval_logps/chosen": -277.7910461425781,
-      "eval_logps/rejected": -236.07266235351562,
-      "eval_loss": 0.6652289628982544,
-      "eval_rewards/accuracies": 0.693965494632721,
-      "eval_rewards/chosen": -0.021870704367756844,
-      "eval_rewards/margins": 0.14809474349021912,
-      "eval_rewards/rejected": -0.16996543109416962,
-      "eval_runtime": 91.08,
-      "eval_samples_per_second": 20.103,
-      "eval_steps_per_second": 0.318,
+      "eval_logits/chosen": -2.537318468093872,
+      "eval_logits/rejected": -2.435882806777954,
+      "eval_logps/chosen": -278.02972412109375,
+      "eval_logps/rejected": -236.3332977294922,
+      "eval_loss": 0.6651818156242371,
+      "eval_rewards/accuracies": 0.6853448152542114,
+      "eval_rewards/chosen": -0.024257637560367584,
+      "eval_rewards/margins": 0.14831387996673584,
+      "eval_rewards/rejected": -0.17257152497768402,
+      "eval_runtime": 90.3676,
+      "eval_samples_per_second": 20.262,
+      "eval_steps_per_second": 0.321,
       "step": 50
     },
     {
       "epoch": 0.13714285714285715,
-      "grad_norm": 6.234314729404,
+      "grad_norm": 6.305194820143998,
       "learning_rate": 4.979579212164186e-07,
-      "logits/chosen": -2.577277421951294,
-      "logits/rejected": -2.4742989540100098,
-      "logps/chosen": -293.23321533203125,
-      "logps/rejected": -273.9915466308594,
-      "loss": 0.6581,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.12727248668670654,
-      "rewards/margins": 0.12712165713310242,
-      "rewards/rejected": -0.2543941140174866,
+      "logits/chosen": -2.5774006843566895,
+      "logits/rejected": -2.4738996028900146,
+      "logps/chosen": -293.5943908691406,
+      "logps/rejected": -274.21575927734375,
+      "loss": 0.6582,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.13088412582874298,
+      "rewards/margins": 0.12575200200080872,
+      "rewards/rejected": -0.2566361129283905,
       "step": 60
     },
     {
       "epoch": 0.16,
-      "grad_norm": 7.736476543325128,
+      "grad_norm": 7.496714062889331,
       "learning_rate": 4.946196886175515e-07,
-      "logits/chosen": -2.597801446914673,
-      "logits/rejected": -2.543898105621338,
-      "logps/chosen": -289.6143493652344,
-      "logps/rejected": -295.7275695800781,
-      "loss": 0.6398,
+      "logits/chosen": -2.6025655269622803,
+      "logits/rejected": -2.5487170219421387,
+      "logps/chosen": -289.283203125,
+      "logps/rejected": -295.7733459472656,
+      "loss": 0.6391,
       "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.13836105167865753,
-      "rewards/margins": 0.22264671325683594,
-      "rewards/rejected": -0.36100777983665466,
+      "rewards/chosen": -0.13504940271377563,
+      "rewards/margins": 0.22641614079475403,
+      "rewards/rejected": -0.36146557331085205,
       "step": 70
     },
     {
       "epoch": 0.18285714285714286,
-      "grad_norm": 8.2018046742501,
+      "grad_norm": 9.182158909337614,
       "learning_rate": 4.897191188239667e-07,
-      "logits/chosen": -2.6352217197418213,
-      "logits/rejected": -2.5961461067199707,
-      "logps/chosen": -297.4925842285156,
-      "logps/rejected": -316.795166015625,
-      "loss": 0.6315,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.30463558435440063,
-      "rewards/margins": 0.3102269768714905,
-      "rewards/rejected": -0.6148625612258911,
+      "logits/chosen": -2.635709047317505,
+      "logits/rejected": -2.6006340980529785,
+      "logps/chosen": -301.3983459472656,
+      "logps/rejected": -321.3275451660156,
+      "loss": 0.6321,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.3436935245990753,
+      "rewards/margins": 0.3164929151535034,
+      "rewards/rejected": -0.6601864099502563,
       "step": 80
     },
     {
       "epoch": 0.2057142857142857,
-      "grad_norm": 10.097131499300193,
+      "grad_norm": 12.866627758013413,
       "learning_rate": 4.832875107981763e-07,
-      "logits/chosen": -2.6919240951538086,
-      "logits/rejected": -2.6384730339050293,
-      "logps/chosen": -297.72381591796875,
-      "logps/rejected": -324.560791015625,
-      "loss": 0.629,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.21438519656658173,
-      "rewards/margins": 0.47574347257614136,
-      "rewards/rejected": -0.6901286840438843,
+      "logits/chosen": -2.6577916145324707,
+      "logits/rejected": -2.6026382446289062,
+      "logps/chosen": -306.7483215332031,
+      "logps/rejected": -331.4389343261719,
+      "loss": 0.6272,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.30463024973869324,
+      "rewards/margins": 0.45427924394607544,
+      "rewards/rejected": -0.7589095830917358,
       "step": 90
     },
     {
       "epoch": 0.22857142857142856,
-      "grad_norm": 11.497879496244352,
+      "grad_norm": 10.120425097316348,
       "learning_rate": 4.753659419387223e-07,
-      "logits/chosen": -2.6669628620147705,
-      "logits/rejected": -2.5793137550354004,
-      "logps/chosen": -332.02459716796875,
-      "logps/rejected": -326.01910400390625,
-      "loss": 0.6193,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.45638221502304077,
-      "rewards/margins": 0.46929416060447693,
-      "rewards/rejected": -0.9256764650344849,
+      "logits/chosen": -2.6471400260925293,
+      "logits/rejected": -2.5577075481414795,
+      "logps/chosen": -336.1173095703125,
+      "logps/rejected": -324.4599609375,
+      "loss": 0.6243,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.49730944633483887,
+      "rewards/margins": 0.41277560591697693,
+      "rewards/rejected": -0.9100850820541382,
       "step": 100
     },
     {
       "epoch": 0.22857142857142856,
-      "eval_logits/chosen": -2.4473772048950195,
-      "eval_logits/rejected": -2.3204526901245117,
-      "eval_logps/chosen": -335.20892333984375,
-      "eval_logps/rejected": -342.3216552734375,
-      "eval_loss": 0.6070090532302856,
+      "eval_logits/chosen": -2.496720790863037,
+      "eval_logits/rejected": -2.3884377479553223,
+      "eval_logps/chosen": -324.6850280761719,
+      "eval_logps/rejected": -321.733154296875,
+      "eval_loss": 0.6148081421852112,
       "eval_rewards/accuracies": 0.7025862336158752,
-      "eval_rewards/chosen": -0.5960496068000793,
-      "eval_rewards/margins": 0.6364051699638367,
-      "eval_rewards/rejected": -1.2324548959732056,
-      "eval_runtime": 90.7362,
-      "eval_samples_per_second": 20.179,
+      "eval_rewards/chosen": -0.49081093072891235,
+      "eval_rewards/margins": 0.5357595682144165,
+      "eval_rewards/rejected": -1.0265703201293945,
+      "eval_runtime": 90.5627,
+      "eval_samples_per_second": 20.218,
       "eval_steps_per_second": 0.32,
       "step": 100
     },
     {
       "epoch": 0.25142857142857145,
-      "grad_norm": 13.479714305351811,
+      "grad_norm": 13.934592452644212,
       "learning_rate": 4.660050057270191e-07,
-      "logits/chosen": -2.1286423206329346,
-      "logits/rejected": -1.9770896434783936,
-      "logps/chosen": -387.36529541015625,
-      "logps/rejected": -417.18206787109375,
-      "loss": 0.5999,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.710616946220398,
-      "rewards/margins": 0.4821074903011322,
-      "rewards/rejected": -1.192724347114563,
+      "logits/chosen": -2.161785364151001,
+      "logits/rejected": -2.060573101043701,
+      "logps/chosen": -390.38104248046875,
+      "logps/rejected": -416.17913818359375,
+      "loss": 0.6043,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.7407739758491516,
+      "rewards/margins": 0.4419216215610504,
+      "rewards/rejected": -1.1826956272125244,
       "step": 110
     },
     {
       "epoch": 0.2742857142857143,
-      "grad_norm": 21.834827090505335,
+      "grad_norm": 18.04191794745382,
       "learning_rate": 4.5526448859687144e-07,
-      "logits/chosen": -1.3608739376068115,
-      "logits/rejected": -0.962149441242218,
-      "logps/chosen": -384.6209716796875,
-      "logps/rejected": -364.0862731933594,
-      "loss": 0.5861,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.7371653914451599,
-      "rewards/margins": 0.5865553617477417,
-      "rewards/rejected": -1.3237206935882568,
+      "logits/chosen": -0.9994190335273743,
+      "logits/rejected": -0.5663596391677856,
+      "logps/chosen": -400.4964904785156,
+      "logps/rejected": -379.3451232910156,
+      "loss": 0.5895,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.8959201574325562,
+      "rewards/margins": 0.580389142036438,
+      "rewards/rejected": -1.4763094186782837,
       "step": 120
     },
     {
       "epoch": 0.29714285714285715,
-      "grad_norm": 17.97880068775228,
+      "grad_norm": 23.569070805460544,
       "learning_rate": 4.432129880904388e-07,
-      "logits/chosen": -0.3695625364780426,
-      "logits/rejected": 0.15285547077655792,
-      "logps/chosen": -405.9339294433594,
-      "logps/rejected": -413.42706298828125,
-      "loss": 0.5653,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -1.0077803134918213,
-      "rewards/margins": 0.6217871904373169,
-      "rewards/rejected": -1.6295677423477173,
+      "logits/chosen": 0.06917886435985565,
+      "logits/rejected": 0.525992751121521,
+      "logps/chosen": -440.16839599609375,
+      "logps/rejected": -436.06341552734375,
+      "loss": 0.5759,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.350124716758728,
+      "rewards/margins": 0.505806565284729,
+      "rewards/rejected": -1.8559315204620361,
       "step": 130
     },
     {
       "epoch": 0.32,
-      "grad_norm": 14.856966772188793,
+      "grad_norm": 17.135407563954015,
       "learning_rate": 4.299274747394055e-07,
-      "logits/chosen": -0.6858156323432922,
-      "logits/rejected": -0.15091530978679657,
-      "logps/chosen": -395.23504638671875,
-      "logps/rejected": -425.66912841796875,
-      "loss": 0.5876,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.7978739738464355,
-      "rewards/margins": 0.7397030591964722,
-      "rewards/rejected": -1.5375769138336182,
+      "logits/chosen": -0.5247443914413452,
+      "logits/rejected": -0.054386675357818604,
+      "logps/chosen": -396.3623352050781,
+      "logps/rejected": -422.71234130859375,
+      "loss": 0.5878,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.809146523475647,
+      "rewards/margins": 0.6988624930381775,
+      "rewards/rejected": -1.5080091953277588,
       "step": 140
     },
     {
       "epoch": 0.34285714285714286,
-      "grad_norm": 15.48919645782762,
+      "grad_norm": 21.61040590906815,
       "learning_rate": 4.1549280046953653e-07,
-      "logits/chosen": 0.14924204349517822,
-      "logits/rejected": 0.8237205743789673,
-      "logps/chosen": -402.83050537109375,
-      "logps/rejected": -480.46337890625,
-      "loss": 0.5552,
+      "logits/chosen": -0.23892001807689667,
+      "logits/rejected": 0.5097410082817078,
+      "logps/chosen": -382.4234924316406,
+      "logps/rejected": -453.66033935546875,
+      "loss": 0.5613,
       "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.1584460735321045,
-      "rewards/margins": 0.8831979036331177,
-      "rewards/rejected": -2.0416440963745117,
+      "rewards/chosen": -0.9543758630752563,
+      "rewards/margins": 0.8192380666732788,
+      "rewards/rejected": -1.7736135721206665,
       "step": 150
     },
     {
       "epoch": 0.34285714285714286,
-      "eval_logits/chosen": 0.6210644245147705,
-      "eval_logits/rejected": 1.7134919166564941,
-      "eval_logps/chosen": -413.1814270019531,
-      "eval_logps/rejected": -475.66424560546875,
-      "eval_loss": 0.5762364268302917,
-      "eval_rewards/accuracies": 0.7456896305084229,
-      "eval_rewards/chosen": -1.3757750988006592,
-      "eval_rewards/margins": 1.19010591506958,
-      "eval_rewards/rejected": -2.5658812522888184,
-      "eval_runtime": 90.6736,
-      "eval_samples_per_second": 20.193,
-      "eval_steps_per_second": 0.32,
+      "eval_logits/chosen": 0.7831615805625916,
+      "eval_logits/rejected": 1.843852162361145,
+      "eval_logps/chosen": -431.0166015625,
+      "eval_logps/rejected": -483.92266845703125,
+      "eval_loss": 0.5827990770339966,
+      "eval_rewards/accuracies": 0.7241379022598267,
+      "eval_rewards/chosen": -1.5541267395019531,
+      "eval_rewards/margins": 1.0943388938903809,
+      "eval_rewards/rejected": -2.648465394973755,
+      "eval_runtime": 90.809,
+      "eval_samples_per_second": 20.163,
+      "eval_steps_per_second": 0.319,
       "step": 150
     },
     {
       "epoch": 0.3657142857142857,
-      "grad_norm": 27.338096513767994,
+      "grad_norm": 21.503001868471035,
       "learning_rate": 4.000011566683401e-07,
-      "logits/chosen": 0.46789199113845825,
-      "logits/rejected": 1.287762999534607,
-      "logps/chosen": -451.74456787109375,
-      "logps/rejected": -507.8985290527344,
-      "loss": 0.5691,
+      "logits/chosen": 0.6514826416969299,
+      "logits/rejected": 1.4523377418518066,
+      "logps/chosen": -458.98016357421875,
+      "logps/rejected": -519.3821411132812,
+      "loss": 0.5709,
       "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.5240428447723389,
-      "rewards/margins": 0.9985151290893555,
-      "rewards/rejected": -2.5225582122802734,
+      "rewards/chosen": -1.5963990688323975,
+      "rewards/margins": 1.04099440574646,
+      "rewards/rejected": -2.6373934745788574,
       "step": 160
     },
     {
       "epoch": 0.38857142857142857,
-      "grad_norm": 20.374282543445702,
+      "grad_norm": 22.124021346867316,
       "learning_rate": 3.8355148537705047e-07,
-      "logits/chosen": 0.05013761669397354,
-      "logits/rejected": 0.7304227352142334,
-      "logps/chosen": -436.2395935058594,
-      "logps/rejected": -469.56884765625,
-      "loss": 0.5685,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.3213472366333008,
-      "rewards/margins": 0.7170308828353882,
-      "rewards/rejected": -2.0383780002593994,
+      "logits/chosen": 0.3275122046470642,
+      "logits/rejected": 1.0541610717773438,
+      "logps/chosen": -431.19708251953125,
+      "logps/rejected": -465.676025390625,
+      "loss": 0.5719,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.2709215879440308,
+      "rewards/margins": 0.7285287976264954,
+      "rewards/rejected": -1.999450445175171,
       "step": 170
     },
     {
       "epoch": 0.4114285714285714,
-      "grad_norm": 21.086569958558634,
+      "grad_norm": 27.446979249488912,
       "learning_rate": 3.662488473675315e-07,
-      "logits/chosen": -0.4792841076850891,
-      "logits/rejected": 0.523780882358551,
-      "logps/chosen": -440.24420166015625,
-      "logps/rejected": -486.8812561035156,
-      "loss": 0.564,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.1006438732147217,
-      "rewards/margins": 1.168663501739502,
-      "rewards/rejected": -2.2693073749542236,
+      "logits/chosen": -0.1673279106616974,
+      "logits/rejected": 0.9503982663154602,
+      "logps/chosen": -435.6683654785156,
+      "logps/rejected": -467.34991455078125,
+      "loss": 0.5746,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.054885745048523,
+      "rewards/margins": 1.0191079378128052,
+      "rewards/rejected": -2.073993682861328,
       "step": 180
     },
     {
       "epoch": 0.4342857142857143,
-      "grad_norm": 19.913089361766232,
+      "grad_norm": 21.261318648058893,
       "learning_rate": 3.48203751140067e-07,
-      "logits/chosen": -0.07349483668804169,
-      "logits/rejected": 0.8528989553451538,
-      "logps/chosen": -430.77471923828125,
-      "logps/rejected": -469.9756774902344,
-      "loss": 0.568,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -1.5451713800430298,
-      "rewards/margins": 0.8096041679382324,
-      "rewards/rejected": -2.3547751903533936,
+      "logits/chosen": 0.8843255043029785,
+      "logits/rejected": 1.7267229557037354,
+      "logps/chosen": -421.56427001953125,
+      "logps/rejected": -454.6175842285156,
+      "loss": 0.5709,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.4530667066574097,
+      "rewards/margins": 0.7481273412704468,
+      "rewards/rejected": -2.2011942863464355,
       "step": 190
     },
     {
       "epoch": 0.45714285714285713,
-      "grad_norm": 16.58015899692404,
+      "grad_norm": 18.016130637147523,
       "learning_rate": 3.2953144712759537e-07,
-      "logits/chosen": -0.06599310785531998,
-      "logits/rejected": 1.1455082893371582,
-      "logps/chosen": -436.7762145996094,
-      "logps/rejected": -503.99127197265625,
-      "loss": 0.5607,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.6843353509902954,
-      "rewards/margins": 1.066771388053894,
-      "rewards/rejected": -2.7511067390441895,
+      "logits/chosen": 0.8405307531356812,
+      "logits/rejected": 1.9338849782943726,
+      "logps/chosen": -396.523193359375,
+      "logps/rejected": -462.48724365234375,
+      "loss": 0.5618,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.2818048000335693,
+      "rewards/margins": 1.0542625188827515,
+      "rewards/rejected": -2.336066961288452,
       "step": 200
     },
     {
       "epoch": 0.45714285714285713,
-      "eval_logits/chosen": -0.7141388654708862,
-      "eval_logits/rejected": 0.6491928100585938,
-      "eval_logps/chosen": -398.3775329589844,
-      "eval_logps/rejected": -464.0081481933594,
-      "eval_loss": 0.5560439229011536,
-      "eval_rewards/accuracies": 0.767241358757019,
-      "eval_rewards/chosen": -1.2277354001998901,
-      "eval_rewards/margins": 1.2215843200683594,
-      "eval_rewards/rejected": -2.449319839477539,
-      "eval_runtime": 90.9021,
-      "eval_samples_per_second": 20.143,
-      "eval_steps_per_second": 0.319,
+      "eval_logits/chosen": 1.353513240814209,
+      "eval_logits/rejected": 2.633384943008423,
+      "eval_logps/chosen": -407.1748962402344,
+      "eval_logps/rejected": -473.4366149902344,
+      "eval_loss": 0.5593964457511902,
+      "eval_rewards/accuracies": 0.7456896305084229,
+      "eval_rewards/chosen": -1.3157094717025757,
+      "eval_rewards/margins": 1.2278952598571777,
+      "eval_rewards/rejected": -2.543604850769043,
+      "eval_runtime": 90.6049,
+      "eval_samples_per_second": 20.209,
+      "eval_steps_per_second": 0.32,
       "step": 200
     },
     {
       "epoch": 0.48,
-      "grad_norm": 21.314455899070882,
+      "grad_norm": 13.884449533300678,
       "learning_rate": 3.103511916141658e-07,
-      "logits/chosen": -0.3136857748031616,
-      "logits/rejected": 0.7702559232711792,
-      "logps/chosen": -407.5398254394531,
-      "logps/rejected": -480.2178649902344,
-      "loss": 0.5537,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.4408626556396484,
-      "rewards/margins": 0.908516526222229,
-      "rewards/rejected": -2.349379062652588,
+      "logits/chosen": 1.2088024616241455,
+      "logits/rejected": 2.1989169120788574,
+      "logps/chosen": -401.5423278808594,
+      "logps/rejected": -482.7976989746094,
+      "loss": 0.5543,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.380887746810913,
+      "rewards/margins": 0.9942899942398071,
+      "rewards/rejected": -2.3751778602600098,
       "step": 210
     },
     {
       "epoch": 0.5028571428571429,
-      "grad_norm": 33.50940356009228,
+      "grad_norm": 22.105068043395047,
       "learning_rate": 2.9078548506882117e-07,
-      "logits/chosen": 0.8876802325248718,
-      "logits/rejected": 2.1045126914978027,
-      "logps/chosen": -468.2164001464844,
-      "logps/rejected": -526.6786499023438,
-      "loss": 0.5741,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.8315035104751587,
-      "rewards/margins": 0.9734483957290649,
-      "rewards/rejected": -2.8049521446228027,
+      "logits/chosen": 1.1457306146621704,
+      "logits/rejected": 2.113302707672119,
+      "logps/chosen": -436.2627868652344,
+      "logps/rejected": -482.525146484375,
+      "loss": 0.573,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5119678974151611,
+      "rewards/margins": 0.8514491319656372,
+      "rewards/rejected": -2.363417387008667,
       "step": 220
     },
     {
       "epoch": 0.5257142857142857,
-      "grad_norm": 30.59316545342991,
+      "grad_norm": 22.834936526556735,
       "learning_rate": 2.709592897595191e-07,
-      "logits/chosen": 1.18466317653656,
-      "logits/rejected": 2.4997925758361816,
-      "logps/chosen": -467.6075744628906,
-      "logps/rejected": -524.8836669921875,
-      "loss": 0.5519,
+      "logits/chosen": 1.5853362083435059,
+      "logits/rejected": 2.6565065383911133,
+      "logps/chosen": -436.3871154785156,
+      "logps/rejected": -490.07977294921875,
+      "loss": 0.5539,
       "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.8654839992523193,
-      "rewards/margins": 0.9904860258102417,
-      "rewards/rejected": -2.8559701442718506,
+      "rewards/chosen": -1.553279995918274,
+      "rewards/margins": 0.9546509981155396,
+      "rewards/rejected": -2.5079312324523926,
       "step": 230
     },
     {
       "epoch": 0.5485714285714286,
-      "grad_norm": 20.516554684285598,
+      "grad_norm": 24.160129408331898,
       "learning_rate": 2.509992316440332e-07,
-      "logits/chosen": 0.2621554434299469,
-      "logits/rejected": 1.582966923713684,
-      "logps/chosen": -450.7454528808594,
-      "logps/rejected": -559.3546142578125,
-      "loss": 0.5496,
+      "logits/chosen": 1.073169469833374,
+      "logits/rejected": 2.0948684215545654,
+      "logps/chosen": -435.13104248046875,
+      "logps/rejected": -528.5726318359375,
+      "loss": 0.5519,
       "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.5203564167022705,
-      "rewards/margins": 1.358971118927002,
-      "rewards/rejected": -2.8793272972106934,
+      "rewards/chosen": -1.3642125129699707,
+      "rewards/margins": 1.2072954177856445,
+      "rewards/rejected": -2.5715081691741943,
       "step": 240
     },
     {
       "epoch": 0.5714285714285714,
-      "grad_norm": 18.381009526967468,
+      "grad_norm": 20.623735608734524,
       "learning_rate": 2.3103279163519918e-07,
-      "logits/chosen": 0.046704571694135666,
-      "logits/rejected": 0.7480655908584595,
-      "logps/chosen": -425.90081787109375,
-      "logps/rejected": -522.4205322265625,
-      "loss": 0.5589,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.4527461528778076,
-      "rewards/margins": 1.0694544315338135,
-      "rewards/rejected": -2.522200584411621,
+      "logits/chosen": 1.123517394065857,
+      "logits/rejected": 1.916009545326233,
+      "logps/chosen": -412.7406311035156,
+      "logps/rejected": -501.85394287109375,
+      "loss": 0.558,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.3211438655853271,
+      "rewards/margins": 0.9953910112380981,
+      "rewards/rejected": -2.316534996032715,
       "step": 250
     },
     {
       "epoch": 0.5714285714285714,
-      "eval_logits/chosen": -0.24794088304042816,
-      "eval_logits/rejected": 1.3226882219314575,
-      "eval_logps/chosen": -408.1035461425781,
-      "eval_logps/rejected": -483.4844665527344,
-      "eval_loss": 0.5523289442062378,
-      "eval_rewards/accuracies": 0.7715517282485962,
-      "eval_rewards/chosen": -1.324995994567871,
-      "eval_rewards/margins": 1.319087028503418,
-      "eval_rewards/rejected": -2.644083023071289,
-      "eval_runtime": 92.8381,
-      "eval_samples_per_second": 19.723,
-      "eval_steps_per_second": 0.312,
+      "eval_logits/chosen": 1.5315722227096558,
+      "eval_logits/rejected": 3.1078054904937744,
+      "eval_logps/chosen": -408.7300720214844,
+      "eval_logps/rejected": -478.3948059082031,
+      "eval_loss": 0.5526223182678223,
+      "eval_rewards/accuracies": 0.7629310488700867,
+      "eval_rewards/chosen": -1.3312608003616333,
+      "eval_rewards/margins": 1.2619256973266602,
+      "eval_rewards/rejected": -2.593186378479004,
+      "eval_runtime": 90.8269,
+      "eval_samples_per_second": 20.159,
+      "eval_steps_per_second": 0.319,
       "step": 250
     },
     {
       "epoch": 0.5942857142857143,
-      "grad_norm": 21.477935114375956,
+      "grad_norm": 18.937383972416836,
       "learning_rate": 2.1118749140573358e-07,
-      "logits/chosen": 0.25936073064804077,
-      "logits/rejected": 1.0254137516021729,
-      "logps/chosen": -452.327880859375,
-      "logps/rejected": -538.3406982421875,
-      "loss": 0.5539,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.7254120111465454,
-      "rewards/margins": 0.924565315246582,
-      "rewards/rejected": -2.649976968765259,
+      "logits/chosen": 2.365237236022949,
+      "logits/rejected": 2.8070249557495117,
+      "logps/chosen": -467.107666015625,
+      "logps/rejected": -548.4678344726562,
+      "loss": 0.5559,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.8732095956802368,
+      "rewards/margins": 0.8780391812324524,
+      "rewards/rejected": -2.751248598098755,
       "step": 260
     },
     {
       "epoch": 0.6171428571428571,
-      "grad_norm": 22.404068631830373,
+      "grad_norm": 21.474579443049855,
       "learning_rate": 1.9159007893272703e-07,
-      "logits/chosen": 0.9826221466064453,
-      "logits/rejected": 2.496065855026245,
-      "logps/chosen": -454.06201171875,
-      "logps/rejected": -520.5513916015625,
-      "loss": 0.5406,
+      "logits/chosen": 2.5897929668426514,
+      "logits/rejected": 3.8603546619415283,
+      "logps/chosen": -472.37420654296875,
+      "logps/rejected": -535.6754150390625,
+      "loss": 0.5468,
       "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.8153613805770874,
-      "rewards/margins": 1.085078477859497,
-      "rewards/rejected": -2.900439739227295,
+      "rewards/chosen": -1.9984840154647827,
+      "rewards/margins": 1.0531952381134033,
+      "rewards/rejected": -3.0516793727874756,
       "step": 270
     },
     {
       "epoch": 0.64,
-      "grad_norm": 24.981220974307632,
+      "grad_norm": 26.380689829844055,
       "learning_rate": 1.7236571898357766e-07,
-      "logits/chosen": 1.0183608531951904,
-      "logits/rejected": 2.100400924682617,
-      "logps/chosen": -443.5811462402344,
-      "logps/rejected": -558.9719848632812,
-      "loss": 0.5461,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.7054297924041748,
-      "rewards/margins": 1.2532217502593994,
-      "rewards/rejected": -2.958651304244995,
+      "logits/chosen": 2.368163824081421,
+      "logits/rejected": 3.2206153869628906,
+      "logps/chosen": -457.65478515625,
+      "logps/rejected": -559.1119384765625,
+      "loss": 0.5472,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.8461668491363525,
+      "rewards/margins": 1.113884687423706,
+      "rewards/rejected": -2.9600515365600586,
       "step": 280
     },
     {
       "epoch": 0.6628571428571428,
-      "grad_norm": 22.42240240560034,
+      "grad_norm": 21.888196354392417,
       "learning_rate": 1.5363719371356882e-07,
-      "logits/chosen": 1.1615647077560425,
-      "logits/rejected": 2.0770411491394043,
-      "logps/chosen": -479.4815979003906,
-      "logps/rejected": -552.2136840820312,
-      "loss": 0.5472,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.7857574224472046,
-      "rewards/margins": 1.0808943510055542,
-      "rewards/rejected": -2.866652011871338,
+      "logits/chosen": 2.094879627227783,
+      "logits/rejected": 2.8882927894592285,
+      "logps/chosen": -477.3121643066406,
+      "logps/rejected": -541.1532592773438,
+      "loss": 0.5476,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.764062523841858,
+      "rewards/margins": 0.991985023021698,
+      "rewards/rejected": -2.7560477256774902,
       "step": 290
     },
     {
       "epoch": 0.6857142857142857,
-      "grad_norm": 17.37829470785684,
+      "grad_norm": 16.876283045841564,
       "learning_rate": 1.3552411848071565e-07,
-      "logits/chosen": 0.5264058709144592,
-      "logits/rejected": 2.128044605255127,
-      "logps/chosen": -466.05633544921875,
-      "logps/rejected": -544.390869140625,
-      "loss": 0.5351,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -1.641471266746521,
-      "rewards/margins": 1.2760158777236938,
-      "rewards/rejected": -2.9174869060516357,
+      "logits/chosen": 2.240018844604492,
+      "logits/rejected": 3.6286048889160156,
+      "logps/chosen": -489.7718200683594,
+      "logps/rejected": -560.1759033203125,
+      "loss": 0.5399,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.8786265850067139,
+      "rewards/margins": 1.196711778640747,
+      "rewards/rejected": -3.075338363647461,
       "step": 300
     },
     {
       "epoch": 0.6857142857142857,
-      "eval_logits/chosen": 0.27849671244621277,
-      "eval_logits/rejected": 1.952508568763733,
-      "eval_logps/chosen": -421.0749206542969,
-      "eval_logps/rejected": -503.1421203613281,
-      "eval_loss": 0.5429859757423401,
-      "eval_rewards/accuracies": 0.7543103694915771,
-      "eval_rewards/chosen": -1.4547101259231567,
-      "eval_rewards/margins": 1.3859494924545288,
-      "eval_rewards/rejected": -2.8406593799591064,
-      "eval_runtime": 90.64,
-      "eval_samples_per_second": 20.201,
-      "eval_steps_per_second": 0.32,
+      "eval_logits/chosen": 2.3192012310028076,
+      "eval_logits/rejected": 3.8099868297576904,
+      "eval_logps/chosen": -449.146484375,
+      "eval_logps/rejected": -527.6614990234375,
+      "eval_loss": 0.5464984178543091,
+      "eval_rewards/accuracies": 0.75,
+      "eval_rewards/chosen": -1.7354251146316528,
+      "eval_rewards/margins": 1.3504279851913452,
+      "eval_rewards/rejected": -3.0858535766601562,
+      "eval_runtime": 89.7327,
+      "eval_samples_per_second": 20.405,
+      "eval_steps_per_second": 0.323,
       "step": 300
     },
     {
       "epoch": 0.7085714285714285,
-      "grad_norm": 20.02714298892451,
+      "grad_norm": 17.86356014365164,
       "learning_rate": 1.1814217788631473e-07,
-      "logits/chosen": 0.3929429054260254,
-      "logits/rejected": 1.4373176097869873,
-      "logps/chosen": -416.38616943359375,
-      "logps/rejected": -497.1776428222656,
-      "loss": 0.5427,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.6008203029632568,
-      "rewards/margins": 0.9990032315254211,
-      "rewards/rejected": -2.599823474884033,
+      "logits/chosen": 2.34942626953125,
+      "logits/rejected": 3.1517879962921143,
+      "logps/chosen": -443.734375,
+      "logps/rejected": -516.211669921875,
+      "loss": 0.5491,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.8743022680282593,
+      "rewards/margins": 0.9158605337142944,
+      "rewards/rejected": -2.790163040161133,
       "step": 310
     },
     {
       "epoch": 0.7314285714285714,
-      "grad_norm": 17.430643463250824,
+      "grad_norm": 18.11439951327569,
       "learning_rate": 1.0160238692045331e-07,
-      "logits/chosen": 0.5950568318367004,
-      "logits/rejected": 1.5454418659210205,
-      "logps/chosen": -417.2794494628906,
-      "logps/rejected": -498.28594970703125,
-      "loss": 0.5494,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.7299703359603882,
-      "rewards/margins": 0.8430070877075195,
-      "rewards/rejected": -2.5729775428771973,
+      "logits/chosen": 2.115510940551758,
+      "logits/rejected": 2.9685635566711426,
+      "logps/chosen": -417.16412353515625,
+      "logps/rejected": -496.8562927246094,
+      "loss": 0.5474,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.728817343711853,
+      "rewards/margins": 0.8298628926277161,
+      "rewards/rejected": -2.558680295944214,
       "step": 320
     },
     {
       "epoch": 0.7542857142857143,
-      "grad_norm": 18.13700521728858,
+      "grad_norm": 23.20251472261412,
       "learning_rate": 8.601038193139438e-08,
-      "logits/chosen": 0.24211910367012024,
-      "logits/rejected": 1.487108826637268,
-      "logps/chosen": -468.9916076660156,
-      "logps/rejected": -524.9107055664062,
-      "loss": 0.5477,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.6977341175079346,
-      "rewards/margins": 1.0734937191009521,
-      "rewards/rejected": -2.7712278366088867,
+      "logits/chosen": 1.6642992496490479,
+      "logits/rejected": 2.858081579208374,
+      "logps/chosen": -455.21453857421875,
+      "logps/rejected": -516.3800048828125,
+      "loss": 0.5498,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.5599634647369385,
+      "rewards/margins": 1.1259580850601196,
+      "rewards/rejected": -2.6859214305877686,
       "step": 330
     },
     {
       "epoch": 0.7771428571428571,
-      "grad_norm": 18.181148717866314,
+      "grad_norm": 17.97386136540539,
       "learning_rate": 7.146574594727572e-08,
-      "logits/chosen": 0.9565867185592651,
-      "logits/rejected": 1.8202543258666992,
-      "logps/chosen": -449.505126953125,
-      "logps/rejected": -544.8054809570312,
-      "loss": 0.537,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.8355414867401123,
-      "rewards/margins": 1.1922752857208252,
-      "rewards/rejected": -3.0278170108795166,
+      "logits/chosen": 2.353365659713745,
+      "logits/rejected": 3.1233067512512207,
+      "logps/chosen": -446.12493896484375,
+      "logps/rejected": -536.3157958984375,
+      "loss": 0.5373,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.801740050315857,
+      "rewards/margins": 1.141181230545044,
+      "rewards/rejected": -2.9429211616516113,
       "step": 340
     },
     {
       "epoch": 0.8,
-      "grad_norm": 22.57819934027554,
+      "grad_norm": 21.871959889194457,
       "learning_rate": 5.8061372659157306e-08,
-      "logits/chosen": 0.5353456139564514,
-      "logits/rejected": 1.7232787609100342,
-      "logps/chosen": -466.8003845214844,
-      "logps/rejected": -523.1851196289062,
-      "loss": 0.548,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.7709314823150635,
-      "rewards/margins": 0.9179432988166809,
-      "rewards/rejected": -2.6888747215270996,
+      "logits/chosen": 2.052710771560669,
+      "logits/rejected": 3.2841498851776123,
+      "logps/chosen": -481.0086975097656,
+      "logps/rejected": -538.4657592773438,
+      "loss": 0.5536,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.9130144119262695,
+      "rewards/margins": 0.9286670684814453,
+      "rewards/rejected": -2.8416812419891357,
       "step": 350
     },
     {
       "epoch": 0.8,
-      "eval_logits/chosen": 0.6372899413108826,
-      "eval_logits/rejected": 2.3928465843200684,
-      "eval_logps/chosen": -433.62774658203125,
-      "eval_logps/rejected": -518.405517578125,
-      "eval_loss": 0.5426377654075623,
-      "eval_rewards/accuracies": 0.7715517282485962,
-      "eval_rewards/chosen": -1.5802381038665771,
-      "eval_rewards/margins": 1.413055419921875,
-      "eval_rewards/rejected": -2.993293523788452,
-      "eval_runtime": 89.749,
-      "eval_samples_per_second": 20.401,
-      "eval_steps_per_second": 0.323,
+      "eval_logits/chosen": 2.5496232509613037,
+      "eval_logits/rejected": 4.100791931152344,
+      "eval_logps/chosen": -459.9457702636719,
+      "eval_logps/rejected": -537.3768310546875,
+      "eval_loss": 0.5457741618156433,
+      "eval_rewards/accuracies": 0.767241358757019,
+      "eval_rewards/chosen": -1.8434182405471802,
+      "eval_rewards/margins": 1.3395885229110718,
+      "eval_rewards/rejected": -3.183006525039673,
+      "eval_runtime": 90.4314,
+      "eval_samples_per_second": 20.247,
+      "eval_steps_per_second": 0.321,
       "step": 350
     },
     {
       "epoch": 0.8228571428571428,
-      "grad_norm": 15.622675608360511,
+      "grad_norm": 18.303747174951123,
       "learning_rate": 4.5882873127531614e-08,
-      "logits/chosen": 0.3670940101146698,
-      "logits/rejected": 1.7792842388153076,
-      "logps/chosen": -446.5591735839844,
-      "logps/rejected": -531.0363159179688,
-      "loss": 0.5314,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.6115410327911377,
-      "rewards/margins": 1.187956690788269,
-      "rewards/rejected": -2.799497604370117,
+      "logits/chosen": 2.120243549346924,
+      "logits/rejected": 3.347181797027588,
+      "logps/chosen": -475.18572998046875,
+      "logps/rejected": -556.6296997070312,
+      "loss": 0.5378,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.8978074789047241,
+      "rewards/margins": 1.1576240062713623,
+      "rewards/rejected": -3.055431604385376,
       "step": 360
     },
     {
       "epoch": 0.8457142857142858,
-      "grad_norm": 19.069433933572203,
+      "grad_norm": 18.37433316843813,
       "learning_rate": 3.500802900154412e-08,
-      "logits/chosen": 0.31656524538993835,
-      "logits/rejected": 1.853607416152954,
-      "logps/chosen": -419.215576171875,
-      "logps/rejected": -512.1829223632812,
-      "loss": 0.5406,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.481541395187378,
-      "rewards/margins": 1.2561315298080444,
-      "rewards/rejected": -2.737673282623291,
+      "logits/chosen": 2.2264809608459473,
+      "logits/rejected": 3.606698513031006,
+      "logps/chosen": -450.2330627441406,
+      "logps/rejected": -542.2530517578125,
+      "loss": 0.5435,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.7917168140411377,
+      "rewards/margins": 1.2466588020324707,
+      "rewards/rejected": -3.0383753776550293,
       "step": 370
     },
     {
       "epoch": 0.8685714285714285,
-      "grad_norm": 23.53092447691508,
+      "grad_norm": 21.62972776738107,
       "learning_rate": 2.550629574310309e-08,
-      "logits/chosen": 0.17315557599067688,
-      "logits/rejected": 1.6630465984344482,
-      "logps/chosen": -496.2525939941406,
-      "logps/rejected": -530.6498413085938,
-      "loss": 0.541,
+      "logits/chosen": 1.8507616519927979,
+      "logits/rejected": 3.276202440261841,
+      "logps/chosen": -521.0817260742188,
+      "logps/rejected": -551.79736328125,
+      "loss": 0.543,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.7819627523422241,
-      "rewards/margins": 0.9505414962768555,
-      "rewards/rejected": -2.732504367828369,
+      "rewards/chosen": -2.0302536487579346,
+      "rewards/margins": 0.913725733757019,
+      "rewards/rejected": -2.943979501724243,
       "step": 380
     },
     {
       "epoch": 0.8914285714285715,
-      "grad_norm": 21.428741178294356,
+      "grad_norm": 21.37326667405599,
       "learning_rate": 1.7438359028687983e-08,
-      "logits/chosen": 0.45238789916038513,
-      "logits/rejected": 1.306971788406372,
-      "logps/chosen": -466.89324951171875,
-      "logps/rejected": -553.9127197265625,
-      "loss": 0.5468,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.5552027225494385,
-      "rewards/margins": 1.0361970663070679,
-      "rewards/rejected": -2.591399669647217,
+      "logits/chosen": 2.1451709270477295,
+      "logits/rejected": 2.937721014022827,
+      "logps/chosen": -487.7579040527344,
+      "logps/rejected": -573.7957763671875,
+      "loss": 0.5485,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.7638496160507202,
+      "rewards/margins": 1.0263822078704834,
+      "rewards/rejected": -2.7902321815490723,
       "step": 390
     },
     {
       "epoch": 0.9142857142857143,
-      "grad_norm": 29.45545415193721,
+      "grad_norm": 36.14381504781887,
       "learning_rate": 1.0855747162029361e-08,
-      "logits/chosen": 0.8573619723320007,
-      "logits/rejected": 1.5074001550674438,
-      "logps/chosen": -456.8833923339844,
-      "logps/rejected": -535.3235473632812,
-      "loss": 0.5602,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.781501054763794,
-      "rewards/margins": 0.9059982299804688,
-      "rewards/rejected": -2.6874990463256836,
+      "logits/chosen": 2.2515556812286377,
+      "logits/rejected": 2.859483480453491,
+      "logps/chosen": -467.8270568847656,
+      "logps/rejected": -549.77392578125,
+      "loss": 0.5612,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.89093816280365,
+      "rewards/margins": 0.9410650134086609,
+      "rewards/rejected": -2.832003116607666,
       "step": 400
     },
     {
       "epoch": 0.9142857142857143,
-      "eval_logits/chosen": 0.7325018048286438,
-      "eval_logits/rejected": 2.4835281372070312,
-      "eval_logps/chosen": -434.1443176269531,
-      "eval_logps/rejected": -519.3194580078125,
-      "eval_loss": 0.5422444343566895,
-      "eval_rewards/accuracies": 0.767241358757019,
-      "eval_rewards/chosen": -1.585403323173523,
-      "eval_rewards/margins": 1.417029857635498,
-      "eval_rewards/rejected": -3.0024335384368896,
-      "eval_runtime": 90.5071,
-      "eval_samples_per_second": 20.23,
+      "eval_logits/chosen": 2.319566249847412,
+      "eval_logits/rejected": 3.925558090209961,
+      "eval_logps/chosen": -443.83612060546875,
+      "eval_logps/rejected": -527.0015258789062,
+      "eval_loss": 0.5425659418106079,
+      "eval_rewards/accuracies": 0.7543103694915771,
+      "eval_rewards/chosen": -1.6823216676712036,
+      "eval_rewards/margins": 1.3969323635101318,
+      "eval_rewards/rejected": -3.079254150390625,
+      "eval_runtime": 90.5136,
+      "eval_samples_per_second": 20.229,
       "eval_steps_per_second": 0.32,
       "step": 400
     },
     {
       "epoch": 0.9371428571428572,
-      "grad_norm": 23.183003691416907,
+      "grad_norm": 20.661658798051754,
       "learning_rate": 5.8005019731033615e-09,
-      "logits/chosen": 0.6646889448165894,
-      "logits/rejected": 1.7684948444366455,
-      "logps/chosen": -474.71209716796875,
-      "logps/rejected": -539.7491455078125,
-      "loss": 0.5368,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -1.8653221130371094,
-      "rewards/margins": 0.9348688125610352,
-      "rewards/rejected": -2.8001909255981445,
+      "logits/chosen": 2.158614158630371,
+      "logits/rejected": 3.273141384124756,
+      "logps/chosen": -486.8451232910156,
+      "logps/rejected": -554.2020263671875,
+      "loss": 0.5334,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.9866514205932617,
+      "rewards/margins": 0.9580680727958679,
+      "rewards/rejected": -2.9447195529937744,
       "step": 410
     },
     {
       "epoch": 0.96,
-      "grad_norm": 20.16157572567649,
+      "grad_norm": 20.631088560363143,
       "learning_rate": 2.3049103053431886e-09,
-      "logits/chosen": 0.5646069645881653,
-      "logits/rejected": 2.1821658611297607,
-      "logps/chosen": -431.48663330078125,
-      "logps/rejected": -519.7050170898438,
-      "loss": 0.5427,
+      "logits/chosen": 2.0465073585510254,
+      "logits/rejected": 3.5103249549865723,
+      "logps/chosen": -440.74224853515625,
+      "logps/rejected": -527.11376953125,
+      "loss": 0.5445,
       "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.4873847961425781,
-      "rewards/margins": 1.369796633720398,
-      "rewards/rejected": -2.857182025909424,
+      "rewards/chosen": -1.5799415111541748,
+      "rewards/margins": 1.351327657699585,
+      "rewards/rejected": -2.9312691688537598,
       "step": 420
     },
     {
       "epoch": 0.9828571428571429,
-      "grad_norm": 18.316684075654734,
+      "grad_norm": 21.077395280178123,
       "learning_rate": 3.9129780600541397e-10,
-      "logits/chosen": 0.780235230922699,
-      "logits/rejected": 1.9581851959228516,
-      "logps/chosen": -448.1625061035156,
-      "logps/rejected": -540.7398681640625,
-      "loss": 0.5411,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.6593061685562134,
-      "rewards/margins": 1.1113975048065186,
-      "rewards/rejected": -2.7707035541534424,
+      "logits/chosen": 2.380171298980713,
+      "logits/rejected": 3.3738162517547607,
+      "logps/chosen": -453.9419860839844,
+      "logps/rejected": -550.107421875,
+      "loss": 0.5377,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.7171010971069336,
+      "rewards/margins": 1.1472762823104858,
+      "rewards/rejected": -2.864377498626709,
       "step": 430
     },
     {
       "epoch": 0.9988571428571429,
       "step": 437,
       "total_flos": 0.0,
-      "train_loss": 0.5784502373134517,
-      "train_runtime": 11646.632,
-      "train_samples_per_second": 4.808,
+      "train_loss": 0.5805802214336614,
+      "train_runtime": 11351.5205,
+      "train_samples_per_second": 4.933,
       "train_steps_per_second": 0.038
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:56915a1e9ee54dec20008e989fbd1cd3a39074027d2a714969f286c80dda42bc
+oid sha256:df9308c326f453e41dac5923fdc0ab4295c88fd6d01fb244d79e02044c538bfd
 size 7608