Model save

README.md

@@ -2,15 +2,9 @@
 license: apache-2.0
 base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
-- trl
-- dpo
-- generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: zephyr-7b-dpo-full
   results: []
@@ -21,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5105
-- Rewards/chosen: -0.1900
-- Rewards/rejected: -1.3026
-- Rewards/accuracies: 0.7812
-- Rewards/margins: 1.1125
-- Logps/rejected: -288.7184
-- Logps/chosen: -266.3937
-- Logits/rejected: -2.4836
-- Logits/chosen: -2.5251
+- Loss: 0.2788
+- Rewards/chosen: 31.8547
+- Rewards/rejected: 27.4761
+- Rewards/accuracies: 0.5781
+- Rewards/margins: 4.3786
+- Logps/rejected: -235.1911
+- Logps/chosen: -230.7387
+- Logits/rejected: -2.6619
+- Logits/chosen: -2.6869
 
 ## Model description
 
@@ -66,17 +60,17 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
-|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.5414        | 0.2092 | 100  | 0.5491          | 0.0981         | -0.5995          | 0.7578             | 0.6977          | -274.6581      | -260.6310    | -2.4578         | -2.4971       |
-| 0.525         | 0.4184 | 200  | 0.5204          | 0.0445         | -0.8986          | 0.7695             | 0.9431          | -280.6389      | -261.7037    | -2.4225         | -2.4648       |
-| 0.4908        | 0.6276 | 300  | 0.5125          | -0.1921        | -1.2467          | 0.7773             | 1.0546          | -287.6011      | -266.4352    | -2.5527         | -2.5937       |
-| 0.4966        | 0.8368 | 400  | 0.5099          | -0.2747        | -1.3633          | 0.7773             | 1.0886          | -289.9331      | -268.0874    | -2.4807         | -2.5231       |
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.3163        | 0.21  | 100  | 0.3129          | 27.2147        | 24.2237          | 0.5820             | 2.9910          | -238.4435      | -235.3788    | -2.6701         | -2.6969       |
+| 0.2918        | 0.42  | 200  | 0.2923          | 30.3522        | 26.4568          | 0.5820             | 3.8954          | -236.2104      | -232.2412    | -2.6826         | -2.7085       |
+| 0.286         | 0.63  | 300  | 0.2921          | 31.4394        | 27.2385          | 0.5820             | 4.2009          | -235.4286      | -231.1540    | -2.6717         | -2.6957       |
+| 0.2819        | 0.84  | 400  | 0.2788          | 31.8547        | 27.4761          | 0.5781             | 4.3786          | -235.1911      | -230.7387    | -2.6619         | -2.6869       |
 
 
 ### Framework versions
 
-- Transformers 4.40.1
+- Transformers 4.38.2
 - Pytorch 2.1.2+cu118
-- Datasets 2.19.0
-- Tokenizers 0.19.1
+- Datasets 2.16.1
+- Tokenizers 0.15.2

all_results.json

@@ -1,22 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.5251381397247314,
-    "eval_logits/rejected": -2.4836020469665527,
-    "eval_logps/chosen": -266.3936767578125,
-    "eval_logps/rejected": -288.7183532714844,
-    "eval_loss": 0.5105125308036804,
-    "eval_rewards/accuracies": 0.78125,
-    "eval_rewards/chosen": -0.190010204911232,
-    "eval_rewards/margins": 1.112548828125,
-    "eval_rewards/rejected": -1.3025591373443604,
-    "eval_runtime": 96.1755,
-    "eval_samples": 2000,
-    "eval_samples_per_second": 20.795,
-    "eval_steps_per_second": 0.333,
-    "total_flos": 0.0,
-    "train_loss": 0.5238834185580329,
-    "train_runtime": 7545.2548,
+    "train_loss": 0.31381568898715734,
+    "train_runtime": 7749.4814,
     "train_samples": 61134,
-    "train_samples_per_second": 8.102,
-    "train_steps_per_second": 0.063
+    "train_samples_per_second": 7.889,
+    "train_steps_per_second": 0.062
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "transformers_version": "4.40.1"
+  "transformers_version": "4.38.2"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6eca9fb78075aaf58b9d4c46fb40e464cf8cea437db3797f59c96d7a5f4cb4dc
+oid sha256:aeaa284c42a7973b56bb02672fa19944707a1761c29070843c6f9f62dd72e4fe
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:19700b0bdf0789d6d5f9666b2d1cd96a34c801a01ffae61d7e26755093d189ba
+oid sha256:5fe9c0ae7edc563803fced44889ee13e00331e230ffe85265ea71dd173dd7599
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d7d52a56a8a28dd535037564b1423e2ad42a207bd763b938826e6862aa71d903
+oid sha256:4eb8c28dbe053ab22cc46c14d5c8cb9815cb514a15461a39bb7903e6e94dd96a
 size 4540516344

train_results.json

@@ -1,9 +1,8 @@
 {
     "epoch": 1.0,
-    "total_flos": 0.0,
-    "train_loss": 0.5238834185580329,
-    "train_runtime": 7545.2548,
+    "train_loss": 0.31381568898715734,
+    "train_runtime": 7749.4814,
     "train_samples": 61134,
-    "train_samples_per_second": 8.102,
-    "train_steps_per_second": 0.063
+    "train_samples_per_second": 7.889,
+    "train_steps_per_second": 0.062
 }

trainer_state.json

@@ -9,8 +9,8 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0020920502092050207,
-      "grad_norm": 46.49109757612532,
+      "epoch": 0.0,
+      "grad_norm": 74.50819179863889,
       "learning_rate": 1.0416666666666666e-08,
       "logits/chosen": -2.7660439014434814,
       "logits/rejected": -2.717564582824707,
@@ -24,782 +24,782 @@
       "step": 1
     },
     {
-      "epoch": 0.02092050209205021,
-      "grad_norm": 45.47385033244497,
+      "epoch": 0.02,
+      "grad_norm": 71.5827858042053,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -2.592388391494751,
-      "logits/rejected": -2.562622547149658,
-      "logps/chosen": -264.7853698730469,
-      "logps/rejected": -251.5283966064453,
-      "loss": 0.6929,
-      "rewards/accuracies": 0.4305555522441864,
-      "rewards/chosen": 0.0006246861885301769,
-      "rewards/margins": -0.000603115709964186,
-      "rewards/rejected": 0.0012278020149096847,
+      "logits/chosen": -2.592801809310913,
+      "logits/rejected": -2.5633366107940674,
+      "logps/chosen": -264.5331726074219,
+      "logps/rejected": -251.33367919921875,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.4444444477558136,
+      "rewards/chosen": 0.2647041380405426,
+      "rewards/margins": 0.0454571396112442,
+      "rewards/rejected": 0.2192470282316208,
       "step": 10
     },
     {
-      "epoch": 0.04184100418410042,
-      "grad_norm": 41.61164521948544,
+      "epoch": 0.04,
+      "grad_norm": 33.37630632393394,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -2.6528677940368652,
-      "logits/rejected": -2.6049089431762695,
-      "logps/chosen": -281.41082763671875,
-      "logps/rejected": -296.81597900390625,
-      "loss": 0.6893,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.0071225822903215885,
-      "rewards/margins": 0.007576107978820801,
-      "rewards/rejected": -0.0004535239131655544,
+      "logits/chosen": -2.6635663509368896,
+      "logits/rejected": -2.6177525520324707,
+      "logps/chosen": -275.1928405761719,
+      "logps/rejected": -290.4365539550781,
+      "loss": 0.5763,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 6.3604888916015625,
+      "rewards/margins": -0.009852093644440174,
+      "rewards/rejected": 6.370340824127197,
       "step": 20
     },
     {
-      "epoch": 0.06276150627615062,
-      "grad_norm": 39.634180213591634,
+      "epoch": 0.06,
+      "grad_norm": 22.1278736890366,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -2.6643600463867188,
-      "logits/rejected": -2.5916037559509277,
-      "logps/chosen": -299.44146728515625,
-      "logps/rejected": -263.7098083496094,
-      "loss": 0.6715,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.06151618808507919,
-      "rewards/margins": 0.049498964101076126,
-      "rewards/rejected": 0.01201721839606762,
+      "logits/chosen": -2.7272486686706543,
+      "logits/rejected": -2.667067527770996,
+      "logps/chosen": -285.1613464355469,
+      "logps/rejected": -249.3108367919922,
+      "loss": 0.4416,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 15.510467529296875,
+      "rewards/margins": 0.8711569905281067,
+      "rewards/rejected": 14.639310836791992,
       "step": 30
     },
     {
-      "epoch": 0.08368200836820083,
-      "grad_norm": 38.153095209098765,
+      "epoch": 0.08,
+      "grad_norm": 17.071895487907064,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.5593624114990234,
-      "logits/rejected": -2.5243585109710693,
-      "logps/chosen": -263.2409973144531,
-      "logps/rejected": -244.7979736328125,
-      "loss": 0.6345,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.19421888887882233,
-      "rewards/margins": 0.21451684832572937,
-      "rewards/rejected": -0.02029796503484249,
+      "logits/chosen": -2.6888694763183594,
+      "logits/rejected": -2.6701016426086426,
+      "logps/chosen": -247.84716796875,
+      "logps/rejected": -227.38131713867188,
+      "loss": 0.3982,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 19.278215408325195,
+      "rewards/margins": 2.267552137374878,
+      "rewards/rejected": 17.010662078857422,
       "step": 40
     },
     {
-      "epoch": 0.10460251046025104,
-      "grad_norm": 36.223426064268466,
+      "epoch": 0.1,
+      "grad_norm": 14.78162706214556,
       "learning_rate": 4.999733114418725e-07,
-      "logits/chosen": -2.508179187774658,
-      "logits/rejected": -2.4701850414276123,
-      "logps/chosen": -277.1423645019531,
-      "logps/rejected": -294.7258605957031,
-      "loss": 0.6007,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.1294766217470169,
-      "rewards/margins": 0.22656401991844177,
-      "rewards/rejected": -0.09708739072084427,
+      "logits/chosen": -2.659508466720581,
+      "logits/rejected": -2.6249804496765137,
+      "logps/chosen": -259.9454650878906,
+      "logps/rejected": -272.14227294921875,
+      "loss": 0.3676,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 19.786420822143555,
+      "rewards/margins": -0.8553922772407532,
+      "rewards/rejected": 20.64181137084961,
       "step": 50
     },
     {
-      "epoch": 0.12552301255230125,
-      "grad_norm": 45.16419531360444,
+      "epoch": 0.13,
+      "grad_norm": 14.285832773490087,
       "learning_rate": 4.990398100856366e-07,
-      "logits/chosen": -2.5453555583953857,
-      "logits/rejected": -2.478151559829712,
-      "logps/chosen": -263.7626953125,
-      "logps/rejected": -298.6721496582031,
-      "loss": 0.5858,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.3921932578086853,
-      "rewards/margins": 0.42173171043395996,
-      "rewards/rejected": -0.02953849732875824,
+      "logits/chosen": -2.6977388858795166,
+      "logits/rejected": -2.654181957244873,
+      "logps/chosen": -247.1780242919922,
+      "logps/rejected": -275.7373962402344,
+      "loss": 0.3521,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 24.428516387939453,
+      "rewards/margins": 2.0845706462860107,
+      "rewards/rejected": 22.343944549560547,
       "step": 60
     },
     {
-      "epoch": 0.14644351464435146,
-      "grad_norm": 41.305043877818754,
+      "epoch": 0.15,
+      "grad_norm": 14.416469937136577,
       "learning_rate": 4.967775735898179e-07,
-      "logits/chosen": -2.4226365089416504,
-      "logits/rejected": -2.4207093715667725,
-      "logps/chosen": -256.3829345703125,
-      "logps/rejected": -258.1474609375,
-      "loss": 0.5542,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.46167653799057007,
-      "rewards/margins": 0.5331880450248718,
-      "rewards/rejected": -0.07151149958372116,
+      "logits/chosen": -2.6118428707122803,
+      "logits/rejected": -2.625479221343994,
+      "logps/chosen": -239.4540252685547,
+      "logps/rejected": -232.90463256835938,
+      "loss": 0.3304,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 26.162424087524414,
+      "rewards/margins": 2.349818706512451,
+      "rewards/rejected": 23.812606811523438,
       "step": 70
     },
     {
-      "epoch": 0.16736401673640167,
-      "grad_norm": 51.35773443134982,
+      "epoch": 0.17,
+      "grad_norm": 15.840881084472352,
       "learning_rate": 4.931986719649298e-07,
-      "logits/chosen": -2.5735816955566406,
-      "logits/rejected": -2.4945335388183594,
-      "logps/chosen": -316.7965393066406,
-      "logps/rejected": -271.0341491699219,
-      "loss": 0.5465,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.3010686933994293,
-      "rewards/margins": 0.6498448252677917,
-      "rewards/rejected": -0.3487761616706848,
+      "logits/chosen": -2.7612788677215576,
+      "logits/rejected": -2.7243030071258545,
+      "logps/chosen": -295.0336608886719,
+      "logps/rejected": -240.8730010986328,
+      "loss": 0.3248,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 27.784252166748047,
+      "rewards/margins": 4.598628997802734,
+      "rewards/rejected": 23.185623168945312,
       "step": 80
     },
     {
-      "epoch": 0.18828451882845187,
-      "grad_norm": 40.17920200162099,
+      "epoch": 0.19,
+      "grad_norm": 13.661268677283298,
       "learning_rate": 4.883222001996351e-07,
-      "logits/chosen": -2.449328660964966,
-      "logits/rejected": -2.3984742164611816,
-      "logps/chosen": -265.48095703125,
-      "logps/rejected": -277.22491455078125,
-      "loss": 0.5443,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.31848278641700745,
-      "rewards/margins": 0.9228578805923462,
-      "rewards/rejected": -1.2413406372070312,
+      "logits/chosen": -2.6661014556884766,
+      "logits/rejected": -2.645249128341675,
+      "logps/chosen": -231.57553100585938,
+      "logps/rejected": -228.09091186523438,
+      "loss": 0.3223,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 27.535770416259766,
+      "rewards/margins": 3.228619337081909,
+      "rewards/rejected": 24.30714988708496,
       "step": 90
     },
     {
-      "epoch": 0.20920502092050208,
-      "grad_norm": 35.58115137888054,
+      "epoch": 0.21,
+      "grad_norm": 11.61288143003843,
       "learning_rate": 4.821741763807186e-07,
-      "logits/chosen": -2.4374217987060547,
-      "logits/rejected": -2.4048655033111572,
-      "logps/chosen": -258.9109802246094,
-      "logps/rejected": -271.645263671875,
-      "loss": 0.5414,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.07237578183412552,
-      "rewards/margins": 0.8032167553901672,
-      "rewards/rejected": -0.7308409810066223,
+      "logits/chosen": -2.6386702060699463,
+      "logits/rejected": -2.6339759826660156,
+      "logps/chosen": -233.39047241210938,
+      "logps/rejected": -232.5922393798828,
+      "loss": 0.3163,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 26.968032836914062,
+      "rewards/margins": 2.5318057537078857,
+      "rewards/rejected": 24.436227798461914,
       "step": 100
     },
     {
-      "epoch": 0.20920502092050208,
-      "eval_logits/chosen": -2.4971094131469727,
-      "eval_logits/rejected": -2.4577691555023193,
-      "eval_logps/chosen": -260.6309814453125,
-      "eval_logps/rejected": -274.6580810546875,
-      "eval_loss": 0.5490534901618958,
-      "eval_rewards/accuracies": 0.7578125,
-      "eval_rewards/chosen": 0.09812341630458832,
-      "eval_rewards/margins": 0.6976690292358398,
-      "eval_rewards/rejected": -0.5995457172393799,
-      "eval_runtime": 96.7939,
-      "eval_samples_per_second": 20.662,
+      "epoch": 0.21,
+      "eval_logits/chosen": -2.6968541145324707,
+      "eval_logits/rejected": -2.670072555541992,
+      "eval_logps/chosen": -235.37875366210938,
+      "eval_logps/rejected": -238.44345092773438,
+      "eval_loss": 0.31289389729499817,
+      "eval_rewards/accuracies": 0.58203125,
+      "eval_rewards/chosen": 27.21471405029297,
+      "eval_rewards/margins": 2.99098801612854,
+      "eval_rewards/rejected": 24.223726272583008,
+      "eval_runtime": 96.735,
+      "eval_samples_per_second": 20.675,
       "eval_steps_per_second": 0.331,
       "step": 100
     },
     {
-      "epoch": 0.2301255230125523,
-      "grad_norm": 48.888412965077826,
+      "epoch": 0.23,
+      "grad_norm": 11.688620320219954,
       "learning_rate": 4.747874028753375e-07,
-      "logits/chosen": -2.507007598876953,
-      "logits/rejected": -2.409585475921631,
-      "logps/chosen": -302.9889221191406,
-      "logps/rejected": -270.8835754394531,
-      "loss": 0.5511,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.07955702394247055,
-      "rewards/margins": 0.7319403886795044,
-      "rewards/rejected": -0.6523833870887756,
+      "logits/chosen": -2.7125041484832764,
+      "logits/rejected": -2.6624934673309326,
+      "logps/chosen": -276.029052734375,
+      "logps/rejected": -234.1141815185547,
+      "loss": 0.3136,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 28.551036834716797,
+      "rewards/margins": 4.829342842102051,
+      "rewards/rejected": 23.72169303894043,
       "step": 110
     },
     {
-      "epoch": 0.2510460251046025,
-      "grad_norm": 73.38684681576858,
+      "epoch": 0.25,
+      "grad_norm": 14.849649400244427,
       "learning_rate": 4.662012913161997e-07,
-      "logits/chosen": -2.4269328117370605,
-      "logits/rejected": -2.3998618125915527,
-      "logps/chosen": -279.83642578125,
-      "logps/rejected": -272.7640380859375,
-      "loss": 0.5288,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 0.0665363073348999,
-      "rewards/margins": 0.7119099497795105,
-      "rewards/rejected": -0.6453736424446106,
+      "logits/chosen": -2.6516470909118652,
+      "logits/rejected": -2.647688865661621,
+      "logps/chosen": -253.4019317626953,
+      "logps/rejected": -234.5045623779297,
+      "loss": 0.3065,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 27.765233993530273,
+      "rewards/margins": 2.4132068157196045,
+      "rewards/rejected": 25.352027893066406,
       "step": 120
     },
     {
-      "epoch": 0.2719665271966527,
-      "grad_norm": 40.38049427137653,
+      "epoch": 0.27,
+      "grad_norm": 12.095477452171375,
       "learning_rate": 4.5646165232345103e-07,
-      "logits/chosen": -2.4314827919006348,
-      "logits/rejected": -2.4052321910858154,
-      "logps/chosen": -284.8794250488281,
-      "logps/rejected": -288.5877990722656,
-      "loss": 0.5134,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.14381402730941772,
-      "rewards/margins": 0.8637838363647461,
-      "rewards/rejected": -1.0075979232788086,
+      "logits/chosen": -2.679412364959717,
+      "logits/rejected": -2.6742541790008545,
+      "logps/chosen": -249.6054229736328,
+      "logps/rejected": -241.8912811279297,
+      "loss": 0.2993,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 32.39772415161133,
+      "rewards/margins": 5.853152275085449,
+      "rewards/rejected": 26.544570922851562,
       "step": 130
     },
     {
-      "epoch": 0.2928870292887029,
-      "grad_norm": 41.38754375370393,
+      "epoch": 0.29,
+      "grad_norm": 13.237989201417717,
       "learning_rate": 4.456204510851956e-07,
-      "logits/chosen": -2.4810681343078613,
-      "logits/rejected": -2.442735195159912,
-      "logps/chosen": -313.91607666015625,
-      "logps/rejected": -316.1298828125,
-      "loss": 0.5118,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": 0.1024663895368576,
-      "rewards/margins": 0.8751181364059448,
-      "rewards/rejected": -0.7726517915725708,
+      "logits/chosen": -2.7010607719421387,
+      "logits/rejected": -2.689103603363037,
+      "logps/chosen": -284.6669921875,
+      "logps/rejected": -270.44970703125,
+      "loss": 0.3016,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 31.298425674438477,
+      "rewards/margins": 1.071274995803833,
+      "rewards/rejected": 30.22715187072754,
       "step": 140
     },
     {
-      "epoch": 0.3138075313807531,
-      "grad_norm": 43.42635489034559,
+      "epoch": 0.31,
+      "grad_norm": 11.533759549255185,
       "learning_rate": 4.337355301007335e-07,
-      "logits/chosen": -2.4657142162323,
-      "logits/rejected": -2.3948910236358643,
-      "logps/chosen": -287.9931335449219,
-      "logps/rejected": -294.61566162109375,
-      "loss": 0.5202,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.23844678699970245,
-      "rewards/margins": 0.6818856000900269,
-      "rewards/rejected": -0.9203322529792786,
+      "logits/chosen": -2.6910300254821777,
+      "logits/rejected": -2.6623480319976807,
+      "logps/chosen": -251.215576171875,
+      "logps/rejected": -248.98348999023438,
+      "loss": 0.2985,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": 32.008628845214844,
+      "rewards/margins": 4.783123970031738,
+      "rewards/rejected": 27.225509643554688,
       "step": 150
     },
     {
-      "epoch": 0.33472803347280333,
-      "grad_norm": 46.56278104633887,
+      "epoch": 0.33,
+      "grad_norm": 13.117822478323479,
       "learning_rate": 4.2087030056579986e-07,
-      "logits/chosen": -2.4711461067199707,
-      "logits/rejected": -2.3905088901519775,
-      "logps/chosen": -273.55096435546875,
-      "logps/rejected": -278.7655334472656,
-      "loss": 0.5435,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.024145102128386497,
-      "rewards/margins": 1.042937994003296,
-      "rewards/rejected": -1.0670831203460693,
+      "logits/chosen": -2.721895217895508,
+      "logits/rejected": -2.675842523574829,
+      "logps/chosen": -242.4053192138672,
+      "logps/rejected": -230.8060302734375,
+      "loss": 0.3009,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 30.662723541259766,
+      "rewards/margins": 4.044883728027344,
+      "rewards/rejected": 26.61783790588379,
       "step": 160
     },
     {
-      "epoch": 0.35564853556485354,
-      "grad_norm": 52.89197089191812,
+      "epoch": 0.36,
+      "grad_norm": 11.340151801902158,
       "learning_rate": 4.070934040463998e-07,
-      "logits/chosen": -2.4202826023101807,
-      "logits/rejected": -2.3639273643493652,
-      "logps/chosen": -253.7214813232422,
-      "logps/rejected": -252.5417938232422,
-      "loss": 0.5259,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.21931107342243195,
-      "rewards/margins": 0.8515389561653137,
-      "rewards/rejected": -1.070850133895874,
+      "logits/chosen": -2.670436382293701,
+      "logits/rejected": -2.632450819015503,
+      "logps/chosen": -220.5222625732422,
+      "logps/rejected": -204.80908203125,
+      "loss": 0.2938,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 28.81294822692871,
+      "rewards/margins": 2.497253179550171,
+      "rewards/rejected": 26.31569480895996,
       "step": 170
     },
     {
-      "epoch": 0.37656903765690375,
-      "grad_norm": 42.71983591222299,
+      "epoch": 0.38,
+      "grad_norm": 11.477634324684333,
       "learning_rate": 3.9247834624635404e-07,
-      "logits/chosen": -2.3343987464904785,
-      "logits/rejected": -2.2835183143615723,
-      "logps/chosen": -259.6838073730469,
-      "logps/rejected": -248.00656127929688,
-      "loss": 0.4919,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.1662406176328659,
-      "rewards/margins": 0.8555324673652649,
-      "rewards/rejected": -1.0217732191085815,
+      "logits/chosen": -2.646768093109131,
+      "logits/rejected": -2.6306955814361572,
+      "logps/chosen": -225.45016479492188,
+      "logps/rejected": -200.42015075683594,
+      "loss": 0.2914,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 30.908817291259766,
+      "rewards/margins": 3.7578415870666504,
+      "rewards/rejected": 27.150976181030273,
       "step": 180
     },
     {
-      "epoch": 0.39748953974895396,
-      "grad_norm": 41.56887324855658,
+      "epoch": 0.4,
+      "grad_norm": 13.566633133843082,
       "learning_rate": 3.7710310482256523e-07,
-      "logits/chosen": -2.401069164276123,
-      "logits/rejected": -2.354982614517212,
-      "logps/chosen": -273.66888427734375,
-      "logps/rejected": -279.1777038574219,
-      "loss": 0.507,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.12130476534366608,
-      "rewards/margins": 0.849841296672821,
-      "rewards/rejected": -0.9711461067199707,
+      "logits/chosen": -2.679771900177002,
+      "logits/rejected": -2.6499440670013428,
+      "logps/chosen": -241.45156860351562,
+      "logps/rejected": -231.2630615234375,
+      "loss": 0.2963,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 29.79128646850586,
+      "rewards/margins": 1.2995483875274658,
+      "rewards/rejected": 28.49173927307129,
       "step": 190
     },
     {
-      "epoch": 0.41841004184100417,
-      "grad_norm": 112.3507311789328,
+      "epoch": 0.42,
+      "grad_norm": 16.736011308973627,
       "learning_rate": 3.610497133404795e-07,
-      "logits/chosen": -2.3550214767456055,
-      "logits/rejected": -2.3447892665863037,
-      "logps/chosen": -263.72918701171875,
-      "logps/rejected": -273.32769775390625,
-      "loss": 0.525,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.19162645936012268,
-      "rewards/margins": 0.8723103404045105,
-      "rewards/rejected": -1.0639369487762451,
+      "logits/chosen": -2.630007028579712,
+      "logits/rejected": -2.6183559894561768,
+      "logps/chosen": -230.09048461914062,
+      "logps/rejected": -223.8180694580078,
+      "loss": 0.2918,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": 29.806177139282227,
+      "rewards/margins": 1.575269341468811,
+      "rewards/rejected": 28.230907440185547,
       "step": 200
     },
     {
-      "epoch": 0.41841004184100417,
-      "eval_logits/chosen": -2.464841604232788,
-      "eval_logits/rejected": -2.422513008117676,
-      "eval_logps/chosen": -261.7037048339844,
-      "eval_logps/rejected": -280.6389465332031,
-      "eval_loss": 0.5203626751899719,
-      "eval_rewards/accuracies": 0.76953125,
-      "eval_rewards/chosen": 0.04448739066720009,
-      "eval_rewards/margins": 0.9430755972862244,
-      "eval_rewards/rejected": -0.8985880613327026,
-      "eval_runtime": 96.5751,
-      "eval_samples_per_second": 20.709,
-      "eval_steps_per_second": 0.331,
+      "epoch": 0.42,
+      "eval_logits/chosen": -2.708475112915039,
+      "eval_logits/rejected": -2.682575225830078,
+      "eval_logps/chosen": -232.24124145507812,
+      "eval_logps/rejected": -236.21038818359375,
+      "eval_loss": 0.29230329394340515,
+      "eval_rewards/accuracies": 0.58203125,
+      "eval_rewards/chosen": 30.35222816467285,
+      "eval_rewards/margins": 3.8954334259033203,
+      "eval_rewards/rejected": 26.45679473876953,
+      "eval_runtime": 96.829,
+      "eval_samples_per_second": 20.655,
+      "eval_steps_per_second": 0.33,
       "step": 200
     },
     {
-      "epoch": 0.4393305439330544,
-      "grad_norm": 51.39723815476762,
+      "epoch": 0.44,
+      "grad_norm": 11.417465496451523,
       "learning_rate": 3.4440382358952115e-07,
-      "logits/chosen": -2.384660243988037,
-      "logits/rejected": -2.3313751220703125,
-      "logps/chosen": -291.1485900878906,
-      "logps/rejected": -272.11468505859375,
-      "loss": 0.5256,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.07282108068466187,
-      "rewards/margins": 0.9705331921577454,
-      "rewards/rejected": -1.0433542728424072,
+      "logits/chosen": -2.6330389976501465,
+      "logits/rejected": -2.6055209636688232,
+      "logps/chosen": -257.6673889160156,
+      "logps/rejected": -225.943359375,
+      "loss": 0.2902,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 32.02475357055664,
+      "rewards/margins": 6.720486640930176,
+      "rewards/rejected": 25.304264068603516,
       "step": 210
     },
     {
-      "epoch": 0.4602510460251046,
-      "grad_norm": 47.88126658740226,
+      "epoch": 0.46,
+      "grad_norm": 12.04727391696027,
       "learning_rate": 3.272542485937368e-07,
-      "logits/chosen": -2.3982672691345215,
-      "logits/rejected": -2.3606152534484863,
-      "logps/chosen": -273.4466857910156,
-      "logps/rejected": -273.1665344238281,
-      "loss": 0.5103,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.40347856283187866,
-      "rewards/margins": 0.9716581106185913,
-      "rewards/rejected": -1.3751367330551147,
+      "logits/chosen": -2.5957412719726562,
+      "logits/rejected": -2.5795822143554688,
+      "logps/chosen": -233.29476928710938,
+      "logps/rejected": -217.3531951904297,
+      "loss": 0.2919,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 32.082313537597656,
+      "rewards/margins": 3.7717392444610596,
+      "rewards/rejected": 28.310577392578125,
       "step": 220
     },
     {
-      "epoch": 0.4811715481171548,
-      "grad_norm": 45.311062314075045,
+      "epoch": 0.48,
+      "grad_norm": 11.505656123665526,
       "learning_rate": 3.096924887558854e-07,
-      "logits/chosen": -2.4156343936920166,
-      "logits/rejected": -2.3753621578216553,
-      "logps/chosen": -252.9831085205078,
-      "logps/rejected": -274.3964538574219,
-      "loss": 0.5175,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.2058834731578827,
-      "rewards/margins": 1.127046823501587,
-      "rewards/rejected": -1.3329302072525024,
+      "logits/chosen": -2.6124305725097656,
+      "logits/rejected": -2.5944228172302246,
+      "logps/chosen": -217.5354461669922,
+      "logps/rejected": -220.5460205078125,
+      "loss": 0.3047,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 31.32999038696289,
+      "rewards/margins": 4.138183116912842,
+      "rewards/rejected": 27.19180679321289,
       "step": 230
     },
     {
-      "epoch": 0.502092050209205,
-      "grad_norm": 47.38487159577979,
+      "epoch": 0.5,
+      "grad_norm": 11.083392566284138,
       "learning_rate": 2.9181224366319943e-07,
-      "logits/chosen": -2.4370293617248535,
-      "logits/rejected": -2.4109508991241455,
-      "logps/chosen": -267.2882385253906,
-      "logps/rejected": -270.58203125,
-      "loss": 0.487,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.1794019192457199,
-      "rewards/margins": 0.8962798118591309,
-      "rewards/rejected": -1.0756816864013672,
+      "logits/chosen": -2.660727024078369,
+      "logits/rejected": -2.6385245323181152,
+      "logps/chosen": -232.0665740966797,
+      "logps/rejected": -219.62210083007812,
+      "loss": 0.2834,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 31.633642196655273,
+      "rewards/margins": 2.1873562335968018,
+      "rewards/rejected": 29.446285247802734,
       "step": 240
     },
     {
-      "epoch": 0.5230125523012552,
-      "grad_norm": 50.43199210692349,
+      "epoch": 0.52,
+      "grad_norm": 11.463127161742676,
       "learning_rate": 2.7370891215954565e-07,
-      "logits/chosen": -2.402012825012207,
-      "logits/rejected": -2.3266422748565674,
-      "logps/chosen": -301.61773681640625,
-      "logps/rejected": -284.40167236328125,
-      "loss": 0.4952,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.1713608205318451,
-      "rewards/margins": 1.072595238685608,
-      "rewards/rejected": -1.2439559698104858,
+      "logits/chosen": -2.6206917762756348,
+      "logits/rejected": -2.576387405395508,
+      "logps/chosen": -264.06439208984375,
+      "logps/rejected": -229.7786865234375,
+      "loss": 0.2818,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 34.12608337402344,
+      "rewards/margins": 4.382205009460449,
+      "rewards/rejected": 29.743881225585938,
       "step": 250
     },
     {
-      "epoch": 0.5439330543933054,
-      "grad_norm": 42.01017847885537,
+      "epoch": 0.54,
+      "grad_norm": 10.661524920447267,
       "learning_rate": 2.55479083351317e-07,
-      "logits/chosen": -2.509986400604248,
-      "logits/rejected": -2.471529245376587,
-      "logps/chosen": -297.0230712890625,
-      "logps/rejected": -275.581787109375,
-      "loss": 0.4949,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.1355748027563095,
-      "rewards/margins": 0.9444950819015503,
-      "rewards/rejected": -1.0800697803497314,
+      "logits/chosen": -2.6774675846099854,
+      "logits/rejected": -2.668527364730835,
+      "logps/chosen": -260.33514404296875,
+      "logps/rejected": -225.80810546875,
+      "loss": 0.2858,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": 33.976402282714844,
+      "rewards/margins": 5.804098606109619,
+      "rewards/rejected": 28.17230224609375,
       "step": 260
     },
     {
-      "epoch": 0.5648535564853556,
-      "grad_norm": 42.0564206770154,
+      "epoch": 0.56,
+      "grad_norm": 11.916616915089687,
       "learning_rate": 2.3722002126275822e-07,
-      "logits/chosen": -2.578350782394409,
-      "logits/rejected": -2.5308547019958496,
-      "logps/chosen": -284.1900939941406,
-      "logps/rejected": -281.2919921875,
-      "loss": 0.5164,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.35970669984817505,
-      "rewards/margins": 0.8226785659790039,
-      "rewards/rejected": -1.1823852062225342,
+      "logits/chosen": -2.6731224060058594,
+      "logits/rejected": -2.6551766395568848,
+      "logps/chosen": -245.6435089111328,
+      "logps/rejected": -228.1649932861328,
+      "loss": 0.2808,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 31.35245704650879,
+      "rewards/margins": 1.8731645345687866,
+      "rewards/rejected": 29.479290008544922,
       "step": 270
     },
     {
-      "epoch": 0.5857740585774058,
-      "grad_norm": 48.57904010890843,
+      "epoch": 0.59,
+      "grad_norm": 11.982078860289866,
       "learning_rate": 2.19029145890313e-07,
-      "logits/chosen": -2.5550217628479004,
-      "logits/rejected": -2.486069679260254,
-      "logps/chosen": -270.565185546875,
-      "logps/rejected": -270.28680419921875,
-      "loss": 0.5324,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.44440436363220215,
-      "rewards/margins": 0.9606183767318726,
-      "rewards/rejected": -1.4050228595733643,
+      "logits/chosen": -2.6452529430389404,
+      "logits/rejected": -2.6127915382385254,
+      "logps/chosen": -229.02554321289062,
+      "logps/rejected": -215.188720703125,
+      "loss": 0.2835,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 32.651554107666016,
+      "rewards/margins": 5.653929233551025,
+      "rewards/rejected": 26.99761962890625,
       "step": 280
     },
     {
-      "epoch": 0.606694560669456,
-      "grad_norm": 42.48920274115382,
+      "epoch": 0.61,
+      "grad_norm": 11.17239233559609,
       "learning_rate": 2.0100351342479216e-07,
-      "logits/chosen": -2.6016674041748047,
-      "logits/rejected": -2.552732467651367,
-      "logps/chosen": -261.68572998046875,
-      "logps/rejected": -269.89862060546875,
-      "loss": 0.509,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.5299187898635864,
-      "rewards/margins": 0.8672173619270325,
-      "rewards/rejected": -1.3971360921859741,
+      "logits/chosen": -2.675553321838379,
+      "logits/rejected": -2.662069082260132,
+      "logps/chosen": -219.8170928955078,
+      "logps/rejected": -211.7806396484375,
+      "loss": 0.2849,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 31.27024269104004,
+      "rewards/margins": 1.0949894189834595,
+      "rewards/rejected": 30.175247192382812,
       "step": 290
     },
     {
-      "epoch": 0.6276150627615062,
-      "grad_norm": 57.28833501038638,
+      "epoch": 0.63,
+      "grad_norm": 9.847053265544167,
       "learning_rate": 1.8323929841460178e-07,
-      "logits/chosen": -2.586317539215088,
-      "logits/rejected": -2.5180938243865967,
-      "logps/chosen": -309.8882751464844,
-      "logps/rejected": -287.2384338378906,
-      "loss": 0.4908,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.4446079730987549,
-      "rewards/margins": 0.9120391607284546,
-      "rewards/rejected": -1.3566471338272095,
+      "logits/chosen": -2.65397572517395,
+      "logits/rejected": -2.6134414672851562,
+      "logps/chosen": -268.84588623046875,
+      "logps/rejected": -232.80752563476562,
+      "loss": 0.286,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 32.15021514892578,
+      "rewards/margins": 4.852233409881592,
+      "rewards/rejected": 27.297988891601562,
       "step": 300
     },
     {
-      "epoch": 0.6276150627615062,
-      "eval_logits/chosen": -2.593677043914795,
-      "eval_logits/rejected": -2.5527396202087402,
-      "eval_logps/chosen": -266.43524169921875,
-      "eval_logps/rejected": -287.60107421875,
-      "eval_loss": 0.5125026702880859,
-      "eval_rewards/accuracies": 0.77734375,
-      "eval_rewards/chosen": -0.19208788871765137,
-      "eval_rewards/margins": 1.0546070337295532,
-      "eval_rewards/rejected": -1.2466949224472046,
-      "eval_runtime": 96.6252,
-      "eval_samples_per_second": 20.699,
+      "epoch": 0.63,
+      "eval_logits/chosen": -2.695726156234741,
+      "eval_logits/rejected": -2.6716713905334473,
+      "eval_logps/chosen": -231.15402221679688,
+      "eval_logps/rejected": -235.42864990234375,
+      "eval_loss": 0.29209351539611816,
+      "eval_rewards/accuracies": 0.58203125,
+      "eval_rewards/chosen": 31.439437866210938,
+      "eval_rewards/margins": 4.200903415679932,
+      "eval_rewards/rejected": 27.238534927368164,
+      "eval_runtime": 96.789,
+      "eval_samples_per_second": 20.664,
       "eval_steps_per_second": 0.331,
       "step": 300
     },
     {
-      "epoch": 0.6485355648535565,
-      "grad_norm": 43.10429882787701,
+      "epoch": 0.65,
+      "grad_norm": 11.299461074514115,
       "learning_rate": 1.6583128063291573e-07,
-      "logits/chosen": -2.4947943687438965,
-      "logits/rejected": -2.4790587425231934,
-      "logps/chosen": -300.5895080566406,
-      "logps/rejected": -282.02764892578125,
-      "loss": 0.4823,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.2626795172691345,
-      "rewards/margins": 0.9128748178482056,
-      "rewards/rejected": -1.1755545139312744,
+      "logits/chosen": -2.6087942123413086,
+      "logits/rejected": -2.607959270477295,
+      "logps/chosen": -263.2939758300781,
+      "logps/rejected": -229.5752716064453,
+      "loss": 0.2804,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 32.041908264160156,
+      "rewards/margins": 3.100654363632202,
+      "rewards/rejected": 28.941247940063477,
       "step": 310
     },
     {
-      "epoch": 0.6694560669456067,
-      "grad_norm": 43.33310818353699,
+      "epoch": 0.67,
+      "grad_norm": 11.979925902064297,
       "learning_rate": 1.488723393865766e-07,
-      "logits/chosen": -2.5217020511627197,
-      "logits/rejected": -2.479039430618286,
-      "logps/chosen": -298.35479736328125,
-      "logps/rejected": -268.5308532714844,
-      "loss": 0.4701,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.23423710465431213,
-      "rewards/margins": 0.9511427879333496,
-      "rewards/rejected": -1.1853798627853394,
+      "logits/chosen": -2.652468204498291,
+      "logits/rejected": -2.6433398723602295,
+      "logps/chosen": -260.83233642578125,
+      "logps/rejected": -216.2664337158203,
+      "loss": 0.2788,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 32.8377571105957,
+      "rewards/margins": 4.280916213989258,
+      "rewards/rejected": 28.556838989257812,
       "step": 320
     },
     {
-      "epoch": 0.6903765690376569,
-      "grad_norm": 41.61026950600333,
+      "epoch": 0.69,
+      "grad_norm": 10.289416601586245,
       "learning_rate": 1.3245295796480788e-07,
-      "logits/chosen": -2.5208044052124023,
-      "logits/rejected": -2.454449415206909,
-      "logps/chosen": -267.09185791015625,
-      "logps/rejected": -282.6268005371094,
-      "loss": 0.4905,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.24408385157585144,
-      "rewards/margins": 0.9123236536979675,
-      "rewards/rejected": -1.1564075946807861,
+      "logits/chosen": -2.678496837615967,
+      "logits/rejected": -2.634920835494995,
+      "logps/chosen": -229.55624389648438,
+      "logps/rejected": -231.64407348632812,
+      "loss": 0.2812,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 32.6539306640625,
+      "rewards/margins": 4.799349784851074,
+      "rewards/rejected": 27.854583740234375,
       "step": 330
     },
     {
-      "epoch": 0.7112970711297071,
-      "grad_norm": 50.989766016006286,
+      "epoch": 0.71,
+      "grad_norm": 12.940304501019066,
       "learning_rate": 1.1666074087171627e-07,
-      "logits/chosen": -2.521392345428467,
-      "logits/rejected": -2.456019639968872,
-      "logps/chosen": -293.96246337890625,
-      "logps/rejected": -306.7515869140625,
-      "loss": 0.495,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.19840189814567566,
-      "rewards/margins": 1.1370878219604492,
-      "rewards/rejected": -1.3354896306991577,
+      "logits/chosen": -2.687782049179077,
+      "logits/rejected": -2.6474757194519043,
+      "logps/chosen": -258.529541015625,
+      "logps/rejected": -247.69125366210938,
+      "loss": 0.2752,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 31.464908599853516,
+      "rewards/margins": -0.8856052160263062,
+      "rewards/rejected": 32.35051727294922,
       "step": 340
     },
     {
-      "epoch": 0.7322175732217573,
-      "grad_norm": 45.02802661804661,
+      "epoch": 0.73,
+      "grad_norm": 13.446019747621028,
       "learning_rate": 1.0157994641835734e-07,
-      "logits/chosen": -2.495126247406006,
-      "logits/rejected": -2.432142734527588,
-      "logps/chosen": -263.77362060546875,
-      "logps/rejected": -267.4032287597656,
-      "loss": 0.4615,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.27811866998672485,
-      "rewards/margins": 1.0566023588180542,
-      "rewards/rejected": -1.3347210884094238,
+      "logits/chosen": -2.6681811809539795,
+      "logits/rejected": -2.6358139514923096,
+      "logps/chosen": -227.58425903320312,
+      "logps/rejected": -212.9467010498047,
+      "loss": 0.2866,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 30.626983642578125,
+      "rewards/margins": 2.8648905754089355,
+      "rewards/rejected": 27.7620906829834,
       "step": 350
     },
     {
-      "epoch": 0.7531380753138075,
-      "grad_norm": 56.589987665603815,
+      "epoch": 0.75,
+      "grad_norm": 10.212615361555141,
       "learning_rate": 8.729103716819111e-08,
-      "logits/chosen": -2.5248653888702393,
-      "logits/rejected": -2.434735059738159,
-      "logps/chosen": -307.5997619628906,
-      "logps/rejected": -288.23223876953125,
-      "loss": 0.5085,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.24538645148277283,
-      "rewards/margins": 1.1009799242019653,
-      "rewards/rejected": -1.3463664054870605,
+      "logits/chosen": -2.691338300704956,
+      "logits/rejected": -2.6329030990600586,
+      "logps/chosen": -269.2547302246094,
+      "logps/rejected": -233.14053344726562,
+      "loss": 0.2785,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 33.437278747558594,
+      "rewards/margins": 5.27285623550415,
+      "rewards/rejected": 28.1644287109375,
       "step": 360
     },
     {
-      "epoch": 0.7740585774058577,
-      "grad_norm": 45.8812566960427,
+      "epoch": 0.77,
+      "grad_norm": 12.701608094493194,
       "learning_rate": 7.387025063449081e-08,
-      "logits/chosen": -2.4673755168914795,
-      "logits/rejected": -2.4324138164520264,
-      "logps/chosen": -282.8244323730469,
-      "logps/rejected": -262.7843322753906,
-      "loss": 0.5091,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.37457117438316345,
-      "rewards/margins": 0.8529427647590637,
-      "rewards/rejected": -1.2275140285491943,
+      "logits/chosen": -2.6507325172424316,
+      "logits/rejected": -2.6226696968078613,
+      "logps/chosen": -243.0960693359375,
+      "logps/rejected": -207.664794921875,
+      "loss": 0.2854,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 32.23695373535156,
+      "rewards/margins": 1.6676933765411377,
+      "rewards/rejected": 30.569263458251953,
       "step": 370
     },
     {
-      "epoch": 0.7949790794979079,
-      "grad_norm": 41.78805473389246,
+      "epoch": 0.79,
+      "grad_norm": 11.004484883830752,
       "learning_rate": 6.138919252022435e-08,
-      "logits/chosen": -2.4137110710144043,
-      "logits/rejected": -2.392765522003174,
-      "logps/chosen": -251.4576416015625,
-      "logps/rejected": -292.29931640625,
-      "loss": 0.495,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.5986499190330505,
-      "rewards/margins": 1.0804951190948486,
-      "rewards/rejected": -1.679145097732544,
+      "logits/chosen": -2.592874526977539,
+      "logits/rejected": -2.5939741134643555,
+      "logps/chosen": -206.689697265625,
+      "logps/rejected": -228.67898559570312,
+      "loss": 0.2774,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 32.79497146606445,
+      "rewards/margins": 2.7575299739837646,
+      "rewards/rejected": 30.037445068359375,
       "step": 380
     },
     {
-      "epoch": 0.8158995815899581,
-      "grad_norm": 44.97232993536685,
+      "epoch": 0.82,
+      "grad_norm": 12.608909298282311,
       "learning_rate": 4.991445467064689e-08,
-      "logits/chosen": -2.47938871383667,
-      "logits/rejected": -2.4436278343200684,
-      "logps/chosen": -314.1337585449219,
-      "logps/rejected": -312.0073547363281,
-      "loss": 0.4841,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.4248424470424652,
-      "rewards/margins": 0.9769999384880066,
-      "rewards/rejected": -1.4018423557281494,
+      "logits/chosen": -2.6360385417938232,
+      "logits/rejected": -2.6261894702911377,
+      "logps/chosen": -270.9910888671875,
+      "logps/rejected": -252.8332977294922,
+      "loss": 0.276,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 34.645816802978516,
+      "rewards/margins": 3.508648633956909,
+      "rewards/rejected": 31.137165069580078,
       "step": 390
     },
     {
-      "epoch": 0.8368200836820083,
-      "grad_norm": 45.25046956895572,
+      "epoch": 0.84,
+      "grad_norm": 11.347134923103408,
       "learning_rate": 3.9507259776993954e-08,
-      "logits/chosen": -2.4646947383880615,
-      "logits/rejected": -2.386026382446289,
-      "logps/chosen": -279.97039794921875,
-      "logps/rejected": -297.86431884765625,
-      "loss": 0.4966,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.471760094165802,
-      "rewards/margins": 1.0708322525024414,
-      "rewards/rejected": -1.5425924062728882,
+      "logits/chosen": -2.632523775100708,
+      "logits/rejected": -2.594832181930542,
+      "logps/chosen": -236.8807830810547,
+      "logps/rejected": -237.6399688720703,
+      "loss": 0.2819,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 33.6544189453125,
+      "rewards/margins": 4.281933784484863,
+      "rewards/rejected": 29.372488021850586,
       "step": 400
     },
     {
-      "epoch": 0.8368200836820083,
-      "eval_logits/chosen": -2.5230555534362793,
-      "eval_logits/rejected": -2.4807281494140625,
-      "eval_logps/chosen": -268.08740234375,
-      "eval_logps/rejected": -289.93310546875,
-      "eval_loss": 0.5098720192909241,
-      "eval_rewards/accuracies": 0.77734375,
-      "eval_rewards/chosen": -0.2746971547603607,
-      "eval_rewards/margins": 1.0885993242263794,
-      "eval_rewards/rejected": -1.3632965087890625,
-      "eval_runtime": 96.6054,
-      "eval_samples_per_second": 20.703,
+      "epoch": 0.84,
+      "eval_logits/chosen": -2.6868975162506104,
+      "eval_logits/rejected": -2.66192626953125,
+      "eval_logps/chosen": -230.7387237548828,
+      "eval_logps/rejected": -235.19105529785156,
+      "eval_loss": 0.2787904143333435,
+      "eval_rewards/accuracies": 0.578125,
+      "eval_rewards/chosen": 31.854747772216797,
+      "eval_rewards/margins": 4.3786234855651855,
+      "eval_rewards/rejected": 27.476125717163086,
+      "eval_runtime": 96.6885,
+      "eval_samples_per_second": 20.685,
       "eval_steps_per_second": 0.331,
       "step": 400
     },
     {
-      "epoch": 0.8577405857740585,
-      "grad_norm": 43.43821318205351,
+      "epoch": 0.86,
+      "grad_norm": 12.175943173191595,
       "learning_rate": 3.022313472693447e-08,
-      "logits/chosen": -2.499673843383789,
-      "logits/rejected": -2.4317073822021484,
-      "logps/chosen": -304.8521728515625,
-      "logps/rejected": -296.1282958984375,
-      "loss": 0.4985,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.29673534631729126,
-      "rewards/margins": 1.0688213109970093,
-      "rewards/rejected": -1.3655567169189453,
+      "logits/chosen": -2.6695199012756348,
+      "logits/rejected": -2.626798152923584,
+      "logps/chosen": -263.4989318847656,
+      "logps/rejected": -240.9721221923828,
+      "loss": 0.2806,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 35.418556213378906,
+      "rewards/margins": 7.573515892028809,
+      "rewards/rejected": 27.845043182373047,
       "step": 410
     },
     {
-      "epoch": 0.8786610878661087,
-      "grad_norm": 47.225420829411256,
+      "epoch": 0.88,
+      "grad_norm": 11.7624491150407,
       "learning_rate": 2.2111614344599684e-08,
-      "logits/chosen": -2.4853806495666504,
-      "logits/rejected": -2.4450175762176514,
-      "logps/chosen": -303.3648986816406,
-      "logps/rejected": -299.51214599609375,
-      "loss": 0.4868,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.3145541548728943,
-      "rewards/margins": 0.9379409551620483,
-      "rewards/rejected": -1.2524950504302979,
+      "logits/chosen": -2.6308817863464355,
+      "logits/rejected": -2.620222568511963,
+      "logps/chosen": -264.280517578125,
+      "logps/rejected": -247.2097625732422,
+      "loss": 0.2882,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 32.79326248168945,
+      "rewards/margins": 5.5407843589782715,
+      "rewards/rejected": 27.252477645874023,
       "step": 420
     },
     {
-      "epoch": 0.899581589958159,
-      "grad_norm": 42.79519877746177,
+      "epoch": 0.9,
+      "grad_norm": 11.16296113559481,
       "learning_rate": 1.521597710086439e-08,
-      "logits/chosen": -2.4045519828796387,
-      "logits/rejected": -2.355051040649414,
-      "logps/chosen": -287.157958984375,
-      "logps/rejected": -285.90185546875,
-      "loss": 0.4696,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.4059911370277405,
-      "rewards/margins": 1.0001002550125122,
-      "rewards/rejected": -1.4060914516448975,
+      "logits/chosen": -2.577580213546753,
+      "logits/rejected": -2.5429909229278564,
+      "logps/chosen": -248.5481719970703,
+      "logps/rejected": -228.4681396484375,
+      "loss": 0.2851,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 30.489971160888672,
+      "rewards/margins": 1.1781085729599,
+      "rewards/rejected": 29.311859130859375,
       "step": 430
     },
     {
-      "epoch": 0.9205020920502092,
-      "grad_norm": 43.203174837210824,
+      "epoch": 0.92,
+      "grad_norm": 10.453636294498436,
       "learning_rate": 9.57301420397924e-09,
-      "logits/chosen": -2.481478214263916,
-      "logits/rejected": -2.426729202270508,
-      "logps/chosen": -287.754638671875,
-      "logps/rejected": -290.429443359375,
-      "loss": 0.4978,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.17087498307228088,
-      "rewards/margins": 0.8950095176696777,
-      "rewards/rejected": -1.0658844709396362,
+      "logits/chosen": -2.654780864715576,
+      "logits/rejected": -2.619481086730957,
+      "logps/chosen": -251.1508026123047,
+      "logps/rejected": -240.0060272216797,
+      "loss": 0.2805,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 33.18633270263672,
+      "rewards/margins": 4.080627918243408,
+      "rewards/rejected": 29.1057071685791,
       "step": 440
     },
     {
-      "epoch": 0.9414225941422594,
-      "grad_norm": 42.158344968140234,
+      "epoch": 0.94,
+      "grad_norm": 10.779162534358996,
       "learning_rate": 5.212833302556258e-09,
-      "logits/chosen": -2.4326705932617188,
-      "logits/rejected": -2.405876636505127,
-      "logps/chosen": -300.6952209472656,
-      "logps/rejected": -333.1911315917969,
-      "loss": 0.485,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.3505518436431885,
-      "rewards/margins": 0.9552817344665527,
-      "rewards/rejected": -1.3058335781097412,
+      "logits/chosen": -2.598240375518799,
+      "logits/rejected": -2.6028037071228027,
+      "logps/chosen": -259.9753112792969,
+      "logps/rejected": -276.95166015625,
+      "loss": 0.2836,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 33.70884323120117,
+      "rewards/margins": 3.5860488414764404,
+      "rewards/rejected": 30.122793197631836,
       "step": 450
     },
     {
-      "epoch": 0.9623430962343096,
-      "grad_norm": 46.29270784779929,
+      "epoch": 0.96,
+      "grad_norm": 12.07874608208951,
       "learning_rate": 2.158697848236607e-09,
-      "logits/chosen": -2.47359561920166,
-      "logits/rejected": -2.4329233169555664,
-      "logps/chosen": -276.7737121582031,
-      "logps/rejected": -266.95977783203125,
-      "loss": 0.4979,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.24045708775520325,
-      "rewards/margins": 0.9731408953666687,
-      "rewards/rejected": -1.2135980129241943,
+      "logits/chosen": -2.6384501457214355,
+      "logits/rejected": -2.618943452835083,
+      "logps/chosen": -240.47885131835938,
+      "logps/rejected": -213.6422882080078,
+      "loss": 0.2815,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 31.485698699951172,
+      "rewards/margins": 2.44018292427063,
+      "rewards/rejected": 29.045513153076172,
       "step": 460
     },
     {
-      "epoch": 0.9832635983263598,
-      "grad_norm": 45.29755120852253,
+      "epoch": 0.98,
+      "grad_norm": 11.390948919388384,
       "learning_rate": 4.269029751107489e-10,
-      "logits/chosen": -2.4899790287017822,
-      "logits/rejected": -2.432119369506836,
-      "logps/chosen": -283.7621154785156,
-      "logps/rejected": -308.806396484375,
-      "loss": 0.474,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.25317153334617615,
-      "rewards/margins": 1.0695946216583252,
-      "rewards/rejected": -1.3227661848068237,
+      "logits/chosen": -2.6327641010284424,
+      "logits/rejected": -2.6079437732696533,
+      "logps/chosen": -245.8006591796875,
+      "logps/rejected": -253.76730346679688,
+      "loss": 0.2778,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 32.898033142089844,
+      "rewards/margins": 4.314266204833984,
+      "rewards/rejected": 28.58376121520996,
       "step": 470
     },
     {
       "epoch": 1.0,
       "step": 478,
       "total_flos": 0.0,
-      "train_loss": 0.5238834185580329,
-      "train_runtime": 7545.2548,
-      "train_samples_per_second": 8.102,
-      "train_steps_per_second": 0.063
+      "train_loss": 0.31381568898715734,
+      "train_runtime": 7749.4814,
+      "train_samples_per_second": 7.889,
+      "train_steps_per_second": 0.062
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e1c2d59e5c027a63352bad711a29580cec536dcd70245ec724a738f805273f0d
+oid sha256:8491be9cba3c1b08383ecee1f5b4b74d53f2e1060af1904c6656dbff18348c02
 size 6264