Model save

README.md

@@ -2,16 +2,10 @@
 license: apache-2.0
 base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
-- alignment-handbook
-- trl
-- dpo
-- generated_from_trainer
 - trl
 - dpo
 - alignment-handbook
 - generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: zephyr-7b-dpo-full-ultrabin-high-margin
   results: []
@@ -22,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full-ultrabin-high-margin
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5565
-- Rewards/chosen: -0.6538
-- Rewards/rejected: -1.5442
-- Rewards/accuracies: 0.7578
-- Rewards/margins: 0.8904
-- Logps/rejected: -417.0811
-- Logps/chosen: -328.0004
-- Logits/rejected: 0.3127
-- Logits/chosen: -0.1625
+- Loss: 0.5598
+- Rewards/chosen: -0.6746
+- Rewards/rejected: -1.5654
+- Rewards/accuracies: 0.75
+- Rewards/margins: 0.8907
+- Logps/rejected: -419.1961
+- Logps/chosen: -330.0835
+- Logits/rejected: 0.2134
+- Logits/chosen: -0.2417
 
 ## Model description
 
@@ -69,7 +63,8 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.3228        | 0.6969 | 100  | 0.5641          | -0.9601        | -1.8247          | 0.7539             | 0.8645          | -445.1261      | -358.6317    | 0.5872          | 0.2254        |
+| 0.4719        | 0.3484 | 50   | 0.5899          | -0.3895        | -0.8981          | 0.7070             | 0.5086          | -352.4708      | -301.5678    | -1.9397         | -1.9963       |
+| 0.3224        | 0.6969 | 100  | 0.5598          | -0.6746        | -1.5654          | 0.75               | 0.8907          | -419.1961      | -330.0835    | 0.2134          | -0.2417       |
 
 
 ### Framework versions

all_results.json

@@ -14,9 +14,9 @@
     "eval_samples_per_second": 19.581,
     "eval_steps_per_second": 0.313,
     "total_flos": 0.0,
-    "train_loss": 0.4296882511018873,
-    "train_runtime": 3529.992,
+    "train_loss": 0.42797933008287337,
+    "train_runtime": 3631.7108,
     "train_samples": 18339,
-    "train_samples_per_second": 5.195,
-    "train_steps_per_second": 0.041
+    "train_samples_per_second": 5.05,
+    "train_steps_per_second": 0.039
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b55076072d3d71f9edcf9b1d358562348023a796ae519c1f0ce9da2573c74ceb
+oid sha256:2850ffba269c9beea7a27e656c559c5f876b38967c7a6b87d210c4d1b66e9185
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:954546cecc8cfffe662fdb29010a136c024abfbf677bfe49a3cc9eada0aae98f
+oid sha256:2f82482a5b53c718a34153ce2321f5c28b587d7da9ba733d4b9e3b74bff5feb2
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:83e63cfe024227217839bcad4c55250906141175fe511bc646f943e2c1c4fd98
+oid sha256:88875900874228283080cbb0b5f0f9444fc6aa58ee90c727879470f81eae1681
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9965156794425087,
     "total_flos": 0.0,
-    "train_loss": 0.4296882511018873,
-    "train_runtime": 3529.992,
+    "train_loss": 0.42797933008287337,
+    "train_runtime": 3631.7108,
     "train_samples": 18339,
-    "train_samples_per_second": 5.195,
-    "train_steps_per_second": 0.041
+    "train_samples_per_second": 5.05,
+    "train_steps_per_second": 0.039
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_metric": null,
   "best_model_checkpoint": null,
   "epoch": 0.9965156794425087,
-  "eval_steps": 100,
+  "eval_steps": 50,
   "global_step": 143,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
@@ -10,238 +10,254 @@
   "log_history": [
     {
       "epoch": 0.06968641114982578,
-      "grad_norm": 8.870989685926396,
+      "grad_norm": 8.879917985004713,
       "learning_rate": 3.333333333333333e-07,
-      "logits/chosen": -2.5107316970825195,
-      "logits/rejected": -2.459897994995117,
-      "logps/chosen": -224.4440460205078,
-      "logps/rejected": -205.32931518554688,
+      "logits/chosen": -2.5109400749206543,
+      "logits/rejected": -2.4602229595184326,
+      "logps/chosen": -224.4826202392578,
+      "logps/rejected": -205.3321075439453,
       "loss": 0.6925,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": 0.0013574643526226282,
-      "rewards/margins": 0.001965512754395604,
-      "rewards/rejected": -0.0006080485763959587,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.0009716759668663144,
+      "rewards/margins": 0.001607558922842145,
+      "rewards/rejected": -0.0006358829559758306,
       "step": 10
     },
     {
       "epoch": 0.13937282229965156,
-      "grad_norm": 8.120676621682783,
+      "grad_norm": 8.119644042984005,
       "learning_rate": 4.981198836496775e-07,
-      "logits/chosen": -2.539044141769409,
-      "logits/rejected": -2.4031107425689697,
-      "logps/chosen": -231.701416015625,
-      "logps/rejected": -203.5116729736328,
+      "logits/chosen": -2.538778305053711,
+      "logits/rejected": -2.402923583984375,
+      "logps/chosen": -231.76394653320312,
+      "logps/rejected": -203.54684448242188,
       "loss": 0.6753,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": 0.00874270312488079,
-      "rewards/margins": 0.029630497097969055,
-      "rewards/rejected": -0.020887792110443115,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.008117455057799816,
+      "rewards/margins": 0.029356980696320534,
+      "rewards/rejected": -0.02123952843248844,
       "step": 20
     },
     {
       "epoch": 0.20905923344947736,
-      "grad_norm": 10.53786145514599,
+      "grad_norm": 10.589687412392658,
       "learning_rate": 4.832481997086846e-07,
-      "logits/chosen": -2.5250794887542725,
-      "logits/rejected": -2.4301116466522217,
-      "logps/chosen": -225.29916381835938,
-      "logps/rejected": -253.11544799804688,
+      "logits/chosen": -2.5253872871398926,
+      "logits/rejected": -2.430739402770996,
+      "logps/chosen": -225.355712890625,
+      "logps/rejected": -253.0909881591797,
       "loss": 0.6114,
       "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 0.030692869797348976,
-      "rewards/margins": 0.19038814306259155,
-      "rewards/rejected": -0.15969529747962952,
+      "rewards/chosen": 0.030127260833978653,
+      "rewards/margins": 0.18957777321338654,
+      "rewards/rejected": -0.15945051610469818,
       "step": 30
     },
     {
       "epoch": 0.2787456445993031,
-      "grad_norm": 19.29362558041151,
+      "grad_norm": 19.230572297481732,
       "learning_rate": 4.543962032878959e-07,
-      "logits/chosen": -2.405980110168457,
-      "logits/rejected": -2.3023486137390137,
-      "logps/chosen": -241.8335418701172,
-      "logps/rejected": -256.7768249511719,
-      "loss": 0.5446,
+      "logits/chosen": -2.4045207500457764,
+      "logits/rejected": -2.3013105392456055,
+      "logps/chosen": -241.72854614257812,
+      "logps/rejected": -256.7030944824219,
+      "loss": 0.5447,
       "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -0.04428701475262642,
-      "rewards/margins": 0.44735702872276306,
-      "rewards/rejected": -0.4916439950466156,
+      "rewards/chosen": -0.04323701187968254,
+      "rewards/margins": 0.447670042514801,
+      "rewards/rejected": -0.4909070134162903,
       "step": 40
     },
     {
       "epoch": 0.34843205574912894,
-      "grad_norm": 21.034007556263575,
+      "grad_norm": 23.777968472263748,
       "learning_rate": 4.1329321073844413e-07,
-      "logits/chosen": -2.3216395378112793,
-      "logits/rejected": -2.171525239944458,
-      "logps/chosen": -253.8367919921875,
-      "logps/rejected": -303.62744140625,
-      "loss": 0.4742,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.29281625151634216,
-      "rewards/margins": 0.6403177380561829,
-      "rewards/rejected": -0.9331340789794922,
+      "logits/chosen": -2.2776694297790527,
+      "logits/rejected": -2.1192574501037598,
+      "logps/chosen": -255.7515411376953,
+      "logps/rejected": -307.02410888671875,
+      "loss": 0.4719,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.31196385622024536,
+      "rewards/margins": 0.6551374197006226,
+      "rewards/rejected": -0.9671012163162231,
+      "step": 50
+    },
+    {
+      "epoch": 0.34843205574912894,
+      "eval_logits/chosen": -1.996337890625,
+      "eval_logits/rejected": -1.9396870136260986,
+      "eval_logps/chosen": -301.5677795410156,
+      "eval_logps/rejected": -352.47076416015625,
+      "eval_loss": 0.5898596048355103,
+      "eval_rewards/accuracies": 0.70703125,
+      "eval_rewards/chosen": -0.3894880414009094,
+      "eval_rewards/margins": 0.5086125135421753,
+      "eval_rewards/rejected": -0.8981005549430847,
+      "eval_runtime": 102.5575,
+      "eval_samples_per_second": 19.501,
+      "eval_steps_per_second": 0.312,
       "step": 50
     },
     {
       "epoch": 0.4181184668989547,
-      "grad_norm": 28.77756194136436,
+      "grad_norm": 33.164641300897365,
       "learning_rate": 3.624028324136517e-07,
-      "logits/chosen": -1.4179085493087769,
-      "logits/rejected": -1.13853919506073,
-      "logps/chosen": -299.9506530761719,
-      "logps/rejected": -362.7635803222656,
-      "loss": 0.4053,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -0.5958151817321777,
-      "rewards/margins": 0.9765597581863403,
-      "rewards/rejected": -1.572374939918518,
+      "logits/chosen": -1.2023751735687256,
+      "logits/rejected": -0.8880468606948853,
+      "logps/chosen": -297.7267150878906,
+      "logps/rejected": -362.5870056152344,
+      "loss": 0.4024,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.5735751986503601,
+      "rewards/margins": 0.9970341920852661,
+      "rewards/rejected": -1.570609211921692,
       "step": 60
     },
     {
       "epoch": 0.4878048780487805,
-      "grad_norm": 24.40968960837773,
+      "grad_norm": 26.00066903060501,
       "learning_rate": 3.047753100392174e-07,
-      "logits/chosen": -0.7467012405395508,
-      "logits/rejected": -0.3752829432487488,
-      "logps/chosen": -266.78118896484375,
-      "logps/rejected": -392.96392822265625,
-      "loss": 0.364,
+      "logits/chosen": -0.6797115802764893,
+      "logits/rejected": -0.30923840403556824,
+      "logps/chosen": -267.3857116699219,
+      "logps/rejected": -395.60101318359375,
+      "loss": 0.362,
       "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -0.5415125489234924,
-      "rewards/margins": 1.2262623310089111,
-      "rewards/rejected": -1.7677749395370483,
+      "rewards/chosen": -0.5475583076477051,
+      "rewards/margins": 1.2465879917144775,
+      "rewards/rejected": -1.7941462993621826,
       "step": 70
     },
     {
       "epoch": 0.5574912891986062,
-      "grad_norm": 35.97929804339313,
+      "grad_norm": 36.007825212896435,
       "learning_rate": 2.4386469286927194e-07,
-      "logits/chosen": -0.13451911509037018,
-      "logits/rejected": 0.4377119541168213,
-      "logps/chosen": -296.5347595214844,
-      "logps/rejected": -490.88177490234375,
-      "loss": 0.3454,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -0.7803608179092407,
-      "rewards/margins": 1.4910542964935303,
-      "rewards/rejected": -2.2714149951934814,
+      "logits/chosen": -0.32210594415664673,
+      "logits/rejected": 0.29763275384902954,
+      "logps/chosen": -286.4994812011719,
+      "logps/rejected": -484.10357666015625,
+      "loss": 0.3448,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -0.6800082921981812,
+      "rewards/margins": 1.5236244201660156,
+      "rewards/rejected": -2.2036328315734863,
       "step": 80
     },
     {
       "epoch": 0.627177700348432,
-      "grad_norm": 33.964744689691656,
+      "grad_norm": 28.429320380976726,
       "learning_rate": 1.8332181063127542e-07,
-      "logits/chosen": -0.3771997094154358,
-      "logits/rejected": 0.5029958486557007,
-      "logps/chosen": -298.05755615234375,
-      "logps/rejected": -438.8995056152344,
+      "logits/chosen": -0.29179516434669495,
+      "logits/rejected": 0.5656725168228149,
+      "logps/chosen": -315.7966613769531,
+      "logps/rejected": -458.8309631347656,
       "loss": 0.3319,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -0.5154351592063904,
-      "rewards/margins": 1.5300105810165405,
-      "rewards/rejected": -2.0454459190368652,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -0.6928261518478394,
+      "rewards/margins": 1.5519336462020874,
+      "rewards/rejected": -2.244760036468506,
       "step": 90
     },
     {
       "epoch": 0.6968641114982579,
-      "grad_norm": 28.73220407106371,
+      "grad_norm": 29.97250123159769,
       "learning_rate": 1.26775451942554e-07,
-      "logits/chosen": 0.3942970931529999,
-      "logits/rejected": 1.039069414138794,
-      "logps/chosen": -326.0721435546875,
-      "logps/rejected": -470.18853759765625,
-      "loss": 0.3228,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -0.9078343510627747,
-      "rewards/margins": 1.4802639484405518,
-      "rewards/rejected": -2.3880982398986816,
+      "logits/chosen": -0.13269878923892975,
+      "logits/rejected": 0.6727190017700195,
+      "logps/chosen": -300.4376220703125,
+      "logps/rejected": -451.68609619140625,
+      "loss": 0.3224,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -0.6514891982078552,
+      "rewards/margins": 1.551584243774414,
+      "rewards/rejected": -2.203073501586914,
       "step": 100
     },
     {
       "epoch": 0.6968641114982579,
-      "eval_logits/chosen": 0.2254416048526764,
-      "eval_logits/rejected": 0.5872303247451782,
-      "eval_logps/chosen": -358.6317443847656,
-      "eval_logps/rejected": -445.1260681152344,
-      "eval_loss": 0.5641274452209473,
-      "eval_rewards/accuracies": 0.75390625,
-      "eval_rewards/chosen": -0.960127592086792,
-      "eval_rewards/margins": 0.8645257353782654,
-      "eval_rewards/rejected": -1.824653148651123,
-      "eval_runtime": 102.0937,
-      "eval_samples_per_second": 19.59,
-      "eval_steps_per_second": 0.313,
+      "eval_logits/chosen": -0.2416563630104065,
+      "eval_logits/rejected": 0.21337364614009857,
+      "eval_logps/chosen": -330.0835266113281,
+      "eval_logps/rejected": -419.1961364746094,
+      "eval_loss": 0.5598118305206299,
+      "eval_rewards/accuracies": 0.75,
+      "eval_rewards/chosen": -0.6746450662612915,
+      "eval_rewards/margins": 0.8907086849212646,
+      "eval_rewards/rejected": -1.5653537511825562,
+      "eval_runtime": 100.9616,
+      "eval_samples_per_second": 19.81,
+      "eval_steps_per_second": 0.317,
       "step": 100
     },
     {
       "epoch": 0.7665505226480837,
-      "grad_norm": 35.093388243905714,
+      "grad_norm": 26.411152004320307,
       "learning_rate": 7.761486381573326e-08,
-      "logits/chosen": 0.3774252235889435,
-      "logits/rejected": 1.4682767391204834,
-      "logps/chosen": -346.6317443847656,
-      "logps/rejected": -460.89837646484375,
-      "loss": 0.3193,
+      "logits/chosen": 0.1635294407606125,
+      "logits/rejected": 1.3954848051071167,
+      "logps/chosen": -325.93487548828125,
+      "logps/rejected": -446.07916259765625,
+      "loss": 0.3142,
       "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -0.9842090606689453,
-      "rewards/margins": 1.5870163440704346,
-      "rewards/rejected": -2.57122540473938,
+      "rewards/chosen": -0.7772396206855774,
+      "rewards/margins": 1.6457939147949219,
+      "rewards/rejected": -2.4230334758758545,
       "step": 110
     },
     {
       "epoch": 0.8362369337979094,
-      "grad_norm": 33.95663558990993,
+      "grad_norm": 32.75502406597345,
       "learning_rate": 3.878660868757322e-08,
-      "logits/chosen": 0.2716614603996277,
-      "logits/rejected": 1.619855284690857,
-      "logps/chosen": -330.23992919921875,
-      "logps/rejected": -440.49334716796875,
-      "loss": 0.309,
+      "logits/chosen": 0.5776845216751099,
+      "logits/rejected": 1.9672679901123047,
+      "logps/chosen": -338.0449523925781,
+      "logps/rejected": -449.5287170410156,
+      "loss": 0.3042,
       "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": -0.8578445315361023,
-      "rewards/margins": 1.694758415222168,
-      "rewards/rejected": -2.552602529525757,
+      "rewards/chosen": -0.9358948469161987,
+      "rewards/margins": 1.7070610523223877,
+      "rewards/rejected": -2.642955780029297,
       "step": 120
     },
     {
       "epoch": 0.9059233449477352,
-      "grad_norm": 29.086065504687443,
+      "grad_norm": 34.08869226634673,
       "learning_rate": 1.261795485174083e-08,
-      "logits/chosen": 0.13383683562278748,
-      "logits/rejected": 1.3482104539871216,
-      "logps/chosen": -283.5831604003906,
-      "logps/rejected": -445.7456970214844,
-      "loss": 0.3327,
+      "logits/chosen": 0.4366391599178314,
+      "logits/rejected": 1.6738389730453491,
+      "logps/chosen": -295.5234375,
+      "logps/rejected": -462.7455139160156,
+      "loss": 0.3275,
       "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": -0.6794359683990479,
-      "rewards/margins": 1.8727607727050781,
-      "rewards/rejected": -2.552196741104126,
+      "rewards/chosen": -0.7988389730453491,
+      "rewards/margins": 1.923356294631958,
+      "rewards/rejected": -2.7221951484680176,
       "step": 130
     },
     {
       "epoch": 0.975609756097561,
-      "grad_norm": 32.68843173564522,
+      "grad_norm": 33.23571195007149,
       "learning_rate": 6.773858303274482e-10,
-      "logits/chosen": 0.3474978804588318,
-      "logits/rejected": 1.376219630241394,
-      "logps/chosen": -289.83441162109375,
-      "logps/rejected": -441.0291442871094,
-      "loss": 0.3297,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -0.7964991331100464,
-      "rewards/margins": 1.6330715417861938,
-      "rewards/rejected": -2.4295706748962402,
+      "logits/chosen": 0.5510319471359253,
+      "logits/rejected": 1.6277908086776733,
+      "logps/chosen": -300.76043701171875,
+      "logps/rejected": -455.19793701171875,
+      "loss": 0.3272,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.9057596325874329,
+      "rewards/margins": 1.6654990911483765,
+      "rewards/rejected": -2.571258544921875,
       "step": 140
     },
     {
       "epoch": 0.9965156794425087,
       "step": 143,
       "total_flos": 0.0,
-      "train_loss": 0.4296882511018873,
-      "train_runtime": 3529.992,
-      "train_samples_per_second": 5.195,
-      "train_steps_per_second": 0.041
+      "train_loss": 0.42797933008287337,
+      "train_runtime": 3631.7108,
+      "train_samples_per_second": 5.05,
+      "train_steps_per_second": 0.039
     }
   ],
   "logging_steps": 10,