Model save

README.md

@@ -2,13 +2,9 @@
 license: mit
 library_name: peft
 tags:
-- alignment-handbook
-- generated_from_trainer
 - trl
 - dpo
 - generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 base_model: microsoft/phi-2
 model-index:
 - name: phi-2-gpo-ultrafeedback-lora
@@ -20,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # phi-2-gpo-ultrafeedback-lora
 
-This model is a fine-tuned version of [lole25/phi-2-sft-ultrachat-lora](https://huggingface.co/lole25/phi-2-sft-ultrachat-lora) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [microsoft/phi-2](https://huggingface.co/microsoft/phi-2) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.0021
-- Rewards/chosen: -0.0083
-- Rewards/rejected: -0.0184
-- Rewards/accuracies: 0.6920
-- Rewards/margins: 0.0101
-- Logps/rejected: -233.2711
-- Logps/chosen: -261.0694
-- Logits/rejected: 0.8833
-- Logits/chosen: 0.7809
+- Loss: 0.0004
+- Rewards/chosen: -0.0084
+- Rewards/rejected: -0.0177
+- Rewards/accuracies: 0.6700
+- Rewards/margins: 0.0093
+- Logps/rejected: -233.2047
+- Logps/chosen: -261.0818
+- Logits/rejected: 0.8824
+- Logits/chosen: 0.7796
 
 ## Model description
 
@@ -65,17 +61,17 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
-|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.0026        | 0.21  | 100  | 0.0025          | 0.0001         | -0.0005          | 0.5080             | 0.0006          | -231.4896      | -260.2373    | 0.9175          | 0.8151        |
-| 0.0023        | 0.42  | 200  | 0.0023          | -0.0015        | -0.0068          | 0.6560             | 0.0053          | -232.1152      | -260.3932    | 0.9120          | 0.8092        |
-| 0.0022        | 0.63  | 300  | 0.0022          | -0.0067        | -0.0141          | 0.6700             | 0.0073          | -232.8447      | -260.9179    | 0.9022          | 0.7992        |
-| 0.0021        | 0.84  | 400  | 0.0022          | -0.0092        | -0.0178          | 0.6640             | 0.0086          | -233.2157      | -261.1620    | 0.8914          | 0.7884        |
-| 0.0022        | 1.05  | 500  | 0.0021          | -0.0094        | -0.0193          | 0.7100             | 0.0098          | -233.3614      | -261.1852    | 0.8853          | 0.7821        |
-| 0.002         | 1.26  | 600  | 0.0021          | -0.0088        | -0.0185          | 0.6940             | 0.0097          | -233.2843      | -261.1207    | 0.8840          | 0.7815        |
-| 0.0021        | 1.47  | 700  | 0.0021          | -0.0083        | -0.0182          | 0.7000             | 0.0099          | -233.2560      | -261.0788    | 0.8816          | 0.7790        |
-| 0.0021        | 1.67  | 800  | 0.0021          | -0.0082        | -0.0184          | 0.6940             | 0.0102          | -233.2740      | -261.0643    | 0.8811          | 0.7781        |
-| 0.0021        | 1.88  | 900  | 0.0021          | -0.0085        | -0.0178          | 0.6900             | 0.0093          | -233.2118      | -261.0922    | 0.8833          | 0.7806        |
+| Training Loss | Epoch | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
+|:-------------:|:-----:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
+| 0.0026        | 0.21  | 100  | 0.8151        | 0.9175          | -260.2373    | -231.4896      | 0.0025          | 0.5080             | 0.0001         | 0.0006          | -0.0005          |
+| 0.0023        | 0.42  | 200  | 0.8092        | 0.9120          | -260.3932    | -232.1152      | 0.0023          | 0.6560             | -0.0015        | 0.0053          | -0.0068          |
+| 0.0022        | 0.63  | 300  | 0.7992        | 0.9022          | -260.9179    | -232.8447      | 0.0022          | 0.6700             | -0.0067        | 0.0073          | -0.0141          |
+| 0.0021        | 0.84  | 400  | 0.7884        | 0.8914          | -261.1620    | -233.2157      | 0.0022          | 0.6640             | -0.0092        | 0.0086          | -0.0178          |
+| 0.0022        | 1.05  | 500  | 0.7821        | 0.8853          | -261.1852    | -233.3614      | 0.0021          | 0.7100             | -0.0094        | 0.0098          | -0.0193          |
+| 0.002         | 1.26  | 600  | 0.7815        | 0.8840          | -261.1207    | -233.2843      | 0.0021          | 0.6940             | -0.0088        | 0.0097          | -0.0185          |
+| 0.0021        | 1.47  | 700  | 0.7790        | 0.8816          | -261.0788    | -233.2560      | 0.0021          | 0.7000             | -0.0083        | 0.0099          | -0.0182          |
+| 0.0021        | 1.67  | 800  | 0.7781        | 0.8811          | -261.0643    | -233.2740      | 0.0021          | 0.6940             | -0.0082        | 0.0102          | -0.0184          |
+| 0.0021        | 1.88  | 900  | 0.7806        | 0.8833          | -261.0922    | -233.2118      | 0.0021          | 0.6900             | -0.0085        | 0.0093          | -0.0178          |
 
 
 ### Framework versions

adapter_config.json

@@ -20,8 +20,8 @@
   "revision": null,
   "target_modules": [
     "q_proj",
-    "k_proj",
     "dense",
+    "k_proj",
     "v_proj"
   ],
   "task_type": "CAUSAL_LM"

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d0dea53272310862afda3712da656b50c1d9a7ad7a46f0642635168a85f6d5a0
+oid sha256:f1a5ec7e556d2b963020bd2793b0483b939a6de653aa4ba0aec480a313b5a507
 size 41977616

all_results.json

@@ -1,21 +1,21 @@
 {
     "epoch": 2.0,
-    "eval_logits/chosen": 0.7808946371078491,
-    "eval_logits/rejected": 0.8833128213882446,
-    "eval_logps/chosen": -261.0694274902344,
-    "eval_logps/rejected": -233.27114868164062,
-    "eval_loss": 0.0021080097649246454,
-    "eval_rewards/accuracies": 0.6919999718666077,
-    "eval_rewards/chosen": -0.008252721279859543,
-    "eval_rewards/margins": 0.01009758934378624,
-    "eval_rewards/rejected": -0.018350308761000633,
-    "eval_runtime": 325.1898,
+    "eval_logits/chosen": 0.7796330451965332,
+    "eval_logits/rejected": 0.8823836445808411,
+    "eval_logps/chosen": -261.0818176269531,
+    "eval_logps/rejected": -233.2046661376953,
+    "eval_loss": 0.0004278263368178159,
+    "eval_rewards/accuracies": 0.6700000166893005,
+    "eval_rewards/chosen": -0.008376287296414375,
+    "eval_rewards/margins": 0.009309147484600544,
+    "eval_rewards/rejected": -0.017685433849692345,
+    "eval_runtime": 324.63,
     "eval_samples": 2000,
-    "eval_samples_per_second": 6.15,
-    "eval_steps_per_second": 0.384,
-    "train_loss": 0.0021909422920118682,
-    "train_runtime": 18127.9992,
+    "eval_samples_per_second": 6.161,
+    "eval_steps_per_second": 0.385,
+    "train_loss": 2.2877212975025803e-05,
+    "train_runtime": 862.5384,
     "train_samples": 30567,
-    "train_samples_per_second": 3.372,
-    "train_steps_per_second": 0.053
+    "train_samples_per_second": 70.877,
+    "train_steps_per_second": 1.106
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 2.0,
-    "eval_logits/chosen": 0.7808946371078491,
-    "eval_logits/rejected": 0.8833128213882446,
-    "eval_logps/chosen": -261.0694274902344,
-    "eval_logps/rejected": -233.27114868164062,
-    "eval_loss": 0.0021080097649246454,
-    "eval_rewards/accuracies": 0.6919999718666077,
-    "eval_rewards/chosen": -0.008252721279859543,
-    "eval_rewards/margins": 0.01009758934378624,
-    "eval_rewards/rejected": -0.018350308761000633,
-    "eval_runtime": 325.1898,
+    "eval_logits/chosen": 0.7796330451965332,
+    "eval_logits/rejected": 0.8823836445808411,
+    "eval_logps/chosen": -261.0818176269531,
+    "eval_logps/rejected": -233.2046661376953,
+    "eval_loss": 0.0004278263368178159,
+    "eval_rewards/accuracies": 0.6700000166893005,
+    "eval_rewards/chosen": -0.008376287296414375,
+    "eval_rewards/margins": 0.009309147484600544,
+    "eval_rewards/rejected": -0.017685433849692345,
+    "eval_runtime": 324.63,
     "eval_samples": 2000,
-    "eval_samples_per_second": 6.15,
-    "eval_steps_per_second": 0.384
+    "eval_samples_per_second": 6.161,
+    "eval_steps_per_second": 0.385
 }

runs/Mar05_10-41-15_gpu4-119-4/events.out.tfevents.1709595839.gpu4-119-4.3155837.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed89e1e527290c4877b057258807d925af0fd9aa366908feb6502245477f3abc
+size 8366

runs/Mar05_10-41-15_gpu4-119-4/events.out.tfevents.1709597026.gpu4-119-4.3155837.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5eeeb7d6c762751ee53da0154e2ad4f754acf3f56d63d6447a6c964c8e649244
+size 828

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 2.0,
-    "train_loss": 0.0021909422920118682,
-    "train_runtime": 18127.9992,
+    "train_loss": 2.2877212975025803e-05,
+    "train_runtime": 862.5384,
     "train_samples": 30567,
-    "train_samples_per_second": 3.372,
-    "train_steps_per_second": 0.053
+    "train_samples_per_second": 70.877,
+    "train_steps_per_second": 1.106
 }

trainer_state.json

@@ -1,7 +1,7 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.9968602825745683,
+  "epoch": 1.998430141287284,
   "eval_steps": 100,
   "global_step": 954,
   "is_hyper_param_search": false,
@@ -1427,83 +1427,83 @@
       "step": 900
     },
     {
-      "epoch": 1.9,
+      "epoch": 1.91,
       "learning_rate": 3.237434340521789e-08,
-      "logits/chosen": 0.7978643774986267,
-      "logits/rejected": 0.8687127828598022,
-      "logps/chosen": -263.38275146484375,
-      "logps/rejected": -247.8026123046875,
-      "loss": 0.0021,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.007231117691844702,
-      "rewards/margins": 0.009652243927121162,
-      "rewards/rejected": -0.016883360221982002,
+      "logits/chosen": 0.8207446932792664,
+      "logits/rejected": 0.8551779985427856,
+      "logps/chosen": -253.6584014892578,
+      "logps/rejected": -244.2725372314453,
+      "loss": 0.0004,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.006646591238677502,
+      "rewards/margins": 0.008468803949654102,
+      "rewards/rejected": -0.015115395188331604,
       "step": 910
     },
     {
       "epoch": 1.93,
       "learning_rate": 1.93478202307823e-08,
-      "logits/chosen": 0.7963850498199463,
-      "logits/rejected": 0.8160678148269653,
-      "logps/chosen": -242.1365966796875,
-      "logps/rejected": -246.0305938720703,
-      "loss": 0.0021,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.0077395932748913765,
-      "rewards/margins": 0.007968437857925892,
-      "rewards/rejected": -0.01570803113281727,
+      "logits/chosen": 0.7895456552505493,
+      "logits/rejected": 0.820245623588562,
+      "logps/chosen": -247.3115692138672,
+      "logps/rejected": -253.6699676513672,
+      "loss": 0.0004,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.008030624128878117,
+      "rewards/margins": 0.007940003648400307,
+      "rewards/rejected": -0.015970628708600998,
       "step": 920
     },
     {
       "epoch": 1.95,
       "learning_rate": 9.646686570697062e-09,
-      "logits/chosen": 0.862303614616394,
-      "logits/rejected": 0.8678015470504761,
-      "logps/chosen": -257.33099365234375,
-      "logps/rejected": -249.9061737060547,
-      "loss": 0.0021,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.007601047400385141,
-      "rewards/margins": 0.009036187082529068,
-      "rewards/rejected": -0.01663723587989807,
+      "logits/chosen": 0.868087112903595,
+      "logits/rejected": 0.8821622729301453,
+      "logps/chosen": -258.50799560546875,
+      "logps/rejected": -248.83169555664062,
+      "loss": 0.0004,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.007558141835033894,
+      "rewards/margins": 0.008687029592692852,
+      "rewards/rejected": -0.016245171427726746,
       "step": 930
     },
     {
       "epoch": 1.97,
       "learning_rate": 3.283947088983663e-09,
-      "logits/chosen": 0.8371657133102417,
-      "logits/rejected": 0.8322643041610718,
-      "logps/chosen": -238.14657592773438,
-      "logps/rejected": -243.3525390625,
-      "loss": 0.0021,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.009062298573553562,
-      "rewards/margins": 0.008633644320070744,
-      "rewards/rejected": -0.017695942893624306,
+      "logits/chosen": 0.8342536687850952,
+      "logits/rejected": 0.8288405537605286,
+      "logps/chosen": -237.41232299804688,
+      "logps/rejected": -237.2918243408203,
+      "loss": 0.0004,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.00896701030433178,
+      "rewards/margins": 0.009420427493751049,
+      "rewards/rejected": -0.018387438729405403,
       "step": 940
     },
     {
       "epoch": 1.99,
       "learning_rate": 2.681312309735229e-10,
-      "logits/chosen": 0.8020931482315063,
-      "logits/rejected": 0.9026565551757812,
-      "logps/chosen": -231.6744842529297,
-      "logps/rejected": -229.53726196289062,
-      "loss": 0.0021,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.007398143410682678,
-      "rewards/margins": 0.009477959014475346,
-      "rewards/rejected": -0.0168761033564806,
+      "logits/chosen": 0.7803130149841309,
+      "logits/rejected": 0.8894654512405396,
+      "logps/chosen": -234.93325805664062,
+      "logps/rejected": -232.20156860351562,
+      "loss": 0.0004,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.008270134218037128,
+      "rewards/margins": 0.00816525612026453,
+      "rewards/rejected": -0.01643539033830166,
       "step": 950
     },
     {
       "epoch": 2.0,
       "step": 954,
       "total_flos": 0.0,
-      "train_loss": 0.0021909422920118682,
-      "train_runtime": 18127.9992,
-      "train_samples_per_second": 3.372,
-      "train_steps_per_second": 0.053
+      "train_loss": 2.2877212975025803e-05,
+      "train_runtime": 862.5384,
+      "train_samples_per_second": 70.877,
+      "train_steps_per_second": 1.106
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6973090f4edc33a748ed65364e59f947a3ecfeb0039599bebec83bce04d70b8
+oid sha256:a0dd2b02ab585a34a9bf39314ee5dc3e74f98b4692efe90bf2da48b933a6a62a
 size 5816