End of training

README.md

@@ -2,10 +2,13 @@
 license: apache-2.0
 library_name: peft
 tags:
+- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: mistralai/Mistral-7B-v0.1
+datasets:
+- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: zephyr-7b-dpo-qlora
   results: []
@@ -16,17 +19,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-qlora
 
-This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on an unknown dataset.
+This model is a fine-tuned version of [ale-bay/zephyr-7b-sft-qlora](https://huggingface.co/ale-bay/zephyr-7b-sft-qlora) on the HuggingFaceH4/ultrafeedback_binarized dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4974
-- Rewards/chosen: -2.4531
-- Rewards/rejected: -3.4740
-- Rewards/accuracies: 0.75
-- Rewards/margins: 1.0209
-- Logps/rejected: -595.1221
-- Logps/chosen: -517.0148
+- Loss: 0.4975
+- Rewards/chosen: -2.4549
+- Rewards/rejected: -3.4757
+- Rewards/accuracies: 0.7490
+- Rewards/margins: 1.0207
+- Logps/rejected: -595.2866
+- Logps/chosen: -517.1966
 - Logits/rejected: -1.3432
-- Logits/chosen: -1.4359
+- Logits/chosen: -1.4358
 
 ## Model description
 

all_results.json

@@ -1,5 +1,18 @@
 {
     "epoch": 1.0,
+    "eval_logits/chosen": -1.4358036518096924,
+    "eval_logits/rejected": -1.3431825637817383,
+    "eval_logps/chosen": -517.1965942382812,
+    "eval_logps/rejected": -595.2865600585938,
+    "eval_loss": 0.4974534213542938,
+    "eval_rewards/accuracies": 0.7490000128746033,
+    "eval_rewards/chosen": -2.4549365043640137,
+    "eval_rewards/margins": 1.0207281112670898,
+    "eval_rewards/rejected": -3.4756648540496826,
+    "eval_runtime": 206.3812,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 9.691,
+    "eval_steps_per_second": 0.606,
     "train_loss": 0.5312882837824796,
     "train_runtime": 19005.9968,
     "train_samples": 61134,

config.json

@@ -19,6 +19,7 @@
     "_load_in_4bit": true,
     "_load_in_8bit": false,
     "bnb_4bit_compute_dtype": "bfloat16",
+    "bnb_4bit_quant_storage": "uint8",
     "bnb_4bit_quant_type": "nf4",
     "bnb_4bit_use_double_quant": false,
     "llm_int8_enable_fp32_cpu_offload": false,
@@ -34,7 +35,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.38.2",
+  "transformers_version": "4.39.3",
   "use_cache": true,
   "vocab_size": 32000
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": 2.2971765995025635,
-    "eval_logits/rejected": 2.509765625,
-    "eval_logps/chosen": -516.2817993164062,
-    "eval_logps/rejected": -600.310302734375,
-    "eval_loss": 0.4919675290584564,
-    "eval_rewards/accuracies": 0.7559999823570251,
-    "eval_rewards/chosen": -2.5097556114196777,
-    "eval_rewards/margins": 1.0807288885116577,
-    "eval_rewards/rejected": -3.590484380722046,
-    "eval_runtime": 449.3054,
+    "eval_logits/chosen": -1.4358036518096924,
+    "eval_logits/rejected": -1.3431825637817383,
+    "eval_logps/chosen": -517.1965942382812,
+    "eval_logps/rejected": -595.2865600585938,
+    "eval_loss": 0.4974534213542938,
+    "eval_rewards/accuracies": 0.7490000128746033,
+    "eval_rewards/chosen": -2.4549365043640137,
+    "eval_rewards/margins": 1.0207281112670898,
+    "eval_rewards/rejected": -3.4756648540496826,
+    "eval_runtime": 206.3812,
     "eval_samples": 2000,
-    "eval_samples_per_second": 4.451,
-    "eval_steps_per_second": 0.278
+    "eval_samples_per_second": 9.691,
+    "eval_steps_per_second": 0.606
 }