Model save

README.md

@@ -2,16 +2,10 @@
 license: mit
 library_name: peft
 tags:
-- alignment-handbook
-- trl
-- dpo
-- generated_from_trainer
 - trl
 - dpo
 - generated_from_trainer
 base_model: HuggingFaceH4/mistral-7b-sft-beta
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: mistral-sft-7b-dpo-qlora
   results: []
@@ -22,17 +16,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # mistral-sft-7b-dpo-qlora
 
-This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the HuggingFaceH4/ultrafeedback_binarized dataset.
-It achieves the following results on the evaluation set:
-- Loss: 0.6939
-- Rewards/chosen: 0.0009
-- Rewards/rejected: 0.0009
-- Rewards/accuracies: 0.5625
-- Rewards/margins: 0.0000
-- Logps/rejected: -122.9066
-- Logps/chosen: -86.4060
-- Logits/rejected: -3.0456
-- Logits/chosen: -2.9840
+This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
 
 ## Model description
 

adapter_config.json

@@ -19,13 +19,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "v_proj",
-    "up_proj",
-    "down_proj",
+    "o_proj",
     "q_proj",
+    "up_proj",
+    "v_proj",
+    "k_proj",
     "gate_proj",
-    "o_proj",
-    "k_proj"
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4f76cf141c42b38a642f31eadded514f94a516ab7e0dea749472df2b2b93a3e8
+oid sha256:fc0793e924d0764c4d8614e0b1426ed310afc447165daf63c4f1609395b2d9b3
 size 671150064

all_results.json

@@ -1,21 +1,8 @@
 {
-    "epoch": 3.0,
-    "eval_logits/chosen": -2.9840264320373535,
-    "eval_logits/rejected": -3.045552968978882,
-    "eval_logps/chosen": -86.40597534179688,
-    "eval_logps/rejected": -122.90658569335938,
-    "eval_loss": 0.6938564777374268,
-    "eval_rewards/accuracies": 0.5625,
-    "eval_rewards/chosen": 0.0008632946410216391,
-    "eval_rewards/margins": 5.57894236408174e-06,
-    "eval_rewards/rejected": 0.000857715611346066,
-    "eval_runtime": 11.9269,
-    "eval_samples": 20,
-    "eval_samples_per_second": 1.677,
-    "eval_steps_per_second": 0.168,
-    "train_loss": 0.3101682186126709,
-    "train_runtime": 67.2839,
-    "train_samples": 20,
-    "train_samples_per_second": 1.486,
-    "train_steps_per_second": 0.074
+    "epoch": 5.0,
+    "train_loss": 0.34364445209503175,
+    "train_runtime": 77.9014,
+    "train_samples": 10,
+    "train_samples_per_second": 0.642,
+    "train_steps_per_second": 0.064
 }

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 3.0,
-    "train_loss": 0.3101682186126709,
-    "train_runtime": 67.2839,
-    "train_samples": 20,
-    "train_samples_per_second": 1.486,
-    "train_steps_per_second": 0.074
+    "epoch": 5.0,
+    "train_loss": 0.34364445209503175,
+    "train_runtime": 77.9014,
+    "train_samples": 10,
+    "train_samples_per_second": 0.642,
+    "train_steps_per_second": 0.064
 }

trainer_state.json

@@ -1,7 +1,7 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 3.0,
+  "epoch": 5.0,
   "eval_steps": 100,
   "global_step": 5,
   "is_hyper_param_search": false,
@@ -11,11 +11,11 @@
     {
       "epoch": 1.0,
       "learning_rate": 5e-06,
-      "logits/chosen": -2.962373733520508,
-      "logits/rejected": -2.9390432834625244,
-      "logps/chosen": -148.1490020751953,
-      "logps/rejected": -152.30950927734375,
-      "loss": 0.5199,
+      "logits/chosen": -2.811583995819092,
+      "logits/rejected": -2.8277242183685303,
+      "logps/chosen": -114.9925765991211,
+      "logps/rejected": -127.65025329589844,
+      "loss": 0.3466,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -23,13 +23,13 @@
       "step": 1
     },
     {
-      "epoch": 3.0,
+      "epoch": 5.0,
       "step": 5,
       "total_flos": 0.0,
-      "train_loss": 0.3101682186126709,
-      "train_runtime": 67.2839,
-      "train_samples_per_second": 1.486,
-      "train_steps_per_second": 0.074
+      "train_loss": 0.34364445209503175,
+      "train_runtime": 77.9014,
+      "train_samples_per_second": 0.642,
+      "train_steps_per_second": 0.064
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8174f88e2193228913c6711cb9f076391851a267eea4184ebbbe8046b7e07bf2
+oid sha256:42394665981968fa2b0c9bacc2ad019c23bfacc4431c755e80b9bb7affd7d253
 size 6200