End of training

README.md

@@ -3,9 +3,15 @@ library_name: transformers
 license: apache-2.0
 base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
+- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
+- trl
+- dpo
+- generated_from_trainer
+datasets:
+- data/rlced_conifer
 model-index:
 - name: rlced-conifer-zephyr-7b-dpo-2e
   results: []
@@ -16,7 +22,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # rlced-conifer-zephyr-7b-dpo-2e
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on an unknown dataset.
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the data/rlced_conifer dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6931
+- Rewards/chosen: 0.0
+- Rewards/rejected: 0.0
+- Rewards/accuracies: 0.0
+- Rewards/margins: 0.0
+- Logps/rejected: -351.6677
+- Logps/chosen: -401.2567
+- Logits/rejected: -2.6879
+- Logits/chosen: -2.8249
 
 ## Model description
 

all_results.json

@@ -1,5 +1,18 @@
 {
     "epoch": 2.0,
+    "eval_logits/chosen": -2.8249309062957764,
+    "eval_logits/rejected": -2.6878879070281982,
+    "eval_logps/chosen": -401.2567138671875,
+    "eval_logps/rejected": -351.667724609375,
+    "eval_loss": 0.6931471228599548,
+    "eval_rewards/accuracies": 0.0,
+    "eval_rewards/chosen": 0.0,
+    "eval_rewards/margins": 0.0,
+    "eval_rewards/rejected": 0.0,
+    "eval_runtime": 2.9241,
+    "eval_samples": 50,
+    "eval_samples_per_second": 17.099,
+    "eval_steps_per_second": 0.342,
     "total_flos": 0.0,
     "train_loss": 0.1732867956161499,
     "train_runtime": 97.0028,

config.json

@@ -22,6 +22,6 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.44.1",
-  "use_cache": false,
+  "use_cache": true,
   "vocab_size": 32000
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
-    "epoch": 1.996886351842242,
-    "eval_logits/chosen": 0.2633172273635864,
-    "eval_logits/rejected": 3.3806047439575195,
-    "eval_logps/chosen": -1345.8763427734375,
-    "eval_logps/rejected": -2649.19384765625,
-    "eval_loss": 0.15931174159049988,
-    "eval_rewards/accuracies": 0.9325980544090271,
-    "eval_rewards/chosen": -9.224682807922363,
-    "eval_rewards/margins": 12.807853698730469,
-    "eval_rewards/rejected": -22.03253746032715,
-    "eval_runtime": 298.1405,
-    "eval_samples": 6491,
-    "eval_samples_per_second": 21.772,
+    "epoch": 2.0,
+    "eval_logits/chosen": -2.8249309062957764,
+    "eval_logits/rejected": -2.6878879070281982,
+    "eval_logps/chosen": -401.2567138671875,
+    "eval_logps/rejected": -351.667724609375,
+    "eval_loss": 0.6931471228599548,
+    "eval_rewards/accuracies": 0.0,
+    "eval_rewards/chosen": 0.0,
+    "eval_rewards/margins": 0.0,
+    "eval_rewards/rejected": 0.0,
+    "eval_runtime": 2.9241,
+    "eval_samples": 50,
+    "eval_samples_per_second": 17.099,
     "eval_steps_per_second": 0.342
 }