End of training

README.md

@@ -2,9 +2,15 @@
 license: apache-2.0
 base_model: nnheui/pythia-1.4b-sft-full
 tags:
+- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
+- trl
+- dpo
+- generated_from_trainer
+datasets:
+- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: pythia-1.4b-dpo-full
   results: []
@@ -15,13 +21,13 @@ should probably proofread and complete it, then remove this comment. -->
 
 # pythia-1.4b-dpo-full
 
-This model is a fine-tuned version of [nnheui/pythia-1.4b-sft-full](https://huggingface.co/nnheui/pythia-1.4b-sft-full) on an unknown dataset.
+This model is a fine-tuned version of [nnheui/pythia-1.4b-sft-full](https://huggingface.co/nnheui/pythia-1.4b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6259
+- Loss: 0.6257
 - Rewards/chosen: -0.5234
-- Rewards/rejected: -0.7852
-- Rewards/accuracies: 0.6567
-- Rewards/margins: 0.2617
+- Rewards/rejected: -0.7812
+- Rewards/accuracies: 0.6597
+- Rewards/margins: 0.2578
 - Logps/rejected: -416.0
 - Logps/chosen: -446.0
 - Logits/rejected: -1.2422

all_results.json

@@ -1,5 +1,22 @@
 {
     "epoch": 0.9990186457311089,
+    "eval_logits/chosen": -1.1953125,
+    "eval_logits/rejected": -1.2421875,
+    "eval_logps/chosen": -446.0,
+    "eval_logps/chosen_bottom_tokens": -14.375,
+    "eval_logps/chosen_top_tokens": -0.00072479248046875,
+    "eval_logps/rejected": -416.0,
+    "eval_logps/rejected_bottom_tokens": -14.3125,
+    "eval_logps/rejected_top_tokens": -0.000728607177734375,
+    "eval_loss": 0.6256738305091858,
+    "eval_rewards/accuracies": 0.6597015857696533,
+    "eval_rewards/chosen": -0.5234375,
+    "eval_rewards/margins": 0.2578125,
+    "eval_rewards/rejected": -0.78125,
+    "eval_runtime": 111.3149,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 17.967,
+    "eval_steps_per_second": 0.602,
     "total_flos": 0.0,
     "train_loss": 0.6464882252961105,
     "train_runtime": 8284.9703,

config.json

@@ -24,7 +24,7 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.40.0",
-  "use_cache": false,
+  "use_cache": true,
   "use_parallel_residual": true,
   "vocab_size": 50304
 }

eval_results.json

@@ -3,16 +3,18 @@
     "eval_logits/chosen": -1.1953125,
     "eval_logits/rejected": -1.2421875,
     "eval_logps/chosen": -446.0,
-    "eval_logps/chosen_bottom_tokens": -0.00072479248046875,
+    "eval_logps/chosen_bottom_tokens": -14.375,
+    "eval_logps/chosen_top_tokens": -0.00072479248046875,
     "eval_logps/rejected": -416.0,
-    "eval_logps/rejected_bottom_tokens": -0.000728607177734375,
+    "eval_logps/rejected_bottom_tokens": -14.3125,
+    "eval_logps/rejected_top_tokens": -0.000728607177734375,
     "eval_loss": 0.6256738305091858,
     "eval_rewards/accuracies": 0.6597015857696533,
     "eval_rewards/chosen": -0.5234375,
     "eval_rewards/margins": 0.2578125,
     "eval_rewards/rejected": -0.78125,
-    "eval_runtime": 103.3574,
+    "eval_runtime": 111.3149,
     "eval_samples": 2000,
-    "eval_samples_per_second": 19.35,
-    "eval_steps_per_second": 0.648
+    "eval_samples_per_second": 17.967,
+    "eval_steps_per_second": 0.602
 }

runs/Jul08_12-10-46_42dbe5cf9ed4/events.out.tfevents.1720449637.42dbe5cf9ed4.850889.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68f8cf6e8007588450a23c06a2bc5fcb4735e4452e85d1d1db6cbb8b40e23c77
+size 1106