End of training

README.md

@@ -1,10 +1,15 @@
 ---
-license: apache-2.0
 base_model: nnheui/pythia-1.4b-sft-full
 tags:
+- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
+- trl
+- dpo
+- generated_from_trainer
+datasets:
+- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: pythia-1.4b-dpo-full
   results: []
@@ -15,13 +20,13 @@ should probably proofread and complete it, then remove this comment. -->
 
 # pythia-1.4b-dpo-full
 
-This model is a fine-tuned version of [nnheui/pythia-1.4b-sft-full](https://huggingface.co/nnheui/pythia-1.4b-sft-full) on an unknown dataset.
+This model is a fine-tuned version of [nnheui/pythia-1.4b-sft-full](https://huggingface.co/nnheui/pythia-1.4b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6259
+- Loss: 0.6257
 - Rewards/chosen: -0.5234
-- Rewards/rejected: -0.7852
-- Rewards/accuracies: 0.6567
-- Rewards/margins: 0.2617
+- Rewards/rejected: -0.7812
+- Rewards/accuracies: 0.6597
+- Rewards/margins: 0.2578
 - Logps/rejected: -416.0
 - Logps/chosen: -446.0
 - Logits/rejected: -1.2422

all_results.json

@@ -1,5 +1,19 @@
 {
     "epoch": 0.9990186457311089,
+    "eval_logits/chosen": -1.1953125,
+    "eval_logits/rejected": -1.2421875,
+    "eval_logps/bottom_tokens": -0.00072479248046875,
+    "eval_logps/chosen": -446.0,
+    "eval_logps/rejected": -416.0,
+    "eval_loss": 0.6256738305091858,
+    "eval_rewards/accuracies": 0.6597015857696533,
+    "eval_rewards/chosen": -0.5234375,
+    "eval_rewards/margins": 0.2578125,
+    "eval_rewards/rejected": -0.78125,
+    "eval_runtime": 94.2502,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 21.22,
+    "eval_steps_per_second": 0.711,
     "total_flos": 0.0,
     "train_loss": 0.6464882252961105,
     "train_runtime": 7224.3746,

config.json

@@ -24,7 +24,7 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.40.0",
-  "use_cache": false,
+  "use_cache": true,
   "use_parallel_residual": true,
   "vocab_size": 50304
 }

eval_results.json

@@ -1,16 +1,17 @@
 {
-    "epoch": 1.0,
-    "eval_logits/chosen": -0.6171875,
-    "eval_logits/rejected": -0.671875,
-    "eval_logps/chosen": -2320.0,
-    "eval_logps/rejected": -2024.0,
-    "eval_loss": 0.6403203010559082,
-    "eval_rewards/accuracies": 0.5892857313156128,
-    "eval_rewards/chosen": 0.609375,
-    "eval_rewards/margins": 0.2001953125,
-    "eval_rewards/rejected": 0.41015625,
-    "eval_runtime": 85.4736,
+    "epoch": 0.9990186457311089,
+    "eval_logits/chosen": -1.1953125,
+    "eval_logits/rejected": -1.2421875,
+    "eval_logps/bottom_tokens": -0.00072479248046875,
+    "eval_logps/chosen": -446.0,
+    "eval_logps/rejected": -416.0,
+    "eval_loss": 0.6256738305091858,
+    "eval_rewards/accuracies": 0.6597015857696533,
+    "eval_rewards/chosen": -0.5234375,
+    "eval_rewards/margins": 0.2578125,
+    "eval_rewards/rejected": -0.78125,
+    "eval_runtime": 94.2502,
     "eval_samples": 2000,
-    "eval_samples_per_second": 23.399,
-    "eval_steps_per_second": 0.491
+    "eval_samples_per_second": 21.22,
+    "eval_steps_per_second": 0.711
 }

runs/Jul08_06-23-29_42dbe5cf9ed4/events.out.tfevents.1720427717.42dbe5cf9ed4.848884.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:700f203101290a9350ce28f5522322255c010a2532a935f5c66075669769ad38
+size 891