End of training

README.md

@@ -1,11 +1,16 @@
 ---
-license: apache-2.0
 base_model: nnheui/pythia-1.4b-sft-full
 tags:
+- alignment-handbook
+- trl
+- dpo
+- generated_from_trainer
 - trl
 - dpo
 - alignment-handbook
 - generated_from_trainer
+datasets:
+- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: pythia-1.4b-dpo-full
   results: []
@@ -16,18 +21,19 @@ should probably proofread and complete it, then remove this comment. -->
 
 # pythia-1.4b-dpo-full
 
-This model is a fine-tuned version of [nnheui/pythia-1.4b-sft-full](https://huggingface.co/nnheui/pythia-1.4b-sft-full) on an unknown dataset.
+This model is a fine-tuned version of [nnheui/pythia-1.4b-sft-full](https://huggingface.co/nnheui/pythia-1.4b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
 It achieves the following results on the evaluation set:
-- Logits/chosen: -1.1953
-- Logits/rejected: -1.2422
-- Logps/bottom Tokens: -0.0007
-- Logps/chosen: -446.0
-- Logps/rejected: -416.0
-- Loss: 0.6259
-- Rewards/accuracies: 0.6567
+- Loss: 0.6257
 - Rewards/chosen: -0.5234
-- Rewards/margins: 0.2617
-- Rewards/rejected: -0.7852
+- Rewards/rejected: -0.7812
+- Rewards/accuracies: 0.6597
+- Rewards/margins: 0.2578
+- Logps/rejected: -416.0
+- Logps/chosen: -446.0
+- Logits/rejected: -1.2422
+- Logits/chosen: -1.1953
+- Logps/chosen Bottom Tokens: -0.0007
+- Logps/rejected Bottom Tokens: -0.0007
 
 ## Model description
 

all_results.json

@@ -4,16 +4,18 @@
     "eval_logits/rejected": -1.2421875,
     "eval_logps/bottom_tokens": -0.00072479248046875,
     "eval_logps/chosen": -446.0,
+    "eval_logps/chosen_bottom_tokens": -0.00072479248046875,
     "eval_logps/rejected": -416.0,
+    "eval_logps/rejected_bottom_tokens": -0.000728607177734375,
     "eval_loss": 0.6256738305091858,
     "eval_rewards/accuracies": 0.6597015857696533,
     "eval_rewards/chosen": -0.5234375,
     "eval_rewards/margins": 0.2578125,
     "eval_rewards/rejected": -0.78125,
-    "eval_runtime": 94.2502,
+    "eval_runtime": 103.3574,
     "eval_samples": 2000,
-    "eval_samples_per_second": 21.22,
-    "eval_steps_per_second": 0.711,
+    "eval_samples_per_second": 19.35,
+    "eval_steps_per_second": 0.648,
     "total_flos": 0.0,
     "train_loss": 0.011100004604616897,
     "train_runtime": 142.2419,

config.json

@@ -24,7 +24,7 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.40.0",
-  "use_cache": false,
+  "use_cache": true,
   "use_parallel_residual": true,
   "vocab_size": 50304
 }

eval_results.json

@@ -2,16 +2,17 @@
     "epoch": 0.9990186457311089,
     "eval_logits/chosen": -1.1953125,
     "eval_logits/rejected": -1.2421875,
-    "eval_logps/bottom_tokens": -0.00072479248046875,
     "eval_logps/chosen": -446.0,
+    "eval_logps/chosen_bottom_tokens": -0.00072479248046875,
     "eval_logps/rejected": -416.0,
+    "eval_logps/rejected_bottom_tokens": -0.000728607177734375,
     "eval_loss": 0.6256738305091858,
     "eval_rewards/accuracies": 0.6597015857696533,
     "eval_rewards/chosen": -0.5234375,
     "eval_rewards/margins": 0.2578125,
     "eval_rewards/rejected": -0.78125,
-    "eval_runtime": 94.2502,
+    "eval_runtime": 103.3574,
     "eval_samples": 2000,
-    "eval_samples_per_second": 21.22,
-    "eval_steps_per_second": 0.711
+    "eval_samples_per_second": 19.35,
+    "eval_steps_per_second": 0.648
 }

runs/Jul08_11-47-26_42dbe5cf9ed4/events.out.tfevents.1720440087.42dbe5cf9ed4.850226.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a90d1a74381a5fc2cb19babf3a6b42b331c04828e5a3ce9ad0eb924fca36400a
+size 970