End of training

README.md

@@ -2,10 +2,16 @@
 license: apache-2.0
 base_model: nnheui/pythia-1.4b-sft-full
 tags:
+- alignment-handbook
+- trl
+- dpo
+- generated_from_trainer
 - trl
 - dpo
 - alignment-handbook
 - generated_from_trainer
+datasets:
+- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: pythia-1.4b-dpo-full
   results: []
@@ -16,19 +22,19 @@ should probably proofread and complete it, then remove this comment. -->
 
 # pythia-1.4b-dpo-full
 
-This model is a fine-tuned version of [nnheui/pythia-1.4b-sft-full](https://huggingface.co/nnheui/pythia-1.4b-sft-full) on an unknown dataset.
+This model is a fine-tuned version of [nnheui/pythia-1.4b-sft-full](https://huggingface.co/nnheui/pythia-1.4b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5967
-- Rewards/chosen: -1.8672
+- Loss: 0.5991
+- Rewards/chosen: -1.875
 - Rewards/rejected: -2.6406
-- Rewards/accuracies: 0.7134
+- Rewards/accuracies: 0.7164
 - Rewards/margins: 0.7734
-- Logps/rejected: -600.0
+- Logps/rejected: -604.0
 - Logps/chosen: -580.0
-- Logits/rejected: -1.4375
+- Logits/rejected: -1.4297
 - Logits/chosen: -1.4062
 - Logps/chosen Top Tokens: -0.0009
-- Logps/rejected Top Tokens: -0.0008
+- Logps/rejected Top Tokens: -0.0009
 - Logps/chosen Bottom Tokens: -13.9375
 - Logps/rejected Bottom Tokens: -13.8125
 

all_results.json

@@ -1,22 +1,22 @@
 {
     "epoch": 9.99018645731109,
-    "eval_logits/chosen": -1.1953125,
-    "eval_logits/rejected": -1.2421875,
-    "eval_logps/chosen": -446.0,
-    "eval_logps/chosen_bottom_tokens": -14.375,
-    "eval_logps/chosen_top_tokens": -0.00072479248046875,
-    "eval_logps/rejected": -416.0,
-    "eval_logps/rejected_bottom_tokens": -14.3125,
-    "eval_logps/rejected_top_tokens": -0.000728607177734375,
-    "eval_loss": 0.6256738305091858,
-    "eval_rewards/accuracies": 0.6597015857696533,
-    "eval_rewards/chosen": -0.5234375,
-    "eval_rewards/margins": 0.2578125,
-    "eval_rewards/rejected": -0.78125,
-    "eval_runtime": 111.3149,
+    "eval_logits/chosen": -1.40625,
+    "eval_logits/rejected": -1.4296875,
+    "eval_logps/chosen": -580.0,
+    "eval_logps/chosen_bottom_tokens": -13.9375,
+    "eval_logps/chosen_top_tokens": -0.000881195068359375,
+    "eval_logps/rejected": -604.0,
+    "eval_logps/rejected_bottom_tokens": -13.8125,
+    "eval_logps/rejected_top_tokens": -0.000858306884765625,
+    "eval_loss": 0.5990539789199829,
+    "eval_rewards/accuracies": 0.7164179086685181,
+    "eval_rewards/chosen": -1.875,
+    "eval_rewards/margins": 0.7734375,
+    "eval_rewards/rejected": -2.640625,
+    "eval_runtime": 107.9788,
     "eval_samples": 2000,
-    "eval_samples_per_second": 17.967,
-    "eval_steps_per_second": 0.602,
+    "eval_samples_per_second": 18.522,
+    "eval_steps_per_second": 0.62,
     "total_flos": 0.0,
     "train_loss": 0.40154263242522953,
     "train_runtime": 75277.8699,

config.json

@@ -24,7 +24,7 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.40.0",
-  "use_cache": false,
+  "use_cache": true,
   "use_parallel_residual": true,
   "vocab_size": 50304
 }

eval_results.json

@@ -1,20 +1,20 @@
 {
-    "epoch": 0.9990186457311089,
-    "eval_logits/chosen": -1.1953125,
-    "eval_logits/rejected": -1.2421875,
-    "eval_logps/chosen": -446.0,
-    "eval_logps/chosen_bottom_tokens": -14.375,
-    "eval_logps/chosen_top_tokens": -0.00072479248046875,
-    "eval_logps/rejected": -416.0,
-    "eval_logps/rejected_bottom_tokens": -14.3125,
-    "eval_logps/rejected_top_tokens": -0.000728607177734375,
-    "eval_loss": 0.6256738305091858,
-    "eval_rewards/accuracies": 0.6597015857696533,
-    "eval_rewards/chosen": -0.5234375,
-    "eval_rewards/margins": 0.2578125,
-    "eval_rewards/rejected": -0.78125,
-    "eval_runtime": 111.3149,
+    "epoch": 9.99018645731109,
+    "eval_logits/chosen": -1.40625,
+    "eval_logits/rejected": -1.4296875,
+    "eval_logps/chosen": -580.0,
+    "eval_logps/chosen_bottom_tokens": -13.9375,
+    "eval_logps/chosen_top_tokens": -0.000881195068359375,
+    "eval_logps/rejected": -604.0,
+    "eval_logps/rejected_bottom_tokens": -13.8125,
+    "eval_logps/rejected_top_tokens": -0.000858306884765625,
+    "eval_loss": 0.5990539789199829,
+    "eval_rewards/accuracies": 0.7164179086685181,
+    "eval_rewards/chosen": -1.875,
+    "eval_rewards/margins": 0.7734375,
+    "eval_rewards/rejected": -2.640625,
+    "eval_runtime": 107.9788,
     "eval_samples": 2000,
-    "eval_samples_per_second": 17.967,
-    "eval_steps_per_second": 0.602
+    "eval_samples_per_second": 18.522,
+    "eval_steps_per_second": 0.62
 }

runs/Jul08_16-05-34_42dbe5cf9ed4/events.out.tfevents.1720531034.42dbe5cf9ed4.852680.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a2d40497be662f3bffb78d4cbba956f7390d6c47331450d2a6d75d7a26254e6
+size 1106