Model save

README.md

@@ -1,15 +1,11 @@
 ---
+license: apache-2.0
 base_model: nnheui/pythia-1.4b-sft-full
 tags:
-- alignment-handbook
-- trl
-- dpo
-- generated_from_trainer
 - trl
 - dpo
+- alignment-handbook
 - generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: pythia-1.4b-dpo-full
   results: []
@@ -20,18 +16,18 @@ should probably proofread and complete it, then remove this comment. -->
 
 # pythia-1.4b-dpo-full
 
-This model is a fine-tuned version of [nnheui/pythia-1.4b-sft-full](https://huggingface.co/nnheui/pythia-1.4b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [nnheui/pythia-1.4b-sft-full](https://huggingface.co/nnheui/pythia-1.4b-sft-full) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6257
-- Rewards/chosen: -0.5234
-- Rewards/rejected: -0.7812
-- Rewards/accuracies: 0.6597
-- Rewards/margins: 0.2578
-- Logps/rejected: -416.0
-- Logps/chosen: -446.0
-- Logits/rejected: -1.2422
 - Logits/chosen: -1.1953
+- Logits/rejected: -1.2422
 - Logps/bottom Tokens: -0.0007
+- Logps/chosen: -446.0
+- Logps/rejected: -416.0
+- Loss: 0.6259
+- Rewards/accuracies: 0.6567
+- Rewards/chosen: -0.5234
+- Rewards/margins: 0.2617
+- Rewards/rejected: -0.7852
 
 ## Model description
 
@@ -66,13 +62,13 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Logps/bottom Tokens |
-|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:-------------------:|
-| 0.678         | 0.1963 | 100  | 0.6789          | -0.0275        | -0.0608          | 0.5881             | 0.0332          | -344.0         | -396.0       | -1.1562         | -1.0938       | -0.0009             |
-| 0.645         | 0.3925 | 200  | 0.6489          | -0.2871        | -0.4238          | 0.6448             | 0.1367          | -380.0         | -422.0       | -1.2031         | -1.1562       | -0.0009             |
-| 0.6396        | 0.5888 | 300  | 0.6304          | -0.4512        | -0.6797          | 0.6627             | 0.2275          | -406.0         | -438.0       | -1.2344         | -1.1875       | -0.0008             |
-| 0.6102        | 0.7851 | 400  | 0.6268          | -0.5039        | -0.7617          | 0.6567             | 0.2578          | -414.0         | -444.0       | -1.2344         | -1.1875       | -0.0007             |
-| 0.6084        | 0.9814 | 500  | 0.6259          | -0.5234        | -0.7852          | 0.6567             | 0.2617          | -416.0         | -446.0       | -1.2422         | -1.1953       | -0.0007             |
+| Training Loss | Epoch  | Step | Logits/chosen | Logits/rejected | Logps/bottom Tokens | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
+|:-------------:|:------:|:----:|:-------------:|:---------------:|:-------------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
+| 0.678         | 0.1963 | 100  | -1.0938       | -1.1562         | -0.0009             | -396.0       | -344.0         | 0.6789          | 0.5881             | -0.0275        | 0.0332          | -0.0608          |
+| 0.645         | 0.3925 | 200  | -1.1562       | -1.2031         | -0.0009             | -422.0       | -380.0         | 0.6489          | 0.6448             | -0.2871        | 0.1367          | -0.4238          |
+| 0.6396        | 0.5888 | 300  | -1.1875       | -1.2344         | -0.0008             | -438.0       | -406.0         | 0.6304          | 0.6627             | -0.4512        | 0.2275          | -0.6797          |
+| 0.6102        | 0.7851 | 400  | -1.1875       | -1.2344         | -0.0007             | -444.0       | -414.0         | 0.6268          | 0.6567             | -0.5039        | 0.2578          | -0.7617          |
+| 0.6084        | 0.9814 | 500  | -1.1953       | -1.2422         | -0.0007             | -446.0       | -416.0         | 0.6259          | 0.6567             | -0.5234        | 0.2617          | -0.7852          |
 
 
 ### Framework versions

all_results.json

@@ -15,9 +15,9 @@
     "eval_samples_per_second": 21.22,
     "eval_steps_per_second": 0.711,
     "total_flos": 0.0,
-    "train_loss": 0.6464882252961105,
-    "train_runtime": 7224.3746,
+    "train_loss": 0.011100004604616897,
+    "train_runtime": 142.2419,
     "train_samples": 61134,
-    "train_samples_per_second": 8.462,
-    "train_steps_per_second": 0.07
+    "train_samples_per_second": 429.789,
+    "train_steps_per_second": 3.578
 }

config.json

@@ -24,7 +24,7 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.40.0",
-  "use_cache": true,
+  "use_cache": false,
   "use_parallel_residual": true,
   "vocab_size": 50304
 }

runs/Jul08_11-47-26_42dbe5cf9ed4/events.out.tfevents.1720439808.42dbe5cf9ed4.850226.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8090350640e29f059617560ec3c10f27830209851a0e1984cba5fc4630feedf6
+size 5344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9990186457311089,
     "total_flos": 0.0,
-    "train_loss": 0.6464882252961105,
-    "train_runtime": 7224.3746,
+    "train_loss": 0.011100004604616897,
+    "train_runtime": 142.2419,
     "train_samples": 61134,
-    "train_samples_per_second": 8.462,
-    "train_steps_per_second": 0.07
+    "train_samples_per_second": 429.789,
+    "train_steps_per_second": 3.578
 }

trainer_state.json

@@ -913,10 +913,10 @@
       "epoch": 0.9990186457311089,
       "step": 509,
       "total_flos": 0.0,
-      "train_loss": 0.6464882252961105,
-      "train_runtime": 7224.3746,
-      "train_samples_per_second": 8.462,
-      "train_steps_per_second": 0.07
+      "train_loss": 0.011100004604616897,
+      "train_runtime": 142.2419,
+      "train_samples_per_second": 429.789,
+      "train_steps_per_second": 3.578
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e11a77604a705654a1e888c55ace8081ed019ade2298cd8d9ce2bfaaba68a65b
+oid sha256:6512ebaae4617131c502f655a94c28ca877471322e354baceb9f8cb3b89b8f9e
 size 6264