Model save

README.md

@@ -15,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 4336.2285
-- Rewards/chosen: -0.7580
-- Rewards/rejected: -0.8224
-- Rewards/accuracies: 0.6084
-- Rewards/margins: 0.0644
-- Rewards/safe Rewards: -0.7526
-- Rewards/unsafe Rewards: -0.7556
-- Logps/rejected: -174.7150
-- Logps/chosen: -206.2398
-- Logits/rejected: 0.0353
-- Logits/chosen: -0.6151
+- Loss: 17200.3730
+- Rewards/chosen: -0.9463
+- Rewards/rejected: -1.0095
+- Rewards/accuracies: 0.6067
+- Rewards/margins: 0.0632
+- Rewards/safe Rewards: -0.9414
+- Rewards/unsafe Rewards: -0.9445
+- Logps/rejected: -294.3703
+- Logps/chosen: -319.7016
+- Logits/rejected: 2.3506
+- Logits/chosen: 1.1396
 
 ## Model description
 
@@ -45,13 +45,13 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 5e-07
-- train_batch_size: 4
+- train_batch_size: 2
 - eval_batch_size: 8
 - seed: 42
 - distributed_type: multi-GPU
 - num_devices: 4
 - gradient_accumulation_steps: 4
-- total_train_batch_size: 64
+- total_train_batch_size: 32
 - total_eval_batch_size: 32
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
@@ -62,7 +62,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Rewards/safe Rewards | Rewards/unsafe Rewards | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------------:|:----------------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 11841.1211    | 0.54  | 500  | 4375.6743       | -0.7240        | -0.8064          | 0.6440             | 0.0823          | -0.7177              | -0.7186                | -173.1074      | -202.8401    | -0.0312         | -0.6399       |
+| 48756.7969    | 0.27  | 500  | 17836.0742      | -0.6517        | -0.6890          | 0.5689             | 0.0374          | -0.6482              | -0.6494                | -230.2809      | -260.7748    | -1.4278         | -1.8042       |
+| 45676.9531    | 0.54  | 1000 | 17429.8281      | -0.7819        | -0.8708          | 0.6481             | 0.0890          | -0.7754              | -0.7778                | -266.6385      | -286.8121    | 1.9950          | 0.8553        |
+| 40856.9313    | 0.81  | 1500 | 17213.6094      | -0.9648        | -1.0237          | 0.6079             | 0.0589          | -0.9605              | -0.9638                | -297.2072      | -323.3928    | 2.5177          | 1.2878        |
 
 
 ### Framework versions

all_results.json

@@ -1,23 +1,23 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -0.6150549054145813,
-    "eval_logits/rejected": 0.035322826355695724,
-    "eval_logps/chosen": -206.2398223876953,
-    "eval_logps/rejected": -174.71498107910156,
-    "eval_loss": 4336.228515625,
-    "eval_rewards/accuracies": 0.6084221005439758,
-    "eval_rewards/chosen": -0.7580091953277588,
-    "eval_rewards/margins": 0.06442829966545105,
-    "eval_rewards/rejected": -0.8224374651908875,
-    "eval_rewards/safe_rewards": -0.7526479363441467,
-    "eval_rewards/unsafe_rewards": -0.755587637424469,
-    "eval_runtime": 1044.2833,
+    "eval_logits/chosen": 1.1395671367645264,
+    "eval_logits/rejected": 2.3505630493164062,
+    "eval_logps/chosen": -319.7015686035156,
+    "eval_logps/rejected": -294.37030029296875,
+    "eval_loss": 17200.373046875,
+    "eval_rewards/accuracies": 0.6067279577255249,
+    "eval_rewards/chosen": -0.946313202381134,
+    "eval_rewards/margins": 0.0631820559501648,
+    "eval_rewards/rejected": -1.0094952583312988,
+    "eval_rewards/safe_rewards": -0.9414083957672119,
+    "eval_rewards/unsafe_rewards": -0.944499135017395,
+    "eval_runtime": 1062.2335,
     "eval_samples": 33044,
-    "eval_samples_per_second": 31.643,
-    "eval_steps_per_second": 0.989,
-    "train_loss": 11837.785513657158,
-    "train_runtime": 18541.4057,
+    "eval_samples_per_second": 31.108,
+    "eval_steps_per_second": 0.972,
+    "train_loss": 47137.349905812705,
+    "train_runtime": 22393.3393,
     "train_samples": 59478,
-    "train_samples_per_second": 3.208,
-    "train_steps_per_second": 0.05
+    "train_samples_per_second": 2.656,
+    "train_steps_per_second": 0.083
 }

eval_results.json

@@ -1,18 +1,18 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -0.6150549054145813,
-    "eval_logits/rejected": 0.035322826355695724,
-    "eval_logps/chosen": -206.2398223876953,
-    "eval_logps/rejected": -174.71498107910156,
-    "eval_loss": 4336.228515625,
-    "eval_rewards/accuracies": 0.6084221005439758,
-    "eval_rewards/chosen": -0.7580091953277588,
-    "eval_rewards/margins": 0.06442829966545105,
-    "eval_rewards/rejected": -0.8224374651908875,
-    "eval_rewards/safe_rewards": -0.7526479363441467,
-    "eval_rewards/unsafe_rewards": -0.755587637424469,
-    "eval_runtime": 1044.2833,
+    "eval_logits/chosen": 1.1395671367645264,
+    "eval_logits/rejected": 2.3505630493164062,
+    "eval_logps/chosen": -319.7015686035156,
+    "eval_logps/rejected": -294.37030029296875,
+    "eval_loss": 17200.373046875,
+    "eval_rewards/accuracies": 0.6067279577255249,
+    "eval_rewards/chosen": -0.946313202381134,
+    "eval_rewards/margins": 0.0631820559501648,
+    "eval_rewards/rejected": -1.0094952583312988,
+    "eval_rewards/safe_rewards": -0.9414083957672119,
+    "eval_rewards/unsafe_rewards": -0.944499135017395,
+    "eval_runtime": 1062.2335,
     "eval_samples": 33044,
-    "eval_samples_per_second": 31.643,
-    "eval_steps_per_second": 0.989
+    "eval_samples_per_second": 31.108,
+    "eval_steps_per_second": 0.972
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a91d9ba47e204876c0ac101e2ef27a0c27d68c135aebaa5c8a60a7e531bd0cfb
+oid sha256:0cae3e9c2f07843bc94eb9798008f8926de896544008b976a659bc33c9ac1466
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3e0125b6268e71d418c96272816b306e8a8ecbeb69a9581073acf5bc4a717edf
+oid sha256:314dd3ad4fa7bfe6ed8ee89722c9965a1117ac67b1818d4aa9e0fd47877a9a67
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bddfd7df2b88d22d9d6b171d69917106ac0155c7c988d14c73740f38f92e0c13
+oid sha256:194ddb5714d660dc426b40b826585d0cbb48b92a2cd27fc443f9b2751b77e61c
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 11837.785513657158,
-    "train_runtime": 18541.4057,
+    "train_loss": 47137.349905812705,
+    "train_runtime": 22393.3393,
     "train_samples": 59478,
-    "train_samples_per_second": 3.208,
-    "train_steps_per_second": 0.05
+    "train_samples_per_second": 2.656,
+    "train_steps_per_second": 0.083
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1bf91ed24d393f3546c828a49f51e81c5c73db31eea06da07d3a5912b056f341
+oid sha256:6300e360d82a0feb906c216b9396bb5a71feec1832af57052022de4a91c93910
 size 6136