Model save

README.md

@@ -15,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 978.6271
-- Rewards/chosen: -0.2485
-- Rewards/rejected: -0.2964
-- Rewards/accuracies: 0.6600
-- Rewards/margins: 0.0478
-- Rewards/safe Rewards: -0.2450
-- Rewards/unsafe Rewards: -0.2427
-- Logps/rejected: -131.7681
-- Logps/chosen: -165.2006
-- Logits/rejected: -2.0856
-- Logits/chosen: -2.2494
+- Loss: 4336.2285
+- Rewards/chosen: -0.7580
+- Rewards/rejected: -0.8224
+- Rewards/accuracies: 0.6084
+- Rewards/margins: 0.0644
+- Rewards/safe Rewards: -0.7526
+- Rewards/unsafe Rewards: -0.7556
+- Logps/rejected: -174.7150
+- Logps/chosen: -206.2398
+- Logits/rejected: 0.0353
+- Logits/chosen: -0.6151
 
 ## Model description
 
@@ -45,13 +45,13 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 5e-07
-- train_batch_size: 2
+- train_batch_size: 4
 - eval_batch_size: 8
 - seed: 42
 - distributed_type: multi-GPU
 - num_devices: 4
 - gradient_accumulation_steps: 4
-- total_train_batch_size: 32
+- total_train_batch_size: 64
 - total_eval_batch_size: 32
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
@@ -62,13 +62,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Rewards/safe Rewards | Rewards/unsafe Rewards | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------------:|:----------------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 1504.5268     | 0.13  | 500  | 1034.4193       | -0.1746        | -0.2243          | 0.6809             | 0.0497          | -0.1711              | -0.1688                | -124.5656      | -157.8098    | -1.9343         | -2.1039       |
-| 1379.832      | 0.27  | 1000 | 1014.7580       | -0.1546        | -0.2013          | 0.6632             | 0.0467          | -0.1522              | -0.1501                | -122.2598      | -155.8100    | -2.1099         | -2.2674       |
-| 1404.9199     | 0.4   | 1500 | 997.0104        | -0.2219        | -0.2678          | 0.6678             | 0.0459          | -0.2189              | -0.2165                | -128.9146      | -162.5368    | -2.0713         | -2.2162       |
-| 1361.9422     | 0.53  | 2000 | 991.2021        | -0.2381        | -0.2863          | 0.6686             | 0.0481          | -0.2356              | -0.2330                | -130.7618      | -164.1645    | -2.1980         | -2.3435       |
-| 1406.6168     | 0.66  | 2500 | 981.6749        | -0.2153        | -0.2602          | 0.6503             | 0.0450          | -0.2126              | -0.2104                | -128.1535      | -161.8747    | -2.0826         | -2.2439       |
-| 1365.8523     | 0.8   | 3000 | 980.2808        | -0.2165        | -0.2645          | 0.6566             | 0.0481          | -0.2132              | -0.2111                | -128.5860      | -161.9975    | -2.0925         | -2.2557       |
-| 1242.352      | 0.93  | 3500 | 978.7930        | -0.2464        | -0.2943          | 0.6613             | 0.0479          | -0.2429              | -0.2405                | -131.5628      | -164.9901    | -2.0840         | -2.2476       |
+| 11841.1211    | 0.54  | 500  | 4375.6743       | -0.7240        | -0.8064          | 0.6440             | 0.0823          | -0.7177              | -0.7186                | -173.1074      | -202.8401    | -0.0312         | -0.6399       |
 
 
 ### Framework versions

all_results.json

@@ -1,23 +1,23 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.249445676803589,
-    "eval_logits/rejected": -2.085613965988159,
-    "eval_logps/chosen": -165.2006378173828,
-    "eval_logps/rejected": -131.76806640625,
-    "eval_loss": 978.6270751953125,
-    "eval_rewards/accuracies": 0.6600136756896973,
-    "eval_rewards/chosen": -0.24851088225841522,
-    "eval_rewards/margins": 0.04783935099840164,
-    "eval_rewards/rejected": -0.2963502109050751,
-    "eval_rewards/safe_rewards": -0.24501556158065796,
-    "eval_rewards/unsafe_rewards": -0.2426704317331314,
-    "eval_runtime": 2349.9274,
-    "eval_samples": 35044,
-    "eval_samples_per_second": 14.913,
-    "eval_steps_per_second": 0.466,
-    "train_loss": 1500.3740138506566,
-    "train_runtime": 40588.9258,
-    "train_samples": 120613,
-    "train_samples_per_second": 2.972,
-    "train_steps_per_second": 0.093
+    "eval_logits/chosen": -0.6150549054145813,
+    "eval_logits/rejected": 0.035322826355695724,
+    "eval_logps/chosen": -206.2398223876953,
+    "eval_logps/rejected": -174.71498107910156,
+    "eval_loss": 4336.228515625,
+    "eval_rewards/accuracies": 0.6084221005439758,
+    "eval_rewards/chosen": -0.7580091953277588,
+    "eval_rewards/margins": 0.06442829966545105,
+    "eval_rewards/rejected": -0.8224374651908875,
+    "eval_rewards/safe_rewards": -0.7526479363441467,
+    "eval_rewards/unsafe_rewards": -0.755587637424469,
+    "eval_runtime": 1044.2833,
+    "eval_samples": 33044,
+    "eval_samples_per_second": 31.643,
+    "eval_steps_per_second": 0.989,
+    "train_loss": 11837.785513657158,
+    "train_runtime": 18541.4057,
+    "train_samples": 59478,
+    "train_samples_per_second": 3.208,
+    "train_steps_per_second": 0.05
 }

eval_results.json

@@ -1,18 +1,18 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.249445676803589,
-    "eval_logits/rejected": -2.085613965988159,
-    "eval_logps/chosen": -165.2006378173828,
-    "eval_logps/rejected": -131.76806640625,
-    "eval_loss": 978.6270751953125,
-    "eval_rewards/accuracies": 0.6600136756896973,
-    "eval_rewards/chosen": -0.24851088225841522,
-    "eval_rewards/margins": 0.04783935099840164,
-    "eval_rewards/rejected": -0.2963502109050751,
-    "eval_rewards/safe_rewards": -0.24501556158065796,
-    "eval_rewards/unsafe_rewards": -0.2426704317331314,
-    "eval_runtime": 2349.9274,
-    "eval_samples": 35044,
-    "eval_samples_per_second": 14.913,
-    "eval_steps_per_second": 0.466
+    "eval_logits/chosen": -0.6150549054145813,
+    "eval_logits/rejected": 0.035322826355695724,
+    "eval_logps/chosen": -206.2398223876953,
+    "eval_logps/rejected": -174.71498107910156,
+    "eval_loss": 4336.228515625,
+    "eval_rewards/accuracies": 0.6084221005439758,
+    "eval_rewards/chosen": -0.7580091953277588,
+    "eval_rewards/margins": 0.06442829966545105,
+    "eval_rewards/rejected": -0.8224374651908875,
+    "eval_rewards/safe_rewards": -0.7526479363441467,
+    "eval_rewards/unsafe_rewards": -0.755587637424469,
+    "eval_runtime": 1044.2833,
+    "eval_samples": 33044,
+    "eval_samples_per_second": 31.643,
+    "eval_steps_per_second": 0.989
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:66ac9d413cabbb2c76c373a804e7399722f74af000828063d871e6b6058bd8c2
+oid sha256:a91d9ba47e204876c0ac101e2ef27a0c27d68c135aebaa5c8a60a7e531bd0cfb
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e930cd85138928e973fe4ea05bc40656880d53ed562dc4340a7a5331c45c49af
+oid sha256:3e0125b6268e71d418c96272816b306e8a8ecbeb69a9581073acf5bc4a717edf
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c1bbe49aa0fb4bf303388388ba17a7836933dd8e445a9e048d71ee389b9399a0
+oid sha256:bddfd7df2b88d22d9d6b171d69917106ac0155c7c988d14c73740f38f92e0c13
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 1500.3740138506566,
-    "train_runtime": 40588.9258,
-    "train_samples": 120613,
-    "train_samples_per_second": 2.972,
-    "train_steps_per_second": 0.093
+    "train_loss": 11837.785513657158,
+    "train_runtime": 18541.4057,
+    "train_samples": 59478,
+    "train_samples_per_second": 3.208,
+    "train_steps_per_second": 0.05
 }