Model save

README.md

@@ -15,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6636
-- Rewards/chosen: 0.0135
-- Rewards/rejected: -0.0160
-- Rewards/accuracies: 0.6280
-- Rewards/margins: 0.0295
-- Logps/rejected: -259.4594
-- Logps/chosen: -284.1223
-- Logits/rejected: -2.8462
-- Logits/chosen: -2.8424
-- Use Label: 11234.4961
-- Pred Label: 4797.5039
+- Loss: 0.6618
+- Rewards/chosen: 0.0116
+- Rewards/rejected: -0.0133
+- Rewards/accuracies: 0.6150
+- Rewards/margins: 0.0249
+- Logps/rejected: -262.4203
+- Logps/chosen: -281.8592
+- Logits/rejected: -2.8344
+- Logits/chosen: -2.8413
+- Use Label: 18709.0195
+- Pred Label: 13352.9805
 
 ## Model description
 
@@ -49,10 +49,10 @@ The following hyperparameters were used during training:
 - eval_batch_size: 4
 - seed: 42
 - distributed_type: multi-GPU
-- num_devices: 4
-- gradient_accumulation_steps: 4
+- num_devices: 2
+- gradient_accumulation_steps: 8
 - total_train_batch_size: 64
-- total_eval_batch_size: 16
+- total_eval_batch_size: 8
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_ratio: 0.1
@@ -62,7 +62,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Use Label  | Pred Label |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:----------:|:----------:|
-| 0.6641        | 1.0   | 955  | 0.6636          | 0.0135         | -0.0160          | 0.6280             | 0.0295          | -259.4594      | -284.1223    | -2.8462         | -2.8424       | 10931.4961 | 4600.5039  |
+| 0.6628        | 1.0   | 955  | 0.6618          | 0.0116         | -0.0133          | 0.6150             | 0.0249          | -262.4203      | -281.8592    | -2.8344         | -2.8413       | 18206.0195 | 12855.9805 |
 
 
 ### Framework versions

adapter_config.json

@@ -8,17 +8,20 @@
   "init_lora_weights": true,
   "layers_pattern": null,
   "layers_to_transform": null,
+  "loftq_config": {},
   "lora_alpha": 16,
   "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
   "r": 64,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "v_proj",
     "k_proj",
     "q_proj",
+    "v_proj",
     "o_proj"
   ],
   "task_type": "CAUSAL_LM"

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8c5c3c2a4be0545137a1202ae0b1a9fe0d6a49f4bb346ccad9136b0b5be4694
-size 218138576
+oid sha256:84a9c6afaad46949167d2bf8c0ff031030a8d02f1006b050e47642be289c99b1
+size 109086672

all_results.json

@@ -1,23 +1,23 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.842418670654297,
-    "eval_logits/rejected": -2.846235752105713,
-    "eval_logps/chosen": -284.122314453125,
-    "eval_logps/rejected": -259.4594421386719,
-    "eval_loss": 0.6635700464248657,
-    "eval_pred_label": 4797.50390625,
-    "eval_rewards/accuracies": 0.628000020980835,
-    "eval_rewards/chosen": 0.013506044633686543,
-    "eval_rewards/margins": 0.029479000717401505,
-    "eval_rewards/rejected": -0.015972958877682686,
-    "eval_runtime": 439.3222,
+    "eval_logits/chosen": -2.841262102127075,
+    "eval_logits/rejected": -2.8343887329101562,
+    "eval_logps/chosen": -281.85919189453125,
+    "eval_logps/rejected": -262.4202880859375,
+    "eval_loss": 0.6618225574493408,
+    "eval_pred_label": 13352.98046875,
+    "eval_rewards/accuracies": 0.6150000095367432,
+    "eval_rewards/chosen": 0.011613711714744568,
+    "eval_rewards/margins": 0.02489962987601757,
+    "eval_rewards/rejected": -0.013285920023918152,
+    "eval_runtime": 834.3056,
     "eval_samples": 2000,
-    "eval_samples_per_second": 4.552,
-    "eval_steps_per_second": 0.285,
-    "eval_use_label": 11234.49609375,
-    "train_loss": 0.6728555943953429,
-    "train_runtime": 24272.064,
+    "eval_samples_per_second": 2.397,
+    "eval_steps_per_second": 0.3,
+    "eval_use_label": 18709.01953125,
+    "train_loss": 0.6705795382954063,
+    "train_runtime": 45840.6595,
     "train_samples": 61135,
-    "train_samples_per_second": 2.519,
-    "train_steps_per_second": 0.039
+    "train_samples_per_second": 1.334,
+    "train_steps_per_second": 0.021
 }

eval_results.json

@@ -1,18 +1,18 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.842418670654297,
-    "eval_logits/rejected": -2.846235752105713,
-    "eval_logps/chosen": -284.122314453125,
-    "eval_logps/rejected": -259.4594421386719,
-    "eval_loss": 0.6635700464248657,
-    "eval_pred_label": 4797.50390625,
-    "eval_rewards/accuracies": 0.628000020980835,
-    "eval_rewards/chosen": 0.013506044633686543,
-    "eval_rewards/margins": 0.029479000717401505,
-    "eval_rewards/rejected": -0.015972958877682686,
-    "eval_runtime": 439.3222,
+    "eval_logits/chosen": -2.841262102127075,
+    "eval_logits/rejected": -2.8343887329101562,
+    "eval_logps/chosen": -281.85919189453125,
+    "eval_logps/rejected": -262.4202880859375,
+    "eval_loss": 0.6618225574493408,
+    "eval_pred_label": 13352.98046875,
+    "eval_rewards/accuracies": 0.6150000095367432,
+    "eval_rewards/chosen": 0.011613711714744568,
+    "eval_rewards/margins": 0.02489962987601757,
+    "eval_rewards/rejected": -0.013285920023918152,
+    "eval_runtime": 834.3056,
     "eval_samples": 2000,
-    "eval_samples_per_second": 4.552,
-    "eval_steps_per_second": 0.285,
-    "eval_use_label": 11234.49609375
+    "eval_samples_per_second": 2.397,
+    "eval_steps_per_second": 0.3,
+    "eval_use_label": 18709.01953125
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6728555943953429,
-    "train_runtime": 24272.064,
+    "train_loss": 0.6705795382954063,
+    "train_runtime": 45840.6595,
     "train_samples": 61135,
-    "train_samples_per_second": 2.519,
-    "train_steps_per_second": 0.039
+    "train_samples_per_second": 1.334,
+    "train_steps_per_second": 0.021
 }

trainer_state.json

@@ -11,10 +11,10 @@
     {
       "epoch": 0.0,
       "learning_rate": 5.208333333333333e-09,
-      "logits/chosen": -2.980285167694092,
-      "logits/rejected": -2.87275767326355,
-      "logps/chosen": -313.4390563964844,
-      "logps/rejected": -236.1754150390625,
+      "logits/chosen": -2.899709463119507,
+      "logits/rejected": -2.879509687423706,
+      "logps/chosen": -314.8815612792969,
+      "logps/rejected": -239.785888671875,
       "loss": 0.6931,
       "pred_label": 0.0,
       "rewards/accuracies": 0.0,
@@ -22,178 +22,178 @@
       "rewards/margins": 0.0,
       "rewards/rejected": 0.0,
       "step": 1,
-      "use_label": 10.0
+      "use_label": 18.0
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.976717112922002e-07,
-      "logits/chosen": -2.8194870948791504,
-      "logits/rejected": -2.8288567066192627,
-      "logps/chosen": -285.2724304199219,
-      "logps/rejected": -270.956298828125,
-      "loss": 0.6838,
-      "pred_label": 150.7020263671875,
-      "rewards/accuracies": 0.5050504803657532,
-      "rewards/chosen": 0.000817809603177011,
-      "rewards/margins": 0.0014873194741085172,
-      "rewards/rejected": -0.0006695101037621498,
+      "logits/chosen": -2.8282251358032227,
+      "logits/rejected": -2.8269264698028564,
+      "logps/chosen": -285.5703430175781,
+      "logps/rejected": -267.9831237792969,
+      "loss": 0.6805,
+      "pred_label": 479.7752380371094,
+      "rewards/accuracies": 0.5012626051902771,
+      "rewards/chosen": 0.0007524320390075445,
+      "rewards/margins": 0.0014539315598085523,
+      "rewards/rejected": -0.0007014995790086687,
       "step": 100,
-      "use_label": 659.2979736328125
+      "use_label": 1138.2247314453125
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.3946449359720607e-07,
-      "logits/chosen": -2.828075647354126,
-      "logits/rejected": -2.816530227661133,
-      "logps/chosen": -278.7549133300781,
-      "logps/rejected": -260.5694274902344,
-      "loss": 0.683,
-      "pred_label": 452.552490234375,
-      "rewards/accuracies": 0.534375011920929,
-      "rewards/chosen": 0.002707230392843485,
-      "rewards/margins": 0.005774380639195442,
-      "rewards/rejected": -0.003067150479182601,
+      "logits/chosen": -2.829744338989258,
+      "logits/rejected": -2.8234996795654297,
+      "logps/chosen": -283.5379333496094,
+      "logps/rejected": -264.45965576171875,
+      "loss": 0.6797,
+      "pred_label": 1464.762451171875,
+      "rewards/accuracies": 0.5268750190734863,
+      "rewards/chosen": 0.0027550552040338516,
+      "rewards/margins": 0.004276593215763569,
+      "rewards/rejected": -0.0015215380117297173,
       "step": 200,
-      "use_label": 1949.447509765625
+      "use_label": 3337.237548828125
     },
     {
       "epoch": 0.31,
       "learning_rate": 3.812572759022118e-07,
-      "logits/chosen": -2.8141846656799316,
-      "logits/rejected": -2.8159701824188232,
-      "logps/chosen": -284.0125732421875,
-      "logps/rejected": -253.9112091064453,
-      "loss": 0.6807,
-      "pred_label": 775.85498046875,
-      "rewards/accuracies": 0.5575000047683716,
-      "rewards/chosen": 0.005504029802978039,
-      "rewards/margins": 0.009370613843202591,
-      "rewards/rejected": -0.0038665838073939085,
+      "logits/chosen": -2.822391986846924,
+      "logits/rejected": -2.821011543273926,
+      "logps/chosen": -290.09552001953125,
+      "logps/rejected": -260.20050048828125,
+      "loss": 0.678,
+      "pred_label": 2522.33740234375,
+      "rewards/accuracies": 0.546875,
+      "rewards/chosen": 0.004526687320321798,
+      "rewards/margins": 0.008910334669053555,
+      "rewards/rejected": -0.004383646883070469,
       "step": 300,
-      "use_label": 3226.14501953125
+      "use_label": 5479.66259765625
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.230500582072177e-07,
-      "logits/chosen": -2.826817512512207,
-      "logits/rejected": -2.8094358444213867,
-      "logps/chosen": -284.3566589355469,
-      "logps/rejected": -262.80731201171875,
-      "loss": 0.6769,
-      "pred_label": 1149.0574951171875,
-      "rewards/accuracies": 0.5774999856948853,
-      "rewards/chosen": 0.007384983357042074,
-      "rewards/margins": 0.015422500669956207,
-      "rewards/rejected": -0.008037514984607697,
+      "logits/chosen": -2.8350696563720703,
+      "logits/rejected": -2.8237721920013428,
+      "logps/chosen": -284.8573303222656,
+      "logps/rejected": -260.8428039550781,
+      "loss": 0.6752,
+      "pred_label": 3636.47509765625,
+      "rewards/accuracies": 0.5778124928474426,
+      "rewards/chosen": 0.00710176769644022,
+      "rewards/margins": 0.013848603703081608,
+      "rewards/rejected": -0.006746836472302675,
       "step": 400,
-      "use_label": 4452.9423828125
+      "use_label": 7565.52490234375
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.648428405122235e-07,
-      "logits/chosen": -2.807734966278076,
-      "logits/rejected": -2.796409845352173,
-      "logps/chosen": -269.9852600097656,
-      "logps/rejected": -252.07232666015625,
-      "loss": 0.6728,
-      "pred_label": 1592.5675048828125,
-      "rewards/accuracies": 0.5756250023841858,
-      "rewards/chosen": 0.006774631794542074,
-      "rewards/margins": 0.01554279588162899,
-      "rewards/rejected": -0.008768163621425629,
+      "logits/chosen": -2.8218374252319336,
+      "logits/rejected": -2.810873508453369,
+      "logps/chosen": -281.2003173828125,
+      "logps/rejected": -257.1551818847656,
+      "loss": 0.6707,
+      "pred_label": 4911.896484375,
+      "rewards/accuracies": 0.5731250047683716,
+      "rewards/chosen": 0.008172390051186085,
+      "rewards/margins": 0.017216255888342857,
+      "rewards/rejected": -0.009043867699801922,
       "step": 500,
-      "use_label": 5609.4326171875
+      "use_label": 9490.103515625
     },
     {
       "epoch": 0.63,
       "learning_rate": 2.0663562281722933e-07,
-      "logits/chosen": -2.8339650630950928,
-      "logits/rejected": -2.82075572013855,
-      "logps/chosen": -285.0927734375,
-      "logps/rejected": -265.4134826660156,
-      "loss": 0.6681,
-      "pred_label": 2111.6650390625,
-      "rewards/accuracies": 0.6206250190734863,
-      "rewards/chosen": 0.013815036043524742,
-      "rewards/margins": 0.0289370846003294,
-      "rewards/rejected": -0.015122047625482082,
+      "logits/chosen": -2.8341524600982666,
+      "logits/rejected": -2.8230907917022705,
+      "logps/chosen": -284.7864685058594,
+      "logps/rejected": -262.0230712890625,
+      "loss": 0.6665,
+      "pred_label": 6349.55859375,
+      "rewards/accuracies": 0.6025000214576721,
+      "rewards/chosen": 0.012129506096243858,
+      "rewards/margins": 0.025582188740372658,
+      "rewards/rejected": -0.0134526826441288,
       "step": 600,
-      "use_label": 6690.3349609375
+      "use_label": 11252.44140625
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.4842840512223514e-07,
-      "logits/chosen": -2.827232599258423,
-      "logits/rejected": -2.811751127243042,
-      "logps/chosen": -281.4178771972656,
-      "logps/rejected": -248.81068420410156,
-      "loss": 0.6659,
-      "pred_label": 2680.2724609375,
-      "rewards/accuracies": 0.6200000047683716,
-      "rewards/chosen": 0.01417633332312107,
-      "rewards/margins": 0.029135096818208694,
-      "rewards/rejected": -0.014958759769797325,
+      "logits/chosen": -2.8274898529052734,
+      "logits/rejected": -2.811511516571045,
+      "logps/chosen": -282.0050048828125,
+      "logps/rejected": -252.4735565185547,
+      "loss": 0.6639,
+      "pred_label": 7854.15869140625,
+      "rewards/accuracies": 0.6112499833106995,
+      "rewards/chosen": 0.013030249625444412,
+      "rewards/margins": 0.026329634711146355,
+      "rewards/rejected": -0.013299385085701942,
       "step": 700,
-      "use_label": 7721.7275390625
+      "use_label": 12947.8408203125
     },
     {
       "epoch": 0.84,
       "learning_rate": 9.022118742724097e-08,
-      "logits/chosen": -2.8300516605377197,
-      "logits/rejected": -2.835542678833008,
-      "logps/chosen": -288.608642578125,
-      "logps/rejected": -261.5773010253906,
-      "loss": 0.6646,
-      "pred_label": 3286.232421875,
-      "rewards/accuracies": 0.6200000047683716,
-      "rewards/chosen": 0.014839441515505314,
-      "rewards/margins": 0.03136582300066948,
-      "rewards/rejected": -0.01652638241648674,
+      "logits/chosen": -2.832928419113159,
+      "logits/rejected": -2.8318238258361816,
+      "logps/chosen": -285.62213134765625,
+      "logps/rejected": -259.8959045410156,
+      "loss": 0.662,
+      "pred_label": 9441.02734375,
+      "rewards/accuracies": 0.6253125071525574,
+      "rewards/chosen": 0.01489347591996193,
+      "rewards/margins": 0.030955424532294273,
+      "rewards/rejected": -0.016061950474977493,
       "step": 800,
-      "use_label": 8715.767578125
+      "use_label": 14560.97265625
     },
     {
       "epoch": 0.94,
       "learning_rate": 3.20139697322468e-08,
-      "logits/chosen": -2.8211710453033447,
-      "logits/rejected": -2.8280835151672363,
-      "logps/chosen": -277.363525390625,
-      "logps/rejected": -256.4843444824219,
-      "loss": 0.6641,
-      "pred_label": 3882.75244140625,
-      "rewards/accuracies": 0.6331250071525574,
-      "rewards/chosen": 0.01446867547929287,
-      "rewards/margins": 0.02932187356054783,
-      "rewards/rejected": -0.014853193424642086,
+      "logits/chosen": -2.824517250061035,
+      "logits/rejected": -2.830662250518799,
+      "logps/chosen": -276.052490234375,
+      "logps/rejected": -262.5692138671875,
+      "loss": 0.6628,
+      "pred_label": 10994.255859375,
+      "rewards/accuracies": 0.6143749952316284,
+      "rewards/chosen": 0.012237527407705784,
+      "rewards/margins": 0.026263901963829994,
+      "rewards/rejected": -0.01402637455612421,
       "step": 900,
-      "use_label": 9719.2470703125
+      "use_label": 16207.744140625
     },
     {
       "epoch": 1.0,
-      "eval_logits/chosen": -2.842418670654297,
-      "eval_logits/rejected": -2.846235752105713,
-      "eval_logps/chosen": -284.122314453125,
-      "eval_logps/rejected": -259.4594421386719,
-      "eval_loss": 0.6635700464248657,
-      "eval_pred_label": 4600.50390625,
-      "eval_rewards/accuracies": 0.628000020980835,
-      "eval_rewards/chosen": 0.013506044633686543,
-      "eval_rewards/margins": 0.029479000717401505,
-      "eval_rewards/rejected": -0.015972958877682686,
-      "eval_runtime": 438.8322,
-      "eval_samples_per_second": 4.558,
-      "eval_steps_per_second": 0.285,
-      "eval_use_label": 10931.49609375,
+      "eval_logits/chosen": -2.841262102127075,
+      "eval_logits/rejected": -2.8343887329101562,
+      "eval_logps/chosen": -281.85919189453125,
+      "eval_logps/rejected": -262.4202880859375,
+      "eval_loss": 0.6618225574493408,
+      "eval_pred_label": 12855.98046875,
+      "eval_rewards/accuracies": 0.6150000095367432,
+      "eval_rewards/chosen": 0.011613711714744568,
+      "eval_rewards/margins": 0.02489962987601757,
+      "eval_rewards/rejected": -0.013285920023918152,
+      "eval_runtime": 826.7995,
+      "eval_samples_per_second": 2.419,
+      "eval_steps_per_second": 0.302,
+      "eval_use_label": 18206.01953125,
       "step": 955
     },
     {
       "epoch": 1.0,
       "step": 955,
       "total_flos": 0.0,
-      "train_loss": 0.6728555943953429,
-      "train_runtime": 24272.064,
-      "train_samples_per_second": 2.519,
-      "train_steps_per_second": 0.039
+      "train_loss": 0.6705795382954063,
+      "train_runtime": 45840.6595,
+      "train_samples_per_second": 1.334,
+      "train_steps_per_second": 0.021
     }
   ],
   "logging_steps": 100,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9103d537bc7a42b0df74010143412e6ee65758ade8a9f94ad18972e25f5db07a
+oid sha256:0cc76df2851d4146f789fd37f9db61180778b9f7f4bb177bcab529cc595e76d3
 size 4792