Model save

README.md

@@ -15,15 +15,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6433
-- Rewards/chosen: 0.0460
-- Rewards/rejected: -0.0949
-- Rewards/accuracies: 0.6740
-- Rewards/margins: 0.1409
-- Logps/rejected: -260.2492
-- Logps/chosen: -283.7975
-- Logits/rejected: -2.8439
-- Logits/chosen: -2.8406
+- Loss: 0.5656
+- Rewards/chosen: 0.0673
+- Rewards/rejected: -0.5344
+- Rewards/accuracies: 0.7000
+- Rewards/margins: 0.6018
+- Logps/rejected: -264.6442
+- Logps/chosen: -283.5841
+- Logits/rejected: -2.8292
+- Logits/chosen: -2.8303
 - Use Label: 0.0
 - Pred Label: 0.0
 
@@ -44,7 +44,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 1e-06
+- learning_rate: 5e-06
 - train_batch_size: 4
 - eval_batch_size: 4
 - seed: 42
@@ -62,7 +62,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Use Label | Pred Label |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:---------:|:----------:|
-| 0.6441        | 1.0   | 955  | 0.6433          | 0.0460         | -0.0949          | 0.6740             | 0.1409          | -260.2492      | -283.7975    | -2.8439         | -2.8406       | 0.0       | 0.0        |
+| 0.5689        | 1.0   | 955  | 0.5656          | 0.0673         | -0.5344          | 0.7000             | 0.6018          | -264.6442      | -283.5841    | -2.8292         | -2.8303       | 0.0       | 0.0        |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:16a42c9d513479f3ee0def22bf154bd218df1ffcf9dc3c0fc98982f6705e3e3b
+oid sha256:52d5571e6c85d40b4ab603d26a4d7d7716eaa1a22f8e22bebc933ad16dbbb8ae
 size 218138576

all_results.json

@@ -1,23 +1,23 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.8405630588531494,
-    "eval_logits/rejected": -2.8439478874206543,
-    "eval_logps/chosen": -283.7974548339844,
-    "eval_logps/rejected": -260.2491760253906,
-    "eval_loss": 0.6432895660400391,
+    "eval_logits/chosen": -2.8302695751190186,
+    "eval_logits/rejected": -2.8291618824005127,
+    "eval_logps/chosen": -283.5841369628906,
+    "eval_logps/rejected": -264.64422607421875,
+    "eval_loss": 0.5655555129051208,
     "eval_pred_label": 0.0,
-    "eval_rewards/accuracies": 0.6740000247955322,
-    "eval_rewards/chosen": 0.04599040374159813,
-    "eval_rewards/margins": 0.14093737304210663,
-    "eval_rewards/rejected": -0.0949469730257988,
-    "eval_runtime": 468.6797,
+    "eval_rewards/accuracies": 0.699999988079071,
+    "eval_rewards/chosen": 0.06732505559921265,
+    "eval_rewards/margins": 0.6017746329307556,
+    "eval_rewards/rejected": -0.534449577331543,
+    "eval_runtime": 457.1465,
     "eval_samples": 2000,
-    "eval_samples_per_second": 4.267,
-    "eval_steps_per_second": 0.267,
+    "eval_samples_per_second": 4.375,
+    "eval_steps_per_second": 0.273,
     "eval_use_label": 0.0,
-    "train_loss": 0.660746982334796,
-    "train_runtime": 25629.2924,
+    "train_loss": 0.5870625535855118,
+    "train_runtime": 25384.4334,
     "train_samples": 61135,
-    "train_samples_per_second": 2.385,
-    "train_steps_per_second": 0.037
+    "train_samples_per_second": 2.408,
+    "train_steps_per_second": 0.038
 }

eval_results.json

@@ -1,18 +1,18 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.8405630588531494,
-    "eval_logits/rejected": -2.8439478874206543,
-    "eval_logps/chosen": -283.7974548339844,
-    "eval_logps/rejected": -260.2491760253906,
-    "eval_loss": 0.6432895660400391,
+    "eval_logits/chosen": -2.8302695751190186,
+    "eval_logits/rejected": -2.8291618824005127,
+    "eval_logps/chosen": -283.5841369628906,
+    "eval_logps/rejected": -264.64422607421875,
+    "eval_loss": 0.5655555129051208,
     "eval_pred_label": 0.0,
-    "eval_rewards/accuracies": 0.6740000247955322,
-    "eval_rewards/chosen": 0.04599040374159813,
-    "eval_rewards/margins": 0.14093737304210663,
-    "eval_rewards/rejected": -0.0949469730257988,
-    "eval_runtime": 468.6797,
+    "eval_rewards/accuracies": 0.699999988079071,
+    "eval_rewards/chosen": 0.06732505559921265,
+    "eval_rewards/margins": 0.6017746329307556,
+    "eval_rewards/rejected": -0.534449577331543,
+    "eval_runtime": 457.1465,
     "eval_samples": 2000,
-    "eval_samples_per_second": 4.267,
-    "eval_steps_per_second": 0.267,
+    "eval_samples_per_second": 4.375,
+    "eval_steps_per_second": 0.273,
     "eval_use_label": 0.0
 }

runs/Jan21_14-07-55_uclaml04.cs.ucla.edu/events.out.tfevents.1705874931.uclaml04.cs.ucla.edu.725178.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dca9082b509c88d4d54225a59df1519b7a85df4f984f73d0fadda08a6aaa13a6
-size 11838
+oid sha256:47c97b55e57bc0004ea9c1ec4e8880902f5488120bd4caddf86cb78fedfcc63f
+size 13039

runs/Jan21_14-07-55_uclaml04.cs.ucla.edu/events.out.tfevents.1705900772.uclaml04.cs.ucla.edu.725178.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:866b58e1ab9cceff104815835a51153047f74404ef944895a1dbbf5ab4c7b723
+size 935

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.660746982334796,
-    "train_runtime": 25629.2924,
+    "train_loss": 0.5870625535855118,
+    "train_runtime": 25384.4334,
     "train_samples": 61135,
-    "train_samples_per_second": 2.385,
-    "train_steps_per_second": 0.037
+    "train_samples_per_second": 2.408,
+    "train_steps_per_second": 0.038
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "learning_rate": 1.0416666666666666e-08,
+      "learning_rate": 5.208333333333333e-08,
       "logits/chosen": -2.980285167694092,
       "logits/rejected": -2.87275767326355,
       "logps/chosen": -313.4390563964844,
@@ -26,163 +26,163 @@
     },
     {
       "epoch": 0.1,
-      "learning_rate": 9.953434225844004e-07,
-      "logits/chosen": -2.8195509910583496,
-      "logits/rejected": -2.8288235664367676,
-      "logps/chosen": -285.27215576171875,
-      "logps/rejected": -270.96209716796875,
-      "loss": 0.6923,
+      "learning_rate": 4.9767171129220025e-06,
+      "logits/chosen": -2.8189077377319336,
+      "logits/rejected": -2.828260660171509,
+      "logps/chosen": -285.1784362792969,
+      "logps/rejected": -271.1445617675781,
+      "loss": 0.6803,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.5183081030845642,
-      "rewards/chosen": 0.0008437540382146835,
-      "rewards/margins": 0.0020915681961923838,
-      "rewards/rejected": -0.0012478141579777002,
+      "rewards/accuracies": 0.5738636255264282,
+      "rewards/chosen": 0.01021653600037098,
+      "rewards/margins": 0.029710056260228157,
+      "rewards/rejected": -0.019493522122502327,
       "step": 100,
       "use_label": 0.0
     },
     {
       "epoch": 0.21,
-      "learning_rate": 8.789289871944121e-07,
-      "logits/chosen": -2.82773494720459,
-      "logits/rejected": -2.816188335418701,
-      "logps/chosen": -278.6949768066406,
-      "logps/rejected": -260.6236572265625,
-      "loss": 0.6855,
+      "learning_rate": 4.394644935972061e-06,
+      "logits/chosen": -2.8255398273468018,
+      "logits/rejected": -2.813203811645508,
+      "logps/chosen": -278.2582092285156,
+      "logps/rejected": -262.1144714355469,
+      "loss": 0.6236,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.5956249833106995,
-      "rewards/chosen": 0.00869889184832573,
-      "rewards/margins": 0.017191100865602493,
-      "rewards/rejected": -0.008492209017276764,
+      "rewards/accuracies": 0.6868749856948853,
+      "rewards/chosen": 0.05237884446978569,
+      "rewards/margins": 0.20995216071605682,
+      "rewards/rejected": -0.15757331252098083,
       "step": 200,
       "use_label": 0.0
     },
     {
       "epoch": 0.31,
-      "learning_rate": 7.625145518044237e-07,
-      "logits/chosen": -2.813408613204956,
-      "logits/rejected": -2.8151955604553223,
-      "logps/chosen": -283.8744812011719,
-      "logps/rejected": -254.1025390625,
-      "loss": 0.675,
+      "learning_rate": 3.812572759022119e-06,
+      "logits/chosen": -2.809633255004883,
+      "logits/rejected": -2.80940580368042,
+      "logps/chosen": -283.6641540527344,
+      "logps/rejected": -257.23779296875,
+      "loss": 0.5901,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.653124988079071,
-      "rewards/chosen": 0.01931346394121647,
-      "rewards/margins": 0.042313817888498306,
-      "rewards/rejected": -0.023000353947281837,
+      "rewards/accuracies": 0.6956250071525574,
+      "rewards/chosen": 0.04034877195954323,
+      "rewards/margins": 0.3768764138221741,
+      "rewards/rejected": -0.33652764558792114,
       "step": 300,
       "use_label": 0.0
     },
     {
       "epoch": 0.42,
-      "learning_rate": 6.461001164144354e-07,
-      "logits/chosen": -2.8259572982788086,
-      "logits/rejected": -2.808323383331299,
-      "logps/chosen": -284.1655578613281,
-      "logps/rejected": -263.1576843261719,
-      "loss": 0.6649,
+      "learning_rate": 3.2305005820721774e-06,
+      "logits/chosen": -2.821267604827881,
+      "logits/rejected": -2.8011868000030518,
+      "logps/chosen": -284.2022399902344,
+      "logps/rejected": -267.48358154296875,
+      "loss": 0.5757,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.6837499737739563,
-      "rewards/chosen": 0.026492329314351082,
-      "rewards/margins": 0.06956954300403595,
-      "rewards/rejected": -0.04307721555233002,
+      "rewards/accuracies": 0.7081249952316284,
+      "rewards/chosen": 0.022827474400401115,
+      "rewards/margins": 0.4984941780567169,
+      "rewards/rejected": -0.47566673159599304,
       "step": 400,
       "use_label": 0.0
     },
     {
       "epoch": 0.52,
-      "learning_rate": 5.29685681024447e-07,
-      "logits/chosen": -2.806447744369507,
-      "logits/rejected": -2.794842004776001,
-      "logps/chosen": -269.8316345214844,
-      "logps/rejected": -252.43292236328125,
-      "loss": 0.6596,
+      "learning_rate": 2.6484284051222353e-06,
+      "logits/chosen": -2.7993886470794678,
+      "logits/rejected": -2.7845959663391113,
+      "logps/chosen": -270.09637451171875,
+      "logps/rejected": -256.13458251953125,
+      "loss": 0.5798,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.628125011920929,
-      "rewards/chosen": 0.022135533392429352,
-      "rewards/margins": 0.06696329265832901,
-      "rewards/rejected": -0.04482775181531906,
+      "rewards/accuracies": 0.6612499952316284,
+      "rewards/chosen": -0.004339172504842281,
+      "rewards/margins": 0.4106566905975342,
+      "rewards/rejected": -0.41499578952789307,
       "step": 500,
       "use_label": 0.0
     },
     {
       "epoch": 0.63,
-      "learning_rate": 4.1327124563445866e-07,
-      "logits/chosen": -2.8327181339263916,
-      "logits/rejected": -2.819202184677124,
-      "logps/chosen": -284.8338623046875,
-      "logps/rejected": -265.977294921875,
-      "loss": 0.6493,
+      "learning_rate": 2.0663562281722936e-06,
+      "logits/chosen": -2.824444532394409,
+      "logits/rejected": -2.808506965637207,
+      "logps/chosen": -284.5413513183594,
+      "logps/rejected": -270.1356201171875,
+      "loss": 0.5643,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.6675000190734863,
-      "rewards/chosen": 0.03970767930150032,
-      "rewards/margins": 0.11121242493391037,
-      "rewards/rejected": -0.07150473445653915,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.06895674020051956,
+      "rewards/margins": 0.5562920570373535,
+      "rewards/rejected": -0.4873352348804474,
       "step": 600,
       "use_label": 0.0
     },
     {
       "epoch": 0.73,
-      "learning_rate": 2.968568102444703e-07,
-      "logits/chosen": -2.8258111476898193,
-      "logits/rejected": -2.8097949028015137,
-      "logps/chosen": -281.1001892089844,
-      "logps/rejected": -249.4017791748047,
-      "loss": 0.6463,
+      "learning_rate": 1.4842840512223516e-06,
+      "logits/chosen": -2.8160746097564697,
+      "logits/rejected": -2.795208692550659,
+      "logps/chosen": -280.79766845703125,
+      "logps/rejected": -253.41058349609375,
+      "loss": 0.5631,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.6806250214576721,
-      "rewards/chosen": 0.04594804719090462,
-      "rewards/margins": 0.12001516669988632,
-      "rewards/rejected": -0.0740671157836914,
+      "rewards/accuracies": 0.7231249809265137,
+      "rewards/chosen": 0.07620371133089066,
+      "rewards/margins": 0.5511507391929626,
+      "rewards/rejected": -0.474947065114975,
       "step": 700,
       "use_label": 0.0
     },
     {
       "epoch": 0.84,
-      "learning_rate": 1.8044237485448194e-07,
-      "logits/chosen": -2.82832407951355,
-      "logits/rejected": -2.833012580871582,
-      "logps/chosen": -288.21307373046875,
-      "logps/rejected": -262.298095703125,
-      "loss": 0.64,
+      "learning_rate": 9.022118742724098e-07,
+      "logits/chosen": -2.8168938159942627,
+      "logits/rejected": -2.8180816173553467,
+      "logps/chosen": -287.77679443359375,
+      "logps/rejected": -266.3511657714844,
+      "loss": 0.5548,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.6831250190734863,
-      "rewards/chosen": 0.05439276248216629,
-      "rewards/margins": 0.14299620687961578,
-      "rewards/rejected": -0.08860345184803009,
+      "rewards/accuracies": 0.7112500071525574,
+      "rewards/chosen": 0.09802371263504028,
+      "rewards/margins": 0.5919383764266968,
+      "rewards/rejected": -0.49391472339630127,
       "step": 800,
       "use_label": 0.0
     },
     {
       "epoch": 0.94,
-      "learning_rate": 6.40279394644936e-08,
-      "logits/chosen": -2.8194103240966797,
-      "logits/rejected": -2.8261044025421143,
-      "logps/chosen": -277.05517578125,
-      "logps/rejected": -257.2059020996094,
-      "loss": 0.6441,
+      "learning_rate": 3.2013969732246806e-07,
+      "logits/chosen": -2.808311700820923,
+      "logits/rejected": -2.812298536300659,
+      "logps/chosen": -276.976806640625,
+      "logps/rejected": -261.3187561035156,
+      "loss": 0.5689,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.6762499809265137,
-      "rewards/chosen": 0.04530614987015724,
-      "rewards/margins": 0.13231344521045685,
-      "rewards/rejected": -0.0870073139667511,
+      "rewards/accuracies": 0.7093750238418579,
+      "rewards/chosen": 0.05313897505402565,
+      "rewards/margins": 0.551435649394989,
+      "rewards/rejected": -0.4982966482639313,
       "step": 900,
       "use_label": 0.0
     },
     {
       "epoch": 1.0,
-      "eval_logits/chosen": -2.8405630588531494,
-      "eval_logits/rejected": -2.8439478874206543,
-      "eval_logps/chosen": -283.7974548339844,
-      "eval_logps/rejected": -260.2491760253906,
-      "eval_loss": 0.6432895660400391,
+      "eval_logits/chosen": -2.8302695751190186,
+      "eval_logits/rejected": -2.8291618824005127,
+      "eval_logps/chosen": -283.5841369628906,
+      "eval_logps/rejected": -264.64422607421875,
+      "eval_loss": 0.5655555129051208,
       "eval_pred_label": 0.0,
-      "eval_rewards/accuracies": 0.6740000247955322,
-      "eval_rewards/chosen": 0.04599040374159813,
-      "eval_rewards/margins": 0.14093737304210663,
-      "eval_rewards/rejected": -0.0949469730257988,
-      "eval_runtime": 468.103,
-      "eval_samples_per_second": 4.273,
-      "eval_steps_per_second": 0.267,
+      "eval_rewards/accuracies": 0.699999988079071,
+      "eval_rewards/chosen": 0.06732505559921265,
+      "eval_rewards/margins": 0.6017746329307556,
+      "eval_rewards/rejected": -0.534449577331543,
+      "eval_runtime": 457.335,
+      "eval_samples_per_second": 4.373,
+      "eval_steps_per_second": 0.273,
       "eval_use_label": 0.0,
       "step": 955
     },
@@ -190,10 +190,10 @@
       "epoch": 1.0,
       "step": 955,
       "total_flos": 0.0,
-      "train_loss": 0.660746982334796,
-      "train_runtime": 25629.2924,
-      "train_samples_per_second": 2.385,
-      "train_steps_per_second": 0.037
+      "train_loss": 0.5870625535855118,
+      "train_runtime": 25384.4334,
+      "train_samples_per_second": 2.408,
+      "train_steps_per_second": 0.038
     }
   ],
   "logging_steps": 100,