Model save

README.md

@@ -15,15 +15,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5656
-- Rewards/chosen: 0.0673
-- Rewards/rejected: -0.5344
-- Rewards/accuracies: 0.7000
-- Rewards/margins: 0.6018
-- Logps/rejected: -264.6442
-- Logps/chosen: -283.5841
-- Logits/rejected: -2.8292
-- Logits/chosen: -2.8303
+- Loss: 0.5460
+- Rewards/chosen: 0.0878
+- Rewards/rejected: -0.6697
+- Rewards/accuracies: 0.7140
+- Rewards/margins: 0.7575
+- Logps/rejected: -265.9970
+- Logps/chosen: -283.3795
+- Logits/rejected: -2.8187
+- Logits/chosen: -2.8274
 - Use Label: 0.0
 - Pred Label: 0.0
 
@@ -44,7 +44,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 5e-06
+- learning_rate: 1e-05
 - train_batch_size: 4
 - eval_batch_size: 4
 - seed: 42
@@ -62,7 +62,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Use Label | Pred Label |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:---------:|:----------:|
-| 0.5689        | 1.0   | 955  | 0.5656          | 0.0673         | -0.5344          | 0.7000             | 0.6018          | -264.6442      | -283.5841    | -2.8292         | -2.8303       | 0.0       | 0.0        |
+| 0.545         | 1.0   | 955  | 0.5460          | 0.0878         | -0.6697          | 0.7140             | 0.7575          | -265.9970      | -283.3795    | -2.8187         | -2.8274       | 0.0       | 0.0        |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:acfc333285ecd363f21ac9a422d31780339b22dc96a535dfa323fa71da01e6dd
+oid sha256:ee44cce906c7a6f93e762633279c05b8cb98e432f3c9ac83bd6f6213cf55c20a
 size 218138576

all_results.json

@@ -1,23 +1,23 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.8302695751190186,
-    "eval_logits/rejected": -2.8291618824005127,
-    "eval_logps/chosen": -283.5841369628906,
-    "eval_logps/rejected": -264.64422607421875,
-    "eval_loss": 0.5655555129051208,
+    "eval_logits/chosen": -2.827404737472534,
+    "eval_logits/rejected": -2.818655014038086,
+    "eval_logps/chosen": -283.37945556640625,
+    "eval_logps/rejected": -265.9969787597656,
+    "eval_loss": 0.5459502935409546,
     "eval_pred_label": 0.0,
-    "eval_rewards/accuracies": 0.699999988079071,
-    "eval_rewards/chosen": 0.06732505559921265,
-    "eval_rewards/margins": 0.6017746329307556,
-    "eval_rewards/rejected": -0.534449577331543,
-    "eval_runtime": 457.1465,
+    "eval_rewards/accuracies": 0.7139999866485596,
+    "eval_rewards/chosen": 0.08778975158929825,
+    "eval_rewards/margins": 0.7575166821479797,
+    "eval_rewards/rejected": -0.6697269678115845,
+    "eval_runtime": 478.4263,
     "eval_samples": 2000,
-    "eval_samples_per_second": 4.375,
-    "eval_steps_per_second": 0.273,
+    "eval_samples_per_second": 4.18,
+    "eval_steps_per_second": 0.261,
     "eval_use_label": 0.0,
-    "train_loss": 0.5870625535855118,
-    "train_runtime": 25384.4334,
+    "train_loss": 0.5628191218950361,
+    "train_runtime": 25746.1298,
     "train_samples": 61135,
-    "train_samples_per_second": 2.408,
-    "train_steps_per_second": 0.038
+    "train_samples_per_second": 2.375,
+    "train_steps_per_second": 0.037
 }

eval_results.json

@@ -1,18 +1,18 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.8302695751190186,
-    "eval_logits/rejected": -2.8291618824005127,
-    "eval_logps/chosen": -283.5841369628906,
-    "eval_logps/rejected": -264.64422607421875,
-    "eval_loss": 0.5655555129051208,
+    "eval_logits/chosen": -2.827404737472534,
+    "eval_logits/rejected": -2.818655014038086,
+    "eval_logps/chosen": -283.37945556640625,
+    "eval_logps/rejected": -265.9969787597656,
+    "eval_loss": 0.5459502935409546,
     "eval_pred_label": 0.0,
-    "eval_rewards/accuracies": 0.699999988079071,
-    "eval_rewards/chosen": 0.06732505559921265,
-    "eval_rewards/margins": 0.6017746329307556,
-    "eval_rewards/rejected": -0.534449577331543,
-    "eval_runtime": 457.1465,
+    "eval_rewards/accuracies": 0.7139999866485596,
+    "eval_rewards/chosen": 0.08778975158929825,
+    "eval_rewards/margins": 0.7575166821479797,
+    "eval_rewards/rejected": -0.6697269678115845,
+    "eval_runtime": 478.4263,
     "eval_samples": 2000,
-    "eval_samples_per_second": 4.375,
-    "eval_steps_per_second": 0.273,
+    "eval_samples_per_second": 4.18,
+    "eval_steps_per_second": 0.261,
     "eval_use_label": 0.0
 }

runs/Jan21_22-53-06_uclaml04.cs.ucla.edu/events.out.tfevents.1705906443.uclaml04.cs.ucla.edu.843444.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1f92826a808e2be9dfed130fa67ad1a0eea1c11f95fe17ca655d1df5bd174f43
-size 11837
+oid sha256:70cef13045c99f0b2e0174d81d676e33577c08d8132d6a7dd5fe8a82051a671a
+size 13038

runs/Jan21_22-53-06_uclaml04.cs.ucla.edu/events.out.tfevents.1705932667.uclaml04.cs.ucla.edu.843444.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e4a8aa0de3f53d33d6f9f03d407834c788dfffa1cfc429c328ae7af5f7003e3
+size 935

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5870625535855118,
-    "train_runtime": 25384.4334,
+    "train_loss": 0.5628191218950361,
+    "train_runtime": 25746.1298,
     "train_samples": 61135,
-    "train_samples_per_second": 2.408,
-    "train_steps_per_second": 0.038
+    "train_samples_per_second": 2.375,
+    "train_steps_per_second": 0.037
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "learning_rate": 5.208333333333333e-08,
+      "learning_rate": 1.0416666666666667e-07,
       "logits/chosen": -2.980285167694092,
       "logits/rejected": -2.87275767326355,
       "logps/chosen": -313.4390563964844,
@@ -26,163 +26,163 @@
     },
     {
       "epoch": 0.1,
-      "learning_rate": 4.9767171129220025e-06,
-      "logits/chosen": -2.8189077377319336,
-      "logits/rejected": -2.828260660171509,
-      "logps/chosen": -285.1784362792969,
-      "logps/rejected": -271.1445617675781,
-      "loss": 0.6803,
+      "learning_rate": 9.953434225844005e-06,
+      "logits/chosen": -2.8180909156799316,
+      "logits/rejected": -2.8273613452911377,
+      "logps/chosen": -285.13623046875,
+      "logps/rejected": -271.66839599609375,
+      "loss": 0.6624,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.5738636255264282,
-      "rewards/chosen": 0.01021653600037098,
-      "rewards/margins": 0.029710056260228157,
-      "rewards/rejected": -0.019493522122502327,
+      "rewards/accuracies": 0.6041666865348816,
+      "rewards/chosen": 0.014434419572353363,
+      "rewards/margins": 0.08631344139575958,
+      "rewards/rejected": -0.07187902927398682,
       "step": 100,
       "use_label": 0.0
     },
     {
       "epoch": 0.21,
-      "learning_rate": 4.394644935972061e-06,
-      "logits/chosen": -2.8255398273468018,
-      "logits/rejected": -2.813203811645508,
-      "logps/chosen": -278.2582092285156,
-      "logps/rejected": -262.1144714355469,
-      "loss": 0.6236,
+      "learning_rate": 8.789289871944122e-06,
+      "logits/chosen": -2.8230364322662354,
+      "logits/rejected": -2.8086395263671875,
+      "logps/chosen": -278.2524108886719,
+      "logps/rejected": -263.9921569824219,
+      "loss": 0.5868,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.6868749856948853,
-      "rewards/chosen": 0.05237884446978569,
-      "rewards/margins": 0.20995216071605682,
-      "rewards/rejected": -0.15757331252098083,
+      "rewards/accuracies": 0.6949999928474426,
+      "rewards/chosen": 0.05295524746179581,
+      "rewards/margins": 0.39829620718955994,
+      "rewards/rejected": -0.3453409671783447,
       "step": 200,
       "use_label": 0.0
     },
     {
       "epoch": 0.31,
-      "learning_rate": 3.812572759022119e-06,
-      "logits/chosen": -2.809633255004883,
-      "logits/rejected": -2.80940580368042,
-      "logps/chosen": -283.6641540527344,
-      "logps/rejected": -257.23779296875,
-      "loss": 0.5901,
+      "learning_rate": 7.625145518044238e-06,
+      "logits/chosen": -2.803905725479126,
+      "logits/rejected": -2.802032232284546,
+      "logps/chosen": -284.01385498046875,
+      "logps/rejected": -259.5546569824219,
+      "loss": 0.562,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.6956250071525574,
-      "rewards/chosen": 0.04034877195954323,
-      "rewards/margins": 0.3768764138221741,
-      "rewards/rejected": -0.33652764558792114,
+      "rewards/accuracies": 0.7056249976158142,
+      "rewards/chosen": 0.005377008114010096,
+      "rewards/margins": 0.5735920667648315,
+      "rewards/rejected": -0.5682151317596436,
       "step": 300,
       "use_label": 0.0
     },
     {
       "epoch": 0.42,
-      "learning_rate": 3.2305005820721774e-06,
-      "logits/chosen": -2.821267604827881,
-      "logits/rejected": -2.8011868000030518,
-      "logps/chosen": -284.2022399902344,
-      "logps/rejected": -267.48358154296875,
-      "loss": 0.5757,
+      "learning_rate": 6.461001164144355e-06,
+      "logits/chosen": -2.8141045570373535,
+      "logits/rejected": -2.7911813259124756,
+      "logps/chosen": -284.3139953613281,
+      "logps/rejected": -269.4837951660156,
+      "loss": 0.5527,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.7081249952316284,
-      "rewards/chosen": 0.022827474400401115,
-      "rewards/margins": 0.4984941780567169,
-      "rewards/rejected": -0.47566673159599304,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.011648621410131454,
+      "rewards/margins": 0.6873368620872498,
+      "rewards/rejected": -0.6756882667541504,
       "step": 400,
       "use_label": 0.0
     },
     {
       "epoch": 0.52,
-      "learning_rate": 2.6484284051222353e-06,
-      "logits/chosen": -2.7993886470794678,
-      "logits/rejected": -2.7845959663391113,
-      "logps/chosen": -270.09637451171875,
-      "logps/rejected": -256.13458251953125,
-      "loss": 0.5798,
+      "learning_rate": 5.2968568102444705e-06,
+      "logits/chosen": -2.7915823459625244,
+      "logits/rejected": -2.776299476623535,
+      "logps/chosen": -269.73016357421875,
+      "logps/rejected": -257.2498474121094,
+      "loss": 0.5556,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.6612499952316284,
-      "rewards/chosen": -0.004339172504842281,
-      "rewards/margins": 0.4106566905975342,
-      "rewards/rejected": -0.41499578952789307,
+      "rewards/accuracies": 0.6862499713897705,
+      "rewards/chosen": 0.03227977454662323,
+      "rewards/margins": 0.5588020086288452,
+      "rewards/rejected": -0.5265222191810608,
       "step": 500,
       "use_label": 0.0
     },
     {
       "epoch": 0.63,
-      "learning_rate": 2.0663562281722936e-06,
-      "logits/chosen": -2.824444532394409,
-      "logits/rejected": -2.808506965637207,
-      "logps/chosen": -284.5413513183594,
-      "logps/rejected": -270.1356201171875,
-      "loss": 0.5643,
+      "learning_rate": 4.132712456344587e-06,
+      "logits/chosen": -2.8199498653411865,
+      "logits/rejected": -2.8022332191467285,
+      "logps/chosen": -284.0947265625,
+      "logps/rejected": -271.3774108886719,
+      "loss": 0.5422,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 0.06895674020051956,
-      "rewards/margins": 0.5562920570373535,
-      "rewards/rejected": -0.4873352348804474,
+      "rewards/accuracies": 0.7143750190734863,
+      "rewards/chosen": 0.11361943930387497,
+      "rewards/margins": 0.7251341938972473,
+      "rewards/rejected": -0.6115147471427917,
       "step": 600,
       "use_label": 0.0
     },
     {
       "epoch": 0.73,
-      "learning_rate": 1.4842840512223516e-06,
-      "logits/chosen": -2.8160746097564697,
-      "logits/rejected": -2.795208692550659,
-      "logps/chosen": -280.79766845703125,
-      "logps/rejected": -253.41058349609375,
-      "loss": 0.5631,
+      "learning_rate": 2.9685681024447033e-06,
+      "logits/chosen": -2.8110527992248535,
+      "logits/rejected": -2.788975477218628,
+      "logps/chosen": -280.3959045410156,
+      "logps/rejected": -254.49673461914062,
+      "loss": 0.5404,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.7231249809265137,
-      "rewards/chosen": 0.07620371133089066,
-      "rewards/margins": 0.5511507391929626,
-      "rewards/rejected": -0.474947065114975,
+      "rewards/accuracies": 0.7212499976158142,
+      "rewards/chosen": 0.11637673527002335,
+      "rewards/margins": 0.6999369263648987,
+      "rewards/rejected": -0.5835601687431335,
       "step": 700,
       "use_label": 0.0
     },
     {
       "epoch": 0.84,
-      "learning_rate": 9.022118742724098e-07,
-      "logits/chosen": -2.8168938159942627,
-      "logits/rejected": -2.8180816173553467,
-      "logps/chosen": -287.77679443359375,
-      "logps/rejected": -266.3511657714844,
-      "loss": 0.5548,
+      "learning_rate": 1.8044237485448196e-06,
+      "logits/chosen": -2.8146722316741943,
+      "logits/rejected": -2.812129020690918,
+      "logps/chosen": -287.4331359863281,
+      "logps/rejected": -267.59161376953125,
+      "loss": 0.5343,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.7112500071525574,
-      "rewards/chosen": 0.09802371263504028,
-      "rewards/margins": 0.5919383764266968,
-      "rewards/rejected": -0.49391472339630127,
+      "rewards/accuracies": 0.721875011920929,
+      "rewards/chosen": 0.13239255547523499,
+      "rewards/margins": 0.7503484487533569,
+      "rewards/rejected": -0.6179558634757996,
       "step": 800,
       "use_label": 0.0
     },
     {
       "epoch": 0.94,
-      "learning_rate": 3.2013969732246806e-07,
-      "logits/chosen": -2.808311700820923,
-      "logits/rejected": -2.812298536300659,
-      "logps/chosen": -276.976806640625,
-      "logps/rejected": -261.3187561035156,
-      "loss": 0.5689,
+      "learning_rate": 6.402793946449361e-07,
+      "logits/chosen": -2.8043179512023926,
+      "logits/rejected": -2.8079681396484375,
+      "logps/chosen": -276.8100891113281,
+      "logps/rejected": -262.690673828125,
+      "loss": 0.545,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.7093750238418579,
-      "rewards/chosen": 0.05313897505402565,
-      "rewards/margins": 0.551435649394989,
-      "rewards/rejected": -0.4982966482639313,
+      "rewards/accuracies": 0.7193750143051147,
+      "rewards/chosen": 0.06981150805950165,
+      "rewards/margins": 0.7052963972091675,
+      "rewards/rejected": -0.6354848742485046,
       "step": 900,
       "use_label": 0.0
     },
     {
       "epoch": 1.0,
-      "eval_logits/chosen": -2.8302695751190186,
-      "eval_logits/rejected": -2.8291618824005127,
-      "eval_logps/chosen": -283.5841369628906,
-      "eval_logps/rejected": -264.64422607421875,
-      "eval_loss": 0.5655555129051208,
+      "eval_logits/chosen": -2.827404737472534,
+      "eval_logits/rejected": -2.818655014038086,
+      "eval_logps/chosen": -283.37945556640625,
+      "eval_logps/rejected": -265.9969787597656,
+      "eval_loss": 0.5459502935409546,
       "eval_pred_label": 0.0,
-      "eval_rewards/accuracies": 0.699999988079071,
-      "eval_rewards/chosen": 0.06732505559921265,
-      "eval_rewards/margins": 0.6017746329307556,
-      "eval_rewards/rejected": -0.534449577331543,
-      "eval_runtime": 457.335,
-      "eval_samples_per_second": 4.373,
-      "eval_steps_per_second": 0.273,
+      "eval_rewards/accuracies": 0.7139999866485596,
+      "eval_rewards/chosen": 0.08778975158929825,
+      "eval_rewards/margins": 0.7575166821479797,
+      "eval_rewards/rejected": -0.6697269678115845,
+      "eval_runtime": 479.5351,
+      "eval_samples_per_second": 4.171,
+      "eval_steps_per_second": 0.261,
       "eval_use_label": 0.0,
       "step": 955
     },
@@ -190,16 +190,16 @@
       "epoch": 1.0,
       "step": 955,
       "total_flos": 0.0,
-      "train_loss": 0.5870625535855118,
-      "train_runtime": 25384.4334,
-      "train_samples_per_second": 2.408,
-      "train_steps_per_second": 0.038
+      "train_loss": 0.5628191218950361,
+      "train_runtime": 25746.1298,
+      "train_samples_per_second": 2.375,
+      "train_steps_per_second": 0.037
     }
   ],
   "logging_steps": 100,
   "max_steps": 955,
   "num_train_epochs": 1,
-  "save_steps": 100,
+  "save_steps": 50,
   "total_flos": 0.0,
   "trial_name": null,
   "trial_params": null