Model save

README.md

@@ -15,15 +15,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6741
-- Rewards/chosen: 0.0220
-- Rewards/rejected: -0.0242
-- Rewards/accuracies: 0.6680
-- Rewards/margins: 0.0462
-- Logps/rejected: -259.5420
-- Logps/chosen: -284.0372
-- Logits/rejected: -2.8454
-- Logits/chosen: -2.8419
+- Loss: 0.6433
+- Rewards/chosen: 0.0460
+- Rewards/rejected: -0.0949
+- Rewards/accuracies: 0.6740
+- Rewards/margins: 0.1409
+- Logps/rejected: -260.2492
+- Logps/chosen: -283.7975
+- Logits/rejected: -2.8439
+- Logits/chosen: -2.8406
 - Use Label: 0.0
 - Pred Label: 0.0
 
@@ -44,7 +44,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 5e-07
+- learning_rate: 1e-06
 - train_batch_size: 4
 - eval_batch_size: 4
 - seed: 42
@@ -62,7 +62,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Use Label | Pred Label |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:---------:|:----------:|
-| 0.6744        | 1.0   | 955  | 0.6741          | 0.0220         | -0.0242          | 0.6680             | 0.0462          | -259.5420      | -284.0372    | -2.8454         | -2.8419       | 0.0       | 0.0        |
+| 0.6441        | 1.0   | 955  | 0.6433          | 0.0460         | -0.0949          | 0.6740             | 0.1409          | -260.2492      | -283.7975    | -2.8439         | -2.8406       | 0.0       | 0.0        |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6a1a6567a4d2a4ccb9f2f43dea6dced325234619f3a1d7ae383e50cdee8187bc
+oid sha256:5cb8cb0361b2338d889c4aeb84f54608df0a08612cd81d53bfe35d6340c2e7b0
 size 218138576

all_results.json

@@ -1,22 +1,22 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.8419151306152344,
-    "eval_logits/rejected": -2.845423936843872,
-    "eval_logps/chosen": -284.0372314453125,
-    "eval_logps/rejected": -259.5419921875,
-    "eval_loss": 0.6741092801094055,
+    "eval_logits/chosen": -2.8405630588531494,
+    "eval_logits/rejected": -2.8439478874206543,
+    "eval_logps/chosen": -283.7974548339844,
+    "eval_logps/rejected": -260.2491760253906,
+    "eval_loss": 0.6432895660400391,
     "eval_pred_label": 0.0,
-    "eval_rewards/accuracies": 0.6679999828338623,
-    "eval_rewards/chosen": 0.02201448194682598,
-    "eval_rewards/margins": 0.04624143987894058,
-    "eval_rewards/rejected": -0.02422695979475975,
-    "eval_runtime": 465.3584,
+    "eval_rewards/accuracies": 0.6740000247955322,
+    "eval_rewards/chosen": 0.04599040374159813,
+    "eval_rewards/margins": 0.14093737304210663,
+    "eval_rewards/rejected": -0.0949469730257988,
+    "eval_runtime": 468.6797,
     "eval_samples": 2000,
-    "eval_samples_per_second": 4.298,
-    "eval_steps_per_second": 0.269,
+    "eval_samples_per_second": 4.267,
+    "eval_steps_per_second": 0.267,
     "eval_use_label": 0.0,
-    "train_loss": 0.6817296707193264,
-    "train_runtime": 25631.2708,
+    "train_loss": 0.660746982334796,
+    "train_runtime": 25629.2924,
     "train_samples": 61135,
     "train_samples_per_second": 2.385,
     "train_steps_per_second": 0.037

eval_results.json

@@ -1,18 +1,18 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.8419151306152344,
-    "eval_logits/rejected": -2.845423936843872,
-    "eval_logps/chosen": -284.0372314453125,
-    "eval_logps/rejected": -259.5419921875,
-    "eval_loss": 0.6741092801094055,
+    "eval_logits/chosen": -2.8405630588531494,
+    "eval_logits/rejected": -2.8439478874206543,
+    "eval_logps/chosen": -283.7974548339844,
+    "eval_logps/rejected": -260.2491760253906,
+    "eval_loss": 0.6432895660400391,
     "eval_pred_label": 0.0,
-    "eval_rewards/accuracies": 0.6679999828338623,
-    "eval_rewards/chosen": 0.02201448194682598,
-    "eval_rewards/margins": 0.04624143987894058,
-    "eval_rewards/rejected": -0.02422695979475975,
-    "eval_runtime": 465.3584,
+    "eval_rewards/accuracies": 0.6740000247955322,
+    "eval_rewards/chosen": 0.04599040374159813,
+    "eval_rewards/margins": 0.14093737304210663,
+    "eval_rewards/rejected": -0.0949469730257988,
+    "eval_runtime": 468.6797,
     "eval_samples": 2000,
-    "eval_samples_per_second": 4.298,
-    "eval_steps_per_second": 0.269,
+    "eval_samples_per_second": 4.267,
+    "eval_steps_per_second": 0.267,
     "eval_use_label": 0.0
 }

runs/Jan21_01-22-38_uclaml04.cs.ucla.edu/events.out.tfevents.1705829022.uclaml04.cs.ucla.edu.475602.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9ee1e43246df245ba58407f0d4e990ae64e2e3765d093c7f0d53e2f0dce4dcf7
-size 11838
+oid sha256:e676c35e95fb49da088feb27d029428aa16e45cec2b9e90b44763b8a1695219a
+size 13039

runs/Jan21_01-22-38_uclaml04.cs.ucla.edu/events.out.tfevents.1705855119.uclaml04.cs.ucla.edu.475602.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7960fe6cc975d99bac6bab44dc4b36c37ee587fc654ea62de4d68de7e46d302a
+size 935

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6817296707193264,
-    "train_runtime": 25631.2708,
+    "train_loss": 0.660746982334796,
+    "train_runtime": 25629.2924,
     "train_samples": 61135,
     "train_samples_per_second": 2.385,
     "train_steps_per_second": 0.037

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "learning_rate": 5.208333333333333e-09,
+      "learning_rate": 1.0416666666666666e-08,
       "logits/chosen": -2.980285167694092,
       "logits/rejected": -2.87275767326355,
       "logps/chosen": -313.4390563964844,
@@ -26,163 +26,163 @@
     },
     {
       "epoch": 0.1,
-      "learning_rate": 4.976717112922002e-07,
-      "logits/chosen": -2.8195502758026123,
-      "logits/rejected": -2.828876495361328,
-      "logps/chosen": -285.2825012207031,
-      "logps/rejected": -270.9394836425781,
-      "loss": 0.6931,
+      "learning_rate": 9.953434225844004e-07,
+      "logits/chosen": -2.8195509910583496,
+      "logits/rejected": -2.8288235664367676,
+      "logps/chosen": -285.27215576171875,
+      "logps/rejected": -270.96209716796875,
+      "loss": 0.6923,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.4873737394809723,
-      "rewards/chosen": -0.0001882972428575158,
-      "rewards/margins": -0.0012037099804729223,
-      "rewards/rejected": 0.0010154128540307283,
+      "rewards/accuracies": 0.5183081030845642,
+      "rewards/chosen": 0.0008437540382146835,
+      "rewards/margins": 0.0020915681961923838,
+      "rewards/rejected": -0.0012478141579777002,
       "step": 100,
       "use_label": 0.0
     },
     {
       "epoch": 0.21,
-      "learning_rate": 4.3946449359720607e-07,
-      "logits/chosen": -2.8279786109924316,
-      "logits/rejected": -2.8164660930633545,
-      "logps/chosen": -278.72576904296875,
-      "logps/rejected": -260.5760498046875,
-      "loss": 0.6901,
+      "learning_rate": 8.789289871944121e-07,
+      "logits/chosen": -2.82773494720459,
+      "logits/rejected": -2.816188335418701,
+      "logps/chosen": -278.6949768066406,
+      "logps/rejected": -260.6236572265625,
+      "loss": 0.6855,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.5506250262260437,
-      "rewards/chosen": 0.0056201983243227005,
-      "rewards/margins": 0.009352817200124264,
-      "rewards/rejected": -0.003732620272785425,
+      "rewards/accuracies": 0.5956249833106995,
+      "rewards/chosen": 0.00869889184832573,
+      "rewards/margins": 0.017191100865602493,
+      "rewards/rejected": -0.008492209017276764,
       "step": 200,
       "use_label": 0.0
     },
     {
       "epoch": 0.31,
-      "learning_rate": 3.812572759022118e-07,
-      "logits/chosen": -2.8140082359313965,
-      "logits/rejected": -2.8158328533172607,
-      "logps/chosen": -284.0040283203125,
-      "logps/rejected": -253.93580627441406,
-      "loss": 0.6875,
+      "learning_rate": 7.625145518044237e-07,
+      "logits/chosen": -2.813408613204956,
+      "logits/rejected": -2.8151955604553223,
+      "logps/chosen": -283.8744812011719,
+      "logps/rejected": -254.1025390625,
+      "loss": 0.675,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.5793750286102295,
-      "rewards/chosen": 0.006361996755003929,
-      "rewards/margins": 0.012689676135778427,
-      "rewards/rejected": -0.006327680312097073,
+      "rewards/accuracies": 0.653124988079071,
+      "rewards/chosen": 0.01931346394121647,
+      "rewards/margins": 0.042313817888498306,
+      "rewards/rejected": -0.023000353947281837,
       "step": 300,
       "use_label": 0.0
     },
     {
       "epoch": 0.42,
-      "learning_rate": 3.230500582072177e-07,
-      "logits/chosen": -2.826840877532959,
-      "logits/rejected": -2.8091540336608887,
-      "logps/chosen": -284.3250427246094,
-      "logps/rejected": -262.8480529785156,
-      "loss": 0.6842,
+      "learning_rate": 6.461001164144354e-07,
+      "logits/chosen": -2.8259572982788086,
+      "logits/rejected": -2.808323383331299,
+      "logps/chosen": -284.1655578613281,
+      "logps/rejected": -263.1576843261719,
+      "loss": 0.6649,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.6031249761581421,
-      "rewards/chosen": 0.010542460717260838,
-      "rewards/margins": 0.02265419438481331,
-      "rewards/rejected": -0.012111731804907322,
+      "rewards/accuracies": 0.6837499737739563,
+      "rewards/chosen": 0.026492329314351082,
+      "rewards/margins": 0.06956954300403595,
+      "rewards/rejected": -0.04307721555233002,
       "step": 400,
       "use_label": 0.0
     },
     {
       "epoch": 0.52,
-      "learning_rate": 2.648428405122235e-07,
-      "logits/chosen": -2.8074240684509277,
-      "logits/rejected": -2.795992136001587,
-      "logps/chosen": -269.97344970703125,
-      "logps/rejected": -252.08152770996094,
-      "loss": 0.6821,
+      "learning_rate": 5.29685681024447e-07,
+      "logits/chosen": -2.806447744369507,
+      "logits/rejected": -2.794842004776001,
+      "logps/chosen": -269.8316345214844,
+      "logps/rejected": -252.43292236328125,
+      "loss": 0.6596,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.007951202802360058,
-      "rewards/margins": 0.017641538754105568,
-      "rewards/rejected": -0.00969033595174551,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": 0.022135533392429352,
+      "rewards/margins": 0.06696329265832901,
+      "rewards/rejected": -0.04482775181531906,
       "step": 500,
       "use_label": 0.0
     },
     {
       "epoch": 0.63,
-      "learning_rate": 2.0663562281722933e-07,
-      "logits/chosen": -2.8337247371673584,
-      "logits/rejected": -2.8203465938568115,
-      "logps/chosen": -285.0716857910156,
-      "logps/rejected": -265.4593811035156,
-      "loss": 0.6782,
+      "learning_rate": 4.1327124563445866e-07,
+      "logits/chosen": -2.8327181339263916,
+      "logits/rejected": -2.819202184677124,
+      "logps/chosen": -284.8338623046875,
+      "logps/rejected": -265.977294921875,
+      "loss": 0.6493,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.6356250047683716,
-      "rewards/chosen": 0.015922056511044502,
-      "rewards/margins": 0.035635706037282944,
-      "rewards/rejected": -0.01971365138888359,
+      "rewards/accuracies": 0.6675000190734863,
+      "rewards/chosen": 0.03970767930150032,
+      "rewards/margins": 0.11121242493391037,
+      "rewards/rejected": -0.07150473445653915,
       "step": 600,
       "use_label": 0.0
     },
     {
       "epoch": 0.73,
-      "learning_rate": 1.4842840512223514e-07,
-      "logits/chosen": -2.8270840644836426,
-      "logits/rejected": -2.8112306594848633,
-      "logps/chosen": -281.3733215332031,
-      "logps/rejected": -248.8465576171875,
-      "loss": 0.6762,
+      "learning_rate": 2.968568102444703e-07,
+      "logits/chosen": -2.8258111476898193,
+      "logits/rejected": -2.8097949028015137,
+      "logps/chosen": -281.1001892089844,
+      "logps/rejected": -249.4017791748047,
+      "loss": 0.6463,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.6324999928474426,
-      "rewards/chosen": 0.018638433888554573,
-      "rewards/margins": 0.03718380257487297,
-      "rewards/rejected": -0.018545370548963547,
+      "rewards/accuracies": 0.6806250214576721,
+      "rewards/chosen": 0.04594804719090462,
+      "rewards/margins": 0.12001516669988632,
+      "rewards/rejected": -0.0740671157836914,
       "step": 700,
       "use_label": 0.0
     },
     {
       "epoch": 0.84,
-      "learning_rate": 9.022118742724097e-08,
-      "logits/chosen": -2.8297488689422607,
-      "logits/rejected": -2.83492374420166,
-      "logps/chosen": -288.54058837890625,
-      "logps/rejected": -261.6487731933594,
-      "loss": 0.6736,
+      "learning_rate": 1.8044237485448194e-07,
+      "logits/chosen": -2.82832407951355,
+      "logits/rejected": -2.833012580871582,
+      "logps/chosen": -288.21307373046875,
+      "logps/rejected": -262.298095703125,
+      "loss": 0.64,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.6393749713897705,
-      "rewards/chosen": 0.021646475419402122,
-      "rewards/margins": 0.04531754553318024,
-      "rewards/rejected": -0.023671068251132965,
+      "rewards/accuracies": 0.6831250190734863,
+      "rewards/chosen": 0.05439276248216629,
+      "rewards/margins": 0.14299620687961578,
+      "rewards/rejected": -0.08860345184803009,
       "step": 800,
       "use_label": 0.0
     },
     {
       "epoch": 0.94,
-      "learning_rate": 3.20139697322468e-08,
-      "logits/chosen": -2.8206725120544434,
-      "logits/rejected": -2.8274974822998047,
-      "logps/chosen": -277.310791015625,
-      "logps/rejected": -256.5542297363281,
-      "loss": 0.6744,
+      "learning_rate": 6.40279394644936e-08,
+      "logits/chosen": -2.8194103240966797,
+      "logits/rejected": -2.8261044025421143,
+      "logps/chosen": -277.05517578125,
+      "logps/rejected": -257.2059020996094,
+      "loss": 0.6441,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.6443750262260437,
-      "rewards/chosen": 0.019743308424949646,
-      "rewards/margins": 0.041582074016332626,
-      "rewards/rejected": -0.021838760003447533,
+      "rewards/accuracies": 0.6762499809265137,
+      "rewards/chosen": 0.04530614987015724,
+      "rewards/margins": 0.13231344521045685,
+      "rewards/rejected": -0.0870073139667511,
       "step": 900,
       "use_label": 0.0
     },
     {
       "epoch": 1.0,
-      "eval_logits/chosen": -2.8419151306152344,
-      "eval_logits/rejected": -2.845423936843872,
-      "eval_logps/chosen": -284.0372314453125,
-      "eval_logps/rejected": -259.5419921875,
-      "eval_loss": 0.6741092801094055,
+      "eval_logits/chosen": -2.8405630588531494,
+      "eval_logits/rejected": -2.8439478874206543,
+      "eval_logps/chosen": -283.7974548339844,
+      "eval_logps/rejected": -260.2491760253906,
+      "eval_loss": 0.6432895660400391,
       "eval_pred_label": 0.0,
-      "eval_rewards/accuracies": 0.6679999828338623,
-      "eval_rewards/chosen": 0.02201448194682598,
-      "eval_rewards/margins": 0.04624143987894058,
-      "eval_rewards/rejected": -0.02422695979475975,
-      "eval_runtime": 469.0597,
-      "eval_samples_per_second": 4.264,
-      "eval_steps_per_second": 0.266,
+      "eval_rewards/accuracies": 0.6740000247955322,
+      "eval_rewards/chosen": 0.04599040374159813,
+      "eval_rewards/margins": 0.14093737304210663,
+      "eval_rewards/rejected": -0.0949469730257988,
+      "eval_runtime": 468.103,
+      "eval_samples_per_second": 4.273,
+      "eval_steps_per_second": 0.267,
       "eval_use_label": 0.0,
       "step": 955
     },
@@ -190,8 +190,8 @@
       "epoch": 1.0,
       "step": 955,
       "total_flos": 0.0,
-      "train_loss": 0.6817296707193264,
-      "train_runtime": 25631.2708,
+      "train_loss": 0.660746982334796,
+      "train_runtime": 25629.2924,
       "train_samples_per_second": 2.385,
       "train_steps_per_second": 0.037
     }