Model save

README.md

@@ -1,5 +1,4 @@
 ---
-base_model: princeton-nlp/Mistral-7B-Base-SFT-SimPO
 tags:
 - trl
 - dpo
@@ -14,7 +13,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model is a fine-tuned version of [princeton-nlp/Mistral-7B-Base-SFT-SimPO](https://huggingface.co/princeton-nlp/Mistral-7B-Base-SFT-SimPO) on the None dataset.
+This model was trained from scratch on the None dataset.
 
 ## Model description
 
@@ -33,7 +32,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 1e-07
+- learning_rate: 1e-08
 - train_batch_size: 4
 - eval_batch_size: 4
 - seed: 42

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5564739256078942,
-    "train_runtime": 6859.6403,
-    "train_samples": 56236,
-    "train_samples_per_second": 8.198,
-    "train_steps_per_second": 0.064
+    "train_loss": 0.6880149315033469,
+    "train_runtime": 4603.368,
+    "train_samples": 38288,
+    "train_samples_per_second": 8.317,
+    "train_steps_per_second": 0.065
 }

config.json

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.41.1",
+  "transformers_version": "4.39.3",
   "use_cache": false,
   "vocab_size": 32000
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3e692100307a70f65bd0d21036b34ef68687d64f8dc88cc7bd88cac3cb499b72
+oid sha256:25a506977023960c048f2933f9aec9fc01ac4cef4294a89c59fb94311d62d0ba
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:22c11cd667919a180be90a638f8180dfe57b3af895c1fe13ee1bda06824ae1db
+oid sha256:f36b418c6f0204d63a482c4d40cf9a099d1c71913006174de004f3468b2600ac
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aee128bd109134555c87391e5c9defecae53a6eb4d7c164396cfda3229777b91
+oid sha256:0098c8b3746d7ba8d54ad9a79ed49c5d66268bbf89e84d00df21498d52d9cb42
 size 4540516344

runs/Jul03_20-00-02_n136-129-074/events.out.tfevents.1720008022.n136-129-074.3083115.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8002adcf5dd4a58d504c1dd4b67ec2f85903dae943432f38436700f0f8a91980
-size 19297
+oid sha256:5c476ad368f70bb551a6766cd2d3c0fa0a42b8470e8b61a505a8c02a979fe20e
+size 25843

tokenizer.json

@@ -134,7 +134,6 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
-    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5564739256078942,
-    "train_runtime": 6859.6403,
-    "train_samples": 56236,
-    "train_samples_per_second": 8.198,
-    "train_steps_per_second": 0.064
+    "train_loss": 0.6880149315033469,
+    "train_runtime": 4603.368,
+    "train_samples": 38288,
+    "train_samples_per_second": 8.317,
+    "train_steps_per_second": 0.065
 }

trainer_state.json

@@ -1,22 +1,22 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9988623435722411,
+  "epoch": 0.9991645781119465,
   "eval_steps": 10000000,
-  "global_step": 439,
+  "global_step": 299,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 113.4577168560686,
-      "learning_rate": 2.2727272727272727e-09,
-      "logits/chosen": -1.6768856048583984,
-      "logits/rejected": -1.7259055376052856,
-      "logps/chosen": -1.2793102264404297,
-      "logps/rejected": -1.2162058353424072,
-      "loss": 0.6934,
+      "grad_norm": 94.51455806402136,
+      "learning_rate": 3.333333333333333e-10,
+      "logits/chosen": -1.693521499633789,
+      "logits/rejected": -1.6753541231155396,
+      "logps/chosen": -1.041430115699768,
+      "logps/rejected": -0.9273841977119446,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -24,662 +24,452 @@
       "step": 1
     },
     {
-      "epoch": 0.02,
-      "grad_norm": 129.24988403709358,
-      "learning_rate": 2.2727272727272725e-08,
-      "logits/chosen": -1.7028687000274658,
-      "logits/rejected": -1.6680525541305542,
-      "logps/chosen": -1.2131904363632202,
-      "logps/rejected": -1.2204842567443848,
+      "epoch": 0.03,
+      "grad_norm": 100.10048800952612,
+      "learning_rate": 3.3333333333333334e-09,
+      "logits/chosen": -1.7426362037658691,
+      "logits/rejected": -1.7463488578796387,
+      "logps/chosen": -1.0522818565368652,
+      "logps/rejected": -1.0174607038497925,
       "loss": 0.6933,
-      "rewards/accuracies": 0.4652777910232544,
-      "rewards/chosen": 0.0011519736144691706,
-      "rewards/margins": -0.001473029493354261,
-      "rewards/rejected": 0.0026250029914081097,
+      "rewards/accuracies": 0.4722222089767456,
+      "rewards/chosen": -0.0006895202095620334,
+      "rewards/margins": -0.0014206942869350314,
+      "rewards/rejected": 0.0007311741355806589,
       "step": 10
     },
     {
-      "epoch": 0.05,
-      "grad_norm": 128.90531745075424,
-      "learning_rate": 4.545454545454545e-08,
-      "logits/chosen": -1.7795600891113281,
-      "logits/rejected": -1.7348560094833374,
-      "logps/chosen": -1.1446261405944824,
-      "logps/rejected": -1.1849511861801147,
-      "loss": 0.6903,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": -0.01403624564409256,
-      "rewards/margins": 0.0008796676993370056,
-      "rewards/rejected": -0.014915913343429565,
+      "epoch": 0.07,
+      "grad_norm": 100.2747717498339,
+      "learning_rate": 6.666666666666667e-09,
+      "logits/chosen": -1.876529335975647,
+      "logits/rejected": -1.8286478519439697,
+      "logps/chosen": -1.0717421770095825,
+      "logps/rejected": -1.0434354543685913,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.0013453494757413864,
+      "rewards/margins": 0.001054912805557251,
+      "rewards/rejected": 0.00029043667018413544,
       "step": 20
     },
     {
-      "epoch": 0.07,
-      "grad_norm": 130.88686983721493,
-      "learning_rate": 6.818181818181817e-08,
-      "logits/chosen": -1.7440669536590576,
-      "logits/rejected": -1.6752160787582397,
-      "logps/chosen": -1.194157600402832,
-      "logps/rejected": -1.246543526649475,
-      "loss": 0.6755,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.11256156116724014,
-      "rewards/margins": 0.05156043916940689,
-      "rewards/rejected": -0.16412200033664703,
+      "epoch": 0.1,
+      "grad_norm": 91.78075458726622,
+      "learning_rate": 1e-08,
+      "logits/chosen": -1.7869594097137451,
+      "logits/rejected": -1.7599306106567383,
+      "logps/chosen": -1.066955327987671,
+      "logps/rejected": -1.0448154211044312,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.0009901206940412521,
+      "rewards/margins": 0.005651786923408508,
+      "rewards/rejected": -0.004661666229367256,
       "step": 30
     },
     {
-      "epoch": 0.09,
-      "grad_norm": 99.69346579706567,
-      "learning_rate": 9.09090909090909e-08,
-      "logits/chosen": -1.7316129207611084,
-      "logits/rejected": -1.6658855676651,
-      "logps/chosen": -1.2354015111923218,
-      "logps/rejected": -1.3086662292480469,
-      "loss": 0.6598,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.28453049063682556,
-      "rewards/margins": 0.20941098034381866,
-      "rewards/rejected": -0.49394145607948303,
+      "epoch": 0.13,
+      "grad_norm": 83.1668176080734,
+      "learning_rate": 9.96594024562513e-09,
+      "logits/chosen": -1.8331626653671265,
+      "logits/rejected": -1.8235479593276978,
+      "logps/chosen": -1.069947361946106,
+      "logps/rejected": -1.0194944143295288,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.00406806543469429,
+      "rewards/margins": 0.006294439546763897,
+      "rewards/rejected": -0.0022263741120696068,
       "step": 40
     },
     {
-      "epoch": 0.11,
-      "grad_norm": 94.76247831424028,
-      "learning_rate": 9.994307990108962e-08,
-      "logits/chosen": -1.6965711116790771,
-      "logits/rejected": -1.6329838037490845,
-      "logps/chosen": -1.254882574081421,
-      "logps/rejected": -1.3045636415481567,
-      "loss": 0.6257,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.4679547846317291,
-      "rewards/margins": 0.29588648676872253,
-      "rewards/rejected": -0.7638412714004517,
+      "epoch": 0.17,
+      "grad_norm": 97.3484905394127,
+      "learning_rate": 9.86422500924775e-09,
+      "logits/chosen": -1.8598533868789673,
+      "logits/rejected": -1.8249794244766235,
+      "logps/chosen": -1.0844773054122925,
+      "logps/rejected": -1.0546021461486816,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0010452494025230408,
+      "rewards/margins": 0.003718096762895584,
+      "rewards/rejected": -0.0026728473603725433,
       "step": 50
     },
     {
-      "epoch": 0.14,
-      "grad_norm": 134.02849477110337,
-      "learning_rate": 9.959570405988094e-08,
-      "logits/chosen": -1.7339792251586914,
-      "logits/rejected": -1.6545541286468506,
-      "logps/chosen": -1.1747747659683228,
-      "logps/rejected": -1.2515872716903687,
-      "loss": 0.628,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.6567646265029907,
-      "rewards/margins": 0.2488430291414261,
-      "rewards/rejected": -0.9056077003479004,
+      "epoch": 0.2,
+      "grad_norm": 89.15270505321199,
+      "learning_rate": 9.696240049254743e-09,
+      "logits/chosen": -1.8162918090820312,
+      "logits/rejected": -1.8095808029174805,
+      "logps/chosen": -1.1147956848144531,
+      "logps/rejected": -1.10584557056427,
+      "loss": 0.692,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.003025440499186516,
+      "rewards/margins": 0.006887376308441162,
+      "rewards/rejected": -0.0038619358092546463,
       "step": 60
     },
     {
-      "epoch": 0.16,
-      "grad_norm": 81.9825685637945,
-      "learning_rate": 9.893476820924666e-08,
-      "logits/chosen": -1.8203418254852295,
-      "logits/rejected": -1.7378566265106201,
-      "logps/chosen": -1.2860430479049683,
-      "logps/rejected": -1.3670638799667358,
-      "loss": 0.5884,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.732010006904602,
-      "rewards/margins": 0.3435601592063904,
-      "rewards/rejected": -1.0755703449249268,
+      "epoch": 0.23,
+      "grad_norm": 92.66102585547152,
+      "learning_rate": 9.464273976236516e-09,
+      "logits/chosen": -1.7991975545883179,
+      "logits/rejected": -1.7613614797592163,
+      "logps/chosen": -1.0598714351654053,
+      "logps/rejected": -1.0530080795288086,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.0042045507580041885,
+      "rewards/margins": 0.0019652042537927628,
+      "rewards/rejected": 0.002239346969872713,
       "step": 70
     },
     {
-      "epoch": 0.18,
-      "grad_norm": 111.43014849080478,
-      "learning_rate": 9.796445099843647e-08,
-      "logits/chosen": -1.8093068599700928,
-      "logits/rejected": -1.7265288829803467,
-      "logps/chosen": -1.2918837070465088,
-      "logps/rejected": -1.3895039558410645,
-      "loss": 0.6282,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.8491527438163757,
-      "rewards/margins": 0.3709031641483307,
-      "rewards/rejected": -1.2200558185577393,
+      "epoch": 0.27,
+      "grad_norm": 194.03821825764928,
+      "learning_rate": 9.171487073181198e-09,
+      "logits/chosen": -1.8440383672714233,
+      "logits/rejected": -1.8225574493408203,
+      "logps/chosen": -1.0219953060150146,
+      "logps/rejected": -1.0044893026351929,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0038851741701364517,
+      "rewards/margins": 0.009898573160171509,
+      "rewards/rejected": -0.006013398990035057,
       "step": 80
     },
     {
-      "epoch": 0.2,
-      "grad_norm": 99.11416934571663,
-      "learning_rate": 9.669088708527066e-08,
-      "logits/chosen": -1.756994605064392,
-      "logits/rejected": -1.6932170391082764,
-      "logps/chosen": -1.3391311168670654,
-      "logps/rejected": -1.3889451026916504,
-      "loss": 0.5898,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.0343393087387085,
-      "rewards/margins": 0.4276931881904602,
-      "rewards/rejected": -1.462032437324524,
+      "epoch": 0.3,
+      "grad_norm": 89.9314461991181,
+      "learning_rate": 8.821868240089676e-09,
+      "logits/chosen": -1.8029550313949585,
+      "logits/rejected": -1.7841434478759766,
+      "logps/chosen": -1.045693039894104,
+      "logps/rejected": -1.0079419612884521,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": 0.005337424576282501,
+      "rewards/margins": 0.004554492421448231,
+      "rewards/rejected": 0.0007829321548342705,
       "step": 90
     },
     {
-      "epoch": 0.23,
-      "grad_norm": 108.94100525225643,
-      "learning_rate": 9.512212835085849e-08,
-      "logits/chosen": -1.7921009063720703,
-      "logits/rejected": -1.7081331014633179,
-      "logps/chosen": -1.2848669290542603,
-      "logps/rejected": -1.379817247390747,
-      "loss": 0.5831,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.1641167402267456,
-      "rewards/margins": 0.5224048495292664,
-      "rewards/rejected": -1.6865215301513672,
+      "epoch": 0.33,
+      "grad_norm": 100.68694545007727,
+      "learning_rate": 8.42018064959393e-09,
+      "logits/chosen": -1.8531291484832764,
+      "logits/rejected": -1.8357194662094116,
+      "logps/chosen": -1.078748345375061,
+      "logps/rejected": -1.0813863277435303,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.007160305976867676,
+      "rewards/margins": 0.007577991578727961,
+      "rewards/rejected": -0.0004176851361989975,
       "step": 100
     },
     {
-      "epoch": 0.25,
-      "grad_norm": 94.58328257186776,
-      "learning_rate": 9.326809299301306e-08,
-      "logits/chosen": -1.7934032678604126,
-      "logits/rejected": -1.692413568496704,
-      "logps/chosen": -1.3193799257278442,
-      "logps/rejected": -1.4346280097961426,
-      "loss": 0.5776,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.1873786449432373,
-      "rewards/margins": 0.6364501714706421,
-      "rewards/rejected": -1.8238286972045898,
+      "epoch": 0.37,
+      "grad_norm": 91.99753152902545,
+      "learning_rate": 7.971896853961043e-09,
+      "logits/chosen": -1.8346866369247437,
+      "logits/rejected": -1.8052761554718018,
+      "logps/chosen": -1.0582704544067383,
+      "logps/rejected": -1.0159928798675537,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0007529235444962978,
+      "rewards/margins": 0.0036198147572577,
+      "rewards/rejected": -0.004372738301753998,
       "step": 110
     },
     {
-      "epoch": 0.27,
-      "grad_norm": 123.65835816529943,
-      "learning_rate": 9.114050282021158e-08,
-      "logits/chosen": -1.7867753505706787,
-      "logits/rejected": -1.7270488739013672,
-      "logps/chosen": -1.2704253196716309,
-      "logps/rejected": -1.373067855834961,
-      "loss": 0.5626,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.228627324104309,
-      "rewards/margins": 0.5525014400482178,
-      "rewards/rejected": -1.7811288833618164,
+      "epoch": 0.4,
+      "grad_norm": 76.03854240063603,
+      "learning_rate": 7.48312422757881e-09,
+      "logits/chosen": -1.8829187154769897,
+      "logits/rejected": -1.8501228094100952,
+      "logps/chosen": -1.0292680263519287,
+      "logps/rejected": -1.019852876663208,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.00865122489631176,
+      "rewards/margins": 0.011260826140642166,
+      "rewards/rejected": -0.0026096017099916935,
       "step": 120
     },
     {
-      "epoch": 0.3,
-      "grad_norm": 95.55607076940454,
-      "learning_rate": 8.875280914254802e-08,
-      "logits/chosen": -1.7855722904205322,
-      "logits/rejected": -1.6952784061431885,
-      "logps/chosen": -1.3185499906539917,
-      "logps/rejected": -1.4162604808807373,
-      "loss": 0.5666,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.2571992874145508,
-      "rewards/margins": 0.6928548812866211,
-      "rewards/rejected": -1.9500541687011719,
+      "epoch": 0.43,
+      "grad_norm": 91.54881621872774,
+      "learning_rate": 6.96052176068713e-09,
+      "logits/chosen": -1.7626311779022217,
+      "logits/rejected": -1.7299768924713135,
+      "logps/chosen": -1.0229580402374268,
+      "logps/rejected": -1.0213210582733154,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.010313736274838448,
+      "rewards/margins": 0.01158633828163147,
+      "rewards/rejected": -0.0012726020067930222,
       "step": 130
     },
     {
-      "epoch": 0.32,
-      "grad_norm": 78.06311583858023,
-      "learning_rate": 8.612010772821971e-08,
-      "logits/chosen": -1.8129308223724365,
-      "logits/rejected": -1.7682344913482666,
-      "logps/chosen": -1.3229562044143677,
-      "logps/rejected": -1.361987829208374,
-      "loss": 0.5656,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.0907716751098633,
-      "rewards/margins": 0.6394159197807312,
-      "rewards/rejected": -1.7301876544952393,
+      "epoch": 0.47,
+      "grad_norm": 88.02230042083127,
+      "learning_rate": 6.4112093379492135e-09,
+      "logits/chosen": -1.796229600906372,
+      "logits/rejected": -1.7824671268463135,
+      "logps/chosen": -1.078906774520874,
+      "logps/rejected": -1.0356519222259521,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.015195205807685852,
+      "rewards/margins": 0.011107077822089195,
+      "rewards/rejected": 0.004088127985596657,
       "step": 140
     },
     {
-      "epoch": 0.34,
-      "grad_norm": 78.88051547861711,
-      "learning_rate": 8.325904336322055e-08,
-      "logits/chosen": -1.7850853204727173,
-      "logits/rejected": -1.7291721105575562,
-      "logps/chosen": -1.2866876125335693,
-      "logps/rejected": -1.402489423751831,
-      "loss": 0.5657,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.1932275295257568,
-      "rewards/margins": 0.6862825155258179,
-      "rewards/rejected": -1.8795099258422852,
+      "epoch": 0.5,
+      "grad_norm": 89.70971937959732,
+      "learning_rate": 5.842670737842467e-09,
+      "logits/chosen": -1.8206145763397217,
+      "logits/rejected": -1.788490891456604,
+      "logps/chosen": -1.0580933094024658,
+      "logps/rejected": -1.0225986242294312,
+      "loss": 0.6877,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.004929685965180397,
+      "rewards/margins": 0.005628877319395542,
+      "rewards/rejected": -0.0006991913542151451,
       "step": 150
     },
     {
-      "epoch": 0.36,
-      "grad_norm": 80.61975465340338,
-      "learning_rate": 8.01877046176447e-08,
-      "logits/chosen": -1.7330862283706665,
-      "logits/rejected": -1.6600639820098877,
-      "logps/chosen": -1.312774419784546,
-      "logps/rejected": -1.419638991355896,
-      "loss": 0.566,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.3447433710098267,
-      "rewards/margins": 0.5547876358032227,
-      "rewards/rejected": -1.8995310068130493,
+      "epoch": 0.53,
+      "grad_norm": 81.17857900193553,
+      "learning_rate": 5.262651674395798e-09,
+      "logits/chosen": -1.8310163021087646,
+      "logits/rejected": -1.8322261571884155,
+      "logps/chosen": -1.0157067775726318,
+      "logps/rejected": -1.0004805326461792,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": 0.010493971407413483,
+      "rewards/margins": 0.013428708538413048,
+      "rewards/rejected": -0.0029347380623221397,
       "step": 160
     },
     {
-      "epoch": 0.39,
-      "grad_norm": 78.96860347790852,
-      "learning_rate": 7.692550948392249e-08,
-      "logits/chosen": -1.7879507541656494,
-      "logits/rejected": -1.7219922542572021,
-      "logps/chosen": -1.3235785961151123,
-      "logps/rejected": -1.3983043432235718,
-      "loss": 0.5589,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.1332305669784546,
-      "rewards/margins": 0.6428099870681763,
-      "rewards/rejected": -1.7760404348373413,
+      "epoch": 0.57,
+      "grad_norm": 87.11260929204627,
+      "learning_rate": 4.679054270342702e-09,
+      "logits/chosen": -1.8449046611785889,
+      "logits/rejected": -1.7946765422821045,
+      "logps/chosen": -1.0548999309539795,
+      "logps/rejected": -1.051992654800415,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.010436683893203735,
+      "rewards/margins": 0.010068513453006744,
+      "rewards/rejected": 0.00036817044019699097,
       "step": 170
     },
     {
-      "epoch": 0.41,
-      "grad_norm": 91.20361542856642,
-      "learning_rate": 7.349308261002021e-08,
-      "logits/chosen": -1.744236707687378,
-      "logits/rejected": -1.685158133506775,
-      "logps/chosen": -1.3169732093811035,
-      "logps/rejected": -1.422654390335083,
-      "loss": 0.5541,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.1988707780838013,
-      "rewards/margins": 0.5603083372116089,
-      "rewards/rejected": -1.7591791152954102,
+      "epoch": 0.6,
+      "grad_norm": 93.41415146032115,
+      "learning_rate": 4.099829399377524e-09,
+      "logits/chosen": -1.8277971744537354,
+      "logits/rejected": -1.7856277227401733,
+      "logps/chosen": -1.0608714818954468,
+      "logps/rejected": -1.0331629514694214,
+      "loss": 0.6855,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.015542459674179554,
+      "rewards/margins": 0.01706361211836338,
+      "rewards/rejected": -0.0015211515128612518,
       "step": 180
     },
     {
-      "epoch": 0.43,
-      "grad_norm": 85.52532511636508,
-      "learning_rate": 6.991212490377531e-08,
-      "logits/chosen": -1.7989540100097656,
-      "logits/rejected": -1.7449232339859009,
-      "logps/chosen": -1.341355562210083,
-      "logps/rejected": -1.4322634935379028,
-      "loss": 0.528,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.112524390220642,
-      "rewards/margins": 0.6936608552932739,
-      "rewards/rejected": -1.8061851263046265,
+      "epoch": 0.63,
+      "grad_norm": 91.20164752256586,
+      "learning_rate": 3.532868364233416e-09,
+      "logits/chosen": -1.8144668340682983,
+      "logits/rejected": -1.7934455871582031,
+      "logps/chosen": -1.0488895177841187,
+      "logps/rejected": -1.0484153032302856,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.015009616501629353,
+      "rewards/margins": 0.011918185278773308,
+      "rewards/rejected": 0.00309143029153347,
       "step": 190
     },
     {
-      "epoch": 0.46,
-      "grad_norm": 103.62294205062797,
-      "learning_rate": 6.620527633276978e-08,
-      "logits/chosen": -1.7379404306411743,
-      "logits/rejected": -1.6623157262802124,
-      "logps/chosen": -1.307138204574585,
-      "logps/rejected": -1.4737988710403442,
-      "loss": 0.54,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.126852035522461,
-      "rewards/margins": 0.7987160682678223,
-      "rewards/rejected": -1.9255679845809937,
+      "epoch": 0.67,
+      "grad_norm": 103.72123679470863,
+      "learning_rate": 2.985895386349233e-09,
+      "logits/chosen": -1.783926248550415,
+      "logits/rejected": -1.7509727478027344,
+      "logps/chosen": -1.033827543258667,
+      "logps/rejected": -1.0074162483215332,
+      "loss": 0.6856,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.018476296216249466,
+      "rewards/margins": 0.019362105056643486,
+      "rewards/rejected": -0.0008858110522851348,
       "step": 200
     },
     {
-      "epoch": 0.48,
-      "grad_norm": 97.12306479641116,
-      "learning_rate": 6.239597278716581e-08,
-      "logits/chosen": -1.8088064193725586,
-      "logits/rejected": -1.74566650390625,
-      "logps/chosen": -1.3601690530776978,
-      "logps/rejected": -1.421120285987854,
-      "loss": 0.5357,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.1483900547027588,
-      "rewards/margins": 0.8161466717720032,
-      "rewards/rejected": -1.9645369052886963,
+      "epoch": 0.7,
+      "grad_norm": 86.58629109761597,
+      "learning_rate": 2.4663623718355446e-09,
+      "logits/chosen": -1.842024803161621,
+      "logits/rejected": -1.8078495264053345,
+      "logps/chosen": -1.0763428211212158,
+      "logps/rejected": -1.0434763431549072,
+      "loss": 0.6853,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.018699195235967636,
+      "rewards/margins": 0.017539886757731438,
+      "rewards/rejected": 0.0011593066155910492,
       "step": 210
     },
     {
-      "epoch": 0.5,
-      "grad_norm": 73.94190937238754,
-      "learning_rate": 5.8508297910462456e-08,
-      "logits/chosen": -1.75725519657135,
-      "logits/rejected": -1.6749064922332764,
-      "logps/chosen": -1.2782678604125977,
-      "logps/rejected": -1.4215964078903198,
-      "loss": 0.5266,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -1.147524356842041,
-      "rewards/margins": 0.8555541038513184,
-      "rewards/rejected": -2.0030786991119385,
+      "epoch": 0.74,
+      "grad_norm": 94.3943636617196,
+      "learning_rate": 1.9813473874379397e-09,
+      "logits/chosen": -1.732317328453064,
+      "logits/rejected": -1.7312743663787842,
+      "logps/chosen": -1.073425054550171,
+      "logps/rejected": -1.0629937648773193,
+      "loss": 0.6847,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.013539738953113556,
+      "rewards/margins": 0.013838117942214012,
+      "rewards/rejected": -0.00029837898910045624,
       "step": 220
     },
     {
-      "epoch": 0.52,
-      "grad_norm": 102.01553665817238,
-      "learning_rate": 5.456683083494731e-08,
-      "logits/chosen": -1.7501509189605713,
-      "logits/rejected": -1.7102434635162354,
-      "logps/chosen": -1.2995601892471313,
-      "logps/rejected": -1.3736200332641602,
-      "loss": 0.5541,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.057513952255249,
-      "rewards/margins": 0.5031803846359253,
-      "rewards/rejected": -1.5606944561004639,
+      "epoch": 0.77,
+      "grad_norm": 92.64370783949347,
+      "learning_rate": 1.5374582296511053e-09,
+      "logits/chosen": -1.7242523431777954,
+      "logits/rejected": -1.6965806484222412,
+      "logps/chosen": -1.0366003513336182,
+      "logps/rejected": -0.9941840171813965,
+      "loss": 0.6857,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.021335098892450333,
+      "rewards/margins": 0.015094568021595478,
+      "rewards/rejected": 0.006240529473870993,
       "step": 230
     },
     {
-      "epoch": 0.55,
-      "grad_norm": 111.25877684367076,
-      "learning_rate": 5.059649078450834e-08,
-      "logits/chosen": -1.7446762323379517,
-      "logits/rejected": -1.6967140436172485,
-      "logps/chosen": -1.2858344316482544,
-      "logps/rejected": -1.407447338104248,
-      "loss": 0.5275,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.174809217453003,
-      "rewards/margins": 0.595741868019104,
-      "rewards/rejected": -1.770551085472107,
+      "epoch": 0.8,
+      "grad_norm": 91.14853265672438,
+      "learning_rate": 1.1407424007485927e-09,
+      "logits/chosen": -1.881166696548462,
+      "logits/rejected": -1.8595256805419922,
+      "logps/chosen": -1.0768239498138428,
+      "logps/rejected": -1.0254974365234375,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.021426241844892502,
+      "rewards/margins": 0.021464312449097633,
+      "rewards/rejected": -3.8067344576120377e-05,
       "step": 240
     },
     {
-      "epoch": 0.57,
-      "grad_norm": 89.74032522739078,
-      "learning_rate": 4.6622379527277186e-08,
-      "logits/chosen": -1.7489802837371826,
-      "logits/rejected": -1.6957324743270874,
-      "logps/chosen": -1.262298345565796,
-      "logps/rejected": -1.361208438873291,
-      "loss": 0.5316,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -1.2340971231460571,
-      "rewards/margins": 0.6583037376403809,
-      "rewards/rejected": -1.8924009799957275,
+      "epoch": 0.84,
+      "grad_norm": 88.07457325362287,
+      "learning_rate": 7.966047182060226e-10,
+      "logits/chosen": -1.8788058757781982,
+      "logits/rejected": -1.881800889968872,
+      "logps/chosen": -1.0638504028320312,
+      "logps/rejected": -1.028044581413269,
+      "loss": 0.6828,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.018116505816578865,
+      "rewards/margins": 0.015344863757491112,
+      "rewards/rejected": 0.0027716427575796843,
       "step": 250
     },
     {
-      "epoch": 0.59,
-      "grad_norm": 75.51753806333156,
-      "learning_rate": 4.26696226741691e-08,
-      "logits/chosen": -1.7651500701904297,
-      "logits/rejected": -1.6915719509124756,
-      "logps/chosen": -1.3747735023498535,
-      "logps/rejected": -1.4673488140106201,
-      "loss": 0.5304,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.4194996356964111,
-      "rewards/margins": 0.7260831594467163,
-      "rewards/rejected": -2.145582914352417,
+      "epoch": 0.87,
+      "grad_norm": 99.87255814500503,
+      "learning_rate": 5.097336799988067e-10,
+      "logits/chosen": -1.8842456340789795,
+      "logits/rejected": -1.8642467260360718,
+      "logps/chosen": -1.0772615671157837,
+      "logps/rejected": -1.0665724277496338,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.023015262559056282,
+      "rewards/margins": 0.02001366578042507,
+      "rewards/rejected": 0.0030015967786312103,
       "step": 260
     },
     {
-      "epoch": 0.61,
-      "grad_norm": 81.81714896589709,
-      "learning_rate": 3.876321082668098e-08,
-      "logits/chosen": -1.8280465602874756,
-      "logits/rejected": -1.7529118061065674,
-      "logps/chosen": -1.3470970392227173,
-      "logps/rejected": -1.435733437538147,
-      "loss": 0.5229,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.0860626697540283,
-      "rewards/margins": 0.7932750582695007,
-      "rewards/rejected": -1.8793376684188843,
+      "epoch": 0.9,
+      "grad_norm": 107.74820920122671,
+      "learning_rate": 2.840375889663871e-10,
+      "logits/chosen": -1.8729417324066162,
+      "logits/rejected": -1.8372119665145874,
+      "logps/chosen": -0.9936866760253906,
+      "logps/rejected": -1.0166294574737549,
+      "loss": 0.6847,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.01751965843141079,
+      "rewards/margins": 0.02036522701382637,
+      "rewards/rejected": -0.0028455646242946386,
       "step": 270
     },
     {
-      "epoch": 0.64,
-      "grad_norm": 88.74965472733419,
-      "learning_rate": 3.492784157826244e-08,
-      "logits/chosen": -1.7529668807983398,
-      "logits/rejected": -1.6530554294586182,
-      "logps/chosen": -1.3322269916534424,
-      "logps/rejected": -1.4489867687225342,
-      "loss": 0.5263,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.0935055017471313,
-      "rewards/margins": 0.8137003779411316,
-      "rewards/rejected": -1.9072058200836182,
+      "epoch": 0.94,
+      "grad_norm": 87.56807066875254,
+      "learning_rate": 1.2259130647833627e-10,
+      "logits/chosen": -1.7625566720962524,
+      "logits/rejected": -1.7224514484405518,
+      "logps/chosen": -1.093827724456787,
+      "logps/rejected": -1.0462143421173096,
+      "loss": 0.6834,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": 0.01200016774237156,
+      "rewards/margins": 0.019106844440102577,
+      "rewards/rejected": -0.0071066757664084435,
       "step": 280
     },
     {
-      "epoch": 0.66,
-      "grad_norm": 76.0414974261225,
-      "learning_rate": 3.118776336817812e-08,
-      "logits/chosen": -1.7995818853378296,
-      "logits/rejected": -1.7317774295806885,
-      "logps/chosen": -1.3204050064086914,
-      "logps/rejected": -1.4101346731185913,
-      "loss": 0.5231,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.1715632677078247,
-      "rewards/margins": 0.8251334428787231,
-      "rewards/rejected": -1.9966968297958374,
+      "epoch": 0.97,
+      "grad_norm": 81.16688874976322,
+      "learning_rate": 2.7594360825166643e-11,
+      "logits/chosen": -1.780118703842163,
+      "logits/rejected": -1.7672898769378662,
+      "logps/chosen": -1.0616685152053833,
+      "logps/rejected": -1.0320245027542114,
+      "loss": 0.6845,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.028133947402238846,
+      "rewards/margins": 0.020768892019987106,
+      "rewards/rejected": 0.007365054450929165,
       "step": 290
     },
-    {
-      "epoch": 0.68,
-      "grad_norm": 87.86932907500237,
-      "learning_rate": 2.7566622175067443e-08,
-      "logits/chosen": -1.7801201343536377,
-      "logits/rejected": -1.7111116647720337,
-      "logps/chosen": -1.3444006443023682,
-      "logps/rejected": -1.494860053062439,
-      "loss": 0.5308,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.3581712245941162,
-      "rewards/margins": 0.7247291803359985,
-      "rewards/rejected": -2.082900285720825,
-      "step": 300
-    },
-    {
-      "epoch": 0.71,
-      "grad_norm": 101.39189225311264,
-      "learning_rate": 2.408731201945432e-08,
-      "logits/chosen": -1.77533757686615,
-      "logits/rejected": -1.7264738082885742,
-      "logps/chosen": -1.313521146774292,
-      "logps/rejected": -1.4012606143951416,
-      "loss": 0.5179,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.1962579488754272,
-      "rewards/margins": 0.6758478283882141,
-      "rewards/rejected": -1.8721058368682861,
-      "step": 310
-    },
-    {
-      "epoch": 0.73,
-      "grad_norm": 97.55308734787654,
-      "learning_rate": 2.0771830220378112e-08,
-      "logits/chosen": -1.7309653759002686,
-      "logits/rejected": -1.673135757446289,
-      "logps/chosen": -1.3468959331512451,
-      "logps/rejected": -1.405505895614624,
-      "loss": 0.5257,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.240114450454712,
-      "rewards/margins": 0.6646026968955994,
-      "rewards/rejected": -1.9047170877456665,
-      "step": 320
-    },
-    {
-      "epoch": 0.75,
-      "grad_norm": 79.56679512217426,
-      "learning_rate": 1.7641138321260257e-08,
-      "logits/chosen": -1.7662233114242554,
-      "logits/rejected": -1.6911147832870483,
-      "logps/chosen": -1.3011645078659058,
-      "logps/rejected": -1.418474555015564,
-      "loss": 0.5175,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.1730973720550537,
-      "rewards/margins": 0.8977500796318054,
-      "rewards/rejected": -2.070847511291504,
-      "step": 330
-    },
-    {
-      "epoch": 0.77,
-      "grad_norm": 78.95443991883164,
-      "learning_rate": 1.4715029564277793e-08,
-      "logits/chosen": -1.8377494812011719,
-      "logits/rejected": -1.785035490989685,
-      "logps/chosen": -1.3052855730056763,
-      "logps/rejected": -1.4216973781585693,
-      "loss": 0.535,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.0062506198883057,
-      "rewards/margins": 0.7934825420379639,
-      "rewards/rejected": -1.7997331619262695,
-      "step": 340
-    },
-    {
-      "epoch": 0.8,
-      "grad_norm": 85.7404253023672,
-      "learning_rate": 1.2012003751113343e-08,
-      "logits/chosen": -1.8215789794921875,
-      "logits/rejected": -1.7556202411651611,
-      "logps/chosen": -1.2944856882095337,
-      "logps/rejected": -1.427002191543579,
-      "loss": 0.5037,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.3231576681137085,
-      "rewards/margins": 0.8232595324516296,
-      "rewards/rejected": -2.1464171409606934,
-      "step": 350
-    },
-    {
-      "epoch": 0.82,
-      "grad_norm": 81.80824877374742,
-      "learning_rate": 9.549150281252633e-09,
-      "logits/chosen": -1.7837785482406616,
-      "logits/rejected": -1.7289537191390991,
-      "logps/chosen": -1.2906091213226318,
-      "logps/rejected": -1.4211328029632568,
-      "loss": 0.5085,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.1847398281097412,
-      "rewards/margins": 0.7879296541213989,
-      "rewards/rejected": -1.9726696014404297,
-      "step": 360
-    },
-    {
-      "epoch": 0.84,
-      "grad_norm": 76.45542191386993,
-      "learning_rate": 7.3420401072985306e-09,
-      "logits/chosen": -1.8155953884124756,
-      "logits/rejected": -1.7616589069366455,
-      "logps/chosen": -1.3416122198104858,
-      "logps/rejected": -1.4825923442840576,
-      "loss": 0.512,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.2699860334396362,
-      "rewards/margins": 0.8351644277572632,
-      "rewards/rejected": -2.1051506996154785,
-      "step": 370
-    },
-    {
-      "epoch": 0.86,
-      "grad_norm": 100.61500636796463,
-      "learning_rate": 5.404627290395369e-09,
-      "logits/chosen": -1.7809003591537476,
-      "logits/rejected": -1.7117547988891602,
-      "logps/chosen": -1.299863338470459,
-      "logps/rejected": -1.4180821180343628,
-      "loss": 0.5139,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.1025749444961548,
-      "rewards/margins": 0.7822272777557373,
-      "rewards/rejected": -1.884802222251892,
-      "step": 380
-    },
-    {
-      "epoch": 0.89,
-      "grad_norm": 143.00077069598655,
-      "learning_rate": 3.74916077816162e-09,
-      "logits/chosen": -1.7901942729949951,
-      "logits/rejected": -1.7254260778427124,
-      "logps/chosen": -1.2824052572250366,
-      "logps/rejected": -1.3659498691558838,
-      "loss": 0.5278,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.326103925704956,
-      "rewards/margins": 0.6741579174995422,
-      "rewards/rejected": -2.0002617835998535,
-      "step": 390
-    },
-    {
-      "epoch": 0.91,
-      "grad_norm": 86.78461219124101,
-      "learning_rate": 2.386106962899165e-09,
-      "logits/chosen": -1.7145483493804932,
-      "logits/rejected": -1.6393749713897705,
-      "logps/chosen": -1.3581172227859497,
-      "logps/rejected": -1.4866178035736084,
-      "loss": 0.5223,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.3169643878936768,
-      "rewards/margins": 0.7216087579727173,
-      "rewards/rejected": -2.0385732650756836,
-      "step": 400
-    },
-    {
-      "epoch": 0.93,
-      "grad_norm": 97.03178172617403,
-      "learning_rate": 1.3240835096913706e-09,
-      "logits/chosen": -1.7489525079727173,
-      "logits/rejected": -1.6416590213775635,
-      "logps/chosen": -1.2802146673202515,
-      "logps/rejected": -1.4399700164794922,
-      "loss": 0.5334,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.086639404296875,
-      "rewards/margins": 0.9034107327461243,
-      "rewards/rejected": -1.9900500774383545,
-      "step": 410
-    },
-    {
-      "epoch": 0.96,
-      "grad_norm": 177.7844461853396,
-      "learning_rate": 5.698048727497462e-10,
-      "logits/chosen": -1.7718130350112915,
-      "logits/rejected": -1.6983397006988525,
-      "logps/chosen": -1.3178266286849976,
-      "logps/rejected": -1.4721015691757202,
-      "loss": 0.5219,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -1.2652876377105713,
-      "rewards/margins": 0.8731800317764282,
-      "rewards/rejected": -2.138467788696289,
-      "step": 420
-    },
-    {
-      "epoch": 0.98,
-      "grad_norm": 191.4766363479098,
-      "learning_rate": 1.2803984447259387e-10,
-      "logits/chosen": -1.7818502187728882,
-      "logits/rejected": -1.721131682395935,
-      "logps/chosen": -1.3105113506317139,
-      "logps/rejected": -1.4450818300247192,
-      "loss": 0.5113,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.2817203998565674,
-      "rewards/margins": 0.869175910949707,
-      "rewards/rejected": -2.1508963108062744,
-      "step": 430
-    },
     {
       "epoch": 1.0,
-      "step": 439,
+      "step": 299,
       "total_flos": 0.0,
-      "train_loss": 0.5564739256078942,
-      "train_runtime": 6859.6403,
-      "train_samples_per_second": 8.198,
-      "train_steps_per_second": 0.064
+      "train_loss": 0.6880149315033469,
+      "train_runtime": 4603.368,
+      "train_samples_per_second": 8.317,
+      "train_steps_per_second": 0.065
     }
   ],
   "logging_steps": 10,
-  "max_steps": 439,
+  "max_steps": 299,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:26744db2cfaad1ba924f198a0f7777d66f0f42a1cfd7425001f46a0c8be3dcff
-size 6520
+oid sha256:5fc17d04ff8999b784673094bfc94f340bf1b82c104ffe64c18c7d2f2ea8b9f4
+size 6328