Model save

README.md

@@ -1,4 +1,5 @@
 ---
+base_model: princeton-nlp/Mistral-7B-Base-SFT-SimPO
 tags:
 - trl
 - dpo
@@ -13,7 +14,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model was trained from scratch on the None dataset.
+This model is a fine-tuned version of [princeton-nlp/Mistral-7B-Base-SFT-SimPO](https://huggingface.co/princeton-nlp/Mistral-7B-Base-SFT-SimPO) on the None dataset.
 
 ## Model description
 
@@ -32,7 +33,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 1e-08
+- learning_rate: 1e-07
 - train_batch_size: 4
 - eval_batch_size: 4
 - seed: 42

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6880149315033469,
-    "train_runtime": 4603.368,
-    "train_samples": 38288,
-    "train_samples_per_second": 8.317,
-    "train_steps_per_second": 0.065
+    "train_loss": 0.5309282744935542,
+    "train_runtime": 6847.998,
+    "train_samples": 56236,
+    "train_samples_per_second": 8.212,
+    "train_steps_per_second": 0.064
 }

config.json

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.41.1",
+  "transformers_version": "4.39.3",
   "use_cache": false,
   "vocab_size": 32000
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:25a506977023960c048f2933f9aec9fc01ac4cef4294a89c59fb94311d62d0ba
+oid sha256:c8d089e259179f6b43c527a368edf999eec073dc01cfa73c90adde6188d1928d
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f36b418c6f0204d63a482c4d40cf9a099d1c71913006174de004f3468b2600ac
+oid sha256:72900a606cc1d8493faaa0946c473002a0400edf1f17c01905fe2a2232dc1be1
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0098c8b3746d7ba8d54ad9a79ed49c5d66268bbf89e84d00df21498d52d9cb42
+oid sha256:2cba821d53166885228308afe33150d7a837fce90e03432661986a37f3edccc4
 size 4540516344

runs/Jul03_22-31-37_n136-129-074/events.out.tfevents.1720017138.n136-129-074.3171670.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a0e66f77295f3829902375295d790633724c034ef007948becbc276f0fa9a23
-size 32937
+oid sha256:80a2a5c55073794b839c2710274799cbb2c7b6ea7aac51eeb82b1adc87fa81e1
+size 35355

tokenizer.json

@@ -134,7 +134,6 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
-    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6880149315033469,
-    "train_runtime": 4603.368,
-    "train_samples": 38288,
-    "train_samples_per_second": 8.317,
-    "train_steps_per_second": 0.065
+    "train_loss": 0.5309282744935542,
+    "train_runtime": 6847.998,
+    "train_samples": 56236,
+    "train_samples_per_second": 8.212,
+    "train_steps_per_second": 0.064
 }

trainer_state.json

@@ -1,21 +1,21 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9991645781119465,
+  "epoch": 0.9988623435722411,
   "eval_steps": 10000000,
-  "global_step": 299,
+  "global_step": 439,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 94.51455806402136,
-      "learning_rate": 3.333333333333333e-10,
-      "logits/chosen": -1.693521499633789,
-      "logits/rejected": -1.6753541231155396,
-      "logps/chosen": -1.041430115699768,
-      "logps/rejected": -0.9273841977119446,
+      "grad_norm": 120.07926705665244,
+      "learning_rate": 2.2727272727272727e-09,
+      "logits/chosen": -1.6768856048583984,
+      "logits/rejected": -1.7259055376052856,
+      "logps/chosen": -394.9654541015625,
+      "logps/rejected": -320.0859069824219,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -24,452 +24,662 @@
       "step": 1
     },
     {
-      "epoch": 0.03,
-      "grad_norm": 100.10048800952612,
-      "learning_rate": 3.3333333333333334e-09,
-      "logits/chosen": -1.7426362037658691,
-      "logits/rejected": -1.7463488578796387,
-      "logps/chosen": -1.0522818565368652,
-      "logps/rejected": -1.0174607038497925,
-      "loss": 0.6933,
-      "rewards/accuracies": 0.4722222089767456,
-      "rewards/chosen": -0.0006895202095620334,
-      "rewards/margins": -0.0014206942869350314,
-      "rewards/rejected": 0.0007311741355806589,
+      "epoch": 0.02,
+      "grad_norm": 160.8036403422587,
+      "learning_rate": 2.2727272727272725e-08,
+      "logits/chosen": -1.703018069267273,
+      "logits/rejected": -1.6685585975646973,
+      "logps/chosen": -429.5360412597656,
+      "logps/rejected": -403.7555236816406,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.0004270696663297713,
+      "rewards/margins": -0.0031983989756554365,
+      "rewards/rejected": 0.00277132960036397,
       "step": 10
     },
     {
-      "epoch": 0.07,
-      "grad_norm": 100.2747717498339,
-      "learning_rate": 6.666666666666667e-09,
-      "logits/chosen": -1.876529335975647,
-      "logits/rejected": -1.8286478519439697,
-      "logps/chosen": -1.0717421770095825,
-      "logps/rejected": -1.0434354543685913,
-      "loss": 0.6934,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": 0.0013453494757413864,
-      "rewards/margins": 0.001054912805557251,
-      "rewards/rejected": 0.00029043667018413544,
+      "epoch": 0.05,
+      "grad_norm": 241.66358094434165,
+      "learning_rate": 4.545454545454545e-08,
+      "logits/chosen": -1.780827283859253,
+      "logits/rejected": -1.7355620861053467,
+      "logps/chosen": -442.1951599121094,
+      "logps/rejected": -401.2707214355469,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.002479883376508951,
+      "rewards/margins": 0.002258532214909792,
+      "rewards/rejected": -0.004738415591418743,
       "step": 20
     },
     {
-      "epoch": 0.1,
-      "grad_norm": 91.78075458726622,
-      "learning_rate": 1e-08,
-      "logits/chosen": -1.7869594097137451,
-      "logits/rejected": -1.7599306106567383,
-      "logps/chosen": -1.066955327987671,
-      "logps/rejected": -1.0448154211044312,
-      "loss": 0.6926,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": 0.0009901206940412521,
-      "rewards/margins": 0.005651786923408508,
-      "rewards/rejected": -0.004661666229367256,
+      "epoch": 0.07,
+      "grad_norm": 160.69914583171965,
+      "learning_rate": 6.818181818181817e-08,
+      "logits/chosen": -1.7514013051986694,
+      "logits/rejected": -1.6820430755615234,
+      "logps/chosen": -439.64373779296875,
+      "logps/rejected": -401.2854919433594,
+      "loss": 0.6738,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.0074474625289440155,
+      "rewards/margins": 0.054817844182252884,
+      "rewards/rejected": -0.04737037047743797,
       "step": 30
     },
     {
-      "epoch": 0.13,
-      "grad_norm": 83.1668176080734,
-      "learning_rate": 9.96594024562513e-09,
-      "logits/chosen": -1.8331626653671265,
-      "logits/rejected": -1.8235479593276978,
-      "logps/chosen": -1.069947361946106,
-      "logps/rejected": -1.0194944143295288,
-      "loss": 0.6933,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.00406806543469429,
-      "rewards/margins": 0.006294439546763897,
-      "rewards/rejected": -0.0022263741120696068,
+      "epoch": 0.09,
+      "grad_norm": 144.27361385062915,
+      "learning_rate": 9.09090909090909e-08,
+      "logits/chosen": -1.7552951574325562,
+      "logits/rejected": -1.6888301372528076,
+      "logps/chosen": -428.416015625,
+      "logps/rejected": -383.068603515625,
+      "loss": 0.6579,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.1923234909772873,
+      "rewards/margins": 0.1899619996547699,
+      "rewards/rejected": 0.0023614875972270966,
       "step": 40
     },
     {
-      "epoch": 0.17,
-      "grad_norm": 97.3484905394127,
-      "learning_rate": 9.86422500924775e-09,
-      "logits/chosen": -1.8598533868789673,
-      "logits/rejected": -1.8249794244766235,
-      "logps/chosen": -1.0844773054122925,
-      "logps/rejected": -1.0546021461486816,
-      "loss": 0.6928,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.0010452494025230408,
-      "rewards/margins": 0.003718096762895584,
-      "rewards/rejected": -0.0026728473603725433,
+      "epoch": 0.11,
+      "grad_norm": 115.83588190821067,
+      "learning_rate": 9.994307990108962e-08,
+      "logits/chosen": -1.7498763799667358,
+      "logits/rejected": -1.6867773532867432,
+      "logps/chosen": -437.7354431152344,
+      "logps/rejected": -383.8248596191406,
+      "loss": 0.6152,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.5490495562553406,
+      "rewards/margins": 0.33619847893714905,
+      "rewards/rejected": 0.2128511667251587,
       "step": 50
     },
     {
-      "epoch": 0.2,
-      "grad_norm": 89.15270505321199,
-      "learning_rate": 9.696240049254743e-09,
-      "logits/chosen": -1.8162918090820312,
-      "logits/rejected": -1.8095808029174805,
-      "logps/chosen": -1.1147956848144531,
-      "logps/rejected": -1.10584557056427,
-      "loss": 0.692,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.003025440499186516,
-      "rewards/margins": 0.006887376308441162,
-      "rewards/rejected": -0.0038619358092546463,
+      "epoch": 0.14,
+      "grad_norm": 111.21906155896082,
+      "learning_rate": 9.959570405988094e-08,
+      "logits/chosen": -1.8246532678604126,
+      "logits/rejected": -1.7464463710784912,
+      "logps/chosen": -379.9985046386719,
+      "logps/rejected": -352.2860412597656,
+      "loss": 0.614,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": 0.7562192678451538,
+      "rewards/margins": 0.2702825665473938,
+      "rewards/rejected": 0.4859367311000824,
       "step": 60
     },
     {
-      "epoch": 0.23,
-      "grad_norm": 92.66102585547152,
-      "learning_rate": 9.464273976236516e-09,
-      "logits/chosen": -1.7991975545883179,
-      "logits/rejected": -1.7613614797592163,
-      "logps/chosen": -1.0598714351654053,
-      "logps/rejected": -1.0530080795288086,
-      "loss": 0.6927,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": 0.0042045507580041885,
-      "rewards/margins": 0.0019652042537927628,
-      "rewards/rejected": 0.002239346969872713,
+      "epoch": 0.16,
+      "grad_norm": 114.01926694263507,
+      "learning_rate": 9.893476820924666e-08,
+      "logits/chosen": -1.9211199283599854,
+      "logits/rejected": -1.8387491703033447,
+      "logps/chosen": -412.0660095214844,
+      "logps/rejected": -374.1966247558594,
+      "loss": 0.5841,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 1.0015352964401245,
+      "rewards/margins": 0.3862503468990326,
+      "rewards/rejected": 0.6152850389480591,
       "step": 70
     },
     {
-      "epoch": 0.27,
-      "grad_norm": 194.03821825764928,
-      "learning_rate": 9.171487073181198e-09,
-      "logits/chosen": -1.8440383672714233,
-      "logits/rejected": -1.8225574493408203,
-      "logps/chosen": -1.0219953060150146,
-      "logps/rejected": -1.0044893026351929,
-      "loss": 0.6914,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.0038851741701364517,
-      "rewards/margins": 0.009898573160171509,
-      "rewards/rejected": -0.006013398990035057,
+      "epoch": 0.18,
+      "grad_norm": 143.95597429946028,
+      "learning_rate": 9.796445099843647e-08,
+      "logits/chosen": -1.9127334356307983,
+      "logits/rejected": -1.8274517059326172,
+      "logps/chosen": -424.8004455566406,
+      "logps/rejected": -390.2203063964844,
+      "loss": 0.6144,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.9376031160354614,
+      "rewards/margins": 0.4474514126777649,
+      "rewards/rejected": 0.4901517331600189,
       "step": 80
     },
     {
-      "epoch": 0.3,
-      "grad_norm": 89.9314461991181,
-      "learning_rate": 8.821868240089676e-09,
-      "logits/chosen": -1.8029550313949585,
-      "logits/rejected": -1.7841434478759766,
-      "logps/chosen": -1.045693039894104,
-      "logps/rejected": -1.0079419612884521,
-      "loss": 0.6912,
-      "rewards/accuracies": 0.48124998807907104,
-      "rewards/chosen": 0.005337424576282501,
-      "rewards/margins": 0.004554492421448231,
-      "rewards/rejected": 0.0007829321548342705,
+      "epoch": 0.2,
+      "grad_norm": 126.42198014014429,
+      "learning_rate": 9.669088708527066e-08,
+      "logits/chosen": -1.8573029041290283,
+      "logits/rejected": -1.7926852703094482,
+      "logps/chosen": -435.7537536621094,
+      "logps/rejected": -401.1055908203125,
+      "loss": 0.5739,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.8144777417182922,
+      "rewards/margins": 0.4378415048122406,
+      "rewards/rejected": 0.376636266708374,
       "step": 90
     },
     {
-      "epoch": 0.33,
-      "grad_norm": 100.68694545007727,
-      "learning_rate": 8.42018064959393e-09,
-      "logits/chosen": -1.8531291484832764,
-      "logits/rejected": -1.8357194662094116,
-      "logps/chosen": -1.078748345375061,
-      "logps/rejected": -1.0813863277435303,
-      "loss": 0.6903,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": 0.007160305976867676,
-      "rewards/margins": 0.007577991578727961,
-      "rewards/rejected": -0.0004176851361989975,
+      "epoch": 0.23,
+      "grad_norm": 132.16216232500156,
+      "learning_rate": 9.512212835085849e-08,
+      "logits/chosen": -1.898048758506775,
+      "logits/rejected": -1.813433051109314,
+      "logps/chosen": -413.4141540527344,
+      "logps/rejected": -398.70526123046875,
+      "loss": 0.5603,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.6169204115867615,
+      "rewards/margins": 0.5242881178855896,
+      "rewards/rejected": 0.09263229370117188,
       "step": 100
     },
     {
-      "epoch": 0.37,
-      "grad_norm": 91.99753152902545,
-      "learning_rate": 7.971896853961043e-09,
-      "logits/chosen": -1.8346866369247437,
-      "logits/rejected": -1.8052761554718018,
-      "logps/chosen": -1.0582704544067383,
-      "logps/rejected": -1.0159928798675537,
-      "loss": 0.6901,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.0007529235444962978,
-      "rewards/margins": 0.0036198147572577,
-      "rewards/rejected": -0.004372738301753998,
+      "epoch": 0.25,
+      "grad_norm": 112.13374239362014,
+      "learning_rate": 9.326809299301306e-08,
+      "logits/chosen": -1.89919114112854,
+      "logits/rejected": -1.7970205545425415,
+      "logps/chosen": -454.9283142089844,
+      "logps/rejected": -405.9847717285156,
+      "loss": 0.5581,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.4878689646720886,
+      "rewards/margins": 0.6205599308013916,
+      "rewards/rejected": -0.1326909363269806,
       "step": 110
     },
     {
-      "epoch": 0.4,
-      "grad_norm": 76.03854240063603,
-      "learning_rate": 7.48312422757881e-09,
-      "logits/chosen": -1.8829187154769897,
-      "logits/rejected": -1.8501228094100952,
-      "logps/chosen": -1.0292680263519287,
-      "logps/rejected": -1.019852876663208,
-      "loss": 0.6887,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.00865122489631176,
-      "rewards/margins": 0.011260826140642166,
-      "rewards/rejected": -0.0026096017099916935,
+      "epoch": 0.27,
+      "grad_norm": 113.80543806203703,
+      "learning_rate": 9.114050282021158e-08,
+      "logits/chosen": -1.8961639404296875,
+      "logits/rejected": -1.836488127708435,
+      "logps/chosen": -453.51898193359375,
+      "logps/rejected": -419.9806213378906,
+      "loss": 0.5372,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.4265638291835785,
+      "rewards/margins": 0.6399323344230652,
+      "rewards/rejected": -0.21336853504180908,
       "step": 120
     },
     {
-      "epoch": 0.43,
-      "grad_norm": 91.54881621872774,
-      "learning_rate": 6.96052176068713e-09,
-      "logits/chosen": -1.7626311779022217,
-      "logits/rejected": -1.7299768924713135,
-      "logps/chosen": -1.0229580402374268,
-      "logps/rejected": -1.0213210582733154,
-      "loss": 0.6874,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.010313736274838448,
-      "rewards/margins": 0.01158633828163147,
-      "rewards/rejected": -0.0012726020067930222,
+      "epoch": 0.3,
+      "grad_norm": 122.38937390797612,
+      "learning_rate": 8.875280914254802e-08,
+      "logits/chosen": -1.9070549011230469,
+      "logits/rejected": -1.8174508810043335,
+      "logps/chosen": -408.0248107910156,
+      "logps/rejected": -365.46185302734375,
+      "loss": 0.5379,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.505743145942688,
+      "rewards/margins": 0.6783953309059143,
+      "rewards/rejected": -0.17265217006206512,
       "step": 130
     },
     {
-      "epoch": 0.47,
-      "grad_norm": 88.02230042083127,
-      "learning_rate": 6.4112093379492135e-09,
-      "logits/chosen": -1.796229600906372,
-      "logits/rejected": -1.7824671268463135,
-      "logps/chosen": -1.078906774520874,
-      "logps/rejected": -1.0356519222259521,
-      "loss": 0.6887,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.015195205807685852,
-      "rewards/margins": 0.011107077822089195,
-      "rewards/rejected": 0.004088127985596657,
+      "epoch": 0.32,
+      "grad_norm": 104.81942476289146,
+      "learning_rate": 8.612010772821971e-08,
+      "logits/chosen": -1.9492871761322021,
+      "logits/rejected": -1.903029203414917,
+      "logps/chosen": -463.1678771972656,
+      "logps/rejected": -408.36138916015625,
+      "loss": 0.5415,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.7421985268592834,
+      "rewards/margins": 0.7196205854415894,
+      "rewards/rejected": 0.02257799357175827,
       "step": 140
     },
     {
-      "epoch": 0.5,
-      "grad_norm": 89.70971937959732,
-      "learning_rate": 5.842670737842467e-09,
-      "logits/chosen": -1.8206145763397217,
-      "logits/rejected": -1.788490891456604,
-      "logps/chosen": -1.0580933094024658,
-      "logps/rejected": -1.0225986242294312,
-      "loss": 0.6877,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.004929685965180397,
-      "rewards/margins": 0.005628877319395542,
-      "rewards/rejected": -0.0006991913542151451,
+      "epoch": 0.34,
+      "grad_norm": 130.65997070751789,
+      "learning_rate": 8.325904336322055e-08,
+      "logits/chosen": -1.9297358989715576,
+      "logits/rejected": -1.872240424156189,
+      "logps/chosen": -408.3719482421875,
+      "logps/rejected": -374.2057189941406,
+      "loss": 0.5461,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.6860557198524475,
+      "rewards/margins": 0.6311514973640442,
+      "rewards/rejected": 0.05490417033433914,
       "step": 150
     },
     {
-      "epoch": 0.53,
-      "grad_norm": 81.17857900193553,
-      "learning_rate": 5.262651674395798e-09,
-      "logits/chosen": -1.8310163021087646,
-      "logits/rejected": -1.8322261571884155,
-      "logps/chosen": -1.0157067775726318,
-      "logps/rejected": -1.0004805326461792,
-      "loss": 0.6876,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": 0.010493971407413483,
-      "rewards/margins": 0.013428708538413048,
-      "rewards/rejected": -0.0029347380623221397,
+      "epoch": 0.36,
+      "grad_norm": 100.26257755554322,
+      "learning_rate": 8.01877046176447e-08,
+      "logits/chosen": -1.8793761730194092,
+      "logits/rejected": -1.8072710037231445,
+      "logps/chosen": -399.7383728027344,
+      "logps/rejected": -373.10791015625,
+      "loss": 0.5278,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.4368124008178711,
+      "rewards/margins": 0.6604553461074829,
+      "rewards/rejected": -0.22364301979541779,
       "step": 160
     },
     {
-      "epoch": 0.57,
-      "grad_norm": 87.11260929204627,
-      "learning_rate": 4.679054270342702e-09,
-      "logits/chosen": -1.8449046611785889,
-      "logits/rejected": -1.7946765422821045,
-      "logps/chosen": -1.0548999309539795,
-      "logps/rejected": -1.051992654800415,
-      "loss": 0.6869,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.010436683893203735,
-      "rewards/margins": 0.010068513453006744,
-      "rewards/rejected": 0.00036817044019699097,
+      "epoch": 0.39,
+      "grad_norm": 89.3633959368459,
+      "learning_rate": 7.692550948392249e-08,
+      "logits/chosen": -1.9449889659881592,
+      "logits/rejected": -1.8793401718139648,
+      "logps/chosen": -430.8158264160156,
+      "logps/rejected": -383.00640869140625,
+      "loss": 0.5275,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.5035308599472046,
+      "rewards/margins": 0.5975342988967896,
+      "rewards/rejected": -0.09400341659784317,
       "step": 170
     },
     {
-      "epoch": 0.6,
-      "grad_norm": 93.41415146032115,
-      "learning_rate": 4.099829399377524e-09,
-      "logits/chosen": -1.8277971744537354,
-      "logits/rejected": -1.7856277227401733,
-      "logps/chosen": -1.0608714818954468,
-      "logps/rejected": -1.0331629514694214,
-      "loss": 0.6855,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.015542459674179554,
-      "rewards/margins": 0.01706361211836338,
-      "rewards/rejected": -0.0015211515128612518,
+      "epoch": 0.41,
+      "grad_norm": 103.70599270965509,
+      "learning_rate": 7.349308261002021e-08,
+      "logits/chosen": -1.9056031703948975,
+      "logits/rejected": -1.8477048873901367,
+      "logps/chosen": -437.502197265625,
+      "logps/rejected": -403.87762451171875,
+      "loss": 0.526,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.6943656206130981,
+      "rewards/margins": 0.6242468357086182,
+      "rewards/rejected": 0.07011876255273819,
       "step": 180
     },
     {
-      "epoch": 0.63,
-      "grad_norm": 91.20164752256586,
-      "learning_rate": 3.532868364233416e-09,
-      "logits/chosen": -1.8144668340682983,
-      "logits/rejected": -1.7934455871582031,
-      "logps/chosen": -1.0488895177841187,
-      "logps/rejected": -1.0484153032302856,
-      "loss": 0.6869,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 0.015009616501629353,
-      "rewards/margins": 0.011918185278773308,
-      "rewards/rejected": 0.00309143029153347,
+      "epoch": 0.43,
+      "grad_norm": 84.20282962041543,
+      "learning_rate": 6.991212490377531e-08,
+      "logits/chosen": -1.9574388265609741,
+      "logits/rejected": -1.906002402305603,
+      "logps/chosen": -464.6742248535156,
+      "logps/rejected": -417.5992126464844,
+      "loss": 0.5064,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.6871397495269775,
+      "rewards/margins": 0.8371666669845581,
+      "rewards/rejected": -0.15002694725990295,
       "step": 190
     },
     {
-      "epoch": 0.67,
-      "grad_norm": 103.72123679470863,
-      "learning_rate": 2.985895386349233e-09,
-      "logits/chosen": -1.783926248550415,
-      "logits/rejected": -1.7509727478027344,
-      "logps/chosen": -1.033827543258667,
-      "logps/rejected": -1.0074162483215332,
-      "loss": 0.6856,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.018476296216249466,
-      "rewards/margins": 0.019362105056643486,
-      "rewards/rejected": -0.0008858110522851348,
+      "epoch": 0.46,
+      "grad_norm": 166.8394060310626,
+      "learning_rate": 6.620527633276978e-08,
+      "logits/chosen": -1.8844044208526611,
+      "logits/rejected": -1.8095699548721313,
+      "logps/chosen": -425.5460510253906,
+      "logps/rejected": -407.5855712890625,
+      "loss": 0.507,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.5698906779289246,
+      "rewards/margins": 0.8980382680892944,
+      "rewards/rejected": -0.3281475901603699,
       "step": 200
     },
     {
-      "epoch": 0.7,
-      "grad_norm": 86.58629109761597,
-      "learning_rate": 2.4663623718355446e-09,
-      "logits/chosen": -1.842024803161621,
-      "logits/rejected": -1.8078495264053345,
-      "logps/chosen": -1.0763428211212158,
-      "logps/rejected": -1.0434763431549072,
-      "loss": 0.6853,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.018699195235967636,
-      "rewards/margins": 0.017539886757731438,
-      "rewards/rejected": 0.0011593066155910492,
+      "epoch": 0.48,
+      "grad_norm": 100.6332916667586,
+      "learning_rate": 6.239597278716581e-08,
+      "logits/chosen": -1.9531447887420654,
+      "logits/rejected": -1.8929874897003174,
+      "logps/chosen": -414.9901428222656,
+      "logps/rejected": -375.222412109375,
+      "loss": 0.4997,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 0.5068634748458862,
+      "rewards/margins": 0.9239821434020996,
+      "rewards/rejected": -0.41711869835853577,
       "step": 210
     },
     {
-      "epoch": 0.74,
-      "grad_norm": 94.3943636617196,
-      "learning_rate": 1.9813473874379397e-09,
-      "logits/chosen": -1.732317328453064,
-      "logits/rejected": -1.7312743663787842,
-      "logps/chosen": -1.073425054550171,
-      "logps/rejected": -1.0629937648773193,
-      "loss": 0.6847,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.013539738953113556,
-      "rewards/margins": 0.013838117942214012,
-      "rewards/rejected": -0.00029837898910045624,
+      "epoch": 0.5,
+      "grad_norm": 109.57190452096494,
+      "learning_rate": 5.8508297910462456e-08,
+      "logits/chosen": -1.8995733261108398,
+      "logits/rejected": -1.818933129310608,
+      "logps/chosen": -414.49176025390625,
+      "logps/rejected": -402.1959228515625,
+      "loss": 0.498,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 0.3541712164878845,
+      "rewards/margins": 0.9701933860778809,
+      "rewards/rejected": -0.6160220503807068,
       "step": 220
     },
     {
-      "epoch": 0.77,
-      "grad_norm": 92.64370783949347,
-      "learning_rate": 1.5374582296511053e-09,
-      "logits/chosen": -1.7242523431777954,
-      "logits/rejected": -1.6965806484222412,
-      "logps/chosen": -1.0366003513336182,
-      "logps/rejected": -0.9941840171813965,
-      "loss": 0.6857,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.021335098892450333,
-      "rewards/margins": 0.015094568021595478,
-      "rewards/rejected": 0.006240529473870993,
+      "epoch": 0.52,
+      "grad_norm": 221.2346866193578,
+      "learning_rate": 5.456683083494731e-08,
+      "logits/chosen": -1.892313003540039,
+      "logits/rejected": -1.8525936603546143,
+      "logps/chosen": -472.8267517089844,
+      "logps/rejected": -455.94305419921875,
+      "loss": 0.5217,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.2913890480995178,
+      "rewards/margins": 0.6712461709976196,
+      "rewards/rejected": -0.3798571825027466,
       "step": 230
     },
     {
-      "epoch": 0.8,
-      "grad_norm": 91.14853265672438,
-      "learning_rate": 1.1407424007485927e-09,
-      "logits/chosen": -1.881166696548462,
-      "logits/rejected": -1.8595256805419922,
-      "logps/chosen": -1.0768239498138428,
-      "logps/rejected": -1.0254974365234375,
-      "loss": 0.6848,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.021426241844892502,
-      "rewards/margins": 0.021464312449097633,
-      "rewards/rejected": -3.8067344576120377e-05,
+      "epoch": 0.55,
+      "grad_norm": 199.8394238496795,
+      "learning_rate": 5.059649078450834e-08,
+      "logits/chosen": -1.8888638019561768,
+      "logits/rejected": -1.8429927825927734,
+      "logps/chosen": -432.7430725097656,
+      "logps/rejected": -422.738037109375,
+      "loss": 0.4908,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.274366557598114,
+      "rewards/margins": 0.8101264834403992,
+      "rewards/rejected": -0.5357599854469299,
       "step": 240
     },
     {
-      "epoch": 0.84,
-      "grad_norm": 88.07457325362287,
-      "learning_rate": 7.966047182060226e-10,
-      "logits/chosen": -1.8788058757781982,
-      "logits/rejected": -1.881800889968872,
-      "logps/chosen": -1.0638504028320312,
-      "logps/rejected": -1.028044581413269,
-      "loss": 0.6828,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.018116505816578865,
-      "rewards/margins": 0.015344863757491112,
-      "rewards/rejected": 0.0027716427575796843,
+      "epoch": 0.57,
+      "grad_norm": 80.39074621948711,
+      "learning_rate": 4.6622379527277186e-08,
+      "logits/chosen": -1.8966060876846313,
+      "logits/rejected": -1.8454921245574951,
+      "logps/chosen": -394.818115234375,
+      "logps/rejected": -373.1236877441406,
+      "loss": 0.4952,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.21977496147155762,
+      "rewards/margins": 0.7499412298202515,
+      "rewards/rejected": -0.5301662683486938,
       "step": 250
     },
     {
-      "epoch": 0.87,
-      "grad_norm": 99.87255814500503,
-      "learning_rate": 5.097336799988067e-10,
-      "logits/chosen": -1.8842456340789795,
-      "logits/rejected": -1.8642467260360718,
-      "logps/chosen": -1.0772615671157837,
-      "logps/rejected": -1.0665724277496338,
-      "loss": 0.6848,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.023015262559056282,
-      "rewards/margins": 0.02001366578042507,
-      "rewards/rejected": 0.0030015967786312103,
+      "epoch": 0.59,
+      "grad_norm": 161.76520518881122,
+      "learning_rate": 4.26696226741691e-08,
+      "logits/chosen": -1.9112564325332642,
+      "logits/rejected": -1.839714765548706,
+      "logps/chosen": -430.2705078125,
+      "logps/rejected": -403.42633056640625,
+      "loss": 0.5027,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.09559208154678345,
+      "rewards/margins": 0.799443244934082,
+      "rewards/rejected": -0.7038511037826538,
       "step": 260
     },
     {
-      "epoch": 0.9,
-      "grad_norm": 107.74820920122671,
-      "learning_rate": 2.840375889663871e-10,
-      "logits/chosen": -1.8729417324066162,
-      "logits/rejected": -1.8372119665145874,
-      "logps/chosen": -0.9936866760253906,
-      "logps/rejected": -1.0166294574737549,
-      "loss": 0.6847,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.01751965843141079,
-      "rewards/margins": 0.02036522701382637,
-      "rewards/rejected": -0.0028455646242946386,
+      "epoch": 0.61,
+      "grad_norm": 92.5112436349587,
+      "learning_rate": 3.876321082668098e-08,
+      "logits/chosen": -1.9705326557159424,
+      "logits/rejected": -1.896384596824646,
+      "logps/chosen": -455.70849609375,
+      "logps/rejected": -424.35528564453125,
+      "loss": 0.4925,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.3973539471626282,
+      "rewards/margins": 0.953347384929657,
+      "rewards/rejected": -0.5559934377670288,
       "step": 270
     },
     {
-      "epoch": 0.94,
-      "grad_norm": 87.56807066875254,
-      "learning_rate": 1.2259130647833627e-10,
-      "logits/chosen": -1.7625566720962524,
-      "logits/rejected": -1.7224514484405518,
-      "logps/chosen": -1.093827724456787,
-      "logps/rejected": -1.0462143421173096,
-      "loss": 0.6834,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": 0.01200016774237156,
-      "rewards/margins": 0.019106844440102577,
-      "rewards/rejected": -0.0071066757664084435,
+      "epoch": 0.64,
+      "grad_norm": 178.08098991416173,
+      "learning_rate": 3.492784157826244e-08,
+      "logits/chosen": -1.8988901376724243,
+      "logits/rejected": -1.8011884689331055,
+      "logps/chosen": -438.60076904296875,
+      "logps/rejected": -388.46978759765625,
+      "loss": 0.5004,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.43949180841445923,
+      "rewards/margins": 1.0134050846099854,
+      "rewards/rejected": -0.5739132165908813,
       "step": 280
     },
     {
-      "epoch": 0.97,
-      "grad_norm": 81.16688874976322,
-      "learning_rate": 2.7594360825166643e-11,
-      "logits/chosen": -1.780118703842163,
-      "logits/rejected": -1.7672898769378662,
-      "logps/chosen": -1.0616685152053833,
-      "logps/rejected": -1.0320245027542114,
-      "loss": 0.6845,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.028133947402238846,
-      "rewards/margins": 0.020768892019987106,
-      "rewards/rejected": 0.007365054450929165,
+      "epoch": 0.66,
+      "grad_norm": 118.63444466442957,
+      "learning_rate": 3.118776336817812e-08,
+      "logits/chosen": -1.9487316608428955,
+      "logits/rejected": -1.8821332454681396,
+      "logps/chosen": -431.9124450683594,
+      "logps/rejected": -391.89208984375,
+      "loss": 0.4867,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.3583192229270935,
+      "rewards/margins": 0.9170917272567749,
+      "rewards/rejected": -0.5587725639343262,
       "step": 290
     },
+    {
+      "epoch": 0.68,
+      "grad_norm": 105.93330787956464,
+      "learning_rate": 2.7566622175067443e-08,
+      "logits/chosen": -1.9368865489959717,
+      "logits/rejected": -1.8702398538589478,
+      "logps/chosen": -427.23101806640625,
+      "logps/rejected": -406.6597595214844,
+      "loss": 0.5099,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.41136330366134644,
+      "rewards/margins": 0.9053149223327637,
+      "rewards/rejected": -0.4939516484737396,
+      "step": 300
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 94.82101284169875,
+      "learning_rate": 2.408731201945432e-08,
+      "logits/chosen": -1.9327596426010132,
+      "logits/rejected": -1.8846778869628906,
+      "logps/chosen": -422.32061767578125,
+      "logps/rejected": -419.91131591796875,
+      "loss": 0.4923,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.4359460473060608,
+      "rewards/margins": 0.7473770380020142,
+      "rewards/rejected": -0.31143102049827576,
+      "step": 310
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 197.8417598175193,
+      "learning_rate": 2.0771830220378112e-08,
+      "logits/chosen": -1.8852752447128296,
+      "logits/rejected": -1.8286488056182861,
+      "logps/chosen": -450.9109802246094,
+      "logps/rejected": -427.3114318847656,
+      "loss": 0.4812,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.44545871019363403,
+      "rewards/margins": 0.8026180267333984,
+      "rewards/rejected": -0.357159286737442,
+      "step": 320
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 115.40932567329016,
+      "learning_rate": 1.7641138321260257e-08,
+      "logits/chosen": -1.9217668771743774,
+      "logits/rejected": -1.8482532501220703,
+      "logps/chosen": -424.6617736816406,
+      "logps/rejected": -386.41943359375,
+      "loss": 0.4888,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": 0.49162572622299194,
+      "rewards/margins": 1.0169591903686523,
+      "rewards/rejected": -0.5253334045410156,
+      "step": 330
+    },
+    {
+      "epoch": 0.77,
+      "grad_norm": 116.14707152688237,
+      "learning_rate": 1.4715029564277793e-08,
+      "logits/chosen": -1.995141625404358,
+      "logits/rejected": -1.9429680109024048,
+      "logps/chosen": -437.26324462890625,
+      "logps/rejected": -408.69403076171875,
+      "loss": 0.4984,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.6876565217971802,
+      "rewards/margins": 0.9710708856582642,
+      "rewards/rejected": -0.283414363861084,
+      "step": 340
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 96.35236055831872,
+      "learning_rate": 1.2012003751113343e-08,
+      "logits/chosen": -1.9784055948257446,
+      "logits/rejected": -1.913835883140564,
+      "logps/chosen": -427.19921875,
+      "logps/rejected": -406.6058044433594,
+      "loss": 0.4675,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.41034063696861267,
+      "rewards/margins": 0.9592168927192688,
+      "rewards/rejected": -0.5488761067390442,
+      "step": 350
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 119.40607594359629,
+      "learning_rate": 9.549150281252633e-09,
+      "logits/chosen": -1.9397361278533936,
+      "logits/rejected": -1.884749174118042,
+      "logps/chosen": -451.00531005859375,
+      "logps/rejected": -423.86553955078125,
+      "loss": 0.481,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.5500633120536804,
+      "rewards/margins": 0.9735897779464722,
+      "rewards/rejected": -0.4235265851020813,
+      "step": 360
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 110.76475497042553,
+      "learning_rate": 7.3420401072985306e-09,
+      "logits/chosen": -1.9737659692764282,
+      "logits/rejected": -1.9217618703842163,
+      "logps/chosen": -427.6600646972656,
+      "logps/rejected": -412.3758850097656,
+      "loss": 0.4918,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.44847536087036133,
+      "rewards/margins": 0.9299023747444153,
+      "rewards/rejected": -0.4814269542694092,
+      "step": 370
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 95.99625394215789,
+      "learning_rate": 5.404627290395369e-09,
+      "logits/chosen": -1.9365133047103882,
+      "logits/rejected": -1.8697948455810547,
+      "logps/chosen": -429.69219970703125,
+      "logps/rejected": -406.6961975097656,
+      "loss": 0.4841,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.6347268223762512,
+      "rewards/margins": 0.9203673601150513,
+      "rewards/rejected": -0.2856404185295105,
+      "step": 380
+    },
+    {
+      "epoch": 0.89,
+      "grad_norm": 157.20747053789427,
+      "learning_rate": 3.74916077816162e-09,
+      "logits/chosen": -1.945406198501587,
+      "logits/rejected": -1.8826377391815186,
+      "logps/chosen": -412.59619140625,
+      "logps/rejected": -386.8226623535156,
+      "loss": 0.4932,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 0.42942380905151367,
+      "rewards/margins": 0.8297268152236938,
+      "rewards/rejected": -0.4003029763698578,
+      "step": 390
+    },
+    {
+      "epoch": 0.91,
+      "grad_norm": 105.21807814382213,
+      "learning_rate": 2.386106962899165e-09,
+      "logits/chosen": -1.871506690979004,
+      "logits/rejected": -1.7978681325912476,
+      "logps/chosen": -419.5323791503906,
+      "logps/rejected": -388.26239013671875,
+      "loss": 0.4821,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.3208548426628113,
+      "rewards/margins": 0.8715551495552063,
+      "rewards/rejected": -0.5507001876831055,
+      "step": 400
+    },
+    {
+      "epoch": 0.93,
+      "grad_norm": 114.13246004934948,
+      "learning_rate": 1.3240835096913706e-09,
+      "logits/chosen": -1.9058313369750977,
+      "logits/rejected": -1.8005132675170898,
+      "logps/chosen": -423.19305419921875,
+      "logps/rejected": -383.4271240234375,
+      "loss": 0.4992,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 0.6571682095527649,
+      "rewards/margins": 0.9992189407348633,
+      "rewards/rejected": -0.3420506417751312,
+      "step": 410
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 88.16218181131795,
+      "learning_rate": 5.698048727497462e-10,
+      "logits/chosen": -1.929990530014038,
+      "logits/rejected": -1.8586419820785522,
+      "logps/chosen": -402.3602294921875,
+      "logps/rejected": -381.64605712890625,
+      "loss": 0.4911,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.5359801650047302,
+      "rewards/margins": 1.0312530994415283,
+      "rewards/rejected": -0.4952728748321533,
+      "step": 420
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 104.24331856340095,
+      "learning_rate": 1.2803984447259387e-10,
+      "logits/chosen": -1.9403343200683594,
+      "logits/rejected": -1.881209135055542,
+      "logps/chosen": -453.416259765625,
+      "logps/rejected": -408.7674865722656,
+      "loss": 0.4731,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.4977526068687439,
+      "rewards/margins": 1.0032169818878174,
+      "rewards/rejected": -0.5054643750190735,
+      "step": 430
+    },
     {
       "epoch": 1.0,
-      "step": 299,
+      "step": 439,
       "total_flos": 0.0,
-      "train_loss": 0.6880149315033469,
-      "train_runtime": 4603.368,
-      "train_samples_per_second": 8.317,
-      "train_steps_per_second": 0.065
+      "train_loss": 0.5309282744935542,
+      "train_runtime": 6847.998,
+      "train_samples_per_second": 8.212,
+      "train_steps_per_second": 0.064
     }
   ],
   "logging_steps": 10,
-  "max_steps": 299,
+  "max_steps": 439,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce791dc81b1b2238243a31a7af35de8e05887362f23dad989bfe1e254d1f7d14
-size 6520
+oid sha256:85030f3e8e8a49982a5e76bc0937f8c7c89ad40820c9fa0bff6598e0cf753ecb
+size 6264