Model save

README.md

@@ -32,7 +32,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 1e-08
+- learning_rate: 5e-08
 - train_batch_size: 8
 - eval_batch_size: 8
 - seed: 42
@@ -44,7 +44,7 @@ The following hyperparameters were used during training:
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
-- num_epochs: 2
+- num_epochs: 1
 
 ### Training results
 

all_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 2.0,
-    "train_loss": 0.6411590957641602,
-    "train_runtime": 8942.072,
-    "train_samples": 38445,
-    "train_samples_per_second": 8.599,
+    "epoch": 1.0,
+    "train_loss": 0.49109079287602353,
+    "train_runtime": 5330.839,
+    "train_samples": 46672,
+    "train_samples_per_second": 8.755,
     "train_steps_per_second": 0.034
 }

generation_config.json

@@ -1,6 +1,6 @@
 {
   "_from_model_config": true,
   "bos_token_id": 1,
-  "eos_token_id": 32000,
+  "eos_token_id": 2,
   "transformers_version": "4.39.3"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e95be6f8c0aa9a00e2dec7409b07bb3971685d185f076a462b8f78d03a1941fd
-size 4943178720
+oid sha256:80c511d14471ab18870f10f923c8787de845cdc70bd963e129a608dd18f3fc60
+size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:65d756c6023aac10ba06a637406ed4d2e2f4752c360d7d38f9d57ab38052466e
+oid sha256:c8c474648050e838c831aabb3699a61ac3756493e671f2a842828d65675367af
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a4dee0a340ae6ecce1b95adc398b7196eb2db45a4ad1ef0dc042944445a81bb6
-size 4540532728
+oid sha256:2335066937ff02bdedb3b53830a851b36f4126b5264e844e5f7337e06e1c6efd
+size 4540516344

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 14483496960
+    "total_size": 14483464192
   },
   "weight_map": {
     "lm_head.weight": "model-00003-of-00003.safetensors",

runs/Jun18_02-31-46_n136-129-074/events.out.tfevents.1718649387.n136-129-074.1961484.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:86f860b0abf43d7f8965a8620682abac5fc820caea5ba6269048d57fe900dc28
-size 12347
+oid sha256:154d3e21a3f1857573cbd485ca2aab1bc0ca7fa6c9015161948c9c3cf1ee365f
+size 18181

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 2.0,
-    "train_loss": 0.6411590957641602,
-    "train_runtime": 8942.072,
-    "train_samples": 38445,
-    "train_samples_per_second": 8.599,
+    "epoch": 1.0,
+    "train_loss": 0.49109079287602353,
+    "train_runtime": 5330.839,
+    "train_samples": 46672,
+    "train_samples_per_second": 8.755,
     "train_steps_per_second": 0.034
 }

trainer_state.json

@@ -1,21 +1,21 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.9966722129783694,
+  "epoch": 0.9972602739726028,
   "eval_steps": 500,
-  "global_step": 300,
+  "global_step": 182,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.01,
-      "grad_norm": 1151.6184415193795,
-      "learning_rate": 3.333333333333333e-10,
-      "logits/chosen": -4.106247425079346,
-      "logits/rejected": -4.200438499450684,
-      "logps/chosen": -382.81439208984375,
-      "logps/rejected": -357.65960693359375,
+      "grad_norm": 450.1434249789045,
+      "learning_rate": 2.6315789473684206e-09,
+      "logits/chosen": -0.7030794620513916,
+      "logits/rejected": -0.3951629400253296,
+      "logps/chosen": -341.73382568359375,
+      "logps/rejected": -292.9862060546875,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -24,469 +24,289 @@
       "step": 1
     },
     {
-      "epoch": 0.07,
-      "grad_norm": 1153.8530059959064,
-      "learning_rate": 3.3333333333333334e-09,
-      "logits/chosen": -4.217168807983398,
-      "logits/rejected": -4.321505069732666,
-      "logps/chosen": -334.6739501953125,
-      "logps/rejected": -313.41986083984375,
-      "loss": 0.7234,
-      "rewards/accuracies": 0.3784722089767456,
-      "rewards/chosen": -0.04607396200299263,
-      "rewards/margins": -0.04357295483350754,
-      "rewards/rejected": -0.0025010076351463795,
+      "epoch": 0.05,
+      "grad_norm": 427.11862533612515,
+      "learning_rate": 2.6315789473684208e-08,
+      "logits/chosen": -0.8544511795043945,
+      "logits/rejected": -0.5010538101196289,
+      "logps/chosen": -343.0840148925781,
+      "logps/rejected": -297.6950378417969,
+      "loss": 0.6989,
+      "rewards/accuracies": 0.4201388955116272,
+      "rewards/chosen": 0.006387188099324703,
+      "rewards/margins": -0.002455425448715687,
+      "rewards/rejected": 0.008842612616717815,
       "step": 10
     },
     {
-      "epoch": 0.13,
-      "grad_norm": 1270.1091311718026,
-      "learning_rate": 6.666666666666667e-09,
-      "logits/chosen": -4.26615571975708,
-      "logits/rejected": -4.41886043548584,
-      "logps/chosen": -313.93829345703125,
-      "logps/rejected": -288.78863525390625,
-      "loss": 0.7299,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -0.012899210676550865,
-      "rewards/margins": -0.021626513451337814,
-      "rewards/rejected": 0.00872730277478695,
+      "epoch": 0.11,
+      "grad_norm": 420.6890953363653,
+      "learning_rate": 4.9995356760283366e-08,
+      "logits/chosen": -0.8882249593734741,
+      "logits/rejected": -0.5170028805732727,
+      "logps/chosen": -333.65777587890625,
+      "logps/rejected": -287.80816650390625,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.012563005089759827,
+      "rewards/margins": 0.03462809696793556,
+      "rewards/rejected": -0.04719109833240509,
       "step": 20
     },
     {
-      "epoch": 0.2,
-      "grad_norm": 1195.4358872902142,
-      "learning_rate": 1e-08,
-      "logits/chosen": -4.3016462326049805,
-      "logits/rejected": -4.365716457366943,
-      "logps/chosen": -308.2979431152344,
-      "logps/rejected": -285.63018798828125,
-      "loss": 0.7311,
-      "rewards/accuracies": 0.49687498807907104,
-      "rewards/chosen": -0.010719490237534046,
-      "rewards/margins": -0.010031750425696373,
-      "rewards/rejected": -0.000687739229761064,
+      "epoch": 0.16,
+      "grad_norm": 398.42327340439533,
+      "learning_rate": 4.944025194969586e-08,
+      "logits/chosen": -0.9182589650154114,
+      "logits/rejected": -0.5222188234329224,
+      "logps/chosen": -332.426025390625,
+      "logps/rejected": -290.18304443359375,
+      "loss": 0.6356,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.035035934299230576,
+      "rewards/margins": 0.15518921613693237,
+      "rewards/rejected": -0.19022515416145325,
       "step": 30
     },
     {
-      "epoch": 0.27,
-      "grad_norm": 1144.0076561795222,
-      "learning_rate": 9.966191788709716e-09,
-      "logits/chosen": -4.187338829040527,
-      "logits/rejected": -4.271176338195801,
-      "logps/chosen": -332.39453125,
-      "logps/rejected": -307.4621276855469,
-      "loss": 0.7199,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.04900529980659485,
-      "rewards/margins": 0.05206792429089546,
-      "rewards/rejected": -0.0030626237858086824,
+      "epoch": 0.22,
+      "grad_norm": 361.5856784181511,
+      "learning_rate": 4.798007040388211e-08,
+      "logits/chosen": -0.8608843684196472,
+      "logits/rejected": -0.5015160441398621,
+      "logps/chosen": -337.7128601074219,
+      "logps/rejected": -307.75726318359375,
+      "loss": 0.5811,
+      "rewards/accuracies": 0.7406250238418579,
+      "rewards/chosen": 0.006648591253906488,
+      "rewards/margins": 0.3918178081512451,
+      "rewards/rejected": -0.38516920804977417,
       "step": 40
     },
     {
-      "epoch": 0.33,
-      "grad_norm": 1135.1845096891816,
-      "learning_rate": 9.86522435289912e-09,
-      "logits/chosen": -4.197329044342041,
-      "logits/rejected": -4.366620063781738,
-      "logps/chosen": -333.64678955078125,
-      "logps/rejected": -309.0525817871094,
-      "loss": 0.7189,
-      "rewards/accuracies": 0.515625,
-      "rewards/chosen": 0.04922889173030853,
-      "rewards/margins": 0.008506924845278263,
-      "rewards/rejected": 0.040721967816352844,
+      "epoch": 0.27,
+      "grad_norm": 334.47329593106986,
+      "learning_rate": 4.5668885883990063e-08,
+      "logits/chosen": -0.8879793286323547,
+      "logits/rejected": -0.4423498511314392,
+      "logps/chosen": -332.4192199707031,
+      "logps/rejected": -296.57025146484375,
+      "loss": 0.5272,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.24052493274211884,
+      "rewards/margins": 0.5968745946884155,
+      "rewards/rejected": -0.3563496172428131,
       "step": 50
     },
     {
-      "epoch": 0.4,
-      "grad_norm": 1124.4015217482226,
-      "learning_rate": 9.698463103929542e-09,
-      "logits/chosen": -4.186924457550049,
-      "logits/rejected": -4.269418239593506,
-      "logps/chosen": -323.7723693847656,
-      "logps/rejected": -308.26971435546875,
-      "loss": 0.7082,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.11511299759149551,
-      "rewards/margins": 0.052407026290893555,
-      "rewards/rejected": 0.06270597130060196,
+      "epoch": 0.33,
+      "grad_norm": 339.682806652983,
+      "learning_rate": 4.2592286678719626e-08,
+      "logits/chosen": -0.8047763109207153,
+      "logits/rejected": -0.4120696485042572,
+      "logps/chosen": -330.0352478027344,
+      "logps/rejected": -295.6581726074219,
+      "loss": 0.5016,
+      "rewards/accuracies": 0.7718750238418579,
+      "rewards/chosen": 0.5128915905952454,
+      "rewards/margins": 0.7231279015541077,
+      "rewards/rejected": -0.21023626625537872,
       "step": 60
     },
     {
-      "epoch": 0.47,
-      "grad_norm": 1133.1247984062034,
-      "learning_rate": 9.468163201617062e-09,
-      "logits/chosen": -4.147147178649902,
-      "logits/rejected": -4.3019304275512695,
-      "logps/chosen": -344.58563232421875,
-      "logps/rejected": -314.4212951660156,
-      "loss": 0.7005,
-      "rewards/accuracies": 0.590624988079071,
-      "rewards/chosen": 0.23254117369651794,
-      "rewards/margins": 0.13268980383872986,
-      "rewards/rejected": 0.09985135495662689,
+      "epoch": 0.38,
+      "grad_norm": 294.8745864628234,
+      "learning_rate": 3.886420608016766e-08,
+      "logits/chosen": -1.0426691770553589,
+      "logits/rejected": -0.600997805595398,
+      "logps/chosen": -348.37408447265625,
+      "logps/rejected": -304.3432922363281,
+      "loss": 0.4851,
+      "rewards/accuracies": 0.753125011920929,
+      "rewards/chosen": 0.823379397392273,
+      "rewards/margins": 0.7953158617019653,
+      "rewards/rejected": 0.02806355059146881,
       "step": 70
     },
     {
-      "epoch": 0.53,
-      "grad_norm": 1057.1503651106625,
-      "learning_rate": 9.177439057064682e-09,
-      "logits/chosen": -4.223555564880371,
-      "logits/rejected": -4.389444828033447,
-      "logps/chosen": -334.3599853515625,
-      "logps/rejected": -305.4422607421875,
-      "loss": 0.6844,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": 0.23962649703025818,
-      "rewards/margins": 0.09747296571731567,
-      "rewards/rejected": 0.1421535313129425,
+      "epoch": 0.44,
+      "grad_norm": 272.6474819647033,
+      "learning_rate": 3.462270318133136e-08,
+      "logits/chosen": -0.9253625869750977,
+      "logits/rejected": -0.3548746705055237,
+      "logps/chosen": -341.5060119628906,
+      "logps/rejected": -294.5276794433594,
+      "loss": 0.4535,
+      "rewards/accuracies": 0.815625011920929,
+      "rewards/chosen": 0.7518247961997986,
+      "rewards/margins": 0.94850093126297,
+      "rewards/rejected": -0.19667614996433258,
       "step": 80
     },
     {
-      "epoch": 0.6,
-      "grad_norm": 1055.57995971272,
-      "learning_rate": 8.830222215594889e-09,
-      "logits/chosen": -4.2292633056640625,
-      "logits/rejected": -4.349828243255615,
-      "logps/chosen": -327.07330322265625,
-      "logps/rejected": -307.81707763671875,
-      "loss": 0.6772,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": 0.2870681881904602,
-      "rewards/margins": 0.06986425817012787,
-      "rewards/rejected": 0.21720390021800995,
+      "epoch": 0.49,
+      "grad_norm": 281.1147370449449,
+      "learning_rate": 3.0024850241696124e-08,
+      "logits/chosen": -1.0233888626098633,
+      "logits/rejected": -0.6329795122146606,
+      "logps/chosen": -337.8768310546875,
+      "logps/rejected": -307.80621337890625,
+      "loss": 0.4505,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": 0.9118101000785828,
+      "rewards/margins": 1.0233700275421143,
+      "rewards/rejected": -0.1115599274635315,
       "step": 90
     },
     {
-      "epoch": 0.67,
-      "grad_norm": 1124.231150209405,
-      "learning_rate": 8.431208189343668e-09,
-      "logits/chosen": -4.214221000671387,
-      "logits/rejected": -4.424824237823486,
-      "logps/chosen": -320.45928955078125,
-      "logps/rejected": -289.71466064453125,
-      "loss": 0.6738,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": 0.3042110502719879,
-      "rewards/margins": 0.12749677896499634,
-      "rewards/rejected": 0.1767142415046692,
+      "epoch": 0.55,
+      "grad_norm": 270.95326246127075,
+      "learning_rate": 2.5240915953099518e-08,
+      "logits/chosen": -0.8435468673706055,
+      "logits/rejected": -0.528495192527771,
+      "logps/chosen": -316.45147705078125,
+      "logps/rejected": -284.6639099121094,
+      "loss": 0.434,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.6949013471603394,
+      "rewards/margins": 0.9432865381240845,
+      "rewards/rejected": -0.24838528037071228,
       "step": 100
     },
     {
-      "epoch": 0.73,
-      "grad_norm": 1045.9935050945592,
-      "learning_rate": 7.98579295851393e-09,
-      "logits/chosen": -4.233684062957764,
-      "logits/rejected": -4.286379814147949,
-      "logps/chosen": -314.39251708984375,
-      "logps/rejected": -303.2556457519531,
-      "loss": 0.6623,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": 0.3848657011985779,
-      "rewards/margins": 0.10032075643539429,
-      "rewards/rejected": 0.2845449447631836,
+      "epoch": 0.6,
+      "grad_norm": 246.8924504601372,
+      "learning_rate": 2.0448060012437142e-08,
+      "logits/chosen": -0.7631230354309082,
+      "logits/rejected": -0.46078410744667053,
+      "logps/chosen": -332.81842041015625,
+      "logps/rejected": -302.547119140625,
+      "loss": 0.4323,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.88267582654953,
+      "rewards/margins": 1.0535722970962524,
+      "rewards/rejected": -0.17089635133743286,
       "step": 110
     },
     {
-      "epoch": 0.8,
-      "grad_norm": 1107.1293654591734,
-      "learning_rate": 7.500000000000001e-09,
-      "logits/chosen": -4.151052951812744,
-      "logits/rejected": -4.307383060455322,
-      "logps/chosen": -338.46759033203125,
-      "logps/rejected": -313.0832214355469,
-      "loss": 0.6634,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.49397316575050354,
-      "rewards/margins": 0.20816774666309357,
-      "rewards/rejected": 0.2858053743839264,
+      "epoch": 0.66,
+      "grad_norm": 249.2911347992777,
+      "learning_rate": 1.5823772505167697e-08,
+      "logits/chosen": -0.8469772338867188,
+      "logits/rejected": -0.47505950927734375,
+      "logps/chosen": -345.8771667480469,
+      "logps/rejected": -308.70025634765625,
+      "loss": 0.421,
+      "rewards/accuracies": 0.8343750238418579,
+      "rewards/chosen": 1.0309433937072754,
+      "rewards/margins": 1.2240039110183716,
+      "rewards/rejected": -0.19306042790412903,
       "step": 120
     },
     {
-      "epoch": 0.87,
-      "grad_norm": 1113.1665283582252,
-      "learning_rate": 6.980398830195784e-09,
-      "logits/chosen": -4.174288272857666,
-      "logits/rejected": -4.36756706237793,
-      "logps/chosen": -323.51800537109375,
-      "logps/rejected": -304.7021484375,
-      "loss": 0.6392,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.5511430501937866,
-      "rewards/margins": 0.23519937694072723,
-      "rewards/rejected": 0.31594371795654297,
+      "epoch": 0.71,
+      "grad_norm": 236.63634286693804,
+      "learning_rate": 1.1539301053796948e-08,
+      "logits/chosen": -0.8259572982788086,
+      "logits/rejected": -0.48908624053001404,
+      "logps/chosen": -326.006591796875,
+      "logps/rejected": -298.5427551269531,
+      "loss": 0.4317,
+      "rewards/accuracies": 0.7406250238418579,
+      "rewards/chosen": 0.7692808508872986,
+      "rewards/margins": 0.9658697843551636,
+      "rewards/rejected": -0.1965889036655426,
       "step": 130
     },
     {
-      "epoch": 0.93,
-      "grad_norm": 1022.5201031232197,
-      "learning_rate": 6.434016163555451e-09,
-      "logits/chosen": -4.292388916015625,
-      "logits/rejected": -4.358359336853027,
-      "logps/chosen": -307.1917419433594,
-      "logps/rejected": -296.78192138671875,
-      "loss": 0.6402,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.5292906761169434,
-      "rewards/margins": 0.16237936913967133,
-      "rewards/rejected": 0.36691129207611084,
+      "epoch": 0.77,
+      "grad_norm": 277.9055914170381,
+      "learning_rate": 7.753309138609703e-09,
+      "logits/chosen": -1.0100657939910889,
+      "logits/rejected": -0.518832266330719,
+      "logps/chosen": -335.1830749511719,
+      "logps/rejected": -293.55389404296875,
+      "loss": 0.4203,
+      "rewards/accuracies": 0.7906249761581421,
+      "rewards/chosen": 0.938341498374939,
+      "rewards/margins": 1.1349132061004639,
+      "rewards/rejected": -0.19657166302204132,
       "step": 140
     },
     {
-      "epoch": 1.0,
-      "grad_norm": 1051.3521300860295,
-      "learning_rate": 5.868240888334653e-09,
-      "logits/chosen": -4.260380268096924,
-      "logits/rejected": -4.333888053894043,
-      "logps/chosen": -311.844482421875,
-      "logps/rejected": -299.74261474609375,
-      "loss": 0.6387,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": 0.5666605830192566,
-      "rewards/margins": 0.1995116025209427,
-      "rewards/rejected": 0.3671489655971527,
+      "epoch": 0.82,
+      "grad_norm": 280.40678633901416,
+      "learning_rate": 4.606000437098476e-09,
+      "logits/chosen": -0.9020172953605652,
+      "logits/rejected": -0.5801359415054321,
+      "logps/chosen": -328.5782775878906,
+      "logps/rejected": -298.58416748046875,
+      "loss": 0.417,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.8313944935798645,
+      "rewards/margins": 1.0033814907073975,
+      "rewards/rejected": -0.1719868779182434,
       "step": 150
     },
     {
-      "epoch": 1.06,
-      "grad_norm": 1029.1548288848146,
-      "learning_rate": 5.290724144552379e-09,
-      "logits/chosen": -4.191515922546387,
-      "logits/rejected": -4.348960876464844,
-      "logps/chosen": -326.90582275390625,
-      "logps/rejected": -305.37689208984375,
-      "loss": 0.6032,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.6744669079780579,
-      "rewards/margins": 0.37094420194625854,
-      "rewards/rejected": 0.3035227358341217,
+      "epoch": 0.88,
+      "grad_norm": 281.95412461514485,
+      "learning_rate": 2.2139267708310455e-09,
+      "logits/chosen": -0.941261887550354,
+      "logits/rejected": -0.5286608934402466,
+      "logps/chosen": -337.65704345703125,
+      "logps/rejected": -302.15557861328125,
+      "loss": 0.4195,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 1.0939249992370605,
+      "rewards/margins": 1.298884391784668,
+      "rewards/rejected": -0.20495939254760742,
       "step": 160
     },
     {
-      "epoch": 1.13,
-      "grad_norm": 991.2604083787021,
-      "learning_rate": 4.709275855447621e-09,
-      "logits/chosen": -4.241927623748779,
-      "logits/rejected": -4.339847564697266,
-      "logps/chosen": -314.4874572753906,
-      "logps/rejected": -295.49151611328125,
-      "loss": 0.6151,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.6504887938499451,
-      "rewards/margins": 0.3018999397754669,
-      "rewards/rejected": 0.3485889136791229,
+      "epoch": 0.93,
+      "grad_norm": 257.0608061838707,
+      "learning_rate": 6.656719329999699e-10,
+      "logits/chosen": -0.6665180921554565,
+      "logits/rejected": -0.36475566029548645,
+      "logps/chosen": -323.7983703613281,
+      "logps/rejected": -292.5258483886719,
+      "loss": 0.4248,
+      "rewards/accuracies": 0.8343750238418579,
+      "rewards/chosen": 1.0974645614624023,
+      "rewards/margins": 1.273900032043457,
+      "rewards/rejected": -0.1764354258775711,
       "step": 170
     },
     {
-      "epoch": 1.2,
-      "grad_norm": 976.0494957890444,
-      "learning_rate": 4.131759111665349e-09,
-      "logits/chosen": -4.174123287200928,
-      "logits/rejected": -4.313396453857422,
-      "logps/chosen": -325.78045654296875,
-      "logps/rejected": -304.82989501953125,
-      "loss": 0.6147,
-      "rewards/accuracies": 0.6656249761581421,
-      "rewards/chosen": 0.6252425909042358,
-      "rewards/margins": 0.27976280450820923,
-      "rewards/rejected": 0.3454797863960266,
+      "epoch": 0.99,
+      "grad_norm": 245.05015884181918,
+      "learning_rate": 1.8571234092507048e-11,
+      "logits/chosen": -0.8604642152786255,
+      "logits/rejected": -0.526264488697052,
+      "logps/chosen": -320.47540283203125,
+      "logps/rejected": -290.77569580078125,
+      "loss": 0.4234,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 0.9940341711044312,
+      "rewards/margins": 1.222551941871643,
+      "rewards/rejected": -0.22851786017417908,
       "step": 180
     },
     {
-      "epoch": 1.26,
-      "grad_norm": 949.77837430784,
-      "learning_rate": 3.56598383644455e-09,
-      "logits/chosen": -4.231461524963379,
-      "logits/rejected": -4.367356777191162,
-      "logps/chosen": -326.8230895996094,
-      "logps/rejected": -299.7027587890625,
-      "loss": 0.5969,
-      "rewards/accuracies": 0.6968749761581421,
-      "rewards/chosen": 0.7501948475837708,
-      "rewards/margins": 0.35921385884284973,
-      "rewards/rejected": 0.39098095893859863,
-      "step": 190
-    },
-    {
-      "epoch": 1.33,
-      "grad_norm": 963.9694294192556,
-      "learning_rate": 3.0196011698042157e-09,
-      "logits/chosen": -4.249307155609131,
-      "logits/rejected": -4.317980766296387,
-      "logps/chosen": -313.9615783691406,
-      "logps/rejected": -293.1615905761719,
-      "loss": 0.5922,
-      "rewards/accuracies": 0.6781250238418579,
-      "rewards/chosen": 0.7425140142440796,
-      "rewards/margins": 0.3114904761314392,
-      "rewards/rejected": 0.4310235381126404,
-      "step": 200
-    },
-    {
-      "epoch": 1.4,
-      "grad_norm": 1038.4899227109647,
-      "learning_rate": 2.5000000000000013e-09,
-      "logits/chosen": -4.174517631530762,
-      "logits/rejected": -4.288783073425293,
-      "logps/chosen": -327.461181640625,
-      "logps/rejected": -304.9690246582031,
-      "loss": 0.5952,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.7787834405899048,
-      "rewards/margins": 0.3926675319671631,
-      "rewards/rejected": 0.3861159086227417,
-      "step": 210
-    },
-    {
-      "epoch": 1.46,
-      "grad_norm": 992.6867793357102,
-      "learning_rate": 2.0142070414860704e-09,
-      "logits/chosen": -4.165514945983887,
-      "logits/rejected": -4.316833019256592,
-      "logps/chosen": -358.1416931152344,
-      "logps/rejected": -323.22998046875,
-      "loss": 0.5994,
-      "rewards/accuracies": 0.6781250238418579,
-      "rewards/chosen": 0.8851108551025391,
-      "rewards/margins": 0.35754603147506714,
-      "rewards/rejected": 0.5275647640228271,
-      "step": 220
-    },
-    {
-      "epoch": 1.53,
-      "grad_norm": 918.0076197376386,
-      "learning_rate": 1.5687918106563326e-09,
-      "logits/chosen": -4.234222412109375,
-      "logits/rejected": -4.369565486907959,
-      "logps/chosen": -328.3675537109375,
-      "logps/rejected": -303.726806640625,
-      "loss": 0.5919,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.8034540414810181,
-      "rewards/margins": 0.2974655032157898,
-      "rewards/rejected": 0.5059884786605835,
-      "step": 230
-    },
-    {
-      "epoch": 1.6,
-      "grad_norm": 944.3107996293463,
-      "learning_rate": 1.1697777844051105e-09,
-      "logits/chosen": -4.202380180358887,
-      "logits/rejected": -4.353602409362793,
-      "logps/chosen": -326.75604248046875,
-      "logps/rejected": -297.03619384765625,
-      "loss": 0.588,
-      "rewards/accuracies": 0.659375011920929,
-      "rewards/chosen": 0.8500032424926758,
-      "rewards/margins": 0.3546527922153473,
-      "rewards/rejected": 0.4953504502773285,
-      "step": 240
-    },
-    {
-      "epoch": 1.66,
-      "grad_norm": 944.2690060176395,
-      "learning_rate": 8.225609429353187e-10,
-      "logits/chosen": -4.173297882080078,
-      "logits/rejected": -4.368635654449463,
-      "logps/chosen": -321.2724914550781,
-      "logps/rejected": -293.50311279296875,
-      "loss": 0.5856,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": 0.8207529783248901,
-      "rewards/margins": 0.40889453887939453,
-      "rewards/rejected": 0.4118584990501404,
-      "step": 250
-    },
-    {
-      "epoch": 1.73,
-      "grad_norm": 957.8767493880096,
-      "learning_rate": 5.318367983829391e-10,
-      "logits/chosen": -4.18589448928833,
-      "logits/rejected": -4.284361839294434,
-      "logps/chosen": -348.6470031738281,
-      "logps/rejected": -326.4360046386719,
-      "loss": 0.5895,
-      "rewards/accuracies": 0.690625011920929,
-      "rewards/chosen": 0.8914071917533875,
-      "rewards/margins": 0.37524908781051636,
-      "rewards/rejected": 0.5161582231521606,
-      "step": 260
-    },
-    {
-      "epoch": 1.8,
-      "grad_norm": 949.249423709826,
-      "learning_rate": 3.015368960704584e-10,
-      "logits/chosen": -4.271857738494873,
-      "logits/rejected": -4.4089555740356445,
-      "logps/chosen": -325.19049072265625,
-      "logps/rejected": -302.7436828613281,
-      "loss": 0.5938,
-      "rewards/accuracies": 0.6781250238418579,
-      "rewards/chosen": 0.9044697880744934,
-      "rewards/margins": 0.38229116797447205,
-      "rewards/rejected": 0.522178590297699,
-      "step": 270
-    },
-    {
-      "epoch": 1.86,
-      "grad_norm": 932.1311904270524,
-      "learning_rate": 1.3477564710088098e-10,
-      "logits/chosen": -4.163296222686768,
-      "logits/rejected": -4.306557655334473,
-      "logps/chosen": -327.57257080078125,
-      "logps/rejected": -306.2264099121094,
-      "loss": 0.5865,
-      "rewards/accuracies": 0.715624988079071,
-      "rewards/chosen": 0.8713304400444031,
-      "rewards/margins": 0.4181024134159088,
-      "rewards/rejected": 0.45322805643081665,
-      "step": 280
-    },
-    {
-      "epoch": 1.93,
-      "grad_norm": 884.8124763448137,
-      "learning_rate": 3.380821129028488e-11,
-      "logits/chosen": -4.1040120124816895,
-      "logits/rejected": -4.260913372039795,
-      "logps/chosen": -334.97686767578125,
-      "logps/rejected": -312.01617431640625,
-      "loss": 0.5924,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.9016163945198059,
-      "rewards/margins": 0.4117993414402008,
-      "rewards/rejected": 0.48981720209121704,
-      "step": 290
-    },
-    {
-      "epoch": 2.0,
-      "grad_norm": 959.7419745329076,
-      "learning_rate": 0.0,
-      "logits/chosen": -4.31270170211792,
-      "logits/rejected": -4.468858242034912,
-      "logps/chosen": -314.4400939941406,
-      "logps/rejected": -284.8276672363281,
-      "loss": 0.5825,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.82757169008255,
-      "rewards/margins": 0.36657077074050903,
-      "rewards/rejected": 0.4610009789466858,
-      "step": 300
-    },
-    {
-      "epoch": 2.0,
-      "step": 300,
+      "epoch": 1.0,
+      "step": 182,
       "total_flos": 0.0,
-      "train_loss": 0.6411590957641602,
-      "train_runtime": 8942.072,
-      "train_samples_per_second": 8.599,
+      "train_loss": 0.49109079287602353,
+      "train_runtime": 5330.839,
+      "train_samples_per_second": 8.755,
       "train_steps_per_second": 0.034
     }
   ],
   "logging_steps": 10,
-  "max_steps": 300,
+  "max_steps": 182,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 2,
+  "num_train_epochs": 1,
   "save_steps": 100,
   "total_flos": 0.0,
   "train_batch_size": 8,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fe7ad1d1edbbc30b461a2666bb026f02d74c0c127226e41e03c561f18f18d157
+oid sha256:511f849ca93a0f3688a329d8a84efe863ebada7f409c9ad22d389e79b9db4345
 size 6328