Model save

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.3992691706388425,
-    "train_runtime": 5832.237,
+    "train_loss": 0.39275855483152927,
+    "train_runtime": 5850.2913,
     "train_samples": 50000,
-    "train_samples_per_second": 8.573,
+    "train_samples_per_second": 8.547,
     "train_steps_per_second": 0.067
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b6ee3663db3dc57816b4a0f27cf1be1f285b7c23c55706f81120aff25150acc6
+oid sha256:1c0dc0c292d489e5548e3b75999fe6cd030919ab96386b961a7432dab2fccb25
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b6d81454b2196ccf4bbf7c9c46175d05eb80be4c1bac3c4412226c1bd4919f25
+oid sha256:78055b666c3b09cd419d126d46142b7e4cbc261fd5db79d249138904dfc1a57d
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e663878eb004471a9e62a96fa047a9faf1c1612a059a5bb36ca7416c9e8dc668
+oid sha256:04307748a9391cbe41e028618c312d319041f81f61a38ff22b6d44f0630d8343
 size 4540516344

runs/Jun22_03-41-55_n136-100-194/events.out.tfevents.1718998971.n136-100-194.1421217.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4f0498ee7060697e1f640444697bdb3effb8b0002b1ef9a219ac3d316d6c28ec
-size 26098
+oid sha256:529e639fdd42d7dbf95d6bf6be4135fb5040103f9df18e1b40660966896a5200
+size 32644

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.3992691706388425,
-    "train_runtime": 5832.237,
+    "train_loss": 0.39275855483152927,
+    "train_runtime": 5850.2913,
     "train_samples": 50000,
-    "train_samples_per_second": 8.573,
+    "train_samples_per_second": 8.547,
     "train_steps_per_second": 0.067
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_metric": null,
   "best_model_checkpoint": null,
   "epoch": 0.9980806142034548,
-  "eval_steps": 10000000,
+  "eval_steps": 1000000,
   "global_step": 390,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 993.0599594653562,
+      "grad_norm": 992.8142742850837,
       "learning_rate": 1.282051282051282e-08,
       "logits/chosen": -2.5583817958831787,
       "logits/rejected": -2.4487552642822266,
       "logps/chosen": -258.1644592285156,
       "logps/rejected": -216.25729370117188,
-      "loss": 0.6931,
+      "loss": 0.6965,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,596 +25,596 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 896.147033393026,
+      "grad_norm": 972.5242632724868,
       "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -2.6060969829559326,
-      "logits/rejected": -2.553100824356079,
-      "logps/chosen": -267.6229553222656,
-      "logps/rejected": -217.69410705566406,
-      "loss": 0.6951,
-      "rewards/accuracies": 0.4305555522441864,
-      "rewards/chosen": -0.04092066362500191,
-      "rewards/margins": -0.0019620400853455067,
-      "rewards/rejected": -0.038958627730607986,
+      "logits/chosen": -2.606093168258667,
+      "logits/rejected": -2.5532331466674805,
+      "logps/chosen": -267.5519714355469,
+      "logps/rejected": -217.63278198242188,
+      "loss": 0.7101,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.008765195496380329,
+      "rewards/margins": 0.004809141159057617,
+      "rewards/rejected": 0.0039560552686452866,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 680.6172429966671,
+      "grad_norm": 659.3539113664132,
       "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -2.6307482719421387,
-      "logits/rejected": -2.56762433052063,
-      "logps/chosen": -260.50250244140625,
-      "logps/rejected": -207.04611206054688,
-      "loss": 0.5259,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 0.5612112283706665,
-      "rewards/margins": 0.5556014776229858,
-      "rewards/rejected": 0.005609664134681225,
+      "logits/chosen": -2.6304450035095215,
+      "logits/rejected": -2.5677618980407715,
+      "logps/chosen": -260.57421875,
+      "logps/rejected": -207.02157592773438,
+      "loss": 0.5261,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.5109978318214417,
+      "rewards/margins": 0.48820367455482483,
+      "rewards/rejected": 0.022794198244810104,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "grad_norm": 988.9163975931574,
+      "grad_norm": 779.0248633950893,
       "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -2.6490671634674072,
-      "logits/rejected": -2.574298620223999,
-      "logps/chosen": -250.87783813476562,
-      "logps/rejected": -198.51046752929688,
-      "loss": 0.3383,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 2.988560438156128,
-      "rewards/margins": 2.737926483154297,
-      "rewards/rejected": 0.25063443183898926,
+      "logits/chosen": -2.648696184158325,
+      "logits/rejected": -2.573786973953247,
+      "logps/chosen": -250.76904296875,
+      "logps/rejected": -198.43540954589844,
+      "loss": 0.3369,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 3.0647289752960205,
+      "rewards/margins": 2.7615580558776855,
+      "rewards/rejected": 0.3031708300113678,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 376.6725294713165,
+      "grad_norm": 384.3988310437092,
       "learning_rate": 4.99989986344963e-07,
-      "logits/chosen": -2.64461612701416,
-      "logits/rejected": -2.5653235912323,
-      "logps/chosen": -243.3360595703125,
-      "logps/rejected": -193.0019989013672,
-      "loss": 0.3088,
+      "logits/chosen": -2.6418910026550293,
+      "logits/rejected": -2.5625271797180176,
+      "logps/chosen": -243.13668823242188,
+      "logps/rejected": -192.8317108154297,
+      "loss": 0.3186,
       "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 4.9092864990234375,
-      "rewards/margins": 4.430295467376709,
-      "rewards/rejected": 0.4789907932281494,
+      "rewards/chosen": 5.048842430114746,
+      "rewards/margins": 4.450641632080078,
+      "rewards/rejected": 0.5982006192207336,
       "step": 40
     },
     {
       "epoch": 0.13,
-      "grad_norm": 533.1420911811614,
+      "grad_norm": 557.6508821014772,
       "learning_rate": 4.987893180827479e-07,
-      "logits/chosen": -2.6562392711639404,
-      "logits/rejected": -2.583529472351074,
-      "logps/chosen": -257.88665771484375,
-      "logps/rejected": -203.4907989501953,
-      "loss": 0.3485,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 7.24533748626709,
-      "rewards/margins": 6.08463716506958,
-      "rewards/rejected": 1.1607000827789307,
+      "logits/chosen": -2.651369333267212,
+      "logits/rejected": -2.5784316062927246,
+      "logps/chosen": -257.6221923828125,
+      "logps/rejected": -203.21920776367188,
+      "loss": 0.3491,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 7.43048620223999,
+      "rewards/margins": 6.079649448394775,
+      "rewards/rejected": 1.3508365154266357,
       "step": 50
     },
     {
       "epoch": 0.15,
-      "grad_norm": 573.7869024935939,
+      "grad_norm": 595.0405769986861,
       "learning_rate": 4.955969343539162e-07,
-      "logits/chosen": -2.6053004264831543,
-      "logits/rejected": -2.530390977859497,
-      "logps/chosen": -261.89007568359375,
-      "logps/rejected": -209.2191925048828,
-      "loss": 0.3313,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 4.856343746185303,
-      "rewards/margins": 5.668840408325195,
-      "rewards/rejected": -0.8124964833259583,
+      "logits/chosen": -2.606095552444458,
+      "logits/rejected": -2.5313313007354736,
+      "logps/chosen": -261.7010192871094,
+      "logps/rejected": -209.05029296875,
+      "loss": 0.3678,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 4.988667011260986,
+      "rewards/margins": 5.682937145233154,
+      "rewards/rejected": -0.6942700147628784,
       "step": 60
     },
     {
       "epoch": 0.18,
-      "grad_norm": 414.65176139322114,
+      "grad_norm": 406.24830016448993,
       "learning_rate": 4.90438392204474e-07,
-      "logits/chosen": -2.568657398223877,
-      "logits/rejected": -2.493375301361084,
-      "logps/chosen": -291.69537353515625,
-      "logps/rejected": -228.1354522705078,
-      "loss": 0.348,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 4.35067081451416,
-      "rewards/margins": 6.412045478820801,
-      "rewards/rejected": -2.0613746643066406,
+      "logits/chosen": -2.57716703414917,
+      "logits/rejected": -2.5019829273223877,
+      "logps/chosen": -291.1893615722656,
+      "logps/rejected": -227.8859100341797,
+      "loss": 0.3366,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 4.704905033111572,
+      "rewards/margins": 6.591601371765137,
+      "rewards/rejected": -1.8866965770721436,
       "step": 70
     },
     {
       "epoch": 0.2,
-      "grad_norm": 729.7756176405182,
+      "grad_norm": 771.1936928093826,
       "learning_rate": 4.83354989019146e-07,
-      "logits/chosen": -2.5291531085968018,
-      "logits/rejected": -2.453605890274048,
-      "logps/chosen": -259.8160400390625,
-      "logps/rejected": -204.7032470703125,
-      "loss": 0.3123,
+      "logits/chosen": -2.534698247909546,
+      "logits/rejected": -2.4588425159454346,
+      "logps/chosen": -259.4831848144531,
+      "logps/rejected": -204.6412353515625,
+      "loss": 0.3162,
       "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 5.419482231140137,
-      "rewards/margins": 6.558361053466797,
-      "rewards/rejected": -1.1388784646987915,
+      "rewards/chosen": 5.652478218078613,
+      "rewards/margins": 6.747942924499512,
+      "rewards/rejected": -1.0954645872116089,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 876.3404421120181,
+      "grad_norm": 746.9475535161084,
       "learning_rate": 4.7440343190975353e-07,
-      "logits/chosen": -2.5618221759796143,
-      "logits/rejected": -2.5039360523223877,
-      "logps/chosen": -257.5802001953125,
-      "logps/rejected": -217.9538116455078,
-      "loss": 0.3037,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 2.9155490398406982,
-      "rewards/margins": 5.336374759674072,
-      "rewards/rejected": -2.420825481414795,
+      "logits/chosen": -2.5645816326141357,
+      "logits/rejected": -2.5068752765655518,
+      "logps/chosen": -257.1772155761719,
+      "logps/rejected": -217.83139038085938,
+      "loss": 0.31,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 3.197655200958252,
+      "rewards/margins": 5.532810211181641,
+      "rewards/rejected": -2.3351550102233887,
       "step": 90
     },
     {
       "epoch": 0.26,
-      "grad_norm": 921.3524245495258,
+      "grad_norm": 823.7911810864297,
       "learning_rate": 4.6365538373900506e-07,
-      "logits/chosen": -2.614607095718384,
-      "logits/rejected": -2.538961172103882,
-      "logps/chosen": -236.43710327148438,
-      "logps/rejected": -201.99563598632812,
-      "loss": 0.3983,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 4.012251853942871,
-      "rewards/margins": 6.339661598205566,
-      "rewards/rejected": -2.3274102210998535,
+      "logits/chosen": -2.6224541664123535,
+      "logits/rejected": -2.5483615398406982,
+      "logps/chosen": -236.20339965820312,
+      "logps/rejected": -201.47940063476562,
+      "loss": 0.5348,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 4.17586088180542,
+      "rewards/margins": 6.1418962478637695,
+      "rewards/rejected": -1.9660362005233765,
       "step": 100
     },
     {
       "epoch": 0.28,
-      "grad_norm": 541.0592397551568,
+      "grad_norm": 579.991739437595,
       "learning_rate": 4.5119688941406386e-07,
-      "logits/chosen": -2.605010509490967,
-      "logits/rejected": -2.523113489151001,
-      "logps/chosen": -256.91888427734375,
-      "logps/rejected": -209.4059600830078,
-      "loss": 0.4244,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 6.128890037536621,
-      "rewards/margins": 6.816291809082031,
-      "rewards/rejected": -0.6874018907546997,
+      "logits/chosen": -2.6264939308166504,
+      "logits/rejected": -2.5470776557922363,
+      "logps/chosen": -256.7718200683594,
+      "logps/rejected": -209.46231079101562,
+      "loss": 0.4091,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 6.23184061050415,
+      "rewards/margins": 6.958665370941162,
+      "rewards/rejected": -0.7268255352973938,
       "step": 110
     },
     {
       "epoch": 0.31,
-      "grad_norm": 751.0517883930627,
+      "grad_norm": 690.1539082640518,
       "learning_rate": 4.3712768704277524e-07,
-      "logits/chosen": -2.575998067855835,
-      "logits/rejected": -2.5046029090881348,
-      "logps/chosen": -261.73760986328125,
-      "logps/rejected": -208.35629272460938,
-      "loss": 0.4211,
+      "logits/chosen": -2.6043858528137207,
+      "logits/rejected": -2.5360493659973145,
+      "logps/chosen": -261.69561767578125,
+      "logps/rejected": -208.3325653076172,
+      "loss": 0.4048,
       "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 5.407345771789551,
-      "rewards/margins": 6.800279140472412,
-      "rewards/rejected": -1.3929327726364136,
+      "rewards/chosen": 5.436749458312988,
+      "rewards/margins": 6.8130621910095215,
+      "rewards/rejected": -1.3763134479522705,
       "step": 120
     },
     {
       "epoch": 0.33,
-      "grad_norm": 469.03377090763667,
+      "grad_norm": 451.1743521072461,
       "learning_rate": 4.2156040946718343e-07,
-      "logits/chosen": -2.55499267578125,
-      "logits/rejected": -2.48675274848938,
-      "logps/chosen": -251.78982543945312,
-      "logps/rejected": -197.8007354736328,
-      "loss": 0.3751,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": 3.859405994415283,
-      "rewards/margins": 6.706671714782715,
-      "rewards/rejected": -2.847266435623169,
+      "logits/chosen": -2.578007698059082,
+      "logits/rejected": -2.5135886669158936,
+      "logps/chosen": -250.7688446044922,
+      "logps/rejected": -196.73779296875,
+      "loss": 0.3698,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 4.574068069458008,
+      "rewards/margins": 6.677270412445068,
+      "rewards/rejected": -2.1032023429870605,
       "step": 130
     },
     {
       "epoch": 0.36,
-      "grad_norm": 469.33299961847337,
+      "grad_norm": 465.41383196085104,
       "learning_rate": 4.046196825665637e-07,
-      "logits/chosen": -2.5828466415405273,
-      "logits/rejected": -2.512634515762329,
-      "logps/chosen": -270.5353088378906,
-      "logps/rejected": -217.965087890625,
-      "loss": 0.4019,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 3.2487106323242188,
-      "rewards/margins": 6.334321022033691,
-      "rewards/rejected": -3.085610866546631,
+      "logits/chosen": -2.600147247314453,
+      "logits/rejected": -2.534235715866089,
+      "logps/chosen": -269.448974609375,
+      "logps/rejected": -216.6135711669922,
+      "loss": 0.4171,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 4.009121417999268,
+      "rewards/margins": 6.148682594299316,
+      "rewards/rejected": -2.139561176300049,
       "step": 140
     },
     {
       "epoch": 0.38,
-      "grad_norm": 519.7264699613345,
+      "grad_norm": 485.4989695160128,
       "learning_rate": 3.864411275486261e-07,
-      "logits/chosen": -2.5669846534729004,
-      "logits/rejected": -2.4968345165252686,
-      "logps/chosen": -263.26190185546875,
-      "logps/rejected": -213.07968139648438,
-      "loss": 0.5855,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 5.130881309509277,
-      "rewards/margins": 7.009912014007568,
-      "rewards/rejected": -1.8790302276611328,
+      "logits/chosen": -2.583752155303955,
+      "logits/rejected": -2.51660418510437,
+      "logps/chosen": -262.35479736328125,
+      "logps/rejected": -212.38021850585938,
+      "loss": 0.429,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 5.765866756439209,
+      "rewards/margins": 7.155278205871582,
+      "rewards/rejected": -1.3894122838974,
       "step": 150
     },
     {
       "epoch": 0.41,
-      "grad_norm": 692.9336367256366,
+      "grad_norm": 546.3479101912284,
       "learning_rate": 3.671702752161759e-07,
-      "logits/chosen": -2.548508405685425,
-      "logits/rejected": -2.476653814315796,
-      "logps/chosen": -244.92520141601562,
-      "logps/rejected": -199.13551330566406,
-      "loss": 0.4272,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 2.8089261054992676,
-      "rewards/margins": 7.088732719421387,
-      "rewards/rejected": -4.279807090759277,
+      "logits/chosen": -2.5821540355682373,
+      "logits/rejected": -2.514200210571289,
+      "logps/chosen": -242.9073944091797,
+      "logps/rejected": -197.12026977539062,
+      "loss": 0.4562,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 4.221369743347168,
+      "rewards/margins": 7.090517520904541,
+      "rewards/rejected": -2.869147777557373,
       "step": 160
     },
     {
       "epoch": 0.44,
-      "grad_norm": 617.441034416213,
+      "grad_norm": 655.017465367729,
       "learning_rate": 3.4696140090121375e-07,
-      "logits/chosen": -2.562197685241699,
-      "logits/rejected": -2.4945685863494873,
-      "logps/chosen": -266.8678283691406,
-      "logps/rejected": -213.10153198242188,
-      "loss": 0.356,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 2.6767470836639404,
-      "rewards/margins": 7.383553504943848,
-      "rewards/rejected": -4.70680570602417,
+      "logits/chosen": -2.582387685775757,
+      "logits/rejected": -2.517115831375122,
+      "logps/chosen": -264.36541748046875,
+      "logps/rejected": -210.4468231201172,
+      "loss": 0.3517,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 4.428440570831299,
+      "rewards/margins": 7.276960849761963,
+      "rewards/rejected": -2.8485207557678223,
       "step": 170
     },
     {
       "epoch": 0.46,
-      "grad_norm": 507.277088262404,
+      "grad_norm": 492.2183630043871,
       "learning_rate": 3.259762893935617e-07,
-      "logits/chosen": -2.642911195755005,
-      "logits/rejected": -2.560927152633667,
-      "logps/chosen": -237.3924102783203,
-      "logps/rejected": -187.8155975341797,
-      "loss": 0.4121,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 3.2487025260925293,
-      "rewards/margins": 6.444317817687988,
-      "rewards/rejected": -3.1956145763397217,
+      "logits/chosen": -2.6398727893829346,
+      "logits/rejected": -2.5528438091278076,
+      "logps/chosen": -235.0208282470703,
+      "logps/rejected": -185.29156494140625,
+      "loss": 0.4065,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 4.908794403076172,
+      "rewards/margins": 6.33758544921875,
+      "rewards/rejected": -1.4287910461425781,
       "step": 180
     },
     {
       "epoch": 0.49,
-      "grad_norm": 561.5062955738938,
+      "grad_norm": 502.0957454157663,
       "learning_rate": 3.0438293975154184e-07,
-      "logits/chosen": -2.609133005142212,
-      "logits/rejected": -2.5395116806030273,
-      "logps/chosen": -259.6526184082031,
-      "logps/rejected": -204.88461303710938,
-      "loss": 0.3414,
+      "logits/chosen": -2.5951266288757324,
+      "logits/rejected": -2.519169330596924,
+      "logps/chosen": -258.04022216796875,
+      "logps/rejected": -202.89727783203125,
+      "loss": 0.3426,
       "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 3.617593288421631,
-      "rewards/margins": 7.505879878997803,
-      "rewards/rejected": -3.8882858753204346,
+      "rewards/chosen": 4.746270179748535,
+      "rewards/margins": 7.24340295791626,
+      "rewards/rejected": -2.4971330165863037,
       "step": 190
     },
     {
       "epoch": 0.51,
-      "grad_norm": 543.0498837871727,
+      "grad_norm": 536.1992163678008,
       "learning_rate": 2.823542203635138e-07,
-      "logits/chosen": -2.6425347328186035,
-      "logits/rejected": -2.5596280097961426,
-      "logps/chosen": -275.01141357421875,
-      "logps/rejected": -219.7457733154297,
-      "loss": 0.3839,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 3.858170747756958,
-      "rewards/margins": 8.147393226623535,
-      "rewards/rejected": -4.289223670959473,
+      "logits/chosen": -2.6266205310821533,
+      "logits/rejected": -2.5365912914276123,
+      "logps/chosen": -273.0335388183594,
+      "logps/rejected": -217.910888671875,
+      "loss": 0.3772,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 5.242685794830322,
+      "rewards/margins": 8.247489929199219,
+      "rewards/rejected": -3.004803419113159,
       "step": 200
     },
     {
       "epoch": 0.54,
-      "grad_norm": 483.69063175339187,
+      "grad_norm": 550.4099704660905,
       "learning_rate": 2.600664850273538e-07,
-      "logits/chosen": -2.6228067874908447,
-      "logits/rejected": -2.551823616027832,
-      "logps/chosen": -268.2137451171875,
-      "logps/rejected": -213.346435546875,
-      "loss": 0.5726,
+      "logits/chosen": -2.6228280067443848,
+      "logits/rejected": -2.550497531890869,
+      "logps/chosen": -265.55755615234375,
+      "logps/rejected": -210.61660766601562,
+      "loss": 0.5254,
       "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 1.9880602359771729,
-      "rewards/margins": 6.878021240234375,
-      "rewards/rejected": -4.889961242675781,
+      "rewards/chosen": 3.847407102584839,
+      "rewards/margins": 6.826489448547363,
+      "rewards/rejected": -2.979081869125366,
       "step": 210
     },
     {
       "epoch": 0.56,
-      "grad_norm": 600.4784540161153,
+      "grad_norm": 581.5523533407168,
       "learning_rate": 2.3769816112703045e-07,
-      "logits/chosen": -2.638782501220703,
-      "logits/rejected": -2.5772552490234375,
-      "logps/chosen": -258.2492370605469,
-      "logps/rejected": -215.2850341796875,
-      "loss": 0.4211,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 2.3993289470672607,
-      "rewards/margins": 6.262051582336426,
-      "rewards/rejected": -3.862722396850586,
+      "logits/chosen": -2.6491611003875732,
+      "logits/rejected": -2.5865907669067383,
+      "logps/chosen": -255.453857421875,
+      "logps/rejected": -212.5009765625,
+      "loss": 0.4229,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 4.356105327606201,
+      "rewards/margins": 6.26999568939209,
+      "rewards/rejected": -1.9138901233673096,
       "step": 220
     },
     {
       "epoch": 0.59,
-      "grad_norm": 498.9901423348722,
+      "grad_norm": 486.64295333011063,
       "learning_rate": 2.1542832120881677e-07,
-      "logits/chosen": -2.672987699508667,
-      "logits/rejected": -2.5890109539031982,
-      "logps/chosen": -267.2917175292969,
-      "logps/rejected": -217.24136352539062,
-      "loss": 0.3733,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 4.277976036071777,
-      "rewards/margins": 7.3069305419921875,
-      "rewards/rejected": -3.0289549827575684,
+      "logits/chosen": -2.6853079795837402,
+      "logits/rejected": -2.600752830505371,
+      "logps/chosen": -264.80615234375,
+      "logps/rejected": -214.556396484375,
+      "loss": 0.3669,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 6.017881393432617,
+      "rewards/margins": 7.167391300201416,
+      "rewards/rejected": -1.1495091915130615,
       "step": 230
     },
     {
       "epoch": 0.61,
-      "grad_norm": 704.2710598090675,
+      "grad_norm": 831.6224733555287,
       "learning_rate": 1.934352493925695e-07,
-      "logits/chosen": -2.6564643383026123,
-      "logits/rejected": -2.612344264984131,
-      "logps/chosen": -262.6011657714844,
-      "logps/rejected": -221.1019287109375,
-      "loss": 0.3611,
+      "logits/chosen": -2.667158603668213,
+      "logits/rejected": -2.620408058166504,
+      "logps/chosen": -261.4100036621094,
+      "logps/rejected": -219.61032104492188,
+      "loss": 0.3738,
       "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 3.5910720825195312,
-      "rewards/margins": 8.862504005432129,
-      "rewards/rejected": -5.271432399749756,
+      "rewards/chosen": 4.424870014190674,
+      "rewards/margins": 8.652185440063477,
+      "rewards/rejected": -4.227315425872803,
       "step": 240
     },
     {
       "epoch": 0.64,
-      "grad_norm": 461.54740219100705,
+      "grad_norm": 563.6526543418182,
       "learning_rate": 1.7189501409486059e-07,
-      "logits/chosen": -2.6683101654052734,
-      "logits/rejected": -2.600257396697998,
-      "logps/chosen": -267.53851318359375,
-      "logps/rejected": -222.7628936767578,
-      "loss": 0.3684,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": 2.9681382179260254,
-      "rewards/margins": 7.435116767883301,
-      "rewards/rejected": -4.466979026794434,
+      "logits/chosen": -2.674989938735962,
+      "logits/rejected": -2.606071949005127,
+      "logps/chosen": -266.0426940917969,
+      "logps/rejected": -221.0687713623047,
+      "loss": 0.367,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 4.015201091766357,
+      "rewards/margins": 7.296302795410156,
+      "rewards/rejected": -3.281102418899536,
       "step": 250
     },
     {
       "epoch": 0.67,
-      "grad_norm": 420.10106625793907,
+      "grad_norm": 457.0558678943042,
       "learning_rate": 1.5098005849021078e-07,
-      "logits/chosen": -2.6590428352355957,
-      "logits/rejected": -2.602717876434326,
-      "logps/chosen": -261.82867431640625,
-      "logps/rejected": -209.0205078125,
-      "loss": 0.3413,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 2.8218843936920166,
-      "rewards/margins": 6.89211893081665,
-      "rewards/rejected": -4.070234298706055,
+      "logits/chosen": -2.6647772789001465,
+      "logits/rejected": -2.6081786155700684,
+      "logps/chosen": -260.12384033203125,
+      "logps/rejected": -207.2174835205078,
+      "loss": 0.3387,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 4.0153021812438965,
+      "rewards/margins": 6.823422908782959,
+      "rewards/rejected": -2.8081207275390625,
       "step": 260
     },
     {
       "epoch": 0.69,
-      "grad_norm": 504.92672454975565,
+      "grad_norm": 478.6275517216598,
       "learning_rate": 1.30857819994673e-07,
-      "logits/chosen": -2.6389431953430176,
-      "logits/rejected": -2.556539297103882,
-      "logps/chosen": -270.43634033203125,
-      "logps/rejected": -230.3980255126953,
-      "loss": 0.5197,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 4.303304195404053,
-      "rewards/margins": 11.129814147949219,
-      "rewards/rejected": -6.826510429382324,
+      "logits/chosen": -2.641366481781006,
+      "logits/rejected": -2.558147668838501,
+      "logps/chosen": -272.80377197265625,
+      "logps/rejected": -228.5403594970703,
+      "loss": 0.4885,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 2.646123170852661,
+      "rewards/margins": 8.172276496887207,
+      "rewards/rejected": -5.526152610778809,
       "step": 270
     },
     {
       "epoch": 0.72,
-      "grad_norm": 390.2906453798093,
+      "grad_norm": 369.20521183996,
       "learning_rate": 1.116893898236716e-07,
-      "logits/chosen": -2.663546323776245,
-      "logits/rejected": -2.610297918319702,
-      "logps/chosen": -270.1681213378906,
-      "logps/rejected": -220.12118530273438,
-      "loss": 0.364,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 2.4767448902130127,
-      "rewards/margins": 7.500287055969238,
-      "rewards/rejected": -5.0235419273376465,
+      "logits/chosen": -2.6526267528533936,
+      "logits/rejected": -2.597562789916992,
+      "logps/chosen": -268.1092224121094,
+      "logps/rejected": -217.9169464111328,
+      "loss": 0.3441,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 3.9179558753967285,
+      "rewards/margins": 7.3985185623168945,
+      "rewards/rejected": -3.480562686920166,
       "step": 280
     },
     {
       "epoch": 0.74,
-      "grad_norm": 476.5711551299184,
+      "grad_norm": 498.11211487860106,
       "learning_rate": 9.362822335518062e-08,
-      "logits/chosen": -2.6236255168914795,
-      "logits/rejected": -2.580749034881592,
-      "logps/chosen": -268.32061767578125,
-      "logps/rejected": -217.591552734375,
-      "loss": 0.3294,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 2.5377907752990723,
-      "rewards/margins": 7.069185733795166,
-      "rewards/rejected": -4.531395435333252,
+      "logits/chosen": -2.6078855991363525,
+      "logits/rejected": -2.5603675842285156,
+      "logps/chosen": -265.8270263671875,
+      "logps/rejected": -215.421142578125,
+      "loss": 0.3216,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 4.283304691314697,
+      "rewards/margins": 7.295420169830322,
+      "rewards/rejected": -3.012115478515625,
       "step": 290
     },
     {
       "epoch": 0.77,
-      "grad_norm": 606.8401904083239,
+      "grad_norm": 433.49223304725587,
       "learning_rate": 7.681891162260015e-08,
-      "logits/chosen": -2.642962694168091,
-      "logits/rejected": -2.5925967693328857,
-      "logps/chosen": -274.8097229003906,
-      "logps/rejected": -221.6527557373047,
-      "loss": 0.369,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 2.635510206222534,
-      "rewards/margins": 7.277352333068848,
-      "rewards/rejected": -4.641841888427734,
+      "logits/chosen": -2.6258552074432373,
+      "logits/rejected": -2.5720603466033936,
+      "logps/chosen": -272.7791442871094,
+      "logps/rejected": -219.44140625,
+      "loss": 0.3717,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 4.056910514831543,
+      "rewards/margins": 7.150809288024902,
+      "rewards/rejected": -3.093899726867676,
       "step": 300
     },
     {
       "epoch": 0.79,
-      "grad_norm": 376.30332763972234,
+      "grad_norm": 369.6165987845038,
       "learning_rate": 6.139602377230247e-08,
-      "logits/chosen": -2.607694149017334,
-      "logits/rejected": -2.543703317642212,
-      "logps/chosen": -278.32476806640625,
-      "logps/rejected": -216.56387329101562,
-      "loss": 0.4215,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 2.7644286155700684,
-      "rewards/margins": 7.646514892578125,
-      "rewards/rejected": -4.882087230682373,
+      "logits/chosen": -2.5929980278015137,
+      "logits/rejected": -2.524878978729248,
+      "logps/chosen": -276.72418212890625,
+      "logps/rejected": -214.73757934570312,
+      "loss": 0.3974,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 3.884862184524536,
+      "rewards/margins": 7.488539695739746,
+      "rewards/rejected": -3.6036782264709473,
       "step": 310
     },
     {
       "epoch": 0.82,
-      "grad_norm": 611.2992882040782,
+      "grad_norm": 542.2853213894015,
       "learning_rate": 4.748302975270837e-08,
-      "logits/chosen": -2.6328682899475098,
-      "logits/rejected": -2.5906100273132324,
-      "logps/chosen": -261.64044189453125,
-      "logps/rejected": -205.4202880859375,
-      "loss": 0.3795,
+      "logits/chosen": -2.618070125579834,
+      "logits/rejected": -2.5723397731781006,
+      "logps/chosen": -259.38433837890625,
+      "logps/rejected": -202.79501342773438,
+      "loss": 0.3593,
       "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 2.2899277210235596,
-      "rewards/margins": 6.90247106552124,
-      "rewards/rejected": -4.612543106079102,
+      "rewards/chosen": 3.869190216064453,
+      "rewards/margins": 6.644041538238525,
+      "rewards/rejected": -2.7748515605926514,
       "step": 320
     },
     {
       "epoch": 0.84,
-      "grad_norm": 530.5762993099619,
+      "grad_norm": 1179.920803140471,
       "learning_rate": 3.5191311859445795e-08,
-      "logits/chosen": -2.6538619995117188,
-      "logits/rejected": -2.6000447273254395,
-      "logps/chosen": -264.9809875488281,
-      "logps/rejected": -218.4214630126953,
-      "loss": 0.3638,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 3.3024134635925293,
-      "rewards/margins": 7.286877632141113,
-      "rewards/rejected": -3.984463930130005,
+      "logits/chosen": -2.639923572540283,
+      "logits/rejected": -2.582639455795288,
+      "logps/chosen": -262.33123779296875,
+      "logps/rejected": -215.505615234375,
+      "loss": 0.3667,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 5.157271385192871,
+      "rewards/margins": 7.100647926330566,
+      "rewards/rejected": -1.9433765411376953,
       "step": 330
     },
     {
       "epoch": 0.87,
-      "grad_norm": 483.450025523063,
+      "grad_norm": 467.7658799716397,
       "learning_rate": 2.4619273049795996e-08,
-      "logits/chosen": -2.638885259628296,
-      "logits/rejected": -2.585780620574951,
-      "logps/chosen": -260.09991455078125,
-      "logps/rejected": -211.5401153564453,
-      "loss": 0.3361,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 3.2979087829589844,
-      "rewards/margins": 7.9664201736450195,
-      "rewards/rejected": -4.668511390686035,
+      "logits/chosen": -2.6265780925750732,
+      "logits/rejected": -2.5702362060546875,
+      "logps/chosen": -257.75689697265625,
+      "logps/rejected": -209.0670623779297,
+      "loss": 0.324,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 4.93801212310791,
+      "rewards/margins": 7.8753981590271,
+      "rewards/rejected": -2.9373855590820312,
       "step": 340
     },
     {
       "epoch": 0.9,
-      "grad_norm": 373.8282686849937,
+      "grad_norm": 345.87107109384476,
       "learning_rate": 1.5851549164932115e-08,
-      "logits/chosen": -2.650052547454834,
-      "logits/rejected": -2.6026833057403564,
-      "logps/chosen": -269.6961364746094,
-      "logps/rejected": -227.0093536376953,
-      "loss": 0.3428,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 3.188291072845459,
-      "rewards/margins": 7.054718971252441,
-      "rewards/rejected": -3.866427183151245,
+      "logits/chosen": -2.637718677520752,
+      "logits/rejected": -2.589808940887451,
+      "logps/chosen": -267.7754821777344,
+      "logps/rejected": -225.0698699951172,
+      "loss": 0.3399,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 4.532751560211182,
+      "rewards/margins": 7.0415191650390625,
+      "rewards/rejected": -2.508768320083618,
       "step": 350
     },
     {
       "epoch": 0.92,
-      "grad_norm": 570.1026903534258,
+      "grad_norm": 491.63656420568554,
       "learning_rate": 8.958331366609423e-09,
-      "logits/chosen": -2.6525673866271973,
-      "logits/rejected": -2.5886600017547607,
-      "logps/chosen": -275.1505432128906,
-      "logps/rejected": -220.02554321289062,
-      "loss": 0.4069,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 2.8464574813842773,
-      "rewards/margins": 7.254122257232666,
-      "rewards/rejected": -4.4076642990112305,
+      "logits/chosen": -2.641171932220459,
+      "logits/rejected": -2.574852705001831,
+      "logps/chosen": -272.898193359375,
+      "logps/rejected": -217.81997680664062,
+      "loss": 0.3962,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 4.423092842102051,
+      "rewards/margins": 7.286849021911621,
+      "rewards/rejected": -2.863755464553833,
       "step": 360
     },
     {
       "epoch": 0.95,
-      "grad_norm": 578.8321276414978,
+      "grad_norm": 863.5117187901297,
       "learning_rate": 3.994804212627461e-09,
-      "logits/chosen": -2.612220287322998,
-      "logits/rejected": -2.578857898712158,
-      "logps/chosen": -273.8528747558594,
-      "logps/rejected": -229.8456268310547,
-      "loss": 0.5088,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 3.7925782203674316,
-      "rewards/margins": 7.483479976654053,
-      "rewards/rejected": -3.690901517868042,
+      "logits/chosen": -2.600586175918579,
+      "logits/rejected": -2.565150737762451,
+      "logps/chosen": -271.3792419433594,
+      "logps/rejected": -227.0088348388672,
+      "loss": 0.4043,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 5.524130821228027,
+      "rewards/margins": 7.229277610778809,
+      "rewards/rejected": -1.7051467895507812,
       "step": 370
     },
     {
       "epoch": 0.97,
-      "grad_norm": 313.4387261721762,
+      "grad_norm": 386.7539067890418,
       "learning_rate": 1.0007038696262516e-09,
-      "logits/chosen": -2.658742904663086,
-      "logits/rejected": -2.6213126182556152,
-      "logps/chosen": -263.07525634765625,
-      "logps/rejected": -230.99319458007812,
-      "loss": 0.3517,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 3.717088222503662,
-      "rewards/margins": 7.349850654602051,
-      "rewards/rejected": -3.6327621936798096,
+      "logits/chosen": -2.649324893951416,
+      "logits/rejected": -2.608961820602417,
+      "logps/chosen": -261.44415283203125,
+      "logps/rejected": -229.10617065429688,
+      "loss": 0.3514,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 4.858848571777344,
+      "rewards/margins": 7.17071008682251,
+      "rewards/rejected": -2.311861276626587,
       "step": 380
     },
     {
       "epoch": 1.0,
-      "grad_norm": 738.0895020493799,
+      "grad_norm": 658.7609084129675,
       "learning_rate": 0.0,
-      "logits/chosen": -2.6698050498962402,
-      "logits/rejected": -2.61210036277771,
-      "logps/chosen": -251.17208862304688,
-      "logps/rejected": -211.110595703125,
-      "loss": 0.4022,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 2.67566180229187,
-      "rewards/margins": 7.086625099182129,
-      "rewards/rejected": -4.410962104797363,
+      "logits/chosen": -2.657970428466797,
+      "logits/rejected": -2.5974597930908203,
+      "logps/chosen": -249.08193969726562,
+      "logps/rejected": -208.86935424804688,
+      "loss": 0.389,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 4.138771057128906,
+      "rewards/margins": 6.9808807373046875,
+      "rewards/rejected": -2.8421101570129395,
       "step": 390
     },
     {
       "epoch": 1.0,
       "step": 390,
       "total_flos": 0.0,
-      "train_loss": 0.3992691706388425,
-      "train_runtime": 5832.237,
-      "train_samples_per_second": 8.573,
+      "train_loss": 0.39275855483152927,
+      "train_runtime": 5850.2913,
+      "train_samples_per_second": 8.547,
       "train_steps_per_second": 0.067
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f0df4506e6cfc7e0993705b9aafa86be9fbe115e2e48aa2a48a1cf5b25398062
+oid sha256:69755ef6c720cdfd7d14b231fbf0ad5a8fc29c369a6e770d1d3031286dc30b37
 size 6264