Model save

README.md

@@ -37,7 +37,7 @@ The following hyperparameters were used during training:
 - learning_rate: 5e-07
 - train_batch_size: 8
 - eval_batch_size: 8
-- seed: 1
+- seed: 2
 - distributed_type: multi-GPU
 - num_devices: 8
 - gradient_accumulation_steps: 2

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.1290441479947832,
-    "train_runtime": 3205.4322,
+    "train_loss": 0.13281457475674005,
+    "train_runtime": 3219.0486,
     "train_samples": 51894,
-    "train_samples_per_second": 16.189,
+    "train_samples_per_second": 16.121,
     "train_steps_per_second": 0.126
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:63eebf66ef032e0273cefa649145ef22ac9d7c7a15a63949bfab2dd82eea3b99
+oid sha256:aab46e8124d9cacc43920ed6943b79f01ec0d55f715b8588dba66c609a2b1d4b
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a18853d43d98a4783811f48aabfdff6a22b3723a0bf973d97dcc5d02e52f01f3
+oid sha256:8d07be1111d3b5bc2c14ffdaf4371c594ea83fb6a105d7bbceaeeb9d43d3f17b
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b07e126a50b2f207cc01c4d63a5c95af42502791e0e97926155fd6c343bfe98
+oid sha256:74f1ed244a41d859eb13abcea7c5eee6a043a8a2c39a674f91dbbfc0cf7e2a35
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.1290441479947832,
-    "train_runtime": 3205.4322,
+    "train_loss": 0.13281457475674005,
+    "train_runtime": 3219.0486,
     "train_samples": 51894,
-    "train_samples_per_second": 16.189,
+    "train_samples_per_second": 16.121,
     "train_steps_per_second": 0.126
 }

trainer_state.json

@@ -11,11 +11,11 @@
     {
       "epoch": 0.0,
       "learning_rate": 1.2195121951219512e-08,
-      "logits/chosen": -2.769179582595825,
-      "logits/rejected": -2.6384379863739014,
-      "logps/chosen": -511.7125244140625,
-      "logps/rejected": -154.70135498046875,
-      "loss": 0.4155,
+      "logits/chosen": -2.8412017822265625,
+      "logits/rejected": -2.798035144805908,
+      "logps/chosen": -481.1488037109375,
+      "logps/rejected": -89.30835723876953,
+      "loss": 0.4176,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,570 +25,570 @@
     {
       "epoch": 0.02,
       "learning_rate": 1.219512195121951e-07,
-      "logits/chosen": -2.7492480278015137,
-      "logits/rejected": -2.7196333408355713,
-      "logps/chosen": -350.17138671875,
-      "logps/rejected": -113.88729095458984,
-      "loss": 0.427,
-      "rewards/accuracies": 0.5833333134651184,
-      "rewards/chosen": 0.0007148745935410261,
-      "rewards/margins": 0.0013887248933315277,
-      "rewards/rejected": -0.0006738504162058234,
+      "logits/chosen": -2.760530948638916,
+      "logits/rejected": -2.7219057083129883,
+      "logps/chosen": -418.49639892578125,
+      "logps/rejected": -116.74214935302734,
+      "loss": 0.4186,
+      "rewards/accuracies": 0.5486111044883728,
+      "rewards/chosen": 0.0008742791833356023,
+      "rewards/margins": 0.0011628220090642571,
+      "rewards/rejected": -0.0002885429421439767,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.439024390243902e-07,
-      "logits/chosen": -2.73565936088562,
-      "logits/rejected": -2.697619676589966,
-      "logps/chosen": -417.7752990722656,
-      "logps/rejected": -126.06368255615234,
-      "loss": 0.4206,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.016129938885569572,
-      "rewards/margins": 0.031221503391861916,
-      "rewards/rejected": -0.015091565437614918,
+      "logits/chosen": -2.785912036895752,
+      "logits/rejected": -2.7400879859924316,
+      "logps/chosen": -395.93634033203125,
+      "logps/rejected": -109.28425598144531,
+      "loss": 0.4276,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.014862718991935253,
+      "rewards/margins": 0.029387209564447403,
+      "rewards/rejected": -0.014524489641189575,
       "step": 20
     },
     {
       "epoch": 0.07,
       "learning_rate": 3.6585365853658536e-07,
-      "logits/chosen": -2.701590061187744,
-      "logits/rejected": -2.6967225074768066,
-      "logps/chosen": -394.0975646972656,
-      "logps/rejected": -112.23805236816406,
-      "loss": 0.4044,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": 0.06890617311000824,
-      "rewards/margins": 0.18525615334510803,
-      "rewards/rejected": -0.1163499727845192,
+      "logits/chosen": -2.70774507522583,
+      "logits/rejected": -2.6736245155334473,
+      "logps/chosen": -375.3507385253906,
+      "logps/rejected": -121.47686767578125,
+      "loss": 0.4137,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.06373313069343567,
+      "rewards/margins": 0.1794833242893219,
+      "rewards/rejected": -0.11575017869472504,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.878048780487804e-07,
-      "logits/chosen": -2.5483059883117676,
-      "logits/rejected": -2.5507187843322754,
-      "logps/chosen": -388.41693115234375,
-      "logps/rejected": -163.46432495117188,
-      "loss": 0.3829,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.02780727483332157,
-      "rewards/margins": 0.41306740045547485,
-      "rewards/rejected": -0.38526007533073425,
+      "logits/chosen": -2.5397536754608154,
+      "logits/rejected": -2.5152153968811035,
+      "logps/chosen": -426.83306884765625,
+      "logps/rejected": -164.6002960205078,
+      "loss": 0.3837,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 0.06933724880218506,
+      "rewards/margins": 0.513908326625824,
+      "rewards/rejected": -0.4445711076259613,
       "step": 40
     },
     {
       "epoch": 0.12,
       "learning_rate": 4.992461696250783e-07,
-      "logits/chosen": -2.425833225250244,
-      "logits/rejected": -2.388296604156494,
-      "logps/chosen": -441.8356018066406,
-      "logps/rejected": -209.2742156982422,
-      "loss": 0.3064,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -0.09178180992603302,
-      "rewards/margins": 0.8215526342391968,
-      "rewards/rejected": -0.9133344888687134,
+      "logits/chosen": -2.425987958908081,
+      "logits/rejected": -2.42197585105896,
+      "logps/chosen": -366.08343505859375,
+      "logps/rejected": -200.18582153320312,
+      "loss": 0.3126,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.1581335961818695,
+      "rewards/margins": 0.757983386516571,
+      "rewards/rejected": -0.9161170721054077,
       "step": 50
     },
     {
       "epoch": 0.15,
       "learning_rate": 4.966461721767899e-07,
-      "logits/chosen": -2.3990914821624756,
-      "logits/rejected": -2.3541452884674072,
-      "logps/chosen": -423.5923767089844,
-      "logps/rejected": -237.95361328125,
-      "loss": 0.2477,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.30225870013237,
-      "rewards/margins": 1.0505648851394653,
-      "rewards/rejected": -1.3528234958648682,
+      "logits/chosen": -2.3848328590393066,
+      "logits/rejected": -2.298985719680786,
+      "logps/chosen": -490.4847106933594,
+      "logps/rejected": -254.928466796875,
+      "loss": 0.2321,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -0.22150389850139618,
+      "rewards/margins": 1.2847968339920044,
+      "rewards/rejected": -1.506300926208496,
       "step": 60
     },
     {
       "epoch": 0.17,
       "learning_rate": 4.922100518015975e-07,
-      "logits/chosen": -2.414358615875244,
-      "logits/rejected": -2.3809196949005127,
-      "logps/chosen": -453.24755859375,
-      "logps/rejected": -317.23309326171875,
-      "loss": 0.1919,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.5829191207885742,
-      "rewards/margins": 1.3952158689498901,
-      "rewards/rejected": -1.978135108947754,
+      "logits/chosen": -2.340442180633545,
+      "logits/rejected": -2.2820117473602295,
+      "logps/chosen": -490.87615966796875,
+      "logps/rejected": -321.3214111328125,
+      "loss": 0.1643,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.6543983817100525,
+      "rewards/margins": 1.5231101512908936,
+      "rewards/rejected": -2.17750883102417,
       "step": 70
     },
     {
       "epoch": 0.2,
       "learning_rate": 4.859708325770919e-07,
-      "logits/chosen": -2.3892292976379395,
-      "logits/rejected": -2.3620591163635254,
-      "logps/chosen": -466.544921875,
-      "logps/rejected": -345.60174560546875,
-      "loss": 0.1845,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.6911368370056152,
-      "rewards/margins": 1.3688609600067139,
-      "rewards/rejected": -2.05999755859375,
+      "logits/chosen": -2.3740787506103516,
+      "logits/rejected": -2.3113033771514893,
+      "logps/chosen": -471.11651611328125,
+      "logps/rejected": -314.3768005371094,
+      "loss": 0.182,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.38558533787727356,
+      "rewards/margins": 1.7214374542236328,
+      "rewards/rejected": -2.107023000717163,
       "step": 80
     },
     {
       "epoch": 0.22,
       "learning_rate": 4.779749614980225e-07,
-      "logits/chosen": -2.3470118045806885,
-      "logits/rejected": -2.2671432495117188,
-      "logps/chosen": -550.2689208984375,
-      "logps/rejected": -383.6993713378906,
-      "loss": 0.1375,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -0.528198778629303,
-      "rewards/margins": 2.0589098930358887,
-      "rewards/rejected": -2.587108612060547,
+      "logits/chosen": -2.2898011207580566,
+      "logits/rejected": -2.192032814025879,
+      "logps/chosen": -496.12384033203125,
+      "logps/rejected": -380.75909423828125,
+      "loss": 0.1376,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.8930786848068237,
+      "rewards/margins": 1.819786787033081,
+      "rewards/rejected": -2.7128653526306152,
       "step": 90
     },
     {
       "epoch": 0.25,
       "learning_rate": 4.682819627081427e-07,
-      "logits/chosen": -2.359837293624878,
-      "logits/rejected": -2.2484238147735596,
-      "logps/chosen": -502.87939453125,
-      "logps/rejected": -352.00054931640625,
-      "loss": 0.1565,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -0.47763410210609436,
-      "rewards/margins": 1.936570167541504,
-      "rewards/rejected": -2.4142043590545654,
+      "logits/chosen": -2.363102436065674,
+      "logits/rejected": -2.2812604904174805,
+      "logps/chosen": -489.04302978515625,
+      "logps/rejected": -326.9526062011719,
+      "loss": 0.1424,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -0.5512069463729858,
+      "rewards/margins": 1.6478255987167358,
+      "rewards/rejected": -2.1990325450897217,
       "step": 100
     },
     {
       "epoch": 0.27,
       "learning_rate": 4.569639943810477e-07,
-      "logits/chosen": -2.303628444671631,
-      "logits/rejected": -2.1916663646698,
-      "logps/chosen": -495.01739501953125,
-      "logps/rejected": -347.0151062011719,
-      "loss": 0.122,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -0.5454806685447693,
-      "rewards/margins": 1.8509422540664673,
-      "rewards/rejected": -2.396422863006592,
+      "logits/chosen": -2.3185834884643555,
+      "logits/rejected": -2.2058892250061035,
+      "logps/chosen": -502.36529541015625,
+      "logps/rejected": -393.0018615722656,
+      "loss": 0.1052,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.6747262477874756,
+      "rewards/margins": 2.091235399246216,
+      "rewards/rejected": -2.7659618854522705,
       "step": 110
     },
     {
       "epoch": 0.3,
       "learning_rate": 4.4410531154874543e-07,
-      "logits/chosen": -2.3315651416778564,
-      "logits/rejected": -2.2395036220550537,
-      "logps/chosen": -450.09100341796875,
-      "logps/rejected": -360.4415588378906,
-      "loss": 0.1182,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -0.5643357038497925,
-      "rewards/margins": 1.957601547241211,
-      "rewards/rejected": -2.521937131881714,
+      "logits/chosen": -2.242546558380127,
+      "logits/rejected": -2.098257541656494,
+      "logps/chosen": -527.9982299804688,
+      "logps/rejected": -425.9187927246094,
+      "loss": 0.1028,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.0199496746063232,
+      "rewards/margins": 2.1254653930664062,
+      "rewards/rejected": -3.1454153060913086,
       "step": 120
     },
     {
       "epoch": 0.32,
       "learning_rate": 4.298016388768561e-07,
-      "logits/chosen": -2.3046891689300537,
-      "logits/rejected": -2.1809306144714355,
-      "logps/chosen": -464.0572814941406,
-      "logps/rejected": -396.9075622558594,
-      "loss": 0.1128,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -0.93115234375,
-      "rewards/margins": 1.978463888168335,
-      "rewards/rejected": -2.909615993499756,
+      "logits/chosen": -2.2639729976654053,
+      "logits/rejected": -2.1410276889801025,
+      "logps/chosen": -474.557373046875,
+      "logps/rejected": -406.2401123046875,
+      "loss": 0.102,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -0.7631456851959229,
+      "rewards/margins": 2.1880364418029785,
+      "rewards/rejected": -2.9511821269989014,
       "step": 130
     },
     {
       "epoch": 0.35,
       "learning_rate": 4.1415945805573005e-07,
-      "logits/chosen": -2.364520311355591,
-      "logits/rejected": -2.2930567264556885,
-      "logps/chosen": -455.96771240234375,
-      "logps/rejected": -342.2510986328125,
-      "loss": 0.1463,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.5802963972091675,
-      "rewards/margins": 1.6709789037704468,
-      "rewards/rejected": -2.2512753009796143,
+      "logits/chosen": -2.2353649139404297,
+      "logits/rejected": -2.103256940841675,
+      "logps/chosen": -494.5367126464844,
+      "logps/rejected": -413.3063049316406,
+      "loss": 0.109,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.6633724570274353,
+      "rewards/margins": 2.3635334968566895,
+      "rewards/rejected": -3.0269057750701904,
       "step": 140
     },
     {
       "epoch": 0.37,
       "learning_rate": 3.972952151123984e-07,
-      "logits/chosen": -2.3566527366638184,
-      "logits/rejected": -2.2735893726348877,
-      "logps/chosen": -478.54693603515625,
-      "logps/rejected": -365.7146911621094,
-      "loss": 0.1316,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.9140907526016235,
-      "rewards/margins": 1.685943365097046,
-      "rewards/rejected": -2.600034236907959,
+      "logits/chosen": -2.243504762649536,
+      "logits/rejected": -2.1586227416992188,
+      "logps/chosen": -485.9297790527344,
+      "logps/rejected": -403.07904052734375,
+      "loss": 0.0988,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.8084269762039185,
+      "rewards/margins": 1.9387495517730713,
+      "rewards/rejected": -2.7471766471862793,
       "step": 150
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.793344535444142e-07,
-      "logits/chosen": -2.2995688915252686,
-      "logits/rejected": -2.1954402923583984,
-      "logps/chosen": -570.6788330078125,
-      "logps/rejected": -437.27130126953125,
-      "loss": 0.0968,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -1.210545301437378,
-      "rewards/margins": 1.947977066040039,
-      "rewards/rejected": -3.158522367477417,
+      "logits/chosen": -2.2556967735290527,
+      "logits/rejected": -2.1410129070281982,
+      "logps/chosen": -517.8903198242188,
+      "logps/rejected": -427.27178955078125,
+      "loss": 0.1095,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.9526297450065613,
+      "rewards/margins": 2.0250723361968994,
+      "rewards/rejected": -2.9777019023895264,
       "step": 160
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.604108797288461e-07,
-      "logits/chosen": -2.310495615005493,
-      "logits/rejected": -2.227719306945801,
-      "logps/chosen": -477.73968505859375,
-      "logps/rejected": -394.82891845703125,
-      "loss": 0.1053,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -1.0732296705245972,
-      "rewards/margins": 1.7702564001083374,
-      "rewards/rejected": -2.8434860706329346,
+      "logits/chosen": -2.2546138763427734,
+      "logits/rejected": -2.162337064743042,
+      "logps/chosen": -484.6184997558594,
+      "logps/rejected": -358.60198974609375,
+      "loss": 0.1069,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -0.6528670787811279,
+      "rewards/margins": 1.8891479969024658,
+      "rewards/rejected": -2.5420150756835938,
       "step": 170
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.40665367563858e-07,
-      "logits/chosen": -2.3077213764190674,
-      "logits/rejected": -2.2177042961120605,
-      "logps/chosen": -518.1863403320312,
-      "logps/rejected": -416.206787109375,
-      "loss": 0.0817,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -0.9166792631149292,
-      "rewards/margins": 2.109100341796875,
-      "rewards/rejected": -3.0257794857025146,
+      "logits/chosen": -2.199352264404297,
+      "logits/rejected": -2.0996298789978027,
+      "logps/chosen": -520.5743408203125,
+      "logps/rejected": -414.6077575683594,
+      "loss": 0.0923,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.9770752787590027,
+      "rewards/margins": 1.9050334692001343,
+      "rewards/rejected": -2.882108688354492,
       "step": 180
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.202449097526798e-07,
-      "logits/chosen": -2.2818238735198975,
-      "logits/rejected": -2.187276840209961,
-      "logps/chosen": -522.6356201171875,
-      "logps/rejected": -490.2731018066406,
-      "loss": 0.067,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -1.4796288013458252,
-      "rewards/margins": 2.3061554431915283,
-      "rewards/rejected": -3.7857837677001953,
+      "logits/chosen": -2.2049007415771484,
+      "logits/rejected": -2.0761523246765137,
+      "logps/chosen": -545.6790771484375,
+      "logps/rejected": -503.28326416015625,
+      "loss": 0.0757,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -1.1332416534423828,
+      "rewards/margins": 2.640087604522705,
+      "rewards/rejected": -3.773329257965088,
       "step": 190
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.993015235369905e-07,
-      "logits/chosen": -2.2447495460510254,
-      "logits/rejected": -2.1707584857940674,
-      "logps/chosen": -549.8089599609375,
-      "logps/rejected": -516.205078125,
-      "loss": 0.0611,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.6697288751602173,
-      "rewards/margins": 2.298145055770874,
-      "rewards/rejected": -3.9678738117218018,
+      "logits/chosen": -2.251674175262451,
+      "logits/rejected": -2.138532876968384,
+      "logps/chosen": -552.5167236328125,
+      "logps/rejected": -459.761962890625,
+      "loss": 0.0843,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -1.0592725276947021,
+      "rewards/margins": 2.3839969635009766,
+      "rewards/rejected": -3.443269729614258,
       "step": 200
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7799111902582693e-07,
-      "logits/chosen": -2.321493625640869,
-      "logits/rejected": -2.215440511703491,
-      "logps/chosen": -575.6776123046875,
-      "logps/rejected": -493.73272705078125,
-      "loss": 0.0744,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.159621000289917,
-      "rewards/margins": 2.540497303009033,
-      "rewards/rejected": -3.70011830329895,
+      "logits/chosen": -2.216552495956421,
+      "logits/rejected": -2.116063356399536,
+      "logps/chosen": -539.1038818359375,
+      "logps/rejected": -468.47869873046875,
+      "loss": 0.0889,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.1180602312088013,
+      "rewards/margins": 2.2188849449157715,
+      "rewards/rejected": -3.336945056915283,
       "step": 210
     },
     {
       "epoch": 0.54,
       "learning_rate": 2.564723385445869e-07,
-      "logits/chosen": -2.2707631587982178,
-      "logits/rejected": -2.2030460834503174,
-      "logps/chosen": -501.503173828125,
-      "logps/rejected": -449.3731384277344,
-      "loss": 0.08,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.1590577363967896,
-      "rewards/margins": 2.1603972911834717,
-      "rewards/rejected": -3.3194549083709717,
+      "logits/chosen": -2.182385206222534,
+      "logits/rejected": -2.0952038764953613,
+      "logps/chosen": -506.44970703125,
+      "logps/rejected": -433.21954345703125,
+      "loss": 0.0928,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -1.1999738216400146,
+      "rewards/margins": 1.8092005252838135,
+      "rewards/rejected": -3.00917387008667,
       "step": 220
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3490537564442845e-07,
-      "logits/chosen": -2.287022352218628,
-      "logits/rejected": -2.1958324909210205,
-      "logps/chosen": -529.8822631835938,
-      "logps/rejected": -451.2649841308594,
-      "loss": 0.089,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.0163233280181885,
-      "rewards/margins": 2.2578654289245605,
-      "rewards/rejected": -3.274188995361328,
+      "logits/chosen": -2.2019972801208496,
+      "logits/rejected": -2.1013846397399902,
+      "logps/chosen": -493.3079528808594,
+      "logps/rejected": -435.74884033203125,
+      "loss": 0.0853,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -1.0892283916473389,
+      "rewards/margins": 2.1229333877563477,
+      "rewards/rejected": -3.2121620178222656,
       "step": 230
     },
     {
       "epoch": 0.59,
       "learning_rate": 2.1345078256378801e-07,
-      "logits/chosen": -2.2853665351867676,
-      "logits/rejected": -2.1983203887939453,
-      "logps/chosen": -590.9471435546875,
-      "logps/rejected": -492.50372314453125,
-      "loss": 0.0612,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -1.4521348476409912,
-      "rewards/margins": 2.383617401123047,
-      "rewards/rejected": -3.835752487182617,
+      "logits/chosen": -2.2013192176818848,
+      "logits/rejected": -2.0689620971679688,
+      "logps/chosen": -503.4147033691406,
+      "logps/rejected": -448.7828674316406,
+      "loss": 0.08,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.28254234790802,
+      "rewards/margins": 2.0458292961120605,
+      "rewards/rejected": -3.32837176322937,
       "step": 240
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.9226827501969865e-07,
-      "logits/chosen": -2.2573533058166504,
-      "logits/rejected": -2.1343369483947754,
-      "logps/chosen": -548.763427734375,
-      "logps/rejected": -487.57623291015625,
-      "loss": 0.0627,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -1.417848825454712,
-      "rewards/margins": 2.380465030670166,
-      "rewards/rejected": -3.798313856124878,
+      "logits/chosen": -2.2620677947998047,
+      "logits/rejected": -2.1344215869903564,
+      "logps/chosen": -535.8228149414062,
+      "logps/rejected": -445.83349609375,
+      "loss": 0.0772,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.9264475107192993,
+      "rewards/margins": 2.3875911235809326,
+      "rewards/rejected": -3.3140385150909424,
       "step": 250
     },
     {
       "epoch": 0.64,
       "learning_rate": 1.715155432264775e-07,
-      "logits/chosen": -2.249309539794922,
-      "logits/rejected": -2.155247688293457,
-      "logps/chosen": -520.6065063476562,
-      "logps/rejected": -490.1532287597656,
-      "loss": 0.0677,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -1.445723295211792,
-      "rewards/margins": 2.3194212913513184,
-      "rewards/rejected": -3.7651443481445312,
+      "logits/chosen": -2.2268154621124268,
+      "logits/rejected": -2.149883270263672,
+      "logps/chosen": -518.2362060546875,
+      "logps/rejected": -444.4808654785156,
+      "loss": 0.0781,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.0662428140640259,
+      "rewards/margins": 2.1827731132507324,
+      "rewards/rejected": -3.249016284942627,
       "step": 260
     },
     {
       "epoch": 0.67,
       "learning_rate": 1.51347077992983e-07,
-      "logits/chosen": -2.260199546813965,
-      "logits/rejected": -2.167729616165161,
-      "logps/chosen": -552.5299072265625,
-      "logps/rejected": -480.06475830078125,
-      "loss": 0.07,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -1.2369455099105835,
-      "rewards/margins": 2.399855375289917,
-      "rewards/rejected": -3.6368002891540527,
+      "logits/chosen": -2.2635481357574463,
+      "logits/rejected": -2.1383774280548096,
+      "logps/chosen": -510.13751220703125,
+      "logps/rejected": -457.0772399902344,
+      "loss": 0.0781,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -1.0047271251678467,
+      "rewards/margins": 2.324492931365967,
+      "rewards/rejected": -3.3292198181152344,
       "step": 270
     },
     {
       "epoch": 0.69,
       "learning_rate": 1.3191302063739906e-07,
-      "logits/chosen": -2.2963688373565674,
-      "logits/rejected": -2.2376794815063477,
-      "logps/chosen": -517.3768310546875,
-      "logps/rejected": -425.8411560058594,
-      "loss": 0.07,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -1.1372196674346924,
-      "rewards/margins": 2.0433435440063477,
-      "rewards/rejected": -3.180562973022461,
+      "logits/chosen": -2.2417304515838623,
+      "logits/rejected": -2.121445417404175,
+      "logps/chosen": -491.12139892578125,
+      "logps/rejected": -425.1968688964844,
+      "loss": 0.0805,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.8460060954093933,
+      "rewards/margins": 2.316087007522583,
+      "rewards/rejected": -3.162093162536621,
       "step": 280
     },
     {
       "epoch": 0.72,
       "learning_rate": 1.1335804528119475e-07,
-      "logits/chosen": -2.256326913833618,
-      "logits/rejected": -2.1535654067993164,
-      "logps/chosen": -577.1251220703125,
-      "logps/rejected": -531.5836791992188,
-      "loss": 0.0683,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -1.3688971996307373,
-      "rewards/margins": 2.631861448287964,
-      "rewards/rejected": -4.000759124755859,
+      "logits/chosen": -2.2970786094665527,
+      "logits/rejected": -2.17289662361145,
+      "logps/chosen": -507.46661376953125,
+      "logps/rejected": -449.6181640625,
+      "loss": 0.0812,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.0554336309432983,
+      "rewards/margins": 2.2787580490112305,
+      "rewards/rejected": -3.3341917991638184,
       "step": 290
     },
     {
       "epoch": 0.74,
       "learning_rate": 9.582028184286423e-08,
-      "logits/chosen": -2.202509880065918,
-      "logits/rejected": -2.116055727005005,
-      "logps/chosen": -553.2191772460938,
-      "logps/rejected": -521.0760498046875,
-      "loss": 0.0577,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.7238006591796875,
-      "rewards/margins": 2.396298885345459,
-      "rewards/rejected": -4.1200995445251465,
+      "logits/chosen": -2.2653486728668213,
+      "logits/rejected": -2.1368870735168457,
+      "logps/chosen": -563.6956787109375,
+      "logps/rejected": -506.1378479003906,
+      "loss": 0.0811,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.0197770595550537,
+      "rewards/margins": 2.7816028594970703,
+      "rewards/rejected": -3.801379680633545,
       "step": 300
     },
     {
       "epoch": 0.76,
       "learning_rate": 7.943028774907065e-08,
-      "logits/chosen": -2.257730484008789,
-      "logits/rejected": -2.1503612995147705,
-      "logps/chosen": -599.2135009765625,
-      "logps/rejected": -497.50384521484375,
-      "loss": 0.0563,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -1.4197520017623901,
-      "rewards/margins": 2.4658217430114746,
-      "rewards/rejected": -3.885573625564575,
+      "logits/chosen": -2.212477445602417,
+      "logits/rejected": -2.083967685699463,
+      "logps/chosen": -543.1881103515625,
+      "logps/rejected": -489.3092346191406,
+      "loss": 0.072,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.234198808670044,
+      "rewards/margins": 2.4262847900390625,
+      "rewards/rejected": -3.6604835987091064,
       "step": 310
     },
     {
       "epoch": 0.79,
       "learning_rate": 6.431007601814637e-08,
-      "logits/chosen": -2.226001262664795,
-      "logits/rejected": -2.1242711544036865,
-      "logps/chosen": -561.7794799804688,
-      "logps/rejected": -525.1495361328125,
-      "loss": 0.0589,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.6232330799102783,
-      "rewards/margins": 2.387523889541626,
-      "rewards/rejected": -4.010756969451904,
+      "logits/chosen": -2.237189531326294,
+      "logits/rejected": -2.118699550628662,
+      "logps/chosen": -533.3051147460938,
+      "logps/rejected": -482.6719665527344,
+      "loss": 0.0764,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.317209243774414,
+      "rewards/margins": 2.3056979179382324,
+      "rewards/rejected": -3.6229069232940674,
       "step": 320
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.0572206951246e-08,
-      "logits/chosen": -2.2513046264648438,
-      "logits/rejected": -2.1654863357543945,
-      "logps/chosen": -513.5191650390625,
-      "logps/rejected": -491.4129333496094,
-      "loss": 0.0596,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.6228710412979126,
-      "rewards/margins": 2.1468024253845215,
-      "rewards/rejected": -3.7696735858917236,
+      "logits/chosen": -2.262988567352295,
+      "logits/rejected": -2.143887996673584,
+      "logps/chosen": -538.3980712890625,
+      "logps/rejected": -492.5909118652344,
+      "loss": 0.0884,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.2063168287277222,
+      "rewards/margins": 2.362534523010254,
+      "rewards/rejected": -3.5688509941101074,
       "step": 330
     },
     {
       "epoch": 0.84,
       "learning_rate": 3.831895019292897e-08,
-      "logits/chosen": -2.284156084060669,
-      "logits/rejected": -2.184755325317383,
-      "logps/chosen": -595.8388061523438,
-      "logps/rejected": -532.6061401367188,
-      "loss": 0.0584,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -1.4855483770370483,
-      "rewards/margins": 2.6151788234710693,
-      "rewards/rejected": -4.100727558135986,
+      "logits/chosen": -2.211160182952881,
+      "logits/rejected": -2.0655343532562256,
+      "logps/chosen": -495.141357421875,
+      "logps/rejected": -464.81646728515625,
+      "loss": 0.0744,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -1.0855491161346436,
+      "rewards/margins": 2.5346646308898926,
+      "rewards/rejected": -3.6202139854431152,
       "step": 340
     },
     {
       "epoch": 0.86,
       "learning_rate": 2.764152339909756e-08,
-      "logits/chosen": -2.272153377532959,
-      "logits/rejected": -2.180081844329834,
-      "logps/chosen": -596.3790283203125,
-      "logps/rejected": -511.85760498046875,
-      "loss": 0.0692,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.242677092552185,
-      "rewards/margins": 2.682408094406128,
-      "rewards/rejected": -3.9250850677490234,
+      "logits/chosen": -2.273918867111206,
+      "logits/rejected": -2.128694534301758,
+      "logps/chosen": -528.7555541992188,
+      "logps/rejected": -469.3017578125,
+      "loss": 0.0642,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.1073048114776611,
+      "rewards/margins": 2.4064698219299316,
+      "rewards/rejected": -3.513774871826172,
       "step": 350
     },
     {
       "epoch": 0.89,
       "learning_rate": 1.861941317991664e-08,
-      "logits/chosen": -2.1758205890655518,
-      "logits/rejected": -2.1184608936309814,
-      "logps/chosen": -529.5315551757812,
-      "logps/rejected": -503.5777893066406,
-      "loss": 0.0542,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.6654908657073975,
-      "rewards/margins": 2.209399700164795,
-      "rewards/rejected": -3.8748905658721924,
+      "logits/chosen": -2.2403626441955566,
+      "logits/rejected": -2.1174542903900146,
+      "logps/chosen": -488.1507873535156,
+      "logps/rejected": -457.27423095703125,
+      "loss": 0.0756,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -1.1823718547821045,
+      "rewards/margins": 2.2865800857543945,
+      "rewards/rejected": -3.46895170211792,
       "step": 360
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.13197833728636e-08,
-      "logits/chosen": -2.2538328170776367,
-      "logits/rejected": -2.175743818283081,
-      "logps/chosen": -569.5551147460938,
-      "logps/rejected": -514.572509765625,
-      "loss": 0.0603,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -1.7610028982162476,
-      "rewards/margins": 2.1565301418304443,
-      "rewards/rejected": -3.9175331592559814,
+      "logits/chosen": -2.2277872562408447,
+      "logits/rejected": -2.0999319553375244,
+      "logps/chosen": -529.3685302734375,
+      "logps/rejected": -459.83551025390625,
+      "loss": 0.0729,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -1.064345121383667,
+      "rewards/margins": 2.42720365524292,
+      "rewards/rejected": -3.491549015045166,
       "step": 370
     },
     {
       "epoch": 0.94,
       "learning_rate": 5.79697505093521e-09,
-      "logits/chosen": -2.308776378631592,
-      "logits/rejected": -2.193223237991333,
-      "logps/chosen": -585.7466430664062,
-      "logps/rejected": -526.7449340820312,
-      "loss": 0.0563,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -1.3321974277496338,
-      "rewards/margins": 2.728519916534424,
-      "rewards/rejected": -4.0607171058654785,
+      "logits/chosen": -2.161315679550171,
+      "logits/rejected": -2.051104784011841,
+      "logps/chosen": -518.0635986328125,
+      "logps/rejected": -463.9219665527344,
+      "loss": 0.0742,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -1.2277439832687378,
+      "rewards/margins": 2.3196287155151367,
+      "rewards/rejected": -3.547372817993164,
       "step": 380
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.092101988131256e-09,
-      "logits/chosen": -2.2692370414733887,
-      "logits/rejected": -2.1535143852233887,
-      "logps/chosen": -583.334716796875,
-      "logps/rejected": -501.6742248535156,
-      "loss": 0.0581,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -1.541758418083191,
-      "rewards/margins": 2.3584675788879395,
-      "rewards/rejected": -3.90022611618042,
+      "logits/chosen": -2.2838692665100098,
+      "logits/rejected": -2.1495554447174072,
+      "logps/chosen": -561.1207275390625,
+      "logps/rejected": -473.84234619140625,
+      "loss": 0.07,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -1.088555932044983,
+      "rewards/margins": 2.4874186515808105,
+      "rewards/rejected": -3.575974702835083,
       "step": 390
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.327445937151673e-10,
-      "logits/chosen": -2.2919979095458984,
-      "logits/rejected": -2.1775660514831543,
-      "logps/chosen": -602.81201171875,
-      "logps/rejected": -531.6156616210938,
-      "loss": 0.0554,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -1.604543924331665,
-      "rewards/margins": 2.4634101390838623,
-      "rewards/rejected": -4.067954063415527,
+      "logits/chosen": -2.214580774307251,
+      "logits/rejected": -2.0943350791931152,
+      "logps/chosen": -564.2920532226562,
+      "logps/rejected": -481.778564453125,
+      "loss": 0.0661,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.2679953575134277,
+      "rewards/margins": 2.343169689178467,
+      "rewards/rejected": -3.6111652851104736,
       "step": 400
     },
     {
       "epoch": 1.0,
       "step": 405,
       "total_flos": 0.0,
-      "train_loss": 0.1290441479947832,
-      "train_runtime": 3205.4322,
-      "train_samples_per_second": 16.189,
+      "train_loss": 0.13281457475674005,
+      "train_runtime": 3219.0486,
+      "train_samples_per_second": 16.121,
       "train_steps_per_second": 0.126
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:34facb59b2833ff9a65d1ea6ca0671f7143189081be77d079ad67a7343d5aa7d
+oid sha256:fb0e23129935740782e931a75e4f8dcc9138d97706187a2a22ccc8a4a4f651ea
 size 5944