Model save

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.37614710375947774,
-    "train_runtime": 6361.5592,
+    "train_loss": 0.42718374404267445,
+    "train_runtime": 6325.1171,
     "train_samples": 61134,
-    "train_samples_per_second": 9.61,
+    "train_samples_per_second": 9.665,
     "train_steps_per_second": 0.075
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8558783426207d481e1b212901b2a745cf326f53087e609ed23dbb2013f59ce2
+oid sha256:c12c9c343db96ae94ef515af0f5da81ed9a6301101432050dd413b94822f22e1
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0de16d834b47e78126680587374bfb8e3f6fff6da5595e08d8bc8c4dfe3f057e
+oid sha256:bfe6406bd258e8fdc18e1f4c122bbb1cfc0b2cd66831b06fda75bf88f2955d87
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bd358506aa4a2ba7d2848486443d68ed78736482c3641750882e51a5423e6c7e
+oid sha256:0a9ea4dc30b2c98ff046a72449873d2850cf575c5ab3894159bbe51207b9cdf1
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1ee534a0b473037a903d7605adf23f1bc01159bba79b0d8f59245eebbe979737
+oid sha256:6e249a9ba96464ca0f1329eea1893d5fff0653b7a9dffc6f54ac8c684320502c
 size 1168138808

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.37614710375947774,
-    "train_runtime": 6361.5592,
+    "train_loss": 0.42718374404267445,
+    "train_runtime": 6325.1171,
     "train_samples": 61134,
-    "train_samples_per_second": 9.61,
+    "train_samples_per_second": 9.665,
     "train_steps_per_second": 0.075
 }

trainer_state.json

@@ -11,668 +11,668 @@
     {
       "epoch": 0.02,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": 0.1770419478416443,
-      "logits/rejected": 0.2540443539619446,
-      "logps/chosen": -354.38037109375,
-      "logps/rejected": -305.27264404296875,
-      "loss": 0.4999,
-      "rewards/accuracies": 0.40625,
-      "rewards/chosen": 0.0009949840605258942,
-      "rewards/margins": 0.003730112686753273,
-      "rewards/rejected": -0.0027351281605660915,
+      "logits/chosen": 0.17655496299266815,
+      "logits/rejected": 0.2531452775001526,
+      "logps/chosen": -354.29669189453125,
+      "logps/rejected": -305.259765625,
+      "loss": 0.5,
+      "rewards/accuracies": 0.4312500059604645,
+      "rewards/chosen": 0.0010361697059124708,
+      "rewards/margins": 0.0014542521676048636,
+      "rewards/rejected": -0.00041808263631537557,
       "step": 10
     },
     {
       "epoch": 0.04,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": 0.07181452214717865,
-      "logits/rejected": 0.19976207613945007,
-      "logps/chosen": -316.61358642578125,
-      "logps/rejected": -276.0943603515625,
-      "loss": 0.4986,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.006085564382374287,
-      "rewards/margins": 0.009029300883412361,
-      "rewards/rejected": -0.002943736733868718,
+      "logits/chosen": 0.07140998542308807,
+      "logits/rejected": 0.19915328919887543,
+      "logps/chosen": -316.61407470703125,
+      "logps/rejected": -276.1783142089844,
+      "loss": 0.4997,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.001211934955790639,
+      "rewards/margins": 0.00264042429625988,
+      "rewards/rejected": -0.0014284893404692411,
       "step": 20
     },
     {
       "epoch": 0.06,
       "learning_rate": 6.249999999999999e-07,
-      "logits/chosen": 0.18359068036079407,
-      "logits/rejected": 0.2548081874847412,
-      "logps/chosen": -294.26654052734375,
-      "logps/rejected": -298.5642395019531,
-      "loss": 0.4893,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 0.03501707315444946,
-      "rewards/margins": 0.048330314457416534,
-      "rewards/rejected": -0.013313241302967072,
+      "logits/chosen": 0.1830858290195465,
+      "logits/rejected": 0.25493288040161133,
+      "logps/chosen": -294.3023376464844,
+      "logps/rejected": -298.47430419921875,
+      "loss": 0.4979,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.00664560217410326,
+      "rewards/margins": 0.008408578112721443,
+      "rewards/rejected": -0.0017629768699407578,
       "step": 30
     },
     {
       "epoch": 0.08,
       "learning_rate": 8.333333333333333e-07,
-      "logits/chosen": 0.11794896423816681,
-      "logits/rejected": 0.23672719299793243,
-      "logps/chosen": -343.41192626953125,
-      "logps/rejected": -318.60638427734375,
-      "loss": 0.4731,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.20801420509815216,
-      "rewards/margins": 0.1149774044752121,
-      "rewards/rejected": 0.09303676337003708,
+      "logits/chosen": 0.1198926791548729,
+      "logits/rejected": 0.2388772964477539,
+      "logps/chosen": -343.3688659667969,
+      "logps/rejected": -318.56866455078125,
+      "loss": 0.4944,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.04203338176012039,
+      "rewards/margins": 0.023049216717481613,
+      "rewards/rejected": 0.01898416317999363,
       "step": 40
     },
     {
       "epoch": 0.1,
       "learning_rate": 9.999463737538052e-07,
-      "logits/chosen": 0.18171748518943787,
-      "logits/rejected": 0.2686474025249481,
-      "logps/chosen": -305.8929443359375,
-      "logps/rejected": -285.5357360839844,
-      "loss": 0.4517,
+      "logits/chosen": 0.19016575813293457,
+      "logits/rejected": 0.2768324613571167,
+      "logps/chosen": -305.9139709472656,
+      "logps/rejected": -285.70263671875,
+      "loss": 0.4888,
       "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.2329844981431961,
-      "rewards/margins": 0.32565948367118835,
-      "rewards/rejected": -0.09267498552799225,
+      "rewards/chosen": 0.0463864728808403,
+      "rewards/margins": 0.06659023463726044,
+      "rewards/rejected": -0.02020375430583954,
       "step": 50
     },
     {
       "epoch": 0.13,
       "learning_rate": 9.980706626858607e-07,
-      "logits/chosen": 0.14295880496501923,
-      "logits/rejected": 0.28016844391822815,
-      "logps/chosen": -290.75335693359375,
-      "logps/rejected": -281.30303955078125,
-      "loss": 0.4337,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.10626886039972305,
-      "rewards/margins": 0.39915287494659424,
-      "rewards/rejected": -0.29288405179977417,
+      "logits/chosen": 0.1583642065525055,
+      "logits/rejected": 0.2964373230934143,
+      "logps/chosen": -292.2091979980469,
+      "logps/rejected": -283.33062744140625,
+      "loss": 0.4823,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.006695735268294811,
+      "rewards/margins": 0.08554854989051819,
+      "rewards/rejected": -0.0788528248667717,
       "step": 60
     },
     {
       "epoch": 0.15,
       "learning_rate": 9.935251313189563e-07,
-      "logits/chosen": 0.1319437325000763,
-      "logits/rejected": 0.21764138340950012,
-      "logps/chosen": -324.74078369140625,
-      "logps/rejected": -324.4031066894531,
-      "loss": 0.4212,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": 0.24914593994617462,
-      "rewards/margins": 0.5219111442565918,
-      "rewards/rejected": -0.27276521921157837,
+      "logits/chosen": 0.1668189913034439,
+      "logits/rejected": 0.25383955240249634,
+      "logps/chosen": -330.51483154296875,
+      "logps/rejected": -332.74249267578125,
+      "loss": 0.476,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.007911129854619503,
+      "rewards/margins": 0.13003569841384888,
+      "rewards/rejected": -0.13794682919979095,
       "step": 70
     },
     {
       "epoch": 0.17,
       "learning_rate": 9.86334145175542e-07,
-      "logits/chosen": 0.17676237225532532,
-      "logits/rejected": 0.27126845717430115,
-      "logps/chosen": -311.08868408203125,
-      "logps/rejected": -297.3030090332031,
-      "loss": 0.4116,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.3787495195865631,
-      "rewards/margins": 0.8905243873596191,
-      "rewards/rejected": -0.5117748379707336,
+      "logits/chosen": 0.22892770171165466,
+      "logits/rejected": 0.32262876629829407,
+      "logps/chosen": -326.62847900390625,
+      "logps/rejected": -321.47064208984375,
+      "loss": 0.4678,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.07964827120304108,
+      "rewards/margins": 0.2643834054470062,
+      "rewards/rejected": -0.3440317213535309,
       "step": 80
     },
     {
       "epoch": 0.19,
       "learning_rate": 9.765362502737097e-07,
-      "logits/chosen": 0.03276940807700157,
-      "logits/rejected": 0.1763920933008194,
-      "logps/chosen": -327.9542541503906,
-      "logps/rejected": -275.962890625,
-      "loss": 0.401,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.4493633210659027,
-      "rewards/margins": 0.8937808871269226,
-      "rewards/rejected": -0.4444176256656647,
+      "logits/chosen": 0.12489993870258331,
+      "logits/rejected": 0.2657889425754547,
+      "logps/chosen": -358.5821838378906,
+      "logps/rejected": -333.71466064453125,
+      "loss": 0.4612,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.21640650928020477,
+      "rewards/margins": 0.4499947130680084,
+      "rewards/rejected": -0.6664012670516968,
       "step": 90
     },
     {
       "epoch": 0.21,
       "learning_rate": 9.641839665080363e-07,
-      "logits/chosen": 0.10452715307474136,
-      "logits/rejected": 0.27761924266815186,
-      "logps/chosen": -323.75360107421875,
-      "logps/rejected": -299.73370361328125,
-      "loss": 0.3917,
+      "logits/chosen": 0.2374851256608963,
+      "logits/rejected": 0.4098134934902191,
+      "logps/chosen": -378.7792053222656,
+      "logps/rejected": -408.1399841308594,
+      "loss": 0.4512,
       "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.5403918027877808,
-      "rewards/margins": 0.907971978187561,
-      "rewards/rejected": -0.3675800561904907,
+      "rewards/chosen": -0.44217753410339355,
+      "rewards/margins": 0.715401291847229,
+      "rewards/rejected": -1.157578706741333,
       "step": 100
     },
     {
       "epoch": 0.23,
       "learning_rate": 9.493435061259129e-07,
-      "logits/chosen": 0.1223590150475502,
-      "logits/rejected": 0.14537492394447327,
-      "logps/chosen": -303.0465087890625,
-      "logps/rejected": -307.5492248535156,
-      "loss": 0.3772,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.4786440432071686,
-      "rewards/margins": 0.8835989236831665,
-      "rewards/rejected": -0.4049549102783203,
+      "logits/chosen": 0.29897215962409973,
+      "logits/rejected": 0.34014248847961426,
+      "logps/chosen": -395.0293884277344,
+      "logps/rejected": -461.2764587402344,
+      "loss": 0.4418,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.8240998983383179,
+      "rewards/margins": 0.7941638231277466,
+      "rewards/rejected": -1.618263602256775,
       "step": 110
     },
     {
       "epoch": 0.25,
       "learning_rate": 9.320944188084241e-07,
-      "logits/chosen": 0.07157851755619049,
-      "logits/rejected": 0.12891840934753418,
-      "logps/chosen": -309.7882385253906,
-      "logps/rejected": -329.6763610839844,
-      "loss": 0.3939,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.538270115852356,
-      "rewards/margins": 0.9622691869735718,
-      "rewards/rejected": -0.4239990711212158,
+      "logits/chosen": 0.18543429672718048,
+      "logits/rejected": 0.282682329416275,
+      "logps/chosen": -440.6853942871094,
+      "logps/rejected": -526.3844604492188,
+      "loss": 0.4495,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.201317548751831,
+      "rewards/margins": 0.8505627512931824,
+      "rewards/rejected": -2.051880359649658,
       "step": 120
     },
     {
       "epoch": 0.27,
       "learning_rate": 9.125291652582547e-07,
-      "logits/chosen": 0.03255900740623474,
-      "logits/rejected": 0.1836429387331009,
-      "logps/chosen": -328.1858825683594,
-      "logps/rejected": -294.7179260253906,
-      "loss": 0.3745,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.5155087113380432,
-      "rewards/margins": 1.008885145187378,
-      "rewards/rejected": -0.4933764338493347,
+      "logits/chosen": 0.10988249629735947,
+      "logits/rejected": 0.2532512843608856,
+      "logps/chosen": -429.30322265625,
+      "logps/rejected": -460.0655822753906,
+      "loss": 0.4407,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.9080715179443359,
+      "rewards/margins": 0.8440803289413452,
+      "rewards/rejected": -1.7521518468856812,
       "step": 130
     },
     {
       "epoch": 0.29,
       "learning_rate": 8.90752621580335e-07,
-      "logits/chosen": 0.008989883586764336,
-      "logits/rejected": 0.17159470915794373,
-      "logps/chosen": -326.74957275390625,
-      "logps/rejected": -284.432373046875,
-      "loss": 0.3671,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.2646927833557129,
-      "rewards/margins": 0.9909642338752747,
-      "rewards/rejected": -0.7262714505195618,
+      "logits/chosen": 0.05259154364466667,
+      "logits/rejected": 0.20351815223693848,
+      "logps/chosen": -478.1226501464844,
+      "logps/rejected": -552.33154296875,
+      "loss": 0.4381,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4607925415039062,
+      "rewards/margins": 1.3634538650512695,
+      "rewards/rejected": -2.8242461681365967,
       "step": 140
     },
     {
       "epoch": 0.31,
       "learning_rate": 8.668815171119019e-07,
-      "logits/chosen": 0.10270164906978607,
-      "logits/rejected": 0.15117475390434265,
-      "logps/chosen": -319.47796630859375,
-      "logps/rejected": -322.017822265625,
-      "loss": 0.3768,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.7911346554756165,
-      "rewards/margins": 1.1445444822311401,
-      "rewards/rejected": -0.35340994596481323,
+      "logits/chosen": 0.1267194300889969,
+      "logits/rejected": 0.16065822541713715,
+      "logps/chosen": -432.47418212890625,
+      "logps/rejected": -556.4413452148438,
+      "loss": 0.4373,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.9717355966567993,
+      "rewards/margins": 1.443182110786438,
+      "rewards/rejected": -2.4149177074432373,
       "step": 150
     },
     {
       "epoch": 0.33,
       "learning_rate": 8.410438087153911e-07,
-      "logits/chosen": 0.07868606597185135,
-      "logits/rejected": 0.12421569973230362,
-      "logps/chosen": -283.022216796875,
-      "logps/rejected": -284.0340881347656,
-      "loss": 0.3737,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 0.9519669413566589,
-      "rewards/margins": 1.1187279224395752,
-      "rewards/rejected": -0.16676095128059387,
+      "logits/chosen": 0.05742305517196655,
+      "logits/rejected": 0.03335579112172127,
+      "logps/chosen": -386.4638366699219,
+      "logps/rejected": -537.6171264648438,
+      "loss": 0.4335,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.8440232276916504,
+      "rewards/margins": 1.7251598834991455,
+      "rewards/rejected": -2.569182872772217,
       "step": 160
     },
     {
       "epoch": 0.36,
       "learning_rate": 8.133779948881513e-07,
-      "logits/chosen": 0.053541384637355804,
-      "logits/rejected": 0.13824662566184998,
-      "logps/chosen": -318.894775390625,
-      "logps/rejected": -313.05755615234375,
-      "loss": 0.3657,
+      "logits/chosen": 0.04388447850942612,
+      "logits/rejected": 0.06478340178728104,
+      "logps/chosen": -450.94049072265625,
+      "logps/rejected": -571.2717895507812,
+      "loss": 0.4268,
       "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 0.8736802339553833,
-      "rewards/margins": 1.134092926979065,
-      "rewards/rejected": -0.26041263341903687,
+      "rewards/chosen": -1.1457209587097168,
+      "rewards/margins": 1.4885038137435913,
+      "rewards/rejected": -2.6342251300811768,
       "step": 170
     },
     {
       "epoch": 0.38,
       "learning_rate": 7.840323733655778e-07,
-      "logits/chosen": 0.01729046180844307,
-      "logits/rejected": 0.1666645109653473,
-      "logps/chosen": -309.3392639160156,
-      "logps/rejected": -307.69378662109375,
-      "loss": 0.3669,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.8346372842788696,
-      "rewards/margins": 1.2255662679672241,
-      "rewards/rejected": -0.39092904329299927,
+      "logits/chosen": 0.03801240772008896,
+      "logits/rejected": 0.0668804943561554,
+      "logps/chosen": -415.9105529785156,
+      "logps/rejected": -594.4246826171875,
+      "loss": 0.426,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.8987852931022644,
+      "rewards/margins": 2.0467095375061035,
+      "rewards/rejected": -2.9454948902130127,
       "step": 180
     },
     {
       "epoch": 0.4,
       "learning_rate": 7.531642461971514e-07,
-      "logits/chosen": 0.07640022039413452,
-      "logits/rejected": 0.1669580638408661,
-      "logps/chosen": -320.5805358886719,
-      "logps/rejected": -321.72894287109375,
-      "loss": 0.3595,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.7039467692375183,
-      "rewards/margins": 1.15414297580719,
-      "rewards/rejected": -0.4501960873603821,
+      "logits/chosen": 0.12394122779369354,
+      "logits/rejected": 0.07622597366571426,
+      "logps/chosen": -482.99774169921875,
+      "logps/rejected": -617.9317626953125,
+      "loss": 0.4148,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4833831787109375,
+      "rewards/margins": 1.5686841011047363,
+      "rewards/rejected": -3.052067279815674,
       "step": 190
     },
     {
       "epoch": 0.42,
       "learning_rate": 7.209390765564318e-07,
-      "logits/chosen": 0.03291007876396179,
-      "logits/rejected": 0.178420752286911,
-      "logps/chosen": -325.4523010253906,
-      "logps/rejected": -307.71624755859375,
-      "loss": 0.3669,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.7518741488456726,
-      "rewards/margins": 1.431354284286499,
-      "rewards/rejected": -0.679480254650116,
+      "logits/chosen": 0.12547728419303894,
+      "logits/rejected": 0.039741553366184235,
+      "logps/chosen": -470.0662536621094,
+      "logps/rejected": -810.3030395507812,
+      "loss": 0.4152,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.2957651615142822,
+      "rewards/margins": 3.8659985065460205,
+      "rewards/rejected": -5.1617631912231445,
       "step": 200
     },
     {
       "epoch": 0.44,
       "learning_rate": 6.875296018047809e-07,
-      "logits/chosen": 0.16855312883853912,
-      "logits/rejected": 0.3136471211910248,
-      "logps/chosen": -296.46148681640625,
-      "logps/rejected": -285.39666748046875,
-      "loss": 0.3694,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": 0.8847891092300415,
-      "rewards/margins": 1.6609344482421875,
-      "rewards/rejected": -0.7761452794075012,
+      "logits/chosen": 0.20153549313545227,
+      "logits/rejected": 0.1317548155784607,
+      "logps/chosen": -447.82562255859375,
+      "logps/rejected": -725.8985595703125,
+      "loss": 0.4249,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.3366836309432983,
+      "rewards/margins": 3.223564863204956,
+      "rewards/rejected": -4.560248374938965,
       "step": 210
     },
     {
       "epoch": 0.46,
       "learning_rate": 6.531149075630796e-07,
-      "logits/chosen": 0.03008892573416233,
-      "logits/rejected": 0.1899140179157257,
-      "logps/chosen": -303.166015625,
-      "logps/rejected": -271.8243408203125,
-      "loss": 0.3701,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.6756522059440613,
-      "rewards/margins": 1.0833475589752197,
-      "rewards/rejected": -0.4076954424381256,
+      "logits/chosen": -0.017775116488337517,
+      "logits/rejected": 0.05367380380630493,
+      "logps/chosen": -476.78790283203125,
+      "logps/rejected": -663.9365844726562,
+      "loss": 0.4167,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.6010878086090088,
+      "rewards/margins": 2.401573419570923,
+      "rewards/rejected": -4.002661228179932,
       "step": 220
     },
     {
       "epoch": 0.48,
       "learning_rate": 6.178794677547137e-07,
-      "logits/chosen": 0.1291465163230896,
-      "logits/rejected": 0.1802050620317459,
-      "logps/chosen": -310.93695068359375,
-      "logps/rejected": -295.5335998535156,
-      "loss": 0.3611,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.7562187314033508,
-      "rewards/margins": 1.3768624067306519,
-      "rewards/rejected": -0.6206437945365906,
+      "logits/chosen": 0.07326556742191315,
+      "logits/rejected": -0.006058653350919485,
+      "logps/chosen": -590.01123046875,
+      "logps/rejected": -870.9129028320312,
+      "loss": 0.4193,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.6394991874694824,
+      "rewards/margins": 3.238422393798828,
+      "rewards/rejected": -5.8779215812683105,
       "step": 230
     },
     {
       "epoch": 0.5,
       "learning_rate": 5.820121557655108e-07,
-      "logits/chosen": 0.07808051258325577,
-      "logits/rejected": 0.12852515280246735,
-      "logps/chosen": -298.18524169921875,
-      "logps/rejected": -307.0373229980469,
-      "loss": 0.3695,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 0.6883242130279541,
-      "rewards/margins": 0.9448167681694031,
-      "rewards/rejected": -0.2564924955368042,
+      "logits/chosen": 0.13632330298423767,
+      "logits/rejected": 0.12085568904876709,
+      "logps/chosen": -450.1314392089844,
+      "logps/rejected": -587.374267578125,
+      "loss": 0.425,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.3817965984344482,
+      "rewards/margins": 1.4728713035583496,
+      "rewards/rejected": -2.854668140411377,
       "step": 240
     },
     {
       "epoch": 0.52,
       "learning_rate": 5.457052320211339e-07,
-      "logits/chosen": 0.028507575392723083,
-      "logits/rejected": 0.13449445366859436,
-      "logps/chosen": -294.77081298828125,
-      "logps/rejected": -290.4283752441406,
-      "loss": 0.363,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 0.863865077495575,
-      "rewards/margins": 1.283084750175476,
-      "rewards/rejected": -0.4192196726799011,
+      "logits/chosen": 0.09744735062122345,
+      "logits/rejected": -0.04311475530266762,
+      "logps/chosen": -561.7251586914062,
+      "logps/rejected": -1082.66064453125,
+      "loss": 0.4126,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.4967703819274902,
+      "rewards/margins": 5.509397029876709,
+      "rewards/rejected": -8.006166458129883,
       "step": 250
     },
     {
       "epoch": 0.54,
       "learning_rate": 5.091533134088387e-07,
-      "logits/chosen": -0.012728470377624035,
-      "logits/rejected": 0.08861465752124786,
-      "logps/chosen": -338.91156005859375,
-      "logps/rejected": -306.783935546875,
-      "loss": 0.3741,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.78661048412323,
-      "rewards/margins": 1.1226966381072998,
-      "rewards/rejected": -0.3360862135887146,
+      "logits/chosen": 0.007685136049985886,
+      "logits/rejected": -0.026540469378232956,
+      "logps/chosen": -681.2808837890625,
+      "logps/rejected": -1102.198486328125,
+      "loss": 0.4237,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -3.266371250152588,
+      "rewards/margins": 4.754992485046387,
+      "rewards/rejected": -8.021364212036133,
       "step": 260
     },
     {
       "epoch": 0.57,
       "learning_rate": 4.7255233006783624e-07,
-      "logits/chosen": 0.08402873575687408,
-      "logits/rejected": 0.13576345145702362,
-      "logps/chosen": -310.37738037109375,
-      "logps/rejected": -325.88531494140625,
-      "loss": 0.3558,
+      "logits/chosen": 0.24146917462348938,
+      "logits/rejected": 0.05772332474589348,
+      "logps/chosen": -437.0887756347656,
+      "logps/rejected": -754.1742553710938,
+      "loss": 0.409,
       "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 0.9378229975700378,
-      "rewards/margins": 1.7096540927886963,
-      "rewards/rejected": -0.7718309164047241,
+      "rewards/chosen": -1.0795494318008423,
+      "rewards/margins": 3.357706069946289,
+      "rewards/rejected": -4.437255859375,
       "step": 270
     },
     {
       "epoch": 0.59,
       "learning_rate": 4.3609847514019763e-07,
-      "logits/chosen": 0.06844338774681091,
-      "logits/rejected": 0.0715598464012146,
-      "logps/chosen": -280.6861877441406,
-      "logps/rejected": -300.75091552734375,
-      "loss": 0.3635,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.7898384928703308,
-      "rewards/margins": 1.2740315198898315,
-      "rewards/rejected": -0.4841931462287903,
+      "logits/chosen": 0.15583154559135437,
+      "logits/rejected": -0.01679980382323265,
+      "logps/chosen": -622.4188232421875,
+      "logps/rejected": -1143.203857421875,
+      "loss": 0.4172,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -3.2593586444854736,
+      "rewards/margins": 5.262009143829346,
+      "rewards/rejected": -8.521368980407715,
       "step": 280
     },
     {
       "epoch": 0.61,
       "learning_rate": 3.9998715311197783e-07,
-      "logits/chosen": 0.03731096163392067,
-      "logits/rejected": 0.12578508257865906,
-      "logps/chosen": -331.48779296875,
-      "logps/rejected": -339.0268859863281,
-      "loss": 0.356,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.7471667528152466,
-      "rewards/margins": 1.5588438510894775,
-      "rewards/rejected": -0.811677098274231,
+      "logits/chosen": 0.12384140491485596,
+      "logits/rejected": -0.03689634054899216,
+      "logps/chosen": -612.9854736328125,
+      "logps/rejected": -1161.8275146484375,
+      "loss": 0.4065,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.665544033050537,
+      "rewards/margins": 5.724797248840332,
+      "rewards/rejected": -8.390340805053711,
       "step": 290
     },
     {
       "epoch": 0.63,
       "learning_rate": 3.6441193238179146e-07,
-      "logits/chosen": 0.10170190036296844,
-      "logits/rejected": 0.125459223985672,
-      "logps/chosen": -276.04132080078125,
-      "logps/rejected": -338.2930603027344,
-      "loss": 0.3543,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.7856809496879578,
-      "rewards/margins": 1.5332136154174805,
-      "rewards/rejected": -0.7475326061248779,
+      "logits/chosen": 0.23247964680194855,
+      "logits/rejected": 0.08442293107509613,
+      "logps/chosen": -644.8258056640625,
+      "logps/rejected": -1333.277099609375,
+      "loss": 0.4067,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -3.5307083129882812,
+      "rewards/margins": 6.568638801574707,
+      "rewards/rejected": -10.099346160888672,
       "step": 300
     },
     {
       "epoch": 0.65,
       "learning_rate": 3.295635076714144e-07,
-      "logits/chosen": 0.10013142973184586,
-      "logits/rejected": 0.12764233350753784,
-      "logps/chosen": -281.08636474609375,
-      "logps/rejected": -327.638916015625,
-      "loss": 0.344,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 0.7322741746902466,
-      "rewards/margins": 1.5288127660751343,
-      "rewards/rejected": -0.7965387105941772,
+      "logits/chosen": 0.21653930842876434,
+      "logits/rejected": -0.010667298920452595,
+      "logps/chosen": -576.2736206054688,
+      "logps/rejected": -1167.0555419921875,
+      "loss": 0.4003,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.805418014526367,
+      "rewards/margins": 5.748055458068848,
+      "rewards/rejected": -8.553472518920898,
       "step": 310
     },
     {
       "epoch": 0.67,
       "learning_rate": 2.956286778402226e-07,
-      "logits/chosen": 0.04914706200361252,
-      "logits/rejected": 0.19642756879329681,
-      "logps/chosen": -297.6860046386719,
-      "logps/rejected": -307.39886474609375,
-      "loss": 0.3589,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.8972498774528503,
-      "rewards/margins": 1.7571271657943726,
-      "rewards/rejected": -0.8598772883415222,
+      "logits/chosen": 0.14956721663475037,
+      "logits/rejected": -0.00617391150444746,
+      "logps/chosen": -499.51556396484375,
+      "logps/rejected": -1073.225830078125,
+      "loss": 0.4081,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.8388452529907227,
+      "rewards/margins": 5.99139928817749,
+      "rewards/rejected": -7.830244541168213,
       "step": 320
     },
     {
       "epoch": 0.69,
       "learning_rate": 2.6278934458271996e-07,
-      "logits/chosen": 0.10303878784179688,
-      "logits/rejected": 0.18232768774032593,
-      "logps/chosen": -279.687744140625,
-      "logps/rejected": -322.4974060058594,
-      "loss": 0.3554,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.9540345072746277,
-      "rewards/margins": 1.5413516759872437,
-      "rewards/rejected": -0.5873170495033264,
+      "logits/chosen": 0.20027479529380798,
+      "logits/rejected": 0.06552217900753021,
+      "logps/chosen": -461.4195861816406,
+      "logps/rejected": -1150.258544921875,
+      "loss": 0.4027,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.6265113353729248,
+      "rewards/margins": 6.768563270568848,
+      "rewards/rejected": -8.395073890686035,
       "step": 330
     },
     {
       "epoch": 0.71,
       "learning_rate": 2.312215373764551e-07,
-      "logits/chosen": 0.05921119451522827,
-      "logits/rejected": 0.1563117355108261,
-      "logps/chosen": -270.2222595214844,
-      "logps/rejected": -273.39544677734375,
-      "loss": 0.3482,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.8237099647521973,
-      "rewards/margins": 1.4518331289291382,
-      "rewards/rejected": -0.6281229853630066,
+      "logits/chosen": 0.1772742122411728,
+      "logits/rejected": 0.058857548981904984,
+      "logps/chosen": -519.1689453125,
+      "logps/rejected": -1075.103759765625,
+      "loss": 0.4056,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.324723720550537,
+      "rewards/margins": 5.8179826736450195,
+      "rewards/rejected": -8.142705917358398,
       "step": 340
     },
     {
       "epoch": 0.73,
       "learning_rate": 2.0109446990692963e-07,
-      "logits/chosen": 0.026022329926490784,
-      "logits/rejected": 0.03612793609499931,
-      "logps/chosen": -287.3631286621094,
-      "logps/rejected": -323.0098876953125,
-      "loss": 0.355,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 0.7069499492645264,
-      "rewards/margins": 1.3287115097045898,
-      "rewards/rejected": -0.6217616200447083,
+      "logits/chosen": 0.09322932362556458,
+      "logits/rejected": -0.021080341190099716,
+      "logps/chosen": -524.8082275390625,
+      "logps/rejected": -1263.429443359375,
+      "loss": 0.404,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -2.233060121536255,
+      "rewards/margins": 7.2954888343811035,
+      "rewards/rejected": -9.528549194335938,
       "step": 350
     },
     {
       "epoch": 0.75,
       "learning_rate": 1.725696330273575e-07,
-      "logits/chosen": -0.008826015517115593,
-      "logits/rejected": 0.11012457311153412,
-      "logps/chosen": -308.5582580566406,
-      "logps/rejected": -291.039794921875,
-      "loss": 0.3408,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.9799526929855347,
-      "rewards/margins": 1.4228074550628662,
-      "rewards/rejected": -0.44285479187965393,
+      "logits/chosen": 0.12329642474651337,
+      "logits/rejected": -0.045363299548625946,
+      "logps/chosen": -477.84747314453125,
+      "logps/rejected": -1159.287353515625,
+      "loss": 0.3987,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.4969019889831543,
+      "rewards/margins": 7.274144172668457,
+      "rewards/rejected": -8.77104663848877,
       "step": 360
     },
     {
       "epoch": 0.77,
       "learning_rate": 1.4579992911531496e-07,
-      "logits/chosen": 0.011949884705245495,
-      "logits/rejected": 0.10477302223443985,
-      "logps/chosen": -325.74664306640625,
-      "logps/rejected": -332.83526611328125,
-      "loss": 0.3372,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 1.134242057800293,
-      "rewards/margins": 2.2450954914093018,
-      "rewards/rejected": -1.1108531951904297,
+      "logits/chosen": 0.13813820481300354,
+      "logits/rejected": 0.06726070493459702,
+      "logps/chosen": -596.8673706054688,
+      "logps/rejected": -1229.910888671875,
+      "loss": 0.3989,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.484358787536621,
+      "rewards/margins": 6.708567142486572,
+      "rewards/rejected": -9.192926406860352,
       "step": 370
     },
     {
       "epoch": 0.8,
       "learning_rate": 1.209288524664029e-07,
-      "logits/chosen": 0.06705882400274277,
-      "logits/rejected": 0.14220719039440155,
-      "logps/chosen": -278.93212890625,
-      "logps/rejected": -287.16900634765625,
-      "loss": 0.3393,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.707054853439331,
-      "rewards/margins": 1.4562907218933105,
-      "rewards/rejected": -0.749235987663269,
+      "logits/chosen": 0.2262219935655594,
+      "logits/rejected": 0.04883592948317528,
+      "logps/chosen": -571.9241333007812,
+      "logps/rejected": -1147.636474609375,
+      "loss": 0.3965,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.7885093688964844,
+      "rewards/margins": 5.966012954711914,
+      "rewards/rejected": -8.754522323608398,
       "step": 380
     },
     {
       "epoch": 0.82,
       "learning_rate": 9.808972011828054e-08,
-      "logits/chosen": -0.041753821074962616,
-      "logits/rejected": 0.10108913481235504,
-      "logps/chosen": -319.83123779296875,
-      "logps/rejected": -312.74029541015625,
-      "loss": 0.3434,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": 1.0930083990097046,
-      "rewards/margins": 1.6807419061660767,
-      "rewards/rejected": -0.5877334475517273,
+      "logits/chosen": 0.13919615745544434,
+      "logits/rejected": 0.08005174249410629,
+      "logps/chosen": -603.2689208984375,
+      "logps/rejected": -1278.978271484375,
+      "loss": 0.3993,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.6157753467559814,
+      "rewards/margins": 7.164151668548584,
+      "rewards/rejected": -9.779927253723145,
       "step": 390
     },
     {
       "epoch": 0.84,
       "learning_rate": 7.740495722810269e-08,
-      "logits/chosen": 0.037683337926864624,
-      "logits/rejected": 0.1319020837545395,
-      "logps/chosen": -298.04736328125,
-      "logps/rejected": -307.0213317871094,
-      "loss": 0.3519,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.9478852152824402,
-      "rewards/margins": 1.5595645904541016,
-      "rewards/rejected": -0.6116792559623718,
+      "logits/chosen": 0.1855761706829071,
+      "logits/rejected": 0.03339262679219246,
+      "logps/chosen": -554.6050415039062,
+      "logps/rejected": -1247.11474609375,
+      "loss": 0.4064,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.375999927520752,
+      "rewards/margins": 7.147269248962402,
+      "rewards/rejected": -9.523270606994629,
       "step": 400
     },
     {
       "epoch": 0.86,
       "learning_rate": 5.898544083397e-08,
-      "logits/chosen": -0.02371780201792717,
-      "logits/rejected": 0.03532714769244194,
-      "logps/chosen": -273.40704345703125,
-      "logps/rejected": -299.4648132324219,
-      "loss": 0.3533,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.789527416229248,
-      "rewards/margins": 1.2326061725616455,
-      "rewards/rejected": -0.44307881593704224,
+      "logits/chosen": 0.10612723976373672,
+      "logits/rejected": -0.03204170614480972,
+      "logps/chosen": -598.8375244140625,
+      "logps/rejected": -1218.921142578125,
+      "loss": 0.4009,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -3.0963997840881348,
+      "rewards/margins": 6.186778545379639,
+      "rewards/rejected": -9.283178329467773,
       "step": 410
     },
     {
       "epoch": 0.88,
       "learning_rate": 4.292990551804171e-08,
-      "logits/chosen": 0.07686875015497208,
-      "logits/rejected": 0.1795254498720169,
-      "logps/chosen": -273.1594543457031,
-      "logps/rejected": -296.47528076171875,
-      "loss": 0.3382,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 1.008718729019165,
-      "rewards/margins": 1.5497512817382812,
-      "rewards/rejected": -0.5410324335098267,
+      "logits/chosen": 0.3134514391422272,
+      "logits/rejected": 0.1133495420217514,
+      "logps/chosen": -560.297607421875,
+      "logps/rejected": -1385.083251953125,
+      "loss": 0.3991,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.669637680053711,
+      "rewards/margins": 8.3246488571167,
+      "rewards/rejected": -10.994285583496094,
       "step": 420
     },
     {
       "epoch": 0.9,
       "learning_rate": 2.9324414157151367e-08,
-      "logits/chosen": 0.06547899544239044,
-      "logits/rejected": 0.09760904312133789,
-      "logps/chosen": -315.00506591796875,
-      "logps/rejected": -338.0772399902344,
-      "loss": 0.3499,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.9535905122756958,
-      "rewards/margins": 1.6683721542358398,
-      "rewards/rejected": -0.7147817015647888,
+      "logits/chosen": 0.14708609879016876,
+      "logits/rejected": 0.05113764852285385,
+      "logps/chosen": -646.3408203125,
+      "logps/rejected": -1521.79345703125,
+      "loss": 0.3999,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.122638702392578,
+      "rewards/margins": 8.8574800491333,
+      "rewards/rejected": -11.980117797851562,
       "step": 430
     },
     {
       "epoch": 0.92,
       "learning_rate": 1.824189659787284e-08,
-      "logits/chosen": 0.051959145814180374,
-      "logits/rejected": 0.0797661691904068,
-      "logps/chosen": -269.6096496582031,
-      "logps/rejected": -288.70709228515625,
-      "loss": 0.3456,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.7473762035369873,
-      "rewards/margins": 1.2938742637634277,
-      "rewards/rejected": -0.5464980006217957,
+      "logits/chosen": 0.19891302287578583,
+      "logits/rejected": 0.057393454015254974,
+      "logps/chosen": -530.86865234375,
+      "logps/rejected": -1372.778076171875,
+      "loss": 0.3979,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.4631145000457764,
+      "rewards/margins": 8.486894607543945,
+      "rewards/rejected": -10.950007438659668,
       "step": 440
     },
     {
       "epoch": 0.94,
       "learning_rate": 9.741758728888217e-09,
-      "logits/chosen": 0.013958173803985119,
-      "logits/rejected": 0.19162164628505707,
-      "logps/chosen": -331.8248596191406,
-      "logps/rejected": -298.86553955078125,
-      "loss": 0.3412,
+      "logits/chosen": 0.20876403152942657,
+      "logits/rejected": 0.052755843847990036,
+      "logps/chosen": -683.3274536132812,
+      "logps/rejected": -1404.552978515625,
+      "loss": 0.3915,
       "rewards/accuracies": 0.71875,
-      "rewards/chosen": 0.9063283801078796,
-      "rewards/margins": 1.6520798206329346,
-      "rewards/rejected": -0.7457513809204102,
+      "rewards/chosen": -3.3337600231170654,
+      "rewards/margins": 7.872265815734863,
+      "rewards/rejected": -11.206026077270508,
       "step": 450
     },
     {
       "epoch": 0.96,
       "learning_rate": 3.869564046156459e-09,
-      "logits/chosen": 0.05520665645599365,
-      "logits/rejected": 0.24095895886421204,
-      "logps/chosen": -279.37713623046875,
-      "logps/rejected": -279.66351318359375,
-      "loss": 0.3525,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": 0.9280216097831726,
-      "rewards/margins": 1.6233961582183838,
-      "rewards/rejected": -0.6953743100166321,
+      "logits/chosen": 0.2985457181930542,
+      "logits/rejected": 0.15650448203086853,
+      "logps/chosen": -468.8932189941406,
+      "logps/rejected": -1197.56201171875,
+      "loss": 0.3987,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7095565795898438,
+      "rewards/margins": 7.608504295349121,
+      "rewards/rejected": -9.318059921264648,
       "step": 460
     },
     {
       "epoch": 0.98,
       "learning_rate": 6.567894177967325e-10,
-      "logits/chosen": 0.008933846838772297,
-      "logits/rejected": 0.1640961915254593,
-      "logps/chosen": -319.6179504394531,
-      "logps/rejected": -314.25787353515625,
-      "loss": 0.3473,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 0.9765602946281433,
-      "rewards/margins": 1.8153190612792969,
-      "rewards/rejected": -0.8387589454650879,
+      "logits/chosen": 0.17393910884857178,
+      "logits/rejected": 0.02789122983813286,
+      "logps/chosen": -607.3438720703125,
+      "logps/rejected": -1505.235595703125,
+      "loss": 0.3978,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -2.6819469928741455,
+      "rewards/margins": 9.39558219909668,
+      "rewards/rejected": -12.07752799987793,
       "step": 470
     },
     {
       "epoch": 1.0,
       "step": 477,
       "total_flos": 0.0,
-      "train_loss": 0.37614710375947774,
-      "train_runtime": 6361.5592,
-      "train_samples_per_second": 9.61,
+      "train_loss": 0.42718374404267445,
+      "train_runtime": 6325.1171,
+      "train_samples_per_second": 9.665,
       "train_steps_per_second": 0.075
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cec4624b9c095040eb8aa52a9ba592de199b303541db109644b3cf58d7c369c4
+oid sha256:b021e642df60d4e058f25e7642bbe07c40ea3fe4bd81ba446202dbd4f17079f4
 size 6648