Model save

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.34605644783883727,
-    "train_runtime": 6207.5812,
+    "train_loss": 0.37614710375947774,
+    "train_runtime": 6361.5592,
     "train_samples": 61134,
-    "train_samples_per_second": 9.848,
-    "train_steps_per_second": 0.077
+    "train_samples_per_second": 9.61,
+    "train_steps_per_second": 0.075
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b48c4d4938d4c04a59f63ceb99963905c7f8b51613ab1c185e59808413f94a24
+oid sha256:8558783426207d481e1b212901b2a745cf326f53087e609ed23dbb2013f59ce2
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0fd34b5e33561f0c1c4e8aba96c533e93794f3b9c20d5d385d7ab01a800026e9
+oid sha256:0de16d834b47e78126680587374bfb8e3f6fff6da5595e08d8bc8c4dfe3f057e
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9471f53484869323cab64ce1a140ea2e9812ea57df01261f12da0a66c195e23f
+oid sha256:bd358506aa4a2ba7d2848486443d68ed78736482c3641750882e51a5423e6c7e
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:955866100db63c46b619b676098124e61b58bb80aedf8c78a341cf53f7450c9d
+oid sha256:1ee534a0b473037a903d7605adf23f1bc01159bba79b0d8f59245eebbe979737
 size 1168138808

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.34605644783883727,
-    "train_runtime": 6207.5812,
+    "train_loss": 0.37614710375947774,
+    "train_runtime": 6361.5592,
     "train_samples": 61134,
-    "train_samples_per_second": 9.848,
-    "train_steps_per_second": 0.077
+    "train_samples_per_second": 9.61,
+    "train_steps_per_second": 0.075
 }

trainer_state.json

@@ -11,669 +11,669 @@
     {
       "epoch": 0.02,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": 0.17659232020378113,
-      "logits/rejected": 0.25393185019493103,
-      "logps/chosen": -354.3299865722656,
-      "logps/rejected": -305.2392883300781,
-      "loss": 0.4997,
-      "rewards/accuracies": 0.4312500059604645,
-      "rewards/chosen": 0.007027293089777231,
-      "rewards/margins": 0.009162568487226963,
-      "rewards/rejected": -0.002135276095941663,
+      "logits/chosen": 0.1770419478416443,
+      "logits/rejected": 0.2540443539619446,
+      "logps/chosen": -354.38037109375,
+      "logps/rejected": -305.27264404296875,
+      "loss": 0.4999,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": 0.0009949840605258942,
+      "rewards/margins": 0.003730112686753273,
+      "rewards/rejected": -0.0027351281605660915,
       "step": 10
     },
     {
       "epoch": 0.04,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": 0.0710873156785965,
-      "logits/rejected": 0.19884119927883148,
-      "logps/chosen": -316.61993408203125,
-      "logps/rejected": -276.21624755859375,
-      "loss": 0.4974,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": 0.011533690616488457,
-      "rewards/margins": 0.02961091324687004,
-      "rewards/rejected": -0.018077218905091286,
+      "logits/chosen": 0.07181452214717865,
+      "logits/rejected": 0.19976207613945007,
+      "logps/chosen": -316.61358642578125,
+      "logps/rejected": -276.0943603515625,
+      "loss": 0.4986,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.006085564382374287,
+      "rewards/margins": 0.009029300883412361,
+      "rewards/rejected": -0.002943736733868718,
       "step": 20
     },
     {
       "epoch": 0.06,
       "learning_rate": 6.249999999999999e-07,
-      "logits/chosen": 0.18181222677230835,
-      "logits/rejected": 0.2527164816856384,
-      "logps/chosen": -294.2743225097656,
-      "logps/rejected": -298.42523193359375,
-      "loss": 0.4791,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.06925608962774277,
-      "rewards/margins": 0.0819806382060051,
-      "rewards/rejected": -0.012724560685455799,
+      "logits/chosen": 0.18359068036079407,
+      "logits/rejected": 0.2548081874847412,
+      "logps/chosen": -294.26654052734375,
+      "logps/rejected": -298.5642395019531,
+      "loss": 0.4893,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.03501707315444946,
+      "rewards/margins": 0.048330314457416534,
+      "rewards/rejected": -0.013313241302967072,
       "step": 30
     },
     {
       "epoch": 0.08,
       "learning_rate": 8.333333333333333e-07,
-      "logits/chosen": 0.11502287536859512,
-      "logits/rejected": 0.23403987288475037,
-      "logps/chosen": -343.6348571777344,
-      "logps/rejected": -318.79791259765625,
-      "loss": 0.4533,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": 0.3937355875968933,
-      "rewards/margins": 0.22681434452533722,
-      "rewards/rejected": 0.16692125797271729,
+      "logits/chosen": 0.11794896423816681,
+      "logits/rejected": 0.23672719299793243,
+      "logps/chosen": -343.41192626953125,
+      "logps/rejected": -318.60638427734375,
+      "loss": 0.4731,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.20801420509815216,
+      "rewards/margins": 0.1149774044752121,
+      "rewards/rejected": 0.09303676337003708,
       "step": 40
     },
     {
       "epoch": 0.1,
       "learning_rate": 9.999463737538052e-07,
-      "logits/chosen": 0.17206831276416779,
-      "logits/rejected": 0.2584429085254669,
-      "logps/chosen": -306.1137390136719,
-      "logps/rejected": -285.19561767578125,
-      "loss": 0.4228,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.4438863694667816,
-      "rewards/margins": 0.5952231287956238,
-      "rewards/rejected": -0.15133680403232574,
+      "logits/chosen": 0.18171748518943787,
+      "logits/rejected": 0.2686474025249481,
+      "logps/chosen": -305.8929443359375,
+      "logps/rejected": -285.5357360839844,
+      "loss": 0.4517,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.2329844981431961,
+      "rewards/margins": 0.32565948367118835,
+      "rewards/rejected": -0.09267498552799225,
       "step": 50
     },
     {
       "epoch": 0.13,
       "learning_rate": 9.980706626858607e-07,
-      "logits/chosen": 0.1263621747493744,
-      "logits/rejected": 0.2632313370704651,
-      "logps/chosen": -289.64898681640625,
-      "logps/rejected": -279.08453369140625,
-      "loss": 0.4049,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.322978675365448,
-      "rewards/margins": 0.6868947744369507,
-      "rewards/rejected": -0.3639160990715027,
+      "logits/chosen": 0.14295880496501923,
+      "logits/rejected": 0.28016844391822815,
+      "logps/chosen": -290.75335693359375,
+      "logps/rejected": -281.30303955078125,
+      "loss": 0.4337,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.10626886039972305,
+      "rewards/margins": 0.39915287494659424,
+      "rewards/rejected": -0.29288405179977417,
       "step": 60
     },
     {
       "epoch": 0.15,
       "learning_rate": 9.935251313189563e-07,
-      "logits/chosen": 0.11648492515087128,
-      "logits/rejected": 0.19881902635097504,
-      "logps/chosen": -322.6625671386719,
-      "logps/rejected": -320.7023620605469,
-      "loss": 0.3918,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.7061153650283813,
-      "rewards/margins": 0.8815712928771973,
-      "rewards/rejected": -0.1754559576511383,
+      "logits/chosen": 0.1319437325000763,
+      "logits/rejected": 0.21764138340950012,
+      "logps/chosen": -324.74078369140625,
+      "logps/rejected": -324.4031066894531,
+      "loss": 0.4212,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.24914593994617462,
+      "rewards/margins": 0.5219111442565918,
+      "rewards/rejected": -0.27276521921157837,
       "step": 70
     },
     {
       "epoch": 0.17,
       "learning_rate": 9.86334145175542e-07,
-      "logits/chosen": 0.1569955050945282,
-      "logits/rejected": 0.25127941370010376,
-      "logps/chosen": -308.86077880859375,
-      "logps/rejected": -291.51300048828125,
-      "loss": 0.3825,
+      "logits/chosen": 0.17676237225532532,
+      "logits/rejected": 0.27126845717430115,
+      "logps/chosen": -311.08868408203125,
+      "logps/rejected": -297.3030090332031,
+      "loss": 0.4116,
       "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.9802907109260559,
-      "rewards/margins": 1.4248442649841309,
-      "rewards/rejected": -0.4445534348487854,
+      "rewards/chosen": 0.3787495195865631,
+      "rewards/margins": 0.8905243873596191,
+      "rewards/rejected": -0.5117748379707336,
       "step": 80
     },
     {
       "epoch": 0.19,
       "learning_rate": 9.765362502737097e-07,
-      "logits/chosen": 0.01096972357481718,
-      "logits/rejected": 0.15417027473449707,
-      "logps/chosen": -325.28692626953125,
-      "logps/rejected": -269.4154968261719,
-      "loss": 0.3708,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 1.165459394454956,
-      "rewards/margins": 1.3995535373687744,
-      "rewards/rejected": -0.2340943068265915,
+      "logits/chosen": 0.03276940807700157,
+      "logits/rejected": 0.1763920933008194,
+      "logps/chosen": -327.9542541503906,
+      "logps/rejected": -275.962890625,
+      "loss": 0.401,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.4493633210659027,
+      "rewards/margins": 0.8937808871269226,
+      "rewards/rejected": -0.4444176256656647,
       "step": 90
     },
     {
       "epoch": 0.21,
       "learning_rate": 9.641839665080363e-07,
-      "logits/chosen": 0.08043601363897324,
-      "logits/rejected": 0.2466708868741989,
-      "logps/chosen": -321.926513671875,
-      "logps/rejected": -292.6954345703125,
-      "loss": 0.3623,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 1.2634952068328857,
-      "rewards/margins": 1.294826626777649,
-      "rewards/rejected": -0.03133126348257065,
+      "logits/chosen": 0.10452715307474136,
+      "logits/rejected": 0.27761924266815186,
+      "logps/chosen": -323.75360107421875,
+      "logps/rejected": -299.73370361328125,
+      "loss": 0.3917,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.5403918027877808,
+      "rewards/margins": 0.907971978187561,
+      "rewards/rejected": -0.3675800561904907,
       "step": 100
     },
     {
       "epoch": 0.23,
       "learning_rate": 9.493435061259129e-07,
-      "logits/chosen": 0.09306775033473969,
-      "logits/rejected": 0.11595858633518219,
-      "logps/chosen": -301.3399963378906,
-      "logps/rejected": -301.2839660644531,
-      "loss": 0.3404,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 1.1279371976852417,
-      "rewards/margins": 1.3113162517547607,
-      "rewards/rejected": -0.18337900936603546,
+      "logits/chosen": 0.1223590150475502,
+      "logits/rejected": 0.14537492394447327,
+      "logps/chosen": -303.0465087890625,
+      "logps/rejected": -307.5492248535156,
+      "loss": 0.3772,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.4786440432071686,
+      "rewards/margins": 0.8835989236831665,
+      "rewards/rejected": -0.4049549102783203,
       "step": 110
     },
     {
       "epoch": 0.25,
       "learning_rate": 9.320944188084241e-07,
-      "logits/chosen": 0.05181150510907173,
-      "logits/rejected": 0.10330178588628769,
-      "logps/chosen": -308.14410400390625,
-      "logps/rejected": -322.2857360839844,
-      "loss": 0.3632,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 1.2409526109695435,
-      "rewards/margins": 1.3498892784118652,
-      "rewards/rejected": -0.10893689095973969,
+      "logits/chosen": 0.07157851755619049,
+      "logits/rejected": 0.12891840934753418,
+      "logps/chosen": -309.7882385253906,
+      "logps/rejected": -329.6763610839844,
+      "loss": 0.3939,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.538270115852356,
+      "rewards/margins": 0.9622691869735718,
+      "rewards/rejected": -0.4239990711212158,
       "step": 120
     },
     {
       "epoch": 0.27,
       "learning_rate": 9.125291652582547e-07,
-      "logits/chosen": 0.020175116136670113,
-      "logits/rejected": 0.16819754242897034,
-      "logps/chosen": -324.732177734375,
-      "logps/rejected": -285.5627136230469,
-      "loss": 0.3408,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 1.376392126083374,
-      "rewards/margins": 1.4476263523101807,
-      "rewards/rejected": -0.07123424857854843,
+      "logits/chosen": 0.03255900740623474,
+      "logits/rejected": 0.1836429387331009,
+      "logps/chosen": -328.1858825683594,
+      "logps/rejected": -294.7179260253906,
+      "loss": 0.3745,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.5155087113380432,
+      "rewards/margins": 1.008885145187378,
+      "rewards/rejected": -0.4933764338493347,
       "step": 130
     },
     {
       "epoch": 0.29,
       "learning_rate": 8.90752621580335e-07,
-      "logits/chosen": -0.007861034944653511,
-      "logits/rejected": 0.14148305356502533,
-      "logps/chosen": -322.4604797363281,
-      "logps/rejected": -273.60321044921875,
-      "loss": 0.3284,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.9582949876785278,
-      "rewards/margins": 1.3279260396957397,
-      "rewards/rejected": -0.36963123083114624,
+      "logits/chosen": 0.008989883586764336,
+      "logits/rejected": 0.17159470915794373,
+      "logps/chosen": -326.74957275390625,
+      "logps/rejected": -284.432373046875,
+      "loss": 0.3671,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.2646927833557129,
+      "rewards/margins": 0.9909642338752747,
+      "rewards/rejected": -0.7262714505195618,
       "step": 140
     },
     {
       "epoch": 0.31,
       "learning_rate": 8.668815171119019e-07,
-      "logits/chosen": 0.07163457572460175,
-      "logits/rejected": 0.11265318095684052,
-      "logps/chosen": -318.3633117675781,
-      "logps/rejected": -314.35321044921875,
-      "loss": 0.3452,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 1.6937291622161865,
-      "rewards/margins": 1.6340910196304321,
-      "rewards/rejected": 0.05963808298110962,
+      "logits/chosen": 0.10270164906978607,
+      "logits/rejected": 0.15117475390434265,
+      "logps/chosen": -319.47796630859375,
+      "logps/rejected": -322.017822265625,
+      "loss": 0.3768,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.7911346554756165,
+      "rewards/margins": 1.1445444822311401,
+      "rewards/rejected": -0.35340994596481323,
       "step": 150
     },
     {
       "epoch": 0.33,
       "learning_rate": 8.410438087153911e-07,
-      "logits/chosen": 0.04666835069656372,
-      "logits/rejected": 0.09292508661746979,
-      "logps/chosen": -283.54595947265625,
-      "logps/rejected": -278.94769287109375,
-      "loss": 0.3399,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 1.8515584468841553,
-      "rewards/margins": 1.676443099975586,
-      "rewards/rejected": 0.17511534690856934,
+      "logits/chosen": 0.07868606597185135,
+      "logits/rejected": 0.12421569973230362,
+      "logps/chosen": -283.022216796875,
+      "logps/rejected": -284.0340881347656,
+      "loss": 0.3737,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.9519669413566589,
+      "rewards/margins": 1.1187279224395752,
+      "rewards/rejected": -0.16676095128059387,
       "step": 160
     },
     {
       "epoch": 0.36,
       "learning_rate": 8.133779948881513e-07,
-      "logits/chosen": 0.017909998074173927,
-      "logits/rejected": 0.0967395007610321,
-      "logps/chosen": -319.28668212890625,
-      "logps/rejected": -307.777099609375,
-      "loss": 0.3344,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 1.7081670761108398,
-      "rewards/margins": 1.7009460926055908,
-      "rewards/rejected": 0.007220864295959473,
+      "logits/chosen": 0.053541384637355804,
+      "logits/rejected": 0.13824662566184998,
+      "logps/chosen": -318.894775390625,
+      "logps/rejected": -313.05755615234375,
+      "loss": 0.3657,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.8736802339553833,
+      "rewards/margins": 1.134092926979065,
+      "rewards/rejected": -0.26041263341903687,
       "step": 170
     },
     {
       "epoch": 0.38,
       "learning_rate": 7.840323733655778e-07,
-      "logits/chosen": -0.01807587407529354,
-      "logits/rejected": 0.12429861724376678,
-      "logps/chosen": -308.74945068359375,
-      "logps/rejected": -301.0330505371094,
-      "loss": 0.3297,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": 1.7282564640045166,
-      "rewards/margins": 1.8440380096435547,
-      "rewards/rejected": -0.1157817393541336,
+      "logits/chosen": 0.01729046180844307,
+      "logits/rejected": 0.1666645109653473,
+      "logps/chosen": -309.3392639160156,
+      "logps/rejected": -307.69378662109375,
+      "loss": 0.3669,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.8346372842788696,
+      "rewards/margins": 1.2255662679672241,
+      "rewards/rejected": -0.39092904329299927,
       "step": 180
     },
     {
       "epoch": 0.4,
       "learning_rate": 7.531642461971514e-07,
-      "logits/chosen": 0.05006791278719902,
-      "logits/rejected": 0.12837380170822144,
-      "logps/chosen": -318.7227478027344,
-      "logps/rejected": -313.162841796875,
-      "loss": 0.3238,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 1.5936723947525024,
-      "rewards/margins": 1.6374588012695312,
-      "rewards/rejected": -0.043786562979221344,
+      "logits/chosen": 0.07640022039413452,
+      "logits/rejected": 0.1669580638408661,
+      "logps/chosen": -320.5805358886719,
+      "logps/rejected": -321.72894287109375,
+      "loss": 0.3595,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.7039467692375183,
+      "rewards/margins": 1.15414297580719,
+      "rewards/rejected": -0.4501960873603821,
       "step": 190
     },
     {
       "epoch": 0.42,
       "learning_rate": 7.209390765564318e-07,
-      "logits/chosen": -0.004646389279514551,
-      "logits/rejected": 0.1289207488298416,
-      "logps/chosen": -325.53973388671875,
-      "logps/rejected": -299.326904296875,
-      "loss": 0.3379,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 1.4950015544891357,
-      "rewards/margins": 2.0150256156921387,
-      "rewards/rejected": -0.5200243592262268,
+      "logits/chosen": 0.03291007876396179,
+      "logits/rejected": 0.178420752286911,
+      "logps/chosen": -325.4523010253906,
+      "logps/rejected": -307.71624755859375,
+      "loss": 0.3669,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.7518741488456726,
+      "rewards/margins": 1.431354284286499,
+      "rewards/rejected": -0.679480254650116,
       "step": 200
     },
     {
       "epoch": 0.44,
       "learning_rate": 6.875296018047809e-07,
-      "logits/chosen": 0.12037472426891327,
-      "logits/rejected": 0.24673119187355042,
-      "logps/chosen": -296.4547119140625,
-      "logps/rejected": -276.714111328125,
-      "loss": 0.337,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 1.770256757736206,
-      "rewards/margins": 2.454294204711914,
-      "rewards/rejected": -0.6840375661849976,
+      "logits/chosen": 0.16855312883853912,
+      "logits/rejected": 0.3136471211910248,
+      "logps/chosen": -296.46148681640625,
+      "logps/rejected": -285.39666748046875,
+      "loss": 0.3694,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.8847891092300415,
+      "rewards/margins": 1.6609344482421875,
+      "rewards/rejected": -0.7761452794075012,
       "step": 210
     },
     {
       "epoch": 0.46,
       "learning_rate": 6.531149075630796e-07,
-      "logits/chosen": -0.00517323287203908,
-      "logits/rejected": 0.13203957676887512,
-      "logps/chosen": -300.7497863769531,
-      "logps/rejected": -265.4754333496094,
-      "loss": 0.3394,
+      "logits/chosen": 0.03008892573416233,
+      "logits/rejected": 0.1899140179157257,
+      "logps/chosen": -303.166015625,
+      "logps/rejected": -271.8243408203125,
+      "loss": 0.3701,
       "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 1.5929324626922607,
-      "rewards/margins": 1.7734311819076538,
-      "rewards/rejected": -0.18049874901771545,
+      "rewards/chosen": 0.6756522059440613,
+      "rewards/margins": 1.0833475589752197,
+      "rewards/rejected": -0.4076954424381256,
       "step": 220
     },
     {
       "epoch": 0.48,
       "learning_rate": 6.178794677547137e-07,
-      "logits/chosen": 0.0826948806643486,
-      "logits/rejected": 0.13583150506019592,
-      "logps/chosen": -309.74224853515625,
-      "logps/rejected": -286.40533447265625,
-      "loss": 0.328,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 1.6319090127944946,
-      "rewards/margins": 1.9603748321533203,
-      "rewards/rejected": -0.32846564054489136,
+      "logits/chosen": 0.1291465163230896,
+      "logits/rejected": 0.1802050620317459,
+      "logps/chosen": -310.93695068359375,
+      "logps/rejected": -295.5335998535156,
+      "loss": 0.3611,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.7562187314033508,
+      "rewards/margins": 1.3768624067306519,
+      "rewards/rejected": -0.6206437945365906,
       "step": 230
     },
     {
       "epoch": 0.5,
       "learning_rate": 5.820121557655108e-07,
-      "logits/chosen": 0.04763277620077133,
-      "logits/rejected": 0.10109977424144745,
-      "logps/chosen": -296.3155822753906,
-      "logps/rejected": -299.51873779296875,
-      "loss": 0.3406,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 1.5636177062988281,
-      "rewards/margins": 1.3247454166412354,
-      "rewards/rejected": 0.23887233436107635,
+      "logits/chosen": 0.07808051258325577,
+      "logits/rejected": 0.12852515280246735,
+      "logps/chosen": -298.18524169921875,
+      "logps/rejected": -307.0373229980469,
+      "loss": 0.3695,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.6883242130279541,
+      "rewards/margins": 0.9448167681694031,
+      "rewards/rejected": -0.2564924955368042,
       "step": 240
     },
     {
       "epoch": 0.52,
       "learning_rate": 5.457052320211339e-07,
-      "logits/chosen": 0.012852217070758343,
-      "logits/rejected": 0.1158476248383522,
-      "logps/chosen": -294.8091125488281,
-      "logps/rejected": -282.0174865722656,
-      "loss": 0.3307,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 1.7239019870758057,
-      "rewards/margins": 1.7212518453598022,
-      "rewards/rejected": 0.002649706555530429,
+      "logits/chosen": 0.028507575392723083,
+      "logits/rejected": 0.13449445366859436,
+      "logps/chosen": -294.77081298828125,
+      "logps/rejected": -290.4283752441406,
+      "loss": 0.363,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.863865077495575,
+      "rewards/margins": 1.283084750175476,
+      "rewards/rejected": -0.4192196726799011,
       "step": 250
     },
     {
       "epoch": 0.54,
       "learning_rate": 5.091533134088387e-07,
-      "logits/chosen": -0.014469897374510765,
-      "logits/rejected": 0.07733233273029327,
-      "logps/chosen": -338.17327880859375,
-      "logps/rejected": -299.3669738769531,
-      "loss": 0.3441,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 1.6470504999160767,
-      "rewards/margins": 1.5775295495986938,
-      "rewards/rejected": 0.06952105462551117,
+      "logits/chosen": -0.012728470377624035,
+      "logits/rejected": 0.08861465752124786,
+      "logps/chosen": -338.91156005859375,
+      "logps/rejected": -306.783935546875,
+      "loss": 0.3741,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.78661048412323,
+      "rewards/margins": 1.1226966381072998,
+      "rewards/rejected": -0.3360862135887146,
       "step": 260
     },
     {
       "epoch": 0.57,
       "learning_rate": 4.7255233006783624e-07,
-      "logits/chosen": 0.07055391371250153,
-      "logits/rejected": 0.12625916302204132,
-      "logps/chosen": -308.22979736328125,
-      "logps/rejected": -314.9990234375,
-      "loss": 0.3276,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 2.090404987335205,
-      "rewards/margins": 2.5454375743865967,
-      "rewards/rejected": -0.4550328850746155,
+      "logits/chosen": 0.08402873575687408,
+      "logits/rejected": 0.13576345145702362,
+      "logps/chosen": -310.37738037109375,
+      "logps/rejected": -325.88531494140625,
+      "loss": 0.3558,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.9378229975700378,
+      "rewards/margins": 1.7096540927886963,
+      "rewards/rejected": -0.7718309164047241,
       "step": 270
     },
     {
       "epoch": 0.59,
       "learning_rate": 4.3609847514019763e-07,
-      "logits/chosen": 0.04763927310705185,
-      "logits/rejected": 0.060921769589185715,
-      "logps/chosen": -279.77642822265625,
-      "logps/rejected": -291.3137512207031,
-      "loss": 0.333,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 1.6706489324569702,
-      "rewards/margins": 1.6953166723251343,
-      "rewards/rejected": -0.02466759644448757,
+      "logits/chosen": 0.06844338774681091,
+      "logits/rejected": 0.0715598464012146,
+      "logps/chosen": -280.6861877441406,
+      "logps/rejected": -300.75091552734375,
+      "loss": 0.3635,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.7898384928703308,
+      "rewards/margins": 1.2740315198898315,
+      "rewards/rejected": -0.4841931462287903,
       "step": 280
     },
     {
       "epoch": 0.61,
       "learning_rate": 3.9998715311197783e-07,
-      "logits/chosen": 0.009870557114481926,
-      "logits/rejected": 0.0942167341709137,
-      "logps/chosen": -330.2633972167969,
-      "logps/rejected": -327.8630065917969,
-      "loss": 0.324,
+      "logits/chosen": 0.03731096163392067,
+      "logits/rejected": 0.12578508257865906,
+      "logps/chosen": -331.48779296875,
+      "logps/rejected": -339.0268859863281,
+      "loss": 0.356,
       "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 1.6167709827423096,
-      "rewards/margins": 2.1237380504608154,
-      "rewards/rejected": -0.5069671869277954,
+      "rewards/chosen": 0.7471667528152466,
+      "rewards/margins": 1.5588438510894775,
+      "rewards/rejected": -0.811677098274231,
       "step": 290
     },
     {
       "epoch": 0.63,
       "learning_rate": 3.6441193238179146e-07,
-      "logits/chosen": 0.07049299776554108,
-      "logits/rejected": 0.08977767825126648,
-      "logps/chosen": -274.5482177734375,
-      "logps/rejected": -326.8363342285156,
-      "loss": 0.3165,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": 1.7206777334213257,
-      "rewards/margins": 2.070065498352051,
-      "rewards/rejected": -0.34938788414001465,
+      "logits/chosen": 0.10170190036296844,
+      "logits/rejected": 0.125459223985672,
+      "logps/chosen": -276.04132080078125,
+      "logps/rejected": -338.2930603027344,
+      "loss": 0.3543,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.7856809496879578,
+      "rewards/margins": 1.5332136154174805,
+      "rewards/rejected": -0.7475326061248779,
       "step": 300
     },
     {
       "epoch": 0.65,
       "learning_rate": 3.295635076714144e-07,
-      "logits/chosen": 0.06275717914104462,
-      "logits/rejected": 0.09255780279636383,
-      "logps/chosen": -279.8996276855469,
-      "logps/rejected": -316.30621337890625,
-      "loss": 0.3135,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 1.5832184553146362,
-      "rewards/margins": 2.043025493621826,
-      "rewards/rejected": -0.4598070979118347,
+      "logits/chosen": 0.10013142973184586,
+      "logits/rejected": 0.12764233350753784,
+      "logps/chosen": -281.08636474609375,
+      "logps/rejected": -327.638916015625,
+      "loss": 0.344,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.7322741746902466,
+      "rewards/margins": 1.5288127660751343,
+      "rewards/rejected": -0.7965387105941772,
       "step": 310
     },
     {
       "epoch": 0.67,
       "learning_rate": 2.956286778402226e-07,
-      "logits/chosen": 0.009205429814755917,
-      "logits/rejected": 0.15460598468780518,
-      "logps/chosen": -296.5576171875,
-      "logps/rejected": -295.835693359375,
-      "loss": 0.3264,
+      "logits/chosen": 0.04914706200361252,
+      "logits/rejected": 0.19642756879329681,
+      "logps/chosen": -297.6860046386719,
+      "logps/rejected": -307.39886474609375,
+      "loss": 0.3589,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 1.9073429107666016,
-      "rewards/margins": 2.4707765579223633,
-      "rewards/rejected": -0.5634336471557617,
+      "rewards/chosen": 0.8972498774528503,
+      "rewards/margins": 1.7571271657943726,
+      "rewards/rejected": -0.8598772883415222,
       "step": 320
     },
     {
       "epoch": 0.69,
       "learning_rate": 2.6278934458271996e-07,
-      "logits/chosen": 0.058381110429763794,
-      "logits/rejected": 0.13819275796413422,
-      "logps/chosen": -281.2006530761719,
-      "logps/rejected": -313.47174072265625,
-      "loss": 0.3232,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 1.7567777633666992,
-      "rewards/margins": 2.0288453102111816,
-      "rewards/rejected": -0.27206745743751526,
+      "logits/chosen": 0.10303878784179688,
+      "logits/rejected": 0.18232768774032593,
+      "logps/chosen": -279.687744140625,
+      "logps/rejected": -322.4974060058594,
+      "loss": 0.3554,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.9540345072746277,
+      "rewards/margins": 1.5413516759872437,
+      "rewards/rejected": -0.5873170495033264,
       "step": 330
     },
     {
       "epoch": 0.71,
       "learning_rate": 2.312215373764551e-07,
-      "logits/chosen": 0.02380923368036747,
-      "logits/rejected": 0.11691228300333023,
-      "logps/chosen": -269.53045654296875,
-      "logps/rejected": -264.8174133300781,
-      "loss": 0.3176,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 1.7166016101837158,
-      "rewards/margins": 2.1150426864624023,
-      "rewards/rejected": -0.39844104647636414,
+      "logits/chosen": 0.05921119451522827,
+      "logits/rejected": 0.1563117355108261,
+      "logps/chosen": -270.2222595214844,
+      "logps/rejected": -273.39544677734375,
+      "loss": 0.3482,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.8237099647521973,
+      "rewards/margins": 1.4518331289291382,
+      "rewards/rejected": -0.6281229853630066,
       "step": 340
     },
     {
       "epoch": 0.73,
       "learning_rate": 2.0109446990692963e-07,
-      "logits/chosen": -0.0028537046164274216,
-      "logits/rejected": 0.010644497349858284,
-      "logps/chosen": -285.66375732421875,
-      "logps/rejected": -311.220703125,
-      "loss": 0.3286,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 1.583837628364563,
-      "rewards/margins": 1.6484451293945312,
-      "rewards/rejected": -0.06460770219564438,
+      "logits/chosen": 0.026022329926490784,
+      "logits/rejected": 0.03612793609499931,
+      "logps/chosen": -287.3631286621094,
+      "logps/rejected": -323.0098876953125,
+      "loss": 0.355,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.7069499492645264,
+      "rewards/margins": 1.3287115097045898,
+      "rewards/rejected": -0.6217616200447083,
       "step": 350
     },
     {
       "epoch": 0.75,
       "learning_rate": 1.725696330273575e-07,
-      "logits/chosen": -0.04485073685646057,
-      "logits/rejected": 0.07074587792158127,
-      "logps/chosen": -308.7581787109375,
-      "logps/rejected": -282.4355773925781,
-      "loss": 0.3048,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 1.9399118423461914,
-      "rewards/margins": 1.9651952981948853,
-      "rewards/rejected": -0.025283390656113625,
+      "logits/chosen": -0.008826015517115593,
+      "logits/rejected": 0.11012457311153412,
+      "logps/chosen": -308.5582580566406,
+      "logps/rejected": -291.039794921875,
+      "loss": 0.3408,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.9799526929855347,
+      "rewards/margins": 1.4228074550628662,
+      "rewards/rejected": -0.44285479187965393,
       "step": 360
     },
     {
       "epoch": 0.77,
       "learning_rate": 1.4579992911531496e-07,
-      "logits/chosen": -0.02086697518825531,
-      "logits/rejected": 0.06267571449279785,
-      "logps/chosen": -326.6328125,
-      "logps/rejected": -318.2734375,
-      "loss": 0.3048,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 2.179868221282959,
-      "rewards/margins": 2.945388078689575,
-      "rewards/rejected": -0.7655196189880371,
+      "logits/chosen": 0.011949884705245495,
+      "logits/rejected": 0.10477302223443985,
+      "logps/chosen": -325.74664306640625,
+      "logps/rejected": -332.83526611328125,
+      "loss": 0.3372,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 1.134242057800293,
+      "rewards/margins": 2.2450954914093018,
+      "rewards/rejected": -1.1108531951904297,
       "step": 370
     },
     {
       "epoch": 0.8,
       "learning_rate": 1.209288524664029e-07,
-      "logits/chosen": 0.030310412868857384,
-      "logits/rejected": 0.11966564506292343,
-      "logps/chosen": -277.7276916503906,
-      "logps/rejected": -276.39739990234375,
-      "loss": 0.303,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 1.5345547199249268,
-      "rewards/margins": 1.9558677673339844,
-      "rewards/rejected": -0.4213130474090576,
+      "logits/chosen": 0.06705882400274277,
+      "logits/rejected": 0.14220719039440155,
+      "logps/chosen": -278.93212890625,
+      "logps/rejected": -287.16900634765625,
+      "loss": 0.3393,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.707054853439331,
+      "rewards/margins": 1.4562907218933105,
+      "rewards/rejected": -0.749235987663269,
       "step": 380
     },
     {
       "epoch": 0.82,
       "learning_rate": 9.808972011828054e-08,
-      "logits/chosen": -0.06816152483224869,
-      "logits/rejected": 0.05935809761285782,
-      "logps/chosen": -320.16424560546875,
-      "logps/rejected": -303.69097900390625,
-      "loss": 0.31,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 2.1527152061462402,
-      "rewards/margins": 2.423252582550049,
-      "rewards/rejected": -0.2705369293689728,
+      "logits/chosen": -0.041753821074962616,
+      "logits/rejected": 0.10108913481235504,
+      "logps/chosen": -319.83123779296875,
+      "logps/rejected": -312.74029541015625,
+      "loss": 0.3434,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 1.0930083990097046,
+      "rewards/margins": 1.6807419061660767,
+      "rewards/rejected": -0.5877334475517273,
       "step": 390
     },
     {
       "epoch": 0.84,
       "learning_rate": 7.740495722810269e-08,
-      "logits/chosen": 0.00943700410425663,
-      "logits/rejected": 0.103798508644104,
-      "logps/chosen": -298.0252685546875,
-      "logps/rejected": -297.83843994140625,
-      "loss": 0.3215,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 1.8979790210723877,
-      "rewards/margins": 2.2030446529388428,
-      "rewards/rejected": -0.30506545305252075,
+      "logits/chosen": 0.037683337926864624,
+      "logits/rejected": 0.1319020837545395,
+      "logps/chosen": -298.04736328125,
+      "logps/rejected": -307.0213317871094,
+      "loss": 0.3519,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.9478852152824402,
+      "rewards/margins": 1.5595645904541016,
+      "rewards/rejected": -0.6116792559623718,
       "step": 400
     },
     {
       "epoch": 0.86,
       "learning_rate": 5.898544083397e-08,
-      "logits/chosen": -0.059524454176425934,
-      "logits/rejected": 0.015174726955592632,
-      "logps/chosen": -273.3992614746094,
-      "logps/rejected": -292.0383605957031,
-      "loss": 0.3228,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 1.579828143119812,
-      "rewards/margins": 1.7233396768569946,
-      "rewards/rejected": -0.14351139962673187,
+      "logits/chosen": -0.02371780201792717,
+      "logits/rejected": 0.03532714769244194,
+      "logps/chosen": -273.40704345703125,
+      "logps/rejected": -299.4648132324219,
+      "loss": 0.3533,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.789527416229248,
+      "rewards/margins": 1.2326061725616455,
+      "rewards/rejected": -0.44307881593704224,
       "step": 410
     },
     {
       "epoch": 0.88,
       "learning_rate": 4.292990551804171e-08,
-      "logits/chosen": 0.0325622633099556,
-      "logits/rejected": 0.11653436720371246,
-      "logps/chosen": -273.19097900390625,
-      "logps/rejected": -286.6775817871094,
-      "loss": 0.3048,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 2.0142855644226074,
-      "rewards/margins": 2.1165807247161865,
-      "rewards/rejected": -0.10229482501745224,
+      "logits/chosen": 0.07686875015497208,
+      "logits/rejected": 0.1795254498720169,
+      "logps/chosen": -273.1594543457031,
+      "logps/rejected": -296.47528076171875,
+      "loss": 0.3382,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 1.008718729019165,
+      "rewards/margins": 1.5497512817382812,
+      "rewards/rejected": -0.5410324335098267,
       "step": 420
     },
     {
       "epoch": 0.9,
       "learning_rate": 2.9324414157151367e-08,
-      "logits/chosen": 0.028907526284456253,
-      "logits/rejected": 0.059953343123197556,
-      "logps/chosen": -314.8848876953125,
-      "logps/rejected": -325.2813720703125,
-      "loss": 0.3156,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 1.919198989868164,
-      "rewards/margins": 2.0691730976104736,
-      "rewards/rejected": -0.14997398853302002,
+      "logits/chosen": 0.06547899544239044,
+      "logits/rejected": 0.09760904312133789,
+      "logps/chosen": -315.00506591796875,
+      "logps/rejected": -338.0772399902344,
+      "loss": 0.3499,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.9535905122756958,
+      "rewards/margins": 1.6683721542358398,
+      "rewards/rejected": -0.7147817015647888,
       "step": 430
     },
     {
       "epoch": 0.92,
       "learning_rate": 1.824189659787284e-08,
-      "logits/chosen": 0.022467201575636864,
-      "logits/rejected": 0.049929820001125336,
-      "logps/chosen": -268.8197937011719,
-      "logps/rejected": -279.419921875,
-      "loss": 0.3137,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 1.5737355947494507,
-      "rewards/margins": 1.738011360168457,
-      "rewards/rejected": -0.16427570581436157,
+      "logits/chosen": 0.051959145814180374,
+      "logits/rejected": 0.0797661691904068,
+      "logps/chosen": -269.6096496582031,
+      "logps/rejected": -288.70709228515625,
+      "loss": 0.3456,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.7473762035369873,
+      "rewards/margins": 1.2938742637634277,
+      "rewards/rejected": -0.5464980006217957,
       "step": 440
     },
     {
       "epoch": 0.94,
       "learning_rate": 9.741758728888217e-09,
-      "logits/chosen": -0.012446149252355099,
-      "logits/rejected": 0.16082307696342468,
-      "logps/chosen": -331.36688232421875,
-      "logps/rejected": -287.3267822265625,
-      "loss": 0.3116,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": 1.8584492206573486,
-      "rewards/margins": 2.196075677871704,
-      "rewards/rejected": -0.337626576423645,
+      "logits/chosen": 0.013958173803985119,
+      "logits/rejected": 0.19162164628505707,
+      "logps/chosen": -331.8248596191406,
+      "logps/rejected": -298.86553955078125,
+      "loss": 0.3412,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.9063283801078796,
+      "rewards/margins": 1.6520798206329346,
+      "rewards/rejected": -0.7457513809204102,
       "step": 450
     },
     {
       "epoch": 0.96,
       "learning_rate": 3.869564046156459e-09,
-      "logits/chosen": 0.03111204504966736,
-      "logits/rejected": 0.199659526348114,
-      "logps/chosen": -279.9968566894531,
-      "logps/rejected": -270.5770568847656,
-      "loss": 0.3212,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 1.7940731048583984,
-      "rewards/margins": 2.2761783599853516,
-      "rewards/rejected": -0.4821050763130188,
+      "logits/chosen": 0.05520665645599365,
+      "logits/rejected": 0.24095895886421204,
+      "logps/chosen": -279.37713623046875,
+      "logps/rejected": -279.66351318359375,
+      "loss": 0.3525,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.9280216097831726,
+      "rewards/margins": 1.6233961582183838,
+      "rewards/rejected": -0.6953743100166321,
       "step": 460
     },
     {
       "epoch": 0.98,
       "learning_rate": 6.567894177967325e-10,
-      "logits/chosen": -0.010620922781527042,
-      "logits/rejected": 0.12738940119743347,
-      "logps/chosen": -319.92535400390625,
-      "logps/rejected": -303.04998779296875,
-      "loss": 0.3148,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 1.9223756790161133,
-      "rewards/margins": 2.479106903076172,
-      "rewards/rejected": -0.5567313432693481,
+      "logits/chosen": 0.008933846838772297,
+      "logits/rejected": 0.1640961915254593,
+      "logps/chosen": -319.6179504394531,
+      "logps/rejected": -314.25787353515625,
+      "loss": 0.3473,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.9765602946281433,
+      "rewards/margins": 1.8153190612792969,
+      "rewards/rejected": -0.8387589454650879,
       "step": 470
     },
     {
       "epoch": 1.0,
       "step": 477,
       "total_flos": 0.0,
-      "train_loss": 0.34605644783883727,
-      "train_runtime": 6207.5812,
-      "train_samples_per_second": 9.848,
-      "train_steps_per_second": 0.077
+      "train_loss": 0.37614710375947774,
+      "train_runtime": 6361.5592,
+      "train_samples_per_second": 9.61,
+      "train_steps_per_second": 0.075
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5489a9492470a4cd73257e10cf655deb8e3a29aaf1d9767180b56bdf4b151b30
+oid sha256:cec4624b9c095040eb8aa52a9ba592de199b303541db109644b3cf58d7c369c4
 size 6648