Model save

README.md

@@ -37,7 +37,7 @@ The following hyperparameters were used during training:
 - learning_rate: 5e-07
 - train_batch_size: 8
 - eval_batch_size: 8
-- seed: 3
+- seed: 4
 - distributed_type: multi-GPU
 - num_devices: 8
 - gradient_accumulation_steps: 2

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.08838768833213383,
-    "train_runtime": 3226.1286,
+    "train_loss": 0.08520494085035206,
+    "train_runtime": 3257.6401,
     "train_samples": 51894,
-    "train_samples_per_second": 16.086,
-    "train_steps_per_second": 0.126
+    "train_samples_per_second": 15.93,
+    "train_steps_per_second": 0.124
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bec2fc3a9f5c1b7f9a8338008c24ba39fa60cc0ce19d81fa4a80fc872b4d1a69
+oid sha256:8cc657a2ee48daf3c730e4aab1598173ac5f267b70b973a7eb21736bbef1e546
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:99b9705334bc445b339c02b18833e0f94ca1f783c8936c7cfc80540fcdb914cc
+oid sha256:d97d51c686e2d180bd32387ba32d6b5c4c1187ea7765c11ec897d31c8c7030e5
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:560b2231901e748076a7cfb0ff2787a4b3f6ab9c45609b9e95f6f3da85ee8e6d
+oid sha256:afb80c7d1ddf42019d5701c07a7752e01c5b98b2bf02d3dc3764e39ca350bc77
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.08838768833213383,
-    "train_runtime": 3226.1286,
+    "train_loss": 0.08520494085035206,
+    "train_runtime": 3257.6401,
     "train_samples": 51894,
-    "train_samples_per_second": 16.086,
-    "train_steps_per_second": 0.126
+    "train_samples_per_second": 15.93,
+    "train_steps_per_second": 0.124
 }

trainer_state.json

@@ -11,11 +11,11 @@
     {
       "epoch": 0.0,
       "learning_rate": 1.2195121951219512e-08,
-      "logits/chosen": -2.8088459968566895,
-      "logits/rejected": -2.7595884799957275,
-      "logps/chosen": -368.90777587890625,
-      "logps/rejected": -133.10202026367188,
-      "loss": 0.4054,
+      "logits/chosen": -2.8681135177612305,
+      "logits/rejected": -2.8858838081359863,
+      "logps/chosen": -518.1907958984375,
+      "logps/rejected": -109.31971740722656,
+      "loss": 0.3939,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,571 +25,571 @@
     {
       "epoch": 0.02,
       "learning_rate": 1.219512195121951e-07,
-      "logits/chosen": -2.838838815689087,
-      "logits/rejected": -2.8251054286956787,
-      "logps/chosen": -433.8114318847656,
-      "logps/rejected": -114.71322631835938,
-      "loss": 0.3815,
-      "rewards/accuracies": 0.5694444179534912,
-      "rewards/chosen": 0.0011257152073085308,
-      "rewards/margins": 0.0019443891942501068,
-      "rewards/rejected": -0.0008186736959032714,
+      "logits/chosen": -2.7987546920776367,
+      "logits/rejected": -2.7523815631866455,
+      "logps/chosen": -434.1848449707031,
+      "logps/rejected": -114.18251037597656,
+      "loss": 0.3844,
+      "rewards/accuracies": 0.4513888955116272,
+      "rewards/chosen": 0.0004987930878996849,
+      "rewards/margins": 0.001031861756928265,
+      "rewards/rejected": -0.000533068785443902,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.439024390243902e-07,
-      "logits/chosen": -2.798201560974121,
-      "logits/rejected": -2.7650632858276367,
-      "logps/chosen": -436.72210693359375,
-      "logps/rejected": -109.3458023071289,
-      "loss": 0.3794,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 0.02019491419196129,
-      "rewards/margins": 0.03630927950143814,
-      "rewards/rejected": -0.016114361584186554,
+      "logits/chosen": -2.8212170600891113,
+      "logits/rejected": -2.7980713844299316,
+      "logps/chosen": -417.22845458984375,
+      "logps/rejected": -118.038330078125,
+      "loss": 0.3804,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.02029377594590187,
+      "rewards/margins": 0.036424122750759125,
+      "rewards/rejected": -0.0161303523927927,
       "step": 20
     },
     {
       "epoch": 0.07,
       "learning_rate": 3.6585365853658536e-07,
-      "logits/chosen": -2.7168636322021484,
-      "logits/rejected": -2.6898279190063477,
-      "logps/chosen": -422.3126525878906,
-      "logps/rejected": -128.2278289794922,
-      "loss": 0.3417,
+      "logits/chosen": -2.656278133392334,
+      "logits/rejected": -2.644028663635254,
+      "logps/chosen": -398.81500244140625,
+      "logps/rejected": -125.97016906738281,
+      "loss": 0.3437,
       "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 0.07048428058624268,
-      "rewards/margins": 0.20025746524333954,
-      "rewards/rejected": -0.12977321445941925,
+      "rewards/chosen": 0.0762760117650032,
+      "rewards/margins": 0.2021373212337494,
+      "rewards/rejected": -0.12586131691932678,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.878048780487804e-07,
-      "logits/chosen": -2.591139554977417,
-      "logits/rejected": -2.57306170463562,
-      "logps/chosen": -396.5892333984375,
-      "logps/rejected": -139.44308471679688,
+      "logits/chosen": -2.5408267974853516,
+      "logits/rejected": -2.5265488624572754,
+      "logps/chosen": -384.7110595703125,
+      "logps/rejected": -169.5616912841797,
       "loss": 0.2883,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": 0.0210564024746418,
-      "rewards/margins": 0.42175084352493286,
-      "rewards/rejected": -0.40069445967674255,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.022841650992631912,
+      "rewards/margins": 0.42677050828933716,
+      "rewards/rejected": -0.40392884612083435,
       "step": 40
     },
     {
       "epoch": 0.12,
       "learning_rate": 4.992461696250783e-07,
-      "logits/chosen": -2.4237170219421387,
-      "logits/rejected": -2.397566318511963,
-      "logps/chosen": -446.45684814453125,
-      "logps/rejected": -203.3081512451172,
-      "loss": 0.2012,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.04143618792295456,
-      "rewards/margins": 0.8411922454833984,
-      "rewards/rejected": -0.8826284408569336,
+      "logits/chosen": -2.42402982711792,
+      "logits/rejected": -2.390493154525757,
+      "logps/chosen": -437.21337890625,
+      "logps/rejected": -221.1737823486328,
+      "loss": 0.204,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.1747589260339737,
+      "rewards/margins": 0.8014341592788696,
+      "rewards/rejected": -0.9761930704116821,
       "step": 50
     },
     {
       "epoch": 0.15,
       "learning_rate": 4.966461721767899e-07,
-      "logits/chosen": -2.3991150856018066,
-      "logits/rejected": -2.3474137783050537,
-      "logps/chosen": -426.8975524902344,
-      "logps/rejected": -257.33331298828125,
-      "loss": 0.1494,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.39810866117477417,
-      "rewards/margins": 0.9240871667861938,
-      "rewards/rejected": -1.3221957683563232,
+      "logits/chosen": -2.378632068634033,
+      "logits/rejected": -2.3337972164154053,
+      "logps/chosen": -439.27764892578125,
+      "logps/rejected": -243.908203125,
+      "loss": 0.1586,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3428526818752289,
+      "rewards/margins": 0.9457007646560669,
+      "rewards/rejected": -1.2885534763336182,
       "step": 60
     },
     {
       "epoch": 0.17,
       "learning_rate": 4.922100518015975e-07,
-      "logits/chosen": -2.436988592147827,
-      "logits/rejected": -2.3888049125671387,
-      "logps/chosen": -421.7872619628906,
-      "logps/rejected": -276.4320068359375,
-      "loss": 0.1308,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.37923428416252136,
-      "rewards/margins": 1.1982433795928955,
-      "rewards/rejected": -1.5774776935577393,
+      "logits/chosen": -2.37459659576416,
+      "logits/rejected": -2.327303409576416,
+      "logps/chosen": -420.84381103515625,
+      "logps/rejected": -267.9267578125,
+      "loss": 0.119,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.40542110800743103,
+      "rewards/margins": 1.2117817401885986,
+      "rewards/rejected": -1.6172027587890625,
       "step": 70
     },
     {
       "epoch": 0.2,
       "learning_rate": 4.859708325770919e-07,
-      "logits/chosen": -2.3866608142852783,
-      "logits/rejected": -2.3412399291992188,
-      "logps/chosen": -472.11474609375,
-      "logps/rejected": -319.96246337890625,
-      "loss": 0.0752,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.8105772733688354,
-      "rewards/margins": 1.3323147296905518,
-      "rewards/rejected": -2.1428916454315186,
+      "logits/chosen": -2.428609609603882,
+      "logits/rejected": -2.370483875274658,
+      "logps/chosen": -474.28582763671875,
+      "logps/rejected": -337.1321716308594,
+      "loss": 0.0755,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.5983784794807434,
+      "rewards/margins": 1.5884076356887817,
+      "rewards/rejected": -2.18678617477417,
       "step": 80
     },
     {
       "epoch": 0.22,
       "learning_rate": 4.779749614980225e-07,
-      "logits/chosen": -2.380894899368286,
-      "logits/rejected": -2.3338356018066406,
-      "logps/chosen": -545.9353637695312,
-      "logps/rejected": -395.8908386230469,
-      "loss": 0.0595,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -0.6987307667732239,
-      "rewards/margins": 1.9613780975341797,
-      "rewards/rejected": -2.660108804702759,
+      "logits/chosen": -2.3890507221221924,
+      "logits/rejected": -2.3450489044189453,
+      "logps/chosen": -500.783203125,
+      "logps/rejected": -368.41094970703125,
+      "loss": 0.0584,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.796062171459198,
+      "rewards/margins": 1.7808799743652344,
+      "rewards/rejected": -2.576941967010498,
       "step": 90
     },
     {
       "epoch": 0.25,
       "learning_rate": 4.682819627081427e-07,
-      "logits/chosen": -2.3579602241516113,
-      "logits/rejected": -2.295790195465088,
-      "logps/chosen": -480.7085876464844,
-      "logps/rejected": -364.87811279296875,
-      "loss": 0.074,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -0.6792080402374268,
-      "rewards/margins": 1.7849966287612915,
-      "rewards/rejected": -2.464204788208008,
+      "logits/chosen": -2.384782075881958,
+      "logits/rejected": -2.3418028354644775,
+      "logps/chosen": -502.3565368652344,
+      "logps/rejected": -362.3828125,
+      "loss": 0.0628,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.5387237071990967,
+      "rewards/margins": 1.9631805419921875,
+      "rewards/rejected": -2.501904249191284,
       "step": 100
     },
     {
       "epoch": 0.27,
       "learning_rate": 4.569639943810477e-07,
-      "logits/chosen": -2.3346972465515137,
-      "logits/rejected": -2.265590190887451,
-      "logps/chosen": -502.68487548828125,
-      "logps/rejected": -394.2782287597656,
-      "loss": 0.0555,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.9552167654037476,
-      "rewards/margins": 1.8180055618286133,
-      "rewards/rejected": -2.7732224464416504,
+      "logits/chosen": -2.3396503925323486,
+      "logits/rejected": -2.2695436477661133,
+      "logps/chosen": -466.52679443359375,
+      "logps/rejected": -395.2033996582031,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.0900439023971558,
+      "rewards/margins": 1.7583458423614502,
+      "rewards/rejected": -2.8483898639678955,
       "step": 110
     },
     {
       "epoch": 0.3,
       "learning_rate": 4.4410531154874543e-07,
-      "logits/chosen": -2.359708547592163,
-      "logits/rejected": -2.275885820388794,
-      "logps/chosen": -540.5257568359375,
-      "logps/rejected": -415.78570556640625,
-      "loss": 0.0608,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.8868741989135742,
-      "rewards/margins": 1.965667486190796,
-      "rewards/rejected": -2.852541923522949,
+      "logits/chosen": -2.3576321601867676,
+      "logits/rejected": -2.3109076023101807,
+      "logps/chosen": -484.6878356933594,
+      "logps/rejected": -393.3995666503906,
+      "loss": 0.0561,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.9047477841377258,
+      "rewards/margins": 1.829101324081421,
+      "rewards/rejected": -2.733849048614502,
       "step": 120
     },
     {
       "epoch": 0.32,
       "learning_rate": 4.298016388768561e-07,
-      "logits/chosen": -2.431809902191162,
-      "logits/rejected": -2.36917781829834,
-      "logps/chosen": -497.15985107421875,
-      "logps/rejected": -360.4634094238281,
-      "loss": 0.0746,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -0.3195730745792389,
-      "rewards/margins": 2.053260326385498,
-      "rewards/rejected": -2.372833251953125,
+      "logits/chosen": -2.3168373107910156,
+      "logits/rejected": -2.277609348297119,
+      "logps/chosen": -474.674560546875,
+      "logps/rejected": -389.1014709472656,
+      "loss": 0.0576,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.033732533454895,
+      "rewards/margins": 1.7740589380264282,
+      "rewards/rejected": -2.807791233062744,
       "step": 130
     },
     {
       "epoch": 0.35,
       "learning_rate": 4.1415945805573005e-07,
-      "logits/chosen": -2.3186304569244385,
-      "logits/rejected": -2.2435660362243652,
-      "logps/chosen": -503.1153259277344,
-      "logps/rejected": -406.16278076171875,
-      "loss": 0.0659,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.8214370012283325,
-      "rewards/margins": 1.914272665977478,
-      "rewards/rejected": -2.7357096672058105,
+      "logits/chosen": -2.27695631980896,
+      "logits/rejected": -2.22804594039917,
+      "logps/chosen": -521.7109985351562,
+      "logps/rejected": -422.5042419433594,
+      "loss": 0.0485,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.9181007146835327,
+      "rewards/margins": 2.139181613922119,
+      "rewards/rejected": -3.0572826862335205,
       "step": 140
     },
     {
       "epoch": 0.37,
       "learning_rate": 3.972952151123984e-07,
-      "logits/chosen": -2.282636880874634,
-      "logits/rejected": -2.1880698204040527,
-      "logps/chosen": -486.3431091308594,
-      "logps/rejected": -408.52734375,
-      "loss": 0.0508,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.11625075340271,
-      "rewards/margins": 1.9597688913345337,
-      "rewards/rejected": -3.076019287109375,
+      "logits/chosen": -2.2969155311584473,
+      "logits/rejected": -2.2290713787078857,
+      "logps/chosen": -517.611328125,
+      "logps/rejected": -414.35491943359375,
+      "loss": 0.0446,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.0711259841918945,
+      "rewards/margins": 1.9469735622406006,
+      "rewards/rejected": -3.018099546432495,
       "step": 150
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.793344535444142e-07,
-      "logits/chosen": -2.278097629547119,
-      "logits/rejected": -2.1780576705932617,
-      "logps/chosen": -555.6715087890625,
-      "logps/rejected": -414.88861083984375,
-      "loss": 0.0418,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -0.8848897218704224,
-      "rewards/margins": 2.1523947715759277,
-      "rewards/rejected": -3.0372846126556396,
+      "logits/chosen": -2.294722080230713,
+      "logits/rejected": -2.2438769340515137,
+      "logps/chosen": -532.6485595703125,
+      "logps/rejected": -426.20513916015625,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.2362943887710571,
+      "rewards/margins": 1.7946383953094482,
+      "rewards/rejected": -3.030932903289795,
       "step": 160
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.604108797288461e-07,
-      "logits/chosen": -2.327190637588501,
-      "logits/rejected": -2.240990400314331,
-      "logps/chosen": -523.722900390625,
-      "logps/rejected": -402.6468200683594,
-      "loss": 0.0573,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -0.8474733233451843,
-      "rewards/margins": 2.0742928981781006,
-      "rewards/rejected": -2.9217662811279297,
+      "logits/chosen": -2.2703588008880615,
+      "logits/rejected": -2.2441458702087402,
+      "logps/chosen": -449.19091796875,
+      "logps/rejected": -373.4166259765625,
+      "loss": 0.059,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.1028985977172852,
+      "rewards/margins": 1.593082070350647,
+      "rewards/rejected": -2.6959805488586426,
       "step": 170
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.40665367563858e-07,
-      "logits/chosen": -2.2963194847106934,
-      "logits/rejected": -2.2069220542907715,
-      "logps/chosen": -533.5665283203125,
-      "logps/rejected": -434.58074951171875,
-      "loss": 0.0479,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -1.2346763610839844,
-      "rewards/margins": 1.9557201862335205,
-      "rewards/rejected": -3.1903960704803467,
+      "logits/chosen": -2.300586700439453,
+      "logits/rejected": -2.2104036808013916,
+      "logps/chosen": -538.3341064453125,
+      "logps/rejected": -435.9353942871094,
+      "loss": 0.0478,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -0.782570481300354,
+      "rewards/margins": 2.3180627822875977,
+      "rewards/rejected": -3.100633144378662,
       "step": 180
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.202449097526798e-07,
-      "logits/chosen": -2.303617000579834,
-      "logits/rejected": -2.2251999378204346,
-      "logps/chosen": -511.34063720703125,
-      "logps/rejected": -417.365966796875,
-      "loss": 0.0478,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.0919907093048096,
-      "rewards/margins": 2.0082223415374756,
-      "rewards/rejected": -3.100213050842285,
+      "logits/chosen": -2.273599624633789,
+      "logits/rejected": -2.2272255420684814,
+      "logps/chosen": -511.9939880371094,
+      "logps/rejected": -422.14044189453125,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.0683417320251465,
+      "rewards/margins": 1.9933398962020874,
+      "rewards/rejected": -3.0616817474365234,
       "step": 190
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.993015235369905e-07,
-      "logits/chosen": -2.2632386684417725,
-      "logits/rejected": -2.1550886631011963,
-      "logps/chosen": -555.7200317382812,
-      "logps/rejected": -457.74981689453125,
-      "loss": 0.0439,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.16446852684021,
-      "rewards/margins": 2.234539270401001,
-      "rewards/rejected": -3.399007797241211,
+      "logits/chosen": -2.2274768352508545,
+      "logits/rejected": -2.178544282913208,
+      "logps/chosen": -531.9685668945312,
+      "logps/rejected": -466.0660095214844,
+      "loss": 0.0415,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.5277751684188843,
+      "rewards/margins": 1.9938457012176514,
+      "rewards/rejected": -3.521620988845825,
       "step": 200
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7799111902582693e-07,
-      "logits/chosen": -2.273411512374878,
-      "logits/rejected": -2.172680377960205,
-      "logps/chosen": -526.6717529296875,
-      "logps/rejected": -403.1504211425781,
-      "loss": 0.0448,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.261858582496643,
-      "rewards/margins": 1.7486225366592407,
-      "rewards/rejected": -3.010481119155884,
+      "logits/chosen": -2.239495038986206,
+      "logits/rejected": -2.14347505569458,
+      "logps/chosen": -574.8614501953125,
+      "logps/rejected": -501.8006896972656,
+      "loss": 0.0305,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.3575702905654907,
+      "rewards/margins": 2.561095714569092,
+      "rewards/rejected": -3.918666124343872,
       "step": 210
     },
     {
       "epoch": 0.54,
       "learning_rate": 2.564723385445869e-07,
-      "logits/chosen": -2.3388514518737793,
-      "logits/rejected": -2.263075351715088,
-      "logps/chosen": -517.6388549804688,
-      "logps/rejected": -412.77325439453125,
-      "loss": 0.0602,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.0001862049102783,
-      "rewards/margins": 1.8844444751739502,
-      "rewards/rejected": -2.8846306800842285,
+      "logits/chosen": -2.3055145740509033,
+      "logits/rejected": -2.2241768836975098,
+      "logps/chosen": -569.2613525390625,
+      "logps/rejected": -487.712158203125,
+      "loss": 0.0477,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.3685243129730225,
+      "rewards/margins": 2.2271132469177246,
+      "rewards/rejected": -3.595637559890747,
       "step": 220
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3490537564442845e-07,
-      "logits/chosen": -2.295351982116699,
-      "logits/rejected": -2.193934202194214,
-      "logps/chosen": -507.52520751953125,
-      "logps/rejected": -393.0706481933594,
-      "loss": 0.0637,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.1557964086532593,
-      "rewards/margins": 1.709519386291504,
-      "rewards/rejected": -2.8653159141540527,
+      "logits/chosen": -2.2792344093322754,
+      "logits/rejected": -2.2088494300842285,
+      "logps/chosen": -494.75286865234375,
+      "logps/rejected": -409.4736328125,
+      "loss": 0.0545,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.0721429586410522,
+      "rewards/margins": 1.7748781442642212,
+      "rewards/rejected": -2.8470211029052734,
       "step": 230
     },
     {
       "epoch": 0.59,
       "learning_rate": 2.1345078256378801e-07,
-      "logits/chosen": -2.326420783996582,
-      "logits/rejected": -2.23372220993042,
-      "logps/chosen": -512.0640258789062,
-      "logps/rejected": -428.93572998046875,
-      "loss": 0.051,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -1.0325157642364502,
-      "rewards/margins": 2.0758230686187744,
-      "rewards/rejected": -3.1083388328552246,
+      "logits/chosen": -2.3294692039489746,
+      "logits/rejected": -2.264793634414673,
+      "logps/chosen": -521.0982666015625,
+      "logps/rejected": -408.40289306640625,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.0901975631713867,
+      "rewards/margins": 1.8291746377944946,
+      "rewards/rejected": -2.919372320175171,
       "step": 240
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.9226827501969865e-07,
-      "logits/chosen": -2.324457883834839,
-      "logits/rejected": -2.2472641468048096,
-      "logps/chosen": -543.7798461914062,
-      "logps/rejected": -454.26043701171875,
-      "loss": 0.0521,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -1.0110582113265991,
-      "rewards/margins": 2.3495380878448486,
-      "rewards/rejected": -3.360596179962158,
+      "logits/chosen": -2.317063808441162,
+      "logits/rejected": -2.2598507404327393,
+      "logps/chosen": -524.32666015625,
+      "logps/rejected": -434.41925048828125,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.0961580276489258,
+      "rewards/margins": 2.1600213050842285,
+      "rewards/rejected": -3.2561793327331543,
       "step": 250
     },
     {
       "epoch": 0.64,
       "learning_rate": 1.715155432264775e-07,
-      "logits/chosen": -2.316281795501709,
-      "logits/rejected": -2.235330581665039,
-      "logps/chosen": -547.55908203125,
-      "logps/rejected": -448.2779235839844,
-      "loss": 0.0399,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -1.1552385091781616,
-      "rewards/margins": 2.1640262603759766,
-      "rewards/rejected": -3.3192646503448486,
+      "logits/chosen": -2.2932469844818115,
+      "logits/rejected": -2.188689708709717,
+      "logps/chosen": -496.59967041015625,
+      "logps/rejected": -417.621826171875,
+      "loss": 0.0481,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.067463755607605,
+      "rewards/margins": 2.0409655570983887,
+      "rewards/rejected": -3.108428955078125,
       "step": 260
     },
     {
       "epoch": 0.67,
       "learning_rate": 1.51347077992983e-07,
-      "logits/chosen": -2.301753520965576,
-      "logits/rejected": -2.2307753562927246,
-      "logps/chosen": -540.0494384765625,
-      "logps/rejected": -464.4815368652344,
-      "loss": 0.0388,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.3635069131851196,
-      "rewards/margins": 2.057849645614624,
-      "rewards/rejected": -3.421356678009033,
+      "logits/chosen": -2.3267765045166016,
+      "logits/rejected": -2.231224775314331,
+      "logps/chosen": -548.4693603515625,
+      "logps/rejected": -429.4169921875,
+      "loss": 0.0427,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -1.0470257997512817,
+      "rewards/margins": 2.1003241539001465,
+      "rewards/rejected": -3.1473500728607178,
       "step": 270
     },
     {
       "epoch": 0.69,
       "learning_rate": 1.3191302063739906e-07,
-      "logits/chosen": -2.272491931915283,
-      "logits/rejected": -2.201686382293701,
-      "logps/chosen": -516.8363037109375,
-      "logps/rejected": -447.940185546875,
-      "loss": 0.0355,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.376404881477356,
-      "rewards/margins": 1.9720706939697266,
-      "rewards/rejected": -3.348475694656372,
+      "logits/chosen": -2.334428310394287,
+      "logits/rejected": -2.2525620460510254,
+      "logps/chosen": -515.8858642578125,
+      "logps/rejected": -440.9449768066406,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.1162147521972656,
+      "rewards/margins": 2.022719383239746,
+      "rewards/rejected": -3.1389341354370117,
       "step": 280
     },
     {
       "epoch": 0.72,
       "learning_rate": 1.1335804528119475e-07,
-      "logits/chosen": -2.3611927032470703,
-      "logits/rejected": -2.2538418769836426,
-      "logps/chosen": -550.6755981445312,
-      "logps/rejected": -446.7557678222656,
-      "loss": 0.0427,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.1491652727127075,
-      "rewards/margins": 2.311260461807251,
-      "rewards/rejected": -3.460425615310669,
+      "logits/chosen": -2.344043016433716,
+      "logits/rejected": -2.2616305351257324,
+      "logps/chosen": -535.1497192382812,
+      "logps/rejected": -425.8858337402344,
+      "loss": 0.0575,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.9090016484260559,
+      "rewards/margins": 2.22568678855896,
+      "rewards/rejected": -3.13468861579895,
       "step": 290
     },
     {
       "epoch": 0.74,
       "learning_rate": 9.582028184286423e-08,
-      "logits/chosen": -2.2688956260681152,
-      "logits/rejected": -2.2058238983154297,
-      "logps/chosen": -497.08392333984375,
-      "logps/rejected": -457.6719665527344,
-      "loss": 0.0469,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.372286319732666,
-      "rewards/margins": 1.9885644912719727,
-      "rewards/rejected": -3.3608508110046387,
+      "logits/chosen": -2.3820149898529053,
+      "logits/rejected": -2.352027416229248,
+      "logps/chosen": -549.5676879882812,
+      "logps/rejected": -470.928466796875,
+      "loss": 0.0489,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.1177103519439697,
+      "rewards/margins": 2.212918758392334,
+      "rewards/rejected": -3.3306288719177246,
       "step": 300
     },
     {
       "epoch": 0.76,
       "learning_rate": 7.943028774907065e-08,
-      "logits/chosen": -2.2832024097442627,
-      "logits/rejected": -2.215949535369873,
-      "logps/chosen": -493.197021484375,
-      "logps/rejected": -424.94073486328125,
-      "loss": 0.0521,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.0872572660446167,
-      "rewards/margins": 1.9770004749298096,
-      "rewards/rejected": -3.0642576217651367,
+      "logits/chosen": -2.3420827388763428,
+      "logits/rejected": -2.2547786235809326,
+      "logps/chosen": -522.0978393554688,
+      "logps/rejected": -427.39801025390625,
+      "loss": 0.0367,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.0316474437713623,
+      "rewards/margins": 2.1541504859924316,
+      "rewards/rejected": -3.185797929763794,
       "step": 310
     },
     {
       "epoch": 0.79,
       "learning_rate": 6.431007601814637e-08,
-      "logits/chosen": -2.300029993057251,
-      "logits/rejected": -2.244724750518799,
-      "logps/chosen": -453.86981201171875,
-      "logps/rejected": -416.4488220214844,
-      "loss": 0.0435,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -1.2616424560546875,
-      "rewards/margins": 1.8299839496612549,
-      "rewards/rejected": -3.0916266441345215,
+      "logits/chosen": -2.299654006958008,
+      "logits/rejected": -2.211099863052368,
+      "logps/chosen": -534.5721435546875,
+      "logps/rejected": -457.3140563964844,
+      "loss": 0.0373,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.5419642925262451,
+      "rewards/margins": 1.9639537334442139,
+      "rewards/rejected": -3.50591778755188,
       "step": 320
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.0572206951246e-08,
-      "logits/chosen": -2.273084878921509,
-      "logits/rejected": -2.188479423522949,
-      "logps/chosen": -499.99957275390625,
-      "logps/rejected": -427.8397521972656,
-      "loss": 0.046,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.324517011642456,
-      "rewards/margins": 1.8908493518829346,
-      "rewards/rejected": -3.215366840362549,
+      "logits/chosen": -2.315218687057495,
+      "logits/rejected": -2.2397372722625732,
+      "logps/chosen": -560.6382446289062,
+      "logps/rejected": -482.7513732910156,
+      "loss": 0.0347,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.36269211769104,
+      "rewards/margins": 2.1836559772491455,
+      "rewards/rejected": -3.5463478565216064,
       "step": 330
     },
     {
       "epoch": 0.84,
       "learning_rate": 3.831895019292897e-08,
-      "logits/chosen": -2.3347887992858887,
-      "logits/rejected": -2.2554373741149902,
-      "logps/chosen": -548.2373046875,
-      "logps/rejected": -488.4436950683594,
-      "loss": 0.0447,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.1382402181625366,
-      "rewards/margins": 2.5688745975494385,
-      "rewards/rejected": -3.7071146965026855,
+      "logits/chosen": -2.3570194244384766,
+      "logits/rejected": -2.2624592781066895,
+      "logps/chosen": -610.1851806640625,
+      "logps/rejected": -510.604248046875,
+      "loss": 0.0387,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.4758695363998413,
+      "rewards/margins": 2.291604518890381,
+      "rewards/rejected": -3.767474412918091,
       "step": 340
     },
     {
       "epoch": 0.86,
       "learning_rate": 2.764152339909756e-08,
-      "logits/chosen": -2.279545545578003,
-      "logits/rejected": -2.195236921310425,
-      "logps/chosen": -536.1104736328125,
-      "logps/rejected": -410.069091796875,
-      "loss": 0.0451,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -1.1084119081497192,
-      "rewards/margins": 2.0049660205841064,
-      "rewards/rejected": -3.1133780479431152,
+      "logits/chosen": -2.341916561126709,
+      "logits/rejected": -2.257068157196045,
+      "logps/chosen": -559.6868286132812,
+      "logps/rejected": -464.418701171875,
+      "loss": 0.0324,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.2571017742156982,
+      "rewards/margins": 2.2919716835021973,
+      "rewards/rejected": -3.5490734577178955,
       "step": 350
     },
     {
       "epoch": 0.89,
       "learning_rate": 1.861941317991664e-08,
-      "logits/chosen": -2.3270392417907715,
-      "logits/rejected": -2.2122137546539307,
-      "logps/chosen": -558.4454956054688,
-      "logps/rejected": -451.0440979003906,
-      "loss": 0.0438,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -1.0215061902999878,
-      "rewards/margins": 2.3432884216308594,
-      "rewards/rejected": -3.3647942543029785,
+      "logits/chosen": -2.354764699935913,
+      "logits/rejected": -2.2795515060424805,
+      "logps/chosen": -564.1604614257812,
+      "logps/rejected": -485.11944580078125,
+      "loss": 0.0352,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.2872124910354614,
+      "rewards/margins": 2.3846843242645264,
+      "rewards/rejected": -3.6718971729278564,
       "step": 360
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.13197833728636e-08,
-      "logits/chosen": -2.287353038787842,
-      "logits/rejected": -2.2039966583251953,
-      "logps/chosen": -510.3124084472656,
-      "logps/rejected": -463.5337829589844,
-      "loss": 0.0384,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -1.076358437538147,
-      "rewards/margins": 2.4394357204437256,
-      "rewards/rejected": -3.515794277191162,
+      "logits/chosen": -2.319873809814453,
+      "logits/rejected": -2.2500181198120117,
+      "logps/chosen": -572.5164794921875,
+      "logps/rejected": -500.48760986328125,
+      "loss": 0.0369,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.1619113683700562,
+      "rewards/margins": 2.508751392364502,
+      "rewards/rejected": -3.6706624031066895,
       "step": 370
     },
     {
       "epoch": 0.94,
       "learning_rate": 5.79697505093521e-09,
-      "logits/chosen": -2.2827162742614746,
-      "logits/rejected": -2.1960558891296387,
-      "logps/chosen": -526.398681640625,
-      "logps/rejected": -435.3667907714844,
-      "loss": 0.0513,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.2410168647766113,
-      "rewards/margins": 2.0592200756073,
-      "rewards/rejected": -3.3002371788024902,
+      "logits/chosen": -2.334282636642456,
+      "logits/rejected": -2.2317328453063965,
+      "logps/chosen": -550.3582153320312,
+      "logps/rejected": -479.588623046875,
+      "loss": 0.0448,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -1.32636559009552,
+      "rewards/margins": 2.201442003250122,
+      "rewards/rejected": -3.5278077125549316,
       "step": 380
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.092101988131256e-09,
-      "logits/chosen": -2.332719087600708,
-      "logits/rejected": -2.206247568130493,
-      "logps/chosen": -562.2529296875,
-      "logps/rejected": -459.3744201660156,
-      "loss": 0.0425,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -0.9911445379257202,
-      "rewards/margins": 2.511375665664673,
-      "rewards/rejected": -3.5025200843811035,
+      "logits/chosen": -2.3516108989715576,
+      "logits/rejected": -2.2636828422546387,
+      "logps/chosen": -587.2901611328125,
+      "logps/rejected": -482.85198974609375,
+      "loss": 0.0351,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.2399132251739502,
+      "rewards/margins": 2.379091739654541,
+      "rewards/rejected": -3.619004726409912,
       "step": 390
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.327445937151673e-10,
-      "logits/chosen": -2.319377899169922,
-      "logits/rejected": -2.2350213527679443,
-      "logps/chosen": -558.1907958984375,
-      "logps/rejected": -479.43963623046875,
-      "loss": 0.0443,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.0957250595092773,
-      "rewards/margins": 2.4137606620788574,
-      "rewards/rejected": -3.5094857215881348,
+      "logits/chosen": -2.351771831512451,
+      "logits/rejected": -2.2641963958740234,
+      "logps/chosen": -545.67822265625,
+      "logps/rejected": -469.13568115234375,
+      "loss": 0.039,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.5256686210632324,
+      "rewards/margins": 2.0123348236083984,
+      "rewards/rejected": -3.538003444671631,
       "step": 400
     },
     {
       "epoch": 1.0,
       "step": 405,
       "total_flos": 0.0,
-      "train_loss": 0.08838768833213383,
-      "train_runtime": 3226.1286,
-      "train_samples_per_second": 16.086,
-      "train_steps_per_second": 0.126
+      "train_loss": 0.08520494085035206,
+      "train_runtime": 3257.6401,
+      "train_samples_per_second": 15.93,
+      "train_steps_per_second": 0.124
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e6fdb171dbedd3c33d747caa4f7d5a15337333684182e1da435907164da14510
+oid sha256:b7deba97c0b442a6672b07e7b7c225827b2e1a1e18ad8c3c0925d66cddd6281e
 size 5944