Model save

README.md

@@ -37,7 +37,7 @@ The following hyperparameters were used during training:
 - learning_rate: 5e-07
 - train_batch_size: 8
 - eval_batch_size: 8
-- seed: 4
+- seed: 5
 - distributed_type: multi-GPU
 - num_devices: 8
 - gradient_accumulation_steps: 2

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.14271833528707056,
-    "train_runtime": 3205.3137,
+    "train_loss": 0.15393988585766452,
+    "train_runtime": 3212.9353,
     "train_samples": 51894,
-    "train_samples_per_second": 16.19,
+    "train_samples_per_second": 16.152,
     "train_steps_per_second": 0.126
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1adf15ad14a70a15f2da17273d04658d31bcb80a7cf71fd8739ea6ece32a9dc0
+oid sha256:5db9799c54db506dd6903dff305f805519c471c8230ee55006d50eaef6fcf71d
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d6840860a9c0b139fea3960dfc4ec0c766e717e85de1ad3968cc0068318f6714
+oid sha256:41c77f02ad4513a905d9fda2f3c23b348a672959edfad115e308cb5ada0dc69b
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:741f22e485d0728e3f4f8f4f6ed06bb1f8dbf7720e43f83f9cde68c7f1dd445c
+oid sha256:ba5c5aaecef82703271e9f4ba884d0379738164b9d77f821c0eac7bfcacdf762
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.14271833528707056,
-    "train_runtime": 3205.3137,
+    "train_loss": 0.15393988585766452,
+    "train_runtime": 3212.9353,
     "train_samples": 51894,
-    "train_samples_per_second": 16.19,
+    "train_samples_per_second": 16.152,
     "train_steps_per_second": 0.126
 }

trainer_state.json

@@ -11,11 +11,11 @@
     {
       "epoch": 0.0,
       "learning_rate": 1.2195121951219512e-08,
-      "logits/chosen": -2.8681135177612305,
-      "logits/rejected": -2.8858838081359863,
-      "logps/chosen": -518.1907958984375,
-      "logps/rejected": -109.31971740722656,
-      "loss": 0.5582,
+      "logits/chosen": -2.8695335388183594,
+      "logits/rejected": -2.8522377014160156,
+      "logps/chosen": -537.80126953125,
+      "logps/rejected": -108.91968536376953,
+      "loss": 0.5601,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,570 +25,570 @@
     {
       "epoch": 0.02,
       "learning_rate": 1.219512195121951e-07,
-      "logits/chosen": -2.7987241744995117,
-      "logits/rejected": -2.7522923946380615,
-      "logps/chosen": -434.1634521484375,
-      "logps/rejected": -114.18580627441406,
-      "loss": 0.5551,
-      "rewards/accuracies": 0.5069444179534912,
-      "rewards/chosen": 0.0007126106647774577,
-      "rewards/margins": 0.0012787013547495008,
-      "rewards/rejected": -0.000566090689972043,
+      "logits/chosen": -2.8006718158721924,
+      "logits/rejected": -2.7512741088867188,
+      "logps/chosen": -339.0959167480469,
+      "logps/rejected": -113.41566467285156,
+      "loss": 0.5529,
+      "rewards/accuracies": 0.5763888955116272,
+      "rewards/chosen": 0.0014224686892703176,
+      "rewards/margins": 0.0021842769347131252,
+      "rewards/rejected": -0.0007618081872351468,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.439024390243902e-07,
-      "logits/chosen": -2.820725917816162,
-      "logits/rejected": -2.7975618839263916,
-      "logps/chosen": -417.19793701171875,
-      "logps/rejected": -118.11334228515625,
-      "loss": 0.5389,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.02059975638985634,
-      "rewards/margins": 0.03748037666082382,
-      "rewards/rejected": -0.016880620270967484,
+      "logits/chosen": -2.815359592437744,
+      "logits/rejected": -2.8071651458740234,
+      "logps/chosen": -435.11260986328125,
+      "logps/rejected": -116.19319152832031,
+      "loss": 0.5366,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 0.022735606878995895,
+      "rewards/margins": 0.04114392399787903,
+      "rewards/rejected": -0.018408317118883133,
       "step": 20
     },
     {
       "epoch": 0.07,
       "learning_rate": 3.6585365853658536e-07,
-      "logits/chosen": -2.654127597808838,
-      "logits/rejected": -2.641984224319458,
-      "logps/chosen": -398.6944580078125,
-      "logps/rejected": -126.52714538574219,
-      "loss": 0.4667,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 0.07748116552829742,
-      "rewards/margins": 0.2089122235774994,
-      "rewards/rejected": -0.13143105804920197,
+      "logits/chosen": -2.7216827869415283,
+      "logits/rejected": -2.68468976020813,
+      "logps/chosen": -437.6546325683594,
+      "logps/rejected": -142.09629821777344,
+      "loss": 0.4702,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.08811721950769424,
+      "rewards/margins": 0.21381433308124542,
+      "rewards/rejected": -0.12569710612297058,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.878048780487804e-07,
-      "logits/chosen": -2.538156509399414,
-      "logits/rejected": -2.523983955383301,
-      "logps/chosen": -384.8261413574219,
-      "logps/rejected": -170.25672912597656,
-      "loss": 0.3743,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.021691232919692993,
-      "rewards/margins": 0.43257051706314087,
-      "rewards/rejected": -0.4108792245388031,
+      "logits/chosen": -2.589413642883301,
+      "logits/rejected": -2.5702805519104004,
+      "logps/chosen": -415.1499938964844,
+      "logps/rejected": -170.38934326171875,
+      "loss": 0.3716,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 0.013831913471221924,
+      "rewards/margins": 0.4765930771827698,
+      "rewards/rejected": -0.4627610743045807,
       "step": 40
     },
     {
       "epoch": 0.12,
       "learning_rate": 4.992461696250783e-07,
-      "logits/chosen": -2.4283156394958496,
-      "logits/rejected": -2.394186496734619,
-      "logps/chosen": -435.05364990234375,
-      "logps/rejected": -218.2919464111328,
-      "loss": 0.2788,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.15316179394721985,
-      "rewards/margins": 0.794213056564331,
-      "rewards/rejected": -0.947374701499939,
+      "logits/chosen": -2.469589948654175,
+      "logits/rejected": -2.456850528717041,
+      "logps/chosen": -421.22100830078125,
+      "logps/rejected": -211.1681671142578,
+      "loss": 0.281,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.12107028812170029,
+      "rewards/margins": 0.793075680732727,
+      "rewards/rejected": -0.9141460657119751,
       "step": 50
     },
     {
       "epoch": 0.15,
       "learning_rate": 4.966461721767899e-07,
-      "logits/chosen": -2.3560433387756348,
-      "logits/rejected": -2.3106768131256104,
-      "logps/chosen": -437.6812438964844,
-      "logps/rejected": -239.7769012451172,
-      "loss": 0.2266,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.3268889784812927,
-      "rewards/margins": 0.9203512072563171,
-      "rewards/rejected": -1.2472401857376099,
+      "logits/chosen": -2.380545139312744,
+      "logits/rejected": -2.347712755203247,
+      "logps/chosen": -379.7464599609375,
+      "logps/rejected": -211.6461944580078,
+      "loss": 0.1977,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.34397271275520325,
+      "rewards/margins": 0.8727855682373047,
+      "rewards/rejected": -1.216758370399475,
       "step": 60
     },
     {
       "epoch": 0.17,
       "learning_rate": 4.922100518015975e-07,
-      "logits/chosen": -2.331510543823242,
-      "logits/rejected": -2.284951686859131,
-      "logps/chosen": -424.73333740234375,
-      "logps/rejected": -268.073486328125,
-      "loss": 0.1727,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.44431695342063904,
-      "rewards/margins": 1.1743532419204712,
-      "rewards/rejected": -1.618670105934143,
+      "logits/chosen": -2.382647752761841,
+      "logits/rejected": -2.3510990142822266,
+      "logps/chosen": -435.24627685546875,
+      "logps/rejected": -257.90399169921875,
+      "loss": 0.1791,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3540270924568176,
+      "rewards/margins": 1.2106399536132812,
+      "rewards/rejected": -1.5646671056747437,
       "step": 70
     },
     {
       "epoch": 0.2,
       "learning_rate": 4.859708325770919e-07,
-      "logits/chosen": -2.393051862716675,
-      "logits/rejected": -2.33725643157959,
-      "logps/chosen": -468.8316345214844,
-      "logps/rejected": -316.95086669921875,
-      "loss": 0.1441,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.5438361167907715,
-      "rewards/margins": 1.4411370754241943,
-      "rewards/rejected": -1.9849731922149658,
+      "logits/chosen": -2.32224178314209,
+      "logits/rejected": -2.277831554412842,
+      "logps/chosen": -448.172607421875,
+      "logps/rejected": -297.2752685546875,
+      "loss": 0.1592,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.37227025628089905,
+      "rewards/margins": 1.5082250833511353,
+      "rewards/rejected": -1.880495309829712,
       "step": 80
     },
     {
       "epoch": 0.22,
       "learning_rate": 4.779749614980225e-07,
-      "logits/chosen": -2.372483491897583,
-      "logits/rejected": -2.3344340324401855,
-      "logps/chosen": -466.607421875,
-      "logps/rejected": -327.7405700683594,
-      "loss": 0.1325,
+      "logits/chosen": -2.3661742210388184,
+      "logits/rejected": -2.33076810836792,
+      "logps/chosen": -458.77081298828125,
+      "logps/rejected": -333.3035888671875,
+      "loss": 0.1222,
       "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.45430439710617065,
-      "rewards/margins": 1.7159345149993896,
-      "rewards/rejected": -2.170238733291626,
+      "rewards/chosen": -0.7345383763313293,
+      "rewards/margins": 1.5108360052108765,
+      "rewards/rejected": -2.2453744411468506,
       "step": 90
     },
     {
       "epoch": 0.25,
       "learning_rate": 4.682819627081427e-07,
-      "logits/chosen": -2.343848466873169,
-      "logits/rejected": -2.300412654876709,
-      "logps/chosen": -498.5199279785156,
-      "logps/rejected": -356.87860107421875,
-      "loss": 0.1141,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.500357449054718,
-      "rewards/margins": 1.946504831314087,
-      "rewards/rejected": -2.446861743927002,
+      "logits/chosen": -2.335977554321289,
+      "logits/rejected": -2.281729221343994,
+      "logps/chosen": -503.5619201660156,
+      "logps/rejected": -364.14862060546875,
+      "loss": 0.1166,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.8016496896743774,
+      "rewards/margins": 1.7226619720458984,
+      "rewards/rejected": -2.5243115425109863,
       "step": 100
     },
     {
       "epoch": 0.27,
       "learning_rate": 4.569639943810477e-07,
-      "logits/chosen": -2.2943625450134277,
-      "logits/rejected": -2.2243731021881104,
-      "logps/chosen": -475.5547790527344,
-      "logps/rejected": -396.2657165527344,
-      "loss": 0.0898,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.1803233623504639,
-      "rewards/margins": 1.67868971824646,
-      "rewards/rejected": -2.859013080596924,
+      "logits/chosen": -2.3253397941589355,
+      "logits/rejected": -2.2732253074645996,
+      "logps/chosen": -540.4392700195312,
+      "logps/rejected": -381.0623474121094,
+      "loss": 0.0958,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.9782747030258179,
+      "rewards/margins": 1.7351051568984985,
+      "rewards/rejected": -2.7133796215057373,
       "step": 110
     },
     {
       "epoch": 0.3,
       "learning_rate": 4.4410531154874543e-07,
-      "logits/chosen": -2.313946485519409,
-      "logits/rejected": -2.2655868530273438,
-      "logps/chosen": -490.6239318847656,
-      "logps/rejected": -387.259765625,
-      "loss": 0.0963,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -0.9641086459159851,
-      "rewards/margins": 1.7083423137664795,
-      "rewards/rejected": -2.6724514961242676,
+      "logits/chosen": -2.319655418395996,
+      "logits/rejected": -2.269864082336426,
+      "logps/chosen": -528.8589477539062,
+      "logps/rejected": -398.9247741699219,
+      "loss": 0.0964,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.9485586881637573,
+      "rewards/margins": 1.865126609802246,
+      "rewards/rejected": -2.813685178756714,
       "step": 120
     },
     {
       "epoch": 0.32,
       "learning_rate": 4.298016388768561e-07,
-      "logits/chosen": -2.270263910293579,
-      "logits/rejected": -2.234313488006592,
-      "logps/chosen": -485.26605224609375,
-      "logps/rejected": -381.05609130859375,
-      "loss": 0.0985,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.1396474838256836,
-      "rewards/margins": 1.5876901149749756,
-      "rewards/rejected": -2.727337598800659,
+      "logits/chosen": -2.3149333000183105,
+      "logits/rejected": -2.247347116470337,
+      "logps/chosen": -502.3334045410156,
+      "logps/rejected": -388.2889404296875,
+      "loss": 0.1271,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.7547552585601807,
+      "rewards/margins": 1.8656883239746094,
+      "rewards/rejected": -2.620443344116211,
       "step": 130
     },
     {
       "epoch": 0.35,
       "learning_rate": 4.1415945805573005e-07,
-      "logits/chosen": -2.252371311187744,
-      "logits/rejected": -2.2070510387420654,
-      "logps/chosen": -512.5037841796875,
-      "logps/rejected": -396.6216735839844,
-      "loss": 0.0999,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -0.8260290026664734,
-      "rewards/margins": 1.9724273681640625,
-      "rewards/rejected": -2.7984564304351807,
+      "logits/chosen": -2.3083794116973877,
+      "logits/rejected": -2.263607978820801,
+      "logps/chosen": -507.48797607421875,
+      "logps/rejected": -399.94720458984375,
+      "loss": 0.1077,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -0.6250351071357727,
+      "rewards/margins": 2.218956470489502,
+      "rewards/rejected": -2.843991756439209,
       "step": 140
     },
     {
       "epoch": 0.37,
       "learning_rate": 3.972952151123984e-07,
-      "logits/chosen": -2.2577097415924072,
-      "logits/rejected": -2.193302869796753,
-      "logps/chosen": -537.8040771484375,
-      "logps/rejected": -436.078369140625,
-      "loss": 0.0769,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.2730529308319092,
-      "rewards/margins": 1.9622818231582642,
-      "rewards/rejected": -3.235334873199463,
+      "logits/chosen": -2.271951198577881,
+      "logits/rejected": -2.212960720062256,
+      "logps/chosen": -497.3548889160156,
+      "logps/rejected": -402.63250732421875,
+      "loss": 0.1228,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -0.729441225528717,
+      "rewards/margins": 2.022250175476074,
+      "rewards/rejected": -2.7516913414001465,
       "step": 150
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.793344535444142e-07,
-      "logits/chosen": -2.242403984069824,
-      "logits/rejected": -2.1921000480651855,
-      "logps/chosen": -575.3533325195312,
-      "logps/rejected": -464.304443359375,
-      "loss": 0.0831,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.6633422374725342,
-      "rewards/margins": 1.7485841512680054,
-      "rewards/rejected": -3.41192626953125,
+      "logits/chosen": -2.3041064739227295,
+      "logits/rejected": -2.241098642349243,
+      "logps/chosen": -538.1851806640625,
+      "logps/rejected": -396.97491455078125,
+      "loss": 0.108,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.7181452512741089,
+      "rewards/margins": 2.0927627086639404,
+      "rewards/rejected": -2.810908079147339,
       "step": 160
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.604108797288461e-07,
-      "logits/chosen": -2.2311742305755615,
-      "logits/rejected": -2.215085983276367,
-      "logps/chosen": -447.465576171875,
-      "logps/rejected": -357.7833557128906,
-      "loss": 0.1106,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.085645318031311,
-      "rewards/margins": 1.4540026187896729,
-      "rewards/rejected": -2.5396475791931152,
+      "logits/chosen": -2.2430710792541504,
+      "logits/rejected": -2.186084270477295,
+      "logps/chosen": -503.18536376953125,
+      "logps/rejected": -424.78863525390625,
+      "loss": 0.0967,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -0.9911813735961914,
+      "rewards/margins": 2.0638327598571777,
+      "rewards/rejected": -3.055014133453369,
       "step": 170
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.40665367563858e-07,
-      "logits/chosen": -2.283315658569336,
-      "logits/rejected": -2.2039597034454346,
-      "logps/chosen": -531.3043212890625,
-      "logps/rejected": -403.9816589355469,
-      "loss": 0.1024,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.712272047996521,
-      "rewards/margins": 2.0688250064849854,
-      "rewards/rejected": -2.781096935272217,
+      "logits/chosen": -2.2562966346740723,
+      "logits/rejected": -2.17518949508667,
+      "logps/chosen": -452.6295471191406,
+      "logps/rejected": -332.67901611328125,
+      "loss": 0.1233,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.7905207276344299,
+      "rewards/margins": 1.5971567630767822,
+      "rewards/rejected": -2.3876776695251465,
       "step": 180
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.202449097526798e-07,
-      "logits/chosen": -2.248110294342041,
-      "logits/rejected": -2.2098121643066406,
-      "logps/chosen": -515.7914428710938,
-      "logps/rejected": -415.6000061035156,
-      "loss": 0.087,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.106316328048706,
-      "rewards/margins": 1.8899608850479126,
-      "rewards/rejected": -2.9962775707244873,
+      "logits/chosen": -2.253117084503174,
+      "logits/rejected": -2.1730282306671143,
+      "logps/chosen": -486.3101501464844,
+      "logps/rejected": -395.2755432128906,
+      "loss": 0.1155,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.9943073987960815,
+      "rewards/margins": 1.831038475036621,
+      "rewards/rejected": -2.825345993041992,
       "step": 190
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.993015235369905e-07,
-      "logits/chosen": -2.2014596462249756,
-      "logits/rejected": -2.1617417335510254,
-      "logps/chosen": -526.7307739257812,
-      "logps/rejected": -452.57415771484375,
-      "loss": 0.0786,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.4753978252410889,
-      "rewards/margins": 1.9113044738769531,
-      "rewards/rejected": -3.386702060699463,
+      "logits/chosen": -2.2392661571502686,
+      "logits/rejected": -2.1586246490478516,
+      "logps/chosen": -517.7527465820312,
+      "logps/rejected": -401.18096923828125,
+      "loss": 0.1106,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.8338342905044556,
+      "rewards/margins": 1.99163818359375,
+      "rewards/rejected": -2.825472354888916,
       "step": 200
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7799111902582693e-07,
-      "logits/chosen": -2.2283272743225098,
-      "logits/rejected": -2.1539306640625,
-      "logps/chosen": -547.9268798828125,
-      "logps/rejected": -442.84716796875,
-      "loss": 0.0712,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.0882245302200317,
-      "rewards/margins": 2.2409071922302246,
-      "rewards/rejected": -3.329131603240967,
+      "logits/chosen": -2.195733070373535,
+      "logits/rejected": -2.118263006210327,
+      "logps/chosen": -480.4593200683594,
+      "logps/rejected": -390.95550537109375,
+      "loss": 0.1078,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.9628806114196777,
+      "rewards/margins": 1.8329284191131592,
+      "rewards/rejected": -2.795809268951416,
       "step": 210
     },
     {
       "epoch": 0.54,
       "learning_rate": 2.564723385445869e-07,
-      "logits/chosen": -2.300952911376953,
-      "logits/rejected": -2.2398998737335205,
-      "logps/chosen": -535.0980224609375,
-      "logps/rejected": -421.75799560546875,
-      "loss": 0.1106,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.0268903970718384,
-      "rewards/margins": 1.9092063903808594,
-      "rewards/rejected": -2.9360969066619873,
+      "logits/chosen": -2.1850428581237793,
+      "logits/rejected": -2.105459213256836,
+      "logps/chosen": -511.00762939453125,
+      "logps/rejected": -415.4864807128906,
+      "loss": 0.1227,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -0.7178577184677124,
+      "rewards/margins": 2.1069018840789795,
+      "rewards/rejected": -2.8247594833374023,
       "step": 220
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3490537564442845e-07,
-      "logits/chosen": -2.2512927055358887,
-      "logits/rejected": -2.1920220851898193,
-      "logps/chosen": -481.20947265625,
-      "logps/rejected": -388.5928649902344,
-      "loss": 0.1072,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.936708927154541,
-      "rewards/margins": 1.7015047073364258,
-      "rewards/rejected": -2.638213634490967,
+      "logits/chosen": -2.2125039100646973,
+      "logits/rejected": -2.1424784660339355,
+      "logps/chosen": -482.50213623046875,
+      "logps/rejected": -388.6355285644531,
+      "loss": 0.1171,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.7609508037567139,
+      "rewards/margins": 1.9426014423370361,
+      "rewards/rejected": -2.703552722930908,
       "step": 230
     },
     {
       "epoch": 0.59,
       "learning_rate": 2.1345078256378801e-07,
-      "logits/chosen": -2.2840850353240967,
-      "logits/rejected": -2.223998546600342,
-      "logps/chosen": -525.611083984375,
-      "logps/rejected": -408.7984924316406,
-      "loss": 0.0977,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.13532555103302,
-      "rewards/margins": 1.788002371788025,
-      "rewards/rejected": -2.923327922821045,
+      "logits/chosen": -2.201641798019409,
+      "logits/rejected": -2.111548900604248,
+      "logps/chosen": -484.5047912597656,
+      "logps/rejected": -418.87945556640625,
+      "loss": 0.0928,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.7977638244628906,
+      "rewards/margins": 2.190457344055176,
+      "rewards/rejected": -2.9882209300994873,
       "step": 240
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.9226827501969865e-07,
-      "logits/chosen": -2.2815299034118652,
-      "logits/rejected": -2.2289319038391113,
-      "logps/chosen": -510.8829040527344,
-      "logps/rejected": -408.6348571777344,
-      "loss": 0.1022,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -0.9617199897766113,
-      "rewards/margins": 2.0366153717041016,
-      "rewards/rejected": -2.998335361480713,
+      "logits/chosen": -2.181918144226074,
+      "logits/rejected": -2.0827794075012207,
+      "logps/chosen": -518.0274658203125,
+      "logps/rejected": -448.84912109375,
+      "loss": 0.1207,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.8979538083076477,
+      "rewards/margins": 2.4133543968200684,
+      "rewards/rejected": -3.3113083839416504,
       "step": 250
     },
     {
       "epoch": 0.64,
       "learning_rate": 1.715155432264775e-07,
-      "logits/chosen": -2.257248640060425,
-      "logits/rejected": -2.162848949432373,
-      "logps/chosen": -476.12603759765625,
-      "logps/rejected": -383.61993408203125,
-      "loss": 0.0978,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.8627266883850098,
-      "rewards/margins": 1.9056837558746338,
-      "rewards/rejected": -2.7684102058410645,
+      "logits/chosen": -2.1671414375305176,
+      "logits/rejected": -2.077252149581909,
+      "logps/chosen": -475.8624572753906,
+      "logps/rejected": -393.2193298339844,
+      "loss": 0.1295,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.9034333229064941,
+      "rewards/margins": 1.9351081848144531,
+      "rewards/rejected": -2.8385415077209473,
       "step": 260
     },
     {
       "epoch": 0.67,
       "learning_rate": 1.51347077992983e-07,
-      "logits/chosen": -2.2847957611083984,
-      "logits/rejected": -2.197834014892578,
-      "logps/chosen": -536.037109375,
-      "logps/rejected": -406.7389221191406,
-      "loss": 0.0889,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.9227035641670227,
-      "rewards/margins": 1.997865915298462,
-      "rewards/rejected": -2.9205691814422607,
+      "logits/chosen": -2.1908605098724365,
+      "logits/rejected": -2.0970406532287598,
+      "logps/chosen": -487.83563232421875,
+      "logps/rejected": -409.66546630859375,
+      "loss": 0.1009,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.9220987558364868,
+      "rewards/margins": 1.9496616125106812,
+      "rewards/rejected": -2.871760845184326,
       "step": 270
     },
     {
       "epoch": 0.69,
       "learning_rate": 1.3191302063739906e-07,
-      "logits/chosen": -2.288609743118286,
-      "logits/rejected": -2.211268424987793,
-      "logps/chosen": -509.29156494140625,
-      "logps/rejected": -429.5159606933594,
-      "loss": 0.1021,
+      "logits/chosen": -2.190361261367798,
+      "logits/rejected": -2.0984115600585938,
+      "logps/chosen": -508.9253845214844,
+      "logps/rejected": -422.1849060058594,
+      "loss": 0.0899,
       "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.05027174949646,
-      "rewards/margins": 1.9743726253509521,
-      "rewards/rejected": -3.024644136428833,
+      "rewards/chosen": -1.112410545349121,
+      "rewards/margins": 1.9581248760223389,
+      "rewards/rejected": -3.070535659790039,
       "step": 280
     },
     {
       "epoch": 0.72,
       "learning_rate": 1.1335804528119475e-07,
-      "logits/chosen": -2.2901360988616943,
-      "logits/rejected": -2.2138819694519043,
-      "logps/chosen": -530.1524658203125,
-      "logps/rejected": -411.786376953125,
-      "loss": 0.1097,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -0.8590289950370789,
-      "rewards/margins": 2.134665012359619,
-      "rewards/rejected": -2.993694305419922,
+      "logits/chosen": -2.1762964725494385,
+      "logits/rejected": -2.0631372928619385,
+      "logps/chosen": -554.19580078125,
+      "logps/rejected": -427.8614196777344,
+      "loss": 0.084,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.0812236070632935,
+      "rewards/margins": 2.109238862991333,
+      "rewards/rejected": -3.190462827682495,
       "step": 290
     },
     {
       "epoch": 0.74,
       "learning_rate": 9.582028184286423e-08,
-      "logits/chosen": -2.3320322036743164,
-      "logits/rejected": -2.3061583042144775,
-      "logps/chosen": -540.7392578125,
-      "logps/rejected": -452.91729736328125,
-      "loss": 0.0991,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.0294259786605835,
-      "rewards/margins": 2.121091365814209,
-      "rewards/rejected": -3.150517225265503,
+      "logits/chosen": -2.1851372718811035,
+      "logits/rejected": -2.1166481971740723,
+      "logps/chosen": -458.4752502441406,
+      "logps/rejected": -388.1811828613281,
+      "loss": 0.1002,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.9944161176681519,
+      "rewards/margins": 1.8174244165420532,
+      "rewards/rejected": -2.811840534210205,
       "step": 300
     },
     {
       "epoch": 0.76,
       "learning_rate": 7.943028774907065e-08,
-      "logits/chosen": -2.298715353012085,
-      "logits/rejected": -2.212947130203247,
-      "logps/chosen": -518.6718139648438,
-      "logps/rejected": -410.0641174316406,
-      "loss": 0.0799,
+      "logits/chosen": -2.173356533050537,
+      "logits/rejected": -2.102245330810547,
+      "logps/chosen": -476.6107482910156,
+      "logps/rejected": -382.61737060546875,
+      "loss": 0.1083,
       "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.9973870515823364,
-      "rewards/margins": 2.0150723457336426,
-      "rewards/rejected": -3.0124592781066895,
+      "rewards/chosen": -0.709411084651947,
+      "rewards/margins": 2.04428768157959,
+      "rewards/rejected": -2.7536988258361816,
       "step": 310
     },
     {
       "epoch": 0.79,
       "learning_rate": 6.431007601814637e-08,
-      "logits/chosen": -2.262784242630005,
-      "logits/rejected": -2.179920196533203,
-      "logps/chosen": -515.6483154296875,
-      "logps/rejected": -430.5453186035156,
-      "loss": 0.0831,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.3527257442474365,
-      "rewards/margins": 1.8855050802230835,
-      "rewards/rejected": -3.2382309436798096,
+      "logits/chosen": -2.2670390605926514,
+      "logits/rejected": -2.171151638031006,
+      "logps/chosen": -525.9238891601562,
+      "logps/rejected": -428.8578186035156,
+      "loss": 0.0967,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.9797876477241516,
+      "rewards/margins": 2.1045024394989014,
+      "rewards/rejected": -3.084290027618408,
       "step": 320
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.0572206951246e-08,
-      "logits/chosen": -2.2783610820770264,
-      "logits/rejected": -2.207493305206299,
-      "logps/chosen": -545.3638916015625,
-      "logps/rejected": -452.2981872558594,
-      "loss": 0.0755,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.209948182106018,
-      "rewards/margins": 2.0318679809570312,
-      "rewards/rejected": -3.241816759109497,
+      "logits/chosen": -2.2066216468811035,
+      "logits/rejected": -2.0978851318359375,
+      "logps/chosen": -530.3206787109375,
+      "logps/rejected": -422.1839904785156,
+      "loss": 0.0962,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.7779099941253662,
+      "rewards/margins": 2.2902636528015137,
+      "rewards/rejected": -3.068173885345459,
       "step": 330
     },
     {
       "epoch": 0.84,
       "learning_rate": 3.831895019292897e-08,
-      "logits/chosen": -2.311371088027954,
-      "logits/rejected": -2.2250900268554688,
-      "logps/chosen": -602.3062133789062,
-      "logps/rejected": -488.07012939453125,
-      "loss": 0.0825,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.3970798254013062,
-      "rewards/margins": 2.1450533866882324,
-      "rewards/rejected": -3.542132616043091,
+      "logits/chosen": -2.2389538288116455,
+      "logits/rejected": -2.1464695930480957,
+      "logps/chosen": -548.3888549804688,
+      "logps/rejected": -455.033935546875,
+      "loss": 0.0911,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.8341192007064819,
+      "rewards/margins": 2.394991874694824,
+      "rewards/rejected": -3.2291111946105957,
       "step": 340
     },
     {
       "epoch": 0.86,
       "learning_rate": 2.764152339909756e-08,
-      "logits/chosen": -2.3025100231170654,
-      "logits/rejected": -2.220672845840454,
-      "logps/chosen": -548.1705322265625,
-      "logps/rejected": -444.21856689453125,
-      "loss": 0.0776,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -1.1419382095336914,
-      "rewards/margins": 2.2051339149475098,
-      "rewards/rejected": -3.347072124481201,
+      "logits/chosen": -2.2050280570983887,
+      "logits/rejected": -2.1107256412506104,
+      "logps/chosen": -484.3163146972656,
+      "logps/rejected": -392.2890319824219,
+      "loss": 0.1003,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.9081208109855652,
+      "rewards/margins": 1.9725834131240845,
+      "rewards/rejected": -2.880704402923584,
       "step": 350
     },
     {
       "epoch": 0.89,
       "learning_rate": 1.861941317991664e-08,
-      "logits/chosen": -2.3118605613708496,
-      "logits/rejected": -2.235429286956787,
-      "logps/chosen": -549.4410400390625,
-      "logps/rejected": -460.76947021484375,
-      "loss": 0.0783,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -1.140019178390503,
-      "rewards/margins": 2.2883784770965576,
-      "rewards/rejected": -3.4283976554870605,
+      "logits/chosen": -2.1940646171569824,
+      "logits/rejected": -2.1099205017089844,
+      "logps/chosen": -481.4151306152344,
+      "logps/rejected": -388.9750061035156,
+      "loss": 0.1004,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.9678090810775757,
+      "rewards/margins": 1.6934540271759033,
+      "rewards/rejected": -2.6612629890441895,
       "step": 360
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.13197833728636e-08,
-      "logits/chosen": -2.2796883583068848,
-      "logits/rejected": -2.2151246070861816,
-      "logps/chosen": -561.2109375,
-      "logps/rejected": -477.9205627441406,
-      "loss": 0.087,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -1.0488560199737549,
-      "rewards/margins": 2.396134853363037,
-      "rewards/rejected": -3.444990873336792,
+      "logits/chosen": -2.192645788192749,
+      "logits/rejected": -2.105945110321045,
+      "logps/chosen": -499.65277099609375,
+      "logps/rejected": -396.80560302734375,
+      "loss": 0.101,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.8237529993057251,
+      "rewards/margins": 2.0019845962524414,
+      "rewards/rejected": -2.825737714767456,
       "step": 370
     },
     {
       "epoch": 0.94,
       "learning_rate": 5.79697505093521e-09,
-      "logits/chosen": -2.2897768020629883,
-      "logits/rejected": -2.192831039428711,
-      "logps/chosen": -544.9513549804688,
-      "logps/rejected": -457.36456298828125,
-      "loss": 0.0966,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.2722972631454468,
-      "rewards/margins": 2.0332696437835693,
-      "rewards/rejected": -3.3055667877197266,
+      "logits/chosen": -2.1727986335754395,
+      "logits/rejected": -2.0996174812316895,
+      "logps/chosen": -452.10504150390625,
+      "logps/rejected": -356.10546875,
+      "loss": 0.0947,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.6749576330184937,
+      "rewards/margins": 1.8276859521865845,
+      "rewards/rejected": -2.5026438236236572,
       "step": 380
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.092101988131256e-09,
-      "logits/chosen": -2.308785915374756,
-      "logits/rejected": -2.224362850189209,
-      "logps/chosen": -571.474609375,
-      "logps/rejected": -455.10498046875,
-      "loss": 0.0847,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.0817571878433228,
-      "rewards/margins": 2.259777545928955,
-      "rewards/rejected": -3.3415348529815674,
+      "logits/chosen": -2.247488498687744,
+      "logits/rejected": -2.1600608825683594,
+      "logps/chosen": -512.2217407226562,
+      "logps/rejected": -402.2645568847656,
+      "loss": 0.0923,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.7571254968643188,
+      "rewards/margins": 2.047520399093628,
+      "rewards/rejected": -2.8046462535858154,
       "step": 390
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.327445937151673e-10,
-      "logits/chosen": -2.303267002105713,
-      "logits/rejected": -2.222599983215332,
-      "logps/chosen": -541.7862548828125,
-      "logps/rejected": -435.1748962402344,
-      "loss": 0.0925,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.486748456954956,
-      "rewards/margins": 1.7116472721099854,
-      "rewards/rejected": -3.1983959674835205,
+      "logits/chosen": -2.178170680999756,
+      "logits/rejected": -2.1037449836730957,
+      "logps/chosen": -498.6328125,
+      "logps/rejected": -398.8753967285156,
+      "loss": 0.11,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -0.8009634017944336,
+      "rewards/margins": 1.9531538486480713,
+      "rewards/rejected": -2.754117250442505,
       "step": 400
     },
     {
       "epoch": 1.0,
       "step": 405,
       "total_flos": 0.0,
-      "train_loss": 0.14271833528707056,
-      "train_runtime": 3205.3137,
-      "train_samples_per_second": 16.19,
+      "train_loss": 0.15393988585766452,
+      "train_runtime": 3212.9353,
+      "train_samples_per_second": 16.152,
       "train_steps_per_second": 0.126
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6820e20a1ed318ae7453ea4028656b79247178ba1388ae104c90cce83fcf4bd3
+oid sha256:76d0210c59ef372732294ad05bab0ecc4970a2b01efd3577d96d30570f2f6789
 size 5944