Model save

README.md

@@ -37,7 +37,7 @@ The following hyperparameters were used during training:
 - learning_rate: 5e-07
 - train_batch_size: 8
 - eval_batch_size: 8
-- seed: 2
+- seed: 3
 - distributed_type: multi-GPU
 - num_devices: 8
 - gradient_accumulation_steps: 2

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.13281457475674005,
-    "train_runtime": 3219.0486,
+    "train_loss": 0.13438091388455145,
+    "train_runtime": 3218.6044,
     "train_samples": 51894,
-    "train_samples_per_second": 16.121,
+    "train_samples_per_second": 16.123,
     "train_steps_per_second": 0.126
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aab46e8124d9cacc43920ed6943b79f01ec0d55f715b8588dba66c609a2b1d4b
+oid sha256:eb8362a83dbb810604c9bafec85943233773008800e5be078eda7676e6a5c17d
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d07be1111d3b5bc2c14ffdaf4371c594ea83fb6a105d7bbceaeeb9d43d3f17b
+oid sha256:d89dd4954ae3d1cf6d18c20fe5c41eace0d5e2cdd13d0e454a7ba4e26f5a2774
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:74f1ed244a41d859eb13abcea7c5eee6a043a8a2c39a674f91dbbfc0cf7e2a35
+oid sha256:3564f1a5524b5df57acdcc790c6cb97db2f4bf1e30dcaacd947fb7902ebd95fa
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.13281457475674005,
-    "train_runtime": 3219.0486,
+    "train_loss": 0.13438091388455145,
+    "train_runtime": 3218.6044,
     "train_samples": 51894,
-    "train_samples_per_second": 16.121,
+    "train_samples_per_second": 16.123,
     "train_steps_per_second": 0.126
 }

trainer_state.json

@@ -11,11 +11,11 @@
     {
       "epoch": 0.0,
       "learning_rate": 1.2195121951219512e-08,
-      "logits/chosen": -2.8412017822265625,
-      "logits/rejected": -2.798035144805908,
-      "logps/chosen": -481.1488037109375,
-      "logps/rejected": -89.30835723876953,
-      "loss": 0.4176,
+      "logits/chosen": -2.8088459968566895,
+      "logits/rejected": -2.7595884799957275,
+      "logps/chosen": -368.90777587890625,
+      "logps/rejected": -133.10202026367188,
+      "loss": 0.4545,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,570 +25,570 @@
     {
       "epoch": 0.02,
       "learning_rate": 1.219512195121951e-07,
-      "logits/chosen": -2.760530948638916,
-      "logits/rejected": -2.7219057083129883,
-      "logps/chosen": -418.49639892578125,
-      "logps/rejected": -116.74214935302734,
-      "loss": 0.4186,
-      "rewards/accuracies": 0.5486111044883728,
-      "rewards/chosen": 0.0008742791833356023,
-      "rewards/margins": 0.0011628220090642571,
-      "rewards/rejected": -0.0002885429421439767,
+      "logits/chosen": -2.83878231048584,
+      "logits/rejected": -2.824958562850952,
+      "logps/chosen": -433.8194580078125,
+      "logps/rejected": -114.66372680664062,
+      "loss": 0.4227,
+      "rewards/accuracies": 0.5416666865348816,
+      "rewards/chosen": 0.0010460919002071023,
+      "rewards/margins": 0.0013696590904146433,
+      "rewards/rejected": -0.000323567190207541,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.439024390243902e-07,
-      "logits/chosen": -2.785912036895752,
-      "logits/rejected": -2.7400879859924316,
-      "logps/chosen": -395.93634033203125,
-      "logps/rejected": -109.28425598144531,
-      "loss": 0.4276,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.014862718991935253,
-      "rewards/margins": 0.029387209564447403,
-      "rewards/rejected": -0.014524489641189575,
+      "logits/chosen": -2.7982840538024902,
+      "logits/rejected": -2.7652382850646973,
+      "logps/chosen": -436.67694091796875,
+      "logps/rejected": -109.33970642089844,
+      "loss": 0.4254,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.020646633580327034,
+      "rewards/margins": 0.03670011833310127,
+      "rewards/rejected": -0.01605348475277424,
       "step": 20
     },
     {
       "epoch": 0.07,
       "learning_rate": 3.6585365853658536e-07,
-      "logits/chosen": -2.70774507522583,
-      "logits/rejected": -2.6736245155334473,
-      "logps/chosen": -375.3507385253906,
-      "logps/rejected": -121.47686767578125,
-      "loss": 0.4137,
+      "logits/chosen": -2.717103958129883,
+      "logits/rejected": -2.6900384426116943,
+      "logps/chosen": -422.26702880859375,
+      "logps/rejected": -128.1683349609375,
+      "loss": 0.4108,
       "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.06373313069343567,
-      "rewards/margins": 0.1794833242893219,
-      "rewards/rejected": -0.11575017869472504,
+      "rewards/chosen": 0.07094015926122665,
+      "rewards/margins": 0.20011821389198303,
+      "rewards/rejected": -0.12917804718017578,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.878048780487804e-07,
-      "logits/chosen": -2.5397536754608154,
-      "logits/rejected": -2.5152153968811035,
-      "logps/chosen": -426.83306884765625,
-      "logps/rejected": -164.6002960205078,
-      "loss": 0.3837,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": 0.06933724880218506,
-      "rewards/margins": 0.513908326625824,
-      "rewards/rejected": -0.4445711076259613,
+      "logits/chosen": -2.590641498565674,
+      "logits/rejected": -2.5721707344055176,
+      "logps/chosen": -396.3973693847656,
+      "logps/rejected": -139.94859313964844,
+      "loss": 0.3881,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.022975314408540726,
+      "rewards/margins": 0.4287249445915222,
+      "rewards/rejected": -0.4057496190071106,
       "step": 40
     },
     {
       "epoch": 0.12,
       "learning_rate": 4.992461696250783e-07,
-      "logits/chosen": -2.425987958908081,
-      "logits/rejected": -2.42197585105896,
-      "logps/chosen": -366.08343505859375,
-      "logps/rejected": -200.18582153320312,
-      "loss": 0.3126,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.1581335961818695,
-      "rewards/margins": 0.757983386516571,
-      "rewards/rejected": -0.9161170721054077,
+      "logits/chosen": -2.42146635055542,
+      "logits/rejected": -2.394202709197998,
+      "logps/chosen": -445.91644287109375,
+      "logps/rejected": -205.5404052734375,
+      "loss": 0.316,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.03603144362568855,
+      "rewards/margins": 0.8689195513725281,
+      "rewards/rejected": -0.9049509763717651,
       "step": 50
     },
     {
       "epoch": 0.15,
       "learning_rate": 4.966461721767899e-07,
-      "logits/chosen": -2.3848328590393066,
-      "logits/rejected": -2.298985719680786,
-      "logps/chosen": -490.4847106933594,
-      "logps/rejected": -254.928466796875,
-      "loss": 0.2321,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -0.22150389850139618,
-      "rewards/margins": 1.2847968339920044,
-      "rewards/rejected": -1.506300926208496,
+      "logits/chosen": -2.417520046234131,
+      "logits/rejected": -2.3663182258605957,
+      "logps/chosen": -422.27215576171875,
+      "logps/rejected": -255.75912475585938,
+      "loss": 0.2661,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.35185474157333374,
+      "rewards/margins": 0.9545990228652954,
+      "rewards/rejected": -1.3064535856246948,
       "step": 60
     },
     {
       "epoch": 0.17,
       "learning_rate": 4.922100518015975e-07,
-      "logits/chosen": -2.340442180633545,
-      "logits/rejected": -2.2820117473602295,
-      "logps/chosen": -490.87615966796875,
-      "logps/rejected": -321.3214111328125,
-      "loss": 0.1643,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.6543983817100525,
-      "rewards/margins": 1.5231101512908936,
-      "rewards/rejected": -2.17750883102417,
+      "logits/chosen": -2.45034122467041,
+      "logits/rejected": -2.397273540496826,
+      "logps/chosen": -428.19207763671875,
+      "logps/rejected": -294.82501220703125,
+      "loss": 0.198,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.4432826638221741,
+      "rewards/margins": 1.3181250095367432,
+      "rewards/rejected": -1.7614076137542725,
       "step": 70
     },
     {
       "epoch": 0.2,
       "learning_rate": 4.859708325770919e-07,
-      "logits/chosen": -2.3740787506103516,
-      "logits/rejected": -2.3113033771514893,
-      "logps/chosen": -471.11651611328125,
-      "logps/rejected": -314.3768005371094,
-      "loss": 0.182,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.38558533787727356,
-      "rewards/margins": 1.7214374542236328,
-      "rewards/rejected": -2.107023000717163,
+      "logits/chosen": -2.3751111030578613,
+      "logits/rejected": -2.321465015411377,
+      "logps/chosen": -468.4130859375,
+      "logps/rejected": -331.666259765625,
+      "loss": 0.162,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.7735603451728821,
+      "rewards/margins": 1.4863694906234741,
+      "rewards/rejected": -2.25993013381958,
       "step": 80
     },
     {
       "epoch": 0.22,
       "learning_rate": 4.779749614980225e-07,
-      "logits/chosen": -2.2898011207580566,
-      "logits/rejected": -2.192032814025879,
-      "logps/chosen": -496.12384033203125,
-      "logps/rejected": -380.75909423828125,
-      "loss": 0.1376,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.8930786848068237,
-      "rewards/margins": 1.819786787033081,
-      "rewards/rejected": -2.7128653526306152,
+      "logits/chosen": -2.381338596343994,
+      "logits/rejected": -2.327340602874756,
+      "logps/chosen": -520.724365234375,
+      "logps/rejected": -380.0218811035156,
+      "loss": 0.1464,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -0.4466208815574646,
+      "rewards/margins": 2.054797649383545,
+      "rewards/rejected": -2.5014188289642334,
       "step": 90
     },
     {
       "epoch": 0.25,
       "learning_rate": 4.682819627081427e-07,
-      "logits/chosen": -2.363102436065674,
-      "logits/rejected": -2.2812604904174805,
-      "logps/chosen": -489.04302978515625,
-      "logps/rejected": -326.9526062011719,
-      "loss": 0.1424,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -0.5512069463729858,
-      "rewards/margins": 1.6478255987167358,
-      "rewards/rejected": -2.1990325450897217,
+      "logits/chosen": -2.3299832344055176,
+      "logits/rejected": -2.2486767768859863,
+      "logps/chosen": -477.24261474609375,
+      "logps/rejected": -372.49017333984375,
+      "loss": 0.1456,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.6445478200912476,
+      "rewards/margins": 1.895777702331543,
+      "rewards/rejected": -2.54032564163208,
       "step": 100
     },
     {
       "epoch": 0.27,
       "learning_rate": 4.569639943810477e-07,
-      "logits/chosen": -2.3185834884643555,
-      "logits/rejected": -2.2058892250061035,
-      "logps/chosen": -502.36529541015625,
-      "logps/rejected": -393.0018615722656,
-      "loss": 0.1052,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.6747262477874756,
-      "rewards/margins": 2.091235399246216,
-      "rewards/rejected": -2.7659618854522705,
+      "logits/chosen": -2.3097102642059326,
+      "logits/rejected": -2.226323127746582,
+      "logps/chosen": -495.50469970703125,
+      "logps/rejected": -389.80078125,
+      "loss": 0.1283,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.8834150433540344,
+      "rewards/margins": 1.8450326919555664,
+      "rewards/rejected": -2.728447675704956,
       "step": 110
     },
     {
       "epoch": 0.3,
       "learning_rate": 4.4410531154874543e-07,
-      "logits/chosen": -2.242546558380127,
-      "logits/rejected": -2.098257541656494,
-      "logps/chosen": -527.9982299804688,
-      "logps/rejected": -425.9187927246094,
-      "loss": 0.1028,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.0199496746063232,
-      "rewards/margins": 2.1254653930664062,
-      "rewards/rejected": -3.1454153060913086,
+      "logits/chosen": -2.3541078567504883,
+      "logits/rejected": -2.2549960613250732,
+      "logps/chosen": -524.7901000976562,
+      "logps/rejected": -398.75775146484375,
+      "loss": 0.1283,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.7295175790786743,
+      "rewards/margins": 1.9527451992034912,
+      "rewards/rejected": -2.682262897491455,
       "step": 120
     },
     {
       "epoch": 0.32,
       "learning_rate": 4.298016388768561e-07,
-      "logits/chosen": -2.2639729976654053,
-      "logits/rejected": -2.1410276889801025,
-      "logps/chosen": -474.557373046875,
-      "logps/rejected": -406.2401123046875,
-      "loss": 0.102,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": -0.7631456851959229,
-      "rewards/margins": 2.1880364418029785,
-      "rewards/rejected": -2.9511821269989014,
+      "logits/chosen": -2.3804497718811035,
+      "logits/rejected": -2.2821872234344482,
+      "logps/chosen": -518.573974609375,
+      "logps/rejected": -398.14306640625,
+      "loss": 0.114,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.5337150692939758,
+      "rewards/margins": 2.2159152030944824,
+      "rewards/rejected": -2.7496302127838135,
       "step": 130
     },
     {
       "epoch": 0.35,
       "learning_rate": 4.1415945805573005e-07,
-      "logits/chosen": -2.2353649139404297,
-      "logits/rejected": -2.103256940841675,
-      "logps/chosen": -494.5367126464844,
-      "logps/rejected": -413.3063049316406,
-      "loss": 0.109,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.6633724570274353,
-      "rewards/margins": 2.3635334968566895,
-      "rewards/rejected": -3.0269057750701904,
+      "logits/chosen": -2.309293270111084,
+      "logits/rejected": -2.2271227836608887,
+      "logps/chosen": -486.838623046875,
+      "logps/rejected": -373.0490417480469,
+      "loss": 0.1246,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.6586702466011047,
+      "rewards/margins": 1.7459022998809814,
+      "rewards/rejected": -2.4045722484588623,
       "step": 140
     },
     {
       "epoch": 0.37,
       "learning_rate": 3.972952151123984e-07,
-      "logits/chosen": -2.243504762649536,
-      "logits/rejected": -2.1586227416992188,
-      "logps/chosen": -485.9297790527344,
-      "logps/rejected": -403.07904052734375,
-      "loss": 0.0988,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -0.8084269762039185,
-      "rewards/margins": 1.9387495517730713,
-      "rewards/rejected": -2.7471766471862793,
+      "logits/chosen": -2.288892984390259,
+      "logits/rejected": -2.1915061473846436,
+      "logps/chosen": -450.01556396484375,
+      "logps/rejected": -368.2213439941406,
+      "loss": 0.1152,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -0.7529748678207397,
+      "rewards/margins": 1.9199845790863037,
+      "rewards/rejected": -2.672959566116333,
       "step": 150
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.793344535444142e-07,
-      "logits/chosen": -2.2556967735290527,
-      "logits/rejected": -2.1410129070281982,
-      "logps/chosen": -517.8903198242188,
-      "logps/rejected": -427.27178955078125,
-      "loss": 0.1095,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.9526297450065613,
-      "rewards/margins": 2.0250723361968994,
-      "rewards/rejected": -2.9777019023895264,
+      "logits/chosen": -2.2575857639312744,
+      "logits/rejected": -2.1550350189208984,
+      "logps/chosen": -547.2183837890625,
+      "logps/rejected": -409.57989501953125,
+      "loss": 0.088,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.8003584146499634,
+      "rewards/margins": 2.1838386058807373,
+      "rewards/rejected": -2.9841971397399902,
       "step": 160
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.604108797288461e-07,
-      "logits/chosen": -2.2546138763427734,
-      "logits/rejected": -2.162337064743042,
-      "logps/chosen": -484.6184997558594,
-      "logps/rejected": -358.60198974609375,
-      "loss": 0.1069,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": -0.6528670787811279,
-      "rewards/margins": 1.8891479969024658,
-      "rewards/rejected": -2.5420150756835938,
+      "logits/chosen": -2.2742323875427246,
+      "logits/rejected": -2.167198419570923,
+      "logps/chosen": -547.2274169921875,
+      "logps/rejected": -456.614501953125,
+      "loss": 0.0776,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -1.0825190544128418,
+      "rewards/margins": 2.3789236545562744,
+      "rewards/rejected": -3.4614429473876953,
       "step": 170
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.40665367563858e-07,
-      "logits/chosen": -2.199352264404297,
-      "logits/rejected": -2.0996298789978027,
-      "logps/chosen": -520.5743408203125,
-      "logps/rejected": -414.6077575683594,
-      "loss": 0.0923,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.9770752787590027,
-      "rewards/margins": 1.9050334692001343,
-      "rewards/rejected": -2.882108688354492,
+      "logits/chosen": -2.2402544021606445,
+      "logits/rejected": -2.1346538066864014,
+      "logps/chosen": -564.0145263671875,
+      "logps/rejected": -489.21160888671875,
+      "loss": 0.0697,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.539156198501587,
+      "rewards/margins": 2.1975486278533936,
+      "rewards/rejected": -3.7367050647735596,
       "step": 180
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.202449097526798e-07,
-      "logits/chosen": -2.2049007415771484,
-      "logits/rejected": -2.0761523246765137,
-      "logps/chosen": -545.6790771484375,
-      "logps/rejected": -503.28326416015625,
-      "loss": 0.0757,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -1.1332416534423828,
-      "rewards/margins": 2.640087604522705,
-      "rewards/rejected": -3.773329257965088,
+      "logits/chosen": -2.3025131225585938,
+      "logits/rejected": -2.224256992340088,
+      "logps/chosen": -505.39520263671875,
+      "logps/rejected": -423.83026123046875,
+      "loss": 0.0811,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -1.0325360298156738,
+      "rewards/margins": 2.132319927215576,
+      "rewards/rejected": -3.16485595703125,
       "step": 190
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.993015235369905e-07,
-      "logits/chosen": -2.251674175262451,
-      "logits/rejected": -2.138532876968384,
-      "logps/chosen": -552.5167236328125,
-      "logps/rejected": -459.761962890625,
-      "loss": 0.0843,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -1.0592725276947021,
-      "rewards/margins": 2.3839969635009766,
-      "rewards/rejected": -3.443269729614258,
+      "logits/chosen": -2.3023552894592285,
+      "logits/rejected": -2.2043874263763428,
+      "logps/chosen": -525.6875610351562,
+      "logps/rejected": -416.1629333496094,
+      "loss": 0.0979,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -0.8641435503959656,
+      "rewards/margins": 2.118994951248169,
+      "rewards/rejected": -2.9831383228302,
       "step": 200
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7799111902582693e-07,
-      "logits/chosen": -2.216552495956421,
-      "logits/rejected": -2.116063356399536,
-      "logps/chosen": -539.1038818359375,
-      "logps/rejected": -468.47869873046875,
-      "loss": 0.0889,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -1.1180602312088013,
-      "rewards/margins": 2.2188849449157715,
-      "rewards/rejected": -3.336945056915283,
+      "logits/chosen": -2.3067820072174072,
+      "logits/rejected": -2.2110161781311035,
+      "logps/chosen": -492.69927978515625,
+      "logps/rejected": -381.31878662109375,
+      "loss": 0.0872,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.9221334457397461,
+      "rewards/margins": 1.870031714439392,
+      "rewards/rejected": -2.7921650409698486,
       "step": 210
     },
     {
       "epoch": 0.54,
       "learning_rate": 2.564723385445869e-07,
-      "logits/chosen": -2.182385206222534,
-      "logits/rejected": -2.0952038764953613,
-      "logps/chosen": -506.44970703125,
-      "logps/rejected": -433.21954345703125,
-      "loss": 0.0928,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -1.1999738216400146,
-      "rewards/margins": 1.8092005252838135,
-      "rewards/rejected": -3.00917387008667,
+      "logits/chosen": -2.3406167030334473,
+      "logits/rejected": -2.2510488033294678,
+      "logps/chosen": -520.8443603515625,
+      "logps/rejected": -442.00732421875,
+      "loss": 0.0908,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.0322405099868774,
+      "rewards/margins": 2.144731044769287,
+      "rewards/rejected": -3.176971912384033,
       "step": 220
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3490537564442845e-07,
-      "logits/chosen": -2.2019972801208496,
-      "logits/rejected": -2.1013846397399902,
-      "logps/chosen": -493.3079528808594,
-      "logps/rejected": -435.74884033203125,
-      "loss": 0.0853,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -1.0892283916473389,
-      "rewards/margins": 2.1229333877563477,
-      "rewards/rejected": -3.2121620178222656,
+      "logits/chosen": -2.284823179244995,
+      "logits/rejected": -2.1653401851654053,
+      "logps/chosen": -511.96929931640625,
+      "logps/rejected": -426.46356201171875,
+      "loss": 0.0967,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.2002372741699219,
+      "rewards/margins": 1.9990075826644897,
+      "rewards/rejected": -3.199244976043701,
       "step": 230
     },
     {
       "epoch": 0.59,
       "learning_rate": 2.1345078256378801e-07,
-      "logits/chosen": -2.2013192176818848,
-      "logits/rejected": -2.0689620971679688,
-      "logps/chosen": -503.4147033691406,
-      "logps/rejected": -448.7828674316406,
-      "loss": 0.08,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -1.28254234790802,
-      "rewards/margins": 2.0458292961120605,
-      "rewards/rejected": -3.32837176322937,
+      "logits/chosen": -2.321927547454834,
+      "logits/rejected": -2.215357780456543,
+      "logps/chosen": -495.8760681152344,
+      "logps/rejected": -439.46282958984375,
+      "loss": 0.0955,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.8706371188163757,
+      "rewards/margins": 2.3429722785949707,
+      "rewards/rejected": -3.213609218597412,
       "step": 240
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.9226827501969865e-07,
-      "logits/chosen": -2.2620677947998047,
-      "logits/rejected": -2.1344215869903564,
-      "logps/chosen": -535.8228149414062,
-      "logps/rejected": -445.83349609375,
-      "loss": 0.0772,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -0.9264475107192993,
-      "rewards/margins": 2.3875911235809326,
-      "rewards/rejected": -3.3140385150909424,
+      "logits/chosen": -2.3428966999053955,
+      "logits/rejected": -2.2573530673980713,
+      "logps/chosen": -526.4675903320312,
+      "logps/rejected": -451.949462890625,
+      "loss": 0.096,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.8379364013671875,
+      "rewards/margins": 2.499549627304077,
+      "rewards/rejected": -3.3374857902526855,
       "step": 250
     },
     {
       "epoch": 0.64,
       "learning_rate": 1.715155432264775e-07,
-      "logits/chosen": -2.2268154621124268,
-      "logits/rejected": -2.149883270263672,
-      "logps/chosen": -518.2362060546875,
-      "logps/rejected": -444.4808654785156,
-      "loss": 0.0781,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.0662428140640259,
-      "rewards/margins": 2.1827731132507324,
-      "rewards/rejected": -3.249016284942627,
+      "logits/chosen": -2.3556008338928223,
+      "logits/rejected": -2.2766494750976562,
+      "logps/chosen": -516.3786010742188,
+      "logps/rejected": -430.13916015625,
+      "loss": 0.0857,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.8434340357780457,
+      "rewards/margins": 2.294442653656006,
+      "rewards/rejected": -3.1378769874572754,
       "step": 260
     },
     {
       "epoch": 0.67,
       "learning_rate": 1.51347077992983e-07,
-      "logits/chosen": -2.2635481357574463,
-      "logits/rejected": -2.1383774280548096,
-      "logps/chosen": -510.13751220703125,
-      "logps/rejected": -457.0772399902344,
-      "loss": 0.0781,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -1.0047271251678467,
-      "rewards/margins": 2.324492931365967,
-      "rewards/rejected": -3.3292198181152344,
+      "logits/chosen": -2.3460044860839844,
+      "logits/rejected": -2.281031370162964,
+      "logps/chosen": -490.55078125,
+      "logps/rejected": -423.6560974121094,
+      "loss": 0.0821,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -0.8685197830200195,
+      "rewards/margins": 2.1445822715759277,
+      "rewards/rejected": -3.0131022930145264,
       "step": 270
     },
     {
       "epoch": 0.69,
       "learning_rate": 1.3191302063739906e-07,
-      "logits/chosen": -2.2417304515838623,
-      "logits/rejected": -2.121445417404175,
-      "logps/chosen": -491.12139892578125,
-      "logps/rejected": -425.1968688964844,
-      "loss": 0.0805,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -0.8460060954093933,
-      "rewards/margins": 2.316087007522583,
-      "rewards/rejected": -3.162093162536621,
+      "logits/chosen": -2.2882773876190186,
+      "logits/rejected": -2.218071699142456,
+      "logps/chosen": -500.769287109375,
+      "logps/rejected": -446.246826171875,
+      "loss": 0.0712,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.2157343626022339,
+      "rewards/margins": 2.1158077716827393,
+      "rewards/rejected": -3.3315422534942627,
       "step": 280
     },
     {
       "epoch": 0.72,
       "learning_rate": 1.1335804528119475e-07,
-      "logits/chosen": -2.2970786094665527,
-      "logits/rejected": -2.17289662361145,
-      "logps/chosen": -507.46661376953125,
-      "logps/rejected": -449.6181640625,
-      "loss": 0.0812,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.0554336309432983,
-      "rewards/margins": 2.2787580490112305,
-      "rewards/rejected": -3.3341917991638184,
+      "logits/chosen": -2.3649039268493652,
+      "logits/rejected": -2.252676486968994,
+      "logps/chosen": -540.1212158203125,
+      "logps/rejected": -467.2939453125,
+      "loss": 0.0686,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -1.0436217784881592,
+      "rewards/margins": 2.6221861839294434,
+      "rewards/rejected": -3.6658082008361816,
       "step": 290
     },
     {
       "epoch": 0.74,
       "learning_rate": 9.582028184286423e-08,
-      "logits/chosen": -2.2653486728668213,
-      "logits/rejected": -2.1368870735168457,
-      "logps/chosen": -563.6956787109375,
-      "logps/rejected": -506.1378479003906,
-      "loss": 0.0811,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.0197770595550537,
-      "rewards/margins": 2.7816028594970703,
-      "rewards/rejected": -3.801379680633545,
+      "logits/chosen": -2.243900775909424,
+      "logits/rejected": -2.1746292114257812,
+      "logps/chosen": -503.1402282714844,
+      "logps/rejected": -486.1592712402344,
+      "loss": 0.0686,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -1.4328491687774658,
+      "rewards/margins": 2.2128751277923584,
+      "rewards/rejected": -3.6457245349884033,
       "step": 300
     },
     {
       "epoch": 0.76,
       "learning_rate": 7.943028774907065e-08,
-      "logits/chosen": -2.212477445602417,
-      "logits/rejected": -2.083967685699463,
-      "logps/chosen": -543.1881103515625,
-      "logps/rejected": -489.3092346191406,
-      "loss": 0.072,
+      "logits/chosen": -2.2528328895568848,
+      "logits/rejected": -2.170386791229248,
+      "logps/chosen": -501.7100524902344,
+      "logps/rejected": -471.88897705078125,
+      "loss": 0.0689,
       "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.234198808670044,
-      "rewards/margins": 2.4262847900390625,
-      "rewards/rejected": -3.6604835987091064,
+      "rewards/chosen": -1.172387719154358,
+      "rewards/margins": 2.3613522052764893,
+      "rewards/rejected": -3.533740282058716,
       "step": 310
     },
     {
       "epoch": 0.79,
       "learning_rate": 6.431007601814637e-08,
-      "logits/chosen": -2.237189531326294,
-      "logits/rejected": -2.118699550628662,
-      "logps/chosen": -533.3051147460938,
-      "logps/rejected": -482.6719665527344,
-      "loss": 0.0764,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.317209243774414,
-      "rewards/margins": 2.3056979179382324,
-      "rewards/rejected": -3.6229069232940674,
+      "logits/chosen": -2.258288860321045,
+      "logits/rejected": -2.1915061473846436,
+      "logps/chosen": -471.57330322265625,
+      "logps/rejected": -461.84417724609375,
+      "loss": 0.0601,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.4386770725250244,
+      "rewards/margins": 2.1069023609161377,
+      "rewards/rejected": -3.545579433441162,
       "step": 320
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.0572206951246e-08,
-      "logits/chosen": -2.262988567352295,
-      "logits/rejected": -2.143887996673584,
-      "logps/chosen": -538.3980712890625,
-      "logps/rejected": -492.5909118652344,
-      "loss": 0.0884,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -1.2063168287277222,
-      "rewards/margins": 2.362534523010254,
-      "rewards/rejected": -3.5688509941101074,
+      "logits/chosen": -2.2368595600128174,
+      "logits/rejected": -2.1402342319488525,
+      "logps/chosen": -522.8599853515625,
+      "logps/rejected": -482.84893798828125,
+      "loss": 0.0626,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.5531214475631714,
+      "rewards/margins": 2.2123360633850098,
+      "rewards/rejected": -3.7654571533203125,
       "step": 330
     },
     {
       "epoch": 0.84,
       "learning_rate": 3.831895019292897e-08,
-      "logits/chosen": -2.211160182952881,
-      "logits/rejected": -2.0655343532562256,
-      "logps/chosen": -495.141357421875,
-      "logps/rejected": -464.81646728515625,
-      "loss": 0.0744,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -1.0855491161346436,
-      "rewards/margins": 2.5346646308898926,
-      "rewards/rejected": -3.6202139854431152,
+      "logits/chosen": -2.308152675628662,
+      "logits/rejected": -2.2120919227600098,
+      "logps/chosen": -565.0369873046875,
+      "logps/rejected": -535.8488159179688,
+      "loss": 0.0642,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -1.306236982345581,
+      "rewards/margins": 2.8749289512634277,
+      "rewards/rejected": -4.181166172027588,
       "step": 340
     },
     {
       "epoch": 0.86,
       "learning_rate": 2.764152339909756e-08,
-      "logits/chosen": -2.273918867111206,
-      "logits/rejected": -2.128694534301758,
-      "logps/chosen": -528.7555541992188,
-      "logps/rejected": -469.3017578125,
-      "loss": 0.0642,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -1.1073048114776611,
-      "rewards/margins": 2.4064698219299316,
-      "rewards/rejected": -3.513774871826172,
+      "logits/chosen": -2.245577573776245,
+      "logits/rejected": -2.1435444355010986,
+      "logps/chosen": -546.0943603515625,
+      "logps/rejected": -454.5082092285156,
+      "loss": 0.0636,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.2082496881484985,
+      "rewards/margins": 2.3495194911956787,
+      "rewards/rejected": -3.5577690601348877,
       "step": 350
     },
     {
       "epoch": 0.89,
       "learning_rate": 1.861941317991664e-08,
-      "logits/chosen": -2.2403626441955566,
-      "logits/rejected": -2.1174542903900146,
-      "logps/chosen": -488.1507873535156,
-      "logps/rejected": -457.27423095703125,
-      "loss": 0.0756,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -1.1823718547821045,
-      "rewards/margins": 2.2865800857543945,
-      "rewards/rejected": -3.46895170211792,
+      "logits/chosen": -2.302865505218506,
+      "logits/rejected": -2.1724164485931396,
+      "logps/chosen": -559.6376953125,
+      "logps/rejected": -483.40771484375,
+      "loss": 0.0675,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.0334274768829346,
+      "rewards/margins": 2.655003070831299,
+      "rewards/rejected": -3.6884307861328125,
       "step": 360
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.13197833728636e-08,
-      "logits/chosen": -2.2277872562408447,
-      "logits/rejected": -2.0999319553375244,
-      "logps/chosen": -529.3685302734375,
-      "logps/rejected": -459.83551025390625,
-      "loss": 0.0729,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -1.064345121383667,
-      "rewards/margins": 2.42720365524292,
-      "rewards/rejected": -3.491549015045166,
+      "logits/chosen": -2.2556536197662354,
+      "logits/rejected": -2.153872013092041,
+      "logps/chosen": -521.9984130859375,
+      "logps/rejected": -505.71673583984375,
+      "loss": 0.06,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -1.1932189464569092,
+      "rewards/margins": 2.7444043159484863,
+      "rewards/rejected": -3.9376235008239746,
       "step": 370
     },
     {
       "epoch": 0.94,
       "learning_rate": 5.79697505093521e-09,
-      "logits/chosen": -2.161315679550171,
-      "logits/rejected": -2.051104784011841,
-      "logps/chosen": -518.0635986328125,
-      "logps/rejected": -463.9219665527344,
-      "loss": 0.0742,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -1.2277439832687378,
-      "rewards/margins": 2.3196287155151367,
-      "rewards/rejected": -3.547372817993164,
+      "logits/chosen": -2.2588906288146973,
+      "logits/rejected": -2.159388303756714,
+      "logps/chosen": -529.9054565429688,
+      "logps/rejected": -461.11700439453125,
+      "loss": 0.0744,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.276084065437317,
+      "rewards/margins": 2.2816543579101562,
+      "rewards/rejected": -3.5577385425567627,
       "step": 380
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.092101988131256e-09,
-      "logits/chosen": -2.2838692665100098,
-      "logits/rejected": -2.1495554447174072,
-      "logps/chosen": -561.1207275390625,
-      "logps/rejected": -473.84234619140625,
-      "loss": 0.07,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -1.088555932044983,
-      "rewards/margins": 2.4874186515808105,
-      "rewards/rejected": -3.575974702835083,
+      "logits/chosen": -2.313697099685669,
+      "logits/rejected": -2.171175003051758,
+      "logps/chosen": -565.225830078125,
+      "logps/rejected": -489.6360778808594,
+      "loss": 0.0609,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -1.0208733081817627,
+      "rewards/margins": 2.784264087677002,
+      "rewards/rejected": -3.8051371574401855,
       "step": 390
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.327445937151673e-10,
-      "logits/chosen": -2.214580774307251,
-      "logits/rejected": -2.0943350791931152,
-      "logps/chosen": -564.2920532226562,
-      "logps/rejected": -481.778564453125,
-      "loss": 0.0661,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -1.2679953575134277,
-      "rewards/margins": 2.343169689178467,
-      "rewards/rejected": -3.6111652851104736,
+      "logits/chosen": -2.29669189453125,
+      "logits/rejected": -2.1986515522003174,
+      "logps/chosen": -561.0698852539062,
+      "logps/rejected": -510.22021484375,
+      "loss": 0.0666,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.1245156526565552,
+      "rewards/margins": 2.6927759647369385,
+      "rewards/rejected": -3.817291736602783,
       "step": 400
     },
     {
       "epoch": 1.0,
       "step": 405,
       "total_flos": 0.0,
-      "train_loss": 0.13281457475674005,
-      "train_runtime": 3219.0486,
-      "train_samples_per_second": 16.121,
+      "train_loss": 0.13438091388455145,
+      "train_runtime": 3218.6044,
+      "train_samples_per_second": 16.123,
       "train_steps_per_second": 0.126
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:34facb59b2833ff9a65d1ea6ca0671f7143189081be77d079ad67a7343d5aa7d
+oid sha256:3d05f2ea4a8f27ac4989592d034e456f8fe99958c58d076bd3ccb965c582e16a
 size 5944