Model save

README.md

@@ -13,7 +13,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model was trained from scratch on an unknown dataset.
+This model was trained from scratch on the None dataset.
 
 ## Model description
 
@@ -54,5 +54,5 @@ The following hyperparameters were used during training:
 
 - Transformers 4.39.3
 - Pytorch 2.1.2+cu118
-- Datasets 2.19.1
+- Datasets 2.16.1
 - Tokenizers 0.15.2

all_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 1.99,
-    "train_loss": 0.6178853964672408,
-    "train_runtime": 10771.3374,
-    "train_samples": 46008,
-    "train_samples_per_second": 8.543,
-    "train_steps_per_second": 0.033
+    "epoch": 2.0,
+    "train_loss": 0.577453197668428,
+    "train_runtime": 9953.5163,
+    "train_samples": 47095,
+    "train_samples_per_second": 9.463,
+    "train_steps_per_second": 0.037
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e2280e1e450848ff611a22e91fa55b332aa9726ecc5d70924cd3609b00fb212
+oid sha256:f386e47c9663e1d94af7a65a98b2f746b252c1555de790a01dd69020c8e6c440
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cf8335c8343439de60d7780dfff5b3af0920ec66940d7c67e6667917b0311519
+oid sha256:5516581d2e7e174cf9a794e5416e1cf5ec5479d8dd1e6b91f5b37a3f3b2f49f1
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:52b7afca158c67eebe0c04a03e046b31ea0cd95308e37ffe3f73753ba89be2b5
+oid sha256:8ac6ed485fbe9f4fb84b4ef9f7c2fa2943cfe663ede4e7e0919c77b6a10f41ce
 size 4540516344

runs/Jun20_00-48-32_n136-100-194/events.out.tfevents.1718815733.n136-100-194.22992.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ad74df8e1bf642da41e3cc71ddffea12555f7bc69af8de440fa0780d2e43d1fb
-size 26105
+oid sha256:b3e5064d1852d30b931bb8808f7549520436641295e04c2552d1505a17858f56
+size 30587

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 1.99,
-    "train_loss": 0.6178853964672408,
-    "train_runtime": 10771.3374,
-    "train_samples": 46008,
-    "train_samples_per_second": 8.543,
-    "train_steps_per_second": 0.033
+    "epoch": 2.0,
+    "train_loss": 0.577453197668428,
+    "train_runtime": 9953.5163,
+    "train_samples": 47095,
+    "train_samples_per_second": 9.463,
+    "train_steps_per_second": 0.037
 }

trainer_state.json

@@ -1,22 +1,22 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.9916550764951322,
+  "epoch": 2.0,
   "eval_steps": 500,
-  "global_step": 358,
+  "global_step": 368,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.01,
-      "grad_norm": 664.5507726524561,
-      "learning_rate": 2.7777777777777777e-10,
-      "logits/chosen": -1.145136833190918,
-      "logits/rejected": -0.7638764977455139,
-      "logps/chosen": -369.05181884765625,
-      "logps/rejected": -372.01507568359375,
-      "loss": 0.6931,
+      "grad_norm": 1359.2177158077461,
+      "learning_rate": 2.702702702702703e-10,
+      "logits/chosen": -1.3332719802856445,
+      "logits/rejected": -1.246394395828247,
+      "logps/chosen": -286.9539794921875,
+      "logps/rejected": -263.3782958984375,
+      "loss": 0.7007,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -24,542 +24,557 @@
       "step": 1
     },
     {
-      "epoch": 0.06,
-      "grad_norm": 784.2820602567579,
-      "learning_rate": 2.777777777777778e-09,
-      "logits/chosen": -1.0929811000823975,
-      "logits/rejected": -0.9427244663238525,
-      "logps/chosen": -372.3557434082031,
-      "logps/rejected": -339.4920959472656,
-      "loss": 0.7141,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.013798276893794537,
-      "rewards/margins": -0.012861602939665318,
-      "rewards/rejected": -0.0009366737212985754,
+      "epoch": 0.05,
+      "grad_norm": 1659.9667867776882,
+      "learning_rate": 2.702702702702703e-09,
+      "logits/chosen": -1.6176425218582153,
+      "logits/rejected": -1.3966516256332397,
+      "logps/chosen": -342.49456787109375,
+      "logps/rejected": -294.5242614746094,
+      "loss": 0.7618,
+      "rewards/accuracies": 0.4340277910232544,
+      "rewards/chosen": 0.040204986929893494,
+      "rewards/margins": 0.027825627475976944,
+      "rewards/rejected": 0.012379361316561699,
       "step": 10
     },
     {
       "epoch": 0.11,
-      "grad_norm": 708.7597502584047,
-      "learning_rate": 5.555555555555556e-09,
-      "logits/chosen": -0.8585885167121887,
-      "logits/rejected": -0.7338771820068359,
-      "logps/chosen": -365.14849853515625,
-      "logps/rejected": -340.0997619628906,
-      "loss": 0.7105,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": -0.00040760039701126516,
-      "rewards/margins": 0.008629530668258667,
-      "rewards/rejected": -0.009037131443619728,
+      "grad_norm": 1365.2835589869765,
+      "learning_rate": 5.405405405405406e-09,
+      "logits/chosen": -1.4892847537994385,
+      "logits/rejected": -1.311715006828308,
+      "logps/chosen": -314.7313537597656,
+      "logps/rejected": -279.33746337890625,
+      "loss": 0.7414,
+      "rewards/accuracies": 0.5406249761581421,
+      "rewards/chosen": 0.03467293456196785,
+      "rewards/margins": 0.06634848564863205,
+      "rewards/rejected": -0.0316755548119545,
       "step": 20
     },
     {
-      "epoch": 0.17,
-      "grad_norm": 671.6766126687812,
-      "learning_rate": 8.333333333333334e-09,
-      "logits/chosen": -1.1642935276031494,
-      "logits/rejected": -0.8622403144836426,
-      "logps/chosen": -386.5867614746094,
-      "logps/rejected": -348.39959716796875,
-      "loss": 0.7069,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": 0.01048402488231659,
-      "rewards/margins": 0.048695411533117294,
-      "rewards/rejected": -0.038211386650800705,
+      "epoch": 0.16,
+      "grad_norm": 1495.215254347967,
+      "learning_rate": 8.108108108108109e-09,
+      "logits/chosen": -1.5478875637054443,
+      "logits/rejected": -1.3803514242172241,
+      "logps/chosen": -324.8534240722656,
+      "logps/rejected": -286.27276611328125,
+      "loss": 0.7533,
+      "rewards/accuracies": 0.5218750238418579,
+      "rewards/chosen": 0.05265723541378975,
+      "rewards/margins": 0.04645369574427605,
+      "rewards/rejected": 0.006203538738191128,
       "step": 30
     },
     {
       "epoch": 0.22,
-      "grad_norm": 642.0282612566415,
-      "learning_rate": 9.99619291237835e-09,
-      "logits/chosen": -1.0643326044082642,
-      "logits/rejected": -0.8505460619926453,
-      "logps/chosen": -362.8887023925781,
-      "logps/rejected": -332.2599792480469,
-      "loss": 0.7045,
-      "rewards/accuracies": 0.503125011920929,
-      "rewards/chosen": -0.04036983847618103,
-      "rewards/margins": 0.002776807639747858,
-      "rewards/rejected": -0.04314664751291275,
+      "grad_norm": 1385.2297985942057,
+      "learning_rate": 9.997973265157192e-09,
+      "logits/chosen": -1.5357484817504883,
+      "logits/rejected": -1.3583762645721436,
+      "logps/chosen": -325.4037780761719,
+      "logps/rejected": -285.64508056640625,
+      "loss": 0.7565,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.010429712943732738,
+      "rewards/margins": -0.014903778210282326,
+      "rewards/rejected": 0.0044740648008883,
       "step": 40
     },
     {
-      "epoch": 0.28,
-      "grad_norm": 708.8205943616936,
-      "learning_rate": 9.953429730181653e-09,
-      "logits/chosen": -1.069666862487793,
-      "logits/rejected": -0.8791624903678894,
-      "logps/chosen": -377.6866760253906,
-      "logps/rejected": -340.76434326171875,
-      "loss": 0.7045,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -0.0037647963035851717,
-      "rewards/margins": 0.01622203178703785,
-      "rewards/rejected": -0.019986826926469803,
+      "epoch": 0.27,
+      "grad_norm": 1517.4779086138956,
+      "learning_rate": 9.961988113473708e-09,
+      "logits/chosen": -1.5413590669631958,
+      "logits/rejected": -1.3948113918304443,
+      "logps/chosen": -337.04327392578125,
+      "logps/rejected": -297.2876892089844,
+      "loss": 0.7454,
+      "rewards/accuracies": 0.47187501192092896,
+      "rewards/chosen": -0.022386690601706505,
+      "rewards/margins": -0.045912474393844604,
+      "rewards/rejected": 0.0235257837921381,
       "step": 50
     },
     {
       "epoch": 0.33,
-      "grad_norm": 615.8031557981225,
-      "learning_rate": 9.863552602006434e-09,
-      "logits/chosen": -1.0530660152435303,
-      "logits/rejected": -0.8627260327339172,
-      "logps/chosen": -368.82427978515625,
-      "logps/rejected": -334.1483154296875,
-      "loss": 0.6907,
-      "rewards/accuracies": 0.559374988079071,
-      "rewards/chosen": 0.020714253187179565,
-      "rewards/margins": 0.061485253274440765,
-      "rewards/rejected": -0.0407710075378418,
+      "grad_norm": 1317.1513962511742,
+      "learning_rate": 9.881337335184878e-09,
+      "logits/chosen": -1.5795971155166626,
+      "logits/rejected": -1.43094801902771,
+      "logps/chosen": -319.79681396484375,
+      "logps/rejected": -285.0555725097656,
+      "loss": 0.6982,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.046097733080387115,
+      "rewards/margins": 0.21919748187065125,
+      "rewards/rejected": -0.17309975624084473,
       "step": 60
     },
     {
-      "epoch": 0.39,
-      "grad_norm": 603.388531475607,
-      "learning_rate": 9.72741638325434e-09,
-      "logits/chosen": -1.0651648044586182,
-      "logits/rejected": -0.9337506294250488,
-      "logps/chosen": -361.4559631347656,
-      "logps/rejected": -338.84661865234375,
-      "loss": 0.6905,
-      "rewards/accuracies": 0.4906249940395355,
-      "rewards/chosen": -0.02459469996392727,
-      "rewards/margins": 0.010977035388350487,
-      "rewards/rejected": -0.035571735352277756,
+      "epoch": 0.38,
+      "grad_norm": 1446.8726182465537,
+      "learning_rate": 9.756746912994832e-09,
+      "logits/chosen": -1.5057274103164673,
+      "logits/rejected": -1.343697190284729,
+      "logps/chosen": -312.12432861328125,
+      "logps/rejected": -275.1330871582031,
+      "loss": 0.6973,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.016833370551466942,
+      "rewards/margins": 0.13973814249038696,
+      "rewards/rejected": -0.15657152235507965,
       "step": 70
     },
     {
-      "epoch": 0.45,
-      "grad_norm": 596.781435384469,
-      "learning_rate": 9.546315917055362e-09,
-      "logits/chosen": -0.9770561456680298,
-      "logits/rejected": -0.7514477372169495,
-      "logps/chosen": -369.4873046875,
-      "logps/rejected": -332.56134033203125,
-      "loss": 0.6689,
-      "rewards/accuracies": 0.609375,
-      "rewards/chosen": 0.04002801328897476,
-      "rewards/margins": 0.09637657552957535,
-      "rewards/rejected": -0.056348562240600586,
+      "epoch": 0.43,
+      "grad_norm": 1216.5073312303066,
+      "learning_rate": 9.589338354885628e-09,
+      "logits/chosen": -1.5987694263458252,
+      "logits/rejected": -1.4453301429748535,
+      "logps/chosen": -323.3362731933594,
+      "logps/rejected": -288.1485595703125,
+      "loss": 0.6695,
+      "rewards/accuracies": 0.565625011920929,
+      "rewards/chosen": 0.04707593470811844,
+      "rewards/margins": 0.22710590064525604,
+      "rewards/rejected": -0.180029958486557,
       "step": 80
     },
     {
-      "epoch": 0.5,
-      "grad_norm": 587.0211902729591,
-      "learning_rate": 9.321973718524472e-09,
-      "logits/chosen": -0.9680253863334656,
-      "logits/rejected": -0.7858943343162537,
-      "logps/chosen": -360.33001708984375,
-      "logps/rejected": -336.4738464355469,
-      "loss": 0.6653,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.001699769520200789,
-      "rewards/margins": 0.08374340832233429,
-      "rewards/rejected": -0.08544318377971649,
+      "epoch": 0.49,
+      "grad_norm": 1161.721503342428,
+      "learning_rate": 9.380618598797472e-09,
+      "logits/chosen": -1.6032607555389404,
+      "logits/rejected": -1.4058634042739868,
+      "logps/chosen": -319.96173095703125,
+      "logps/rejected": -281.8211364746094,
+      "loss": 0.6501,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": 0.11853907257318497,
+      "rewards/margins": 0.32674410939216614,
+      "rewards/rejected": -0.20820502936840057,
       "step": 90
     },
     {
-      "epoch": 0.56,
-      "grad_norm": 589.9118067767693,
-      "learning_rate": 9.056523591268063e-09,
-      "logits/chosen": -1.0888255834579468,
-      "logits/rejected": -0.9123393893241882,
-      "logps/chosen": -381.3514099121094,
-      "logps/rejected": -355.5039978027344,
-      "loss": 0.6515,
-      "rewards/accuracies": 0.609375,
-      "rewards/chosen": 0.051872074604034424,
-      "rewards/margins": 0.133183091878891,
-      "rewards/rejected": -0.08131101727485657,
+      "epoch": 0.54,
+      "grad_norm": 1255.206435160775,
+      "learning_rate": 9.132466447838596e-09,
+      "logits/chosen": -1.5419257879257202,
+      "logits/rejected": -1.3666443824768066,
+      "logps/chosen": -321.9004821777344,
+      "logps/rejected": -282.74346923828125,
+      "loss": 0.6241,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.21292057633399963,
+      "rewards/margins": 0.4663107991218567,
+      "rewards/rejected": -0.25339022278785706,
       "step": 100
     },
     {
-      "epoch": 0.61,
-      "grad_norm": 589.5691633116944,
-      "learning_rate": 8.752490331969806e-09,
-      "logits/chosen": -1.048461675643921,
-      "logits/rejected": -0.8349950909614563,
-      "logps/chosen": -379.940185546875,
-      "logps/rejected": -349.4921569824219,
-      "loss": 0.6436,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 0.09967100620269775,
-      "rewards/margins": 0.1681671440601349,
-      "rewards/rejected": -0.06849612295627594,
+      "epoch": 0.6,
+      "grad_norm": 1196.6092743226689,
+      "learning_rate": 8.847115658129039e-09,
+      "logits/chosen": -1.50592839717865,
+      "logits/rejected": -1.3777477741241455,
+      "logps/chosen": -318.1675720214844,
+      "logps/rejected": -287.3236083984375,
+      "loss": 0.6068,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.17643409967422485,
+      "rewards/margins": 0.4380553662776947,
+      "rewards/rejected": -0.26162129640579224,
       "step": 110
     },
     {
-      "epoch": 0.67,
-      "grad_norm": 556.6131343001705,
-      "learning_rate": 8.41276571609327e-09,
-      "logits/chosen": -1.1043516397476196,
-      "logits/rejected": -0.8103276491165161,
-      "logps/chosen": -378.1798095703125,
-      "logps/rejected": -348.0905456542969,
-      "loss": 0.6435,
-      "rewards/accuracies": 0.640625,
-      "rewards/chosen": 0.10052381455898285,
-      "rewards/margins": 0.18633946776390076,
-      "rewards/rejected": -0.08581562340259552,
+      "epoch": 0.65,
+      "grad_norm": 1069.9581355208816,
+      "learning_rate": 8.527134831514116e-09,
+      "logits/chosen": -1.5797988176345825,
+      "logits/rejected": -1.4248679876327515,
+      "logps/chosen": -331.4310302734375,
+      "logps/rejected": -297.9786682128906,
+      "loss": 0.613,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": 0.19524307548999786,
+      "rewards/margins": 0.3385895788669586,
+      "rewards/rejected": -0.14334650337696075,
       "step": 120
     },
     {
-      "epoch": 0.72,
-      "grad_norm": 585.5616649485556,
-      "learning_rate": 8.040580993110404e-09,
-      "logits/chosen": -1.119443416595459,
-      "logits/rejected": -0.9243392944335938,
-      "logps/chosen": -374.62371826171875,
-      "logps/rejected": -351.63323974609375,
-      "loss": 0.6349,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 0.19561262428760529,
-      "rewards/margins": 0.23775608837604523,
-      "rewards/rejected": -0.04214347153902054,
+      "epoch": 0.71,
+      "grad_norm": 1001.0236709718766,
+      "learning_rate": 8.175404294144481e-09,
+      "logits/chosen": -1.6208512783050537,
+      "logits/rejected": -1.4344335794448853,
+      "logps/chosen": -317.17388916015625,
+      "logps/rejected": -271.62255859375,
+      "loss": 0.5838,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.3348919153213501,
+      "rewards/margins": 0.4819467067718506,
+      "rewards/rejected": -0.14705480635166168,
       "step": 130
     },
     {
-      "epoch": 0.78,
-      "grad_norm": 568.4650526780266,
-      "learning_rate": 7.639476152864163e-09,
-      "logits/chosen": -1.0983831882476807,
-      "logits/rejected": -0.987181544303894,
-      "logps/chosen": -356.64117431640625,
-      "logps/rejected": -332.7178039550781,
-      "loss": 0.6295,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 0.1680435836315155,
-      "rewards/margins": 0.22499410808086395,
-      "rewards/rejected": -0.05695053189992905,
+      "epoch": 0.76,
+      "grad_norm": 1056.0149896521477,
+      "learning_rate": 7.79509016905158e-09,
+      "logits/chosen": -1.566674828529358,
+      "logits/rejected": -1.417965292930603,
+      "logps/chosen": -331.0799255371094,
+      "logps/rejected": -294.3246154785156,
+      "loss": 0.5738,
+      "rewards/accuracies": 0.7281249761581421,
+      "rewards/chosen": 0.5121167898178101,
+      "rewards/margins": 0.6085957288742065,
+      "rewards/rejected": -0.09647894650697708,
       "step": 140
     },
     {
-      "epoch": 0.83,
-      "grad_norm": 565.1483875486134,
-      "learning_rate": 7.21326625538456e-09,
-      "logits/chosen": -1.0306283235549927,
-      "logits/rejected": -0.7497280836105347,
-      "logps/chosen": -373.5386047363281,
-      "logps/rejected": -344.0720520019531,
-      "loss": 0.6098,
-      "rewards/accuracies": 0.690625011920929,
-      "rewards/chosen": 0.2403784692287445,
-      "rewards/margins": 0.28130704164505005,
-      "rewards/rejected": -0.04092860221862793,
+      "epoch": 0.82,
+      "grad_norm": 1085.2712824812697,
+      "learning_rate": 7.389615876105773e-09,
+      "logits/chosen": -1.5494105815887451,
+      "logits/rejected": -1.4207683801651,
+      "logps/chosen": -314.55694580078125,
+      "logps/rejected": -291.8951110839844,
+      "loss": 0.5814,
+      "rewards/accuracies": 0.6656249761581421,
+      "rewards/chosen": 0.5086601972579956,
+      "rewards/margins": 0.5659324526786804,
+      "rewards/rejected": -0.057272158563137054,
       "step": 150
     },
     {
-      "epoch": 0.89,
-      "grad_norm": 532.1675415448219,
-      "learning_rate": 6.766005144407989e-09,
-      "logits/chosen": -1.1308726072311401,
-      "logits/rejected": -0.8597946166992188,
-      "logps/chosen": -368.006591796875,
-      "logps/rejected": -335.4107360839844,
-      "loss": 0.6206,
-      "rewards/accuracies": 0.6656249761581421,
-      "rewards/chosen": 0.24870140850543976,
-      "rewards/margins": 0.258108526468277,
-      "rewards/rejected": -0.00940710585564375,
+      "epoch": 0.87,
+      "grad_norm": 1147.2094257934837,
+      "learning_rate": 6.962631315901861e-09,
+      "logits/chosen": -1.5222028493881226,
+      "logits/rejected": -1.406696081161499,
+      "logps/chosen": -317.96502685546875,
+      "logps/rejected": -291.0884704589844,
+      "loss": 0.5667,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.5517348051071167,
+      "rewards/margins": 0.5213624835014343,
+      "rewards/rejected": 0.030372310429811478,
       "step": 160
     },
     {
-      "epoch": 0.95,
-      "grad_norm": 740.3690117109163,
-      "learning_rate": 6.301946889734301e-09,
-      "logits/chosen": -0.9349819421768188,
-      "logits/rejected": -0.8212488293647766,
-      "logps/chosen": -363.19781494140625,
-      "logps/rejected": -337.5945739746094,
-      "loss": 0.6152,
-      "rewards/accuracies": 0.6781250238418579,
-      "rewards/chosen": 0.2966246008872986,
-      "rewards/margins": 0.3543737530708313,
-      "rewards/rejected": -0.05774913355708122,
+      "epoch": 0.92,
+      "grad_norm": 1097.0692572479243,
+      "learning_rate": 6.517980014965139e-09,
+      "logits/chosen": -1.5993129014968872,
+      "logits/rejected": -1.41109299659729,
+      "logps/chosen": -331.37066650390625,
+      "logps/rejected": -289.573486328125,
+      "loss": 0.5518,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": 0.6186414957046509,
+      "rewards/margins": 0.6974190473556519,
+      "rewards/rejected": -0.0787774994969368,
       "step": 170
     },
     {
-      "epoch": 1.0,
-      "grad_norm": 511.7643196913959,
-      "learning_rate": 5.825505325157961e-09,
-      "logits/chosen": -1.017749547958374,
-      "logits/rejected": -0.809060275554657,
-      "logps/chosen": -368.81463623046875,
-      "logps/rejected": -349.66790771484375,
-      "loss": 0.6065,
-      "rewards/accuracies": 0.690625011920929,
-      "rewards/chosen": 0.3305915594100952,
-      "rewards/margins": 0.28414201736450195,
-      "rewards/rejected": 0.04644955322146416,
+      "epoch": 0.98,
+      "grad_norm": 1005.7280013223308,
+      "learning_rate": 6.059664528022266e-09,
+      "logits/chosen": -1.6032111644744873,
+      "logits/rejected": -1.4532817602157593,
+      "logps/chosen": -315.0304260253906,
+      "logps/rejected": -276.7928771972656,
+      "loss": 0.5485,
+      "rewards/accuracies": 0.746874988079071,
+      "rewards/chosen": 0.6357426047325134,
+      "rewards/margins": 0.7282391786575317,
+      "rewards/rejected": -0.09249657392501831,
       "step": 180
     },
     {
-      "epoch": 1.06,
-      "grad_norm": 540.5118356148005,
-      "learning_rate": 5.341212066823356e-09,
-      "logits/chosen": -1.1874797344207764,
-      "logits/rejected": -0.9048231244087219,
-      "logps/chosen": -382.1863708496094,
-      "logps/rejected": -338.0008239746094,
-      "loss": 0.5876,
-      "rewards/accuracies": 0.715624988079071,
-      "rewards/chosen": 0.40876778960227966,
-      "rewards/margins": 0.41164499521255493,
-      "rewards/rejected": -0.0028771907091140747,
+      "epoch": 1.03,
+      "grad_norm": 1081.11663192182,
+      "learning_rate": 5.591810408770492e-09,
+      "logits/chosen": -1.5532522201538086,
+      "logits/rejected": -1.3794432878494263,
+      "logps/chosen": -315.5338439941406,
+      "logps/rejected": -278.7769470214844,
+      "loss": 0.5324,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.6404975056648254,
+      "rewards/margins": 0.7475987076759338,
+      "rewards/rejected": -0.10710116475820541,
       "step": 190
     },
     {
-      "epoch": 1.11,
-      "grad_norm": 574.532818545015,
-      "learning_rate": 4.853673411307564e-09,
-      "logits/chosen": -1.0419094562530518,
-      "logits/rejected": -0.8011609315872192,
-      "logps/chosen": -383.00933837890625,
-      "logps/rejected": -344.86761474609375,
-      "loss": 0.5989,
-      "rewards/accuracies": 0.6968749761581421,
-      "rewards/chosen": 0.4167671799659729,
-      "rewards/margins": 0.3280712068080902,
-      "rewards/rejected": 0.08869597315788269,
+      "epoch": 1.09,
+      "grad_norm": 994.8740373609369,
+      "learning_rate": 5.118629073464423e-09,
+      "logits/chosen": -1.5689371824264526,
+      "logits/rejected": -1.3581821918487549,
+      "logps/chosen": -325.8009033203125,
+      "logps/rejected": -282.7605895996094,
+      "loss": 0.5279,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": 0.8169169425964355,
+      "rewards/margins": 0.8118550181388855,
+      "rewards/rejected": 0.005061971955001354,
       "step": 200
     },
     {
-      "epoch": 1.17,
-      "grad_norm": 567.697203170915,
-      "learning_rate": 4.367526523389253e-09,
-      "logits/chosen": -1.1011512279510498,
-      "logits/rejected": -0.8666337132453918,
-      "logps/chosen": -370.8396911621094,
-      "logps/rejected": -342.78558349609375,
-      "loss": 0.5881,
-      "rewards/accuracies": 0.6781250238418579,
-      "rewards/chosen": 0.4040229916572571,
-      "rewards/margins": 0.35025185346603394,
-      "rewards/rejected": 0.05377109721302986,
+      "epoch": 1.14,
+      "grad_norm": 1046.062764450064,
+      "learning_rate": 4.644379891605983e-09,
+      "logits/chosen": -1.6105190515518188,
+      "logits/rejected": -1.4331891536712646,
+      "logps/chosen": -324.65234375,
+      "logps/rejected": -291.387451171875,
+      "loss": 0.5257,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.7297837138175964,
+      "rewards/margins": 0.761337399482727,
+      "rewards/rejected": -0.03155365586280823,
       "step": 210
     },
     {
-      "epoch": 1.22,
-      "grad_norm": 541.6922856310176,
-      "learning_rate": 3.887395330218429e-09,
-      "logits/chosen": -1.077133297920227,
-      "logits/rejected": -0.8909082412719727,
-      "logps/chosen": -376.4487609863281,
-      "logps/rejected": -351.6236877441406,
-      "loss": 0.5866,
-      "rewards/accuracies": 0.659375011920929,
-      "rewards/chosen": 0.4883548319339752,
-      "rewards/margins": 0.3798820674419403,
-      "rewards/rejected": 0.10847274214029312,
+      "epoch": 1.2,
+      "grad_norm": 1062.5909355592705,
+      "learning_rate": 4.173331844980362e-09,
+      "logits/chosen": -1.542718529701233,
+      "logits/rejected": -1.4185158014297485,
+      "logps/chosen": -323.8753356933594,
+      "logps/rejected": -293.48626708984375,
+      "loss": 0.5141,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 0.7222188711166382,
+      "rewards/margins": 0.7484906911849976,
+      "rewards/rejected": -0.026271820068359375,
       "step": 220
     },
     {
-      "epoch": 1.28,
-      "grad_norm": 527.0093020370833,
-      "learning_rate": 3.4178465413942626e-09,
-      "logits/chosen": -1.108720302581787,
-      "logits/rejected": -0.8026013374328613,
-      "logps/chosen": -378.4242248535156,
-      "logps/rejected": -342.5303649902344,
-      "loss": 0.5738,
-      "rewards/accuracies": 0.721875011920929,
-      "rewards/chosen": 0.5319411158561707,
-      "rewards/margins": 0.44675689935684204,
-      "rewards/rejected": 0.08518422394990921,
+      "epoch": 1.25,
+      "grad_norm": 1054.001314243816,
+      "learning_rate": 3.7097251001664824e-09,
+      "logits/chosen": -1.5353752374649048,
+      "logits/rejected": -1.3762162923812866,
+      "logps/chosen": -323.80047607421875,
+      "logps/rejected": -287.00726318359375,
+      "loss": 0.5117,
+      "rewards/accuracies": 0.715624988079071,
+      "rewards/chosen": 0.8148177266120911,
+      "rewards/margins": 0.8335307240486145,
+      "rewards/rejected": -0.018712949007749557,
       "step": 230
     },
     {
-      "epoch": 1.34,
-      "grad_norm": 486.03500376882494,
-      "learning_rate": 2.963346213260737e-09,
-      "logits/chosen": -0.9698505401611328,
-      "logits/rejected": -0.8225222826004028,
-      "logps/chosen": -367.175048828125,
-      "logps/rejected": -342.53631591796875,
-      "loss": 0.5707,
-      "rewards/accuracies": 0.703125,
-      "rewards/chosen": 0.5555700063705444,
-      "rewards/margins": 0.44443902373313904,
-      "rewards/rejected": 0.11113103479146957,
+      "epoch": 1.3,
+      "grad_norm": 1057.7644150535455,
+      "learning_rate": 3.2577328404292057e-09,
+      "logits/chosen": -1.5480402708053589,
+      "logits/rejected": -1.4182308912277222,
+      "logps/chosen": -312.3599548339844,
+      "logps/rejected": -285.9615478515625,
+      "loss": 0.5004,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.8805425763130188,
+      "rewards/margins": 0.8217647671699524,
+      "rewards/rejected": 0.05877774953842163,
       "step": 240
     },
     {
-      "epoch": 1.39,
-      "grad_norm": 534.3215854444298,
-      "learning_rate": 2.528217270553501e-09,
-      "logits/chosen": -0.9723398089408875,
-      "logits/rejected": -0.8604954481124878,
-      "logps/chosen": -359.05828857421875,
-      "logps/rejected": -328.1695556640625,
-      "loss": 0.5785,
-      "rewards/accuracies": 0.690625011920929,
-      "rewards/chosen": 0.47152179479599,
-      "rewards/margins": 0.378196656703949,
-      "rewards/rejected": 0.09332513064146042,
+      "epoch": 1.36,
+      "grad_norm": 1134.7078580204877,
+      "learning_rate": 2.821423700565763e-09,
+      "logits/chosen": -1.597772479057312,
+      "logits/rejected": -1.4192153215408325,
+      "logps/chosen": -350.49432373046875,
+      "logps/rejected": -306.66290283203125,
+      "loss": 0.497,
+      "rewards/accuracies": 0.8031250238418579,
+      "rewards/chosen": 1.035740613937378,
+      "rewards/margins": 1.0768238306045532,
+      "rewards/rejected": -0.04108327627182007,
       "step": 250
     },
     {
-      "epoch": 1.45,
-      "grad_norm": 511.2341258250477,
-      "learning_rate": 2.1165983894256645e-09,
-      "logits/chosen": -0.9806705713272095,
-      "logits/rejected": -0.8469393849372864,
-      "logps/chosen": -367.417724609375,
-      "logps/rejected": -341.4565124511719,
-      "loss": 0.5765,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.540740430355072,
-      "rewards/margins": 0.34819895029067993,
-      "rewards/rejected": 0.19254149496555328,
+      "epoch": 1.41,
+      "grad_norm": 1065.7113049843235,
+      "learning_rate": 2.4047251428513483e-09,
+      "logits/chosen": -1.615321397781372,
+      "logits/rejected": -1.4609696865081787,
+      "logps/chosen": -325.0920104980469,
+      "logps/rejected": -291.08197021484375,
+      "loss": 0.513,
+      "rewards/accuracies": 0.7406250238418579,
+      "rewards/chosen": 0.9730992317199707,
+      "rewards/margins": 0.9078540802001953,
+      "rewards/rejected": 0.06524516642093658,
       "step": 260
     },
     {
-      "epoch": 1.5,
-      "grad_norm": 481.01667031921556,
-      "learning_rate": 1.7324046329316252e-09,
-      "logits/chosen": -1.0429353713989258,
-      "logits/rejected": -0.745051920413971,
-      "logps/chosen": -371.4905090332031,
-      "logps/rejected": -336.188720703125,
-      "loss": 0.5641,
-      "rewards/accuracies": 0.7093750238418579,
-      "rewards/chosen": 0.5642050504684448,
-      "rewards/margins": 0.4432891011238098,
-      "rewards/rejected": 0.12091599404811859,
+      "epoch": 1.47,
+      "grad_norm": 864.6386915761193,
+      "learning_rate": 2.011388103757442e-09,
+      "logits/chosen": -1.5269956588745117,
+      "logits/rejected": -1.3828452825546265,
+      "logps/chosen": -316.21124267578125,
+      "logps/rejected": -285.77667236328125,
+      "loss": 0.4948,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.9972602725028992,
+      "rewards/margins": 0.9109752774238586,
+      "rewards/rejected": 0.08628497272729874,
       "step": 270
     },
     {
-      "epoch": 1.56,
-      "grad_norm": 476.885773993008,
-      "learning_rate": 1.3792902133797692e-09,
-      "logits/chosen": -1.0373092889785767,
-      "logits/rejected": -0.8002932667732239,
-      "logps/chosen": -375.8576354980469,
-      "logps/rejected": -339.48870849609375,
-      "loss": 0.5606,
-      "rewards/accuracies": 0.7406250238418579,
-      "rewards/chosen": 0.6401156187057495,
-      "rewards/margins": 0.4620915353298187,
-      "rewards/rejected": 0.178024023771286,
+      "epoch": 1.52,
+      "grad_norm": 990.2860964249656,
+      "learning_rate": 1.644953229677474e-09,
+      "logits/chosen": -1.601299524307251,
+      "logits/rejected": -1.4185166358947754,
+      "logps/chosen": -325.8226623535156,
+      "logps/rejected": -284.75567626953125,
+      "loss": 0.5046,
+      "rewards/accuracies": 0.7906249761581421,
+      "rewards/chosen": 1.0880569219589233,
+      "rewards/margins": 1.0006572008132935,
+      "rewards/rejected": 0.08739979565143585,
       "step": 280
     },
     {
-      "epoch": 1.61,
-      "grad_norm": 571.353263468623,
-      "learning_rate": 1.060613735735384e-09,
-      "logits/chosen": -0.9055244326591492,
-      "logits/rejected": -0.6923736333847046,
-      "logps/chosen": -358.05694580078125,
-      "logps/rejected": -331.3091125488281,
-      "loss": 0.5778,
-      "rewards/accuracies": 0.7093750238418579,
-      "rewards/chosen": 0.5616304278373718,
-      "rewards/margins": 0.4886326193809509,
-      "rewards/rejected": 0.0729978084564209,
+      "epoch": 1.58,
+      "grad_norm": 1035.2441434858022,
+      "learning_rate": 1.308719005590957e-09,
+      "logits/chosen": -1.5150429010391235,
+      "logits/rejected": -1.400657057762146,
+      "logps/chosen": -318.3056640625,
+      "logps/rejected": -282.54803466796875,
+      "loss": 0.5012,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.9477123022079468,
+      "rewards/margins": 0.9480409622192383,
+      "rewards/rejected": -0.00032869577989913523,
       "step": 290
     },
     {
-      "epoch": 1.67,
-      "grad_norm": 521.7732616506195,
-      "learning_rate": 7.794062526569734e-10,
-      "logits/chosen": -1.1269298791885376,
-      "logits/rejected": -0.7582497596740723,
-      "logps/chosen": -392.01593017578125,
-      "logps/rejected": -350.402587890625,
-      "loss": 0.5693,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": 0.6667734980583191,
-      "rewards/margins": 0.451296329498291,
-      "rewards/rejected": 0.2154771387577057,
+      "epoch": 1.63,
+      "grad_norm": 952.917065524399,
+      "learning_rate": 1.005712063557776e-09,
+      "logits/chosen": -1.6322643756866455,
+      "logits/rejected": -1.4532310962677002,
+      "logps/chosen": -323.9804382324219,
+      "logps/rejected": -290.63629150390625,
+      "loss": 0.505,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 0.903466522693634,
+      "rewards/margins": 0.8820658922195435,
+      "rewards/rejected": 0.02140064910054207,
       "step": 300
     },
     {
-      "epoch": 1.72,
-      "grad_norm": 590.1584228378923,
-      "learning_rate": 5.383424350065824e-10,
-      "logits/chosen": -1.1271040439605713,
-      "logits/rejected": -0.9299942255020142,
-      "logps/chosen": -392.9803161621094,
-      "logps/rejected": -365.2398376464844,
-      "loss": 0.5713,
-      "rewards/accuracies": 0.721875011920929,
-      "rewards/chosen": 0.710626482963562,
-      "rewards/margins": 0.4400349259376526,
-      "rewards/rejected": 0.27059149742126465,
+      "epoch": 1.68,
+      "grad_norm": 946.6373160602382,
+      "learning_rate": 7.386599383124321e-10,
+      "logits/chosen": -1.5624831914901733,
+      "logits/rejected": -1.3783166408538818,
+      "logps/chosen": -321.67547607421875,
+      "logps/rejected": -285.8205261230469,
+      "loss": 0.5001,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.9382842779159546,
+      "rewards/margins": 0.9216762781143188,
+      "rewards/rejected": 0.01660792902112007,
       "step": 310
     },
     {
-      "epoch": 1.78,
-      "grad_norm": 547.631467225418,
-      "learning_rate": 3.397151320423647e-10,
-      "logits/chosen": -1.1201112270355225,
-      "logits/rejected": -0.8372467756271362,
-      "logps/chosen": -377.406982421875,
-      "logps/rejected": -341.136474609375,
-      "loss": 0.5715,
-      "rewards/accuracies": 0.7093750238418579,
-      "rewards/chosen": 0.6100808382034302,
-      "rewards/margins": 0.44269323348999023,
-      "rewards/rejected": 0.16738756000995636,
+      "epoch": 1.74,
+      "grad_norm": 995.8621536271032,
+      "learning_rate": 5.099665152003929e-10,
+      "logits/chosen": -1.5874156951904297,
+      "logits/rejected": -1.3745605945587158,
+      "logps/chosen": -333.58453369140625,
+      "logps/rejected": -289.9964294433594,
+      "loss": 0.4994,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 1.030099868774414,
+      "rewards/margins": 1.0678800344467163,
+      "rewards/rejected": -0.03778017312288284,
       "step": 320
     },
     {
-      "epoch": 1.84,
-      "grad_norm": 532.2336174316262,
-      "learning_rate": 1.8541356326100434e-10,
-      "logits/chosen": -1.0350492000579834,
-      "logits/rejected": -0.8257778882980347,
-      "logps/chosen": -369.24896240234375,
-      "logps/rejected": -344.9882507324219,
-      "loss": 0.5662,
-      "rewards/accuracies": 0.7406250238418579,
-      "rewards/chosen": 0.6600748896598816,
-      "rewards/margins": 0.4713415205478668,
-      "rewards/rejected": 0.18873335421085358,
+      "epoch": 1.79,
+      "grad_norm": 920.3253000772652,
+      "learning_rate": 3.216903914633745e-10,
+      "logits/chosen": -1.5468734502792358,
+      "logits/rejected": -1.4234826564788818,
+      "logps/chosen": -325.117919921875,
+      "logps/rejected": -296.07086181640625,
+      "loss": 0.5015,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.9095224142074585,
+      "rewards/margins": 0.8073747754096985,
+      "rewards/rejected": 0.1021476536989212,
       "step": 330
     },
     {
-      "epoch": 1.89,
-      "grad_norm": 513.0198840165194,
-      "learning_rate": 7.690534931565518e-11,
-      "logits/chosen": -1.0103225708007812,
-      "logits/rejected": -0.7807376384735107,
-      "logps/chosen": -372.0942687988281,
-      "logps/rejected": -339.8793029785156,
-      "loss": 0.5724,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.6672757863998413,
-      "rewards/margins": 0.49458056688308716,
-      "rewards/rejected": 0.1726953089237213,
+      "epoch": 1.85,
+      "grad_norm": 988.4564193799931,
+      "learning_rate": 1.7552634565570324e-10,
+      "logits/chosen": -1.5551540851593018,
+      "logits/rejected": -1.3869084119796753,
+      "logps/chosen": -329.73553466796875,
+      "logps/rejected": -292.8804931640625,
+      "loss": 0.4969,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 1.0978221893310547,
+      "rewards/margins": 1.0354284048080444,
+      "rewards/rejected": 0.0623936764895916,
       "step": 340
     },
     {
-      "epoch": 1.95,
-      "grad_norm": 524.4542591483576,
-      "learning_rate": 1.5222552920138853e-11,
-      "logits/chosen": -1.0104167461395264,
-      "logits/rejected": -0.8294402360916138,
-      "logps/chosen": -391.20452880859375,
-      "logps/rejected": -356.1845703125,
-      "loss": 0.5659,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 0.7216753959655762,
-      "rewards/margins": 0.5309287309646606,
-      "rewards/rejected": 0.19074663519859314,
+      "epoch": 1.9,
+      "grad_norm": 978.2289689829179,
+      "learning_rate": 7.279008199590543e-11,
+      "logits/chosen": -1.5398132801055908,
+      "logits/rejected": -1.3765273094177246,
+      "logps/chosen": -326.29425048828125,
+      "logps/rejected": -292.0558166503906,
+      "loss": 0.4951,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 1.0201548337936401,
+      "rewards/margins": 1.052316665649414,
+      "rewards/rejected": -0.03216180205345154,
       "step": 350
     },
     {
-      "epoch": 1.99,
-      "step": 358,
+      "epoch": 1.96,
+      "grad_norm": 921.2000894790062,
+      "learning_rate": 1.4406386978128017e-11,
+      "logits/chosen": -1.629494309425354,
+      "logits/rejected": -1.434570074081421,
+      "logps/chosen": -330.90911865234375,
+      "logps/rejected": -291.73638916015625,
+      "loss": 0.4873,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 1.1673272848129272,
+      "rewards/margins": 1.1191251277923584,
+      "rewards/rejected": 0.048202164471149445,
+      "step": 360
+    },
+    {
+      "epoch": 2.0,
+      "step": 368,
       "total_flos": 0.0,
-      "train_loss": 0.6178853964672408,
-      "train_runtime": 10771.3374,
-      "train_samples_per_second": 8.543,
-      "train_steps_per_second": 0.033
+      "train_loss": 0.577453197668428,
+      "train_runtime": 9953.5163,
+      "train_samples_per_second": 9.463,
+      "train_steps_per_second": 0.037
     }
   ],
   "logging_steps": 10,
-  "max_steps": 358,
+  "max_steps": 368,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 2,
   "save_steps": 100,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fdeda5a8d4bfbc0a95a9fc5768806e3f1fdf2e395d6ef47d92af3478e9533580
+oid sha256:6f94bf159e3c09d5755b9d7f6e04d5c212f5352b8cf00853f316d50c78e1c689
 size 6328