Model save

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 2.0,
-    "train_loss": 0.577453197668428,
-    "train_runtime": 9953.5163,
+    "train_loss": 1.1851136749205382,
+    "train_runtime": 9917.443,
     "train_samples": 47095,
-    "train_samples_per_second": 9.463,
+    "train_samples_per_second": 9.497,
     "train_steps_per_second": 0.037
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f386e47c9663e1d94af7a65a98b2f746b252c1555de790a01dd69020c8e6c440
+oid sha256:ee032bde6fa6760b2c2a0c6fd438619c8124f4fd40426fd16ea7103737c8b3a1
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5516581d2e7e174cf9a794e5416e1cf5ec5479d8dd1e6b91f5b37a3f3b2f49f1
+oid sha256:21ad507b64d6a5bac882bc60d28e30ace07790d869be8ebc579669104f91f2a7
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8ac6ed485fbe9f4fb84b4ef9f7c2fa2943cfe663ede4e7e0919c77b6a10f41ce
+oid sha256:cda087bfd4990777fd9795c39e3bef165695ce638c3fa33eeb2a9a659aabaad9
 size 4540516344

runs/Jun20_00-51-25_n136-129-074/events.out.tfevents.1718816157.n136-129-074.3397724.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:59027ddf51a64ff93b49478afd24363cd9b21a2ba9ccc8bfbbade06fc4a6ec37
-size 26105
+oid sha256:da365c70b759535a51ff3b960447e822731f825fe1df916819fd3f3cee7d22a3
+size 30587

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 2.0,
-    "train_loss": 0.577453197668428,
-    "train_runtime": 9953.5163,
+    "train_loss": 1.1851136749205382,
+    "train_runtime": 9917.443,
     "train_samples": 47095,
-    "train_samples_per_second": 9.463,
+    "train_samples_per_second": 9.497,
     "train_steps_per_second": 0.037
 }

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.01,
-      "grad_norm": 1359.2177158077461,
+      "grad_norm": 6710.683257043726,
       "learning_rate": 2.702702702702703e-10,
       "logits/chosen": -1.3332719802856445,
       "logits/rejected": -1.246394395828247,
       "logps/chosen": -286.9539794921875,
       "logps/rejected": -263.3782958984375,
-      "loss": 0.7007,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,551 +25,551 @@
     },
     {
       "epoch": 0.05,
-      "grad_norm": 1659.9667867776882,
+      "grad_norm": 8627.07198024136,
       "learning_rate": 2.702702702702703e-09,
-      "logits/chosen": -1.6176425218582153,
-      "logits/rejected": -1.3966516256332397,
-      "logps/chosen": -342.49456787109375,
-      "logps/rejected": -294.5242614746094,
-      "loss": 0.7618,
-      "rewards/accuracies": 0.4340277910232544,
-      "rewards/chosen": 0.040204986929893494,
-      "rewards/margins": 0.027825627475976944,
-      "rewards/rejected": 0.012379361316561699,
+      "logits/chosen": -1.6131304502487183,
+      "logits/rejected": -1.3906824588775635,
+      "logps/chosen": -342.4801025390625,
+      "logps/rejected": -294.5570068359375,
+      "loss": 1.5693,
+      "rewards/accuracies": 0.4861111044883728,
+      "rewards/chosen": 0.2733865976333618,
+      "rewards/margins": 0.37529557943344116,
+      "rewards/rejected": -0.10190902650356293,
       "step": 10
     },
     {
       "epoch": 0.11,
-      "grad_norm": 1365.2835589869765,
+      "grad_norm": 7866.070005970393,
       "learning_rate": 5.405405405405406e-09,
-      "logits/chosen": -1.4892847537994385,
-      "logits/rejected": -1.311715006828308,
-      "logps/chosen": -314.7313537597656,
-      "logps/rejected": -279.33746337890625,
-      "loss": 0.7414,
-      "rewards/accuracies": 0.5406249761581421,
-      "rewards/chosen": 0.03467293456196785,
-      "rewards/margins": 0.06634848564863205,
-      "rewards/rejected": -0.0316755548119545,
+      "logits/chosen": -1.4882968664169312,
+      "logits/rejected": -1.310754418373108,
+      "logps/chosen": -314.77069091796875,
+      "logps/rejected": -279.344970703125,
+      "loss": 1.5992,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.023275405168533325,
+      "rewards/margins": 0.17254583537578583,
+      "rewards/rejected": -0.19582125544548035,
       "step": 20
     },
     {
       "epoch": 0.16,
-      "grad_norm": 1495.215254347967,
+      "grad_norm": 9136.009378303972,
       "learning_rate": 8.108108108108109e-09,
-      "logits/chosen": -1.5478875637054443,
-      "logits/rejected": -1.3803514242172241,
-      "logps/chosen": -324.8534240722656,
-      "logps/rejected": -286.27276611328125,
-      "loss": 0.7533,
-      "rewards/accuracies": 0.5218750238418579,
-      "rewards/chosen": 0.05265723541378975,
-      "rewards/margins": 0.04645369574427605,
-      "rewards/rejected": 0.006203538738191128,
+      "logits/chosen": -1.5432701110839844,
+      "logits/rejected": -1.3749229907989502,
+      "logps/chosen": -324.8939208984375,
+      "logps/rejected": -286.3031005859375,
+      "loss": 1.7105,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": 0.060706138610839844,
+      "rewards/margins": 0.18149089813232422,
+      "rewards/rejected": -0.12078475952148438,
       "step": 30
     },
     {
       "epoch": 0.22,
-      "grad_norm": 1385.2297985942057,
+      "grad_norm": 8253.74733424752,
       "learning_rate": 9.997973265157192e-09,
-      "logits/chosen": -1.5357484817504883,
-      "logits/rejected": -1.3583762645721436,
-      "logps/chosen": -325.4037780761719,
-      "logps/rejected": -285.64508056640625,
-      "loss": 0.7565,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": -0.010429712943732738,
-      "rewards/margins": -0.014903778210282326,
-      "rewards/rejected": 0.0044740648008883,
+      "logits/chosen": -1.530524730682373,
+      "logits/rejected": -1.3523082733154297,
+      "logps/chosen": -325.4083557128906,
+      "logps/rejected": -285.64794921875,
+      "loss": 1.688,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.07491829246282578,
+      "rewards/margins": -0.08290421962738037,
+      "rewards/rejected": 0.007985919713973999,
       "step": 40
     },
     {
       "epoch": 0.27,
-      "grad_norm": 1517.4779086138956,
+      "grad_norm": 7989.013903323287,
       "learning_rate": 9.961988113473708e-09,
-      "logits/chosen": -1.5413590669631958,
-      "logits/rejected": -1.3948113918304443,
-      "logps/chosen": -337.04327392578125,
-      "logps/rejected": -297.2876892089844,
-      "loss": 0.7454,
-      "rewards/accuracies": 0.47187501192092896,
-      "rewards/chosen": -0.022386690601706505,
-      "rewards/margins": -0.045912474393844604,
-      "rewards/rejected": 0.0235257837921381,
+      "logits/chosen": -1.537957787513733,
+      "logits/rejected": -1.390915870666504,
+      "logps/chosen": -336.9923400878906,
+      "logps/rejected": -297.2927551269531,
+      "loss": 1.5308,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": 0.14283373951911926,
+      "rewards/margins": 0.05056797340512276,
+      "rewards/rejected": 0.09226574748754501,
       "step": 50
     },
     {
       "epoch": 0.33,
-      "grad_norm": 1317.1513962511742,
+      "grad_norm": 7487.424543983984,
       "learning_rate": 9.881337335184878e-09,
-      "logits/chosen": -1.5795971155166626,
-      "logits/rejected": -1.43094801902771,
-      "logps/chosen": -319.79681396484375,
-      "logps/rejected": -285.0555725097656,
-      "loss": 0.6982,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": 0.046097733080387115,
-      "rewards/margins": 0.21919748187065125,
-      "rewards/rejected": -0.17309975624084473,
+      "logits/chosen": -1.5855934619903564,
+      "logits/rejected": -1.4371620416641235,
+      "logps/chosen": -319.7807922363281,
+      "logps/rejected": -284.994140625,
+      "loss": 1.4965,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": 0.3105394244194031,
+      "rewards/margins": 0.8686850666999817,
+      "rewards/rejected": -0.5581457018852234,
       "step": 60
     },
     {
       "epoch": 0.38,
-      "grad_norm": 1446.8726182465537,
+      "grad_norm": 8830.269579570355,
       "learning_rate": 9.756746912994832e-09,
-      "logits/chosen": -1.5057274103164673,
-      "logits/rejected": -1.343697190284729,
-      "logps/chosen": -312.12432861328125,
-      "logps/rejected": -275.1330871582031,
-      "loss": 0.6973,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.016833370551466942,
-      "rewards/margins": 0.13973814249038696,
-      "rewards/rejected": -0.15657152235507965,
+      "logits/chosen": -1.5150383710861206,
+      "logits/rejected": -1.354196310043335,
+      "logps/chosen": -312.10052490234375,
+      "logps/rejected": -275.0794982910156,
+      "loss": 1.409,
+      "rewards/accuracies": 0.565625011920929,
+      "rewards/chosen": 0.03486788272857666,
+      "rewards/margins": 0.5497146844863892,
+      "rewards/rejected": -0.5148467421531677,
       "step": 70
     },
     {
       "epoch": 0.43,
-      "grad_norm": 1216.5073312303066,
+      "grad_norm": 7046.728190784739,
       "learning_rate": 9.589338354885628e-09,
-      "logits/chosen": -1.5987694263458252,
-      "logits/rejected": -1.4453301429748535,
-      "logps/chosen": -323.3362731933594,
-      "logps/rejected": -288.1485595703125,
-      "loss": 0.6695,
-      "rewards/accuracies": 0.565625011920929,
-      "rewards/chosen": 0.04707593470811844,
-      "rewards/margins": 0.22710590064525604,
-      "rewards/rejected": -0.180029958486557,
+      "logits/chosen": -1.593528389930725,
+      "logits/rejected": -1.4396848678588867,
+      "logps/chosen": -323.3490295410156,
+      "logps/rejected": -288.1017150878906,
+      "loss": 1.4073,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.1715652495622635,
+      "rewards/margins": 0.8374398946762085,
+      "rewards/rejected": -0.6658747792243958,
       "step": 80
     },
     {
       "epoch": 0.49,
-      "grad_norm": 1161.721503342428,
+      "grad_norm": 6929.551893900645,
       "learning_rate": 9.380618598797472e-09,
-      "logits/chosen": -1.6032607555389404,
-      "logits/rejected": -1.4058634042739868,
-      "logps/chosen": -319.96173095703125,
-      "logps/rejected": -281.8211364746094,
-      "loss": 0.6501,
-      "rewards/accuracies": 0.628125011920929,
-      "rewards/chosen": 0.11853907257318497,
-      "rewards/margins": 0.32674410939216614,
-      "rewards/rejected": -0.20820502936840057,
+      "logits/chosen": -1.6050440073013306,
+      "logits/rejected": -1.4080655574798584,
+      "logps/chosen": -319.95452880859375,
+      "logps/rejected": -281.72833251953125,
+      "loss": 1.3787,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.6286121606826782,
+      "rewards/margins": 1.2058827877044678,
+      "rewards/rejected": -0.5772705078125,
       "step": 90
     },
     {
       "epoch": 0.54,
-      "grad_norm": 1255.206435160775,
+      "grad_norm": 6275.615806226439,
       "learning_rate": 9.132466447838596e-09,
-      "logits/chosen": -1.5419257879257202,
-      "logits/rejected": -1.3666443824768066,
-      "logps/chosen": -321.9004821777344,
-      "logps/rejected": -282.74346923828125,
-      "loss": 0.6241,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 0.21292057633399963,
-      "rewards/margins": 0.4663107991218567,
-      "rewards/rejected": -0.25339022278785706,
+      "logits/chosen": -1.5423895120620728,
+      "logits/rejected": -1.3670966625213623,
+      "logps/chosen": -321.89300537109375,
+      "logps/rejected": -282.581787109375,
+      "loss": 1.2362,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 1.1018999814987183,
+      "rewards/margins": 1.560381531715393,
+      "rewards/rejected": -0.45848163962364197,
       "step": 100
     },
     {
       "epoch": 0.6,
-      "grad_norm": 1196.6092743226689,
+      "grad_norm": 6651.119283999509,
       "learning_rate": 8.847115658129039e-09,
-      "logits/chosen": -1.50592839717865,
-      "logits/rejected": -1.3777477741241455,
-      "logps/chosen": -318.1675720214844,
-      "logps/rejected": -287.3236083984375,
-      "loss": 0.6068,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.17643409967422485,
-      "rewards/margins": 0.4380553662776947,
-      "rewards/rejected": -0.26162129640579224,
+      "logits/chosen": -1.512652039527893,
+      "logits/rejected": -1.3851889371871948,
+      "logps/chosen": -318.17474365234375,
+      "logps/rejected": -287.15032958984375,
+      "loss": 1.2039,
+      "rewards/accuracies": 0.6343749761581421,
+      "rewards/chosen": 0.8460339307785034,
+      "rewards/margins": 1.2877717018127441,
+      "rewards/rejected": -0.44173774123191833,
       "step": 110
     },
     {
       "epoch": 0.65,
-      "grad_norm": 1069.9581355208816,
+      "grad_norm": 6722.274057952871,
       "learning_rate": 8.527134831514116e-09,
-      "logits/chosen": -1.5797988176345825,
-      "logits/rejected": -1.4248679876327515,
-      "logps/chosen": -331.4310302734375,
-      "logps/rejected": -297.9786682128906,
-      "loss": 0.613,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": 0.19524307548999786,
-      "rewards/margins": 0.3385895788669586,
-      "rewards/rejected": -0.14334650337696075,
+      "logits/chosen": -1.5804601907730103,
+      "logits/rejected": -1.4257571697235107,
+      "logps/chosen": -331.40728759765625,
+      "logps/rejected": -297.8313903808594,
+      "loss": 1.2367,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 1.094892978668213,
+      "rewards/margins": 1.075182318687439,
+      "rewards/rejected": 0.019710630178451538,
       "step": 120
     },
     {
       "epoch": 0.71,
-      "grad_norm": 1001.0236709718766,
+      "grad_norm": 5891.724908819839,
       "learning_rate": 8.175404294144481e-09,
-      "logits/chosen": -1.6208512783050537,
-      "logits/rejected": -1.4344335794448853,
-      "logps/chosen": -317.17388916015625,
-      "logps/rejected": -271.62255859375,
-      "loss": 0.5838,
+      "logits/chosen": -1.6209514141082764,
+      "logits/rejected": -1.4345628023147583,
+      "logps/chosen": -317.20465087890625,
+      "logps/rejected": -271.5218811035156,
+      "loss": 1.1341,
       "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.3348919153213501,
-      "rewards/margins": 0.4819467067718506,
-      "rewards/rejected": -0.14705480635166168,
+      "rewards/chosen": 1.5204100608825684,
+      "rewards/margins": 1.7521209716796875,
+      "rewards/rejected": -0.23171091079711914,
       "step": 130
     },
     {
       "epoch": 0.76,
-      "grad_norm": 1056.0149896521477,
+      "grad_norm": 6773.700242649663,
       "learning_rate": 7.79509016905158e-09,
-      "logits/chosen": -1.566674828529358,
-      "logits/rejected": -1.417965292930603,
-      "logps/chosen": -331.0799255371094,
-      "logps/rejected": -294.3246154785156,
-      "loss": 0.5738,
-      "rewards/accuracies": 0.7281249761581421,
-      "rewards/chosen": 0.5121167898178101,
-      "rewards/margins": 0.6085957288742065,
-      "rewards/rejected": -0.09647894650697708,
+      "logits/chosen": -1.5730321407318115,
+      "logits/rejected": -1.4252164363861084,
+      "logps/chosen": -331.2398681640625,
+      "logps/rejected": -294.24530029296875,
+      "loss": 1.2492,
+      "rewards/accuracies": 0.6781250238418579,
+      "rewards/chosen": 1.7606897354125977,
+      "rewards/margins": 1.846710443496704,
+      "rewards/rejected": -0.08602098375558853,
       "step": 140
     },
     {
       "epoch": 0.82,
-      "grad_norm": 1085.2712824812697,
+      "grad_norm": 6236.159184960732,
       "learning_rate": 7.389615876105773e-09,
-      "logits/chosen": -1.5494105815887451,
-      "logits/rejected": -1.4207683801651,
-      "logps/chosen": -314.55694580078125,
-      "logps/rejected": -291.8951110839844,
-      "loss": 0.5814,
-      "rewards/accuracies": 0.6656249761581421,
-      "rewards/chosen": 0.5086601972579956,
-      "rewards/margins": 0.5659324526786804,
-      "rewards/rejected": -0.057272158563137054,
+      "logits/chosen": -1.551709532737732,
+      "logits/rejected": -1.4236745834350586,
+      "logps/chosen": -314.64178466796875,
+      "logps/rejected": -291.77093505859375,
+      "loss": 1.1633,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": 2.1191415786743164,
+      "rewards/margins": 1.7847158908843994,
+      "rewards/rejected": 0.33442583680152893,
       "step": 150
     },
     {
       "epoch": 0.87,
-      "grad_norm": 1147.2094257934837,
+      "grad_norm": 6133.347836072311,
       "learning_rate": 6.962631315901861e-09,
-      "logits/chosen": -1.5222028493881226,
-      "logits/rejected": -1.406696081161499,
-      "logps/chosen": -317.96502685546875,
-      "logps/rejected": -291.0884704589844,
-      "loss": 0.5667,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.5517348051071167,
-      "rewards/margins": 0.5213624835014343,
-      "rewards/rejected": 0.030372310429811478,
+      "logits/chosen": -1.526906967163086,
+      "logits/rejected": -1.411745548248291,
+      "logps/chosen": -318.1049499511719,
+      "logps/rejected": -290.96685791015625,
+      "loss": 1.1827,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 2.0592172145843506,
+      "rewards/margins": 1.2991172075271606,
+      "rewards/rejected": 0.7601001262664795,
       "step": 160
     },
     {
       "epoch": 0.92,
-      "grad_norm": 1097.0692572479243,
+      "grad_norm": 5643.626720023826,
       "learning_rate": 6.517980014965139e-09,
-      "logits/chosen": -1.5993129014968872,
-      "logits/rejected": -1.41109299659729,
-      "logps/chosen": -331.37066650390625,
-      "logps/rejected": -289.573486328125,
-      "loss": 0.5518,
-      "rewards/accuracies": 0.734375,
-      "rewards/chosen": 0.6186414957046509,
-      "rewards/margins": 0.6974190473556519,
-      "rewards/rejected": -0.0787774994969368,
+      "logits/chosen": -1.599163293838501,
+      "logits/rejected": -1.4116249084472656,
+      "logps/chosen": -331.5318603515625,
+      "logps/rejected": -289.4364929199219,
+      "loss": 1.1298,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 2.287135601043701,
+      "rewards/margins": 1.9960601329803467,
+      "rewards/rejected": 0.29107528924942017,
       "step": 170
     },
     {
       "epoch": 0.98,
-      "grad_norm": 1005.7280013223308,
+      "grad_norm": 5637.630920957477,
       "learning_rate": 6.059664528022266e-09,
-      "logits/chosen": -1.6032111644744873,
-      "logits/rejected": -1.4532817602157593,
-      "logps/chosen": -315.0304260253906,
-      "logps/rejected": -276.7928771972656,
-      "loss": 0.5485,
-      "rewards/accuracies": 0.746874988079071,
-      "rewards/chosen": 0.6357426047325134,
-      "rewards/margins": 0.7282391786575317,
-      "rewards/rejected": -0.09249657392501831,
+      "logits/chosen": -1.5944277048110962,
+      "logits/rejected": -1.4438059329986572,
+      "logps/chosen": -315.2278747558594,
+      "logps/rejected": -276.7435607910156,
+      "loss": 1.0907,
+      "rewards/accuracies": 0.6968749761581421,
+      "rewards/chosen": 2.1914913654327393,
+      "rewards/margins": 2.4074912071228027,
+      "rewards/rejected": -0.21599988639354706,
       "step": 180
     },
     {
       "epoch": 1.03,
-      "grad_norm": 1081.11663192182,
+      "grad_norm": 6180.318875375917,
       "learning_rate": 5.591810408770492e-09,
-      "logits/chosen": -1.5532522201538086,
-      "logits/rejected": -1.3794432878494263,
-      "logps/chosen": -315.5338439941406,
-      "logps/rejected": -278.7769470214844,
-      "loss": 0.5324,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.6404975056648254,
-      "rewards/margins": 0.7475987076759338,
-      "rewards/rejected": -0.10710116475820541,
+      "logits/chosen": -1.5520689487457275,
+      "logits/rejected": -1.3785512447357178,
+      "logps/chosen": -315.73565673828125,
+      "logps/rejected": -278.69659423828125,
+      "loss": 1.0339,
+      "rewards/accuracies": 0.6656249761581421,
+      "rewards/chosen": 2.1935009956359863,
+      "rewards/margins": 2.327456474304199,
+      "rewards/rejected": -0.1339556723833084,
       "step": 190
     },
     {
       "epoch": 1.09,
-      "grad_norm": 994.8740373609369,
+      "grad_norm": 6463.364092385962,
       "learning_rate": 5.118629073464423e-09,
-      "logits/chosen": -1.5689371824264526,
-      "logits/rejected": -1.3581821918487549,
-      "logps/chosen": -325.8009033203125,
-      "logps/rejected": -282.7605895996094,
-      "loss": 0.5279,
-      "rewards/accuracies": 0.703125,
-      "rewards/chosen": 0.8169169425964355,
-      "rewards/margins": 0.8118550181388855,
-      "rewards/rejected": 0.005061971955001354,
+      "logits/chosen": -1.56984543800354,
+      "logits/rejected": -1.3597148656845093,
+      "logps/chosen": -326.0714416503906,
+      "logps/rejected": -282.65838623046875,
+      "loss": 1.0602,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 2.7320990562438965,
+      "rewards/margins": 2.1956944465637207,
+      "rewards/rejected": 0.5364044904708862,
       "step": 200
     },
     {
       "epoch": 1.14,
-      "grad_norm": 1046.062764450064,
+      "grad_norm": 5690.544515251583,
       "learning_rate": 4.644379891605983e-09,
-      "logits/chosen": -1.6105190515518188,
-      "logits/rejected": -1.4331891536712646,
-      "logps/chosen": -324.65234375,
-      "logps/rejected": -291.387451171875,
-      "loss": 0.5257,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.7297837138175964,
-      "rewards/margins": 0.761337399482727,
-      "rewards/rejected": -0.03155365586280823,
+      "logits/chosen": -1.6162197589874268,
+      "logits/rejected": -1.4398715496063232,
+      "logps/chosen": -324.8927001953125,
+      "logps/rejected": -291.322998046875,
+      "loss": 1.0255,
+      "rewards/accuracies": 0.690625011920929,
+      "rewards/chosen": 2.447072982788086,
+      "rewards/margins": 2.282660961151123,
+      "rewards/rejected": 0.1644122153520584,
       "step": 210
     },
     {
       "epoch": 1.2,
-      "grad_norm": 1062.5909355592705,
+      "grad_norm": 6400.665884048493,
       "learning_rate": 4.173331844980362e-09,
-      "logits/chosen": -1.542718529701233,
-      "logits/rejected": -1.4185158014297485,
-      "logps/chosen": -323.8753356933594,
-      "logps/rejected": -293.48626708984375,
-      "loss": 0.5141,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": 0.7222188711166382,
-      "rewards/margins": 0.7484906911849976,
-      "rewards/rejected": -0.026271820068359375,
+      "logits/chosen": -1.5352128744125366,
+      "logits/rejected": -1.4104220867156982,
+      "logps/chosen": -324.0686340332031,
+      "logps/rejected": -293.37646484375,
+      "loss": 1.0832,
+      "rewards/accuracies": 0.6968749761581421,
+      "rewards/chosen": 2.6446428298950195,
+      "rewards/margins": 2.2270336151123047,
+      "rewards/rejected": 0.41760945320129395,
       "step": 220
     },
     {
       "epoch": 1.25,
-      "grad_norm": 1054.001314243816,
+      "grad_norm": 7190.025557983883,
       "learning_rate": 3.7097251001664824e-09,
-      "logits/chosen": -1.5353752374649048,
-      "logits/rejected": -1.3762162923812866,
-      "logps/chosen": -323.80047607421875,
-      "logps/rejected": -287.00726318359375,
-      "loss": 0.5117,
-      "rewards/accuracies": 0.715624988079071,
-      "rewards/chosen": 0.8148177266120911,
-      "rewards/margins": 0.8335307240486145,
-      "rewards/rejected": -0.018712949007749557,
+      "logits/chosen": -1.5286749601364136,
+      "logits/rejected": -1.3688592910766602,
+      "logps/chosen": -324.03509521484375,
+      "logps/rejected": -286.93658447265625,
+      "loss": 0.9913,
+      "rewards/accuracies": 0.684374988079071,
+      "rewards/chosen": 2.901033401489258,
+      "rewards/margins": 2.64115834236145,
+      "rewards/rejected": 0.25987523794174194,
       "step": 230
     },
     {
       "epoch": 1.3,
-      "grad_norm": 1057.7644150535455,
+      "grad_norm": 6451.277776389874,
       "learning_rate": 3.2577328404292057e-09,
-      "logits/chosen": -1.5480402708053589,
-      "logits/rejected": -1.4182308912277222,
-      "logps/chosen": -312.3599548339844,
-      "logps/rejected": -285.9615478515625,
-      "loss": 0.5004,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 0.8805425763130188,
-      "rewards/margins": 0.8217647671699524,
-      "rewards/rejected": 0.05877774953842163,
+      "logits/chosen": -1.5419480800628662,
+      "logits/rejected": -1.4117484092712402,
+      "logps/chosen": -312.6917724609375,
+      "logps/rejected": -286.0001220703125,
+      "loss": 0.9838,
+      "rewards/accuracies": 0.684374988079071,
+      "rewards/chosen": 2.7435173988342285,
+      "rewards/margins": 2.642423629760742,
+      "rewards/rejected": 0.101093590259552,
       "step": 240
     },
     {
       "epoch": 1.36,
-      "grad_norm": 1134.7078580204877,
+      "grad_norm": 6314.813043200908,
       "learning_rate": 2.821423700565763e-09,
-      "logits/chosen": -1.597772479057312,
-      "logits/rejected": -1.4192153215408325,
-      "logps/chosen": -350.49432373046875,
-      "logps/rejected": -306.66290283203125,
-      "loss": 0.497,
-      "rewards/accuracies": 0.8031250238418579,
-      "rewards/chosen": 1.035740613937378,
-      "rewards/margins": 1.0768238306045532,
-      "rewards/rejected": -0.04108327627182007,
+      "logits/chosen": -1.597633719444275,
+      "logits/rejected": -1.419690728187561,
+      "logps/chosen": -350.88946533203125,
+      "logps/rejected": -306.6169128417969,
+      "loss": 0.9826,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": 3.2030205726623535,
+      "rewards/margins": 3.1783461570739746,
+      "rewards/rejected": 0.024674177169799805,
       "step": 250
     },
     {
       "epoch": 1.41,
-      "grad_norm": 1065.7113049843235,
+      "grad_norm": 6095.456225806253,
       "learning_rate": 2.4047251428513483e-09,
-      "logits/chosen": -1.615321397781372,
-      "logits/rejected": -1.4609696865081787,
-      "logps/chosen": -325.0920104980469,
-      "logps/rejected": -291.08197021484375,
-      "loss": 0.513,
-      "rewards/accuracies": 0.7406250238418579,
-      "rewards/chosen": 0.9730992317199707,
-      "rewards/margins": 0.9078540802001953,
-      "rewards/rejected": 0.06524516642093658,
+      "logits/chosen": -1.6133763790130615,
+      "logits/rejected": -1.4590764045715332,
+      "logps/chosen": -325.47283935546875,
+      "logps/rejected": -291.05609130859375,
+      "loss": 1.0523,
+      "rewards/accuracies": 0.6781250238418579,
+      "rewards/chosen": 2.9614436626434326,
+      "rewards/margins": 2.505828619003296,
+      "rewards/rejected": 0.45561495423316956,
       "step": 260
     },
     {
       "epoch": 1.47,
-      "grad_norm": 864.6386915761193,
+      "grad_norm": 4454.374261141978,
       "learning_rate": 2.011388103757442e-09,
-      "logits/chosen": -1.5269956588745117,
-      "logits/rejected": -1.3828452825546265,
-      "logps/chosen": -316.21124267578125,
-      "logps/rejected": -285.77667236328125,
-      "loss": 0.4948,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.9972602725028992,
-      "rewards/margins": 0.9109752774238586,
-      "rewards/rejected": 0.08628497272729874,
+      "logits/chosen": -1.5234829187393188,
+      "logits/rejected": -1.3793760538101196,
+      "logps/chosen": -316.57562255859375,
+      "logps/rejected": -285.84161376953125,
+      "loss": 1.0123,
+      "rewards/accuracies": 0.7406250238418579,
+      "rewards/chosen": 3.164306640625,
+      "rewards/margins": 3.0578055381774902,
+      "rewards/rejected": 0.10650081932544708,
       "step": 270
     },
     {
       "epoch": 1.52,
-      "grad_norm": 990.2860964249656,
+      "grad_norm": 6732.22761580186,
       "learning_rate": 1.644953229677474e-09,
-      "logits/chosen": -1.601299524307251,
-      "logits/rejected": -1.4185166358947754,
-      "logps/chosen": -325.8226623535156,
-      "logps/rejected": -284.75567626953125,
-      "loss": 0.5046,
-      "rewards/accuracies": 0.7906249761581421,
-      "rewards/chosen": 1.0880569219589233,
-      "rewards/margins": 1.0006572008132935,
-      "rewards/rejected": 0.08739979565143585,
+      "logits/chosen": -1.5963881015777588,
+      "logits/rejected": -1.4128029346466064,
+      "logps/chosen": -326.2952575683594,
+      "logps/rejected": -284.8047790527344,
+      "loss": 1.0398,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 3.0773532390594482,
+      "rewards/margins": 2.8858485221862793,
+      "rewards/rejected": 0.1915048062801361,
       "step": 280
     },
     {
       "epoch": 1.58,
-      "grad_norm": 1035.2441434858022,
+      "grad_norm": 5790.846466596631,
       "learning_rate": 1.308719005590957e-09,
-      "logits/chosen": -1.5150429010391235,
-      "logits/rejected": -1.400657057762146,
-      "logps/chosen": -318.3056640625,
-      "logps/rejected": -282.54803466796875,
-      "loss": 0.5012,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 0.9477123022079468,
-      "rewards/margins": 0.9480409622192383,
-      "rewards/rejected": -0.00032869577989913523,
+      "logits/chosen": -1.508599042892456,
+      "logits/rejected": -1.3934084177017212,
+      "logps/chosen": -318.65557861328125,
+      "logps/rejected": -282.54144287109375,
+      "loss": 1.0287,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": 2.989049196243286,
+      "rewards/margins": 2.957663059234619,
+      "rewards/rejected": 0.031385958194732666,
       "step": 290
     },
     {
       "epoch": 1.63,
-      "grad_norm": 952.917065524399,
+      "grad_norm": 5615.161677680287,
       "learning_rate": 1.005712063557776e-09,
-      "logits/chosen": -1.6322643756866455,
-      "logits/rejected": -1.4532310962677002,
-      "logps/chosen": -323.9804382324219,
-      "logps/rejected": -290.63629150390625,
-      "loss": 0.505,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": 0.903466522693634,
-      "rewards/margins": 0.8820658922195435,
-      "rewards/rejected": 0.02140064910054207,
+      "logits/chosen": -1.6266453266143799,
+      "logits/rejected": -1.4476075172424316,
+      "logps/chosen": -324.32855224609375,
+      "logps/rejected": -290.5617370605469,
+      "loss": 1.0124,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.7768070697784424,
+      "rewards/margins": 2.2968738079071045,
+      "rewards/rejected": 0.47993311285972595,
       "step": 300
     },
     {
       "epoch": 1.68,
-      "grad_norm": 946.6373160602382,
+      "grad_norm": 5693.104301118471,
       "learning_rate": 7.386599383124321e-10,
-      "logits/chosen": -1.5624831914901733,
-      "logits/rejected": -1.3783166408538818,
-      "logps/chosen": -321.67547607421875,
-      "logps/rejected": -285.8205261230469,
-      "loss": 0.5001,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 0.9382842779159546,
-      "rewards/margins": 0.9216762781143188,
-      "rewards/rejected": 0.01660792902112007,
+      "logits/chosen": -1.5692319869995117,
+      "logits/rejected": -1.3874424695968628,
+      "logps/chosen": -322.054443359375,
+      "logps/rejected": -285.78363037109375,
+      "loss": 1.0301,
+      "rewards/accuracies": 0.7093750238418579,
+      "rewards/chosen": 2.796619176864624,
+      "rewards/margins": 2.5291390419006348,
+      "rewards/rejected": 0.26748019456863403,
       "step": 310
     },
     {
       "epoch": 1.74,
-      "grad_norm": 995.8621536271032,
+      "grad_norm": 5750.226909350172,
       "learning_rate": 5.099665152003929e-10,
-      "logits/chosen": -1.5874156951904297,
-      "logits/rejected": -1.3745605945587158,
-      "logps/chosen": -333.58453369140625,
-      "logps/rejected": -289.9964294433594,
-      "loss": 0.4994,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 1.030099868774414,
-      "rewards/margins": 1.0678800344467163,
-      "rewards/rejected": -0.03778017312288284,
+      "logits/chosen": -1.5892612934112549,
+      "logits/rejected": -1.377362608909607,
+      "logps/chosen": -333.9684143066406,
+      "logps/rejected": -289.87786865234375,
+      "loss": 1.0507,
+      "rewards/accuracies": 0.715624988079071,
+      "rewards/chosen": 3.231071949005127,
+      "rewards/margins": 2.8270695209503174,
+      "rewards/rejected": 0.40400204062461853,
       "step": 320
     },
     {
       "epoch": 1.79,
-      "grad_norm": 920.3253000772652,
+      "grad_norm": 5901.981652001965,
       "learning_rate": 3.216903914633745e-10,
-      "logits/chosen": -1.5468734502792358,
-      "logits/rejected": -1.4234826564788818,
-      "logps/chosen": -325.117919921875,
-      "logps/rejected": -296.07086181640625,
-      "loss": 0.5015,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.9095224142074585,
-      "rewards/margins": 0.8073747754096985,
-      "rewards/rejected": 0.1021476536989212,
+      "logits/chosen": -1.5573378801345825,
+      "logits/rejected": -1.4351418018341064,
+      "logps/chosen": -325.4165954589844,
+      "logps/rejected": -296.0995178222656,
+      "loss": 1.0543,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 3.054168939590454,
+      "rewards/margins": 2.6867241859436035,
+      "rewards/rejected": 0.367445170879364,
       "step": 330
     },
     {
       "epoch": 1.85,
-      "grad_norm": 988.4564193799931,
+      "grad_norm": 5320.494017847622,
       "learning_rate": 1.7552634565570324e-10,
-      "logits/chosen": -1.5551540851593018,
-      "logits/rejected": -1.3869084119796753,
-      "logps/chosen": -329.73553466796875,
-      "logps/rejected": -292.8804931640625,
-      "loss": 0.4969,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 1.0978221893310547,
-      "rewards/margins": 1.0354284048080444,
-      "rewards/rejected": 0.0623936764895916,
+      "logits/chosen": -1.5548150539398193,
+      "logits/rejected": -1.3869941234588623,
+      "logps/chosen": -330.1687316894531,
+      "logps/rejected": -292.9322509765625,
+      "loss": 0.9934,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 3.3230159282684326,
+      "rewards/margins": 3.269806385040283,
+      "rewards/rejected": 0.053209513425827026,
       "step": 340
     },
     {
       "epoch": 1.9,
-      "grad_norm": 978.2289689829179,
+      "grad_norm": 5796.257294954858,
       "learning_rate": 7.279008199590543e-11,
-      "logits/chosen": -1.5398132801055908,
-      "logits/rejected": -1.3765273094177246,
-      "logps/chosen": -326.29425048828125,
-      "logps/rejected": -292.0558166503906,
-      "loss": 0.4951,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 1.0201548337936401,
-      "rewards/margins": 1.052316665649414,
-      "rewards/rejected": -0.03216180205345154,
+      "logits/chosen": -1.5367571115493774,
+      "logits/rejected": -1.3734115362167358,
+      "logps/chosen": -326.6300048828125,
+      "logps/rejected": -291.9888916015625,
+      "loss": 1.0303,
+      "rewards/accuracies": 0.715624988079071,
+      "rewards/chosen": 3.4221725463867188,
+      "rewards/margins": 3.2483201026916504,
+      "rewards/rejected": 0.1738525927066803,
       "step": 350
     },
     {
       "epoch": 1.96,
-      "grad_norm": 921.2000894790062,
+      "grad_norm": 5941.322275258092,
       "learning_rate": 1.4406386978128017e-11,
-      "logits/chosen": -1.629494309425354,
-      "logits/rejected": -1.434570074081421,
-      "logps/chosen": -330.90911865234375,
-      "logps/rejected": -291.73638916015625,
-      "loss": 0.4873,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 1.1673272848129272,
-      "rewards/margins": 1.1191251277923584,
-      "rewards/rejected": 0.048202164471149445,
+      "logits/chosen": -1.6203804016113281,
+      "logits/rejected": -1.4243013858795166,
+      "logps/chosen": -331.37005615234375,
+      "logps/rejected": -291.6743469238281,
+      "loss": 0.9963,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 3.5317084789276123,
+      "rewards/margins": 2.9804892539978027,
+      "rewards/rejected": 0.5512194633483887,
       "step": 360
     },
     {
       "epoch": 2.0,
       "step": 368,
       "total_flos": 0.0,
-      "train_loss": 0.577453197668428,
-      "train_runtime": 9953.5163,
-      "train_samples_per_second": 9.463,
+      "train_loss": 1.1851136749205382,
+      "train_runtime": 9917.443,
+      "train_samples_per_second": 9.497,
       "train_steps_per_second": 0.037
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6f94bf159e3c09d5755b9d7f6e04d5c212f5352b8cf00853f316d50c78e1c689
+oid sha256:fdeda5a8d4bfbc0a95a9fc5768806e3f1fdf2e395d6ef47d92af3478e9533580
 size 6328