Model save

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 2.0,
-    "train_loss": 1.1851136749205382,
-    "train_runtime": 9917.443,
+    "train_loss": 2.18696221968402,
+    "train_runtime": 9935.8575,
     "train_samples": 47095,
-    "train_samples_per_second": 9.497,
+    "train_samples_per_second": 9.48,
     "train_steps_per_second": 0.037
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee032bde6fa6760b2c2a0c6fd438619c8124f4fd40426fd16ea7103737c8b3a1
+oid sha256:968c07796cbf011ade16175ecaf2a7282954a19f0f65d71b98de391ae153d590
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:21ad507b64d6a5bac882bc60d28e30ace07790d869be8ebc579669104f91f2a7
+oid sha256:580125ee0c6c30674d33019962d59b99eff37a67138f1c229a205dcf74fd1460
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cda087bfd4990777fd9795c39e3bef165695ce638c3fa33eeb2a9a659aabaad9
+oid sha256:9ab37cd78594d5ee181eb3674af0003a5c70a68f3b9135f60d053d84a3912ebf
 size 4540516344

runs/Jun20_04-22-11_n136-129-074/events.out.tfevents.1718828554.n136-129-074.3454650.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b91bf15530cda1f9f3d52b7365e3ea1aa4d3b990b393a844846b949f834eb0e2
-size 26109
+oid sha256:11ed4983e41960c8a2bd371b18f0708389224337cf6a997f60d75179551aa46c
+size 30591

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 2.0,
-    "train_loss": 1.1851136749205382,
-    "train_runtime": 9917.443,
+    "train_loss": 2.18696221968402,
+    "train_runtime": 9935.8575,
     "train_samples": 47095,
-    "train_samples_per_second": 9.497,
+    "train_samples_per_second": 9.48,
     "train_steps_per_second": 0.037
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.01,
-      "grad_norm": 6710.683257043726,
+      "grad_norm": 13421.071613225095,
       "learning_rate": 2.702702702702703e-10,
       "logits/chosen": -1.3332719802856445,
       "logits/rejected": -1.246394395828247,
@@ -25,551 +25,551 @@
     },
     {
       "epoch": 0.05,
-      "grad_norm": 8627.07198024136,
+      "grad_norm": 20347.808666602745,
       "learning_rate": 2.702702702702703e-09,
-      "logits/chosen": -1.6131304502487183,
-      "logits/rejected": -1.3906824588775635,
-      "logps/chosen": -342.4801025390625,
-      "logps/rejected": -294.5570068359375,
-      "loss": 1.5693,
-      "rewards/accuracies": 0.4861111044883728,
-      "rewards/chosen": 0.2733865976333618,
-      "rewards/margins": 0.37529557943344116,
-      "rewards/rejected": -0.10190902650356293,
+      "logits/chosen": -1.6190177202224731,
+      "logits/rejected": -1.3982949256896973,
+      "logps/chosen": -342.49090576171875,
+      "logps/rejected": -294.5392761230469,
+      "loss": 2.904,
+      "rewards/accuracies": 0.4548611044883728,
+      "rewards/chosen": 0.43856528401374817,
+      "rewards/margins": 0.4652646481990814,
+      "rewards/rejected": -0.026699384674429893,
       "step": 10
     },
     {
       "epoch": 0.11,
-      "grad_norm": 7866.070005970393,
+      "grad_norm": 14961.634137592142,
       "learning_rate": 5.405405405405406e-09,
-      "logits/chosen": -1.4882968664169312,
-      "logits/rejected": -1.310754418373108,
-      "logps/chosen": -314.77069091796875,
-      "logps/rejected": -279.344970703125,
-      "loss": 1.5992,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": -0.023275405168533325,
-      "rewards/margins": 0.17254583537578583,
-      "rewards/rejected": -0.19582125544548035,
+      "logits/chosen": -1.4918005466461182,
+      "logits/rejected": -1.3144338130950928,
+      "logps/chosen": -314.73779296875,
+      "logps/rejected": -279.31842041015625,
+      "loss": 2.9253,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.2821510434150696,
+      "rewards/margins": 0.4083018898963928,
+      "rewards/rejected": -0.12615084648132324,
       "step": 20
     },
     {
       "epoch": 0.16,
-      "grad_norm": 9136.009378303972,
+      "grad_norm": 18492.420742404967,
       "learning_rate": 8.108108108108109e-09,
-      "logits/chosen": -1.5432701110839844,
-      "logits/rejected": -1.3749229907989502,
-      "logps/chosen": -324.8939208984375,
-      "logps/rejected": -286.3031005859375,
-      "loss": 1.7105,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": 0.060706138610839844,
-      "rewards/margins": 0.18149089813232422,
-      "rewards/rejected": -0.12078475952148438,
+      "logits/chosen": -1.5456618070602417,
+      "logits/rejected": -1.377851128578186,
+      "logps/chosen": -324.8970642089844,
+      "logps/rejected": -286.29644775390625,
+      "loss": 3.069,
+      "rewards/accuracies": 0.484375,
+      "rewards/chosen": 0.0898141860961914,
+      "rewards/margins": 0.26539188623428345,
+      "rewards/rejected": -0.17557768523693085,
       "step": 30
     },
     {
       "epoch": 0.22,
-      "grad_norm": 8253.74733424752,
+      "grad_norm": 16172.032582850023,
       "learning_rate": 9.997973265157192e-09,
-      "logits/chosen": -1.530524730682373,
-      "logits/rejected": -1.3523082733154297,
-      "logps/chosen": -325.4083557128906,
-      "logps/rejected": -285.64794921875,
-      "loss": 1.688,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.07491829246282578,
-      "rewards/margins": -0.08290421962738037,
-      "rewards/rejected": 0.007985919713973999,
+      "logits/chosen": -1.5278599262237549,
+      "logits/rejected": -1.348915934562683,
+      "logps/chosen": -325.442626953125,
+      "logps/rejected": -285.65606689453125,
+      "loss": 3.1235,
+      "rewards/accuracies": 0.49687498807907104,
+      "rewards/chosen": -0.4925743043422699,
+      "rewards/margins": -0.42746010422706604,
+      "rewards/rejected": -0.06511423736810684,
       "step": 40
     },
     {
       "epoch": 0.27,
-      "grad_norm": 7989.013903323287,
+      "grad_norm": 16865.011165666692,
       "learning_rate": 9.961988113473708e-09,
-      "logits/chosen": -1.537957787513733,
-      "logits/rejected": -1.390915870666504,
-      "logps/chosen": -336.9923400878906,
-      "logps/rejected": -297.2927551269531,
-      "loss": 1.5308,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": 0.14283373951911926,
-      "rewards/margins": 0.05056797340512276,
-      "rewards/rejected": 0.09226574748754501,
+      "logits/chosen": -1.5399911403656006,
+      "logits/rejected": -1.3937371969223022,
+      "logps/chosen": -337.0230712890625,
+      "logps/rejected": -297.31866455078125,
+      "loss": 2.8389,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.021488964557647705,
+      "rewards/margins": 0.0530308373272419,
+      "rewards/rejected": -0.0745197981595993,
       "step": 50
     },
     {
       "epoch": 0.33,
-      "grad_norm": 7487.424543983984,
+      "grad_norm": 14690.810465030829,
       "learning_rate": 9.881337335184878e-09,
-      "logits/chosen": -1.5855934619903564,
-      "logits/rejected": -1.4371620416641235,
-      "logps/chosen": -319.7807922363281,
-      "logps/rejected": -284.994140625,
-      "loss": 1.4965,
-      "rewards/accuracies": 0.6031249761581421,
-      "rewards/chosen": 0.3105394244194031,
-      "rewards/margins": 0.8686850666999817,
-      "rewards/rejected": -0.5581457018852234,
+      "logits/chosen": -1.58315908908844,
+      "logits/rejected": -1.4344873428344727,
+      "logps/chosen": -319.7967529296875,
+      "logps/rejected": -285.00897216796875,
+      "loss": 2.6638,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.46145668625831604,
+      "rewards/margins": 1.7262779474258423,
+      "rewards/rejected": -1.2648210525512695,
       "step": 60
     },
     {
       "epoch": 0.38,
-      "grad_norm": 8830.269579570355,
+      "grad_norm": 19235.210685128586,
       "learning_rate": 9.756746912994832e-09,
-      "logits/chosen": -1.5150383710861206,
-      "logits/rejected": -1.354196310043335,
-      "logps/chosen": -312.10052490234375,
-      "logps/rejected": -275.0794982910156,
-      "loss": 1.409,
-      "rewards/accuracies": 0.565625011920929,
-      "rewards/chosen": 0.03486788272857666,
-      "rewards/margins": 0.5497146844863892,
-      "rewards/rejected": -0.5148467421531677,
+      "logits/chosen": -1.5140897035598755,
+      "logits/rejected": -1.353376865386963,
+      "logps/chosen": -312.042236328125,
+      "logps/rejected": -275.03875732421875,
+      "loss": 2.6382,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.6530985236167908,
+      "rewards/margins": 1.2754881381988525,
+      "rewards/rejected": -0.6223896741867065,
       "step": 70
     },
     {
       "epoch": 0.43,
-      "grad_norm": 7046.728190784739,
+      "grad_norm": 14005.352374602855,
       "learning_rate": 9.589338354885628e-09,
-      "logits/chosen": -1.593528389930725,
-      "logits/rejected": -1.4396848678588867,
-      "logps/chosen": -323.3490295410156,
-      "logps/rejected": -288.1017150878906,
-      "loss": 1.4073,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.1715652495622635,
-      "rewards/margins": 0.8374398946762085,
-      "rewards/rejected": -0.6658747792243958,
+      "logits/chosen": -1.5981298685073853,
+      "logits/rejected": -1.4447776079177856,
+      "logps/chosen": -323.3012390136719,
+      "logps/rejected": -288.0871887207031,
+      "loss": 2.5625,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.8214060664176941,
+      "rewards/margins": 2.007897138595581,
+      "rewards/rejected": -1.186490774154663,
       "step": 80
     },
     {
       "epoch": 0.49,
-      "grad_norm": 6929.551893900645,
+      "grad_norm": 13300.546208456732,
       "learning_rate": 9.380618598797472e-09,
-      "logits/chosen": -1.6050440073013306,
-      "logits/rejected": -1.4080655574798584,
-      "logps/chosen": -319.95452880859375,
-      "logps/rejected": -281.72833251953125,
-      "loss": 1.3787,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": 0.6286121606826782,
-      "rewards/margins": 1.2058827877044678,
-      "rewards/rejected": -0.5772705078125,
+      "logits/chosen": -1.614319086074829,
+      "logits/rejected": -1.418944239616394,
+      "logps/chosen": -319.93804931640625,
+      "logps/rejected": -281.74871826171875,
+      "loss": 2.4466,
+      "rewards/accuracies": 0.590624988079071,
+      "rewards/chosen": 1.4227933883666992,
+      "rewards/margins": 2.7808287143707275,
+      "rewards/rejected": -1.3580353260040283,
       "step": 90
     },
     {
       "epoch": 0.54,
-      "grad_norm": 6275.615806226439,
+      "grad_norm": 14640.023348603876,
       "learning_rate": 9.132466447838596e-09,
-      "logits/chosen": -1.5423895120620728,
-      "logits/rejected": -1.3670966625213623,
-      "logps/chosen": -321.89300537109375,
-      "logps/rejected": -282.581787109375,
-      "loss": 1.2362,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 1.1018999814987183,
-      "rewards/margins": 1.560381531715393,
-      "rewards/rejected": -0.45848163962364197,
+      "logits/chosen": -1.5410282611846924,
+      "logits/rejected": -1.3655275106430054,
+      "logps/chosen": -321.95489501953125,
+      "logps/rejected": -282.6141662597656,
+      "loss": 2.5501,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": 1.5853115320205688,
+      "rewards/margins": 2.8258020877838135,
+      "rewards/rejected": -1.240490198135376,
       "step": 100
     },
     {
       "epoch": 0.6,
-      "grad_norm": 6651.119283999509,
+      "grad_norm": 13236.809764267844,
       "learning_rate": 8.847115658129039e-09,
-      "logits/chosen": -1.512652039527893,
-      "logits/rejected": -1.3851889371871948,
-      "logps/chosen": -318.17474365234375,
-      "logps/rejected": -287.15032958984375,
-      "loss": 1.2039,
-      "rewards/accuracies": 0.6343749761581421,
-      "rewards/chosen": 0.8460339307785034,
-      "rewards/margins": 1.2877717018127441,
-      "rewards/rejected": -0.44173774123191833,
+      "logits/chosen": -1.505202293395996,
+      "logits/rejected": -1.3764159679412842,
+      "logps/chosen": -318.1921081542969,
+      "logps/rejected": -287.1940002441406,
+      "loss": 2.1932,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": 1.5191149711608887,
+      "rewards/margins": 2.8392601013183594,
+      "rewards/rejected": -1.3201450109481812,
       "step": 110
     },
     {
       "epoch": 0.65,
-      "grad_norm": 6722.274057952871,
+      "grad_norm": 14131.088229346033,
       "learning_rate": 8.527134831514116e-09,
-      "logits/chosen": -1.5804601907730103,
-      "logits/rejected": -1.4257571697235107,
-      "logps/chosen": -331.40728759765625,
-      "logps/rejected": -297.8313903808594,
-      "loss": 1.2367,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 1.094892978668213,
-      "rewards/margins": 1.075182318687439,
-      "rewards/rejected": 0.019710630178451538,
+      "logits/chosen": -1.5827970504760742,
+      "logits/rejected": -1.4286653995513916,
+      "logps/chosen": -331.36956787109375,
+      "logps/rejected": -297.8145446777344,
+      "loss": 2.3092,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": 2.5670523643493652,
+      "rewards/margins": 2.359330654144287,
+      "rewards/rejected": 0.20772182941436768,
       "step": 120
     },
     {
       "epoch": 0.71,
-      "grad_norm": 5891.724908819839,
+      "grad_norm": 12432.379320236165,
       "learning_rate": 8.175404294144481e-09,
-      "logits/chosen": -1.6209514141082764,
-      "logits/rejected": -1.4345628023147583,
-      "logps/chosen": -317.20465087890625,
-      "logps/rejected": -271.5218811035156,
-      "loss": 1.1341,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 1.5204100608825684,
-      "rewards/margins": 1.7521209716796875,
-      "rewards/rejected": -0.23171091079711914,
+      "logits/chosen": -1.6129405498504639,
+      "logits/rejected": -1.4252656698226929,
+      "logps/chosen": -317.1763610839844,
+      "logps/rejected": -271.4945068359375,
+      "loss": 2.1552,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": 3.3237526416778564,
+      "rewards/margins": 3.513510227203369,
+      "rewards/rejected": -0.1897575855255127,
       "step": 130
     },
     {
       "epoch": 0.76,
-      "grad_norm": 6773.700242649663,
+      "grad_norm": 14125.607348595095,
       "learning_rate": 7.79509016905158e-09,
-      "logits/chosen": -1.5730321407318115,
-      "logits/rejected": -1.4252164363861084,
-      "logps/chosen": -331.2398681640625,
-      "logps/rejected": -294.24530029296875,
-      "loss": 1.2492,
-      "rewards/accuracies": 0.6781250238418579,
-      "rewards/chosen": 1.7606897354125977,
-      "rewards/margins": 1.846710443496704,
-      "rewards/rejected": -0.08602098375558853,
+      "logits/chosen": -1.5669504404067993,
+      "logits/rejected": -1.4181562662124634,
+      "logps/chosen": -331.1703186035156,
+      "logps/rejected": -294.15667724609375,
+      "loss": 2.238,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": 4.217373847961426,
+      "rewards/margins": 3.5030651092529297,
+      "rewards/rejected": 0.7143087387084961,
       "step": 140
     },
     {
       "epoch": 0.82,
-      "grad_norm": 6236.159184960732,
+      "grad_norm": 13250.148461496135,
       "learning_rate": 7.389615876105773e-09,
-      "logits/chosen": -1.551709532737732,
-      "logits/rejected": -1.4236745834350586,
-      "logps/chosen": -314.64178466796875,
-      "logps/rejected": -291.77093505859375,
-      "loss": 1.1633,
-      "rewards/accuracies": 0.628125011920929,
-      "rewards/chosen": 2.1191415786743164,
-      "rewards/margins": 1.7847158908843994,
-      "rewards/rejected": 0.33442583680152893,
+      "logits/chosen": -1.5488533973693848,
+      "logits/rejected": -1.4203182458877563,
+      "logps/chosen": -314.6199645996094,
+      "logps/rejected": -291.7601623535156,
+      "loss": 2.2017,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 4.456366539001465,
+      "rewards/margins": 3.679431200027466,
+      "rewards/rejected": 0.7769355773925781,
       "step": 150
     },
     {
       "epoch": 0.87,
-      "grad_norm": 6133.347836072311,
+      "grad_norm": 12949.50039152227,
       "learning_rate": 6.962631315901861e-09,
-      "logits/chosen": -1.526906967163086,
-      "logits/rejected": -1.411745548248291,
-      "logps/chosen": -318.1049499511719,
-      "logps/rejected": -290.96685791015625,
-      "loss": 1.1827,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 2.0592172145843506,
-      "rewards/margins": 1.2991172075271606,
-      "rewards/rejected": 0.7601001262664795,
+      "logits/chosen": -1.5129776000976562,
+      "logits/rejected": -1.396781325340271,
+      "logps/chosen": -318.1156005859375,
+      "logps/rejected": -291.0067138671875,
+      "loss": 2.2242,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": 4.011959075927734,
+      "rewards/margins": 2.890659809112549,
+      "rewards/rejected": 1.1212995052337646,
       "step": 160
     },
     {
       "epoch": 0.92,
-      "grad_norm": 5643.626720023826,
+      "grad_norm": 12255.51409729144,
       "learning_rate": 6.517980014965139e-09,
-      "logits/chosen": -1.599163293838501,
-      "logits/rejected": -1.4116249084472656,
-      "logps/chosen": -331.5318603515625,
-      "logps/rejected": -289.4364929199219,
-      "loss": 1.1298,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 2.287135601043701,
-      "rewards/margins": 1.9960601329803467,
-      "rewards/rejected": 0.29107528924942017,
+      "logits/chosen": -1.5933173894882202,
+      "logits/rejected": -1.4045393466949463,
+      "logps/chosen": -331.5554504394531,
+      "logps/rejected": -289.4686584472656,
+      "loss": 2.1595,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": 4.3387274742126465,
+      "rewards/margins": 4.077757358551025,
+      "rewards/rejected": 0.2609703540802002,
       "step": 170
     },
     {
       "epoch": 0.98,
-      "grad_norm": 5637.630920957477,
+      "grad_norm": 11568.19856890153,
       "learning_rate": 6.059664528022266e-09,
-      "logits/chosen": -1.5944277048110962,
-      "logits/rejected": -1.4438059329986572,
-      "logps/chosen": -315.2278747558594,
-      "logps/rejected": -276.7435607910156,
-      "loss": 1.0907,
-      "rewards/accuracies": 0.6968749761581421,
-      "rewards/chosen": 2.1914913654327393,
-      "rewards/margins": 2.4074912071228027,
-      "rewards/rejected": -0.21599988639354706,
+      "logits/chosen": -1.5902655124664307,
+      "logits/rejected": -1.4390876293182373,
+      "logps/chosen": -315.20379638671875,
+      "logps/rejected": -276.70684814453125,
+      "loss": 2.1416,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 4.6242899894714355,
+      "rewards/margins": 4.689143180847168,
+      "rewards/rejected": -0.06485319137573242,
       "step": 180
     },
     {
       "epoch": 1.03,
-      "grad_norm": 6180.318875375917,
+      "grad_norm": 11270.506557264835,
       "learning_rate": 5.591810408770492e-09,
-      "logits/chosen": -1.5520689487457275,
-      "logits/rejected": -1.3785512447357178,
-      "logps/chosen": -315.73565673828125,
-      "logps/rejected": -278.69659423828125,
-      "loss": 1.0339,
-      "rewards/accuracies": 0.6656249761581421,
-      "rewards/chosen": 2.1935009956359863,
-      "rewards/margins": 2.327456474304199,
-      "rewards/rejected": -0.1339556723833084,
+      "logits/chosen": -1.553971529006958,
+      "logits/rejected": -1.380243182182312,
+      "logps/chosen": -315.759033203125,
+      "logps/rejected": -278.6817932128906,
+      "loss": 1.8829,
+      "rewards/accuracies": 0.659375011920929,
+      "rewards/chosen": 4.153166770935059,
+      "rewards/margins": 4.272480010986328,
+      "rewards/rejected": -0.11931288242340088,
       "step": 190
     },
     {
       "epoch": 1.09,
-      "grad_norm": 6463.364092385962,
+      "grad_norm": 13205.342425147532,
       "learning_rate": 5.118629073464423e-09,
-      "logits/chosen": -1.56984543800354,
-      "logits/rejected": -1.3597148656845093,
-      "logps/chosen": -326.0714416503906,
-      "logps/rejected": -282.65838623046875,
-      "loss": 1.0602,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 2.7320990562438965,
-      "rewards/margins": 2.1956944465637207,
-      "rewards/rejected": 0.5364044904708862,
+      "logits/chosen": -1.5635509490966797,
+      "logits/rejected": -1.3522040843963623,
+      "logps/chosen": -326.10589599609375,
+      "logps/rejected": -282.65509033203125,
+      "loss": 2.0122,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 5.118946075439453,
+      "rewards/margins": 4.013725757598877,
+      "rewards/rejected": 1.105220079421997,
       "step": 200
     },
     {
       "epoch": 1.14,
-      "grad_norm": 5690.544515251583,
+      "grad_norm": 12686.21644324199,
       "learning_rate": 4.644379891605983e-09,
-      "logits/chosen": -1.6162197589874268,
-      "logits/rejected": -1.4398715496063232,
-      "logps/chosen": -324.8927001953125,
-      "logps/rejected": -291.322998046875,
-      "loss": 1.0255,
-      "rewards/accuracies": 0.690625011920929,
-      "rewards/chosen": 2.447072982788086,
-      "rewards/margins": 2.282660961151123,
-      "rewards/rejected": 0.1644122153520584,
+      "logits/chosen": -1.6106517314910889,
+      "logits/rejected": -1.4335637092590332,
+      "logps/chosen": -324.8961486816406,
+      "logps/rejected": -291.36163330078125,
+      "loss": 1.9558,
+      "rewards/accuracies": 0.6656249761581421,
+      "rewards/chosen": 4.859737873077393,
+      "rewards/margins": 4.917017459869385,
+      "rewards/rejected": -0.05727982521057129,
       "step": 210
     },
     {
       "epoch": 1.2,
-      "grad_norm": 6400.665884048493,
+      "grad_norm": 12142.898576557665,
       "learning_rate": 4.173331844980362e-09,
-      "logits/chosen": -1.5352128744125366,
-      "logits/rejected": -1.4104220867156982,
-      "logps/chosen": -324.0686340332031,
-      "logps/rejected": -293.37646484375,
-      "loss": 1.0832,
-      "rewards/accuracies": 0.6968749761581421,
-      "rewards/chosen": 2.6446428298950195,
-      "rewards/margins": 2.2270336151123047,
-      "rewards/rejected": 0.41760945320129395,
+      "logits/chosen": -1.5359071493148804,
+      "logits/rejected": -1.4112781286239624,
+      "logps/chosen": -324.1145324707031,
+      "logps/rejected": -293.42950439453125,
+      "loss": 1.9354,
+      "rewards/accuracies": 0.6781250238418579,
+      "rewards/chosen": 4.830060005187988,
+      "rewards/margins": 4.525102138519287,
+      "rewards/rejected": 0.3049588203430176,
       "step": 220
     },
     {
       "epoch": 1.25,
-      "grad_norm": 7190.025557983883,
+      "grad_norm": 12759.193686594554,
       "learning_rate": 3.7097251001664824e-09,
-      "logits/chosen": -1.5286749601364136,
-      "logits/rejected": -1.3688592910766602,
-      "logps/chosen": -324.03509521484375,
-      "logps/rejected": -286.93658447265625,
-      "loss": 0.9913,
-      "rewards/accuracies": 0.684374988079071,
-      "rewards/chosen": 2.901033401489258,
-      "rewards/margins": 2.64115834236145,
-      "rewards/rejected": 0.25987523794174194,
+      "logits/chosen": -1.5307817459106445,
+      "logits/rejected": -1.3716728687286377,
+      "logps/chosen": -324.10162353515625,
+      "logps/rejected": -286.9132995605469,
+      "loss": 1.856,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 5.137416362762451,
+      "rewards/margins": 4.384617805480957,
+      "rewards/rejected": 0.7527987360954285,
       "step": 230
     },
     {
       "epoch": 1.3,
-      "grad_norm": 6451.277776389874,
+      "grad_norm": 13355.016786310944,
       "learning_rate": 3.2577328404292057e-09,
-      "logits/chosen": -1.5419480800628662,
-      "logits/rejected": -1.4117484092712402,
-      "logps/chosen": -312.6917724609375,
-      "logps/rejected": -286.0001220703125,
-      "loss": 0.9838,
-      "rewards/accuracies": 0.684374988079071,
-      "rewards/chosen": 2.7435173988342285,
-      "rewards/margins": 2.642423629760742,
-      "rewards/rejected": 0.101093590259552,
+      "logits/chosen": -1.5365327596664429,
+      "logits/rejected": -1.4058964252471924,
+      "logps/chosen": -312.64398193359375,
+      "logps/rejected": -285.9228515625,
+      "loss": 1.8652,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 5.964874267578125,
+      "rewards/margins": 4.990009307861328,
+      "rewards/rejected": 0.9748651385307312,
       "step": 240
     },
     {
       "epoch": 1.36,
-      "grad_norm": 6314.813043200908,
+      "grad_norm": 13138.491204289365,
       "learning_rate": 2.821423700565763e-09,
-      "logits/chosen": -1.597633719444275,
-      "logits/rejected": -1.419690728187561,
-      "logps/chosen": -350.88946533203125,
-      "logps/rejected": -306.6169128417969,
-      "loss": 0.9826,
-      "rewards/accuracies": 0.734375,
-      "rewards/chosen": 3.2030205726623535,
-      "rewards/margins": 3.1783461570739746,
-      "rewards/rejected": 0.024674177169799805,
+      "logits/chosen": -1.5989090204238892,
+      "logits/rejected": -1.4208343029022217,
+      "logps/chosen": -350.8832092285156,
+      "logps/rejected": -306.54180908203125,
+      "loss": 1.7277,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 6.4689040184021,
+      "rewards/margins": 5.668680191040039,
+      "rewards/rejected": 0.8002230525016785,
       "step": 250
     },
     {
       "epoch": 1.41,
-      "grad_norm": 6095.456225806253,
+      "grad_norm": 13320.645189206805,
       "learning_rate": 2.4047251428513483e-09,
-      "logits/chosen": -1.6133763790130615,
-      "logits/rejected": -1.4590764045715332,
-      "logps/chosen": -325.47283935546875,
-      "logps/rejected": -291.05609130859375,
-      "loss": 1.0523,
-      "rewards/accuracies": 0.6781250238418579,
-      "rewards/chosen": 2.9614436626434326,
-      "rewards/margins": 2.505828619003296,
-      "rewards/rejected": 0.45561495423316956,
+      "logits/chosen": -1.6121330261230469,
+      "logits/rejected": -1.4577114582061768,
+      "logps/chosen": -325.4482421875,
+      "logps/rejected": -291.07183837890625,
+      "loss": 1.9519,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": 6.1689534187316895,
+      "rewards/margins": 5.415326118469238,
+      "rewards/rejected": 0.7536273002624512,
       "step": 260
     },
     {
       "epoch": 1.47,
-      "grad_norm": 4454.374261141978,
+      "grad_norm": 10367.972989627691,
       "learning_rate": 2.011388103757442e-09,
-      "logits/chosen": -1.5234829187393188,
-      "logits/rejected": -1.3793760538101196,
-      "logps/chosen": -316.57562255859375,
-      "logps/rejected": -285.84161376953125,
-      "loss": 1.0123,
-      "rewards/accuracies": 0.7406250238418579,
-      "rewards/chosen": 3.164306640625,
-      "rewards/margins": 3.0578055381774902,
-      "rewards/rejected": 0.10650081932544708,
+      "logits/chosen": -1.5253089666366577,
+      "logits/rejected": -1.381744146347046,
+      "logps/chosen": -316.5883483886719,
+      "logps/rejected": -285.75799560546875,
+      "loss": 1.8335,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 6.201399803161621,
+      "rewards/margins": 5.151947021484375,
+      "rewards/rejected": 1.0494521856307983,
       "step": 270
     },
     {
       "epoch": 1.52,
-      "grad_norm": 6732.22761580186,
+      "grad_norm": 11861.49317635369,
       "learning_rate": 1.644953229677474e-09,
-      "logits/chosen": -1.5963881015777588,
-      "logits/rejected": -1.4128029346466064,
-      "logps/chosen": -326.2952575683594,
-      "logps/rejected": -284.8047790527344,
-      "loss": 1.0398,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 3.0773532390594482,
-      "rewards/margins": 2.8858485221862793,
-      "rewards/rejected": 0.1915048062801361,
+      "logits/chosen": -1.5998437404632568,
+      "logits/rejected": -1.4171390533447266,
+      "logps/chosen": -326.30133056640625,
+      "logps/rejected": -284.77996826171875,
+      "loss": 1.8816,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 6.093815803527832,
+      "rewards/margins": 5.462450981140137,
+      "rewards/rejected": 0.6313648223876953,
       "step": 280
     },
     {
       "epoch": 1.58,
-      "grad_norm": 5790.846466596631,
+      "grad_norm": 12967.35903281612,
       "learning_rate": 1.308719005590957e-09,
-      "logits/chosen": -1.508599042892456,
-      "logits/rejected": -1.3934084177017212,
-      "logps/chosen": -318.65557861328125,
-      "logps/rejected": -282.54144287109375,
-      "loss": 1.0287,
-      "rewards/accuracies": 0.703125,
-      "rewards/chosen": 2.989049196243286,
-      "rewards/margins": 2.957663059234619,
-      "rewards/rejected": 0.031385958194732666,
+      "logits/chosen": -1.5077544450759888,
+      "logits/rejected": -1.392764687538147,
+      "logps/chosen": -318.66082763671875,
+      "logps/rejected": -282.51806640625,
+      "loss": 1.8195,
+      "rewards/accuracies": 0.7093750238418579,
+      "rewards/chosen": 5.925144195556641,
+      "rewards/margins": 5.62863826751709,
+      "rewards/rejected": 0.29650676250457764,
       "step": 290
     },
     {
       "epoch": 1.63,
-      "grad_norm": 5615.161677680287,
+      "grad_norm": 10720.865204139644,
       "learning_rate": 1.005712063557776e-09,
-      "logits/chosen": -1.6266453266143799,
-      "logits/rejected": -1.4476075172424316,
-      "logps/chosen": -324.32855224609375,
-      "logps/rejected": -290.5617370605469,
-      "loss": 1.0124,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 2.7768070697784424,
-      "rewards/margins": 2.2968738079071045,
-      "rewards/rejected": 0.47993311285972595,
+      "logits/chosen": -1.632625937461853,
+      "logits/rejected": -1.4543178081512451,
+      "logps/chosen": -324.3389587402344,
+      "logps/rejected": -290.5496520996094,
+      "loss": 1.7503,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": 5.449400901794434,
+      "rewards/margins": 4.368684768676758,
+      "rewards/rejected": 1.0807160139083862,
       "step": 300
     },
     {
       "epoch": 1.68,
-      "grad_norm": 5693.104301118471,
+      "grad_norm": 10876.995374840744,
       "learning_rate": 7.386599383124321e-10,
-      "logits/chosen": -1.5692319869995117,
-      "logits/rejected": -1.3874424695968628,
-      "logps/chosen": -322.054443359375,
-      "logps/rejected": -285.78363037109375,
-      "loss": 1.0301,
-      "rewards/accuracies": 0.7093750238418579,
-      "rewards/chosen": 2.796619176864624,
-      "rewards/margins": 2.5291390419006348,
-      "rewards/rejected": 0.26748019456863403,
+      "logits/chosen": -1.5634690523147583,
+      "logits/rejected": -1.381116509437561,
+      "logps/chosen": -322.0839538574219,
+      "logps/rejected": -285.80072021484375,
+      "loss": 1.9509,
+      "rewards/accuracies": 0.6656249761581421,
+      "rewards/chosen": 5.297797203063965,
+      "rewards/margins": 4.9336748123168945,
+      "rewards/rejected": 0.36412325501441956,
       "step": 310
     },
     {
       "epoch": 1.74,
-      "grad_norm": 5750.226909350172,
+      "grad_norm": 11694.435027751537,
       "learning_rate": 5.099665152003929e-10,
-      "logits/chosen": -1.5892612934112549,
-      "logits/rejected": -1.377362608909607,
-      "logps/chosen": -333.9684143066406,
-      "logps/rejected": -289.87786865234375,
-      "loss": 1.0507,
+      "logits/chosen": -1.5935719013214111,
+      "logits/rejected": -1.3827770948410034,
+      "logps/chosen": -334.022705078125,
+      "logps/rejected": -289.9593505859375,
+      "loss": 1.8646,
       "rewards/accuracies": 0.715624988079071,
-      "rewards/chosen": 3.231071949005127,
-      "rewards/margins": 2.8270695209503174,
-      "rewards/rejected": 0.40400204062461853,
+      "rewards/chosen": 5.919422149658203,
+      "rewards/margins": 5.926436424255371,
+      "rewards/rejected": -0.00701451301574707,
       "step": 320
     },
     {
       "epoch": 1.79,
-      "grad_norm": 5901.981652001965,
+      "grad_norm": 13425.084683268695,
       "learning_rate": 3.216903914633745e-10,
-      "logits/chosen": -1.5573378801345825,
-      "logits/rejected": -1.4351418018341064,
-      "logps/chosen": -325.4165954589844,
-      "logps/rejected": -296.0995178222656,
-      "loss": 1.0543,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 3.054168939590454,
-      "rewards/margins": 2.6867241859436035,
-      "rewards/rejected": 0.367445170879364,
+      "logits/chosen": -1.5508067607879639,
+      "logits/rejected": -1.4279029369354248,
+      "logps/chosen": -325.4474792480469,
+      "logps/rejected": -296.10894775390625,
+      "loss": 1.9418,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 5.799554347991943,
+      "rewards/margins": 5.1590423583984375,
+      "rewards/rejected": 0.6405118703842163,
       "step": 330
     },
     {
       "epoch": 1.85,
-      "grad_norm": 5320.494017847622,
+      "grad_norm": 12502.360923512058,
       "learning_rate": 1.7552634565570324e-10,
-      "logits/chosen": -1.5548150539398193,
-      "logits/rejected": -1.3869941234588623,
-      "logps/chosen": -330.1687316894531,
-      "logps/rejected": -292.9322509765625,
-      "loss": 0.9934,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 3.3230159282684326,
-      "rewards/margins": 3.269806385040283,
-      "rewards/rejected": 0.053209513425827026,
+      "logits/chosen": -1.5481187105178833,
+      "logits/rejected": -1.379558801651001,
+      "logps/chosen": -330.1790466308594,
+      "logps/rejected": -292.8612365722656,
+      "loss": 1.8007,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 6.542909145355225,
+      "rewards/margins": 5.726317405700684,
+      "rewards/rejected": 0.8165918588638306,
       "step": 340
     },
     {
       "epoch": 1.9,
-      "grad_norm": 5796.257294954858,
+      "grad_norm": 12479.355591476906,
       "learning_rate": 7.279008199590543e-11,
-      "logits/chosen": -1.5367571115493774,
-      "logits/rejected": -1.3734115362167358,
-      "logps/chosen": -326.6300048828125,
-      "logps/rejected": -291.9888916015625,
-      "loss": 1.0303,
+      "logits/chosen": -1.5400382280349731,
+      "logits/rejected": -1.3772073984146118,
+      "logps/chosen": -326.68316650390625,
+      "logps/rejected": -291.93536376953125,
+      "loss": 1.8342,
       "rewards/accuracies": 0.715624988079071,
-      "rewards/chosen": 3.4221725463867188,
-      "rewards/margins": 3.2483201026916504,
-      "rewards/rejected": 0.1738525927066803,
+      "rewards/chosen": 6.312959671020508,
+      "rewards/margins": 5.429690361022949,
+      "rewards/rejected": 0.8832691311836243,
       "step": 350
     },
     {
       "epoch": 1.96,
-      "grad_norm": 5941.322275258092,
+      "grad_norm": 12177.012050569681,
       "learning_rate": 1.4406386978128017e-11,
-      "logits/chosen": -1.6203804016113281,
-      "logits/rejected": -1.4243013858795166,
-      "logps/chosen": -331.37005615234375,
-      "logps/rejected": -291.6743469238281,
-      "loss": 0.9963,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": 3.5317084789276123,
-      "rewards/margins": 2.9804892539978027,
-      "rewards/rejected": 0.5512194633483887,
+      "logits/chosen": -1.6199939250946045,
+      "logits/rejected": -1.4238550662994385,
+      "logps/chosen": -331.38580322265625,
+      "logps/rejected": -291.71551513671875,
+      "loss": 1.8517,
+      "rewards/accuracies": 0.7406250238418579,
+      "rewards/chosen": 6.9059858322143555,
+      "rewards/margins": 6.215287208557129,
+      "rewards/rejected": 0.6906987428665161,
       "step": 360
     },
     {
       "epoch": 2.0,
       "step": 368,
       "total_flos": 0.0,
-      "train_loss": 1.1851136749205382,
-      "train_runtime": 9917.443,
-      "train_samples_per_second": 9.497,
+      "train_loss": 2.18696221968402,
+      "train_runtime": 9935.8575,
+      "train_samples_per_second": 9.48,
       "train_steps_per_second": 0.037
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6eab4d0ccccc9c8df6826f88462a25d51fd04bbfc05973c90a0a7d5bc9deb1b9
+oid sha256:74d1a823b5a40d5d1a36cbfb31b324eb89870bcbf02b551d8574efc18e76332d
 size 6328