Model save

README.md

@@ -32,7 +32,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 5e-07
+- learning_rate: 5e-09
 - train_batch_size: 8
 - eval_batch_size: 8
 - seed: 42

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 1.2810401298381664,
-    "train_runtime": 5417.1403,
-    "train_samples": 48530,
-    "train_samples_per_second": 8.959,
-    "train_steps_per_second": 0.035
+    "train_loss": 0.7162073644002279,
+    "train_runtime": 4512.4874,
+    "train_samples": 38445,
+    "train_samples_per_second": 8.52,
+    "train_steps_per_second": 0.033
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b008443197fefaac049fe7a0eabc01b0151b73564d92afc7767c1fa8c71e1ca8
+oid sha256:c198eb9df08ec1da45189177d9165731677bcf8abcb6d6938bc58bf75d03977d
 size 4943178720

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9c3ff196fe6ec2e2cfd1db6b1aeb3c9df6fa0eb8b2d7025d4ac80bcd6c50ae14
+oid sha256:57087d3701236ddd033204862662fca1d328a1d40449da21fe52584fb50fefff
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:026ec669932b9619a7d776b17104d80d5e127fd258a8759a65b68cadf6158a75
+oid sha256:d22d542aa9e2e706d1339613cd7b76b3452024d5fd48f29b7543bbc9e61fc003
 size 4540532728

runs/Jun17_01-47-24_n136-112-146/events.out.tfevents.1718560498.n136-112-146.891340.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3d3f8e8736abd04ccc11ca702964740dd849bf1b59def5f9d196dc08153f92d7
-size 12362
+oid sha256:dd3d21db7f59f214e4271167668304d02bf7010ba64ca49e16ae58fc95cdcd30
+size 16132

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 1.2810401298381664,
-    "train_runtime": 5417.1403,
-    "train_samples": 48530,
-    "train_samples_per_second": 8.959,
-    "train_steps_per_second": 0.035
+    "train_loss": 0.7162073644002279,
+    "train_runtime": 4512.4874,
+    "train_samples": 38445,
+    "train_samples_per_second": 8.52,
+    "train_steps_per_second": 0.033
 }

trainer_state.json

@@ -1,22 +1,22 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9960474308300395,
+  "epoch": 0.9983361064891847,
   "eval_steps": 500,
-  "global_step": 189,
+  "global_step": 150,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.01,
-      "grad_norm": 4737.779382861946,
-      "learning_rate": 2.6315789473684208e-08,
-      "logits/chosen": -4.638427734375,
-      "logits/rejected": -4.891327857971191,
-      "logps/chosen": -198.52749633789062,
-      "logps/rejected": -147.3392791748047,
-      "loss": 2.1269,
+      "grad_norm": 1151.6129333090275,
+      "learning_rate": 3.333333333333333e-10,
+      "logits/chosen": -4.106247425079346,
+      "logits/rejected": -4.200438499450684,
+      "logps/chosen": -382.81439208984375,
+      "logps/rejected": -357.65960693359375,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -24,287 +24,242 @@
       "step": 1
     },
     {
-      "epoch": 0.05,
-      "grad_norm": 3019.619079402316,
-      "learning_rate": 2.631578947368421e-07,
-      "logits/chosen": -4.496801376342773,
-      "logits/rejected": -4.816222190856934,
-      "logps/chosen": -224.27357482910156,
-      "logps/rejected": -168.04739379882812,
-      "loss": 1.9212,
-      "rewards/accuracies": 0.5381944179534912,
-      "rewards/chosen": 0.3541475236415863,
-      "rewards/margins": 0.37169286608695984,
-      "rewards/rejected": -0.017545383423566818,
+      "epoch": 0.07,
+      "grad_norm": 1175.0279383615512,
+      "learning_rate": 3.3333333333333334e-09,
+      "logits/chosen": -4.217203617095947,
+      "logits/rejected": -4.32081413269043,
+      "logps/chosen": -334.6714172363281,
+      "logps/rejected": -313.4322509765625,
+      "loss": 0.7317,
+      "rewards/accuracies": 0.3923611044883728,
+      "rewards/chosen": -0.044815655797719955,
+      "rewards/margins": -0.03612741455435753,
+      "rewards/rejected": -0.008688241243362427,
       "step": 10
     },
     {
-      "epoch": 0.11,
-      "grad_norm": 1863.4195630562826,
-      "learning_rate": 4.999573126145131e-07,
-      "logits/chosen": -4.533459663391113,
-      "logits/rejected": -4.848563194274902,
-      "logps/chosen": -220.4309539794922,
-      "logps/rejected": -180.72413635253906,
-      "loss": 1.1783,
-      "rewards/accuracies": 0.8343750238418579,
-      "rewards/chosen": 5.777209281921387,
-      "rewards/margins": 6.331713676452637,
-      "rewards/rejected": -0.5545047521591187,
+      "epoch": 0.13,
+      "grad_norm": 1253.9538230101048,
+      "learning_rate": 4.983095894354858e-09,
+      "logits/chosen": -4.266427040100098,
+      "logits/rejected": -4.4187798500061035,
+      "logps/chosen": -313.9143371582031,
+      "logps/rejected": -288.782470703125,
+      "loss": 0.732,
+      "rewards/accuracies": 0.4781250059604645,
+      "rewards/chosen": -0.000932177877984941,
+      "rewards/margins": -0.012739461846649647,
+      "rewards/rejected": 0.011807283386588097,
       "step": 20
     },
     {
-      "epoch": 0.16,
-      "grad_norm": 1882.3555396757283,
-      "learning_rate": 4.948524419003415e-07,
-      "logits/chosen": -4.54370641708374,
-      "logits/rejected": -4.812285423278809,
-      "logps/chosen": -213.49411010742188,
-      "logps/rejected": -177.16848754882812,
-      "loss": 1.3518,
-      "rewards/accuracies": 0.8531249761581421,
-      "rewards/chosen": 8.2464017868042,
-      "rewards/margins": 12.621076583862305,
-      "rewards/rejected": -4.3746747970581055,
+      "epoch": 0.2,
+      "grad_norm": 1147.308654200848,
+      "learning_rate": 4.849231551964771e-09,
+      "logits/chosen": -4.301184177398682,
+      "logits/rejected": -4.36545991897583,
+      "logps/chosen": -308.3140563964844,
+      "logps/rejected": -285.6829833984375,
+      "loss": 0.7352,
+      "rewards/accuracies": 0.46562498807907104,
+      "rewards/chosen": -0.018803134560585022,
+      "rewards/margins": 0.008298242464661598,
+      "rewards/rejected": -0.02710137702524662,
       "step": 30
     },
     {
-      "epoch": 0.21,
-      "grad_norm": 1979.651061288252,
-      "learning_rate": 4.81409414945389e-07,
-      "logits/chosen": -4.619187831878662,
-      "logits/rejected": -4.8958845138549805,
-      "logps/chosen": -221.00082397460938,
-      "logps/rejected": -184.62203979492188,
-      "loss": 1.4689,
-      "rewards/accuracies": 0.8343750238418579,
-      "rewards/chosen": 7.427975654602051,
-      "rewards/margins": 15.529205322265625,
-      "rewards/rejected": -8.101228713989258,
+      "epoch": 0.27,
+      "grad_norm": 1152.01480268075,
+      "learning_rate": 4.588719528532341e-09,
+      "logits/chosen": -4.186487674713135,
+      "logits/rejected": -4.270204544067383,
+      "logps/chosen": -332.467041015625,
+      "logps/rejected": -307.530517578125,
+      "loss": 0.722,
+      "rewards/accuracies": 0.5531250238418579,
+      "rewards/chosen": 0.012754167430102825,
+      "rewards/margins": 0.04999501258134842,
+      "rewards/rejected": -0.03724084421992302,
       "step": 40
     },
     {
-      "epoch": 0.26,
-      "grad_norm": 2122.235483955093,
-      "learning_rate": 4.6008601790947314e-07,
-      "logits/chosen": -4.608691215515137,
-      "logits/rejected": -4.925226211547852,
-      "logps/chosen": -210.32058715820312,
-      "logps/rejected": -179.0367431640625,
-      "loss": 1.3821,
-      "rewards/accuracies": 0.859375,
-      "rewards/chosen": 6.428221225738525,
-      "rewards/margins": 16.42898941040039,
-      "rewards/rejected": -10.00076961517334,
+      "epoch": 0.33,
+      "grad_norm": 1161.340877574017,
+      "learning_rate": 4.215604094671834e-09,
+      "logits/chosen": -4.197475910186768,
+      "logits/rejected": -4.366654396057129,
+      "logps/chosen": -333.69940185546875,
+      "logps/rejected": -309.05511474609375,
+      "loss": 0.7263,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.022939234972000122,
+      "rewards/margins": -0.01653190515935421,
+      "rewards/rejected": 0.03947114199399948,
       "step": 50
     },
     {
-      "epoch": 0.32,
-      "grad_norm": 1692.2277360562514,
-      "learning_rate": 4.3160839350405605e-07,
-      "logits/chosen": -4.665585994720459,
-      "logits/rejected": -4.9272074699401855,
-      "logps/chosen": -205.7926788330078,
-      "logps/rejected": -178.56011962890625,
-      "loss": 1.3465,
-      "rewards/accuracies": 0.859375,
-      "rewards/chosen": 7.834652900695801,
-      "rewards/margins": 16.5399169921875,
-      "rewards/rejected": -8.705263137817383,
+      "epoch": 0.4,
+      "grad_norm": 1135.5326964714952,
+      "learning_rate": 3.7500000000000005e-09,
+      "logits/chosen": -4.186155796051025,
+      "logits/rejected": -4.269167900085449,
+      "logps/chosen": -323.9843444824219,
+      "logps/rejected": -308.364990234375,
+      "loss": 0.7133,
+      "rewards/accuracies": 0.484375,
+      "rewards/chosen": 0.009113344363868237,
+      "rewards/margins": -0.005971288774162531,
+      "rewards/rejected": 0.015084633603692055,
       "step": 60
     },
     {
-      "epoch": 0.37,
-      "grad_norm": 1810.8868167381333,
-      "learning_rate": 3.9694631307311825e-07,
-      "logits/chosen": -4.6464009284973145,
-      "logits/rejected": -4.913968086242676,
-      "logps/chosen": -207.1618194580078,
-      "logps/rejected": -182.61012268066406,
-      "loss": 1.3564,
-      "rewards/accuracies": 0.8656250238418579,
-      "rewards/chosen": 8.192334175109863,
-      "rewards/margins": 16.401655197143555,
-      "rewards/rejected": -8.209321975708008,
+      "epoch": 0.47,
+      "grad_norm": 1170.598014061488,
+      "learning_rate": 3.2170080817777257e-09,
+      "logits/chosen": -4.14601469039917,
+      "logits/rejected": -4.301178932189941,
+      "logps/chosen": -344.8753967285156,
+      "logps/rejected": -314.6096496582031,
+      "loss": 0.7182,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.08767497539520264,
+      "rewards/margins": 0.08200599253177643,
+      "rewards/rejected": 0.005668987520039082,
       "step": 70
     },
     {
-      "epoch": 0.42,
-      "grad_norm": 1395.6821844604426,
-      "learning_rate": 3.572801521931522e-07,
-      "logits/chosen": -4.674800395965576,
-      "logits/rejected": -4.932587623596191,
-      "logps/chosen": -202.7789764404297,
-      "logps/rejected": -184.74395751953125,
-      "loss": 1.312,
-      "rewards/accuracies": 0.846875011920929,
-      "rewards/chosen": 5.3585076332092285,
-      "rewards/margins": 16.119762420654297,
-      "rewards/rejected": -10.761255264282227,
+      "epoch": 0.53,
+      "grad_norm": 1088.822875661876,
+      "learning_rate": 2.6453620722761897e-09,
+      "logits/chosen": -4.220850944519043,
+      "logits/rejected": -4.3867106437683105,
+      "logps/chosen": -334.7141418457031,
+      "logps/rejected": -305.6958923339844,
+      "loss": 0.7175,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": 0.06255482137203217,
+      "rewards/margins": 0.04721928387880325,
+      "rewards/rejected": 0.015335534699261189,
       "step": 80
     },
     {
-      "epoch": 0.47,
-      "grad_norm": 1761.719146022038,
-      "learning_rate": 3.139606943986089e-07,
-      "logits/chosen": -4.721759796142578,
-      "logits/rejected": -4.953747272491455,
-      "logps/chosen": -199.81448364257812,
-      "logps/rejected": -178.44004821777344,
-      "loss": 1.3425,
-      "rewards/accuracies": 0.815625011920929,
-      "rewards/chosen": 7.849789619445801,
-      "rewards/margins": 15.739909172058105,
-      "rewards/rejected": -7.890120029449463,
+      "epoch": 0.6,
+      "grad_norm": 1120.1472438016,
+      "learning_rate": 2.0658795558326744e-09,
+      "logits/chosen": -4.226521968841553,
+      "logits/rejected": -4.347161293029785,
+      "logps/chosen": -327.5687561035156,
+      "logps/rejected": -308.19744873046875,
+      "loss": 0.7128,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.03935312479734421,
+      "rewards/margins": 0.012340927496552467,
+      "rewards/rejected": 0.02701219543814659,
       "step": 90
     },
     {
-      "epoch": 0.53,
-      "grad_norm": 1641.4466240114464,
-      "learning_rate": 2.684631318687185e-07,
-      "logits/chosen": -4.7313385009765625,
-      "logits/rejected": -4.984685897827148,
-      "logps/chosen": -213.2564239501953,
-      "logps/rejected": -190.69088745117188,
-      "loss": 1.3623,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 5.4120659828186035,
-      "rewards/margins": 16.620161056518555,
-      "rewards/rejected": -11.208093643188477,
+      "epoch": 0.67,
+      "grad_norm": 1147.49194388717,
+      "learning_rate": 1.5098005849021078e-09,
+      "logits/chosen": -4.208071708679199,
+      "logits/rejected": -4.417771339416504,
+      "logps/chosen": -320.9470520019531,
+      "logps/rejected": -290.018310546875,
+      "loss": 0.712,
+      "rewards/accuracies": 0.5218750238418579,
+      "rewards/chosen": 0.060337893664836884,
+      "rewards/margins": 0.03545590117573738,
+      "rewards/rejected": 0.0248819962143898,
       "step": 100
     },
     {
-      "epoch": 0.58,
-      "grad_norm": 1346.9601711684072,
-      "learning_rate": 2.2233682952712483e-07,
-      "logits/chosen": -4.668034553527832,
-      "logits/rejected": -4.953825950622559,
-      "logps/chosen": -216.8499298095703,
-      "logps/rejected": -186.10470581054688,
-      "loss": 1.1234,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 7.529428005218506,
-      "rewards/margins": 15.865753173828125,
-      "rewards/rejected": -8.336324691772461,
+      "epoch": 0.73,
+      "grad_norm": 1131.801124941311,
+      "learning_rate": 1.0071035207430352e-09,
+      "logits/chosen": -4.222798824310303,
+      "logits/rejected": -4.275721549987793,
+      "logps/chosen": -315.00030517578125,
+      "logps/rejected": -303.7385559082031,
+      "loss": 0.7022,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.08098876476287842,
+      "rewards/margins": 0.037886131554841995,
+      "rewards/rejected": 0.04310264065861702,
       "step": 110
     },
     {
-      "epoch": 0.63,
-      "grad_norm": 1999.3900490416042,
-      "learning_rate": 1.7715256327766884e-07,
-      "logits/chosen": -4.720789909362793,
-      "logits/rejected": -5.025943279266357,
-      "logps/chosen": -207.7978973388672,
-      "logps/rejected": -178.0445098876953,
-      "loss": 1.1185,
-      "rewards/accuracies": 0.859375,
-      "rewards/chosen": 8.364091873168945,
-      "rewards/margins": 16.011329650878906,
-      "rewards/rejected": -7.647237300872803,
+      "epoch": 0.8,
+      "grad_norm": 1201.8232728734756,
+      "learning_rate": 5.848888922025553e-10,
+      "logits/chosen": -4.147335052490234,
+      "logits/rejected": -4.3030242919921875,
+      "logps/chosen": -339.2781677246094,
+      "logps/rejected": -313.5749206542969,
+      "loss": 0.7129,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.08869560062885284,
+      "rewards/margins": 0.048744406551122665,
+      "rewards/rejected": 0.03995119035243988,
       "step": 120
     },
     {
-      "epoch": 0.69,
-      "grad_norm": 1568.9083661238265,
-      "learning_rate": 1.3444902911492174e-07,
-      "logits/chosen": -4.702408790588379,
-      "logits/rejected": -4.98063325881958,
-      "logps/chosen": -215.88174438476562,
-      "logps/rejected": -188.39645385742188,
-      "loss": 1.2748,
-      "rewards/accuracies": 0.8531249761581421,
-      "rewards/chosen": 7.107934474945068,
-      "rewards/margins": 16.361108779907227,
-      "rewards/rejected": -9.253173828125,
+      "epoch": 0.87,
+      "grad_norm": 1182.4206440855232,
+      "learning_rate": 2.659183991914696e-10,
+      "logits/chosen": -4.173482418060303,
+      "logits/rejected": -4.367284297943115,
+      "logps/chosen": -324.4493713378906,
+      "logps/rejected": -305.2012634277344,
+      "loss": 0.7,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": 0.08546491712331772,
+      "rewards/margins": 0.01908993348479271,
+      "rewards/rejected": 0.06637498736381531,
       "step": 130
     },
     {
-      "epoch": 0.74,
-      "grad_norm": 1390.666498149427,
-      "learning_rate": 9.56804446775518e-08,
-      "logits/chosen": -4.6129560470581055,
-      "logits/rejected": -4.923257350921631,
-      "logps/chosen": -208.4700469970703,
-      "logps/rejected": -178.78623962402344,
-      "loss": 1.0987,
-      "rewards/accuracies": 0.859375,
-      "rewards/chosen": 6.485724449157715,
-      "rewards/margins": 17.193899154663086,
-      "rewards/rejected": -10.708174705505371,
+      "epoch": 0.93,
+      "grad_norm": 1158.9214495930962,
+      "learning_rate": 6.738782355044049e-11,
+      "logits/chosen": -4.290203094482422,
+      "logits/rejected": -4.35637092590332,
+      "logps/chosen": -308.0101318359375,
+      "logps/rejected": -297.37701416015625,
+      "loss": 0.7082,
+      "rewards/accuracies": 0.5406249761581421,
+      "rewards/chosen": 0.12010886520147324,
+      "rewards/margins": 0.05074785277247429,
+      "rewards/rejected": 0.06936100870370865,
       "step": 140
     },
     {
-      "epoch": 0.79,
-      "grad_norm": 1296.5821049110084,
-      "learning_rate": 6.216702761078166e-08,
-      "logits/chosen": -4.699868202209473,
-      "logits/rejected": -4.9864583015441895,
-      "logps/chosen": -196.650146484375,
-      "logps/rejected": -168.93551635742188,
-      "loss": 1.0856,
-      "rewards/accuracies": 0.8968750238418579,
-      "rewards/chosen": 7.758223056793213,
-      "rewards/margins": 17.15636444091797,
-      "rewards/rejected": -9.398139953613281,
+      "epoch": 1.0,
+      "grad_norm": 1208.2615961890754,
+      "learning_rate": 0.0,
+      "logits/chosen": -4.257304668426514,
+      "logits/rejected": -4.3310017585754395,
+      "logps/chosen": -312.7613830566406,
+      "logps/rejected": -300.3546447753906,
+      "loss": 0.7025,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.10821112245321274,
+      "rewards/margins": 0.047078292816877365,
+      "rewards/rejected": 0.06113281846046448,
       "step": 150
     },
-    {
-      "epoch": 0.84,
-      "grad_norm": 1769.0071097352081,
-      "learning_rate": 3.5050037137906885e-08,
-      "logits/chosen": -4.634187698364258,
-      "logits/rejected": -4.958773612976074,
-      "logps/chosen": -211.03591918945312,
-      "logps/rejected": -176.72067260742188,
-      "loss": 1.1749,
-      "rewards/accuracies": 0.8531249761581421,
-      "rewards/chosen": 8.515033721923828,
-      "rewards/margins": 16.594724655151367,
-      "rewards/rejected": -8.079689979553223,
-      "step": 160
-    },
-    {
-      "epoch": 0.9,
-      "grad_norm": 1606.699013433802,
-      "learning_rate": 1.5252909846235894e-08,
-      "logits/chosen": -4.62954044342041,
-      "logits/rejected": -4.913142204284668,
-      "logps/chosen": -209.8083953857422,
-      "logps/rejected": -184.52127075195312,
-      "loss": 1.2059,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 9.714839935302734,
-      "rewards/margins": 18.106443405151367,
-      "rewards/rejected": -8.391606330871582,
-      "step": 170
-    },
-    {
-      "epoch": 0.95,
-      "grad_norm": 1242.5060745172418,
-      "learning_rate": 3.4498131616493565e-09,
-      "logits/chosen": -4.616083145141602,
-      "logits/rejected": -4.87780237197876,
-      "logps/chosen": -215.27685546875,
-      "logps/rejected": -193.55332946777344,
-      "loss": 1.1132,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 6.19677209854126,
-      "rewards/margins": 13.694157600402832,
-      "rewards/rejected": -7.497385501861572,
-      "step": 180
-    },
     {
       "epoch": 1.0,
-      "step": 189,
+      "step": 150,
       "total_flos": 0.0,
-      "train_loss": 1.2810401298381664,
-      "train_runtime": 5417.1403,
-      "train_samples_per_second": 8.959,
-      "train_steps_per_second": 0.035
+      "train_loss": 0.7162073644002279,
+      "train_runtime": 4512.4874,
+      "train_samples_per_second": 8.52,
+      "train_steps_per_second": 0.033
     }
   ],
   "logging_steps": 10,
-  "max_steps": 189,
+  "max_steps": 150,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8bb6c258154054c1d311e020aa1a0b2699cd9e54ea2ade0bdf6796fef174dbcd
+oid sha256:a9928d76a7e7521b67335c6d05e83d75eb13e444587e18aa0436ce552c75b5fd
 size 6328