Model save

README.md

@@ -37,7 +37,7 @@ The following hyperparameters were used during training:
 - learning_rate: 5e-07
 - train_batch_size: 8
 - eval_batch_size: 8
-- seed: 5
+- seed: 1
 - distributed_type: multi-GPU
 - num_devices: 8
 - gradient_accumulation_steps: 2

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.30924020654561063,
-    "train_runtime": 2368.0064,
-    "train_samples": 39494,
-    "train_samples_per_second": 16.678,
-    "train_steps_per_second": 0.13
+    "train_loss": 0.14623940638553948,
+    "train_runtime": 3207.041,
+    "train_samples": 51894,
+    "train_samples_per_second": 16.181,
+    "train_steps_per_second": 0.126
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a10ca011a1c2a1c658772aa56d4cc4ac2385ee09b49f4b3ed20c74c4fd1555ac
+oid sha256:d2703b55dff69031b3f3ecaf50c0ed868a1d94df7abec9bb57ae4ecb8a13ad10
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f1448fe3337ffa6090a6b661a18e2ec20e095ab48bb2c147b7179935cc7941a3
+oid sha256:89d3d7e08ba2c7ed0fe3b144a8cf69336851d2f0fefd997aba0e399ce8456238
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c2a49167fad0108c937be618534f5af6119f899d333ecda5b356cac8822f5b69
+oid sha256:3e9453a6a9af4586367b2e727b6ccebbe9a638c7ea0dd62394a3b9b2a2f0db79
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.30924020654561063,
-    "train_runtime": 2368.0064,
-    "train_samples": 39494,
-    "train_samples_per_second": 16.678,
-    "train_steps_per_second": 0.13
+    "train_loss": 0.14623940638553948,
+    "train_runtime": 3207.041,
+    "train_samples": 51894,
+    "train_samples_per_second": 16.181,
+    "train_steps_per_second": 0.126
 }

trainer_state.json

@@ -1,21 +1,21 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 0.998766954377312,
   "eval_steps": 1000,
-  "global_step": 309,
+  "global_step": 405,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.0,
-      "learning_rate": 1.6129032258064514e-08,
-      "logits/chosen": -2.771798610687256,
-      "logits/rejected": -2.786311149597168,
-      "logps/chosen": -64.84683227539062,
-      "logps/rejected": -78.83343505859375,
-      "loss": 0.465,
+      "learning_rate": 1.2195121951219512e-08,
+      "logits/chosen": -2.769179582595825,
+      "logits/rejected": -2.6384379863739014,
+      "logps/chosen": -511.7125244140625,
+      "logps/rejected": -154.70135498046875,
+      "loss": 0.5584,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -23,437 +23,577 @@
       "step": 1
     },
     {
-      "epoch": 0.03,
-      "learning_rate": 1.6129032258064515e-07,
-      "logits/chosen": -2.861445426940918,
-      "logits/rejected": -2.8537797927856445,
-      "logps/chosen": -97.3335952758789,
-      "logps/rejected": -99.28689575195312,
-      "loss": 0.4711,
-      "rewards/accuracies": 0.4652777910232544,
-      "rewards/chosen": 0.00041951198363676667,
-      "rewards/margins": 0.00024795616627670825,
-      "rewards/rejected": 0.0001715558028081432,
+      "epoch": 0.02,
+      "learning_rate": 1.219512195121951e-07,
+      "logits/chosen": -2.7494113445281982,
+      "logits/rejected": -2.719766616821289,
+      "logps/chosen": -350.1738586425781,
+      "logps/rejected": -113.84426879882812,
+      "loss": 0.5524,
+      "rewards/accuracies": 0.5486111044883728,
+      "rewards/chosen": 0.0006897298735566437,
+      "rewards/margins": 0.0009333027992397547,
+      "rewards/rejected": -0.00024357291113119572,
       "step": 10
     },
     {
-      "epoch": 0.06,
-      "learning_rate": 3.225806451612903e-07,
-      "logits/chosen": -2.805128812789917,
-      "logits/rejected": -2.798733711242676,
-      "logps/chosen": -87.49668884277344,
-      "logps/rejected": -88.37379455566406,
-      "loss": 0.4545,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -0.00021584751084446907,
-      "rewards/margins": 0.00035835974267683923,
-      "rewards/rejected": -0.0005742072244174778,
+      "epoch": 0.05,
+      "learning_rate": 2.439024390243902e-07,
+      "logits/chosen": -2.7349042892456055,
+      "logits/rejected": -2.6970717906951904,
+      "logps/chosen": -417.70599365234375,
+      "logps/rejected": -126.1716079711914,
+      "loss": 0.5335,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.016823332756757736,
+      "rewards/margins": 0.03299414739012718,
+      "rewards/rejected": -0.016170816496014595,
       "step": 20
     },
     {
-      "epoch": 0.1,
-      "learning_rate": 4.838709677419355e-07,
-      "logits/chosen": -2.8101882934570312,
-      "logits/rejected": -2.8058152198791504,
-      "logps/chosen": -101.92448425292969,
-      "logps/rejected": -103.37696838378906,
-      "loss": 0.4629,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": 0.002588495146483183,
-      "rewards/margins": 0.0013554933248087764,
-      "rewards/rejected": 0.0012330020545050502,
+      "epoch": 0.07,
+      "learning_rate": 3.6585365853658536e-07,
+      "logits/chosen": -2.699075222015381,
+      "logits/rejected": -2.694394588470459,
+      "logps/chosen": -394.01434326171875,
+      "logps/rejected": -112.81089782714844,
+      "loss": 0.4665,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.06973811239004135,
+      "rewards/margins": 0.1918165385723114,
+      "rewards/rejected": -0.12207841873168945,
       "step": 30
     },
     {
-      "epoch": 0.13,
-      "learning_rate": 4.987080943856886e-07,
-      "logits/chosen": -2.792264223098755,
-      "logits/rejected": -2.785740375518799,
-      "logps/chosen": -108.3989028930664,
-      "logps/rejected": -109.5976791381836,
-      "loss": 0.4756,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": 0.015416177920997143,
-      "rewards/margins": 0.006553493440151215,
-      "rewards/rejected": 0.008862685412168503,
+      "epoch": 0.1,
+      "learning_rate": 4.878048780487804e-07,
+      "logits/chosen": -2.5450072288513184,
+      "logits/rejected": -2.547879934310913,
+      "logps/chosen": -388.6388854980469,
+      "logps/rejected": -163.53250122070312,
+      "loss": 0.3715,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.025587651878595352,
+      "rewards/margins": 0.4115292429924011,
+      "rewards/rejected": -0.38594159483909607,
       "step": 40
     },
     {
-      "epoch": 0.16,
-      "learning_rate": 4.942593872763566e-07,
-      "logits/chosen": -2.7586538791656494,
-      "logits/rejected": -2.7508249282836914,
-      "logps/chosen": -99.14311981201172,
-      "logps/rejected": -102.03758239746094,
-      "loss": 0.4882,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.03316986560821533,
-      "rewards/margins": 0.01143565308302641,
-      "rewards/rejected": 0.02173421159386635,
+      "epoch": 0.12,
+      "learning_rate": 4.992461696250783e-07,
+      "logits/chosen": -2.4270787239074707,
+      "logits/rejected": -2.39143705368042,
+      "logps/chosen": -440.15228271484375,
+      "logps/rejected": -204.70932006835938,
+      "loss": 0.2689,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -0.07494890689849854,
+      "rewards/margins": 0.7927365303039551,
+      "rewards/rejected": -0.8676854372024536,
       "step": 50
     },
     {
-      "epoch": 0.19,
-      "learning_rate": 4.866946677079314e-07,
-      "logits/chosen": -2.772948741912842,
-      "logits/rejected": -2.7423453330993652,
-      "logps/chosen": -114.72819519042969,
-      "logps/rejected": -116.7930679321289,
-      "loss": 0.4922,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.024583298712968826,
-      "rewards/margins": 0.016203362494707108,
-      "rewards/rejected": 0.008379933424293995,
+      "epoch": 0.15,
+      "learning_rate": 4.966461721767899e-07,
+      "logits/chosen": -2.369778633117676,
+      "logits/rejected": -2.3275959491729736,
+      "logps/chosen": -423.04144287109375,
+      "logps/rejected": -231.8262939453125,
+      "loss": 0.1992,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.29674965143203735,
+      "rewards/margins": 0.9948006868362427,
+      "rewards/rejected": -1.2915503978729248,
       "step": 60
     },
     {
-      "epoch": 0.23,
-      "learning_rate": 4.7611043866720737e-07,
-      "logits/chosen": -2.700876474380493,
-      "logits/rejected": -2.698098659515381,
-      "logps/chosen": -104.4677734375,
-      "logps/rejected": -109.02482604980469,
-      "loss": 0.4562,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.018551770597696304,
-      "rewards/margins": 0.03420226648449898,
-      "rewards/rejected": -0.05275403708219528,
+      "epoch": 0.17,
+      "learning_rate": 4.922100518015975e-07,
+      "logits/chosen": -2.3612518310546875,
+      "logits/rejected": -2.331784725189209,
+      "logps/chosen": -452.7647399902344,
+      "logps/rejected": -300.84552001953125,
+      "loss": 0.1691,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.578091025352478,
+      "rewards/margins": 1.2361681461334229,
+      "rewards/rejected": -1.8142591714859009,
       "step": 70
     },
     {
-      "epoch": 0.26,
-      "learning_rate": 4.6264172296714e-07,
-      "logits/chosen": -2.6664352416992188,
-      "logits/rejected": -2.6628241539001465,
-      "logps/chosen": -104.18912506103516,
-      "logps/rejected": -108.33952331542969,
-      "loss": 0.4006,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.09273780882358551,
-      "rewards/margins": 0.031208738684654236,
-      "rewards/rejected": -0.12394653260707855,
+      "epoch": 0.2,
+      "learning_rate": 4.859708325770919e-07,
+      "logits/chosen": -2.3421552181243896,
+      "logits/rejected": -2.3237850666046143,
+      "logps/chosen": -463.72821044921875,
+      "logps/rejected": -334.5154724121094,
+      "loss": 0.1669,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.6629701852798462,
+      "rewards/margins": 1.2861653566360474,
+      "rewards/rejected": -1.949135422706604,
       "step": 80
     },
     {
-      "epoch": 0.29,
-      "learning_rate": 4.4646034076333254e-07,
-      "logits/chosen": -2.652411937713623,
-      "logits/rejected": -2.649259090423584,
-      "logps/chosen": -121.21761322021484,
-      "logps/rejected": -129.4744415283203,
-      "loss": 0.3289,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": -0.2388344258069992,
-      "rewards/margins": 0.04864976182579994,
-      "rewards/rejected": -0.28748419880867004,
+      "epoch": 0.22,
+      "learning_rate": 4.779749614980225e-07,
+      "logits/chosen": -2.3320186138153076,
+      "logits/rejected": -2.277460813522339,
+      "logps/chosen": -532.4778442382812,
+      "logps/rejected": -346.4347839355469,
+      "loss": 0.1202,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -0.35028788447380066,
+      "rewards/margins": 1.8641752004623413,
+      "rewards/rejected": -2.214462995529175,
       "step": 90
     },
     {
-      "epoch": 0.32,
-      "learning_rate": 4.27772717647508e-07,
-      "logits/chosen": -2.66510272026062,
-      "logits/rejected": -2.647733688354492,
-      "logps/chosen": -123.56236267089844,
-      "logps/rejected": -126.4317398071289,
-      "loss": 0.3661,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.1472521871328354,
-      "rewards/margins": 0.0753973126411438,
-      "rewards/rejected": -0.22264952957630157,
+      "epoch": 0.25,
+      "learning_rate": 4.682819627081427e-07,
+      "logits/chosen": -2.311751127243042,
+      "logits/rejected": -2.2163002490997314,
+      "logps/chosen": -536.274658203125,
+      "logps/rejected": -369.44781494140625,
+      "loss": 0.1156,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.8115866780281067,
+      "rewards/margins": 1.7770898342132568,
+      "rewards/rejected": -2.588676691055298,
       "step": 100
     },
     {
-      "epoch": 0.36,
-      "learning_rate": 4.068172512800759e-07,
-      "logits/chosen": -2.6250503063201904,
-      "logits/rejected": -2.6321260929107666,
-      "logps/chosen": -114.4910659790039,
-      "logps/rejected": -132.10842895507812,
-      "loss": 0.4089,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.1267862319946289,
-      "rewards/margins": 0.08127979934215546,
-      "rewards/rejected": -0.20806603133678436,
+      "epoch": 0.27,
+      "learning_rate": 4.569639943810477e-07,
+      "logits/chosen": -2.2905831336975098,
+      "logits/rejected": -2.210843324661255,
+      "logps/chosen": -507.51702880859375,
+      "logps/rejected": -328.5555725097656,
+      "loss": 0.1103,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.670477032661438,
+      "rewards/margins": 1.5413503646850586,
+      "rewards/rejected": -2.211827516555786,
       "step": 110
     },
     {
-      "epoch": 0.39,
-      "learning_rate": 3.8386127015561377e-07,
-      "logits/chosen": -2.547484874725342,
-      "logits/rejected": -2.527634859085083,
-      "logps/chosen": -124.6208724975586,
-      "logps/rejected": -142.69337463378906,
-      "loss": 0.3772,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.24697920680046082,
-      "rewards/margins": 0.0980183333158493,
-      "rewards/rejected": -0.3449975550174713,
+      "epoch": 0.3,
+      "learning_rate": 4.4410531154874543e-07,
+      "logits/chosen": -2.3034403324127197,
+      "logits/rejected": -2.234853982925415,
+      "logps/chosen": -464.66168212890625,
+      "logps/rejected": -350.87933349609375,
+      "loss": 0.0998,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.7100422978401184,
+      "rewards/margins": 1.7162730693817139,
+      "rewards/rejected": -2.4263155460357666,
       "step": 120
     },
     {
-      "epoch": 0.42,
-      "learning_rate": 3.591976232982355e-07,
-      "logits/chosen": -2.5323774814605713,
-      "logits/rejected": -2.5387911796569824,
-      "logps/chosen": -106.24836730957031,
-      "logps/rejected": -122.51704406738281,
-      "loss": 0.4003,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.18509183824062347,
-      "rewards/margins": 0.08678923547267914,
-      "rewards/rejected": -0.2718810439109802,
+      "epoch": 0.32,
+      "learning_rate": 4.298016388768561e-07,
+      "logits/chosen": -2.2450332641601562,
+      "logits/rejected": -2.133479356765747,
+      "logps/chosen": -486.155029296875,
+      "logps/rejected": -390.749755859375,
+      "loss": 0.0957,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.152130365371704,
+      "rewards/margins": 1.6959073543548584,
+      "rewards/rejected": -2.8480374813079834,
       "step": 130
     },
     {
-      "epoch": 0.45,
-      "learning_rate": 3.33140944392039e-07,
-      "logits/chosen": -2.489406108856201,
-      "logits/rejected": -2.470911741256714,
-      "logps/chosen": -117.50244140625,
-      "logps/rejected": -122.6696548461914,
-      "loss": 0.3715,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.18632158637046814,
-      "rewards/margins": 0.10030879080295563,
-      "rewards/rejected": -0.2866303622722626,
+      "epoch": 0.35,
+      "learning_rate": 4.1415945805573005e-07,
+      "logits/chosen": -2.2405707836151123,
+      "logits/rejected": -2.163384199142456,
+      "logps/chosen": -506.66583251953125,
+      "logps/rejected": -383.2144470214844,
+      "loss": 0.112,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.087276577949524,
+      "rewards/margins": 1.5736323595046997,
+      "rewards/rejected": -2.6609089374542236,
       "step": 140
     },
     {
-      "epoch": 0.49,
-      "learning_rate": 3.060236380050519e-07,
-      "logits/chosen": -2.52604079246521,
-      "logits/rejected": -2.4881179332733154,
-      "logps/chosen": -148.2981414794922,
-      "logps/rejected": -160.26583862304688,
-      "loss": 0.2992,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.37317097187042236,
-      "rewards/margins": 0.10073389858007431,
-      "rewards/rejected": -0.47390490770339966,
+      "epoch": 0.37,
+      "learning_rate": 3.972952151123984e-07,
+      "logits/chosen": -2.2938637733459473,
+      "logits/rejected": -2.209303379058838,
+      "logps/chosen": -459.609375,
+      "logps/rejected": -328.35955810546875,
+      "loss": 0.1276,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.7247152924537659,
+      "rewards/margins": 1.5017669200897217,
+      "rewards/rejected": -2.2264821529388428,
       "step": 150
     },
     {
-      "epoch": 0.52,
-      "learning_rate": 2.781916391103417e-07,
-      "logits/chosen": -2.474472761154175,
-      "logits/rejected": -2.4683144092559814,
-      "logps/chosen": -175.12594604492188,
-      "logps/rejected": -180.5726776123047,
-      "loss": 0.2327,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.6483449935913086,
-      "rewards/margins": 0.06987190991640091,
-      "rewards/rejected": -0.7182168960571289,
+      "epoch": 0.39,
+      "learning_rate": 3.793344535444142e-07,
+      "logits/chosen": -2.2822673320770264,
+      "logits/rejected": -2.1807644367218018,
+      "logps/chosen": -501.0845642089844,
+      "logps/rejected": -365.5765075683594,
+      "loss": 0.1175,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -0.5146036744117737,
+      "rewards/margins": 1.9269710779190063,
+      "rewards/rejected": -2.441574811935425,
       "step": 160
     },
     {
-      "epoch": 0.55,
-      "learning_rate": 2.5e-07,
-      "logits/chosen": -2.5102310180664062,
-      "logits/rejected": -2.4696249961853027,
-      "logps/chosen": -177.8674774169922,
-      "logps/rejected": -190.3830108642578,
-      "loss": 0.2095,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.7225574254989624,
-      "rewards/margins": 0.14976230263710022,
-      "rewards/rejected": -0.8723198175430298,
+      "epoch": 0.42,
+      "learning_rate": 3.604108797288461e-07,
+      "logits/chosen": -2.2316362857818604,
+      "logits/rejected": -2.135136127471924,
+      "logps/chosen": -474.1004333496094,
+      "logps/rejected": -387.35906982421875,
+      "loss": 0.1052,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.0368373394012451,
+      "rewards/margins": 1.7319509983062744,
+      "rewards/rejected": -2.7687880992889404,
       "step": 170
     },
     {
-      "epoch": 0.58,
-      "learning_rate": 2.218083608896583e-07,
-      "logits/chosen": -2.46416974067688,
-      "logits/rejected": -2.461364984512329,
-      "logps/chosen": -192.23269653320312,
-      "logps/rejected": -195.1028594970703,
-      "loss": 0.2071,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.8408095240592957,
-      "rewards/margins": 0.03609101474285126,
-      "rewards/rejected": -0.8769005537033081,
+      "epoch": 0.44,
+      "learning_rate": 3.40665367563858e-07,
+      "logits/chosen": -2.197690010070801,
+      "logits/rejected": -2.077615976333618,
+      "logps/chosen": -538.2199096679688,
+      "logps/rejected": -420.12158203125,
+      "loss": 0.0723,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.1170153617858887,
+      "rewards/margins": 1.9479118585586548,
+      "rewards/rejected": -3.064927339553833,
       "step": 180
     },
     {
-      "epoch": 0.61,
-      "learning_rate": 1.9397636199494806e-07,
-      "logits/chosen": -2.4198787212371826,
-      "logits/rejected": -2.410118818283081,
-      "logps/chosen": -204.03204345703125,
-      "logps/rejected": -210.47055053710938,
-      "loss": 0.1992,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.9554036855697632,
-      "rewards/margins": 0.09068112075328827,
-      "rewards/rejected": -1.0460847616195679,
+      "epoch": 0.47,
+      "learning_rate": 3.202449097526798e-07,
+      "logits/chosen": -2.184736967086792,
+      "logits/rejected": -2.0798563957214355,
+      "logps/chosen": -509.679443359375,
+      "logps/rejected": -437.552978515625,
+      "loss": 0.0815,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.3500667810440063,
+      "rewards/margins": 1.90851628780365,
+      "rewards/rejected": -3.2585830688476562,
       "step": 190
     },
     {
-      "epoch": 0.65,
-      "learning_rate": 1.6685905560796098e-07,
-      "logits/chosen": -2.4162700176239014,
-      "logits/rejected": -2.4063594341278076,
-      "logps/chosen": -207.71646118164062,
-      "logps/rejected": -210.524169921875,
-      "loss": 0.1957,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -0.9324175119400024,
-      "rewards/margins": 0.046879496425390244,
-      "rewards/rejected": -0.9792969822883606,
+      "epoch": 0.49,
+      "learning_rate": 2.993015235369905e-07,
+      "logits/chosen": -2.149742603302002,
+      "logits/rejected": -2.0623302459716797,
+      "logps/chosen": -521.0450439453125,
+      "logps/rejected": -463.20843505859375,
+      "loss": 0.0798,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.382089376449585,
+      "rewards/margins": 2.0558183193206787,
+      "rewards/rejected": -3.4379076957702637,
       "step": 200
     },
     {
-      "epoch": 0.68,
-      "learning_rate": 1.4080237670176453e-07,
-      "logits/chosen": -2.4201414585113525,
-      "logits/rejected": -2.411137580871582,
-      "logps/chosen": -180.5799102783203,
-      "logps/rejected": -200.37741088867188,
-      "loss": 0.2065,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.8232952356338501,
-      "rewards/margins": 0.11933133751153946,
-      "rewards/rejected": -0.9426265954971313,
+      "epoch": 0.52,
+      "learning_rate": 2.7799111902582693e-07,
+      "logits/chosen": -2.225651264190674,
+      "logits/rejected": -2.0984432697296143,
+      "logps/chosen": -563.5731811523438,
+      "logps/rejected": -442.27191162109375,
+      "loss": 0.0997,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.038576364517212,
+      "rewards/margins": 2.1469340324401855,
+      "rewards/rejected": -3.1855106353759766,
       "step": 210
     },
     {
-      "epoch": 0.71,
-      "learning_rate": 1.1613872984438628e-07,
-      "logits/chosen": -2.4377281665802,
-      "logits/rejected": -2.423133373260498,
-      "logps/chosen": -177.68234252929688,
-      "logps/rejected": -193.7434539794922,
-      "loss": 0.2017,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.7646304368972778,
-      "rewards/margins": 0.09790431708097458,
-      "rewards/rejected": -0.8625347018241882,
+      "epoch": 0.54,
+      "learning_rate": 2.564723385445869e-07,
+      "logits/chosen": -2.173677444458008,
+      "logits/rejected": -2.1013877391815186,
+      "logps/chosen": -484.97259521484375,
+      "logps/rejected": -392.4546203613281,
+      "loss": 0.1033,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.9937517046928406,
+      "rewards/margins": 1.7565183639526367,
+      "rewards/rejected": -2.750270128250122,
       "step": 220
     },
     {
-      "epoch": 0.74,
-      "learning_rate": 9.318274871992407e-08,
-      "logits/chosen": -2.424259901046753,
-      "logits/rejected": -2.407381534576416,
-      "logps/chosen": -189.03651428222656,
-      "logps/rejected": -192.30810546875,
-      "loss": 0.2018,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.7987578511238098,
-      "rewards/margins": 0.09162624180316925,
-      "rewards/rejected": -0.8903840780258179,
+      "epoch": 0.57,
+      "learning_rate": 2.3490537564442845e-07,
+      "logits/chosen": -2.194636821746826,
+      "logits/rejected": -2.0953433513641357,
+      "logps/chosen": -513.7042236328125,
+      "logps/rejected": -395.5045166015625,
+      "loss": 0.1263,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.8545423746109009,
+      "rewards/margins": 1.8620414733886719,
+      "rewards/rejected": -2.716583728790283,
       "step": 230
     },
     {
-      "epoch": 0.78,
-      "learning_rate": 7.222728235249195e-08,
-      "logits/chosen": -2.4359657764434814,
-      "logits/rejected": -2.420992612838745,
-      "logps/chosen": -181.5802459716797,
-      "logps/rejected": -196.73719787597656,
-      "loss": 0.2143,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.742468535900116,
-      "rewards/margins": 0.11282582581043243,
-      "rewards/rejected": -0.855294406414032,
+      "epoch": 0.59,
+      "learning_rate": 2.1345078256378801e-07,
+      "logits/chosen": -2.207188844680786,
+      "logits/rejected": -2.108193874359131,
+      "logps/chosen": -526.3621826171875,
+      "logps/rejected": -398.5421142578125,
+      "loss": 0.1013,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.8062858581542969,
+      "rewards/margins": 2.0898499488830566,
+      "rewards/rejected": -2.8961360454559326,
       "step": 240
     },
     {
-      "epoch": 0.81,
-      "learning_rate": 5.353965923666742e-08,
-      "logits/chosen": -2.4418349266052246,
-      "logits/rejected": -2.4159674644470215,
-      "logps/chosen": -181.2228546142578,
-      "logps/rejected": -187.01864624023438,
-      "loss": 0.2171,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.7495532035827637,
-      "rewards/margins": 0.05978519842028618,
-      "rewards/rejected": -0.809338390827179,
+      "epoch": 0.62,
+      "learning_rate": 1.9226827501969865e-07,
+      "logits/chosen": -2.140782594680786,
+      "logits/rejected": -1.9999908208847046,
+      "logps/chosen": -523.0418701171875,
+      "logps/rejected": -424.337890625,
+      "loss": 0.0789,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.1606338024139404,
+      "rewards/margins": 2.0052971839904785,
+      "rewards/rejected": -3.165931224822998,
       "step": 250
     },
     {
-      "epoch": 0.84,
-      "learning_rate": 3.7358277032860016e-08,
-      "logits/chosen": -2.394810438156128,
-      "logits/rejected": -2.3954341411590576,
-      "logps/chosen": -177.30311584472656,
-      "logps/rejected": -178.42190551757812,
-      "loss": 0.2077,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.7291509509086609,
-      "rewards/margins": 0.0840456634759903,
-      "rewards/rejected": -0.8131965398788452,
+      "epoch": 0.64,
+      "learning_rate": 1.715155432264775e-07,
+      "logits/chosen": -2.1131765842437744,
+      "logits/rejected": -1.9959485530853271,
+      "logps/chosen": -505.00689697265625,
+      "logps/rejected": -455.15850830078125,
+      "loss": 0.0781,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.289726972579956,
+      "rewards/margins": 2.1254703998565674,
+      "rewards/rejected": -3.4151973724365234,
       "step": 260
     },
     {
-      "epoch": 0.87,
-      "learning_rate": 2.3889561332792657e-08,
-      "logits/chosen": -2.3990871906280518,
-      "logits/rejected": -2.380483388900757,
-      "logps/chosen": -178.64584350585938,
-      "logps/rejected": -192.37232971191406,
-      "loss": 0.2112,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.7460119724273682,
-      "rewards/margins": 0.0882803276181221,
-      "rewards/rejected": -0.8342922329902649,
+      "epoch": 0.67,
+      "learning_rate": 1.51347077992983e-07,
+      "logits/chosen": -2.159238338470459,
+      "logits/rejected": -2.043375253677368,
+      "logps/chosen": -543.9025268554688,
+      "logps/rejected": -457.2245178222656,
+      "loss": 0.0875,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.1506714820861816,
+      "rewards/margins": 2.2577271461486816,
+      "rewards/rejected": -3.408398389816284,
       "step": 270
     },
     {
-      "epoch": 0.91,
-      "learning_rate": 1.3305332292068705e-08,
-      "logits/chosen": -2.412071943283081,
-      "logits/rejected": -2.416182041168213,
-      "logps/chosen": -186.1865997314453,
-      "logps/rejected": -212.6334991455078,
-      "loss": 0.2083,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.7327592968940735,
-      "rewards/margins": 0.17295940220355988,
-      "rewards/rejected": -0.9057186841964722,
+      "epoch": 0.69,
+      "learning_rate": 1.3191302063739906e-07,
+      "logits/chosen": -2.2006659507751465,
+      "logits/rejected": -2.11677885055542,
+      "logps/chosen": -509.25091552734375,
+      "logps/rejected": -413.2759704589844,
+      "loss": 0.0897,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.0559613704681396,
+      "rewards/margins": 1.9989497661590576,
+      "rewards/rejected": -3.0549111366271973,
       "step": 280
     },
     {
-      "epoch": 0.94,
-      "learning_rate": 5.740612723643401e-09,
-      "logits/chosen": -2.406705856323242,
-      "logits/rejected": -2.417464017868042,
-      "logps/chosen": -191.5905303955078,
-      "logps/rejected": -209.446533203125,
-      "loss": 0.2026,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.8136159181594849,
-      "rewards/margins": 0.10502395778894424,
-      "rewards/rejected": -0.9186398386955261,
+      "epoch": 0.72,
+      "learning_rate": 1.1335804528119475e-07,
+      "logits/chosen": -2.1818597316741943,
+      "logits/rejected": -2.0668787956237793,
+      "logps/chosen": -554.3623657226562,
+      "logps/rejected": -471.38043212890625,
+      "loss": 0.1041,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.1412699222564697,
+      "rewards/margins": 2.2574570178985596,
+      "rewards/rejected": -3.39872670173645,
       "step": 290
     },
     {
-      "epoch": 0.97,
-      "learning_rate": 1.2919056143113061e-09,
-      "logits/chosen": -2.4019103050231934,
-      "logits/rejected": -2.395653009414673,
-      "logps/chosen": -198.51217651367188,
-      "logps/rejected": -210.8072967529297,
-      "loss": 0.2113,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.8107719421386719,
-      "rewards/margins": 0.11891667544841766,
-      "rewards/rejected": -0.9296887516975403,
+      "epoch": 0.74,
+      "learning_rate": 9.582028184286423e-08,
+      "logits/chosen": -2.153825283050537,
+      "logits/rejected": -2.071503162384033,
+      "logps/chosen": -486.72186279296875,
+      "logps/rejected": -409.9599609375,
+      "loss": 0.1008,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.0588269233703613,
+      "rewards/margins": 1.9501113891601562,
+      "rewards/rejected": -3.0089378356933594,
       "step": 300
     },
+    {
+      "epoch": 0.76,
+      "learning_rate": 7.943028774907065e-08,
+      "logits/chosen": -2.2051730155944824,
+      "logits/rejected": -2.0893607139587402,
+      "logps/chosen": -538.9556274414062,
+      "logps/rejected": -393.6221618652344,
+      "loss": 0.1001,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.817173182964325,
+      "rewards/margins": 2.02958345413208,
+      "rewards/rejected": -2.84675669670105,
+      "step": 310
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.431007601814637e-08,
+      "logits/chosen": -2.155592441558838,
+      "logits/rejected": -2.048361301422119,
+      "logps/chosen": -504.9908142089844,
+      "logps/rejected": -441.20257568359375,
+      "loss": 0.099,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -1.055345892906189,
+      "rewards/margins": 2.115941286087036,
+      "rewards/rejected": -3.1712870597839355,
+      "step": 320
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.0572206951246e-08,
+      "logits/chosen": -2.172407627105713,
+      "logits/rejected": -2.0862064361572266,
+      "logps/chosen": -459.5045471191406,
+      "logps/rejected": -408.4109802246094,
+      "loss": 0.0885,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.082724690437317,
+      "rewards/margins": 1.8569291830062866,
+      "rewards/rejected": -2.9396538734436035,
+      "step": 330
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.831895019292897e-08,
+      "logits/chosen": -2.197789430618286,
+      "logits/rejected": -2.0894722938537598,
+      "logps/chosen": -554.7324829101562,
+      "logps/rejected": -459.5433654785156,
+      "loss": 0.0897,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.07448410987854,
+      "rewards/margins": 2.295614719390869,
+      "rewards/rejected": -3.3700993061065674,
+      "step": 340
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.764152339909756e-08,
+      "logits/chosen": -2.1942572593688965,
+      "logits/rejected": -2.0833990573883057,
+      "logps/chosen": -550.0361938476562,
+      "logps/rejected": -428.2833557128906,
+      "loss": 0.1036,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.7792487144470215,
+      "rewards/margins": 2.310093879699707,
+      "rewards/rejected": -3.0893425941467285,
+      "step": 350
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.861941317991664e-08,
+      "logits/chosen": -2.0975775718688965,
+      "logits/rejected": -2.0279173851013184,
+      "logps/chosen": -480.5646057128906,
+      "logps/rejected": -423.53277587890625,
+      "loss": 0.0817,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.1758220195770264,
+      "rewards/margins": 1.8986190557479858,
+      "rewards/rejected": -3.0744411945343018,
+      "step": 360
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.13197833728636e-08,
+      "logits/chosen": -2.1713950634002686,
+      "logits/rejected": -2.0815377235412598,
+      "logps/chosen": -517.8831787109375,
+      "logps/rejected": -435.673828125,
+      "loss": 0.0966,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.24428391456604,
+      "rewards/margins": 1.8842618465423584,
+      "rewards/rejected": -3.1285459995269775,
+      "step": 370
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.79697505093521e-09,
+      "logits/chosen": -2.2110981941223145,
+      "logits/rejected": -2.0913751125335693,
+      "logps/chosen": -547.6890258789062,
+      "logps/rejected": -447.9027404785156,
+      "loss": 0.0901,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.9516210556030273,
+      "rewards/margins": 2.320674180984497,
+      "rewards/rejected": -3.2722949981689453,
+      "step": 380
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.092101988131256e-09,
+      "logits/chosen": -2.1908106803894043,
+      "logits/rejected": -2.0610384941101074,
+      "logps/chosen": -536.3062744140625,
+      "logps/rejected": -412.360595703125,
+      "loss": 0.09,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.071474552154541,
+      "rewards/margins": 1.9356151819229126,
+      "rewards/rejected": -3.007089614868164,
+      "step": 390
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.327445937151673e-10,
+      "logits/chosen": -2.1998062133789062,
+      "logits/rejected": -2.0742921829223633,
+      "logps/chosen": -562.770751953125,
+      "logps/rejected": -443.3938903808594,
+      "loss": 0.0877,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.2041308879852295,
+      "rewards/margins": 1.9816064834594727,
+      "rewards/rejected": -3.185737133026123,
+      "step": 400
+    },
     {
       "epoch": 1.0,
-      "step": 309,
+      "step": 405,
       "total_flos": 0.0,
-      "train_loss": 0.30924020654561063,
-      "train_runtime": 2368.0064,
-      "train_samples_per_second": 16.678,
-      "train_steps_per_second": 0.13
+      "train_loss": 0.14623940638553948,
+      "train_runtime": 3207.041,
+      "train_samples_per_second": 16.181,
+      "train_steps_per_second": 0.126
     }
   ],
   "logging_steps": 10,
-  "max_steps": 309,
+  "max_steps": 405,
   "num_train_epochs": 1,
   "save_steps": 1000,
   "total_flos": 0.0,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8d736344af27fdfcfb8add7ed663f4aa74f90a545b0f9265f1a0e9b81247cd8
+oid sha256:0c8773390caa70c528e762ee16dcaba2dddc8cacf6287d6b9ba470b75637e007
 size 5944