Model save

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 2.0,
-    "train_loss": 0.5404228230597268,
-    "train_runtime": 10286.8427,
-    "train_samples": 45548,
-    "train_samples_per_second": 8.856,
-    "train_steps_per_second": 0.069
+    "train_loss": 0.31343152379167494,
+    "train_runtime": 8630.1644,
+    "train_samples": 37141,
+    "train_samples_per_second": 8.607,
+    "train_steps_per_second": 0.067
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c2c61a6dd8fe7793da54bd1b79b4b9221e1239f3363810e66b5a5d06060ba9ea
+oid sha256:336e82446271b1f84cf981b43382c3329448f66e3165029498821a5bd11498d5
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:16ffea690a41a6d122d002cf3f1420b6241fdbe0c7510caec4cf0fb567b0f671
+oid sha256:04920783a09dd11a6700fc1ac2fdc19eacf04737ea07cae97a703a48df72397c
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b30b2e5bf06a6e6d91362379d4a10d366cdc868fe8553eb7081ba311ffefc064
+oid sha256:578fe292401e4d92076f630656f260d30707a4d6564a9d36b1aa5ef1982b6f52
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:07b05c0a9bdb631a5b59a47c4205ca4650eab53c87308a2af4d3e84183f0bda1
+oid sha256:56f45d6dc596c0804896541624466d9d983344958d85ebc4464d33f48bf36439
 size 1168138808

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 2.0,
-    "train_loss": 0.5404228230597268,
-    "train_runtime": 10286.8427,
-    "train_samples": 45548,
-    "train_samples_per_second": 8.856,
-    "train_steps_per_second": 0.069
+    "train_loss": 0.31343152379167494,
+    "train_runtime": 8630.1644,
+    "train_samples": 37141,
+    "train_samples_per_second": 8.607,
+    "train_steps_per_second": 0.067
 }

trainer_state.json

@@ -1,1019 +1,837 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.9950825430277486,
+  "epoch": 1.9982773471145565,
   "eval_steps": 10000,
-  "global_step": 710,
+  "global_step": 580,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.03,
-      "learning_rate": 1.4084507042253522e-07,
-      "logits/chosen": -0.08252329379320145,
-      "logits/rejected": -0.03875836357474327,
-      "logps/chosen": -327.2303466796875,
-      "logps/rejected": -244.4869384765625,
-      "loss": 0.6931,
-      "rewards/accuracies": 0.41874998807907104,
-      "rewards/chosen": -0.00044718230492435396,
-      "rewards/margins": 0.00022346181503962725,
-      "rewards/rejected": -0.0006706439307890832,
+      "learning_rate": 1.7241379310344828e-07,
+      "logits/chosen": -0.04663122445344925,
+      "logits/rejected": -0.04833713918924332,
+      "logps/chosen": -182.5131072998047,
+      "logps/rejected": -183.75173950195312,
+      "loss": 0.6541,
+      "rewards/accuracies": 0.39375001192092896,
+      "rewards/chosen": 1.5550864191027358e-05,
+      "rewards/margins": 4.5274849981069565e-05,
+      "rewards/rejected": -2.9723969419137575e-05,
       "step": 10
     },
     {
-      "epoch": 0.06,
-      "learning_rate": 2.8169014084507043e-07,
-      "logits/chosen": -0.05704839155077934,
-      "logits/rejected": -0.027086356654763222,
-      "logps/chosen": -291.45880126953125,
-      "logps/rejected": -194.75648498535156,
-      "loss": 0.6917,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.0008408628636971116,
-      "rewards/margins": 0.0024045060854405165,
-      "rewards/rejected": -0.0032453692983835936,
+      "epoch": 0.07,
+      "learning_rate": 3.4482758620689656e-07,
+      "logits/chosen": -0.04483399540185928,
+      "logits/rejected": -0.040133703500032425,
+      "logps/chosen": -175.80125427246094,
+      "logps/rejected": -183.30113220214844,
+      "loss": 0.6465,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.0010477929608896375,
+      "rewards/margins": -0.0001325715275015682,
+      "rewards/rejected": -0.000915221287868917,
       "step": 20
     },
     {
-      "epoch": 0.08,
-      "learning_rate": 4.225352112676056e-07,
-      "logits/chosen": -0.024302149191498756,
-      "logits/rejected": 0.019717644900083542,
-      "logps/chosen": -348.6321716308594,
-      "logps/rejected": -206.6336669921875,
-      "loss": 0.6844,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.0045356592163443565,
-      "rewards/margins": 0.02465306967496872,
-      "rewards/rejected": -0.020117413252592087,
+      "epoch": 0.1,
+      "learning_rate": 5.172413793103448e-07,
+      "logits/chosen": 0.03275300934910774,
+      "logits/rejected": 0.05599850416183472,
+      "logps/chosen": -188.52236938476562,
+      "logps/rejected": -177.93060302734375,
+      "loss": 0.6521,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.0008330008713528514,
+      "rewards/margins": -0.000871331081725657,
+      "rewards/rejected": 3.833030496025458e-05,
       "step": 30
     },
     {
-      "epoch": 0.11,
-      "learning_rate": 5.633802816901409e-07,
-      "logits/chosen": -0.021973121911287308,
-      "logits/rejected": 0.018863247707486153,
-      "logps/chosen": -305.7060241699219,
-      "logps/rejected": -227.82315063476562,
-      "loss": 0.6768,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.02298508957028389,
-      "rewards/margins": 0.028871804475784302,
-      "rewards/rejected": -0.05185689404606819,
+      "epoch": 0.14,
+      "learning_rate": 6.896551724137931e-07,
+      "logits/chosen": 0.0016825118800625205,
+      "logits/rejected": 0.0037804923485964537,
+      "logps/chosen": -163.9655303955078,
+      "logps/rejected": -169.550048828125,
+      "loss": 0.6509,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.0026731090620160103,
+      "rewards/margins": -0.0011917275842279196,
+      "rewards/rejected": 0.00386483664624393,
       "step": 40
     },
     {
-      "epoch": 0.14,
-      "learning_rate": 7.04225352112676e-07,
-      "logits/chosen": 0.037532515823841095,
-      "logits/rejected": 0.0720117837190628,
-      "logps/chosen": -321.4078063964844,
-      "logps/rejected": -257.11822509765625,
-      "loss": 0.6655,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.0590963289141655,
-      "rewards/margins": 0.08719820529222488,
-      "rewards/rejected": -0.14629453420639038,
+      "epoch": 0.17,
+      "learning_rate": 8.620689655172412e-07,
+      "logits/chosen": -0.002694876166060567,
+      "logits/rejected": -0.017489472404122353,
+      "logps/chosen": -177.7022705078125,
+      "logps/rejected": -171.62660217285156,
+      "loss": 0.6509,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.004064758308231831,
+      "rewards/margins": 0.001050427439622581,
+      "rewards/rejected": 0.0030143309850245714,
       "step": 50
     },
     {
-      "epoch": 0.17,
-      "learning_rate": 8.450704225352112e-07,
-      "logits/chosen": -0.029829170554876328,
-      "logits/rejected": 0.031527455896139145,
-      "logps/chosen": -393.5614318847656,
-      "logps/rejected": -230.4453887939453,
-      "loss": 0.647,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.06647895276546478,
-      "rewards/margins": 0.2127491980791092,
-      "rewards/rejected": -0.279228150844574,
+      "epoch": 0.21,
+      "learning_rate": 9.999637795788383e-07,
+      "logits/chosen": 0.04514119774103165,
+      "logits/rejected": 0.0163094662129879,
+      "logps/chosen": -154.91921997070312,
+      "logps/rejected": -172.60826110839844,
+      "loss": 0.6471,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": 0.006378197111189365,
+      "rewards/margins": -0.00023460984812118113,
+      "rewards/rejected": 0.006612806580960751,
       "step": 60
     },
     {
-      "epoch": 0.2,
-      "learning_rate": 9.859154929577465e-07,
-      "logits/chosen": -0.03452586010098457,
-      "logits/rejected": 0.050171900540590286,
-      "logps/chosen": -376.34027099609375,
-      "logps/rejected": -247.0965576171875,
-      "loss": 0.6429,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.0781661719083786,
-      "rewards/margins": 0.16850998997688293,
-      "rewards/rejected": -0.24667616188526154,
+      "epoch": 0.24,
+      "learning_rate": 9.98696615758975e-07,
+      "logits/chosen": -0.01900678686797619,
+      "logits/rejected": 0.04069110006093979,
+      "logps/chosen": -155.2123565673828,
+      "logps/rejected": -151.2286834716797,
+      "loss": 0.6484,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.005074621178209782,
+      "rewards/margins": 0.0047805835492908955,
+      "rewards/rejected": -0.009855205193161964,
       "step": 70
     },
     {
-      "epoch": 0.22,
-      "learning_rate": 9.995106132599868e-07,
-      "logits/chosen": 0.01499585248529911,
-      "logits/rejected": 0.02722088061273098,
-      "logps/chosen": -268.036865234375,
-      "logps/rejected": -205.2001953125,
-      "loss": 0.6501,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.08637014031410217,
-      "rewards/margins": 0.16527670621871948,
-      "rewards/rejected": -0.25164681673049927,
+      "epoch": 0.28,
+      "learning_rate": 9.95623675140179e-07,
+      "logits/chosen": 0.014727266505360603,
+      "logits/rejected": 0.035828378051519394,
+      "logps/chosen": -157.910400390625,
+      "logps/rejected": -170.37710571289062,
+      "loss": 0.6375,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.009923619218170643,
+      "rewards/margins": 0.011426478624343872,
+      "rewards/rejected": -0.0015028597554191947,
       "step": 80
     },
     {
-      "epoch": 0.25,
-      "learning_rate": 9.978201358980644e-07,
-      "logits/chosen": -0.08599165827035904,
-      "logits/rejected": -0.023884322494268417,
-      "logps/chosen": -365.6874694824219,
-      "logps/rejected": -217.9360809326172,
-      "loss": 0.6489,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 0.04203636571764946,
-      "rewards/margins": 0.3163127303123474,
-      "rewards/rejected": -0.27427637577056885,
+      "epoch": 0.31,
+      "learning_rate": 9.907560848179605e-07,
+      "logits/chosen": -0.06330171972513199,
+      "logits/rejected": -0.05511590838432312,
+      "logps/chosen": -170.8775634765625,
+      "logps/rejected": -178.7204132080078,
+      "loss": 0.6279,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.0003136039595119655,
+      "rewards/margins": 0.006801925599575043,
+      "rewards/rejected": -0.0071155303157866,
       "step": 90
     },
     {
-      "epoch": 0.28,
-      "learning_rate": 9.949266103908894e-07,
-      "logits/chosen": -0.05627519637346268,
-      "logits/rejected": -0.03349987417459488,
-      "logps/chosen": -342.11798095703125,
-      "logps/rejected": -253.5501251220703,
-      "loss": 0.6607,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": 0.010883918963372707,
-      "rewards/margins": 0.12967781722545624,
-      "rewards/rejected": -0.11879388988018036,
+      "epoch": 0.34,
+      "learning_rate": 9.841114703012817e-07,
+      "logits/chosen": -0.08814354985952377,
+      "logits/rejected": -0.053762950003147125,
+      "logps/chosen": -177.0261993408203,
+      "logps/rejected": -170.6063690185547,
+      "loss": 0.6162,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.01691436395049095,
+      "rewards/margins": 0.0037760145496577024,
+      "rewards/rejected": -0.020690377801656723,
       "step": 100
     },
     {
-      "epoch": 0.31,
-      "learning_rate": 9.908370293252287e-07,
-      "logits/chosen": -0.14977756142616272,
-      "logits/rejected": -0.09158390015363693,
-      "logps/chosen": -292.13519287109375,
-      "logps/rejected": -243.66629028320312,
-      "loss": 0.6515,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": 0.0016441673506051302,
-      "rewards/margins": 0.08432444930076599,
-      "rewards/rejected": -0.08268027007579803,
+      "epoch": 0.38,
+      "learning_rate": 9.757138916907184e-07,
+      "logits/chosen": -0.058143556118011475,
+      "logits/rejected": -0.0431932732462883,
+      "logps/chosen": -171.3651885986328,
+      "logps/rejected": -169.97354125976562,
+      "loss": 0.5997,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.02524380013346672,
+      "rewards/margins": 0.02074708789587021,
+      "rewards/rejected": -0.04599089175462723,
       "step": 110
     },
     {
-      "epoch": 0.34,
-      "learning_rate": 9.855612757141654e-07,
-      "logits/chosen": -0.139164537191391,
-      "logits/rejected": -0.109003446996212,
-      "logps/chosen": -311.84649658203125,
-      "logps/rejected": -195.47445678710938,
-      "loss": 0.6485,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.06000605970621109,
-      "rewards/margins": 0.16552844643592834,
-      "rewards/rejected": -0.10552239418029785,
+      "epoch": 0.41,
+      "learning_rate": 9.655937565570122e-07,
+      "logits/chosen": -0.037062786519527435,
+      "logits/rejected": -0.0404152050614357,
+      "logps/chosen": -160.91107177734375,
+      "logps/rejected": -170.44813537597656,
+      "loss": 0.5917,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.07813913375139236,
+      "rewards/margins": 0.017309704795479774,
+      "rewards/rejected": -0.09544883668422699,
       "step": 120
     },
     {
-      "epoch": 0.37,
-      "learning_rate": 9.791120991134902e-07,
-      "logits/chosen": -0.14722749590873718,
-      "logits/rejected": -0.09714435786008835,
-      "logps/chosen": -347.22283935546875,
-      "logps/rejected": -209.6717987060547,
-      "loss": 0.6375,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.028411392122507095,
-      "rewards/margins": 0.26581087708473206,
-      "rewards/rejected": -0.23739945888519287,
+      "epoch": 0.45,
+      "learning_rate": 9.537877098354784e-07,
+      "logits/chosen": 0.01685944013297558,
+      "logits/rejected": -0.009876708500087261,
+      "logps/chosen": -193.56317138671875,
+      "logps/rejected": -199.8650360107422,
+      "loss": 0.5874,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.1443156749010086,
+      "rewards/margins": 0.01459985226392746,
+      "rewards/rejected": -0.15891551971435547,
       "step": 130
     },
     {
-      "epoch": 0.39,
-      "learning_rate": 9.715050848107168e-07,
-      "logits/chosen": -0.11946950852870941,
-      "logits/rejected": -0.0962797999382019,
-      "logps/chosen": -304.10626220703125,
-      "logps/rejected": -229.857421875,
-      "loss": 0.6413,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.05194922536611557,
-      "rewards/margins": 0.15714497864246368,
-      "rewards/rejected": -0.20909419655799866,
+      "epoch": 0.48,
+      "learning_rate": 9.403385011349638e-07,
+      "logits/chosen": -0.08319505304098129,
+      "logits/rejected": -0.04671981930732727,
+      "logps/chosen": -184.2541961669922,
+      "logps/rejected": -192.4634552001953,
+      "loss": 0.574,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.1738511472940445,
+      "rewards/margins": 0.02675568498671055,
+      "rewards/rejected": -0.2006068229675293,
       "step": 140
     },
     {
-      "epoch": 0.42,
-      "learning_rate": 9.627586161611731e-07,
-      "logits/chosen": -0.0967506542801857,
-      "logits/rejected": -0.05451072379946709,
-      "logps/chosen": -280.56353759765625,
-      "logps/rejected": -237.4569854736328,
-      "loss": 0.6309,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.10683991014957428,
-      "rewards/margins": 0.1551331728696823,
-      "rewards/rejected": -0.2619730830192566,
+      "epoch": 0.52,
+      "learning_rate": 9.252948299418253e-07,
+      "logits/chosen": -0.029505770653486252,
+      "logits/rejected": 0.007474844343960285,
+      "logps/chosen": -201.96896362304688,
+      "logps/rejected": -209.3897247314453,
+      "loss": 0.5615,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.2171604186296463,
+      "rewards/margins": 0.049543753266334534,
+      "rewards/rejected": -0.26670414209365845,
       "step": 150
     },
     {
-      "epoch": 0.45,
-      "learning_rate": 9.528938301621955e-07,
-      "logits/chosen": -0.19875934720039368,
-      "logits/rejected": -0.11256257444620132,
-      "logps/chosen": -352.45086669921875,
-      "logps/rejected": -242.0645751953125,
-      "loss": 0.6411,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.03587179258465767,
-      "rewards/margins": 0.228188157081604,
-      "rewards/rejected": -0.26405996084213257,
+      "epoch": 0.55,
+      "learning_rate": 9.087111692794459e-07,
+      "logits/chosen": -0.0830586701631546,
+      "logits/rejected": -0.05916399508714676,
+      "logps/chosen": -188.63107299804688,
+      "logps/rejected": -191.2251434326172,
+      "loss": 0.5454,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.2526788115501404,
+      "rewards/margins": 0.013631664216518402,
+      "rewards/rejected": -0.2663104832172394,
       "step": 160
     },
     {
-      "epoch": 0.48,
-      "learning_rate": 9.419345663727804e-07,
-      "logits/chosen": -0.14901363849639893,
-      "logits/rejected": -0.11375929415225983,
-      "logps/chosen": -305.3213806152344,
-      "logps/rejected": -258.25732421875,
-      "loss": 0.6253,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.09901336580514908,
-      "rewards/margins": 0.1916494369506836,
-      "rewards/rejected": -0.29066282510757446,
+      "epoch": 0.59,
+      "learning_rate": 8.906475684618158e-07,
+      "logits/chosen": -0.0008124202722683549,
+      "logits/rejected": -0.014014181680977345,
+      "logps/chosen": -185.88009643554688,
+      "logps/rejected": -198.26937866210938,
+      "loss": 0.5244,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.2561153471469879,
+      "rewards/margins": 0.03350779041647911,
+      "rewards/rejected": -0.28962308168411255,
       "step": 170
     },
     {
-      "epoch": 0.51,
-      "learning_rate": 9.299073093021404e-07,
-      "logits/chosen": -0.2564430832862854,
-      "logits/rejected": -0.1882060468196869,
-      "logps/chosen": -320.2134704589844,
-      "logps/rejected": -250.32595825195312,
-      "loss": 0.6422,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.18239301443099976,
-      "rewards/margins": 0.17805449664592743,
-      "rewards/rejected": -0.3604475259780884,
+      "epoch": 0.62,
+      "learning_rate": 8.711694356554049e-07,
+      "logits/chosen": -0.02522989548742771,
+      "logits/rejected": 0.017963331192731857,
+      "logps/chosen": -186.06103515625,
+      "logps/rejected": -192.4676971435547,
+      "loss": 0.5036,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.26322150230407715,
+      "rewards/margins": 0.049817826598882675,
+      "rewards/rejected": -0.3130393326282501,
       "step": 180
     },
     {
-      "epoch": 0.53,
-      "learning_rate": 9.168411244063861e-07,
-      "logits/chosen": -0.18668004870414734,
-      "logits/rejected": -0.138667032122612,
-      "logps/chosen": -332.6545715332031,
-      "logps/rejected": -223.46337890625,
-      "loss": 0.6247,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.1231316328048706,
-      "rewards/margins": 0.29294314980506897,
-      "rewards/rejected": -0.41607481241226196,
+      "epoch": 0.65,
+      "learning_rate": 8.503473010366712e-07,
+      "logits/chosen": -0.03940283879637718,
+      "logits/rejected": 0.012682870030403137,
+      "logps/chosen": -197.4193572998047,
+      "logps/rejected": -196.120849609375,
+      "loss": 0.4869,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.30093440413475037,
+      "rewards/margins": 0.04106402397155762,
+      "rewards/rejected": -0.341998428106308,
       "step": 190
     },
     {
-      "epoch": 0.56,
-      "learning_rate": 9.02767587848013e-07,
-      "logits/chosen": -0.22853071987628937,
-      "logits/rejected": -0.14494441449642181,
-      "logps/chosen": -314.8374328613281,
-      "logps/rejected": -249.9915313720703,
-      "loss": 0.626,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.25344473123550415,
-      "rewards/margins": 0.199028879404068,
-      "rewards/rejected": -0.45247355103492737,
+      "epoch": 0.69,
+      "learning_rate": 8.282565614028067e-07,
+      "logits/chosen": -0.011284901760518551,
+      "logits/rejected": -0.018011104315519333,
+      "logps/chosen": -197.7900848388672,
+      "logps/rejected": -208.4326171875,
+      "loss": 0.4659,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.30557769536972046,
+      "rewards/margins": 0.04261254146695137,
+      "rewards/rejected": -0.34819021821022034,
       "step": 200
     },
     {
-      "epoch": 0.59,
-      "learning_rate": 8.877207101879301e-07,
-      "logits/chosen": -0.11853925883769989,
-      "logits/rejected": -0.06579209864139557,
-      "logps/chosen": -379.6506652832031,
-      "logps/rejected": -270.2796630859375,
-      "loss": 0.6266,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.182673841714859,
-      "rewards/margins": 0.2815098166465759,
-      "rewards/rejected": -0.46418365836143494,
+      "epoch": 0.72,
+      "learning_rate": 8.049772071604863e-07,
+      "logits/chosen": 0.025003302842378616,
+      "logits/rejected": 0.056857384741306305,
+      "logps/chosen": -208.2877960205078,
+      "logps/rejected": -204.39129638671875,
+      "loss": 0.4466,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.3003290891647339,
+      "rewards/margins": 0.05031699687242508,
+      "rewards/rejected": -0.35064607858657837,
       "step": 210
     },
     {
-      "epoch": 0.62,
-      "learning_rate": 8.717368541944452e-07,
-      "logits/chosen": -0.26754170656204224,
-      "logits/rejected": -0.20888149738311768,
-      "logps/chosen": -353.4117126464844,
-      "logps/rejected": -238.6965789794922,
-      "loss": 0.6277,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.19892700016498566,
-      "rewards/margins": 0.19706733524799347,
-      "rewards/rejected": -0.39599430561065674,
+      "epoch": 0.76,
+      "learning_rate": 7.805935326811912e-07,
+      "logits/chosen": -0.040947750210762024,
+      "logits/rejected": 0.005281107500195503,
+      "logps/chosen": -198.28762817382812,
+      "logps/rejected": -201.60983276367188,
+      "loss": 0.4448,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.3105833828449249,
+      "rewards/margins": 0.04565637558698654,
+      "rewards/rejected": -0.3562397360801697,
       "step": 220
     },
     {
-      "epoch": 0.65,
-      "learning_rate": 8.54854646967831e-07,
-      "logits/chosen": -0.1852068156003952,
-      "logits/rejected": -0.1229841336607933,
-      "logps/chosen": -324.0721435546875,
-      "logps/rejected": -241.095703125,
-      "loss": 0.6275,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.20178286731243134,
-      "rewards/margins": 0.23417548835277557,
-      "rewards/rejected": -0.4359583854675293,
+      "epoch": 0.79,
+      "learning_rate": 7.551938310719042e-07,
+      "logits/chosen": 0.004909530282020569,
+      "logits/rejected": 0.02475576661527157,
+      "logps/chosen": -217.17431640625,
+      "logps/rejected": -222.0946044921875,
+      "loss": 0.4414,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.3833041191101074,
+      "rewards/margins": 0.03259982913732529,
+      "rewards/rejected": -0.4159039556980133,
       "step": 230
     },
     {
-      "epoch": 0.67,
-      "learning_rate": 8.371148865928318e-07,
-      "logits/chosen": -0.23117892444133759,
-      "logits/rejected": -0.14788521826267242,
-      "logps/chosen": -371.8868713378906,
-      "logps/rejected": -260.60723876953125,
-      "loss": 0.6221,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.24495843052864075,
-      "rewards/margins": 0.22593311965465546,
-      "rewards/rejected": -0.470891535282135,
+      "epoch": 0.83,
+      "learning_rate": 7.288700744664167e-07,
+      "logits/chosen": 0.020644571632146835,
+      "logits/rejected": 0.05471833795309067,
+      "logps/chosen": -204.6312713623047,
+      "logps/rejected": -212.8608856201172,
+      "loss": 0.4231,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.40746060013771057,
+      "rewards/margins": 0.06532532721757889,
+      "rewards/rejected": -0.47278594970703125,
       "step": 240
     },
     {
-      "epoch": 0.7,
-      "learning_rate": 8.185604435447001e-07,
-      "logits/chosen": -0.2556711733341217,
-      "logits/rejected": -0.20389270782470703,
-      "logps/chosen": -375.0079345703125,
-      "logps/rejected": -259.3301696777344,
-      "loss": 0.6136,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.2166837453842163,
-      "rewards/margins": 0.361177921295166,
-      "rewards/rejected": -0.5778616666793823,
+      "epoch": 0.86,
+      "learning_rate": 7.017175809949043e-07,
+      "logits/chosen": -0.00010571479651844129,
+      "logits/rejected": 0.024410605430603027,
+      "logps/chosen": -214.3887481689453,
+      "logps/rejected": -221.22830200195312,
+      "loss": 0.4142,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.41884541511535645,
+      "rewards/margins": 0.052488796412944794,
+      "rewards/rejected": -0.47133421897888184,
       "step": 250
     },
     {
-      "epoch": 0.73,
-      "learning_rate": 7.992361570870287e-07,
-      "logits/chosen": -0.28021442890167236,
-      "logits/rejected": -0.21745452284812927,
-      "logps/chosen": -365.2284851074219,
-      "logps/rejected": -272.9545593261719,
-      "loss": 0.6145,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.3231395184993744,
-      "rewards/margins": 0.3174718916416168,
-      "rewards/rejected": -0.6406115293502808,
+      "epoch": 0.9,
+      "learning_rate": 6.738346696376738e-07,
+      "logits/chosen": -0.027939433231949806,
+      "logits/rejected": 0.005430951714515686,
+      "logps/chosen": -215.529296875,
+      "logps/rejected": -218.4357452392578,
+      "loss": 0.418,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.45902904868125916,
+      "rewards/margins": 0.07481794059276581,
+      "rewards/rejected": -0.5338469743728638,
       "step": 260
     },
     {
-      "epoch": 0.76,
-      "learning_rate": 7.791887269117441e-07,
-      "logits/chosen": -0.21335843205451965,
-      "logits/rejected": -0.20840346813201904,
-      "logps/chosen": -342.4277648925781,
-      "logps/rejected": -276.76861572265625,
-      "loss": 0.6102,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.44275951385498047,
-      "rewards/margins": 0.28791165351867676,
-      "rewards/rejected": -0.7306711673736572,
+      "epoch": 0.93,
+      "learning_rate": 6.453223042128555e-07,
+      "logits/chosen": 0.020816553384065628,
+      "logits/rejected": 0.034314144402742386,
+      "logps/chosen": -229.1612548828125,
+      "logps/rejected": -241.5037078857422,
+      "loss": 0.4108,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.48783358931541443,
+      "rewards/margins": 0.08682344853878021,
+      "rewards/rejected": -0.5746570229530334,
       "step": 270
     },
     {
-      "epoch": 0.79,
-      "learning_rate": 7.584666002831294e-07,
-      "logits/chosen": -0.2197401523590088,
-      "logits/rejected": -0.16780361533164978,
-      "logps/chosen": -389.8875732421875,
-      "logps/rejected": -275.64178466796875,
-      "loss": 0.6237,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.36678892374038696,
-      "rewards/margins": 0.36119475960731506,
-      "rewards/rejected": -0.7279836535453796,
+      "epoch": 0.96,
+      "learning_rate": 6.162837277871552e-07,
+      "logits/chosen": 0.009753930382430553,
+      "logits/rejected": 0.015020027756690979,
+      "logps/chosen": -243.6470184326172,
+      "logps/rejected": -253.1077117919922,
+      "loss": 0.4073,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.5220585465431213,
+      "rewards/margins": 0.08642096817493439,
+      "rewards/rejected": -0.6084794998168945,
       "step": 280
     },
     {
-      "epoch": 0.81,
-      "learning_rate": 7.37119854958609e-07,
-      "logits/chosen": -0.20696017146110535,
-      "logits/rejected": -0.13257133960723877,
-      "logps/chosen": -353.92718505859375,
-      "logps/rejected": -285.9951171875,
-      "loss": 0.6255,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.41903942823410034,
-      "rewards/margins": 0.22267785668373108,
-      "rewards/rejected": -0.6417173147201538,
+      "epoch": 1.0,
+      "learning_rate": 5.868240888334652e-07,
+      "logits/chosen": 0.04999474436044693,
+      "logits/rejected": 0.09121865034103394,
+      "logps/chosen": -241.92050170898438,
+      "logps/rejected": -245.2644805908203,
+      "loss": 0.3966,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.5882852673530579,
+      "rewards/margins": 0.03265915438532829,
+      "rewards/rejected": -0.6209444403648376,
       "step": 290
     },
     {
-      "epoch": 0.84,
-      "learning_rate": 7.152000781692285e-07,
-      "logits/chosen": -0.2640349268913269,
-      "logits/rejected": -0.14995057880878448,
-      "logps/chosen": -372.60443115234375,
-      "logps/rejected": -262.364013671875,
-      "loss": 0.6187,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.2642083764076233,
-      "rewards/margins": 0.3148954510688782,
-      "rewards/rejected": -0.5791038274765015,
+      "epoch": 1.03,
+      "learning_rate": 5.570500604890124e-07,
+      "logits/chosen": 0.004443037323653698,
+      "logits/rejected": 0.03949873894453049,
+      "logps/chosen": -231.2201385498047,
+      "logps/rejected": -267.2882385253906,
+      "loss": 0.296,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.4168514311313629,
+      "rewards/margins": 0.3946831226348877,
+      "rewards/rejected": -0.811534583568573,
       "step": 300
     },
     {
-      "epoch": 0.87,
-      "learning_rate": 6.927602419522946e-07,
-      "logits/chosen": -0.28869348764419556,
-      "logits/rejected": -0.18395274877548218,
-      "logps/chosen": -348.57696533203125,
-      "logps/rejected": -290.04876708984375,
-      "loss": 0.6139,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.3691590130329132,
-      "rewards/margins": 0.294549822807312,
-      "rewards/rejected": -0.6637088060379028,
+      "epoch": 1.07,
+      "learning_rate": 5.270694542927088e-07,
+      "logits/chosen": 0.04513490945100784,
+      "logits/rejected": 0.04028277471661568,
+      "logps/chosen": -236.4629669189453,
+      "logps/rejected": -286.91162109375,
+      "loss": 0.2439,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.6017101407051086,
+      "rewards/margins": 0.4379204213619232,
+      "rewards/rejected": -1.03963041305542,
       "step": 310
     },
     {
-      "epoch": 0.9,
-      "learning_rate": 6.698545751374463e-07,
-      "logits/chosen": -0.2199905812740326,
-      "logits/rejected": -0.19674551486968994,
-      "logps/chosen": -327.25982666015625,
-      "logps/rejected": -276.5052185058594,
-      "loss": 0.614,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.3490336835384369,
-      "rewards/margins": 0.22011037170886993,
-      "rewards/rejected": -0.5691440105438232,
+      "epoch": 1.1,
+      "learning_rate": 4.969908298003572e-07,
+      "logits/chosen": 0.08646970987319946,
+      "logits/rejected": 0.07388903200626373,
+      "logps/chosen": -243.06735229492188,
+      "logps/rejected": -291.15106201171875,
+      "loss": 0.1796,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.9106127023696899,
+      "rewards/margins": 0.41714897751808167,
+      "rewards/rejected": -1.3277615308761597,
       "step": 320
     },
     {
-      "epoch": 0.93,
-      "learning_rate": 6.465384322955224e-07,
-      "logits/chosen": -0.35627591609954834,
-      "logits/rejected": -0.29421260952949524,
-      "logps/chosen": -424.39013671875,
-      "logps/rejected": -319.91070556640625,
-      "loss": 0.614,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.39154383540153503,
-      "rewards/margins": 0.3266575038433075,
-      "rewards/rejected": -0.7182014584541321,
+      "epoch": 1.14,
+      "learning_rate": 4.669231014912942e-07,
+      "logits/chosen": 0.14787010848522186,
+      "logits/rejected": 0.15399041771888733,
+      "logps/chosen": -333.15264892578125,
+      "logps/rejected": -407.36944580078125,
+      "loss": 0.1341,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.4717319011688232,
+      "rewards/margins": 0.6132253408432007,
+      "rewards/rejected": -2.0849571228027344,
       "step": 330
     },
     {
-      "epoch": 0.96,
-      "learning_rate": 6.228681599669248e-07,
-      "logits/chosen": -0.2920376658439636,
-      "logits/rejected": -0.23990678787231445,
-      "logps/chosen": -375.3729248046875,
-      "logps/rejected": -287.75689697265625,
-      "loss": 0.6142,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.4391535818576813,
-      "rewards/margins": 0.2904636561870575,
-      "rewards/rejected": -0.7296172380447388,
+      "epoch": 1.17,
+      "learning_rate": 4.3697514438985533e-07,
+      "logits/chosen": 0.2917874753475189,
+      "logits/rejected": 0.3329147696495056,
+      "logps/chosen": -340.0228576660156,
+      "logps/rejected": -400.05853271484375,
+      "loss": 0.1011,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.6781260967254639,
+      "rewards/margins": 0.6559454202651978,
+      "rewards/rejected": -2.334071636199951,
       "step": 340
     },
     {
-      "epoch": 0.98,
-      "learning_rate": 5.989009604927586e-07,
-      "logits/chosen": -0.30733591318130493,
-      "logits/rejected": -0.3105633854866028,
-      "logps/chosen": -364.541748046875,
-      "logps/rejected": -291.7525329589844,
-      "loss": 0.6029,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.47139501571655273,
-      "rewards/margins": 0.3316076695919037,
-      "rewards/rejected": -0.803002655506134,
+      "epoch": 1.21,
+      "learning_rate": 4.0725539982971027e-07,
+      "logits/chosen": 0.22039537131786346,
+      "logits/rejected": 0.22784602642059326,
+      "logps/chosen": -332.1070556640625,
+      "logps/rejected": -402.21575927734375,
+      "loss": 0.0947,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.765737533569336,
+      "rewards/margins": 0.6189795732498169,
+      "rewards/rejected": -2.384716749191284,
       "step": 350
     },
     {
-      "epoch": 1.01,
-      "learning_rate": 5.74694753777815e-07,
-      "logits/chosen": -0.2973925471305847,
-      "logits/rejected": -0.24487097561359406,
-      "logps/chosen": -331.7513427734375,
-      "logps/rejected": -334.37347412109375,
-      "loss": 0.5713,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.46736210584640503,
-      "rewards/margins": 0.4294673502445221,
-      "rewards/rejected": -0.8968294262886047,
+      "epoch": 1.24,
+      "learning_rate": 3.778714827885845e-07,
+      "logits/chosen": 0.2562865614891052,
+      "logits/rejected": 0.28801122307777405,
+      "logps/chosen": -332.604248046875,
+      "logps/rejected": -385.0444030761719,
+      "loss": 0.0888,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.6409450769424438,
+      "rewards/margins": 0.5473521947860718,
+      "rewards/rejected": -2.1882972717285156,
       "step": 360
     },
     {
-      "epoch": 1.04,
-      "learning_rate": 5.503080373194666e-07,
-      "logits/chosen": -0.3142127990722656,
-      "logits/rejected": -0.26799526810646057,
-      "logps/chosen": -371.78662109375,
-      "logps/rejected": -302.34722900390625,
-      "loss": 0.4951,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -0.4359899163246155,
-      "rewards/margins": 0.5236090421676636,
-      "rewards/rejected": -0.9595988988876343,
+      "epoch": 1.27,
+      "learning_rate": 3.489297922152136e-07,
+      "logits/chosen": 0.29671263694763184,
+      "logits/rejected": 0.2544638216495514,
+      "logps/chosen": -340.67266845703125,
+      "logps/rejected": -433.8108825683594,
+      "loss": 0.0848,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.7547416687011719,
+      "rewards/margins": 0.7966595888137817,
+      "rewards/rejected": -2.551401138305664,
       "step": 370
     },
     {
-      "epoch": 1.07,
-      "learning_rate": 5.257997448407366e-07,
-      "logits/chosen": -0.2998116910457611,
-      "logits/rejected": -0.18579967319965363,
-      "logps/chosen": -355.5238342285156,
-      "logps/rejected": -342.79376220703125,
-      "loss": 0.4881,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.7259389758110046,
-      "rewards/margins": 0.529077410697937,
-      "rewards/rejected": -1.255016565322876,
+      "epoch": 1.31,
+      "learning_rate": 3.205351257595271e-07,
+      "logits/chosen": 0.3430071473121643,
+      "logits/rejected": 0.3394010663032532,
+      "logps/chosen": -339.52386474609375,
+      "logps/rejected": -415.0911560058594,
+      "loss": 0.075,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.862532377243042,
+      "rewards/margins": 0.7082081437110901,
+      "rewards/rejected": -2.5707404613494873,
       "step": 380
     },
     {
-      "epoch": 1.1,
-      "learning_rate": 5.012291038691665e-07,
-      "logits/chosen": -0.34251856803894043,
-      "logits/rejected": -0.23830440640449524,
-      "logps/chosen": -356.92022705078125,
-      "logps/rejected": -346.2132263183594,
-      "loss": 0.4881,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.9590240716934204,
-      "rewards/margins": 0.6022399067878723,
-      "rewards/rejected": -1.561263918876648,
+      "epoch": 1.34,
+      "learning_rate": 2.9279030030112407e-07,
+      "logits/chosen": 0.4035106599330902,
+      "logits/rejected": 0.34052735567092896,
+      "logps/chosen": -378.32696533203125,
+      "logps/rejected": -467.59075927734375,
+      "loss": 0.0663,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.0151519775390625,
+      "rewards/margins": 0.8498624563217163,
+      "rewards/rejected": -2.8650143146514893,
       "step": 390
     },
     {
-      "epoch": 1.12,
-      "learning_rate": 4.7665549260567063e-07,
-      "logits/chosen": -0.32043999433517456,
-      "logits/rejected": -0.27651286125183105,
-      "logps/chosen": -383.32647705078125,
-      "logps/rejected": -371.6776428222656,
-      "loss": 0.4688,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": -0.7162632942199707,
-      "rewards/margins": 0.7107415199279785,
-      "rewards/rejected": -1.4270049333572388,
+      "epoch": 1.38,
+      "learning_rate": 2.65795779650105e-07,
+      "logits/chosen": 0.6060320138931274,
+      "logits/rejected": 0.5839828252792358,
+      "logps/chosen": -379.7890625,
+      "logps/rejected": -466.09075927734375,
+      "loss": 0.0648,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -2.1252145767211914,
+      "rewards/margins": 0.841218113899231,
+      "rewards/rejected": -2.966432809829712,
       "step": 400
     },
     {
-      "epoch": 1.15,
-      "learning_rate": 4.521382964292663e-07,
-      "logits/chosen": -0.2228749692440033,
-      "logits/rejected": -0.20508511364459991,
-      "logps/chosen": -382.3846435546875,
-      "logps/rejected": -355.65057373046875,
-      "loss": 0.4602,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -0.8095337152481079,
-      "rewards/margins": 0.7768293023109436,
-      "rewards/rejected": -1.5863628387451172,
+      "epoch": 1.41,
+      "learning_rate": 2.3964931076834876e-07,
+      "logits/chosen": 0.5519194602966309,
+      "logits/rejected": 0.6366934776306152,
+      "logps/chosen": -366.50872802734375,
+      "logps/rejected": -428.6471252441406,
+      "loss": 0.0666,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.167186737060547,
+      "rewards/margins": 0.6175845861434937,
+      "rewards/rejected": -2.78477144241333,
       "step": 410
     },
     {
-      "epoch": 1.18,
-      "learning_rate": 4.277367643844574e-07,
-      "logits/chosen": -0.29616856575012207,
-      "logits/rejected": -0.22625382244586945,
-      "logps/chosen": -401.6692810058594,
-      "logps/rejected": -380.3138122558594,
-      "loss": 0.4765,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -1.0269509553909302,
-      "rewards/margins": 0.5805720090866089,
-      "rewards/rejected": -1.607522964477539,
+      "epoch": 1.45,
+      "learning_rate": 2.1444556982847995e-07,
+      "logits/chosen": 0.6113961338996887,
+      "logits/rejected": 0.6564093828201294,
+      "logps/chosen": -409.9825134277344,
+      "logps/rejected": -469.96697998046875,
+      "loss": 0.06,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.302380323410034,
+      "rewards/margins": 0.7064192295074463,
+      "rewards/rejected": -3.0087997913360596,
       "step": 420
     },
     {
-      "epoch": 1.21,
-      "learning_rate": 4.035098659980891e-07,
-      "logits/chosen": -0.1935046911239624,
-      "logits/rejected": -0.20023953914642334,
-      "logps/chosen": -454.99993896484375,
-      "logps/rejected": -372.5246887207031,
-      "loss": 0.4619,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -0.960999608039856,
-      "rewards/margins": 0.8414300680160522,
-      "rewards/rejected": -1.8024297952651978,
+      "epoch": 1.48,
+      "learning_rate": 1.902758193921385e-07,
+      "logits/chosen": 0.5821201205253601,
+      "logits/rejected": 0.6013978123664856,
+      "logps/chosen": -398.11602783203125,
+      "logps/rejected": -498.4593811035156,
+      "loss": 0.0628,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.178372859954834,
+      "rewards/margins": 0.9250782132148743,
+      "rewards/rejected": -3.1034510135650635,
       "step": 430
     },
     {
-      "epoch": 1.24,
-      "learning_rate": 3.795161487716928e-07,
-      "logits/chosen": -0.18740960955619812,
-      "logits/rejected": -0.10208537429571152,
-      "logps/chosen": -380.5172119140625,
-      "logps/rejected": -369.58697509765625,
-      "loss": 0.4518,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -0.9771830439567566,
-      "rewards/margins": 0.7713623046875,
-      "rewards/rejected": -1.7485454082489014,
+      "epoch": 1.52,
+      "learning_rate": 1.6722757794891285e-07,
+      "logits/chosen": 0.7665516138076782,
+      "logits/rejected": 0.7662865519523621,
+      "logps/chosen": -377.22186279296875,
+      "logps/rejected": -439.4972229003906,
+      "loss": 0.0685,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.1608002185821533,
+      "rewards/margins": 0.674599289894104,
+      "rewards/rejected": -2.835399866104126,
       "step": 440
     },
     {
-      "epoch": 1.26,
-      "learning_rate": 3.5581359669371223e-07,
-      "logits/chosen": -0.22033481299877167,
-      "logits/rejected": -0.13506947457790375,
-      "logps/chosen": -451.69439697265625,
-      "logps/rejected": -405.4228820800781,
-      "loss": 0.4406,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -1.060671329498291,
-      "rewards/margins": 0.8252478837966919,
-      "rewards/rejected": -1.8859189748764038,
+      "epoch": 1.55,
+      "learning_rate": 1.453843030125278e-07,
+      "logits/chosen": 0.6384638547897339,
+      "logits/rejected": 0.6626998782157898,
+      "logps/chosen": -406.65338134765625,
+      "logps/rejected": -464.52374267578125,
+      "loss": 0.065,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.227891683578491,
+      "rewards/margins": 0.6393032073974609,
+      "rewards/rejected": -2.8671953678131104,
       "step": 450
     },
     {
-      "epoch": 1.29,
-      "learning_rate": 3.324594901135326e-07,
-      "logits/chosen": -0.11018607765436172,
-      "logits/rejected": -0.008564871735870838,
-      "logps/chosen": -405.7510681152344,
-      "logps/rejected": -418.47515869140625,
-      "loss": 0.4644,
+      "epoch": 1.58,
+      "learning_rate": 1.2482508892179883e-07,
+      "logits/chosen": 0.6284786462783813,
+      "logits/rejected": 0.5958371162414551,
+      "logps/chosen": -404.72796630859375,
+      "logps/rejected": -500.9256286621094,
+      "loss": 0.0628,
       "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.1530125141143799,
-      "rewards/margins": 0.7224055528640747,
-      "rewards/rejected": -1.8754180669784546,
+      "rewards/chosen": -2.1921005249023438,
+      "rewards/margins": 0.9022598266601562,
+      "rewards/rejected": -3.094360113143921,
       "step": 460
     },
     {
-      "epoch": 1.32,
-      "learning_rate": 3.095102673159463e-07,
-      "logits/chosen": -0.26194238662719727,
-      "logits/rejected": -0.25700998306274414,
-      "logps/chosen": -428.1478576660156,
-      "logps/rejected": -398.10845947265625,
-      "loss": 0.4541,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -0.9932273030281067,
-      "rewards/margins": 0.7858660817146301,
-      "rewards/rejected": -1.7790933847427368,
+      "epoch": 1.62,
+      "learning_rate": 1.0562438044060846e-07,
+      "logits/chosen": 0.6781089901924133,
+      "logits/rejected": 0.6540604829788208,
+      "logps/chosen": -373.59014892578125,
+      "logps/rejected": -451.7066345214844,
+      "loss": 0.0619,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.1520633697509766,
+      "rewards/margins": 0.7549977898597717,
+      "rewards/rejected": -2.9070611000061035,
       "step": 470
     },
     {
-      "epoch": 1.35,
-      "learning_rate": 2.870213881305802e-07,
-      "logits/chosen": -0.18018680810928345,
-      "logits/rejected": -0.09376071393489838,
-      "logps/chosen": -409.2381286621094,
-      "logps/rejected": -400.8353576660156,
-      "loss": 0.4345,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -0.9986583590507507,
-      "rewards/margins": 0.9575240015983582,
-      "rewards/rejected": -1.9561822414398193,
+      "epoch": 1.65,
+      "learning_rate": 8.785170319396174e-08,
+      "logits/chosen": 0.6746602058410645,
+      "logits/rejected": 0.6840069890022278,
+      "logps/chosen": -407.03350830078125,
+      "logps/rejected": -509.2193298339844,
+      "loss": 0.0583,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -2.3068630695343018,
+      "rewards/margins": 0.9839746356010437,
+      "rewards/rejected": -3.2908377647399902,
       "step": 480
     },
     {
-      "epoch": 1.38,
-      "learning_rate": 2.6504719990588745e-07,
-      "logits/chosen": -0.16257253289222717,
-      "logits/rejected": -0.12673404812812805,
-      "logps/chosen": -485.07177734375,
-      "logps/rejected": -450.83453369140625,
-      "loss": 0.4298,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -1.2001688480377197,
-      "rewards/margins": 0.920021653175354,
-      "rewards/rejected": -2.120190382003784,
+      "epoch": 1.69,
+      "learning_rate": 7.157141191620548e-08,
+      "logits/chosen": 0.7601083517074585,
+      "logits/rejected": 0.8036757707595825,
+      "logps/chosen": -383.3311462402344,
+      "logps/rejected": -455.77447509765625,
+      "loss": 0.0588,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.1426267623901367,
+      "rewards/margins": 0.7608777284622192,
+      "rewards/rejected": -2.9035048484802246,
       "step": 490
     },
     {
-      "epoch": 1.4,
-      "learning_rate": 2.436408061715988e-07,
-      "logits/chosen": -0.18359680473804474,
-      "logits/rejected": -0.10836654901504517,
-      "logps/chosen": -500.3644104003906,
-      "logps/rejected": -464.4107971191406,
-      "loss": 0.4383,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.3315287828445435,
-      "rewards/margins": 0.9130539894104004,
-      "rewards/rejected": -2.2445826530456543,
+      "epoch": 1.72,
+      "learning_rate": 5.684245742300625e-08,
+      "logits/chosen": 0.7304960489273071,
+      "logits/rejected": 0.7055848240852356,
+      "logps/chosen": -382.73931884765625,
+      "logps/rejected": -500.84515380859375,
+      "loss": 0.058,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.192786931991577,
+      "rewards/margins": 1.0639842748641968,
+      "rewards/rejected": -3.2567710876464844,
       "step": 500
     },
     {
-      "epoch": 1.43,
-      "learning_rate": 2.22853938307025e-07,
-      "logits/chosen": -0.06143376976251602,
-      "logits/rejected": 0.023818308487534523,
-      "logps/chosen": -437.2125549316406,
-      "logps/rejected": -424.68536376953125,
-      "loss": 0.4522,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.1669594049453735,
-      "rewards/margins": 0.7936140894889832,
-      "rewards/rejected": -1.9605735540390015,
+      "epoch": 1.76,
+      "learning_rate": 4.3718173150878446e-08,
+      "logits/chosen": 0.9688064455986023,
+      "logits/rejected": 1.001210331916809,
+      "logps/chosen": -388.23193359375,
+      "logps/rejected": -464.37371826171875,
+      "loss": 0.0555,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.178377866744995,
+      "rewards/margins": 0.7957491874694824,
+      "rewards/rejected": -2.9741270542144775,
       "step": 510
     },
     {
-      "epoch": 1.46,
-      "learning_rate": 2.0273683052534173e-07,
-      "logits/chosen": -0.08646591007709503,
-      "logits/rejected": -0.04847399890422821,
-      "logps/chosen": -420.911376953125,
-      "logps/rejected": -395.58245849609375,
-      "loss": 0.4313,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.1632864475250244,
-      "rewards/margins": 0.857226550579071,
-      "rewards/rejected": -2.0205130577087402,
+      "epoch": 1.79,
+      "learning_rate": 3.2246082037199526e-08,
+      "logits/chosen": 0.8018981218338013,
+      "logits/rejected": 0.8608972430229187,
+      "logps/chosen": -382.80157470703125,
+      "logps/rejected": -468.1249084472656,
+      "loss": 0.0604,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.2035393714904785,
+      "rewards/margins": 0.8492420315742493,
+      "rewards/rejected": -3.052781343460083,
       "step": 520
     },
     {
-      "epoch": 1.49,
-      "learning_rate": 1.833380984759764e-07,
-      "logits/chosen": -0.06111500784754753,
-      "logits/rejected": 0.017475496977567673,
-      "logps/chosen": -416.8744201660156,
-      "logps/rejected": -397.92266845703125,
-      "loss": 0.4405,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -1.094646692276001,
-      "rewards/margins": 0.8134492635726929,
-      "rewards/rejected": -1.9080959558486938,
+      "epoch": 1.83,
+      "learning_rate": 2.2467724440002333e-08,
+      "logits/chosen": 0.7628926038742065,
+      "logits/rejected": 0.7621763944625854,
+      "logps/chosen": -394.536376953125,
+      "logps/rejected": -478.94598388671875,
+      "loss": 0.0564,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.241116523742676,
+      "rewards/margins": 0.7960697412490845,
+      "rewards/rejected": -3.0371861457824707,
       "step": 530
     },
     {
-      "epoch": 1.52,
-      "learning_rate": 1.6470462175846606e-07,
-      "logits/chosen": -0.09185104817152023,
-      "logits/rejected": 0.040125779807567596,
-      "logps/chosen": -480.3749084472656,
-      "logps/rejected": -440.147705078125,
-      "loss": 0.428,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -1.2493479251861572,
-      "rewards/margins": 0.8988983035087585,
-      "rewards/rejected": -2.1482462882995605,
+      "epoch": 1.86,
+      "learning_rate": 1.4418507720641793e-08,
+      "logits/chosen": 0.8032611012458801,
+      "logits/rejected": 0.8268760442733765,
+      "logps/chosen": -413.1278381347656,
+      "logps/rejected": -507.7405700683594,
+      "loss": 0.0575,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.3634586334228516,
+      "rewards/margins": 0.9177207946777344,
+      "rewards/rejected": -3.281179428100586,
       "step": 540
     },
     {
-      "epoch": 1.55,
-      "learning_rate": 1.468814306317092e-07,
-      "logits/chosen": -0.02560562826693058,
-      "logits/rejected": 0.08234803378582001,
-      "logps/chosen": -440.58673095703125,
-      "logps/rejected": -397.36761474609375,
-      "loss": 0.435,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -1.3546863794326782,
-      "rewards/margins": 0.9014855623245239,
-      "rewards/rejected": -2.256171703338623,
+      "epoch": 1.89,
+      "learning_rate": 8.127578033998661e-09,
+      "logits/chosen": 0.8457974195480347,
+      "logits/rejected": 0.8470609784126282,
+      "logps/chosen": -351.919677734375,
+      "logps/rejected": -441.13238525390625,
+      "loss": 0.0589,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.1192312240600586,
+      "rewards/margins": 0.815728485584259,
+      "rewards/rejected": -2.934959888458252,
       "step": 550
     },
     {
-      "epoch": 1.57,
-      "learning_rate": 1.299115971923958e-07,
-      "logits/chosen": -0.08617626875638962,
-      "logits/rejected": 0.016537964344024658,
-      "logps/chosen": -470.5138244628906,
-      "logps/rejected": -446.70556640625,
-      "loss": 0.4417,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -1.5232818126678467,
-      "rewards/margins": 0.9088877439498901,
-      "rewards/rejected": -2.4321696758270264,
+      "epoch": 1.93,
+      "learning_rate": 3.6177147904655757e-09,
+      "logits/chosen": 0.8455848693847656,
+      "logits/rejected": 0.8221101760864258,
+      "logps/chosen": -420.8208923339844,
+      "logps/rejected": -514.7294311523438,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -2.299787998199463,
+      "rewards/margins": 0.9560101628303528,
+      "rewards/rejected": -3.255797863006592,
       "step": 560
     },
     {
-      "epoch": 1.6,
-      "learning_rate": 1.1383613128559305e-07,
-      "logits/chosen": -0.03411116451025009,
-      "logits/rejected": 0.08535107225179672,
-      "logps/chosen": -473.811767578125,
-      "logps/rejected": -449.7967834472656,
-      "loss": 0.4229,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -1.225787878036499,
-      "rewards/margins": 1.0088860988616943,
-      "rewards/rejected": -2.2346739768981934,
+      "epoch": 1.96,
+      "learning_rate": 9.052481718690997e-10,
+      "logits/chosen": 0.8840595483779907,
+      "logits/rejected": 0.8795121908187866,
+      "logps/chosen": -424.8587951660156,
+      "logps/rejected": -513.06591796875,
+      "loss": 0.0555,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.4113454818725586,
+      "rewards/margins": 0.8990417718887329,
+      "rewards/rejected": -3.310387372970581,
       "step": 570
     },
-    {
-      "epoch": 1.63,
-      "learning_rate": 9.869388139903495e-08,
-      "logits/chosen": -0.04067504033446312,
-      "logits/rejected": -0.04381244629621506,
-      "logps/chosen": -403.59710693359375,
-      "logps/rejected": -443.3935546875,
-      "loss": 0.4333,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.2790496349334717,
-      "rewards/margins": 0.8235654830932617,
-      "rewards/rejected": -2.1026148796081543,
-      "step": 580
-    },
-    {
-      "epoch": 1.66,
-      "learning_rate": 8.452144078061818e-08,
-      "logits/chosen": -0.06195799633860588,
-      "logits/rejected": 0.00023120641708374023,
-      "logps/chosen": -473.6097717285156,
-      "logps/rejected": -475.43475341796875,
-      "loss": 0.4288,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -1.3708075284957886,
-      "rewards/margins": 1.0009411573410034,
-      "rewards/rejected": -2.371748447418213,
-      "step": 590
-    },
-    {
-      "epoch": 1.69,
-      "learning_rate": 7.135305900598321e-08,
-      "logits/chosen": -0.07134322822093964,
-      "logits/rejected": 0.04816692695021629,
-      "logps/chosen": -485.545166015625,
-      "logps/rejected": -423.0818786621094,
-      "loss": 0.422,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -1.2954851388931274,
-      "rewards/margins": 0.9698074460029602,
-      "rewards/rejected": -2.2652926445007324,
-      "step": 600
-    },
-    {
-      "epoch": 1.71,
-      "learning_rate": 5.9220559209888166e-08,
-      "logits/chosen": -0.0719083845615387,
-      "logits/rejected": -0.051315806806087494,
-      "logps/chosen": -447.87518310546875,
-      "logps/rejected": -441.8182678222656,
-      "loss": 0.4367,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.3435814380645752,
-      "rewards/margins": 0.8819317817687988,
-      "rewards/rejected": -2.225512981414795,
-      "step": 610
-    },
-    {
-      "epoch": 1.74,
-      "learning_rate": 4.815326118139812e-08,
-      "logits/chosen": -0.03743662312626839,
-      "logits/rejected": 0.02814921736717224,
-      "logps/chosen": -447.53411865234375,
-      "logps/rejected": -402.9500427246094,
-      "loss": 0.425,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.2780933380126953,
-      "rewards/margins": 0.9174167513847351,
-      "rewards/rejected": -2.195510149002075,
-      "step": 620
-    },
-    {
-      "epoch": 1.77,
-      "learning_rate": 3.81779105087407e-08,
-      "logits/chosen": -0.12636077404022217,
-      "logits/rejected": 0.03690079599618912,
-      "logps/chosen": -432.38177490234375,
-      "logps/rejected": -432.5584411621094,
-      "loss": 0.4391,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.4293471574783325,
-      "rewards/margins": 0.8169783353805542,
-      "rewards/rejected": -2.2463252544403076,
-      "step": 630
-    },
-    {
-      "epoch": 1.8,
-      "learning_rate": 2.9318613945057637e-08,
-      "logits/chosen": -0.1318369209766388,
-      "logits/rejected": -0.046459414064884186,
-      "logps/chosen": -508.367431640625,
-      "logps/rejected": -467.4312438964844,
-      "loss": 0.4208,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": -1.3762736320495605,
-      "rewards/margins": 1.1156460046768188,
-      "rewards/rejected": -2.49191951751709,
-      "step": 640
-    },
-    {
-      "epoch": 1.83,
-      "learning_rate": 2.1596781151249523e-08,
-      "logits/chosen": 0.05323456600308418,
-      "logits/rejected": 0.11673180758953094,
-      "logps/chosen": -460.07366943359375,
-      "logps/rejected": -404.09979248046875,
-      "loss": 0.4408,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.3242613077163696,
-      "rewards/margins": 0.9012883305549622,
-      "rewards/rejected": -2.2255492210388184,
-      "step": 650
-    },
-    {
-      "epoch": 1.85,
-      "learning_rate": 1.5031072956701695e-08,
-      "logits/chosen": -0.09114273637533188,
-      "logits/rejected": -0.030487876385450363,
-      "logps/chosen": -523.2674560546875,
-      "logps/rejected": -505.872802734375,
-      "loss": 0.4338,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": -1.4451870918273926,
-      "rewards/margins": 1.1003198623657227,
-      "rewards/rejected": -2.5455069541931152,
-      "step": 660
-    },
-    {
-      "epoch": 1.88,
-      "learning_rate": 9.637356262923723e-09,
-      "logits/chosen": -0.018620457500219345,
-      "logits/rejected": 0.09086041152477264,
-      "logps/chosen": -455.99371337890625,
-      "logps/rejected": -447.4140625,
-      "loss": 0.4191,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -1.377251386642456,
-      "rewards/margins": 0.9726886749267578,
-      "rewards/rejected": -2.3499398231506348,
-      "step": 670
-    },
-    {
-      "epoch": 1.91,
-      "learning_rate": 5.428665699084789e-09,
-      "logits/chosen": -0.15896455943584442,
-      "logits/rejected": 0.003171744989231229,
-      "logps/chosen": -446.88519287109375,
-      "logps/rejected": -409.6089782714844,
-      "loss": 0.417,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -1.368312120437622,
-      "rewards/margins": 0.9707133173942566,
-      "rewards/rejected": -2.3390254974365234,
-      "step": 680
-    },
-    {
-      "epoch": 1.94,
-      "learning_rate": 2.415172122110343e-09,
-      "logits/chosen": -0.183979794383049,
-      "logits/rejected": -0.08970221877098083,
-      "logps/chosen": -522.4932861328125,
-      "logps/rejected": -475.81060791015625,
-      "loss": 0.4177,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -1.4173470735549927,
-      "rewards/margins": 0.9275003671646118,
-      "rewards/rejected": -2.3448474407196045,
-      "step": 690
-    },
-    {
-      "epoch": 1.97,
-      "learning_rate": 6.041580374618327e-10,
-      "logits/chosen": -0.12467072904109955,
-      "logits/rejected": 0.008299602195620537,
-      "logps/chosen": -485.9302673339844,
-      "logps/rejected": -479.005126953125,
-      "loss": 0.4332,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -1.5329062938690186,
-      "rewards/margins": 0.9002411961555481,
-      "rewards/rejected": -2.433147430419922,
-      "step": 700
-    },
     {
       "epoch": 2.0,
       "learning_rate": 0.0,
-      "logits/chosen": 0.022132104262709618,
-      "logits/rejected": 0.0691557303071022,
-      "logps/chosen": -448.744140625,
-      "logps/rejected": -475.7118225097656,
-      "loss": 0.428,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -1.5141645669937134,
-      "rewards/margins": 0.9551412463188171,
-      "rewards/rejected": -2.469305992126465,
-      "step": 710
+      "logits/chosen": 0.9808861017227173,
+      "logits/rejected": 0.9314306378364563,
+      "logps/chosen": -372.04034423828125,
+      "logps/rejected": -452.5663146972656,
+      "loss": 0.0544,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -2.2187602519989014,
+      "rewards/margins": 0.7363004684448242,
+      "rewards/rejected": -2.9550607204437256,
+      "step": 580
     },
     {
       "epoch": 2.0,
-      "step": 710,
+      "step": 580,
       "total_flos": 0.0,
-      "train_loss": 0.5404228230597268,
-      "train_runtime": 10286.8427,
-      "train_samples_per_second": 8.856,
-      "train_steps_per_second": 0.069
+      "train_loss": 0.31343152379167494,
+      "train_runtime": 8630.1644,
+      "train_samples_per_second": 8.607,
+      "train_steps_per_second": 0.067
     }
   ],
   "logging_steps": 10,
-  "max_steps": 710,
+  "max_steps": 580,
   "num_train_epochs": 2,
   "save_steps": 10000,
   "total_flos": 0.0,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f85d16e0a98ee7e9ff3b237d19533e381dd47dd87c8b37846af1e6e031f0f6b0
+oid sha256:bc9d7c7d0bacc27c393dcf6748242363d5a7ffc1ed7d1bdf238aa7708ab53c34
 size 6648