Model save

README.md

@@ -35,7 +35,7 @@ The following hyperparameters were used during training:
 - learning_rate: 1e-06
 - train_batch_size: 2
 - eval_batch_size: 8
-- seed: 1
+- seed: 2
 - distributed_type: multi-GPU
 - num_devices: 8
 - gradient_accumulation_steps: 8

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 2.0,
-    "train_loss": 0.31343152379167494,
-    "train_runtime": 8630.1644,
-    "train_samples": 37141,
-    "train_samples_per_second": 8.607,
-    "train_steps_per_second": 0.067
+    "train_loss": 0.5407548575334146,
+    "train_runtime": 10382.7821,
+    "train_samples": 45548,
+    "train_samples_per_second": 8.774,
+    "train_steps_per_second": 0.068
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:336e82446271b1f84cf981b43382c3329448f66e3165029498821a5bd11498d5
+oid sha256:f5fe4288e0a548624dde43de0a52827f920f75036371a401f4802c6559686b85
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:04920783a09dd11a6700fc1ac2fdc19eacf04737ea07cae97a703a48df72397c
+oid sha256:05a91ac86fed729b227a85c23a32b80f447f76f43e88b8fc71ab73e747c36331
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:578fe292401e4d92076f630656f260d30707a4d6564a9d36b1aa5ef1982b6f52
+oid sha256:b3527625878dae3b5d2ff8b6e7b96aa2f5fd8b0df8ba1c760622ef3f9fc8f4e5
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:56f45d6dc596c0804896541624466d9d983344958d85ebc4464d33f48bf36439
+oid sha256:7cb34bcd04c59efbf8ba150596a40f797ad20adebd35f851806967b3528b4756
 size 1168138808

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 2.0,
-    "train_loss": 0.31343152379167494,
-    "train_runtime": 8630.1644,
-    "train_samples": 37141,
-    "train_samples_per_second": 8.607,
-    "train_steps_per_second": 0.067
+    "train_loss": 0.5407548575334146,
+    "train_runtime": 10382.7821,
+    "train_samples": 45548,
+    "train_samples_per_second": 8.774,
+    "train_steps_per_second": 0.068
 }

trainer_state.json

@@ -1,837 +1,1019 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.9982773471145565,
+  "epoch": 1.9950825430277486,
   "eval_steps": 10000,
-  "global_step": 580,
+  "global_step": 710,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.03,
-      "learning_rate": 1.7241379310344828e-07,
-      "logits/chosen": -0.04663122445344925,
-      "logits/rejected": -0.04833713918924332,
-      "logps/chosen": -182.5131072998047,
-      "logps/rejected": -183.75173950195312,
-      "loss": 0.6541,
-      "rewards/accuracies": 0.39375001192092896,
-      "rewards/chosen": 1.5550864191027358e-05,
-      "rewards/margins": 4.5274849981069565e-05,
-      "rewards/rejected": -2.9723969419137575e-05,
+      "learning_rate": 1.4084507042253522e-07,
+      "logits/chosen": -0.024068962782621384,
+      "logits/rejected": 0.04506533965468407,
+      "logps/chosen": -317.1720275878906,
+      "logps/rejected": -207.3963623046875,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": -0.0009543737396597862,
+      "rewards/margins": -0.0014725492801517248,
+      "rewards/rejected": 0.0005181756569072604,
       "step": 10
     },
     {
-      "epoch": 0.07,
-      "learning_rate": 3.4482758620689656e-07,
-      "logits/chosen": -0.04483399540185928,
-      "logits/rejected": -0.040133703500032425,
-      "logps/chosen": -175.80125427246094,
-      "logps/rejected": -183.30113220214844,
-      "loss": 0.6465,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.0010477929608896375,
-      "rewards/margins": -0.0001325715275015682,
-      "rewards/rejected": -0.000915221287868917,
+      "epoch": 0.06,
+      "learning_rate": 2.8169014084507043e-07,
+      "logits/chosen": -0.04674551263451576,
+      "logits/rejected": -0.010908829048275948,
+      "logps/chosen": -294.0361328125,
+      "logps/rejected": -201.79061889648438,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0003093578852713108,
+      "rewards/margins": 0.004586204886436462,
+      "rewards/rejected": -0.004276847001165152,
       "step": 20
     },
     {
-      "epoch": 0.1,
-      "learning_rate": 5.172413793103448e-07,
-      "logits/chosen": 0.03275300934910774,
-      "logits/rejected": 0.05599850416183472,
-      "logps/chosen": -188.52236938476562,
-      "logps/rejected": -177.93060302734375,
-      "loss": 0.6521,
-      "rewards/accuracies": 0.48124998807907104,
-      "rewards/chosen": -0.0008330008713528514,
-      "rewards/margins": -0.000871331081725657,
-      "rewards/rejected": 3.833030496025458e-05,
+      "epoch": 0.08,
+      "learning_rate": 4.225352112676056e-07,
+      "logits/chosen": -0.04512999951839447,
+      "logits/rejected": 0.007197662256658077,
+      "logps/chosen": -361.146240234375,
+      "logps/rejected": -249.73080444335938,
+      "loss": 0.6841,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.001304158358834684,
+      "rewards/margins": 0.015663906931877136,
+      "rewards/rejected": -0.01696806587278843,
       "step": 30
     },
     {
-      "epoch": 0.14,
-      "learning_rate": 6.896551724137931e-07,
-      "logits/chosen": 0.0016825118800625205,
-      "logits/rejected": 0.0037804923485964537,
-      "logps/chosen": -163.9655303955078,
-      "logps/rejected": -169.550048828125,
-      "loss": 0.6509,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.0026731090620160103,
-      "rewards/margins": -0.0011917275842279196,
-      "rewards/rejected": 0.00386483664624393,
+      "epoch": 0.11,
+      "learning_rate": 5.633802816901409e-07,
+      "logits/chosen": -0.04949279874563217,
+      "logits/rejected": 0.00024662315263412893,
+      "logps/chosen": -291.42913818359375,
+      "logps/rejected": -196.7120361328125,
+      "loss": 0.6759,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.015403245575726032,
+      "rewards/margins": 0.05249834060668945,
+      "rewards/rejected": -0.06790158897638321,
       "step": 40
     },
     {
-      "epoch": 0.17,
-      "learning_rate": 8.620689655172412e-07,
-      "logits/chosen": -0.002694876166060567,
-      "logits/rejected": -0.017489472404122353,
-      "logps/chosen": -177.7022705078125,
-      "logps/rejected": -171.62660217285156,
-      "loss": 0.6509,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": 0.004064758308231831,
-      "rewards/margins": 0.001050427439622581,
-      "rewards/rejected": 0.0030143309850245714,
+      "epoch": 0.14,
+      "learning_rate": 7.04225352112676e-07,
+      "logits/chosen": 0.02841530181467533,
+      "logits/rejected": 0.0736595019698143,
+      "logps/chosen": -373.4976806640625,
+      "logps/rejected": -227.70068359375,
+      "loss": 0.665,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.05390547588467598,
+      "rewards/margins": 0.11358483880758286,
+      "rewards/rejected": -0.16749031841754913,
       "step": 50
     },
     {
-      "epoch": 0.21,
-      "learning_rate": 9.999637795788383e-07,
-      "logits/chosen": 0.04514119774103165,
-      "logits/rejected": 0.0163094662129879,
-      "logps/chosen": -154.91921997070312,
-      "logps/rejected": -172.60826110839844,
-      "loss": 0.6471,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": 0.006378197111189365,
-      "rewards/margins": -0.00023460984812118113,
-      "rewards/rejected": 0.006612806580960751,
+      "epoch": 0.17,
+      "learning_rate": 8.450704225352112e-07,
+      "logits/chosen": -0.03540544956922531,
+      "logits/rejected": 0.025740886107087135,
+      "logps/chosen": -323.0583801269531,
+      "logps/rejected": -230.3257293701172,
+      "loss": 0.6567,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.03229106217622757,
+      "rewards/margins": 0.1074444055557251,
+      "rewards/rejected": -0.13973546028137207,
       "step": 60
     },
     {
-      "epoch": 0.24,
-      "learning_rate": 9.98696615758975e-07,
-      "logits/chosen": -0.01900678686797619,
-      "logits/rejected": 0.04069110006093979,
-      "logps/chosen": -155.2123565673828,
-      "logps/rejected": -151.2286834716797,
-      "loss": 0.6484,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.005074621178209782,
-      "rewards/margins": 0.0047805835492908955,
-      "rewards/rejected": -0.009855205193161964,
+      "epoch": 0.2,
+      "learning_rate": 9.859154929577465e-07,
+      "logits/chosen": -0.06292358785867691,
+      "logits/rejected": -0.009262708015739918,
+      "logps/chosen": -354.81341552734375,
+      "logps/rejected": -255.65560913085938,
+      "loss": 0.6477,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.03358080983161926,
+      "rewards/margins": 0.17669352889060974,
+      "rewards/rejected": -0.210274338722229,
       "step": 70
     },
     {
-      "epoch": 0.28,
-      "learning_rate": 9.95623675140179e-07,
-      "logits/chosen": 0.014727266505360603,
-      "logits/rejected": 0.035828378051519394,
-      "logps/chosen": -157.910400390625,
-      "logps/rejected": -170.37710571289062,
-      "loss": 0.6375,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.009923619218170643,
-      "rewards/margins": 0.011426478624343872,
-      "rewards/rejected": -0.0015028597554191947,
+      "epoch": 0.22,
+      "learning_rate": 9.995106132599868e-07,
+      "logits/chosen": 0.03767090290784836,
+      "logits/rejected": 0.09468533098697662,
+      "logps/chosen": -319.8956298828125,
+      "logps/rejected": -235.5646514892578,
+      "loss": 0.667,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.07292340695858002,
+      "rewards/margins": 0.1298426389694214,
+      "rewards/rejected": -0.2027660608291626,
       "step": 80
     },
     {
-      "epoch": 0.31,
-      "learning_rate": 9.907560848179605e-07,
-      "logits/chosen": -0.06330171972513199,
-      "logits/rejected": -0.05511590838432312,
-      "logps/chosen": -170.8775634765625,
-      "logps/rejected": -178.7204132080078,
-      "loss": 0.6279,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.0003136039595119655,
-      "rewards/margins": 0.006801925599575043,
-      "rewards/rejected": -0.0071155303157866,
+      "epoch": 0.25,
+      "learning_rate": 9.978201358980644e-07,
+      "logits/chosen": -0.016046693548560143,
+      "logits/rejected": 0.04123395308852196,
+      "logps/chosen": -361.1316223144531,
+      "logps/rejected": -279.39520263671875,
+      "loss": 0.6491,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.030158694833517075,
+      "rewards/margins": 0.1393767148256302,
+      "rewards/rejected": -0.10921802371740341,
       "step": 90
     },
     {
-      "epoch": 0.34,
-      "learning_rate": 9.841114703012817e-07,
-      "logits/chosen": -0.08814354985952377,
-      "logits/rejected": -0.053762950003147125,
-      "logps/chosen": -177.0261993408203,
-      "logps/rejected": -170.6063690185547,
-      "loss": 0.6162,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.01691436395049095,
-      "rewards/margins": 0.0037760145496577024,
-      "rewards/rejected": -0.020690377801656723,
+      "epoch": 0.28,
+      "learning_rate": 9.949266103908894e-07,
+      "logits/chosen": -0.01002789568156004,
+      "logits/rejected": 0.055082082748413086,
+      "logps/chosen": -275.36773681640625,
+      "logps/rejected": -211.86721801757812,
+      "loss": 0.6472,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.04291190952062607,
+      "rewards/margins": 0.11093775182962418,
+      "rewards/rejected": -0.06802584230899811,
       "step": 100
     },
     {
-      "epoch": 0.38,
-      "learning_rate": 9.757138916907184e-07,
-      "logits/chosen": -0.058143556118011475,
-      "logits/rejected": -0.0431932732462883,
-      "logps/chosen": -171.3651885986328,
-      "logps/rejected": -169.97354125976562,
-      "loss": 0.5997,
+      "epoch": 0.31,
+      "learning_rate": 9.908370293252287e-07,
+      "logits/chosen": -0.008449924178421497,
+      "logits/rejected": 0.018720177933573723,
+      "logps/chosen": -323.53802490234375,
+      "logps/rejected": -244.490966796875,
+      "loss": 0.6407,
       "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.02524380013346672,
-      "rewards/margins": 0.02074708789587021,
-      "rewards/rejected": -0.04599089175462723,
+      "rewards/chosen": 0.07049892842769623,
+      "rewards/margins": 0.1456976681947708,
+      "rewards/rejected": -0.07519873231649399,
       "step": 110
     },
     {
-      "epoch": 0.41,
-      "learning_rate": 9.655937565570122e-07,
-      "logits/chosen": -0.037062786519527435,
-      "logits/rejected": -0.0404152050614357,
-      "logps/chosen": -160.91107177734375,
-      "logps/rejected": -170.44813537597656,
-      "loss": 0.5917,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.07813913375139236,
-      "rewards/margins": 0.017309704795479774,
-      "rewards/rejected": -0.09544883668422699,
+      "epoch": 0.34,
+      "learning_rate": 9.855612757141654e-07,
+      "logits/chosen": -0.07508296519517899,
+      "logits/rejected": -0.012953217141330242,
+      "logps/chosen": -336.1876525878906,
+      "logps/rejected": -208.52011108398438,
+      "loss": 0.6333,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.03278186544775963,
+      "rewards/margins": 0.26016712188720703,
+      "rewards/rejected": -0.2273852527141571,
       "step": 120
     },
     {
-      "epoch": 0.45,
-      "learning_rate": 9.537877098354784e-07,
-      "logits/chosen": 0.01685944013297558,
-      "logits/rejected": -0.009876708500087261,
-      "logps/chosen": -193.56317138671875,
-      "logps/rejected": -199.8650360107422,
-      "loss": 0.5874,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.1443156749010086,
-      "rewards/margins": 0.01459985226392746,
-      "rewards/rejected": -0.15891551971435547,
+      "epoch": 0.37,
+      "learning_rate": 9.791120991134902e-07,
+      "logits/chosen": -0.10753818601369858,
+      "logits/rejected": -0.01082658488303423,
+      "logps/chosen": -376.611083984375,
+      "logps/rejected": -251.7242889404297,
+      "loss": 0.6491,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.026872599497437477,
+      "rewards/margins": 0.2888794541358948,
+      "rewards/rejected": -0.3157520890235901,
       "step": 130
     },
     {
-      "epoch": 0.48,
-      "learning_rate": 9.403385011349638e-07,
-      "logits/chosen": -0.08319505304098129,
-      "logits/rejected": -0.04671981930732727,
-      "logps/chosen": -184.2541961669922,
-      "logps/rejected": -192.4634552001953,
-      "loss": 0.574,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.1738511472940445,
-      "rewards/margins": 0.02675568498671055,
-      "rewards/rejected": -0.2006068229675293,
+      "epoch": 0.39,
+      "learning_rate": 9.715050848107168e-07,
+      "logits/chosen": -0.17280681431293488,
+      "logits/rejected": -0.0874427855014801,
+      "logps/chosen": -342.14593505859375,
+      "logps/rejected": -229.78652954101562,
+      "loss": 0.6346,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.014674955978989601,
+      "rewards/margins": 0.2581936717033386,
+      "rewards/rejected": -0.2728686034679413,
       "step": 140
     },
     {
-      "epoch": 0.52,
-      "learning_rate": 9.252948299418253e-07,
-      "logits/chosen": -0.029505770653486252,
-      "logits/rejected": 0.007474844343960285,
-      "logps/chosen": -201.96896362304688,
-      "logps/rejected": -209.3897247314453,
-      "loss": 0.5615,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.2171604186296463,
-      "rewards/margins": 0.049543753266334534,
-      "rewards/rejected": -0.26670414209365845,
+      "epoch": 0.42,
+      "learning_rate": 9.627586161611731e-07,
+      "logits/chosen": -0.09239596873521805,
+      "logits/rejected": -0.035311758518218994,
+      "logps/chosen": -282.2192077636719,
+      "logps/rejected": -214.80810546875,
+      "loss": 0.6501,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.03641432523727417,
+      "rewards/margins": 0.19125035405158997,
+      "rewards/rejected": -0.22766467928886414,
       "step": 150
     },
     {
-      "epoch": 0.55,
-      "learning_rate": 9.087111692794459e-07,
-      "logits/chosen": -0.0830586701631546,
-      "logits/rejected": -0.05916399508714676,
-      "logps/chosen": -188.63107299804688,
-      "logps/rejected": -191.2251434326172,
-      "loss": 0.5454,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": -0.2526788115501404,
-      "rewards/margins": 0.013631664216518402,
-      "rewards/rejected": -0.2663104832172394,
+      "epoch": 0.45,
+      "learning_rate": 9.528938301621955e-07,
+      "logits/chosen": -0.168453648686409,
+      "logits/rejected": -0.1073572039604187,
+      "logps/chosen": -341.64599609375,
+      "logps/rejected": -223.13101196289062,
+      "loss": 0.6342,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.01668923906981945,
+      "rewards/margins": 0.19806070625782013,
+      "rewards/rejected": -0.18137145042419434,
       "step": 160
     },
     {
-      "epoch": 0.59,
-      "learning_rate": 8.906475684618158e-07,
-      "logits/chosen": -0.0008124202722683549,
-      "logits/rejected": -0.014014181680977345,
-      "logps/chosen": -185.88009643554688,
-      "logps/rejected": -198.26937866210938,
-      "loss": 0.5244,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": -0.2561153471469879,
-      "rewards/margins": 0.03350779041647911,
-      "rewards/rejected": -0.28962308168411255,
+      "epoch": 0.48,
+      "learning_rate": 9.419345663727804e-07,
+      "logits/chosen": -0.13118991255760193,
+      "logits/rejected": -0.0702921599149704,
+      "logps/chosen": -344.593505859375,
+      "logps/rejected": -231.71804809570312,
+      "loss": 0.6221,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.004311258438974619,
+      "rewards/margins": 0.2375575602054596,
+      "rewards/rejected": -0.23324629664421082,
       "step": 170
     },
     {
-      "epoch": 0.62,
-      "learning_rate": 8.711694356554049e-07,
-      "logits/chosen": -0.02522989548742771,
-      "logits/rejected": 0.017963331192731857,
-      "logps/chosen": -186.06103515625,
-      "logps/rejected": -192.4676971435547,
-      "loss": 0.5036,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.26322150230407715,
-      "rewards/margins": 0.049817826598882675,
-      "rewards/rejected": -0.3130393326282501,
+      "epoch": 0.51,
+      "learning_rate": 9.299073093021404e-07,
+      "logits/chosen": -0.05861334875226021,
+      "logits/rejected": 0.004674489144235849,
+      "logps/chosen": -355.00872802734375,
+      "logps/rejected": -255.2248992919922,
+      "loss": 0.6285,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.13683992624282837,
+      "rewards/margins": 0.26262304186820984,
+      "rewards/rejected": -0.3994629979133606,
       "step": 180
     },
     {
-      "epoch": 0.65,
-      "learning_rate": 8.503473010366712e-07,
-      "logits/chosen": -0.03940283879637718,
-      "logits/rejected": 0.012682870030403137,
-      "logps/chosen": -197.4193572998047,
-      "logps/rejected": -196.120849609375,
-      "loss": 0.4869,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.30093440413475037,
-      "rewards/margins": 0.04106402397155762,
-      "rewards/rejected": -0.341998428106308,
+      "epoch": 0.53,
+      "learning_rate": 9.168411244063861e-07,
+      "logits/chosen": -0.15201464295387268,
+      "logits/rejected": -0.08699367940425873,
+      "logps/chosen": -369.53936767578125,
+      "logps/rejected": -261.6515197753906,
+      "loss": 0.6253,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.18763788044452667,
+      "rewards/margins": 0.2461237609386444,
+      "rewards/rejected": -0.4337615966796875,
       "step": 190
     },
     {
-      "epoch": 0.69,
-      "learning_rate": 8.282565614028067e-07,
-      "logits/chosen": -0.011284901760518551,
-      "logits/rejected": -0.018011104315519333,
-      "logps/chosen": -197.7900848388672,
-      "logps/rejected": -208.4326171875,
-      "loss": 0.4659,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.30557769536972046,
-      "rewards/margins": 0.04261254146695137,
-      "rewards/rejected": -0.34819021821022034,
+      "epoch": 0.56,
+      "learning_rate": 9.02767587848013e-07,
+      "logits/chosen": -0.1659875363111496,
+      "logits/rejected": -0.11425626277923584,
+      "logps/chosen": -322.26971435546875,
+      "logps/rejected": -269.41571044921875,
+      "loss": 0.6377,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.1545395851135254,
+      "rewards/margins": 0.16578510403633118,
+      "rewards/rejected": -0.32032471895217896,
       "step": 200
     },
     {
-      "epoch": 0.72,
-      "learning_rate": 8.049772071604863e-07,
-      "logits/chosen": 0.025003302842378616,
-      "logits/rejected": 0.056857384741306305,
-      "logps/chosen": -208.2877960205078,
-      "logps/rejected": -204.39129638671875,
-      "loss": 0.4466,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.3003290891647339,
-      "rewards/margins": 0.05031699687242508,
-      "rewards/rejected": -0.35064607858657837,
+      "epoch": 0.59,
+      "learning_rate": 8.877207101879301e-07,
+      "logits/chosen": -0.12081719934940338,
+      "logits/rejected": -0.055270951241254807,
+      "logps/chosen": -303.1000061035156,
+      "logps/rejected": -226.3379364013672,
+      "loss": 0.6265,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.1504133641719818,
+      "rewards/margins": 0.2184509038925171,
+      "rewards/rejected": -0.3688642680644989,
       "step": 210
     },
     {
-      "epoch": 0.76,
-      "learning_rate": 7.805935326811912e-07,
-      "logits/chosen": -0.040947750210762024,
-      "logits/rejected": 0.005281107500195503,
-      "logps/chosen": -198.28762817382812,
-      "logps/rejected": -201.60983276367188,
-      "loss": 0.4448,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.3105833828449249,
-      "rewards/margins": 0.04565637558698654,
-      "rewards/rejected": -0.3562397360801697,
+      "epoch": 0.62,
+      "learning_rate": 8.717368541944452e-07,
+      "logits/chosen": -0.14912042021751404,
+      "logits/rejected": -0.11508848518133163,
+      "logps/chosen": -313.95745849609375,
+      "logps/rejected": -232.803955078125,
+      "loss": 0.6245,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.17989172041416168,
+      "rewards/margins": 0.21268931031227112,
+      "rewards/rejected": -0.3925810158252716,
       "step": 220
     },
     {
-      "epoch": 0.79,
-      "learning_rate": 7.551938310719042e-07,
-      "logits/chosen": 0.004909530282020569,
-      "logits/rejected": 0.02475576661527157,
-      "logps/chosen": -217.17431640625,
-      "logps/rejected": -222.0946044921875,
-      "loss": 0.4414,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": -0.3833041191101074,
-      "rewards/margins": 0.03259982913732529,
-      "rewards/rejected": -0.4159039556980133,
+      "epoch": 0.65,
+      "learning_rate": 8.54854646967831e-07,
+      "logits/chosen": -0.20850150287151337,
+      "logits/rejected": -0.14839352667331696,
+      "logps/chosen": -351.56768798828125,
+      "logps/rejected": -286.0954284667969,
+      "loss": 0.6232,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.20482465624809265,
+      "rewards/margins": 0.2566435933113098,
+      "rewards/rejected": -0.46146830916404724,
       "step": 230
     },
     {
-      "epoch": 0.83,
-      "learning_rate": 7.288700744664167e-07,
-      "logits/chosen": 0.020644571632146835,
-      "logits/rejected": 0.05471833795309067,
-      "logps/chosen": -204.6312713623047,
-      "logps/rejected": -212.8608856201172,
-      "loss": 0.4231,
+      "epoch": 0.67,
+      "learning_rate": 8.371148865928318e-07,
+      "logits/chosen": -0.13414549827575684,
+      "logits/rejected": -0.09452676773071289,
+      "logps/chosen": -334.4315490722656,
+      "logps/rejected": -271.9152526855469,
+      "loss": 0.6104,
       "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.40746060013771057,
-      "rewards/margins": 0.06532532721757889,
-      "rewards/rejected": -0.47278594970703125,
+      "rewards/chosen": -0.18954138457775116,
+      "rewards/margins": 0.24784322082996368,
+      "rewards/rejected": -0.43738460540771484,
       "step": 240
     },
     {
-      "epoch": 0.86,
-      "learning_rate": 7.017175809949043e-07,
-      "logits/chosen": -0.00010571479651844129,
-      "logits/rejected": 0.024410605430603027,
-      "logps/chosen": -214.3887481689453,
-      "logps/rejected": -221.22830200195312,
-      "loss": 0.4142,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.41884541511535645,
-      "rewards/margins": 0.052488796412944794,
-      "rewards/rejected": -0.47133421897888184,
+      "epoch": 0.7,
+      "learning_rate": 8.185604435447001e-07,
+      "logits/chosen": -0.08637334406375885,
+      "logits/rejected": -0.03546728193759918,
+      "logps/chosen": -349.38787841796875,
+      "logps/rejected": -278.61761474609375,
+      "loss": 0.6238,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.2417462319135666,
+      "rewards/margins": 0.2916719615459442,
+      "rewards/rejected": -0.5334181785583496,
       "step": 250
     },
     {
-      "epoch": 0.9,
-      "learning_rate": 6.738346696376738e-07,
-      "logits/chosen": -0.027939433231949806,
-      "logits/rejected": 0.005430951714515686,
-      "logps/chosen": -215.529296875,
-      "logps/rejected": -218.4357452392578,
-      "loss": 0.418,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.45902904868125916,
-      "rewards/margins": 0.07481794059276581,
-      "rewards/rejected": -0.5338469743728638,
+      "epoch": 0.73,
+      "learning_rate": 7.992361570870287e-07,
+      "logits/chosen": -0.15081752836704254,
+      "logits/rejected": -0.11529238522052765,
+      "logps/chosen": -311.26507568359375,
+      "logps/rejected": -253.03659057617188,
+      "loss": 0.625,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.34321683645248413,
+      "rewards/margins": 0.19419686496257782,
+      "rewards/rejected": -0.5374137163162231,
       "step": 260
     },
     {
-      "epoch": 0.93,
-      "learning_rate": 6.453223042128555e-07,
-      "logits/chosen": 0.020816553384065628,
-      "logits/rejected": 0.034314144402742386,
-      "logps/chosen": -229.1612548828125,
-      "logps/rejected": -241.5037078857422,
-      "loss": 0.4108,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.48783358931541443,
-      "rewards/margins": 0.08682344853878021,
-      "rewards/rejected": -0.5746570229530334,
+      "epoch": 0.76,
+      "learning_rate": 7.791887269117441e-07,
+      "logits/chosen": -0.08736265450716019,
+      "logits/rejected": 0.010209694504737854,
+      "logps/chosen": -350.77197265625,
+      "logps/rejected": -249.4473114013672,
+      "loss": 0.6198,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.33235999941825867,
+      "rewards/margins": 0.3289110064506531,
+      "rewards/rejected": -0.6612709760665894,
       "step": 270
     },
     {
-      "epoch": 0.96,
-      "learning_rate": 6.162837277871552e-07,
-      "logits/chosen": 0.009753930382430553,
-      "logits/rejected": 0.015020027756690979,
-      "logps/chosen": -243.6470184326172,
-      "logps/rejected": -253.1077117919922,
-      "loss": 0.4073,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.5220585465431213,
-      "rewards/margins": 0.08642096817493439,
-      "rewards/rejected": -0.6084794998168945,
+      "epoch": 0.79,
+      "learning_rate": 7.584666002831294e-07,
+      "logits/chosen": -0.17468181252479553,
+      "logits/rejected": -0.10544611513614655,
+      "logps/chosen": -393.7838439941406,
+      "logps/rejected": -275.46636962890625,
+      "loss": 0.6284,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.20970232784748077,
+      "rewards/margins": 0.31893718242645264,
+      "rewards/rejected": -0.5286394357681274,
       "step": 280
     },
     {
-      "epoch": 1.0,
-      "learning_rate": 5.868240888334652e-07,
-      "logits/chosen": 0.04999474436044693,
-      "logits/rejected": 0.09121865034103394,
-      "logps/chosen": -241.92050170898438,
-      "logps/rejected": -245.2644805908203,
-      "loss": 0.3966,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": -0.5882852673530579,
-      "rewards/margins": 0.03265915438532829,
-      "rewards/rejected": -0.6209444403648376,
+      "epoch": 0.81,
+      "learning_rate": 7.37119854958609e-07,
+      "logits/chosen": -0.18520912528038025,
+      "logits/rejected": -0.12915001809597015,
+      "logps/chosen": -372.53961181640625,
+      "logps/rejected": -301.4205017089844,
+      "loss": 0.6426,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3479081690311432,
+      "rewards/margins": 0.25431200861930847,
+      "rewards/rejected": -0.6022201776504517,
       "step": 290
     },
     {
-      "epoch": 1.03,
-      "learning_rate": 5.570500604890124e-07,
-      "logits/chosen": 0.004443037323653698,
-      "logits/rejected": 0.03949873894453049,
-      "logps/chosen": -231.2201385498047,
-      "logps/rejected": -267.2882385253906,
-      "loss": 0.296,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -0.4168514311313629,
-      "rewards/margins": 0.3946831226348877,
-      "rewards/rejected": -0.811534583568573,
+      "epoch": 0.84,
+      "learning_rate": 7.152000781692285e-07,
+      "logits/chosen": -0.20307299494743347,
+      "logits/rejected": -0.1757517158985138,
+      "logps/chosen": -368.33392333984375,
+      "logps/rejected": -302.0033874511719,
+      "loss": 0.6219,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2545304298400879,
+      "rewards/margins": 0.2229403257369995,
+      "rewards/rejected": -0.4774707853794098,
       "step": 300
     },
     {
-      "epoch": 1.07,
-      "learning_rate": 5.270694542927088e-07,
-      "logits/chosen": 0.04513490945100784,
-      "logits/rejected": 0.04028277471661568,
-      "logps/chosen": -236.4629669189453,
-      "logps/rejected": -286.91162109375,
-      "loss": 0.2439,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -0.6017101407051086,
-      "rewards/margins": 0.4379204213619232,
-      "rewards/rejected": -1.03963041305542,
+      "epoch": 0.87,
+      "learning_rate": 6.927602419522946e-07,
+      "logits/chosen": -0.15039893984794617,
+      "logits/rejected": -0.1383979469537735,
+      "logps/chosen": -319.64349365234375,
+      "logps/rejected": -265.85198974609375,
+      "loss": 0.624,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.3667163848876953,
+      "rewards/margins": 0.18039202690124512,
+      "rewards/rejected": -0.5471083521842957,
       "step": 310
     },
     {
-      "epoch": 1.1,
-      "learning_rate": 4.969908298003572e-07,
-      "logits/chosen": 0.08646970987319946,
-      "logits/rejected": 0.07388903200626373,
-      "logps/chosen": -243.06735229492188,
-      "logps/rejected": -291.15106201171875,
-      "loss": 0.1796,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.9106127023696899,
-      "rewards/margins": 0.41714897751808167,
-      "rewards/rejected": -1.3277615308761597,
+      "epoch": 0.9,
+      "learning_rate": 6.698545751374463e-07,
+      "logits/chosen": -0.25044673681259155,
+      "logits/rejected": -0.12538060545921326,
+      "logps/chosen": -413.25946044921875,
+      "logps/rejected": -303.199951171875,
+      "loss": 0.6216,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.3125981390476227,
+      "rewards/margins": 0.38619184494018555,
+      "rewards/rejected": -0.6987899541854858,
       "step": 320
     },
     {
-      "epoch": 1.14,
-      "learning_rate": 4.669231014912942e-07,
-      "logits/chosen": 0.14787010848522186,
-      "logits/rejected": 0.15399041771888733,
-      "logps/chosen": -333.15264892578125,
-      "logps/rejected": -407.36944580078125,
-      "loss": 0.1341,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -1.4717319011688232,
-      "rewards/margins": 0.6132253408432007,
-      "rewards/rejected": -2.0849571228027344,
+      "epoch": 0.93,
+      "learning_rate": 6.465384322955224e-07,
+      "logits/chosen": -0.1757752150297165,
+      "logits/rejected": -0.11078636348247528,
+      "logps/chosen": -362.4681091308594,
+      "logps/rejected": -267.5201416015625,
+      "loss": 0.6074,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.33040475845336914,
+      "rewards/margins": 0.37607377767562866,
+      "rewards/rejected": -0.7064785957336426,
       "step": 330
     },
     {
-      "epoch": 1.17,
-      "learning_rate": 4.3697514438985533e-07,
-      "logits/chosen": 0.2917874753475189,
-      "logits/rejected": 0.3329147696495056,
-      "logps/chosen": -340.0228576660156,
-      "logps/rejected": -400.05853271484375,
-      "loss": 0.1011,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.6781260967254639,
-      "rewards/margins": 0.6559454202651978,
-      "rewards/rejected": -2.334071636199951,
+      "epoch": 0.96,
+      "learning_rate": 6.228681599669248e-07,
+      "logits/chosen": -0.19059506058692932,
+      "logits/rejected": -0.09184812009334564,
+      "logps/chosen": -430.0191345214844,
+      "logps/rejected": -296.02557373046875,
+      "loss": 0.617,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.3559810519218445,
+      "rewards/margins": 0.3992285132408142,
+      "rewards/rejected": -0.7552096247673035,
       "step": 340
     },
     {
-      "epoch": 1.21,
-      "learning_rate": 4.0725539982971027e-07,
-      "logits/chosen": 0.22039537131786346,
-      "logits/rejected": 0.22784602642059326,
-      "logps/chosen": -332.1070556640625,
-      "logps/rejected": -402.21575927734375,
-      "loss": 0.0947,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.765737533569336,
-      "rewards/margins": 0.6189795732498169,
-      "rewards/rejected": -2.384716749191284,
+      "epoch": 0.98,
+      "learning_rate": 5.989009604927586e-07,
+      "logits/chosen": -0.1337263584136963,
+      "logits/rejected": -0.01602357253432274,
+      "logps/chosen": -391.5013122558594,
+      "logps/rejected": -306.56982421875,
+      "loss": 0.5985,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.3254554867744446,
+      "rewards/margins": 0.4386405944824219,
+      "rewards/rejected": -0.7640960812568665,
       "step": 350
     },
     {
-      "epoch": 1.24,
-      "learning_rate": 3.778714827885845e-07,
-      "logits/chosen": 0.2562865614891052,
-      "logits/rejected": 0.28801122307777405,
-      "logps/chosen": -332.604248046875,
-      "logps/rejected": -385.0444030761719,
-      "loss": 0.0888,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.6409450769424438,
-      "rewards/margins": 0.5473521947860718,
-      "rewards/rejected": -2.1882972717285156,
+      "epoch": 1.01,
+      "learning_rate": 5.74694753777815e-07,
+      "logits/chosen": -0.13974535465240479,
+      "logits/rejected": -0.11217441409826279,
+      "logps/chosen": -327.7983703613281,
+      "logps/rejected": -279.513916015625,
+      "loss": 0.545,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.42400461435317993,
+      "rewards/margins": 0.4070449769496918,
+      "rewards/rejected": -0.8310495615005493,
       "step": 360
     },
     {
-      "epoch": 1.27,
-      "learning_rate": 3.489297922152136e-07,
-      "logits/chosen": 0.29671263694763184,
-      "logits/rejected": 0.2544638216495514,
-      "logps/chosen": -340.67266845703125,
-      "logps/rejected": -433.8108825683594,
-      "loss": 0.0848,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.7547416687011719,
-      "rewards/margins": 0.7966595888137817,
-      "rewards/rejected": -2.551401138305664,
+      "epoch": 1.04,
+      "learning_rate": 5.503080373194666e-07,
+      "logits/chosen": -0.2235272228717804,
+      "logits/rejected": -0.12439509481191635,
+      "logps/chosen": -374.24261474609375,
+      "logps/rejected": -333.127685546875,
+      "loss": 0.4827,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.4394102096557617,
+      "rewards/margins": 0.6707636713981628,
+      "rewards/rejected": -1.1101738214492798,
       "step": 370
     },
     {
-      "epoch": 1.31,
-      "learning_rate": 3.205351257595271e-07,
-      "logits/chosen": 0.3430071473121643,
-      "logits/rejected": 0.3394010663032532,
-      "logps/chosen": -339.52386474609375,
-      "logps/rejected": -415.0911560058594,
-      "loss": 0.075,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.862532377243042,
-      "rewards/margins": 0.7082081437110901,
-      "rewards/rejected": -2.5707404613494873,
+      "epoch": 1.07,
+      "learning_rate": 5.257997448407366e-07,
+      "logits/chosen": -0.12490369379520416,
+      "logits/rejected": 0.020254041999578476,
+      "logps/chosen": -408.45367431640625,
+      "logps/rejected": -308.180908203125,
+      "loss": 0.4897,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.4755324423313141,
+      "rewards/margins": 0.6596079468727112,
+      "rewards/rejected": -1.1351404190063477,
       "step": 380
     },
     {
-      "epoch": 1.34,
-      "learning_rate": 2.9279030030112407e-07,
-      "logits/chosen": 0.4035106599330902,
-      "logits/rejected": 0.34052735567092896,
-      "logps/chosen": -378.32696533203125,
-      "logps/rejected": -467.59075927734375,
-      "loss": 0.0663,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -2.0151519775390625,
-      "rewards/margins": 0.8498624563217163,
-      "rewards/rejected": -2.8650143146514893,
+      "epoch": 1.1,
+      "learning_rate": 5.012291038691665e-07,
+      "logits/chosen": -0.15194548666477203,
+      "logits/rejected": -0.03352439031004906,
+      "logps/chosen": -419.17730712890625,
+      "logps/rejected": -391.656982421875,
+      "loss": 0.4746,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.6772257089614868,
+      "rewards/margins": 0.8299336433410645,
+      "rewards/rejected": -1.5071594715118408,
       "step": 390
     },
     {
-      "epoch": 1.38,
-      "learning_rate": 2.65795779650105e-07,
-      "logits/chosen": 0.6060320138931274,
-      "logits/rejected": 0.5839828252792358,
-      "logps/chosen": -379.7890625,
-      "logps/rejected": -466.09075927734375,
-      "loss": 0.0648,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -2.1252145767211914,
-      "rewards/margins": 0.841218113899231,
-      "rewards/rejected": -2.966432809829712,
+      "epoch": 1.12,
+      "learning_rate": 4.7665549260567063e-07,
+      "logits/chosen": -0.20493462681770325,
+      "logits/rejected": -0.13634856045246124,
+      "logps/chosen": -411.919189453125,
+      "logps/rejected": -344.78759765625,
+      "loss": 0.4713,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.7057473063468933,
+      "rewards/margins": 0.824458122253418,
+      "rewards/rejected": -1.530205249786377,
       "step": 400
     },
     {
-      "epoch": 1.41,
-      "learning_rate": 2.3964931076834876e-07,
-      "logits/chosen": 0.5519194602966309,
-      "logits/rejected": 0.6366934776306152,
-      "logps/chosen": -366.50872802734375,
-      "logps/rejected": -428.6471252441406,
-      "loss": 0.0666,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -2.167186737060547,
-      "rewards/margins": 0.6175845861434937,
-      "rewards/rejected": -2.78477144241333,
+      "epoch": 1.15,
+      "learning_rate": 4.521382964292663e-07,
+      "logits/chosen": -0.14551517367362976,
+      "logits/rejected": -0.07351706176996231,
+      "logps/chosen": -394.6050109863281,
+      "logps/rejected": -345.28082275390625,
+      "loss": 0.4741,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.6415594816207886,
+      "rewards/margins": 0.7878659963607788,
+      "rewards/rejected": -1.4294254779815674,
       "step": 410
     },
     {
-      "epoch": 1.45,
-      "learning_rate": 2.1444556982847995e-07,
-      "logits/chosen": 0.6113961338996887,
-      "logits/rejected": 0.6564093828201294,
-      "logps/chosen": -409.9825134277344,
-      "logps/rejected": -469.96697998046875,
-      "loss": 0.06,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -2.302380323410034,
-      "rewards/margins": 0.7064192295074463,
-      "rewards/rejected": -3.0087997913360596,
+      "epoch": 1.18,
+      "learning_rate": 4.277367643844574e-07,
+      "logits/chosen": -0.1973930448293686,
+      "logits/rejected": -0.10258449614048004,
+      "logps/chosen": -446.4418029785156,
+      "logps/rejected": -366.457763671875,
+      "loss": 0.4605,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.8334900736808777,
+      "rewards/margins": 0.7750234603881836,
+      "rewards/rejected": -1.608513593673706,
       "step": 420
     },
     {
-      "epoch": 1.48,
-      "learning_rate": 1.902758193921385e-07,
-      "logits/chosen": 0.5821201205253601,
-      "logits/rejected": 0.6013978123664856,
-      "logps/chosen": -398.11602783203125,
-      "logps/rejected": -498.4593811035156,
-      "loss": 0.0628,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -2.178372859954834,
-      "rewards/margins": 0.9250782132148743,
-      "rewards/rejected": -3.1034510135650635,
+      "epoch": 1.21,
+      "learning_rate": 4.035098659980891e-07,
+      "logits/chosen": -0.1986023634672165,
+      "logits/rejected": -0.06475992500782013,
+      "logps/chosen": -419.17010498046875,
+      "logps/rejected": -395.53509521484375,
+      "loss": 0.465,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.0268628597259521,
+      "rewards/margins": 0.7243725657463074,
+      "rewards/rejected": -1.7512352466583252,
       "step": 430
     },
     {
-      "epoch": 1.52,
-      "learning_rate": 1.6722757794891285e-07,
-      "logits/chosen": 0.7665516138076782,
-      "logits/rejected": 0.7662865519523621,
-      "logps/chosen": -377.22186279296875,
-      "logps/rejected": -439.4972229003906,
-      "loss": 0.0685,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -2.1608002185821533,
-      "rewards/margins": 0.674599289894104,
-      "rewards/rejected": -2.835399866104126,
+      "epoch": 1.24,
+      "learning_rate": 3.795161487716928e-07,
+      "logits/chosen": -0.20193979144096375,
+      "logits/rejected": -0.09314943850040436,
+      "logps/chosen": -491.21405029296875,
+      "logps/rejected": -423.76654052734375,
+      "loss": 0.4524,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.0318418741226196,
+      "rewards/margins": 0.9399174451828003,
+      "rewards/rejected": -1.9717592000961304,
       "step": 440
     },
     {
-      "epoch": 1.55,
-      "learning_rate": 1.453843030125278e-07,
-      "logits/chosen": 0.6384638547897339,
-      "logits/rejected": 0.6626998782157898,
-      "logps/chosen": -406.65338134765625,
-      "logps/rejected": -464.52374267578125,
-      "loss": 0.065,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -2.227891683578491,
-      "rewards/margins": 0.6393032073974609,
-      "rewards/rejected": -2.8671953678131104,
+      "epoch": 1.26,
+      "learning_rate": 3.5581359669371223e-07,
+      "logits/chosen": -0.11288833618164062,
+      "logits/rejected": -0.1004786491394043,
+      "logps/chosen": -417.3036193847656,
+      "logps/rejected": -372.6753845214844,
+      "loss": 0.4561,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.969109058380127,
+      "rewards/margins": 0.6991706490516663,
+      "rewards/rejected": -1.6682794094085693,
       "step": 450
     },
     {
-      "epoch": 1.58,
-      "learning_rate": 1.2482508892179883e-07,
-      "logits/chosen": 0.6284786462783813,
-      "logits/rejected": 0.5958371162414551,
-      "logps/chosen": -404.72796630859375,
-      "logps/rejected": -500.9256286621094,
-      "loss": 0.0628,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.1921005249023438,
-      "rewards/margins": 0.9022598266601562,
-      "rewards/rejected": -3.094360113143921,
+      "epoch": 1.29,
+      "learning_rate": 3.324594901135326e-07,
+      "logits/chosen": -0.17446324229240417,
+      "logits/rejected": -0.0842059999704361,
+      "logps/chosen": -438.217041015625,
+      "logps/rejected": -390.52874755859375,
+      "loss": 0.459,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.0395996570587158,
+      "rewards/margins": 0.7797072529792786,
+      "rewards/rejected": -1.8193069696426392,
       "step": 460
     },
     {
-      "epoch": 1.62,
-      "learning_rate": 1.0562438044060846e-07,
-      "logits/chosen": 0.6781089901924133,
-      "logits/rejected": 0.6540604829788208,
-      "logps/chosen": -373.59014892578125,
-      "logps/rejected": -451.7066345214844,
-      "loss": 0.0619,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -2.1520633697509766,
-      "rewards/margins": 0.7549977898597717,
-      "rewards/rejected": -2.9070611000061035,
+      "epoch": 1.32,
+      "learning_rate": 3.095102673159463e-07,
+      "logits/chosen": -0.13965001702308655,
+      "logits/rejected": -0.09614251554012299,
+      "logps/chosen": -477.21337890625,
+      "logps/rejected": -420.8619079589844,
+      "loss": 0.4336,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -1.1968367099761963,
+      "rewards/margins": 0.8374635577201843,
+      "rewards/rejected": -2.0343000888824463,
       "step": 470
     },
     {
-      "epoch": 1.65,
-      "learning_rate": 8.785170319396174e-08,
-      "logits/chosen": 0.6746602058410645,
-      "logits/rejected": 0.6840069890022278,
-      "logps/chosen": -407.03350830078125,
-      "logps/rejected": -509.2193298339844,
-      "loss": 0.0583,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -2.3068630695343018,
-      "rewards/margins": 0.9839746356010437,
-      "rewards/rejected": -3.2908377647399902,
+      "epoch": 1.35,
+      "learning_rate": 2.870213881305802e-07,
+      "logits/chosen": -0.06153956800699234,
+      "logits/rejected": 0.04337477311491966,
+      "logps/chosen": -457.060302734375,
+      "logps/rejected": -406.51300048828125,
+      "loss": 0.4411,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.1784467697143555,
+      "rewards/margins": 0.8234208226203918,
+      "rewards/rejected": -2.0018675327301025,
       "step": 480
     },
     {
-      "epoch": 1.69,
-      "learning_rate": 7.157141191620548e-08,
-      "logits/chosen": 0.7601083517074585,
-      "logits/rejected": 0.8036757707595825,
-      "logps/chosen": -383.3311462402344,
-      "logps/rejected": -455.77447509765625,
-      "loss": 0.0588,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -2.1426267623901367,
-      "rewards/margins": 0.7608777284622192,
-      "rewards/rejected": -2.9035048484802246,
+      "epoch": 1.38,
+      "learning_rate": 2.6504719990588745e-07,
+      "logits/chosen": -0.05881907790899277,
+      "logits/rejected": 0.013253748416900635,
+      "logps/chosen": -455.1761169433594,
+      "logps/rejected": -401.30133056640625,
+      "loss": 0.4396,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -1.2112154960632324,
+      "rewards/margins": 0.8642939329147339,
+      "rewards/rejected": -2.0755093097686768,
       "step": 490
     },
     {
-      "epoch": 1.72,
-      "learning_rate": 5.684245742300625e-08,
-      "logits/chosen": 0.7304960489273071,
-      "logits/rejected": 0.7055848240852356,
-      "logps/chosen": -382.73931884765625,
-      "logps/rejected": -500.84515380859375,
-      "loss": 0.058,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -2.192786931991577,
-      "rewards/margins": 1.0639842748641968,
-      "rewards/rejected": -3.2567710876464844,
+      "epoch": 1.4,
+      "learning_rate": 2.436408061715988e-07,
+      "logits/chosen": 0.04037608206272125,
+      "logits/rejected": 0.047458432614803314,
+      "logps/chosen": -382.39141845703125,
+      "logps/rejected": -422.6522521972656,
+      "loss": 0.4409,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -1.066866159439087,
+      "rewards/margins": 0.8686450719833374,
+      "rewards/rejected": -1.9355109930038452,
       "step": 500
     },
     {
-      "epoch": 1.76,
-      "learning_rate": 4.3718173150878446e-08,
-      "logits/chosen": 0.9688064455986023,
-      "logits/rejected": 1.001210331916809,
-      "logps/chosen": -388.23193359375,
-      "logps/rejected": -464.37371826171875,
-      "loss": 0.0555,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -2.178377866744995,
-      "rewards/margins": 0.7957491874694824,
-      "rewards/rejected": -2.9741270542144775,
+      "epoch": 1.43,
+      "learning_rate": 2.22853938307025e-07,
+      "logits/chosen": 0.09550214558839798,
+      "logits/rejected": 0.1434255838394165,
+      "logps/chosen": -393.2830810546875,
+      "logps/rejected": -355.2184753417969,
+      "loss": 0.4469,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.153294324874878,
+      "rewards/margins": 0.6956819891929626,
+      "rewards/rejected": -1.8489763736724854,
       "step": 510
     },
     {
-      "epoch": 1.79,
-      "learning_rate": 3.2246082037199526e-08,
-      "logits/chosen": 0.8018981218338013,
-      "logits/rejected": 0.8608972430229187,
-      "logps/chosen": -382.80157470703125,
-      "logps/rejected": -468.1249084472656,
-      "loss": 0.0604,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -2.2035393714904785,
-      "rewards/margins": 0.8492420315742493,
-      "rewards/rejected": -3.052781343460083,
+      "epoch": 1.46,
+      "learning_rate": 2.0273683052534173e-07,
+      "logits/chosen": 0.14489802718162537,
+      "logits/rejected": 0.22391514480113983,
+      "logps/chosen": -465.29595947265625,
+      "logps/rejected": -435.1458435058594,
+      "loss": 0.4414,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.2600550651550293,
+      "rewards/margins": 0.9257229566574097,
+      "rewards/rejected": -2.1857781410217285,
       "step": 520
     },
     {
-      "epoch": 1.83,
-      "learning_rate": 2.2467724440002333e-08,
-      "logits/chosen": 0.7628926038742065,
-      "logits/rejected": 0.7621763944625854,
-      "logps/chosen": -394.536376953125,
-      "logps/rejected": -478.94598388671875,
-      "loss": 0.0564,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -2.241116523742676,
-      "rewards/margins": 0.7960697412490845,
-      "rewards/rejected": -3.0371861457824707,
+      "epoch": 1.49,
+      "learning_rate": 1.833380984759764e-07,
+      "logits/chosen": 0.07251317799091339,
+      "logits/rejected": 0.17206279933452606,
+      "logps/chosen": -410.1485900878906,
+      "logps/rejected": -438.0814514160156,
+      "loss": 0.4487,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.2980937957763672,
+      "rewards/margins": 0.8264884948730469,
+      "rewards/rejected": -2.124582290649414,
       "step": 530
     },
     {
-      "epoch": 1.86,
-      "learning_rate": 1.4418507720641793e-08,
-      "logits/chosen": 0.8032611012458801,
-      "logits/rejected": 0.8268760442733765,
-      "logps/chosen": -413.1278381347656,
-      "logps/rejected": -507.7405700683594,
-      "loss": 0.0575,
+      "epoch": 1.52,
+      "learning_rate": 1.6470462175846606e-07,
+      "logits/chosen": 0.09354039281606674,
+      "logits/rejected": 0.13454324007034302,
+      "logps/chosen": -464.46990966796875,
+      "logps/rejected": -444.4037170410156,
+      "loss": 0.4383,
       "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -2.3634586334228516,
-      "rewards/margins": 0.9177207946777344,
-      "rewards/rejected": -3.281179428100586,
+      "rewards/chosen": -1.0875951051712036,
+      "rewards/margins": 0.913548469543457,
+      "rewards/rejected": -2.001143455505371,
       "step": 540
     },
     {
-      "epoch": 1.89,
-      "learning_rate": 8.127578033998661e-09,
-      "logits/chosen": 0.8457974195480347,
-      "logits/rejected": 0.8470609784126282,
-      "logps/chosen": -351.919677734375,
-      "logps/rejected": -441.13238525390625,
-      "loss": 0.0589,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.1192312240600586,
-      "rewards/margins": 0.815728485584259,
-      "rewards/rejected": -2.934959888458252,
+      "epoch": 1.55,
+      "learning_rate": 1.468814306317092e-07,
+      "logits/chosen": 0.12879224121570587,
+      "logits/rejected": 0.1979069709777832,
+      "logps/chosen": -426.343505859375,
+      "logps/rejected": -380.24493408203125,
+      "loss": 0.4398,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -1.0752108097076416,
+      "rewards/margins": 0.9150044322013855,
+      "rewards/rejected": -1.9902150630950928,
       "step": 550
     },
     {
-      "epoch": 1.93,
-      "learning_rate": 3.6177147904655757e-09,
-      "logits/chosen": 0.8455848693847656,
-      "logits/rejected": 0.8221101760864258,
-      "logps/chosen": -420.8208923339844,
-      "logps/rejected": -514.7294311523438,
-      "loss": 0.0538,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -2.299787998199463,
-      "rewards/margins": 0.9560101628303528,
-      "rewards/rejected": -3.255797863006592,
+      "epoch": 1.57,
+      "learning_rate": 1.299115971923958e-07,
+      "logits/chosen": 0.12415747344493866,
+      "logits/rejected": 0.20912082493305206,
+      "logps/chosen": -448.64031982421875,
+      "logps/rejected": -442.22576904296875,
+      "loss": 0.4317,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -1.2348341941833496,
+      "rewards/margins": 0.9574505090713501,
+      "rewards/rejected": -2.1922848224639893,
       "step": 560
     },
     {
-      "epoch": 1.96,
-      "learning_rate": 9.052481718690997e-10,
-      "logits/chosen": 0.8840595483779907,
-      "logits/rejected": 0.8795121908187866,
-      "logps/chosen": -424.8587951660156,
-      "logps/rejected": -513.06591796875,
-      "loss": 0.0555,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -2.4113454818725586,
-      "rewards/margins": 0.8990417718887329,
-      "rewards/rejected": -3.310387372970581,
+      "epoch": 1.6,
+      "learning_rate": 1.1383613128559305e-07,
+      "logits/chosen": 0.024107109755277634,
+      "logits/rejected": 0.1119920164346695,
+      "logps/chosen": -505.82928466796875,
+      "logps/rejected": -505.4091796875,
+      "loss": 0.426,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -1.2975566387176514,
+      "rewards/margins": 1.085707187652588,
+      "rewards/rejected": -2.3832640647888184,
       "step": 570
     },
+    {
+      "epoch": 1.63,
+      "learning_rate": 9.869388139903495e-08,
+      "logits/chosen": 0.012987576425075531,
+      "logits/rejected": 0.06353282183408737,
+      "logps/chosen": -541.9244995117188,
+      "logps/rejected": -499.44110107421875,
+      "loss": 0.4104,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.3797237873077393,
+      "rewards/margins": 0.9557130932807922,
+      "rewards/rejected": -2.335437059402466,
+      "step": 580
+    },
+    {
+      "epoch": 1.66,
+      "learning_rate": 8.452144078061818e-08,
+      "logits/chosen": 0.14748625457286835,
+      "logits/rejected": 0.21033573150634766,
+      "logps/chosen": -428.7500915527344,
+      "logps/rejected": -418.6878356933594,
+      "loss": 0.4248,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.361243724822998,
+      "rewards/margins": 0.893297553062439,
+      "rewards/rejected": -2.2545411586761475,
+      "step": 590
+    },
+    {
+      "epoch": 1.69,
+      "learning_rate": 7.135305900598321e-08,
+      "logits/chosen": 0.012422094121575356,
+      "logits/rejected": 0.08027663081884384,
+      "logps/chosen": -506.3257751464844,
+      "logps/rejected": -460.52777099609375,
+      "loss": 0.4245,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -1.5179449319839478,
+      "rewards/margins": 1.0011281967163086,
+      "rewards/rejected": -2.5190727710723877,
+      "step": 600
+    },
+    {
+      "epoch": 1.71,
+      "learning_rate": 5.9220559209888166e-08,
+      "logits/chosen": 0.12760373950004578,
+      "logits/rejected": 0.20078882575035095,
+      "logps/chosen": -452.77734375,
+      "logps/rejected": -487.36016845703125,
+      "loss": 0.4266,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -1.444874882698059,
+      "rewards/margins": 1.0283567905426025,
+      "rewards/rejected": -2.473231792449951,
+      "step": 610
+    },
+    {
+      "epoch": 1.74,
+      "learning_rate": 4.815326118139812e-08,
+      "logits/chosen": 0.14059333503246307,
+      "logits/rejected": 0.2606312930583954,
+      "logps/chosen": -415.8949279785156,
+      "logps/rejected": -418.4481506347656,
+      "loss": 0.4339,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.5133248567581177,
+      "rewards/margins": 0.8190656900405884,
+      "rewards/rejected": -2.332390546798706,
+      "step": 620
+    },
+    {
+      "epoch": 1.77,
+      "learning_rate": 3.81779105087407e-08,
+      "logits/chosen": 0.08177526295185089,
+      "logits/rejected": 0.21824178099632263,
+      "logps/chosen": -476.4537658691406,
+      "logps/rejected": -467.0603942871094,
+      "loss": 0.4436,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.559093713760376,
+      "rewards/margins": 0.9111778140068054,
+      "rewards/rejected": -2.470271587371826,
+      "step": 630
+    },
+    {
+      "epoch": 1.8,
+      "learning_rate": 2.9318613945057637e-08,
+      "logits/chosen": 0.12043829262256622,
+      "logits/rejected": 0.19589121639728546,
+      "logps/chosen": -504.67608642578125,
+      "logps/rejected": -489.88330078125,
+      "loss": 0.4181,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -1.4812028408050537,
+      "rewards/margins": 1.078580617904663,
+      "rewards/rejected": -2.559783458709717,
+      "step": 640
+    },
+    {
+      "epoch": 1.83,
+      "learning_rate": 2.1596781151249523e-08,
+      "logits/chosen": 0.15832188725471497,
+      "logits/rejected": 0.2765315771102905,
+      "logps/chosen": -461.1602478027344,
+      "logps/rejected": -456.65826416015625,
+      "loss": 0.4321,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.4598538875579834,
+      "rewards/margins": 0.8676374554634094,
+      "rewards/rejected": -2.327491283416748,
+      "step": 650
+    },
+    {
+      "epoch": 1.85,
+      "learning_rate": 1.5031072956701695e-08,
+      "logits/chosen": 0.13437607884407043,
+      "logits/rejected": 0.19320687651634216,
+      "logps/chosen": -503.2486267089844,
+      "logps/rejected": -498.3321228027344,
+      "loss": 0.4196,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -1.5257575511932373,
+      "rewards/margins": 1.106710433959961,
+      "rewards/rejected": -2.632467746734619,
+      "step": 660
+    },
+    {
+      "epoch": 1.88,
+      "learning_rate": 9.637356262923723e-09,
+      "logits/chosen": 0.24336127936840057,
+      "logits/rejected": 0.2697839140892029,
+      "logps/chosen": -468.03289794921875,
+      "logps/rejected": -444.1060485839844,
+      "loss": 0.4387,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -1.5293312072753906,
+      "rewards/margins": 0.842644214630127,
+      "rewards/rejected": -2.3719754219055176,
+      "step": 670
+    },
+    {
+      "epoch": 1.91,
+      "learning_rate": 5.428665699084789e-09,
+      "logits/chosen": 0.10800081491470337,
+      "logits/rejected": 0.1402808427810669,
+      "logps/chosen": -492.41448974609375,
+      "logps/rejected": -494.878173828125,
+      "loss": 0.4088,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -1.4665706157684326,
+      "rewards/margins": 1.0468090772628784,
+      "rewards/rejected": -2.5133795738220215,
+      "step": 680
+    },
+    {
+      "epoch": 1.94,
+      "learning_rate": 2.415172122110343e-09,
+      "logits/chosen": 0.13986334204673767,
+      "logits/rejected": 0.26917481422424316,
+      "logps/chosen": -483.06024169921875,
+      "logps/rejected": -482.5399475097656,
+      "loss": 0.4382,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -1.4039775133132935,
+      "rewards/margins": 1.0037094354629517,
+      "rewards/rejected": -2.407686948776245,
+      "step": 690
+    },
+    {
+      "epoch": 1.97,
+      "learning_rate": 6.041580374618327e-10,
+      "logits/chosen": 0.105182945728302,
+      "logits/rejected": 0.17750731110572815,
+      "logps/chosen": -485.490478515625,
+      "logps/rejected": -456.46063232421875,
+      "loss": 0.4411,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -1.5071966648101807,
+      "rewards/margins": 0.8808605074882507,
+      "rewards/rejected": -2.388056993484497,
+      "step": 700
+    },
     {
       "epoch": 2.0,
       "learning_rate": 0.0,
-      "logits/chosen": 0.9808861017227173,
-      "logits/rejected": 0.9314306378364563,
-      "logps/chosen": -372.04034423828125,
-      "logps/rejected": -452.5663146972656,
-      "loss": 0.0544,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -2.2187602519989014,
-      "rewards/margins": 0.7363004684448242,
-      "rewards/rejected": -2.9550607204437256,
-      "step": 580
+      "logits/chosen": 0.1599053144454956,
+      "logits/rejected": 0.2173675298690796,
+      "logps/chosen": -413.2229919433594,
+      "logps/rejected": -438.8427734375,
+      "loss": 0.4277,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.4848761558532715,
+      "rewards/margins": 0.8370813131332397,
+      "rewards/rejected": -2.3219573497772217,
+      "step": 710
     },
     {
       "epoch": 2.0,
-      "step": 580,
+      "step": 710,
       "total_flos": 0.0,
-      "train_loss": 0.31343152379167494,
-      "train_runtime": 8630.1644,
-      "train_samples_per_second": 8.607,
-      "train_steps_per_second": 0.067
+      "train_loss": 0.5407548575334146,
+      "train_runtime": 10382.7821,
+      "train_samples_per_second": 8.774,
+      "train_steps_per_second": 0.068
     }
   ],
   "logging_steps": 10,
-  "max_steps": 580,
+  "max_steps": 710,
   "num_train_epochs": 2,
   "save_steps": 10000,
   "total_flos": 0.0,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc9d7c7d0bacc27c393dcf6748242363d5a7ffc1ed7d1bdf238aa7708ab53c34
+oid sha256:1091900189f2bf20317d25f99163db88a112c122ef6171e24251c1551cf023ed
 size 6648