Model save

README.md

@@ -32,7 +32,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 1e-06
+- learning_rate: 5e-06
 - train_batch_size: 2
 - eval_batch_size: 8
 - seed: 1

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 2.0,
-    "train_loss": 0.25192113166124047,
-    "train_runtime": 8113.4503,
+    "train_loss": 0.1250429530667377,
+    "train_runtime": 7954.4917,
     "train_samples": 45548,
-    "train_samples_per_second": 11.228,
-    "train_steps_per_second": 0.088
+    "train_samples_per_second": 11.452,
+    "train_steps_per_second": 0.089
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3402aa3938e1c31d822d0e375cf01dcc5f8377b7a31a3b6cf0575f1fb7953d4a
+oid sha256:bb89283c64a9aa24f68c2e6ff899db2a4eff0089ec7f651d36a630d06ead65c6
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:655f95ca04b0961ed4b515a7348c05cf72428837d0d1656bb20de7415dd6d23b
+oid sha256:2e4e4da391be8ea6eb88812a2d53d80df1fe445d950f4dab4b0ab10af79c538e
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8c376df770dfdaedca5de952735c527acff55b46dffecb9de5c18634541d6e2
+oid sha256:91ff76793cb8b35d103a71c2a1031a7a34debec12bd3585181646e51a457a68e
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a996e33d039658928f6a70a590d6caec1f6957e1db80bebabe1c6f9186fd855d
+oid sha256:d07159171f547d0196276801d699317fda916d2e68e7f02ad52368f2eca2348e
 size 1168138808

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 2.0,
-    "train_loss": 0.25192113166124047,
-    "train_runtime": 8113.4503,
+    "train_loss": 0.1250429530667377,
+    "train_runtime": 7954.4917,
     "train_samples": 45548,
-    "train_samples_per_second": 11.228,
-    "train_steps_per_second": 0.088
+    "train_samples_per_second": 11.452,
+    "train_steps_per_second": 0.089
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "learning_rate": 1.408450704225352e-08,
+      "learning_rate": 7.042253521126761e-08,
       "logits/chosen": -0.20257116854190826,
       "logits/rejected": -0.07568661123514175,
       "logps/chosen": -521.0269775390625,
@@ -24,1006 +24,1006 @@
     },
     {
       "epoch": 0.03,
-      "learning_rate": 1.4084507042253522e-07,
-      "logits/chosen": -0.13786724209785461,
-      "logits/rejected": -0.08816312253475189,
-      "logps/chosen": -345.8097229003906,
-      "logps/rejected": -280.6612854003906,
-      "loss": 0.3323,
-      "rewards/accuracies": 0.4444444477558136,
-      "rewards/chosen": 0.0006422583828680217,
-      "rewards/margins": 0.0004346017958596349,
-      "rewards/rejected": 0.00020765644148923457,
+      "learning_rate": 7.042253521126762e-07,
+      "logits/chosen": -0.13681186735630035,
+      "logits/rejected": -0.0872005894780159,
+      "logps/chosen": -345.9254150390625,
+      "logps/rejected": -280.7078857421875,
+      "loss": 0.3327,
+      "rewards/accuracies": 0.4791666567325592,
+      "rewards/chosen": -0.000514834770001471,
+      "rewards/margins": -0.00025620695669203997,
+      "rewards/rejected": -0.00025862775510177016,
       "step": 10
     },
     {
       "epoch": 0.06,
-      "learning_rate": 2.8169014084507043e-07,
-      "logits/chosen": -0.09473811089992523,
-      "logits/rejected": -0.056791383773088455,
-      "logps/chosen": -331.81280517578125,
-      "logps/rejected": -241.23910522460938,
-      "loss": 0.3272,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": 0.0002513823565095663,
-      "rewards/margins": 0.0021114428527653217,
-      "rewards/rejected": -0.001860060729086399,
+      "learning_rate": 1.4084507042253523e-06,
+      "logits/chosen": -0.09434644877910614,
+      "logits/rejected": -0.054363738745450974,
+      "logps/chosen": -332.7166442871094,
+      "logps/rejected": -246.1228790283203,
+      "loss": 0.3405,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.008787451311945915,
+      "rewards/margins": 0.04191022366285324,
+      "rewards/rejected": -0.05069767311215401,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "learning_rate": 4.225352112676056e-07,
-      "logits/chosen": -0.05887429043650627,
-      "logits/rejected": -0.0021190904080867767,
-      "logps/chosen": -387.2514953613281,
-      "logps/rejected": -252.19833374023438,
-      "loss": 0.3291,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.004256925545632839,
-      "rewards/margins": 0.018127940595149994,
-      "rewards/rejected": -0.01387101411819458,
+      "learning_rate": 2.1126760563380285e-06,
+      "logits/chosen": -0.06431654840707779,
+      "logits/rejected": -0.008967995643615723,
+      "logps/chosen": -398.9429931640625,
+      "logps/rejected": -280.65277099609375,
+      "loss": 0.3833,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.11265812814235687,
+      "rewards/margins": 0.18575695157051086,
+      "rewards/rejected": -0.29841506481170654,
       "step": 30
     },
     {
       "epoch": 0.11,
-      "learning_rate": 5.633802816901409e-07,
-      "logits/chosen": -0.07777061313390732,
-      "logits/rejected": -0.023820143193006516,
-      "logps/chosen": -335.6570129394531,
-      "logps/rejected": -263.23907470703125,
-      "loss": 0.3421,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.004789065103977919,
-      "rewards/margins": 0.021946147084236145,
-      "rewards/rejected": -0.02673521265387535,
+      "learning_rate": 2.8169014084507046e-06,
+      "logits/chosen": -0.10810466855764389,
+      "logits/rejected": -0.04957962408661842,
+      "logps/chosen": -347.4810485839844,
+      "logps/rejected": -286.8487854003906,
+      "loss": 0.4038,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.12302961200475693,
+      "rewards/margins": 0.13980238139629364,
+      "rewards/rejected": -0.26283198595046997,
       "step": 40
     },
     {
       "epoch": 0.14,
-      "learning_rate": 7.04225352112676e-07,
-      "logits/chosen": -0.03277156129479408,
-      "logits/rejected": 0.012101896107196808,
-      "logps/chosen": -336.37176513671875,
-      "logps/rejected": -272.0014953613281,
-      "loss": 0.358,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.026565855368971825,
-      "rewards/margins": 0.06764300167560577,
-      "rewards/rejected": -0.09420885145664215,
+      "learning_rate": 3.5211267605633804e-06,
+      "logits/chosen": -0.09582972526550293,
+      "logits/rejected": -0.03566538169980049,
+      "logps/chosen": -331.12353515625,
+      "logps/rejected": -272.82281494140625,
+      "loss": 0.3735,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.02591649256646633,
+      "rewards/margins": 0.12833817303180695,
+      "rewards/rejected": -0.10242166370153427,
       "step": 50
     },
     {
       "epoch": 0.17,
-      "learning_rate": 8.450704225352112e-07,
-      "logits/chosen": -0.11417659372091293,
-      "logits/rejected": -0.05113930255174637,
-      "logps/chosen": -427.59600830078125,
-      "logps/rejected": -268.9815673828125,
-      "loss": 0.3784,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.052729975432157516,
-      "rewards/margins": 0.18380871415138245,
-      "rewards/rejected": -0.23653869330883026,
+      "learning_rate": 4.225352112676057e-06,
+      "logits/chosen": -0.28920671343803406,
+      "logits/rejected": -0.21953177452087402,
+      "logps/chosen": -414.81640625,
+      "logps/rejected": -273.1141052246094,
+      "loss": 0.3825,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.07506568729877472,
+      "rewards/margins": 0.3529297113418579,
+      "rewards/rejected": -0.2778640389442444,
       "step": 60
     },
     {
       "epoch": 0.2,
-      "learning_rate": 9.859154929577465e-07,
-      "logits/chosen": -0.12808646261692047,
-      "logits/rejected": -0.009413952939212322,
-      "logps/chosen": -417.78680419921875,
-      "logps/rejected": -287.1832580566406,
-      "loss": 0.3848,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.10650447756052017,
-      "rewards/margins": 0.15690955519676208,
-      "rewards/rejected": -0.26341402530670166,
+      "learning_rate": 4.929577464788733e-06,
+      "logits/chosen": -0.1399366408586502,
+      "logits/rejected": -0.017617041245102882,
+      "logps/chosen": -428.28375244140625,
+      "logps/rejected": -310.7579650878906,
+      "loss": 0.3466,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.21147418022155762,
+      "rewards/margins": 0.28768691420555115,
+      "rewards/rejected": -0.49916109442710876,
       "step": 70
     },
     {
       "epoch": 0.22,
-      "learning_rate": 9.995106132599868e-07,
-      "logits/chosen": -0.0769750103354454,
-      "logits/rejected": -0.06951722502708435,
-      "logps/chosen": -294.3128662109375,
-      "logps/rejected": -227.47463989257812,
-      "loss": 0.3948,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.06992264091968536,
-      "rewards/margins": 0.16703224182128906,
-      "rewards/rejected": -0.23695488274097443,
+      "learning_rate": 4.997553066299934e-06,
+      "logits/chosen": 0.08324304968118668,
+      "logits/rejected": 0.10611347109079361,
+      "logps/chosen": -343.00885009765625,
+      "logps/rejected": -283.32366943359375,
+      "loss": 0.3241,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.5568825006484985,
+      "rewards/margins": 0.2385629415512085,
+      "rewards/rejected": -0.795445442199707,
       "step": 80
     },
     {
       "epoch": 0.25,
-      "learning_rate": 9.978201358980644e-07,
-      "logits/chosen": -0.2559957504272461,
-      "logits/rejected": -0.20053008198738098,
-      "logps/chosen": -389.33929443359375,
-      "logps/rejected": -247.1188201904297,
-      "loss": 0.4103,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.11513955891132355,
-      "rewards/margins": 0.3396330177783966,
-      "rewards/rejected": -0.22449345886707306,
+      "learning_rate": 4.989100679490323e-06,
+      "logits/chosen": -0.06007497385144234,
+      "logits/rejected": 0.01979098841547966,
+      "logps/chosen": -443.93310546875,
+      "logps/rejected": -310.85113525390625,
+      "loss": 0.3178,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.4307987093925476,
+      "rewards/margins": 0.43101730942726135,
+      "rewards/rejected": -0.8618160486221313,
       "step": 90
     },
     {
       "epoch": 0.28,
-      "learning_rate": 9.949266103908894e-07,
-      "logits/chosen": -0.18818387389183044,
-      "logits/rejected": -0.16776612401008606,
-      "logps/chosen": -360.2618103027344,
-      "logps/rejected": -283.3966979980469,
-      "loss": 0.4011,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.07560204714536667,
-      "rewards/margins": 0.13260915875434875,
-      "rewards/rejected": -0.05700710415840149,
+      "learning_rate": 4.974633051954447e-06,
+      "logits/chosen": 0.1131998747587204,
+      "logits/rejected": 0.13339917361736298,
+      "logps/chosen": -433.76629638671875,
+      "logps/rejected": -361.7824401855469,
+      "loss": 0.2628,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.6594427824020386,
+      "rewards/margins": 0.18142201006412506,
+      "rewards/rejected": -0.8408648371696472,
       "step": 100
     },
     {
       "epoch": 0.31,
-      "learning_rate": 9.908370293252287e-07,
-      "logits/chosen": -0.2396635115146637,
-      "logits/rejected": -0.16480541229248047,
-      "logps/chosen": -319.79925537109375,
-      "logps/rejected": -279.1387023925781,
-      "loss": 0.3716,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.08118756860494614,
-      "rewards/margins": 0.0795254111289978,
-      "rewards/rejected": 0.0016621515387669206,
+      "learning_rate": 4.954185146626145e-06,
+      "logits/chosen": -0.03576885908842087,
+      "logits/rejected": 0.0655001848936081,
+      "logps/chosen": -414.16192626953125,
+      "logps/rejected": -375.30816650390625,
+      "loss": 0.2445,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.8624384999275208,
+      "rewards/margins": 0.09759419411420822,
+      "rewards/rejected": -0.9600327610969543,
       "step": 110
     },
     {
       "epoch": 0.34,
-      "learning_rate": 9.855612757141654e-07,
-      "logits/chosen": -0.2065432071685791,
-      "logits/rejected": -0.17942771315574646,
-      "logps/chosen": -320.2702941894531,
-      "logps/rejected": -222.2958221435547,
-      "loss": 0.3645,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.12114707380533218,
-      "rewards/margins": 0.17006908357143402,
-      "rewards/rejected": -0.04892200976610184,
+      "learning_rate": 4.927806378570827e-06,
+      "logits/chosen": -0.13169847428798676,
+      "logits/rejected": -0.09171108901500702,
+      "logps/chosen": -411.78973388671875,
+      "logps/rejected": -328.10693359375,
+      "loss": 0.2521,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.7940471768379211,
+      "rewards/margins": 0.3129858374595642,
+      "rewards/rejected": -1.1070330142974854,
       "step": 120
     },
     {
       "epoch": 0.37,
-      "learning_rate": 9.791120991134902e-07,
-      "logits/chosen": -0.162322998046875,
-      "logits/rejected": -0.08893263339996338,
-      "logps/chosen": -376.00360107421875,
-      "logps/rejected": -244.6248016357422,
-      "loss": 0.3721,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.10144136846065521,
-      "rewards/margins": 0.25357523560523987,
-      "rewards/rejected": -0.15213388204574585,
+      "learning_rate": 4.895560495567452e-06,
+      "logits/chosen": 0.02038062922656536,
+      "logits/rejected": 0.09609419107437134,
+      "logps/chosen": -509.9686584472656,
+      "logps/rejected": -392.2916564941406,
+      "loss": 0.2267,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.2382087707519531,
+      "rewards/margins": 0.39059334993362427,
+      "rewards/rejected": -1.6288020610809326,
       "step": 130
     },
     {
       "epoch": 0.39,
-      "learning_rate": 9.715050848107168e-07,
-      "logits/chosen": -0.14301994442939758,
-      "logits/rejected": -0.13107290863990784,
-      "logps/chosen": -337.5934753417969,
-      "logps/rejected": -264.13153076171875,
-      "loss": 0.3693,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.008726155385375023,
-      "rewards/margins": 0.13979385793209076,
-      "rewards/rejected": -0.14851999282836914,
+      "learning_rate": 4.857525424053584e-06,
+      "logits/chosen": 0.11588418483734131,
+      "logits/rejected": 0.14068534970283508,
+      "logps/chosen": -441.15704345703125,
+      "logps/rejected": -364.00799560546875,
+      "loss": 0.2176,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.0443618297576904,
+      "rewards/margins": 0.1029224619269371,
+      "rewards/rejected": -1.1472842693328857,
       "step": 140
     },
     {
       "epoch": 0.42,
-      "learning_rate": 9.627586161611731e-07,
-      "logits/chosen": -0.25995808839797974,
-      "logits/rejected": -0.22309823334217072,
-      "logps/chosen": -305.5233459472656,
-      "logps/rejected": -264.06097412109375,
-      "loss": 0.387,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.004295418970286846,
-      "rewards/margins": 0.1561904400587082,
-      "rewards/rejected": -0.16048584878444672,
+      "learning_rate": 4.813793080805866e-06,
+      "logits/chosen": 0.1739293485879898,
+      "logits/rejected": 0.2377185821533203,
+      "logps/chosen": -406.53631591796875,
+      "logps/rejected": -369.4870910644531,
+      "loss": 0.217,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.0144250392913818,
+      "rewards/margins": 0.20032191276550293,
+      "rewards/rejected": -1.2147470712661743,
       "step": 150
     },
     {
       "epoch": 0.45,
-      "learning_rate": 9.528938301621955e-07,
-      "logits/chosen": -0.17277738451957703,
-      "logits/rejected": -0.054037339985370636,
-      "logps/chosen": -388.6319885253906,
-      "logps/rejected": -281.8338317871094,
-      "loss": 0.3624,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.027996420860290527,
-      "rewards/margins": 0.2142985761165619,
-      "rewards/rejected": -0.2422950267791748,
+      "learning_rate": 4.764469150810978e-06,
+      "logits/chosen": 0.1326332688331604,
+      "logits/rejected": 0.22836916148662567,
+      "logps/chosen": -519.8853759765625,
+      "logps/rejected": -416.75482177734375,
+      "loss": 0.1838,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.3405306339263916,
+      "rewards/margins": 0.2509741485118866,
+      "rewards/rejected": -1.5915048122406006,
       "step": 160
     },
     {
       "epoch": 0.48,
-      "learning_rate": 9.419345663727804e-07,
-      "logits/chosen": 0.1668136566877365,
-      "logits/rejected": 0.24267610907554626,
-      "logps/chosen": -348.5519104003906,
-      "logps/rejected": -298.996826171875,
-      "loss": 0.3338,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.16783149540424347,
-      "rewards/margins": 0.18178348243236542,
-      "rewards/rejected": -0.3496149778366089,
+      "learning_rate": 4.709672831863903e-06,
+      "logits/chosen": 0.29795151948928833,
+      "logits/rejected": 0.3732362687587738,
+      "logps/chosen": -471.41204833984375,
+      "logps/rejected": -430.14678955078125,
+      "loss": 0.173,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.3964329957962036,
+      "rewards/margins": 0.2646811604499817,
+      "rewards/rejected": -1.6611143350601196,
       "step": 170
     },
     {
       "epoch": 0.51,
-      "learning_rate": 9.299073093021404e-07,
-      "logits/chosen": 0.047697070986032486,
-      "logits/rejected": 0.1468953937292099,
-      "logps/chosen": -363.13934326171875,
-      "logps/rejected": -292.49822998046875,
-      "loss": 0.3368,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.21875329315662384,
-      "rewards/margins": 0.1854187548160553,
-      "rewards/rejected": -0.40417203307151794,
+      "learning_rate": 4.649536546510702e-06,
+      "logits/chosen": 0.10790077596902847,
+      "logits/rejected": 0.2056044638156891,
+      "logps/chosen": -483.7994689941406,
+      "logps/rejected": -419.7510681152344,
+      "loss": 0.1935,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.4253547191619873,
+      "rewards/margins": 0.25134578347206116,
+      "rewards/rejected": -1.6767005920410156,
       "step": 180
     },
     {
       "epoch": 0.53,
-      "learning_rate": 9.168411244063861e-07,
-      "logits/chosen": 0.016937877982854843,
-      "logits/rejected": 0.10573382675647736,
-      "logps/chosen": -369.8836975097656,
-      "logps/rejected": -270.32574462890625,
-      "loss": 0.3274,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.134426087141037,
-      "rewards/margins": 0.2750268578529358,
-      "rewards/rejected": -0.40945297479629517,
+      "learning_rate": 4.584205622031932e-06,
+      "logits/chosen": 0.11415908485651016,
+      "logits/rejected": 0.17238232493400574,
+      "logps/chosen": -478.956298828125,
+      "logps/rejected": -395.60382080078125,
+      "loss": 0.1794,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.2251521348953247,
+      "rewards/margins": 0.4370814859867096,
+      "rewards/rejected": -1.6622337102890015,
       "step": 190
     },
     {
       "epoch": 0.56,
-      "learning_rate": 9.02767587848013e-07,
-      "logits/chosen": -0.031068842858076096,
-      "logits/rejected": 0.08885425329208374,
-      "logps/chosen": -358.65374755859375,
-      "logps/rejected": -290.309814453125,
-      "loss": 0.3447,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.28612247109413147,
-      "rewards/margins": 0.17394599318504333,
-      "rewards/rejected": -0.4600685238838196,
+      "learning_rate": 4.513837939240066e-06,
+      "logits/chosen": -0.009004750289022923,
+      "logits/rejected": 0.11390223354101181,
+      "logps/chosen": -502.33441162109375,
+      "logps/rejected": -436.2867736816406,
+      "loss": 0.1964,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.7229286432266235,
+      "rewards/margins": 0.19690918922424316,
+      "rewards/rejected": -1.9198377132415771,
       "step": 200
     },
     {
       "epoch": 0.59,
-      "learning_rate": 8.877207101879301e-07,
-      "logits/chosen": 0.1502365916967392,
-      "logits/rejected": 0.24661250412464142,
-      "logps/chosen": -428.3617248535156,
-      "logps/rejected": -313.82257080078125,
-      "loss": 0.3299,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -0.25656384229660034,
-      "rewards/margins": 0.19923393428325653,
-      "rewards/rejected": -0.4557977616786957,
+      "learning_rate": 4.438603550939651e-06,
+      "logits/chosen": 0.11682312190532684,
+      "logits/rejected": 0.21160845458507538,
+      "logps/chosen": -531.32421875,
+      "logps/rejected": -420.9443359375,
+      "loss": 0.2037,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2861896753311157,
+      "rewards/margins": 0.2408260852098465,
+      "rewards/rejected": -1.5270155668258667,
       "step": 210
     },
     {
       "epoch": 0.62,
-      "learning_rate": 8.717368541944452e-07,
-      "logits/chosen": 0.08803655207157135,
-      "logits/rejected": 0.1639718860387802,
-      "logps/chosen": -423.03228759765625,
-      "logps/rejected": -313.6676025390625,
-      "loss": 0.3218,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.3232230246067047,
-      "rewards/margins": 0.1633337289094925,
-      "rewards/rejected": -0.4865567088127136,
+      "learning_rate": 4.358684270972226e-06,
+      "logits/chosen": -0.03151511400938034,
+      "logits/rejected": 0.05730857327580452,
+      "logps/chosen": -499.53924560546875,
+      "logps/rejected": -399.32147216796875,
+      "loss": 0.222,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.0882928371429443,
+      "rewards/margins": 0.2548028826713562,
+      "rewards/rejected": -1.3430957794189453,
       "step": 220
     },
     {
       "epoch": 0.65,
-      "learning_rate": 8.54854646967831e-07,
-      "logits/chosen": -0.0030194534920156,
-      "logits/rejected": 0.14094045758247375,
-      "logps/chosen": -369.0701904296875,
-      "logps/rejected": -286.33990478515625,
-      "loss": 0.3382,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.18699464201927185,
-      "rewards/margins": 0.23964881896972656,
-      "rewards/rejected": -0.4266434609889984,
+      "learning_rate": 4.2742732348391555e-06,
+      "logits/chosen": 0.026254689320921898,
+      "logits/rejected": 0.15401390194892883,
+      "logps/chosen": -465.4871520996094,
+      "logps/rejected": -386.40081787109375,
+      "loss": 0.2377,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.151163935661316,
+      "rewards/margins": 0.27608826756477356,
+      "rewards/rejected": -1.427252173423767,
       "step": 230
     },
     {
       "epoch": 0.67,
-      "learning_rate": 8.371148865928318e-07,
-      "logits/chosen": -0.2055165320634842,
-      "logits/rejected": -0.08643672615289688,
-      "logps/chosen": -377.8648986816406,
-      "logps/rejected": -279.52703857421875,
-      "loss": 0.3443,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.19047299027442932,
-      "rewards/margins": 0.21315133571624756,
-      "rewards/rejected": -0.40362435579299927,
+      "learning_rate": 4.18557443296416e-06,
+      "logits/chosen": -0.057366080582141876,
+      "logits/rejected": 0.04457206651568413,
+      "logps/chosen": -472.771728515625,
+      "logps/rejected": -376.9158020019531,
+      "loss": 0.1975,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.139541745185852,
+      "rewards/margins": 0.23796971142292023,
+      "rewards/rejected": -1.3775115013122559,
       "step": 240
     },
     {
       "epoch": 0.7,
-      "learning_rate": 8.185604435447001e-07,
-      "logits/chosen": -0.21048621833324432,
-      "logits/rejected": -0.12320221960544586,
-      "logps/chosen": -396.4952392578125,
-      "logps/rejected": -284.85693359375,
-      "loss": 0.3476,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.14602257311344147,
-      "rewards/margins": 0.29507723450660706,
-      "rewards/rejected": -0.4410998225212097,
+      "learning_rate": 4.092802217723501e-06,
+      "logits/chosen": -0.03958164528012276,
+      "logits/rejected": 0.07425127923488617,
+      "logps/chosen": -530.5653076171875,
+      "logps/rejected": -422.4510192871094,
+      "loss": 0.1985,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.486722707748413,
+      "rewards/margins": 0.33031830191612244,
+      "rewards/rejected": -1.817041039466858,
       "step": 250
     },
     {
       "epoch": 0.73,
-      "learning_rate": 7.992361570870287e-07,
-      "logits/chosen": -0.09357413649559021,
-      "logits/rejected": -0.02147941291332245,
-      "logps/chosen": -391.3370666503906,
-      "logps/rejected": -304.73248291015625,
-      "loss": 0.3457,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.30016764998435974,
-      "rewards/margins": 0.2906726002693176,
-      "rewards/rejected": -0.590840220451355,
+      "learning_rate": 3.9961807854351446e-06,
+      "logits/chosen": 0.07857771217823029,
+      "logits/rejected": 0.13072842359542847,
+      "logps/chosen": -527.210693359375,
+      "logps/rejected": -451.3309020996094,
+      "loss": 0.2017,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.6589043140411377,
+      "rewards/margins": 0.3979204297065735,
+      "rewards/rejected": -2.0568246841430664,
       "step": 260
     },
     {
       "epoch": 0.76,
-      "learning_rate": 7.791887269117441e-07,
-      "logits/chosen": -0.01912299357354641,
-      "logits/rejected": 0.015499010682106018,
-      "logps/chosen": -385.39788818359375,
-      "logps/rejected": -318.03045654296875,
-      "loss": 0.3332,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.46580857038497925,
-      "rewards/margins": 0.2723786234855652,
-      "rewards/rejected": -0.7381871938705444,
+      "learning_rate": 3.895943634558721e-06,
+      "logits/chosen": 0.26124829053878784,
+      "logits/rejected": 0.3160572052001953,
+      "logps/chosen": -498.911376953125,
+      "logps/rejected": -438.7411193847656,
+      "loss": 0.1763,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.6009438037872314,
+      "rewards/margins": 0.3443499505519867,
+      "rewards/rejected": -1.9452937841415405,
       "step": 270
     },
     {
       "epoch": 0.79,
-      "learning_rate": 7.584666002831294e-07,
-      "logits/chosen": -0.1838226318359375,
-      "logits/rejected": -0.11540188640356064,
-      "logps/chosen": -431.819580078125,
-      "logps/rejected": -314.17315673828125,
-      "loss": 0.3741,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.38983967900276184,
-      "rewards/margins": 0.3283058702945709,
-      "rewards/rejected": -0.718145489692688,
+      "learning_rate": 3.792333001415648e-06,
+      "logits/chosen": 0.2600861191749573,
+      "logits/rejected": 0.38268446922302246,
+      "logps/chosen": -588.9376220703125,
+      "logps/rejected": -477.60101318359375,
+      "loss": 0.1838,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.961019515991211,
+      "rewards/margins": 0.3914053440093994,
+      "rewards/rejected": -2.3524250984191895,
       "step": 280
     },
     {
       "epoch": 0.81,
-      "learning_rate": 7.37119854958609e-07,
-      "logits/chosen": -0.16166582703590393,
-      "logits/rejected": -0.06856708228588104,
-      "logps/chosen": -389.5946960449219,
-      "logps/rejected": -316.12677001953125,
-      "loss": 0.3638,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.3782878816127777,
-      "rewards/margins": 0.20532508194446564,
-      "rewards/rejected": -0.5836129188537598,
+      "learning_rate": 3.6855992747930456e-06,
+      "logits/chosen": 0.34141284227371216,
+      "logits/rejected": 0.470449835062027,
+      "logps/chosen": -519.5052490234375,
+      "logps/rejected": -442.36871337890625,
+      "loss": 0.1849,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.677393913269043,
+      "rewards/margins": 0.16863858699798584,
+      "rewards/rejected": -1.8460325002670288,
       "step": 290
     },
     {
       "epoch": 0.84,
-      "learning_rate": 7.152000781692285e-07,
-      "logits/chosen": -0.15225966274738312,
-      "logits/rejected": 0.023759862408041954,
-      "logps/chosen": -398.35040283203125,
-      "logps/rejected": -296.66046142578125,
-      "loss": 0.3393,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.21167917549610138,
-      "rewards/margins": 0.28109148144721985,
-      "rewards/rejected": -0.4927706718444824,
+      "learning_rate": 3.5760003908461432e-06,
+      "logits/chosen": 0.2685919404029846,
+      "logits/rejected": 0.3938666582107544,
+      "logps/chosen": -494.66009521484375,
+      "logps/rejected": -392.2904052734375,
+      "loss": 0.2002,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.1747759580612183,
+      "rewards/margins": 0.2742941677570343,
+      "rewards/rejected": -1.4490702152252197,
       "step": 300
     },
     {
       "epoch": 0.87,
-      "learning_rate": 6.927602419522946e-07,
-      "logits/chosen": -0.1469275802373886,
-      "logits/rejected": 0.009778399951756,
-      "logps/chosen": -370.21051025390625,
-      "logps/rejected": -298.3064880371094,
-      "loss": 0.3295,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.29429182410240173,
-      "rewards/margins": 0.266606867313385,
-      "rewards/rejected": -0.5608987212181091,
+      "learning_rate": 3.4638012097614735e-06,
+      "logits/chosen": 0.2535479664802551,
+      "logits/rejected": 0.37561532855033875,
+      "logps/chosen": -460.380859375,
+      "logps/rejected": -389.4805603027344,
+      "loss": 0.2132,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.1959959268569946,
+      "rewards/margins": 0.27664345502853394,
+      "rewards/rejected": -1.4726393222808838,
       "step": 310
     },
     {
       "epoch": 0.9,
-      "learning_rate": 6.698545751374463e-07,
-      "logits/chosen": -0.008808034472167492,
-      "logits/rejected": 0.06853709369897842,
-      "logps/chosen": -371.8343200683594,
-      "logps/rejected": -314.5874328613281,
-      "loss": 0.3437,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.3911176323890686,
-      "rewards/margins": 0.1818263977766037,
-      "rewards/rejected": -0.5729440450668335,
+      "learning_rate": 3.3492728756872324e-06,
+      "logits/chosen": 0.29194751381874084,
+      "logits/rejected": 0.34319376945495605,
+      "logps/chosen": -467.84783935546875,
+      "logps/rejected": -414.1121520996094,
+      "loss": 0.2141,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.3512526750564575,
+      "rewards/margins": 0.21693870425224304,
+      "rewards/rejected": -1.5681912899017334,
       "step": 320
     },
     {
       "epoch": 0.93,
-      "learning_rate": 6.465384322955224e-07,
-      "logits/chosen": -0.17208532989025116,
-      "logits/rejected": -0.05994819477200508,
-      "logps/chosen": -450.02850341796875,
-      "logps/rejected": -345.80419921875,
-      "loss": 0.3342,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.3475356698036194,
-      "rewards/margins": 0.32027798891067505,
-      "rewards/rejected": -0.6678135991096497,
+      "learning_rate": 3.2326921614776125e-06,
+      "logits/chosen": 0.12696528434753418,
+      "logits/rejected": 0.2001853734254837,
+      "logps/chosen": -572.5255126953125,
+      "logps/rejected": -473.86932373046875,
+      "loss": 0.1812,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.5725057125091553,
+      "rewards/margins": 0.37595897912979126,
+      "rewards/rejected": -1.9484646320343018,
       "step": 330
     },
     {
       "epoch": 0.96,
-      "learning_rate": 6.228681599669248e-07,
-      "logits/chosen": 0.0015781521797180176,
-      "logits/rejected": 0.043049950152635574,
-      "logps/chosen": -387.4454345703125,
-      "logps/rejected": -302.407958984375,
-      "loss": 0.3188,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.3666529059410095,
-      "rewards/margins": 0.25499850511550903,
-      "rewards/rejected": -0.6216514110565186,
+      "learning_rate": 3.114340799834624e-06,
+      "logits/chosen": 0.24421611428260803,
+      "logits/rejected": 0.309607595205307,
+      "logps/chosen": -510.90948486328125,
+      "logps/rejected": -431.14642333984375,
+      "loss": 0.1856,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.6012935638427734,
+      "rewards/margins": 0.30774253606796265,
+      "rewards/rejected": -1.9090360403060913,
       "step": 340
     },
     {
       "epoch": 0.98,
-      "learning_rate": 5.989009604927586e-07,
-      "logits/chosen": -0.03641257435083389,
-      "logits/rejected": 0.037856753915548325,
-      "logps/chosen": -401.0362854003906,
-      "logps/rejected": -317.9659729003906,
-      "loss": 0.3231,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.3948978781700134,
-      "rewards/margins": 0.2911498546600342,
-      "rewards/rejected": -0.6860476732254028,
+      "learning_rate": 2.9945048024637935e-06,
+      "logits/chosen": 0.3243614733219147,
+      "logits/rejected": 0.42463597655296326,
+      "logps/chosen": -533.7662353515625,
+      "logps/rejected": -459.022705078125,
+      "loss": 0.1946,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.722198247909546,
+      "rewards/margins": 0.3744170069694519,
+      "rewards/rejected": -2.0966150760650635,
       "step": 350
     },
     {
       "epoch": 1.01,
-      "learning_rate": 5.74694753777815e-07,
-      "logits/chosen": -0.12373703718185425,
-      "logits/rejected": -0.014805403538048267,
-      "logps/chosen": -356.97100830078125,
-      "logps/rejected": -344.86102294921875,
-      "loss": 0.3052,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.41758236289024353,
-      "rewards/margins": 0.3036467730998993,
-      "rewards/rejected": -0.721229076385498,
+      "learning_rate": 2.8734737688890752e-06,
+      "logits/chosen": 0.3470906615257263,
+      "logits/rejected": 0.42189717292785645,
+      "logps/chosen": -492.981689453125,
+      "logps/rejected": -519.1780395507812,
+      "loss": 0.1261,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.7776886224746704,
+      "rewards/margins": 0.6867099404335022,
+      "rewards/rejected": -2.4643986225128174,
       "step": 360
     },
     {
       "epoch": 1.04,
-      "learning_rate": 5.503080373194666e-07,
-      "logits/chosen": -0.13851913809776306,
-      "logits/rejected": -0.07055823504924774,
-      "logps/chosen": -396.57769775390625,
-      "logps/rejected": -335.1228942871094,
-      "loss": 0.2391,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.39428186416625977,
-      "rewards/margins": 0.44222506880760193,
-      "rewards/rejected": -0.8365070223808289,
+      "learning_rate": 2.7515401865973333e-06,
+      "logits/chosen": 0.29653865098953247,
+      "logits/rejected": 0.31189900636672974,
+      "logps/chosen": -541.0857543945312,
+      "logps/rejected": -558.9527587890625,
+      "loss": 0.0325,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.839362382888794,
+      "rewards/margins": 1.2354429960250854,
+      "rewards/rejected": -3.074805498123169,
       "step": 370
     },
     {
       "epoch": 1.07,
-      "learning_rate": 5.257997448407366e-07,
-      "logits/chosen": -0.13537368178367615,
-      "logits/rejected": 0.009506871923804283,
-      "logps/chosen": -375.4892578125,
-      "logps/rejected": -355.96136474609375,
-      "loss": 0.232,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.5489501357078552,
-      "rewards/margins": 0.44577375054359436,
-      "rewards/rejected": -0.9947239756584167,
+      "learning_rate": 2.628998724203683e-06,
+      "logits/chosen": 0.24393782019615173,
+      "logits/rejected": 0.3541429042816162,
+      "logps/chosen": -601.4793090820312,
+      "logps/rejected": -695.1011352539062,
+      "loss": 0.0162,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.808850049972534,
+      "rewards/margins": 1.5772719383239746,
+      "rewards/rejected": -4.386122226715088,
       "step": 380
     },
     {
       "epoch": 1.1,
-      "learning_rate": 5.012291038691665e-07,
-      "logits/chosen": -0.292557954788208,
-      "logits/rejected": -0.12162590026855469,
-      "logps/chosen": -359.124755859375,
-      "logps/rejected": -338.5267333984375,
-      "loss": 0.2175,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.7096319794654846,
-      "rewards/margins": 0.47353801131248474,
-      "rewards/rejected": -1.1831700801849365,
+      "learning_rate": 2.5061455193458326e-06,
+      "logits/chosen": 0.2903648018836975,
+      "logits/rejected": 0.4108067452907562,
+      "logps/chosen": -613.734619140625,
+      "logps/rejected": -712.7540893554688,
+      "loss": 0.0109,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -3.2557308673858643,
+      "rewards/margins": 1.669713020324707,
+      "rewards/rejected": -4.92544412612915,
       "step": 390
     },
     {
       "epoch": 1.12,
-      "learning_rate": 4.7665549260567063e-07,
-      "logits/chosen": -0.24239158630371094,
-      "logits/rejected": -0.13934944570064545,
-      "logps/chosen": -426.60369873046875,
-      "logps/rejected": -403.0608215332031,
-      "loss": 0.1717,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.8361096382141113,
-      "rewards/margins": 0.5920412540435791,
-      "rewards/rejected": -1.4281507730484009,
+      "learning_rate": 2.3832774630283534e-06,
+      "logits/chosen": 0.3819536864757538,
+      "logits/rejected": 0.4252726137638092,
+      "logps/chosen": -615.9334716796875,
+      "logps/rejected": -717.3001098632812,
+      "loss": 0.0112,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.7294070720672607,
+      "rewards/margins": 1.8411362171173096,
+      "rewards/rejected": -4.57054328918457,
       "step": 400
     },
     {
       "epoch": 1.15,
-      "learning_rate": 4.521382964292663e-07,
-      "logits/chosen": -0.040043365210294724,
-      "logits/rejected": -0.01921779289841652,
-      "logps/chosen": -417.27215576171875,
-      "logps/rejected": -388.7860412597656,
-      "loss": 0.1755,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.9553691744804382,
-      "rewards/margins": 0.6799647212028503,
-      "rewards/rejected": -1.6353340148925781,
+      "learning_rate": 2.2606914821463317e-06,
+      "logits/chosen": 0.5641288161277771,
+      "logits/rejected": 0.5737628936767578,
+      "logps/chosen": -588.9856567382812,
+      "logps/rejected": -711.607421875,
+      "loss": 0.0108,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.6725032329559326,
+      "rewards/margins": 2.191044330596924,
+      "rewards/rejected": -4.8635478019714355,
       "step": 410
     },
     {
       "epoch": 1.18,
-      "learning_rate": 4.277367643844574e-07,
-      "logits/chosen": -0.097591333091259,
-      "logits/rejected": 0.017194192856550217,
-      "logps/chosen": -437.2616271972656,
-      "logps/rejected": -411.5907287597656,
-      "loss": 0.1823,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.0141613483428955,
-      "rewards/margins": 0.5529431700706482,
-      "rewards/rejected": -1.567104458808899,
+      "learning_rate": 2.138683821922287e-06,
+      "logits/chosen": 0.44424986839294434,
+      "logits/rejected": 0.5702834129333496,
+      "logps/chosen": -649.6632080078125,
+      "logps/rejected": -763.0523681640625,
+      "loss": 0.0084,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -3.1381773948669434,
+      "rewards/margins": 1.943543791770935,
+      "rewards/rejected": -5.081721305847168,
       "step": 420
     },
     {
       "epoch": 1.21,
-      "learning_rate": 4.035098659980891e-07,
-      "logits/chosen": 0.12445954233407974,
-      "logits/rejected": 0.16896803677082062,
-      "logps/chosen": -484.2261657714844,
-      "logps/rejected": -403.1665344238281,
-      "loss": 0.1577,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.9584755897521973,
-      "rewards/margins": 0.7136095762252808,
-      "rewards/rejected": -1.672085165977478,
+      "learning_rate": 2.0175493299904456e-06,
+      "logits/chosen": 0.5278526544570923,
+      "logits/rejected": 0.5699671506881714,
+      "logps/chosen": -760.1556396484375,
+      "logps/rejected": -837.56298828125,
+      "loss": 0.0057,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -3.7177700996398926,
+      "rewards/margins": 2.2982795238494873,
+      "rewards/rejected": -6.016049385070801,
       "step": 430
     },
     {
       "epoch": 1.24,
-      "learning_rate": 3.795161487716928e-07,
-      "logits/chosen": 0.10622860491275787,
-      "logits/rejected": 0.20460453629493713,
-      "logps/chosen": -416.31878662109375,
-      "logps/rejected": -390.43353271484375,
-      "loss": 0.1601,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.0357643365859985,
-      "rewards/margins": 0.6402472257614136,
-      "rewards/rejected": -1.6760118007659912,
+      "learning_rate": 1.8975807438584643e-06,
+      "logits/chosen": 0.4808652400970459,
+      "logits/rejected": 0.5634757280349731,
+      "logps/chosen": -693.7796020507812,
+      "logps/rejected": -791.9578247070312,
+      "loss": 0.0054,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.8103713989257812,
+      "rewards/margins": 1.8808832168579102,
+      "rewards/rejected": -5.69125509262085,
       "step": 440
     },
     {
       "epoch": 1.26,
-      "learning_rate": 3.5581359669371223e-07,
-      "logits/chosen": 0.04869859293103218,
-      "logits/rejected": 0.1988603174686432,
-      "logps/chosen": -459.97161865234375,
-      "logps/rejected": -406.51129150390625,
-      "loss": 0.1485,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.9077569246292114,
-      "rewards/margins": 0.6647528409957886,
-      "rewards/rejected": -1.572509527206421,
+      "learning_rate": 1.7790679834685615e-06,
+      "logits/chosen": 0.48697033524513245,
+      "logits/rejected": 0.6213686466217041,
+      "logps/chosen": -726.90185546875,
+      "logps/rejected": -848.7301635742188,
+      "loss": 0.0053,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -3.577058792114258,
+      "rewards/margins": 2.4176392555236816,
+      "rewards/rejected": -5.9946980476379395,
       "step": 450
     },
     {
       "epoch": 1.29,
-      "learning_rate": 3.324594901135326e-07,
-      "logits/chosen": 0.09600436687469482,
-      "logits/rejected": 0.2769695520401001,
-      "logps/chosen": -412.62481689453125,
-      "logps/rejected": -427.94573974609375,
-      "loss": 0.1594,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.8976230621337891,
-      "rewards/margins": 0.6307999491691589,
-      "rewards/rejected": -1.5284230709075928,
+      "learning_rate": 1.662297450567663e-06,
+      "logits/chosen": 0.6364110708236694,
+      "logits/rejected": 0.7476580739021301,
+      "logps/chosen": -679.8997802734375,
+      "logps/rejected": -860.2080078125,
+      "loss": 0.0055,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -3.570373058319092,
+      "rewards/margins": 2.280672073364258,
+      "rewards/rejected": -5.85104513168335,
       "step": 460
     },
     {
       "epoch": 1.32,
-      "learning_rate": 3.095102673159463e-07,
-      "logits/chosen": -0.04745341092348099,
-      "logits/rejected": 0.04035920277237892,
-      "logps/chosen": -448.31304931640625,
-      "logps/rejected": -434.83026123046875,
-      "loss": 0.162,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -1.009493112564087,
-      "rewards/margins": 0.8154042363166809,
-      "rewards/rejected": -1.8248971700668335,
+      "learning_rate": 1.5475513365797317e-06,
+      "logits/chosen": 0.47412365674972534,
+      "logits/rejected": 0.47869664430618286,
+      "logps/chosen": -706.4371337890625,
+      "logps/rejected": -842.859375,
+      "loss": 0.0054,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -3.590733766555786,
+      "rewards/margins": 2.3144545555114746,
+      "rewards/rejected": -5.90518856048584,
       "step": 470
     },
     {
       "epoch": 1.35,
-      "learning_rate": 2.870213881305802e-07,
-      "logits/chosen": 0.003581273602321744,
-      "logits/rejected": 0.15165765583515167,
-      "logps/chosen": -442.5594177246094,
-      "logps/rejected": -411.17144775390625,
-      "loss": 0.1555,
+      "learning_rate": 1.435106940652901e-06,
+      "logits/chosen": 0.5485504865646362,
+      "logits/rejected": 0.681022584438324,
+      "logps/chosen": -682.8555908203125,
+      "logps/rejected": -810.1474609375,
+      "loss": 0.0055,
       "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.9550750851631165,
-      "rewards/margins": 0.7856963872909546,
-      "rewards/rejected": -1.7407715320587158,
+      "rewards/chosen": -3.3580360412597656,
+      "rewards/margins": 2.372495174407959,
+      "rewards/rejected": -5.730531215667725,
       "step": 480
     },
     {
       "epoch": 1.38,
-      "learning_rate": 2.6504719990588745e-07,
-      "logits/chosen": -0.00950574316084385,
-      "logits/rejected": 0.15859754383563995,
-      "logps/chosen": -484.21954345703125,
-      "logps/rejected": -441.0923767089844,
-      "loss": 0.1608,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.0694897174835205,
-      "rewards/margins": 0.7761608362197876,
-      "rewards/rejected": -1.8456504344940186,
+      "learning_rate": 1.3252359995294376e-06,
+      "logits/chosen": 0.5862436890602112,
+      "logits/rejected": 0.6748989820480347,
+      "logps/chosen": -704.3018188476562,
+      "logps/rejected": -828.2868041992188,
+      "loss": 0.0047,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -3.270312786102295,
+      "rewards/margins": 2.447282075881958,
+      "rewards/rejected": -5.717595100402832,
       "step": 490
     },
     {
       "epoch": 1.4,
-      "learning_rate": 2.436408061715988e-07,
-      "logits/chosen": -0.03855917602777481,
-      "logits/rejected": 0.09556344896554947,
-      "logps/chosen": -491.30523681640625,
-      "logps/rejected": -452.349609375,
-      "loss": 0.1511,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.0780155658721924,
-      "rewards/margins": 0.823270320892334,
-      "rewards/rejected": -1.9012857675552368,
+      "learning_rate": 1.2182040308579942e-06,
+      "logits/chosen": 0.5599916577339172,
+      "logits/rejected": 0.6381617784500122,
+      "logps/chosen": -757.0291748046875,
+      "logps/rejected": -892.974609375,
+      "loss": 0.0041,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -3.7352547645568848,
+      "rewards/margins": 2.5722806453704834,
+      "rewards/rejected": -6.307535171508789,
       "step": 500
     },
     {
       "epoch": 1.43,
-      "learning_rate": 2.22853938307025e-07,
-      "logits/chosen": 0.043701499700546265,
-      "logits/rejected": 0.22214682400226593,
-      "logps/chosen": -456.72662353515625,
-      "logps/rejected": -426.0232849121094,
-      "loss": 0.1501,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -0.9664266705513,
-      "rewards/margins": 0.6386443376541138,
-      "rewards/rejected": -1.6050708293914795,
+      "learning_rate": 1.114269691535125e-06,
+      "logits/chosen": 0.6536546945571899,
+      "logits/rejected": 0.7505389451980591,
+      "logps/chosen": -729.1384887695312,
+      "logps/rejected": -864.0009765625,
+      "loss": 0.004,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.6905455589294434,
+      "rewards/margins": 2.2943015098571777,
+      "rewards/rejected": -5.984846591949463,
       "step": 510
     },
     {
       "epoch": 1.46,
-      "learning_rate": 2.0273683052534173e-07,
-      "logits/chosen": 0.0689583569765091,
-      "logits/rejected": 0.1506797820329666,
-      "logps/chosen": -445.80133056640625,
-      "logps/rejected": -414.3836975097656,
-      "loss": 0.1396,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -1.0923160314559937,
-      "rewards/margins": 0.7477865219116211,
-      "rewards/rejected": -1.8401025533676147,
+      "learning_rate": 1.013684152626709e-06,
+      "logits/chosen": 0.5734939575195312,
+      "logits/rejected": 0.6782180070877075,
+      "logps/chosen": -710.2030029296875,
+      "logps/rejected": -866.6712646484375,
+      "loss": 0.0042,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -3.7363319396972656,
+      "rewards/margins": 2.6266465187072754,
+      "rewards/rejected": -6.362977981567383,
       "step": 520
     },
     {
       "epoch": 1.49,
-      "learning_rate": 1.833380984759764e-07,
-      "logits/chosen": 0.10293974727392197,
-      "logits/rejected": 0.26486995816230774,
-      "logps/chosen": -436.48907470703125,
-      "logps/rejected": -422.7872009277344,
-      "loss": 0.1299,
+      "learning_rate": 9.166904923798822e-07,
+      "logits/chosen": 0.6636993885040283,
+      "logits/rejected": 0.7554783225059509,
+      "logps/chosen": -705.7659912109375,
+      "logps/rejected": -842.1114501953125,
+      "loss": 0.0035,
       "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.0291588306427002,
-      "rewards/margins": 0.720792829990387,
-      "rewards/rejected": -1.749951720237732,
+      "rewards/chosen": -3.721928358078003,
+      "rewards/margins": 2.2212657928466797,
+      "rewards/rejected": -5.943194389343262,
       "step": 530
     },
     {
       "epoch": 1.52,
-      "learning_rate": 1.6470462175846606e-07,
-      "logits/chosen": 0.07386808097362518,
-      "logits/rejected": 0.2429957389831543,
-      "logps/chosen": -517.3760986328125,
-      "logps/rejected": -460.2138671875,
-      "loss": 0.1413,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -1.3018550872802734,
-      "rewards/margins": 0.7603974342346191,
-      "rewards/rejected": -2.0622525215148926,
+      "learning_rate": 8.235231087923304e-07,
+      "logits/chosen": 0.5900656580924988,
+      "logits/rejected": 0.7390304803848267,
+      "logps/chosen": -762.967041015625,
+      "logps/rejected": -907.9431762695312,
+      "loss": 0.0039,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -3.7577643394470215,
+      "rewards/margins": 2.781780242919922,
+      "rewards/rejected": -6.539544582366943,
       "step": 540
     },
     {
       "epoch": 1.55,
-      "learning_rate": 1.468814306317092e-07,
-      "logits/chosen": 0.11437557637691498,
-      "logits/rejected": 0.2697022557258606,
-      "logps/chosen": -451.03326416015625,
-      "logps/rejected": -420.84100341796875,
-      "loss": 0.1376,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.2259981632232666,
-      "rewards/margins": 0.9535650014877319,
-      "rewards/rejected": -2.179563045501709,
+      "learning_rate": 7.344071531585462e-07,
+      "logits/chosen": 0.6735460162162781,
+      "logits/rejected": 0.7701358795166016,
+      "logps/chosen": -721.6229248046875,
+      "logps/rejected": -840.623046875,
+      "loss": 0.0035,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -3.9318947792053223,
+      "rewards/margins": 2.4454891681671143,
+      "rewards/rejected": -6.377383232116699,
       "step": 550
     },
     {
       "epoch": 1.57,
-      "learning_rate": 1.299115971923958e-07,
-      "logits/chosen": 0.07330118119716644,
-      "logits/rejected": 0.24497468769550323,
-      "logps/chosen": -482.2550354003906,
-      "logps/rejected": -460.70367431640625,
-      "loss": 0.1354,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.2959102392196655,
-      "rewards/margins": 0.8086190223693848,
-      "rewards/rejected": -2.10452938079834,
+      "learning_rate": 6.495579859619791e-07,
+      "logits/chosen": 0.6034525632858276,
+      "logits/rejected": 0.7575315237045288,
+      "logps/chosen": -774.0142211914062,
+      "logps/rejected": -933.1998291015625,
+      "loss": 0.0038,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -4.213502407073975,
+      "rewards/margins": 2.61598801612854,
+      "rewards/rejected": -6.829490661621094,
       "step": 560
     },
     {
       "epoch": 1.6,
-      "learning_rate": 1.1383613128559305e-07,
-      "logits/chosen": 0.15279309451580048,
-      "logits/rejected": 0.3158939480781555,
-      "logps/chosen": -485.0721740722656,
-      "logps/rejected": -450.25799560546875,
-      "loss": 0.1225,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.0827850103378296,
-      "rewards/margins": 0.839277446269989,
-      "rewards/rejected": -1.9220625162124634,
+      "learning_rate": 5.691806564279653e-07,
+      "logits/chosen": 0.6900167465209961,
+      "logits/rejected": 0.8468635678291321,
+      "logps/chosen": -746.2769775390625,
+      "logps/rejected": -889.0632934570312,
+      "loss": 0.0034,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -3.694833278656006,
+      "rewards/margins": 2.6152822971343994,
+      "rewards/rejected": -6.310114860534668,
       "step": 570
     },
     {
       "epoch": 1.63,
-      "learning_rate": 9.869388139903495e-08,
-      "logits/chosen": 0.09564773738384247,
-      "logits/rejected": 0.150350421667099,
-      "logps/chosen": -427.63616943359375,
-      "logps/rejected": -459.97064208984375,
-      "loss": 0.1297,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.125893473625183,
-      "rewards/margins": 0.7602401971817017,
-      "rewards/rejected": -1.8861335515975952,
+      "learning_rate": 4.934694069951748e-07,
+      "logits/chosen": 0.6488009691238403,
+      "logits/rejected": 0.6377809643745422,
+      "logps/chosen": -676.0171508789062,
+      "logps/rejected": -875.9207763671875,
+      "loss": 0.0033,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -3.6097030639648438,
+      "rewards/margins": 2.43593168258667,
+      "rewards/rejected": -6.04563570022583,
       "step": 580
     },
     {
       "epoch": 1.66,
-      "learning_rate": 8.452144078061818e-08,
-      "logits/chosen": 0.08417952060699463,
-      "logits/rejected": 0.249791219830513,
-      "logps/chosen": -476.9566345214844,
-      "logps/rejected": -474.86944580078125,
-      "loss": 0.1333,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.174066185951233,
-      "rewards/margins": 0.9380697011947632,
-      "rewards/rejected": -2.112135648727417,
+      "learning_rate": 4.22607203903091e-07,
+      "logits/chosen": 0.6412608623504639,
+      "logits/rejected": 0.7388615608215332,
+      "logps/chosen": -764.0328369140625,
+      "logps/rejected": -968.5056762695312,
+      "loss": 0.0035,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -4.044827461242676,
+      "rewards/margins": 3.0036702156066895,
+      "rewards/rejected": -7.048497676849365,
       "step": 590
     },
     {
       "epoch": 1.69,
-      "learning_rate": 7.135305900598321e-08,
-      "logits/chosen": 0.11366138607263565,
-      "logits/rejected": 0.24674567580223083,
-      "logps/chosen": -484.5184631347656,
-      "logps/rejected": -416.90863037109375,
-      "loss": 0.1273,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.1246851682662964,
-      "rewards/margins": 0.781345546245575,
-      "rewards/rejected": -1.9060306549072266,
+      "learning_rate": 3.5676529502991606e-07,
+      "logits/chosen": 0.6487208604812622,
+      "logits/rejected": 0.7878293395042419,
+      "logps/chosen": -745.95556640625,
+      "logps/rejected": -838.0792236328125,
+      "loss": 0.0039,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -3.73905611038208,
+      "rewards/margins": 2.3786799907684326,
+      "rewards/rejected": -6.117735862731934,
       "step": 600
     },
     {
       "epoch": 1.71,
-      "learning_rate": 5.9220559209888166e-08,
-      "logits/chosen": 0.10064254701137543,
-      "logits/rejected": 0.16818474233150482,
-      "logps/chosen": -491.97662353515625,
-      "logps/rejected": -483.4784240722656,
-      "loss": 0.1357,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.2799218893051147,
-      "rewards/margins": 0.9080696105957031,
-      "rewards/rejected": -2.1879916191101074,
+      "learning_rate": 2.961027960494409e-07,
+      "logits/chosen": 0.6219756603240967,
+      "logits/rejected": 0.6644929647445679,
+      "logps/chosen": -760.4959106445312,
+      "logps/rejected": -936.97265625,
+      "loss": 0.0036,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -3.9651145935058594,
+      "rewards/margins": 2.757819652557373,
+      "rewards/rejected": -6.722934722900391,
       "step": 610
     },
     {
       "epoch": 1.74,
-      "learning_rate": 4.815326118139812e-08,
-      "logits/chosen": 0.08698529005050659,
-      "logits/rejected": 0.18599075078964233,
-      "logps/chosen": -458.9814453125,
-      "logps/rejected": -416.03424072265625,
-      "loss": 0.1329,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.0884288549423218,
-      "rewards/margins": 0.8256447911262512,
-      "rewards/rejected": -1.9140735864639282,
+      "learning_rate": 2.4076630590699065e-07,
+      "logits/chosen": 0.6320921182632446,
+      "logits/rejected": 0.7059046626091003,
+      "logps/chosen": -700.6550903320312,
+      "logps/rejected": -826.9192504882812,
+      "loss": 0.0037,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -3.5051655769348145,
+      "rewards/margins": 2.5177574157714844,
+      "rewards/rejected": -6.022922515869141,
       "step": 620
     },
     {
       "epoch": 1.77,
-      "learning_rate": 3.81779105087407e-08,
-      "logits/chosen": 0.028252029791474342,
-      "logits/rejected": 0.24228385090827942,
-      "logps/chosen": -447.22283935546875,
-      "logps/rejected": -428.1376953125,
-      "loss": 0.1349,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.2487438917160034,
-      "rewards/margins": 0.7152423858642578,
-      "rewards/rejected": -1.9639861583709717,
+      "learning_rate": 1.9088955254370351e-07,
+      "logits/chosen": 0.5824363827705383,
+      "logits/rejected": 0.765023946762085,
+      "logps/chosen": -715.6721801757812,
+      "logps/rejected": -855.0099487304688,
+      "loss": 0.0042,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -3.933236598968506,
+      "rewards/margins": 2.2994723320007324,
+      "rewards/rejected": -6.232708930969238,
       "step": 630
     },
     {
       "epoch": 1.8,
-      "learning_rate": 2.9318613945057637e-08,
-      "logits/chosen": 0.03168308362364769,
-      "logits/rejected": 0.14438685774803162,
-      "logps/chosen": -530.6989135742188,
-      "logps/rejected": -481.19317626953125,
-      "loss": 0.1283,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -1.2953087091445923,
-      "rewards/margins": 0.9182836413383484,
-      "rewards/rejected": -2.213592529296875,
+      "learning_rate": 1.465930697252882e-07,
+      "logits/chosen": 0.5748051404953003,
+      "logits/rejected": 0.668735146522522,
+      "logps/chosen": -811.7765502929688,
+      "logps/rejected": -966.9187622070312,
+      "loss": 0.0032,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -4.106084823608398,
+      "rewards/margins": 2.9647626876831055,
+      "rewards/rejected": -7.070847988128662,
       "step": 640
     },
     {
       "epoch": 1.83,
-      "learning_rate": 2.1596781151249523e-08,
-      "logits/chosen": 0.15784808993339539,
-      "logits/rejected": 0.28222331404685974,
-      "logps/chosen": -461.800048828125,
-      "logps/rejected": -412.47320556640625,
-      "loss": 0.136,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -1.0977789163589478,
-      "rewards/margins": 0.8101053237915039,
-      "rewards/rejected": -1.9078842401504517,
+      "learning_rate": 1.0798390575624762e-07,
+      "logits/chosen": 0.7111840844154358,
+      "logits/rejected": 0.8005073666572571,
+      "logps/chosen": -721.6022338867188,
+      "logps/rejected": -854.2897338867188,
+      "loss": 0.0033,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -3.6958014965057373,
+      "rewards/margins": 2.6302475929260254,
+      "rewards/rejected": -6.326048851013184,
       "step": 650
     },
     {
       "epoch": 1.85,
-      "learning_rate": 1.5031072956701695e-08,
-      "logits/chosen": 0.04679325968027115,
-      "logits/rejected": 0.1709231436252594,
-      "logps/chosen": -501.66656494140625,
-      "logps/rejected": -491.91925048828125,
-      "loss": 0.1335,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -1.1601709127426147,
-      "rewards/margins": 1.024567723274231,
-      "rewards/rejected": -2.1847386360168457,
+      "learning_rate": 7.515536478350849e-08,
+      "logits/chosen": 0.6084551215171814,
+      "logits/rejected": 0.6986690163612366,
+      "logps/chosen": -774.9620361328125,
+      "logps/rejected": -953.1961669921875,
+      "loss": 0.0036,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -3.89312481880188,
+      "rewards/margins": 2.904383420944214,
+      "rewards/rejected": -6.79750919342041,
       "step": 660
     },
     {
       "epoch": 1.88,
-      "learning_rate": 9.637356262923723e-09,
-      "logits/chosen": 0.12124764919281006,
-      "logits/rejected": 0.2632105350494385,
-      "logps/chosen": -449.41064453125,
-      "logps/rejected": -440.3700256347656,
-      "loss": 0.125,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -1.137796401977539,
-      "rewards/margins": 0.832270622253418,
-      "rewards/rejected": -1.970067024230957,
+      "learning_rate": 4.818678131461862e-08,
+      "logits/chosen": 0.7103031277656555,
+      "logits/rejected": 0.8091765642166138,
+      "logps/chosen": -720.9351806640625,
+      "logps/rejected": -879.3292846679688,
+      "loss": 0.0028,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -3.853041887283325,
+      "rewards/margins": 2.506618022918701,
+      "rewards/rejected": -6.359659671783447,
       "step": 670
     },
     {
       "epoch": 1.91,
-      "learning_rate": 5.428665699084789e-09,
-      "logits/chosen": -0.006133219692856073,
-      "logits/rejected": 0.22374041378498077,
-      "logps/chosen": -471.9480895996094,
-      "logps/rejected": -433.7080078125,
-      "loss": 0.1286,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -1.1709232330322266,
-      "rewards/margins": 0.926952064037323,
-      "rewards/rejected": -2.0978751182556152,
+      "learning_rate": 2.7143328495423948e-08,
+      "logits/chosen": 0.5255075693130493,
+      "logits/rejected": 0.7452009320259094,
+      "logps/chosen": -735.2042236328125,
+      "logps/rejected": -863.3488159179688,
+      "loss": 0.0034,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -3.803483247756958,
+      "rewards/margins": 2.5907998085021973,
+      "rewards/rejected": -6.394282817840576,
       "step": 680
     },
     {
       "epoch": 1.94,
-      "learning_rate": 2.415172122110343e-09,
-      "logits/chosen": -0.02816038951277733,
-      "logits/rejected": 0.07123323529958725,
-      "logps/chosen": -522.4472045898438,
-      "logps/rejected": -474.63934326171875,
-      "loss": 0.1306,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.2836827039718628,
-      "rewards/margins": 0.8452218174934387,
-      "rewards/rejected": -2.1289045810699463,
+      "learning_rate": 1.2075860610551715e-08,
+      "logits/chosen": 0.4812963604927063,
+      "logits/rejected": 0.5726746320724487,
+      "logps/chosen": -796.3458251953125,
+      "logps/rejected": -932.7238159179688,
+      "loss": 0.0033,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -4.022668361663818,
+      "rewards/margins": 2.6870803833007812,
+      "rewards/rejected": -6.7097487449646,
       "step": 690
     },
     {
       "epoch": 1.97,
-      "learning_rate": 6.041580374618327e-10,
-      "logits/chosen": 0.06725062429904938,
-      "logits/rejected": 0.20876435935497284,
-      "logps/chosen": -496.45098876953125,
-      "logps/rejected": -499.33856201171875,
-      "loss": 0.1292,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.3299496173858643,
-      "rewards/margins": 0.8479114770889282,
-      "rewards/rejected": -2.177860975265503,
+      "learning_rate": 3.0207901873091638e-09,
+      "logits/chosen": 0.6011410355567932,
+      "logits/rejected": 0.7141892313957214,
+      "logps/chosen": -788.8591918945312,
+      "logps/rejected": -992.2294921875,
+      "loss": 0.0025,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -4.254032611846924,
+      "rewards/margins": 2.852738380432129,
+      "rewards/rejected": -7.1067705154418945,
       "step": 700
     },
     {
       "epoch": 2.0,
       "learning_rate": 0.0,
-      "logits/chosen": 0.11434557288885117,
-      "logits/rejected": 0.21849770843982697,
-      "logps/chosen": -448.5828552246094,
-      "logps/rejected": -464.2832946777344,
-      "loss": 0.1326,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.2470009326934814,
-      "rewards/margins": 0.9275137782096863,
-      "rewards/rejected": -2.1745147705078125,
+      "logits/chosen": 0.6989771127700806,
+      "logits/rejected": 0.749651312828064,
+      "logps/chosen": -713.0648803710938,
+      "logps/rejected": -953.052734375,
+      "loss": 0.0041,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -3.8918204307556152,
+      "rewards/margins": 3.1703882217407227,
+      "rewards/rejected": -7.062208652496338,
       "step": 710
     },
     {
       "epoch": 2.0,
       "step": 710,
       "total_flos": 0.0,
-      "train_loss": 0.25192113166124047,
-      "train_runtime": 8113.4503,
-      "train_samples_per_second": 11.228,
-      "train_steps_per_second": 0.088
+      "train_loss": 0.1250429530667377,
+      "train_runtime": 7954.4917,
+      "train_samples_per_second": 11.452,
+      "train_steps_per_second": 0.089
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c481ba85df8845bff76e10943520a1179a74d1197923c5d013b878850f576711
+oid sha256:9ca9dc28a8826421823e18d437752c8a513c4321a73f387682f187d9117b4213
 size 6008