Model save

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.4350618153878388,
-    "train_runtime": 5287.1692,
+    "train_loss": 0.5807303016441016,
+    "train_runtime": 6860.7908,
     "train_samples": 56236,
-    "train_samples_per_second": 10.636,
-    "train_steps_per_second": 0.083
+    "train_samples_per_second": 8.197,
+    "train_steps_per_second": 0.064
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1aad48cb8aa7b2903f93b8cfd7cb4737c66ec3d00b09189ab8e7eb0e36e7f63d
+oid sha256:82d958e991ee8e8b81e29d04fcc8041bd5590fa21a2b4b16d78ac83600e78932
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df0b2b647126bc8b703c31af35bd8e2148134aa0e7a73730fa9484c71e1715ba
+oid sha256:ded0292a572c378c561e70cb2c28e9ce4fd32ea16fbfbec8c2b36978e671b88f
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2c4087aac41dc68561e56069eabd2d1e6f707422d96ba377ee692b1d30a1ec69
+oid sha256:23081b1de882d06c5f50e034e626fcca1f28e623615e921b1870ad9b5ff57021
 size 4540516344

runs/Jul03_01-53-04_n136-100-194/events.out.tfevents.1719942806.n136-100-194.1930716.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e5f8eb1dd7e34790c767b8227852e30128a8a582ad5c501bc88020d16318e17b
-size 32997
+oid sha256:29dfb50f1ac3f975f3605a500c6d2377cafa56d1723892e0483456ad85387b5a
+size 35415

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.4350618153878388,
-    "train_runtime": 5287.1692,
+    "train_loss": 0.5807303016441016,
+    "train_runtime": 6860.7908,
     "train_samples": 56236,
-    "train_samples_per_second": 10.636,
-    "train_steps_per_second": 0.083
+    "train_samples_per_second": 8.197,
+    "train_steps_per_second": 0.064
 }

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 22.786040256127208,
+      "grad_norm": 34.16262092771155,
       "learning_rate": 2.2727272727272727e-09,
       "logits/chosen": -1.6768856048583984,
       "logits/rejected": -1.7259055376052856,
       "logps/chosen": -1.2793102264404297,
       "logps/rejected": -1.2162058353424072,
-      "loss": 0.6931,
+      "loss": 0.6932,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,657 +25,657 @@
     },
     {
       "epoch": 0.02,
-      "grad_norm": 25.6836964844408,
+      "grad_norm": 38.39373462126025,
       "learning_rate": 2.2727272727272725e-08,
-      "logits/chosen": -1.7025811672210693,
-      "logits/rejected": -1.6681201457977295,
-      "logps/chosen": -1.2131770849227905,
-      "logps/rejected": -1.2202790975570679,
-      "loss": 0.6931,
-      "rewards/accuracies": 0.4166666567325592,
-      "rewards/chosen": 0.00025717043899931014,
-      "rewards/margins": -0.0006778165698051453,
-      "rewards/rejected": 0.0009349870379082859,
+      "logits/chosen": -1.703444004058838,
+      "logits/rejected": -1.6687698364257812,
+      "logps/chosen": -1.2133899927139282,
+      "logps/rejected": -1.2203023433685303,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4236111044883728,
+      "rewards/chosen": -0.00025349482893943787,
+      "rewards/margins": -0.0015861267456784844,
+      "rewards/rejected": 0.0013326319167390466,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 26.405806920127038,
+      "grad_norm": 39.25857800494746,
       "learning_rate": 4.545454545454545e-08,
-      "logits/chosen": -1.7794504165649414,
-      "logits/rejected": -1.734635591506958,
-      "logps/chosen": -1.144683599472046,
-      "logps/rejected": -1.1852140426635742,
-      "loss": 0.6925,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.0029223989695310593,
-      "rewards/margins": 0.0005865652346983552,
-      "rewards/rejected": -0.0035089640878140926,
+      "logits/chosen": -1.7795432806015015,
+      "logits/rejected": -1.734358549118042,
+      "logps/chosen": -1.1447466611862183,
+      "logps/rejected": -1.185644507408142,
+      "loss": 0.692,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.004572628065943718,
+      "rewards/margins": 0.001982099376618862,
+      "rewards/rejected": -0.00655472744256258,
       "step": 20
     },
     {
       "epoch": 0.07,
-      "grad_norm": 28.068883928310647,
+      "grad_norm": 42.49760342477406,
       "learning_rate": 6.818181818181817e-08,
-      "logits/chosen": -1.7446922063827515,
-      "logits/rejected": -1.676011085510254,
-      "logps/chosen": -1.1949479579925537,
-      "logps/rejected": -1.2484194040298462,
-      "loss": 0.6882,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.024093259125947952,
-      "rewards/margins": 0.012483139522373676,
-      "rewards/rejected": -0.036576397716999054,
+      "logits/chosen": -1.7439743280410767,
+      "logits/rejected": -1.6747764348983765,
+      "logps/chosen": -1.195011019706726,
+      "logps/rejected": -1.247870922088623,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.03632856160402298,
+      "rewards/margins": 0.01689038798213005,
+      "rewards/rejected": -0.05321894958615303,
       "step": 30
     },
     {
       "epoch": 0.09,
-      "grad_norm": 19.593533440172358,
+      "grad_norm": 28.94059188031956,
       "learning_rate": 9.09090909090909e-08,
-      "logits/chosen": -1.7312889099121094,
-      "logits/rejected": -1.6649024486541748,
-      "logps/chosen": -1.2515684366226196,
-      "logps/rejected": -1.3352997303009033,
-      "loss": 0.6797,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.08923971652984619,
-      "rewards/margins": 0.06281541287899017,
-      "rewards/rejected": -0.15205512940883636,
+      "logits/chosen": -1.7316467761993408,
+      "logits/rejected": -1.665168046951294,
+      "logps/chosen": -1.2489526271820068,
+      "logps/rejected": -1.330916166305542,
+      "loss": 0.6749,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.12601229548454285,
+      "rewards/margins": 0.08891994506120682,
+      "rewards/rejected": -0.21493223309516907,
       "step": 40
     },
     {
       "epoch": 0.11,
-      "grad_norm": 24.263100694250465,
+      "grad_norm": 34.449211690195526,
       "learning_rate": 9.994307990108962e-08,
-      "logits/chosen": -1.6921930313110352,
-      "logits/rejected": -1.6271032094955444,
-      "logps/chosen": -1.298782467842102,
-      "logps/rejected": -1.3565475940704346,
-      "loss": 0.6637,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.1813907027244568,
-      "rewards/margins": 0.07534531503915787,
-      "rewards/rejected": -0.25673604011535645,
+      "logits/chosen": -1.693706750869751,
+      "logits/rejected": -1.628753662109375,
+      "logps/chosen": -1.2911241054534912,
+      "logps/rejected": -1.3476231098175049,
+      "loss": 0.6547,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.2491108626127243,
+      "rewards/margins": 0.10921992361545563,
+      "rewards/rejected": -0.35833072662353516,
       "step": 50
     },
     {
       "epoch": 0.14,
-      "grad_norm": 28.402662229181594,
+      "grad_norm": 45.18662841636004,
       "learning_rate": 9.959570405988094e-08,
-      "logits/chosen": -1.7207958698272705,
-      "logits/rejected": -1.6402089595794678,
-      "logps/chosen": -1.2883288860321045,
-      "logps/rejected": -1.378244161605835,
-      "loss": 0.6599,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.35846152901649475,
-      "rewards/margins": 0.07597381621599197,
-      "rewards/rejected": -0.4344353675842285,
+      "logits/chosen": -1.7251135110855103,
+      "logits/rejected": -1.644749402999878,
+      "logps/chosen": -1.2596534490585327,
+      "logps/rejected": -1.3467203378677368,
+      "loss": 0.6529,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.4516659379005432,
+      "rewards/margins": 0.10541460663080215,
+      "rewards/rejected": -0.5570805668830872,
       "step": 60
     },
     {
       "epoch": 0.16,
-      "grad_norm": 21.201003636828116,
+      "grad_norm": 31.773982345964285,
       "learning_rate": 9.893476820924666e-08,
-      "logits/chosen": -1.801526665687561,
-      "logits/rejected": -1.7130231857299805,
-      "logps/chosen": -1.4759318828582764,
-      "logps/rejected": -1.5954515933990479,
-      "loss": 0.6498,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.5261799097061157,
-      "rewards/margins": 0.14570936560630798,
-      "rewards/rejected": -0.6718892455101013,
+      "logits/chosen": -1.8094285726547241,
+      "logits/rejected": -1.723013162612915,
+      "logps/chosen": -1.4289662837982178,
+      "logps/rejected": -1.5425760746002197,
+      "loss": 0.6391,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.648373007774353,
+      "rewards/margins": 0.20083490014076233,
+      "rewards/rejected": -0.849207878112793,
       "step": 70
     },
     {
       "epoch": 0.18,
-      "grad_norm": 26.86998789465821,
+      "grad_norm": 34.373704859871594,
       "learning_rate": 9.796445099843647e-08,
-      "logits/chosen": -1.7849477529525757,
-      "logits/rejected": -1.6990400552749634,
-      "logps/chosen": -1.5336757898330688,
-      "logps/rejected": -1.6886417865753174,
-      "loss": 0.654,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.6534146070480347,
-      "rewards/margins": 0.18887227773666382,
-      "rewards/rejected": -0.8422868847846985,
+      "logits/chosen": -1.8001670837402344,
+      "logits/rejected": -1.7168426513671875,
+      "logps/chosen": -1.458465337753296,
+      "logps/rejected": -1.5951015949249268,
+      "loss": 0.6452,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.7544907331466675,
+      "rewards/margins": 0.22831876575946808,
+      "rewards/rejected": -0.9828095436096191,
       "step": 80
     },
     {
       "epoch": 0.2,
-      "grad_norm": 25.145260129960253,
+      "grad_norm": 33.91159834305824,
       "learning_rate": 9.669088708527066e-08,
-      "logits/chosen": -1.7331278324127197,
-      "logits/rejected": -1.6643707752227783,
-      "logps/chosen": -1.604683518409729,
-      "logps/rejected": -1.6745525598526,
-      "loss": 0.6397,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.7379721999168396,
-      "rewards/margins": 0.12564878165721893,
-      "rewards/rejected": -0.8636210560798645,
+      "logits/chosen": -1.7510133981704712,
+      "logits/rejected": -1.6854064464569092,
+      "logps/chosen": -1.5039077997207642,
+      "logps/rejected": -1.5803452730178833,
+      "loss": 0.6218,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.8046310544013977,
+      "rewards/margins": 0.20817911624908447,
+      "rewards/rejected": -1.0128101110458374,
       "step": 90
     },
     {
       "epoch": 0.23,
-      "grad_norm": 28.208412906348087,
+      "grad_norm": 37.79881627068083,
       "learning_rate": 9.512212835085849e-08,
-      "logits/chosen": -1.770500898361206,
-      "logits/rejected": -1.6813795566558838,
-      "logps/chosen": -1.557369351387024,
-      "logps/rejected": -1.7003123760223389,
-      "loss": 0.623,
+      "logits/chosen": -1.7936757802963257,
+      "logits/rejected": -1.7088886499404907,
+      "logps/chosen": -1.4624369144439697,
+      "logps/rejected": -1.5909606218338013,
+      "loss": 0.6102,
       "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.7778279781341553,
-      "rewards/margins": 0.200466588139534,
-      "rewards/rejected": -0.9782945513725281,
+      "rewards/chosen": -0.881945013999939,
+      "rewards/margins": 0.25744158029556274,
+      "rewards/rejected": -1.139386534690857,
       "step": 100
     },
     {
       "epoch": 0.25,
-      "grad_norm": 25.857658698377822,
+      "grad_norm": 38.73339240736025,
       "learning_rate": 9.326809299301306e-08,
-      "logits/chosen": -1.7706959247589111,
-      "logits/rejected": -1.666680932044983,
-      "logps/chosen": -1.6019655466079712,
-      "logps/rejected": -1.7736393213272095,
-      "loss": 0.6212,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.8026469945907593,
-      "rewards/margins": 0.240141361951828,
-      "rewards/rejected": -1.0427882671356201,
+      "logits/chosen": -1.7911834716796875,
+      "logits/rejected": -1.6914081573486328,
+      "logps/chosen": -1.4954617023468018,
+      "logps/rejected": -1.65456223487854,
+      "loss": 0.6056,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.8844586610794067,
+      "rewards/margins": 0.3224923014640808,
+      "rewards/rejected": -1.2069510221481323,
       "step": 110
     },
     {
       "epoch": 0.27,
-      "grad_norm": 27.113925141570313,
+      "grad_norm": 35.446910248333666,
       "learning_rate": 9.114050282021158e-08,
-      "logits/chosen": -1.7661209106445312,
-      "logits/rejected": -1.7044270038604736,
-      "logps/chosen": -1.5668163299560547,
-      "logps/rejected": -1.7313486337661743,
-      "loss": 0.6144,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.8385075330734253,
-      "rewards/margins": 0.2342797964811325,
-      "rewards/rejected": -1.0727872848510742,
+      "logits/chosen": -1.786220908164978,
+      "logits/rejected": -1.7271583080291748,
+      "logps/chosen": -1.4743096828460693,
+      "logps/rejected": -1.6279948949813843,
+      "loss": 0.6015,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.9802410006523132,
+      "rewards/margins": 0.3188786804676056,
+      "rewards/rejected": -1.2991197109222412,
       "step": 120
     },
     {
       "epoch": 0.3,
-      "grad_norm": 25.192318558632277,
+      "grad_norm": 34.33852298554088,
       "learning_rate": 8.875280914254802e-08,
-      "logits/chosen": -1.7651408910751343,
-      "logits/rejected": -1.6746151447296143,
-      "logps/chosen": -1.7618343830108643,
-      "logps/rejected": -1.9648230075836182,
-      "loss": 0.5995,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -1.1380090713500977,
-      "rewards/margins": 0.34912705421447754,
-      "rewards/rejected": -1.4871362447738647,
+      "logits/chosen": -1.7842994928359985,
+      "logits/rejected": -1.6957006454467773,
+      "logps/chosen": -1.6290218830108643,
+      "logps/rejected": -1.7964956760406494,
+      "loss": 0.5858,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.308574914932251,
+      "rewards/margins": 0.41714730858802795,
+      "rewards/rejected": -1.725722312927246,
       "step": 130
     },
     {
       "epoch": 0.32,
-      "grad_norm": 24.555074276571457,
+      "grad_norm": 31.7434623187325,
       "learning_rate": 8.612010772821971e-08,
-      "logits/chosen": -1.7977707386016846,
-      "logits/rejected": -1.7540991306304932,
-      "logps/chosen": -1.8272440433502197,
-      "logps/rejected": -1.9743133783340454,
-      "loss": 0.5993,
+      "logits/chosen": -1.8148624897003174,
+      "logits/rejected": -1.7723850011825562,
+      "logps/chosen": -1.629787802696228,
+      "logps/rejected": -1.7449651956558228,
+      "loss": 0.5865,
       "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.2267296314239502,
-      "rewards/margins": 0.34395888447761536,
-      "rewards/rejected": -1.5706886053085327,
+      "rewards/chosen": -1.2477266788482666,
+      "rewards/margins": 0.4202614724636078,
+      "rewards/rejected": -1.6679881811141968,
       "step": 140
     },
     {
       "epoch": 0.34,
-      "grad_norm": 29.798327176387467,
+      "grad_norm": 30.04410305195209,
       "learning_rate": 8.325904336322055e-08,
-      "logits/chosen": -1.7762537002563477,
-      "logits/rejected": -1.722198247909546,
-      "logps/chosen": -1.9330543279647827,
-      "logps/rejected": -2.1491074562072754,
-      "loss": 0.6088,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -1.5313791036605835,
-      "rewards/margins": 0.3377588093280792,
-      "rewards/rejected": -1.8691380023956299,
+      "logits/chosen": -1.7936302423477173,
+      "logits/rejected": -1.740534782409668,
+      "logps/chosen": -1.6605161428451538,
+      "logps/rejected": -1.848489761352539,
+      "loss": 0.5948,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.4794538021087646,
+      "rewards/margins": 0.4223996102809906,
+      "rewards/rejected": -1.901853322982788,
       "step": 150
     },
     {
       "epoch": 0.36,
-      "grad_norm": 22.379243172505653,
+      "grad_norm": 31.6961444390199,
       "learning_rate": 8.01877046176447e-08,
-      "logits/chosen": -1.7237749099731445,
-      "logits/rejected": -1.6540390253067017,
-      "logps/chosen": -1.9900233745574951,
-      "logps/rejected": -2.1875462532043457,
-      "loss": 0.5903,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.6234468221664429,
-      "rewards/margins": 0.29227370023727417,
-      "rewards/rejected": -1.9157205820083618,
+      "logits/chosen": -1.7392256259918213,
+      "logits/rejected": -1.6678926944732666,
+      "logps/chosen": -1.707990288734436,
+      "logps/rejected": -1.8706386089324951,
+      "loss": 0.5828,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.5890707969665527,
+      "rewards/margins": 0.33378690481185913,
+      "rewards/rejected": -1.922857642173767,
       "step": 160
     },
     {
       "epoch": 0.39,
-      "grad_norm": 19.935363442551864,
+      "grad_norm": 29.03821659608711,
       "learning_rate": 7.692550948392249e-08,
-      "logits/chosen": -1.7721624374389648,
-      "logits/rejected": -1.7196706533432007,
-      "logps/chosen": -1.960934042930603,
-      "logps/rejected": -2.164457321166992,
-      "loss": 0.5921,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.501357078552246,
-      "rewards/margins": 0.38615694642066956,
-      "rewards/rejected": -1.887514352798462,
+      "logits/chosen": -1.7871320247650146,
+      "logits/rejected": -1.7301868200302124,
+      "logps/chosen": -1.6618130207061768,
+      "logps/rejected": -1.81256902217865,
+      "loss": 0.584,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.3546721935272217,
+      "rewards/margins": 0.42093437910079956,
+      "rewards/rejected": -1.7756065130233765,
       "step": 170
     },
     {
       "epoch": 0.41,
-      "grad_norm": 27.99675949527233,
+      "grad_norm": 36.01706358849117,
       "learning_rate": 7.349308261002021e-08,
-      "logits/chosen": -1.7341270446777344,
-      "logits/rejected": -1.6832844018936157,
-      "logps/chosen": -1.9921613931655884,
-      "logps/rejected": -2.2205045223236084,
-      "loss": 0.5854,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.5901505947113037,
-      "rewards/margins": 0.35738545656204224,
-      "rewards/rejected": -1.9475358724594116,
+      "logits/chosen": -1.743697166442871,
+      "logits/rejected": -1.6886732578277588,
+      "logps/chosen": -1.658620834350586,
+      "logps/rejected": -1.8341089487075806,
+      "loss": 0.5784,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.3846042156219482,
+      "rewards/margins": 0.3775123953819275,
+      "rewards/rejected": -1.7621164321899414,
       "step": 180
     },
     {
       "epoch": 0.43,
-      "grad_norm": 24.529175323127856,
+      "grad_norm": 31.031052349259785,
       "learning_rate": 6.991212490377531e-08,
-      "logits/chosen": -1.7890892028808594,
-      "logits/rejected": -1.746469259262085,
-      "logps/chosen": -2.087700843811035,
-      "logps/rejected": -2.3541736602783203,
-      "loss": 0.5608,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.7151954174041748,
-      "rewards/margins": 0.4898623526096344,
-      "rewards/rejected": -2.2050578594207764,
+      "logits/chosen": -1.797126054763794,
+      "logits/rejected": -1.7482073307037354,
+      "logps/chosen": -1.731537103652954,
+      "logps/rejected": -1.9256778955459595,
+      "loss": 0.5544,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.504301905632019,
+      "rewards/margins": 0.517795979976654,
+      "rewards/rejected": -2.0220980644226074,
       "step": 190
     },
     {
       "epoch": 0.46,
-      "grad_norm": 30.651962115004057,
+      "grad_norm": 41.499506266425385,
       "learning_rate": 6.620527633276978e-08,
-      "logits/chosen": -1.7287298440933228,
-      "logits/rejected": -1.6635551452636719,
-      "logps/chosen": -2.149373769760132,
-      "logps/rejected": -2.5112643241882324,
-      "loss": 0.5803,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.909841537475586,
-      "rewards/margins": 0.5502029657363892,
-      "rewards/rejected": -2.4600441455841064,
+      "logits/chosen": -1.7355127334594727,
+      "logits/rejected": -1.6644805669784546,
+      "logps/chosen": -1.7263151407241821,
+      "logps/rejected": -2.0103392601013184,
+      "loss": 0.5689,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.5955860614776611,
+      "rewards/margins": 0.591705858707428,
+      "rewards/rejected": -2.1872916221618652,
       "step": 200
     },
     {
       "epoch": 0.48,
-      "grad_norm": 27.408811645376396,
+      "grad_norm": 34.74514581193027,
       "learning_rate": 6.239597278716581e-08,
-      "logits/chosen": -1.7830358743667603,
-      "logits/rejected": -1.7272449731826782,
-      "logps/chosen": -2.303990125656128,
-      "logps/rejected": -2.549621105194092,
-      "loss": 0.5627,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -2.1173198223114014,
-      "rewards/margins": 0.5325894355773926,
-      "rewards/rejected": -2.649909496307373,
+      "logits/chosen": -1.7984533309936523,
+      "logits/rejected": -1.7380235195159912,
+      "logps/chosen": -1.856172800064087,
+      "logps/rejected": -2.031567096710205,
+      "loss": 0.5577,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.8325283527374268,
+      "rewards/margins": 0.5881733894348145,
+      "rewards/rejected": -2.420701742172241,
       "step": 210
     },
     {
       "epoch": 0.5,
-      "grad_norm": 25.141892376540234,
+      "grad_norm": 30.0311431190145,
       "learning_rate": 5.8508297910462456e-08,
-      "logits/chosen": -1.729365587234497,
-      "logits/rejected": -1.6552003622055054,
-      "logps/chosen": -2.227386713027954,
-      "logps/rejected": -2.558889627456665,
-      "loss": 0.5666,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -2.1277427673339844,
-      "rewards/margins": 0.5474593043327332,
-      "rewards/rejected": -2.675201892852783,
+      "logits/chosen": -1.7461389303207397,
+      "logits/rejected": -1.665144681930542,
+      "logps/chosen": -1.78778874874115,
+      "logps/rejected": -2.0407111644744873,
+      "loss": 0.5611,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.872820258140564,
+      "rewards/margins": 0.5854478478431702,
+      "rewards/rejected": -2.4582679271698,
       "step": 220
     },
     {
       "epoch": 0.52,
-      "grad_norm": 25.203206141016988,
+      "grad_norm": 33.05734006337458,
       "learning_rate": 5.456683083494731e-08,
-      "logits/chosen": -1.7191989421844482,
-      "logits/rejected": -1.6832917928695679,
-      "logps/chosen": -2.193159580230713,
-      "logps/rejected": -2.4023752212524414,
-      "loss": 0.5847,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -1.9987010955810547,
-      "rewards/margins": 0.3709481358528137,
-      "rewards/rejected": -2.3696494102478027,
+      "logits/chosen": -1.7369384765625,
+      "logits/rejected": -1.69870924949646,
+      "logps/chosen": -1.7765703201293945,
+      "logps/rejected": -1.9211543798446655,
+      "loss": 0.5821,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.7482845783233643,
+      "rewards/margins": 0.3625262379646301,
+      "rewards/rejected": -2.1108105182647705,
       "step": 230
     },
     {
       "epoch": 0.55,
-      "grad_norm": 31.30257665339357,
+      "grad_norm": 42.868772014524744,
       "learning_rate": 5.059649078450834e-08,
-      "logits/chosen": -1.7015504837036133,
-      "logits/rejected": -1.662376046180725,
-      "logps/chosen": -2.2371976375579834,
-      "logps/rejected": -2.5139238834381104,
-      "loss": 0.5533,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -2.137688398361206,
-      "rewards/margins": 0.4293746054172516,
-      "rewards/rejected": -2.567063570022583,
+      "logits/chosen": -1.7265160083770752,
+      "logits/rejected": -1.6816940307617188,
+      "logps/chosen": -1.7579532861709595,
+      "logps/rejected": -1.9656188488006592,
+      "loss": 0.5499,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.7687995433807373,
+      "rewards/margins": 0.43688035011291504,
+      "rewards/rejected": -2.2056798934936523,
       "step": 240
     },
     {
       "epoch": 0.57,
-      "grad_norm": 24.832978144124045,
+      "grad_norm": 32.81853514392731,
       "learning_rate": 4.6622379527277186e-08,
-      "logits/chosen": -1.7127622365951538,
-      "logits/rejected": -1.6652097702026367,
-      "logps/chosen": -2.328359603881836,
-      "logps/rejected": -2.558023452758789,
-      "loss": 0.5645,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -2.378941535949707,
-      "rewards/margins": 0.3931688666343689,
-      "rewards/rejected": -2.7721104621887207,
+      "logits/chosen": -1.7367656230926514,
+      "logits/rejected": -1.685956358909607,
+      "logps/chosen": -1.7992569208145142,
+      "logps/rejected": -1.9660634994506836,
+      "loss": 0.5585,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.981104850769043,
+      "rewards/margins": 0.4011806845664978,
+      "rewards/rejected": -2.3822855949401855,
       "step": 250
     },
     {
       "epoch": 0.59,
-      "grad_norm": 28.702758237373775,
+      "grad_norm": 33.27003433225899,
       "learning_rate": 4.26696226741691e-08,
-      "logits/chosen": -1.725927710533142,
-      "logits/rejected": -1.6632360219955444,
-      "logps/chosen": -2.4254117012023926,
-      "logps/rejected": -2.7066590785980225,
-      "loss": 0.5648,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -2.385176181793213,
-      "rewards/margins": 0.522560715675354,
-      "rewards/rejected": -2.9077370166778564,
+      "logits/chosen": -1.7512362003326416,
+      "logits/rejected": -1.6821905374526978,
+      "logps/chosen": -1.897579550743103,
+      "logps/rejected": -2.0977420806884766,
+      "loss": 0.5601,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.9942684173583984,
+      "rewards/margins": 0.5405869483947754,
+      "rewards/rejected": -2.534855365753174,
       "step": 260
     },
     {
       "epoch": 0.61,
-      "grad_norm": 24.57643832571065,
+      "grad_norm": 28.394830302289282,
       "learning_rate": 3.876321082668098e-08,
-      "logits/chosen": -1.7822297811508179,
-      "logits/rejected": -1.7207348346710205,
-      "logps/chosen": -2.3284859657287598,
-      "logps/rejected": -2.6171371936798096,
-      "loss": 0.5581,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -2.179990530014038,
-      "rewards/margins": 0.5586847066879272,
-      "rewards/rejected": -2.7386748790740967,
+      "logits/chosen": -1.8098827600479126,
+      "logits/rejected": -1.7416326999664307,
+      "logps/chosen": -1.8279718160629272,
+      "logps/rejected": -2.0366756916046143,
+      "loss": 0.5516,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.7684434652328491,
+      "rewards/margins": 0.5981845855712891,
+      "rewards/rejected": -2.3666281700134277,
       "step": 270
     },
     {
       "epoch": 0.64,
-      "grad_norm": 26.182300162855046,
+      "grad_norm": 33.9712584613191,
       "learning_rate": 3.492784157826244e-08,
-      "logits/chosen": -1.721687912940979,
-      "logits/rejected": -1.6329882144927979,
-      "logps/chosen": -2.33394193649292,
-      "logps/rejected": -2.6748275756835938,
-      "loss": 0.5564,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -2.2221310138702393,
-      "rewards/margins": 0.610991895198822,
-      "rewards/rejected": -2.833122730255127,
+      "logits/chosen": -1.7408695220947266,
+      "logits/rejected": -1.6451225280761719,
+      "logps/chosen": -1.844485878944397,
+      "logps/rejected": -2.0920097827911377,
+      "loss": 0.5537,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.864828109741211,
+      "rewards/margins": 0.6364026665687561,
+      "rewards/rejected": -2.5012307167053223,
       "step": 280
     },
     {
       "epoch": 0.66,
-      "grad_norm": 22.815010055443274,
+      "grad_norm": 26.97002465180278,
       "learning_rate": 3.118776336817812e-08,
-      "logits/chosen": -1.755319595336914,
-      "logits/rejected": -1.7038185596466064,
-      "logps/chosen": -2.277468204498291,
-      "logps/rejected": -2.616783618927002,
-      "loss": 0.5473,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -2.1484389305114746,
-      "rewards/margins": 0.6641989946365356,
-      "rewards/rejected": -2.8126378059387207,
+      "logits/chosen": -1.782135009765625,
+      "logits/rejected": -1.72174870967865,
+      "logps/chosen": -1.8031295537948608,
+      "logps/rejected": -2.0307796001434326,
+      "loss": 0.5443,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.7996422052383423,
+      "rewards/margins": 0.6613016128540039,
+      "rewards/rejected": -2.4609439373016357,
       "step": 290
     },
     {
       "epoch": 0.68,
-      "grad_norm": 23.65537068910306,
+      "grad_norm": 31.17322900966105,
       "learning_rate": 2.7566622175067443e-08,
-      "logits/chosen": -1.7356256246566772,
-      "logits/rejected": -1.6781995296478271,
-      "logps/chosen": -2.3750412464141846,
-      "logps/rejected": -2.7629873752593994,
-      "loss": 0.5497,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -2.3329153060913086,
-      "rewards/margins": 0.6199190616607666,
-      "rewards/rejected": -2.9528346061706543,
+      "logits/chosen": -1.7659027576446533,
+      "logits/rejected": -1.7027981281280518,
+      "logps/chosen": -1.8818680047988892,
+      "logps/rejected": -2.163147449493408,
+      "loss": 0.5502,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.0198538303375244,
+      "rewards/margins": 0.6098783612251282,
+      "rewards/rejected": -2.629732131958008,
       "step": 300
     },
     {
       "epoch": 0.71,
-      "grad_norm": 27.623729083981935,
+      "grad_norm": 36.262422147908055,
       "learning_rate": 2.408731201945432e-08,
-      "logits/chosen": -1.7345046997070312,
-      "logits/rejected": -1.6898084878921509,
-      "logps/chosen": -2.304429531097412,
-      "logps/rejected": -2.555560827255249,
-      "loss": 0.5555,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -2.2210681438446045,
-      "rewards/margins": 0.46195346117019653,
-      "rewards/rejected": -2.6830217838287354,
+      "logits/chosen": -1.761371374130249,
+      "logits/rejected": -1.7142196893692017,
+      "logps/chosen": -1.8247020244598389,
+      "logps/rejected": -2.0095391273498535,
+      "loss": 0.5485,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.8924195766448975,
+      "rewards/margins": 0.49404746294021606,
+      "rewards/rejected": -2.386467218399048,
       "step": 310
     },
     {
       "epoch": 0.73,
-      "grad_norm": 26.649151417142534,
+      "grad_norm": 32.89905332615021,
       "learning_rate": 2.0771830220378112e-08,
-      "logits/chosen": -1.68793523311615,
-      "logits/rejected": -1.6301701068878174,
-      "logps/chosen": -2.3206005096435547,
-      "logps/rejected": -2.559023380279541,
-      "loss": 0.5553,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -2.195432424545288,
-      "rewards/margins": 0.4925463795661926,
-      "rewards/rejected": -2.687978744506836,
+      "logits/chosen": -1.7170169353485107,
+      "logits/rejected": -1.6577152013778687,
+      "logps/chosen": -1.8545100688934326,
+      "logps/rejected": -2.0157198905944824,
+      "loss": 0.5515,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.8948768377304077,
+      "rewards/margins": 0.5071803331375122,
+      "rewards/rejected": -2.40205717086792,
       "step": 320
     },
     {
       "epoch": 0.75,
-      "grad_norm": 24.875498436093228,
+      "grad_norm": 29.752828533537873,
       "learning_rate": 1.7641138321260257e-08,
-      "logits/chosen": -1.722652792930603,
-      "logits/rejected": -1.6583467721939087,
-      "logps/chosen": -2.2422232627868652,
-      "logps/rejected": -2.6411871910095215,
-      "loss": 0.5479,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -2.116737127304077,
-      "rewards/margins": 0.7428576946258545,
-      "rewards/rejected": -2.8595950603485107,
+      "logits/chosen": -1.7509727478027344,
+      "logits/rejected": -1.6813408136367798,
+      "logps/chosen": -1.8006902933120728,
+      "logps/rejected": -2.068765878677368,
+      "loss": 0.5463,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.8505061864852905,
+      "rewards/margins": 0.7216218113899231,
+      "rewards/rejected": -2.5721280574798584,
       "step": 330
     },
     {
       "epoch": 0.77,
-      "grad_norm": 26.307918213457313,
+      "grad_norm": 29.865199582942665,
       "learning_rate": 1.4715029564277793e-08,
-      "logits/chosen": -1.7855126857757568,
-      "logits/rejected": -1.7428696155548096,
-      "logps/chosen": -2.2222747802734375,
-      "logps/rejected": -2.578981399536133,
-      "loss": 0.5581,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -2.0352282524108887,
-      "rewards/margins": 0.6392861604690552,
-      "rewards/rejected": -2.6745142936706543,
+      "logits/chosen": -1.8179073333740234,
+      "logits/rejected": -1.771498680114746,
+      "logps/chosen": -1.7910722494125366,
+      "logps/rejected": -2.0462353229522705,
+      "loss": 0.5539,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.759235143661499,
+      "rewards/margins": 0.6542983651161194,
+      "rewards/rejected": -2.4135334491729736,
       "step": 340
     },
     {
       "epoch": 0.8,
-      "grad_norm": 25.341211676404754,
+      "grad_norm": 33.27484680109776,
       "learning_rate": 1.2012003751113343e-08,
-      "logits/chosen": -1.7763115167617798,
-      "logits/rejected": -1.7206395864486694,
-      "logps/chosen": -2.328850269317627,
-      "logps/rejected": -2.70664119720459,
-      "loss": 0.5366,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -2.3333606719970703,
-      "rewards/margins": 0.6552005410194397,
-      "rewards/rejected": -2.9885611534118652,
+      "logits/chosen": -1.806565284729004,
+      "logits/rejected": -1.745749831199646,
+      "logps/chosen": -1.8502833843231201,
+      "logps/rejected": -2.1222567558288574,
+      "loss": 0.5334,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.0643410682678223,
+      "rewards/margins": 0.6653475761413574,
+      "rewards/rejected": -2.729688882827759,
       "step": 350
     },
     {
       "epoch": 0.82,
-      "grad_norm": 31.147010925293188,
+      "grad_norm": 35.608600933807864,
       "learning_rate": 9.549150281252633e-09,
-      "logits/chosen": -1.725211501121521,
-      "logits/rejected": -1.6820595264434814,
-      "logps/chosen": -2.2657017707824707,
-      "logps/rejected": -2.6082816123962402,
-      "loss": 0.5395,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -2.1871337890625,
-      "rewards/margins": 0.5816982388496399,
-      "rewards/rejected": -2.768831729888916,
+      "logits/chosen": -1.762521505355835,
+      "logits/rejected": -1.7128925323486328,
+      "logps/chosen": -1.8086223602294922,
+      "logps/rejected": -2.0623652935028076,
+      "loss": 0.5356,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.9094616174697876,
+      "rewards/margins": 0.6060369610786438,
+      "rewards/rejected": -2.515498399734497,
       "step": 360
     },
     {
       "epoch": 0.84,
-      "grad_norm": 24.868596217446985,
+      "grad_norm": 31.739203329110374,
       "learning_rate": 7.3420401072985306e-09,
-      "logits/chosen": -1.7670882940292358,
-      "logits/rejected": -1.7162415981292725,
-      "logps/chosen": -2.3288838863372803,
-      "logps/rejected": -2.694032669067383,
-      "loss": 0.5377,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -2.2285408973693848,
-      "rewards/margins": 0.6153696179389954,
-      "rewards/rejected": -2.8439104557037354,
+      "logits/chosen": -1.7991466522216797,
+      "logits/rejected": -1.746799111366272,
+      "logps/chosen": -1.8659775257110596,
+      "logps/rejected": -2.1334667205810547,
+      "loss": 0.5356,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.9540916681289673,
+      "rewards/margins": 0.6300768256187439,
+      "rewards/rejected": -2.5841686725616455,
       "step": 370
     },
     {
       "epoch": 0.86,
-      "grad_norm": 28.264277931849893,
+      "grad_norm": 37.231267463423904,
       "learning_rate": 5.404627290395369e-09,
-      "logits/chosen": -1.7286021709442139,
-      "logits/rejected": -1.6685943603515625,
-      "logps/chosen": -2.2551302909851074,
-      "logps/rejected": -2.6050727367401123,
-      "loss": 0.537,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -2.1310482025146484,
-      "rewards/margins": 0.619892954826355,
-      "rewards/rejected": -2.750941514968872,
+      "logits/chosen": -1.7623279094696045,
+      "logits/rejected": -1.6976515054702759,
+      "logps/chosen": -1.7954782247543335,
+      "logps/rejected": -2.0490074157714844,
+      "loss": 0.5377,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.8176171779632568,
+      "rewards/margins": 0.6405991315841675,
+      "rewards/rejected": -2.4582161903381348,
       "step": 380
     },
     {
       "epoch": 0.89,
-      "grad_norm": 32.571402526944446,
+      "grad_norm": 48.31129307964873,
       "learning_rate": 3.74916077816162e-09,
-      "logits/chosen": -1.734569787979126,
-      "logits/rejected": -1.678733468055725,
-      "logps/chosen": -2.2776572704315186,
-      "logps/rejected": -2.573931932449341,
-      "loss": 0.5505,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -2.255725145339966,
-      "rewards/margins": 0.5602914690971375,
-      "rewards/rejected": -2.816016674041748,
+      "logits/chosen": -1.7699306011199951,
+      "logits/rejected": -1.7098052501678467,
+      "logps/chosen": -1.7985498905181885,
+      "logps/rejected": -1.9994351863861084,
+      "loss": 0.5502,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.946265459060669,
+      "rewards/margins": 0.5542693734169006,
+      "rewards/rejected": -2.5005345344543457,
       "step": 390
     },
     {
       "epoch": 0.91,
-      "grad_norm": 30.76786815731744,
+      "grad_norm": 36.09144092538641,
       "learning_rate": 2.386106962899165e-09,
-      "logits/chosen": -1.6618547439575195,
-      "logits/rejected": -1.599841594696045,
-      "logps/chosen": -2.417301654815674,
-      "logps/rejected": -2.7463181018829346,
-      "loss": 0.5418,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -2.381761312484741,
-      "rewards/margins": 0.5453529357910156,
-      "rewards/rejected": -2.927114725112915,
+      "logits/chosen": -1.6945453882217407,
+      "logits/rejected": -1.62551748752594,
+      "logps/chosen": -1.9125568866729736,
+      "logps/rejected": -2.1548168659210205,
+      "loss": 0.5447,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.05840802192688,
+      "rewards/margins": 0.5577607154846191,
+      "rewards/rejected": -2.61616849899292,
       "step": 400
     },
     {
       "epoch": 0.93,
-      "grad_norm": 31.25431184934112,
+      "grad_norm": 35.37496893135356,
       "learning_rate": 1.3240835096913706e-09,
-      "logits/chosen": -1.6889197826385498,
-      "logits/rejected": -1.5975219011306763,
-      "logps/chosen": -2.266620397567749,
-      "logps/rejected": -2.7067856788635254,
-      "loss": 0.5607,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -2.190139055252075,
-      "rewards/margins": 0.7415019273757935,
-      "rewards/rejected": -2.931641101837158,
+      "logits/chosen": -1.7273473739624023,
+      "logits/rejected": -1.6288312673568726,
+      "logps/chosen": -1.787335753440857,
+      "logps/rejected": -2.1021180152893066,
+      "loss": 0.5546,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.8473552465438843,
+      "rewards/margins": 0.7361037731170654,
+      "rewards/rejected": -2.5834591388702393,
       "step": 410
     },
     {
       "epoch": 0.96,
-      "grad_norm": 23.39234165392022,
+      "grad_norm": 32.802494867000924,
       "learning_rate": 5.698048727497462e-10,
-      "logits/chosen": -1.7234703302383423,
-      "logits/rejected": -1.65776789188385,
-      "logps/chosen": -2.363593578338623,
-      "logps/rejected": -2.7682502269744873,
-      "loss": 0.5431,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -2.3445918560028076,
-      "rewards/margins": 0.6753993034362793,
-      "rewards/rejected": -3.019990921020508,
+      "logits/chosen": -1.7570117712020874,
+      "logits/rejected": -1.6864467859268188,
+      "logps/chosen": -1.8803859949111938,
+      "logps/rejected": -2.1749351024627686,
+      "loss": 0.5408,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -2.0672643184661865,
+      "rewards/margins": 0.6827760338783264,
+      "rewards/rejected": -2.7500405311584473,
       "step": 420
     },
     {
       "epoch": 0.98,
-      "grad_norm": 24.387678586587597,
+      "grad_norm": 30.44073541133352,
       "learning_rate": 1.2803984447259387e-10,
-      "logits/chosen": -1.7296276092529297,
-      "logits/rejected": -1.68634033203125,
-      "logps/chosen": -2.347764730453491,
-      "logps/rejected": -2.731468677520752,
-      "loss": 0.5333,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -2.330850601196289,
-      "rewards/margins": 0.6721019744873047,
-      "rewards/rejected": -3.002953052520752,
+      "logits/chosen": -1.7648996114730835,
+      "logits/rejected": -1.713091254234314,
+      "logps/chosen": -1.8600318431854248,
+      "logps/rejected": -2.127103805541992,
+      "loss": 0.5345,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.0330777168273926,
+      "rewards/margins": 0.6582567095756531,
+      "rewards/rejected": -2.6913342475891113,
       "step": 430
     },
     {
       "epoch": 1.0,
       "step": 439,
       "total_flos": 0.0,
-      "train_loss": 0.4350618153878388,
-      "train_runtime": 5287.1692,
-      "train_samples_per_second": 10.636,
-      "train_steps_per_second": 0.083
+      "train_loss": 0.5807303016441016,
+      "train_runtime": 6860.7908,
+      "train_samples_per_second": 8.197,
+      "train_steps_per_second": 0.064
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a7e7e48ed484507dc4bf687f99b9afd74522a0ce37982f50ef4c64c14f4f420c
+oid sha256:52fd958028d2a8e77c3db64f0cef52c04c727dbfa54965417b4be2b66b01aefd
 size 6328