Model save

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5807303016441016,
-    "train_runtime": 6860.7908,
+    "train_loss": 0.5484831045320202,
+    "train_runtime": 6843.1048,
     "train_samples": 56236,
-    "train_samples_per_second": 8.197,
+    "train_samples_per_second": 8.218,
     "train_steps_per_second": 0.064
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:82d958e991ee8e8b81e29d04fcc8041bd5590fa21a2b4b16d78ac83600e78932
+oid sha256:a94f57df315bdddf8185ae3fb818101aee3b42c76690c4d7fb8251fe4adf4bb0
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ded0292a572c378c561e70cb2c28e9ce4fd32ea16fbfbec8c2b36978e671b88f
+oid sha256:24cdff7b85c4211e27a3c617cf68911989d809a7209cbb8dfa47f4c19addf19d
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:23081b1de882d06c5f50e034e626fcca1f28e623615e921b1870ad9b5ff57021
+oid sha256:2670039c24a55fcd15a7121e0de7f3764137b155c19366c74c849c7b54ec5a47
 size 4540516344

runs/Jul03_01-53-39_n136-129-074/events.out.tfevents.1719942853.n136-129-074.2600724.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:96cffad0b73203df2dbc5cb8d809df9c5e6f2fd7768645cecd74e61368c63922
-size 33001
+oid sha256:96a5741efa0c60fa2942bc0ded9517e0573db2e9ef95f00017284a3916df3204
+size 35419

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5807303016441016,
-    "train_runtime": 6860.7908,
+    "train_loss": 0.5484831045320202,
+    "train_runtime": 6843.1048,
     "train_samples": 56236,
-    "train_samples_per_second": 8.197,
+    "train_samples_per_second": 8.218,
     "train_steps_per_second": 0.064
 }

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 34.16262092771155,
+      "grad_norm": 170.90539309098384,
       "learning_rate": 2.2727272727272727e-09,
       "logits/chosen": -1.6768856048583984,
       "logits/rejected": -1.7259055376052856,
       "logps/chosen": -1.2793102264404297,
       "logps/rejected": -1.2162058353424072,
-      "loss": 0.6932,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,656 +25,656 @@
     },
     {
       "epoch": 0.02,
-      "grad_norm": 38.39373462126025,
+      "grad_norm": 192.8004690883836,
       "learning_rate": 2.2727272727272725e-08,
-      "logits/chosen": -1.703444004058838,
-      "logits/rejected": -1.6687698364257812,
-      "logps/chosen": -1.2133899927139282,
-      "logps/rejected": -1.2203023433685303,
-      "loss": 0.6932,
-      "rewards/accuracies": 0.4236111044883728,
-      "rewards/chosen": -0.00025349482893943787,
-      "rewards/margins": -0.0015861267456784844,
-      "rewards/rejected": 0.0013326319167390466,
+      "logits/chosen": -1.7027912139892578,
+      "logits/rejected": -1.6683564186096191,
+      "logps/chosen": -1.2133431434631348,
+      "logps/rejected": -1.2206003665924072,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4861111044883728,
+      "rewards/chosen": -0.0005598987336270511,
+      "rewards/margins": -0.0027534286491572857,
+      "rewards/rejected": 0.0021935300901532173,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 39.25857800494746,
+      "grad_norm": 189.74036986732978,
       "learning_rate": 4.545454545454545e-08,
-      "logits/chosen": -1.7795432806015015,
-      "logits/rejected": -1.734358549118042,
-      "logps/chosen": -1.1447466611862183,
-      "logps/rejected": -1.185644507408142,
-      "loss": 0.692,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": -0.004572628065943718,
-      "rewards/margins": 0.001982099376618862,
-      "rewards/rejected": -0.00655472744256258,
+      "logits/chosen": -1.7791492938995361,
+      "logits/rejected": -1.7344741821289062,
+      "logps/chosen": -1.1445863246917725,
+      "logps/rejected": -1.1852391958236694,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.02045648917555809,
+      "rewards/margins": 0.006238006055355072,
+      "rewards/rejected": -0.026694495230913162,
       "step": 20
     },
     {
       "epoch": 0.07,
-      "grad_norm": 42.49760342477406,
+      "grad_norm": 201.2210648540721,
       "learning_rate": 6.818181818181817e-08,
-      "logits/chosen": -1.7439743280410767,
-      "logits/rejected": -1.6747764348983765,
-      "logps/chosen": -1.195011019706726,
-      "logps/rejected": -1.247870922088623,
-      "loss": 0.6863,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.03632856160402298,
-      "rewards/margins": 0.01689038798213005,
-      "rewards/rejected": -0.05321894958615303,
+      "logits/chosen": -1.7444446086883545,
+      "logits/rejected": -1.6758038997650146,
+      "logps/chosen": -1.1940228939056396,
+      "logps/rejected": -1.2459399700164795,
+      "loss": 0.6716,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.16682226955890656,
+      "rewards/margins": 0.07031063735485077,
+      "rewards/rejected": -0.23713290691375732,
       "step": 30
     },
     {
       "epoch": 0.09,
-      "grad_norm": 28.94059188031956,
+      "grad_norm": 157.5785989108621,
       "learning_rate": 9.09090909090909e-08,
-      "logits/chosen": -1.7316467761993408,
-      "logits/rejected": -1.665168046951294,
-      "logps/chosen": -1.2489526271820068,
-      "logps/rejected": -1.330916166305542,
-      "loss": 0.6749,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.12601229548454285,
-      "rewards/margins": 0.08891994506120682,
-      "rewards/rejected": -0.21493223309516907,
+      "logits/chosen": -1.7311503887176514,
+      "logits/rejected": -1.6649019718170166,
+      "logps/chosen": -1.2304141521453857,
+      "logps/rejected": -1.2984638214111328,
+      "loss": 0.6586,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.3519838750362396,
+      "rewards/margins": 0.23589149117469788,
+      "rewards/rejected": -0.5878753662109375,
       "step": 40
     },
     {
       "epoch": 0.11,
-      "grad_norm": 34.449211690195526,
+      "grad_norm": 130.7345765168361,
       "learning_rate": 9.994307990108962e-08,
-      "logits/chosen": -1.693706750869751,
-      "logits/rejected": -1.628753662109375,
-      "logps/chosen": -1.2911241054534912,
-      "logps/rejected": -1.3476231098175049,
-      "loss": 0.6547,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.2491108626127243,
-      "rewards/margins": 0.10921992361545563,
-      "rewards/rejected": -0.35833072662353516,
+      "logits/chosen": -1.6965749263763428,
+      "logits/rejected": -1.6330372095108032,
+      "logps/chosen": -1.2411986589431763,
+      "logps/rejected": -1.2848317623138428,
+      "loss": 0.6183,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.4966731071472168,
+      "rewards/margins": 0.3531120717525482,
+      "rewards/rejected": -0.8497850298881531,
       "step": 50
     },
     {
       "epoch": 0.14,
-      "grad_norm": 45.18662841636004,
+      "grad_norm": 196.7850573404173,
       "learning_rate": 9.959570405988094e-08,
-      "logits/chosen": -1.7251135110855103,
-      "logits/rejected": -1.644749402999878,
-      "logps/chosen": -1.2596534490585327,
-      "logps/rejected": -1.3467203378677368,
-      "loss": 0.6529,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.4516659379005432,
-      "rewards/margins": 0.10541460663080215,
-      "rewards/rejected": -0.5570805668830872,
+      "logits/chosen": -1.7333612442016602,
+      "logits/rejected": -1.6539825201034546,
+      "logps/chosen": -1.1546742916107178,
+      "logps/rejected": -1.2285583019256592,
+      "loss": 0.6171,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.6836405396461487,
+      "rewards/margins": 0.3293355703353882,
+      "rewards/rejected": -1.012976050376892,
       "step": 60
     },
     {
       "epoch": 0.16,
-      "grad_norm": 31.773982345964285,
+      "grad_norm": 118.03026103477566,
       "learning_rate": 9.893476820924666e-08,
-      "logits/chosen": -1.8094285726547241,
-      "logits/rejected": -1.723013162612915,
-      "logps/chosen": -1.4289662837982178,
-      "logps/rejected": -1.5425760746002197,
-      "loss": 0.6391,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.648373007774353,
-      "rewards/margins": 0.20083490014076233,
-      "rewards/rejected": -0.849207878112793,
+      "logits/chosen": -1.8179279565811157,
+      "logits/rejected": -1.735210657119751,
+      "logps/chosen": -1.2660285234451294,
+      "logps/rejected": -1.3405873775482178,
+      "loss": 0.5806,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.7977959513664246,
+      "rewards/margins": 0.41841205954551697,
+      "rewards/rejected": -1.2162081003189087,
       "step": 70
     },
     {
       "epoch": 0.18,
-      "grad_norm": 34.373704859871594,
+      "grad_norm": 163.94509974087566,
       "learning_rate": 9.796445099843647e-08,
-      "logits/chosen": -1.8001670837402344,
-      "logits/rejected": -1.7168426513671875,
-      "logps/chosen": -1.458465337753296,
-      "logps/rejected": -1.5951015949249268,
-      "loss": 0.6452,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.7544907331466675,
-      "rewards/margins": 0.22831876575946808,
-      "rewards/rejected": -0.9828095436096191,
+      "logits/chosen": -1.8049923181533813,
+      "logits/rejected": -1.7208877801895142,
+      "logps/chosen": -1.266593098640442,
+      "logps/rejected": -1.3546384572982788,
+      "loss": 0.6281,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.8943712115287781,
+      "rewards/margins": 0.4127298891544342,
+      "rewards/rejected": -1.3071010112762451,
       "step": 80
     },
     {
       "epoch": 0.2,
-      "grad_norm": 33.91159834305824,
+      "grad_norm": 141.7645640906373,
       "learning_rate": 9.669088708527066e-08,
-      "logits/chosen": -1.7510133981704712,
-      "logits/rejected": -1.6854064464569092,
-      "logps/chosen": -1.5039077997207642,
-      "logps/rejected": -1.5803452730178833,
-      "loss": 0.6218,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.8046310544013977,
-      "rewards/margins": 0.20817911624908447,
-      "rewards/rejected": -1.0128101110458374,
+      "logits/chosen": -1.751291275024414,
+      "logits/rejected": -1.6858265399932861,
+      "logps/chosen": -1.2966631650924683,
+      "logps/rejected": -1.3356841802597046,
+      "loss": 0.5844,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.9144881963729858,
+      "rewards/margins": 0.47964733839035034,
+      "rewards/rejected": -1.3941354751586914,
       "step": 90
     },
     {
       "epoch": 0.23,
-      "grad_norm": 37.79881627068083,
+      "grad_norm": 162.44708527833532,
       "learning_rate": 9.512212835085849e-08,
-      "logits/chosen": -1.7936757802963257,
-      "logits/rejected": -1.7088886499404907,
-      "logps/chosen": -1.4624369144439697,
-      "logps/rejected": -1.5909606218338013,
-      "loss": 0.6102,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.881945013999939,
-      "rewards/margins": 0.25744158029556274,
-      "rewards/rejected": -1.139386534690857,
+      "logits/chosen": -1.7844059467315674,
+      "logits/rejected": -1.6998250484466553,
+      "logps/chosen": -1.2385976314544678,
+      "logps/rejected": -1.3161624670028687,
+      "loss": 0.5751,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.05213463306427,
+      "rewards/margins": 0.5228256583213806,
+      "rewards/rejected": -1.5749603509902954,
       "step": 100
     },
     {
       "epoch": 0.25,
-      "grad_norm": 38.73339240736025,
+      "grad_norm": 135.05375086299483,
       "learning_rate": 9.326809299301306e-08,
-      "logits/chosen": -1.7911834716796875,
-      "logits/rejected": -1.6914081573486328,
-      "logps/chosen": -1.4954617023468018,
-      "logps/rejected": -1.65456223487854,
-      "loss": 0.6056,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.8844586610794067,
-      "rewards/margins": 0.3224923014640808,
-      "rewards/rejected": -1.2069510221481323,
+      "logits/chosen": -1.786421537399292,
+      "logits/rejected": -1.6838346719741821,
+      "logps/chosen": -1.2664103507995605,
+      "logps/rejected": -1.3627694845199585,
+      "loss": 0.567,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.9865251779556274,
+      "rewards/margins": 0.6713417768478394,
+      "rewards/rejected": -1.6578670740127563,
       "step": 110
     },
     {
       "epoch": 0.27,
-      "grad_norm": 35.446910248333666,
+      "grad_norm": 177.39824546583225,
       "learning_rate": 9.114050282021158e-08,
-      "logits/chosen": -1.786220908164978,
-      "logits/rejected": -1.7271583080291748,
-      "logps/chosen": -1.4743096828460693,
-      "logps/rejected": -1.6279948949813843,
-      "loss": 0.6015,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.9802410006523132,
-      "rewards/margins": 0.3188786804676056,
-      "rewards/rejected": -1.2991197109222412,
+      "logits/chosen": -1.7802947759628296,
+      "logits/rejected": -1.719653844833374,
+      "logps/chosen": -1.2173160314559937,
+      "logps/rejected": -1.3028596639633179,
+      "loss": 0.549,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.0463018417358398,
+      "rewards/margins": 0.5722708106040955,
+      "rewards/rejected": -1.6185725927352905,
       "step": 120
     },
     {
       "epoch": 0.3,
-      "grad_norm": 34.33852298554088,
+      "grad_norm": 135.6561906635664,
       "learning_rate": 8.875280914254802e-08,
-      "logits/chosen": -1.7842994928359985,
-      "logits/rejected": -1.6957006454467773,
-      "logps/chosen": -1.6290218830108643,
-      "logps/rejected": -1.7964956760406494,
-      "loss": 0.5858,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.308574914932251,
-      "rewards/margins": 0.41714730858802795,
-      "rewards/rejected": -1.725722312927246,
+      "logits/chosen": -1.7779016494750977,
+      "logits/rejected": -1.6867297887802124,
+      "logps/chosen": -1.2577775716781616,
+      "logps/rejected": -1.333316683769226,
+      "loss": 0.5544,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.9742139577865601,
+      "rewards/margins": 0.7067134976387024,
+      "rewards/rejected": -1.6809275150299072,
       "step": 130
     },
     {
       "epoch": 0.32,
-      "grad_norm": 31.7434623187325,
+      "grad_norm": 120.17349986733925,
       "learning_rate": 8.612010772821971e-08,
-      "logits/chosen": -1.8148624897003174,
-      "logits/rejected": -1.7723850011825562,
-      "logps/chosen": -1.629787802696228,
-      "logps/rejected": -1.7449651956558228,
-      "loss": 0.5865,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.2477266788482666,
-      "rewards/margins": 0.4202614724636078,
-      "rewards/rejected": -1.6679881811141968,
+      "logits/chosen": -1.806403398513794,
+      "logits/rejected": -1.7602646350860596,
+      "logps/chosen": -1.27701735496521,
+      "logps/rejected": -1.2981541156768799,
+      "loss": 0.5617,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.9470755457878113,
+      "rewards/margins": 0.6906994581222534,
+      "rewards/rejected": -1.6377748250961304,
       "step": 140
     },
     {
       "epoch": 0.34,
-      "grad_norm": 30.04410305195209,
+      "grad_norm": 125.85356406550791,
       "learning_rate": 8.325904336322055e-08,
-      "logits/chosen": -1.7936302423477173,
-      "logits/rejected": -1.740534782409668,
-      "logps/chosen": -1.6605161428451538,
-      "logps/rejected": -1.848489761352539,
-      "loss": 0.5948,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.4794538021087646,
-      "rewards/margins": 0.4223996102809906,
-      "rewards/rejected": -1.901853322982788,
+      "logits/chosen": -1.7769775390625,
+      "logits/rejected": -1.719050645828247,
+      "logps/chosen": -1.2321232557296753,
+      "logps/rejected": -1.3308744430541992,
+      "loss": 0.5593,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.9713759422302246,
+      "rewards/margins": 0.7736625671386719,
+      "rewards/rejected": -1.745038628578186,
       "step": 150
     },
     {
       "epoch": 0.36,
-      "grad_norm": 31.6961444390199,
+      "grad_norm": 121.99191139870062,
       "learning_rate": 8.01877046176447e-08,
-      "logits/chosen": -1.7392256259918213,
-      "logits/rejected": -1.6678926944732666,
-      "logps/chosen": -1.707990288734436,
-      "logps/rejected": -1.8706386089324951,
-      "loss": 0.5828,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -1.5890707969665527,
-      "rewards/margins": 0.33378690481185913,
-      "rewards/rejected": -1.922857642173767,
+      "logits/chosen": -1.7220207452774048,
+      "logits/rejected": -1.6478888988494873,
+      "logps/chosen": -1.2581905126571655,
+      "logps/rejected": -1.3523187637329102,
+      "loss": 0.5554,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.1983577013015747,
+      "rewards/margins": 0.6411346197128296,
+      "rewards/rejected": -1.8394920825958252,
       "step": 160
     },
     {
       "epoch": 0.39,
-      "grad_norm": 29.03821659608711,
+      "grad_norm": 121.3180476954759,
       "learning_rate": 7.692550948392249e-08,
-      "logits/chosen": -1.7871320247650146,
-      "logits/rejected": -1.7301868200302124,
-      "logps/chosen": -1.6618130207061768,
-      "logps/rejected": -1.81256902217865,
-      "loss": 0.584,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.3546721935272217,
-      "rewards/margins": 0.42093437910079956,
-      "rewards/rejected": -1.7756065130233765,
+      "logits/chosen": -1.7813163995742798,
+      "logits/rejected": -1.7118024826049805,
+      "logps/chosen": -1.2790229320526123,
+      "logps/rejected": -1.3363419771194458,
+      "loss": 0.5532,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.0315128564834595,
+      "rewards/margins": 0.7031131386756897,
+      "rewards/rejected": -1.7346258163452148,
       "step": 170
     },
     {
       "epoch": 0.41,
-      "grad_norm": 36.01706358849117,
+      "grad_norm": 130.86455724997884,
       "learning_rate": 7.349308261002021e-08,
-      "logits/chosen": -1.743697166442871,
-      "logits/rejected": -1.6886732578277588,
-      "logps/chosen": -1.658620834350586,
-      "logps/rejected": -1.8341089487075806,
-      "loss": 0.5784,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.3846042156219482,
-      "rewards/margins": 0.3775123953819275,
-      "rewards/rejected": -1.7621164321899414,
+      "logits/chosen": -1.738299012184143,
+      "logits/rejected": -1.6767507791519165,
+      "logps/chosen": -1.2677620649337769,
+      "logps/rejected": -1.3575375080108643,
+      "loss": 0.5458,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.0601390600204468,
+      "rewards/margins": 0.6018749475479126,
+      "rewards/rejected": -1.6620140075683594,
       "step": 180
     },
     {
       "epoch": 0.43,
-      "grad_norm": 31.031052349259785,
+      "grad_norm": 129.0255507321099,
       "learning_rate": 6.991212490377531e-08,
-      "logits/chosen": -1.797126054763794,
-      "logits/rejected": -1.7482073307037354,
-      "logps/chosen": -1.731537103652954,
-      "logps/rejected": -1.9256778955459595,
-      "loss": 0.5544,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.504301905632019,
-      "rewards/margins": 0.517795979976654,
-      "rewards/rejected": -2.0220980644226074,
+      "logits/chosen": -1.7946367263793945,
+      "logits/rejected": -1.7395241260528564,
+      "logps/chosen": -1.2883760929107666,
+      "logps/rejected": -1.35979163646698,
+      "loss": 0.5244,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.8740962743759155,
+      "rewards/margins": 0.748104989528656,
+      "rewards/rejected": -1.6222013235092163,
       "step": 190
     },
     {
       "epoch": 0.46,
-      "grad_norm": 41.499506266425385,
+      "grad_norm": 147.0180883475064,
       "learning_rate": 6.620527633276978e-08,
-      "logits/chosen": -1.7355127334594727,
-      "logits/rejected": -1.6644805669784546,
-      "logps/chosen": -1.7263151407241821,
-      "logps/rejected": -2.0103392601013184,
-      "loss": 0.5689,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.5955860614776611,
-      "rewards/margins": 0.591705858707428,
-      "rewards/rejected": -2.1872916221618652,
+      "logits/chosen": -1.732683539390564,
+      "logits/rejected": -1.6549829244613647,
+      "logps/chosen": -1.2525498867034912,
+      "logps/rejected": -1.396055817604065,
+      "loss": 0.5295,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.8714515566825867,
+      "rewards/margins": 0.850756049156189,
+      "rewards/rejected": -1.72220778465271,
       "step": 200
     },
     {
       "epoch": 0.48,
-      "grad_norm": 34.74514581193027,
+      "grad_norm": 171.87128796879827,
       "learning_rate": 6.239597278716581e-08,
-      "logits/chosen": -1.7984533309936523,
-      "logits/rejected": -1.7380235195159912,
-      "logps/chosen": -1.856172800064087,
-      "logps/rejected": -2.031567096710205,
-      "loss": 0.5577,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.8325283527374268,
-      "rewards/margins": 0.5881733894348145,
-      "rewards/rejected": -2.420701742172241,
+      "logits/chosen": -1.8040691614151,
+      "logits/rejected": -1.7398675680160522,
+      "logps/chosen": -1.3047817945480347,
+      "logps/rejected": -1.3454277515411377,
+      "loss": 0.5269,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.891778826713562,
+      "rewards/margins": 0.9196379780769348,
+      "rewards/rejected": -1.8114169836044312,
       "step": 210
     },
     {
       "epoch": 0.5,
-      "grad_norm": 30.0311431190145,
+      "grad_norm": 105.17069566737707,
       "learning_rate": 5.8508297910462456e-08,
-      "logits/chosen": -1.7461389303207397,
-      "logits/rejected": -1.665144681930542,
-      "logps/chosen": -1.78778874874115,
-      "logps/rejected": -2.0407111644744873,
-      "loss": 0.5611,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.872820258140564,
-      "rewards/margins": 0.5854478478431702,
-      "rewards/rejected": -2.4582679271698,
+      "logits/chosen": -1.752892255783081,
+      "logits/rejected": -1.66922926902771,
+      "logps/chosen": -1.2225987911224365,
+      "logps/rejected": -1.3446890115737915,
+      "loss": 0.5101,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.8862507939338684,
+      "rewards/margins": 0.9647538065910339,
+      "rewards/rejected": -1.8510046005249023,
       "step": 220
     },
     {
       "epoch": 0.52,
-      "grad_norm": 33.05734006337458,
+      "grad_norm": 163.22912298655683,
       "learning_rate": 5.456683083494731e-08,
-      "logits/chosen": -1.7369384765625,
-      "logits/rejected": -1.69870924949646,
-      "logps/chosen": -1.7765703201293945,
-      "logps/rejected": -1.9211543798446655,
-      "loss": 0.5821,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -1.7482845783233643,
-      "rewards/margins": 0.3625262379646301,
-      "rewards/rejected": -2.1108105182647705,
+      "logits/chosen": -1.749047040939331,
+      "logits/rejected": -1.7083661556243896,
+      "logps/chosen": -1.2520840167999268,
+      "logps/rejected": -1.3157161474227905,
+      "loss": 0.5452,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.8741270303726196,
+      "rewards/margins": 0.5983553528785706,
+      "rewards/rejected": -1.4724823236465454,
       "step": 230
     },
     {
       "epoch": 0.55,
-      "grad_norm": 42.868772014524744,
+      "grad_norm": 166.72716973375287,
       "learning_rate": 5.059649078450834e-08,
-      "logits/chosen": -1.7265160083770752,
-      "logits/rejected": -1.6816940307617188,
-      "logps/chosen": -1.7579532861709595,
-      "logps/rejected": -1.9656188488006592,
-      "loss": 0.5499,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -1.7687995433807373,
-      "rewards/margins": 0.43688035011291504,
-      "rewards/rejected": -2.2056798934936523,
+      "logits/chosen": -1.744289755821228,
+      "logits/rejected": -1.6957895755767822,
+      "logps/chosen": -1.237776517868042,
+      "logps/rejected": -1.3457074165344238,
+      "loss": 0.5212,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.041345477104187,
+      "rewards/margins": 0.6883827447891235,
+      "rewards/rejected": -1.729728102684021,
       "step": 240
     },
     {
       "epoch": 0.57,
-      "grad_norm": 32.81853514392731,
+      "grad_norm": 144.21705239974517,
       "learning_rate": 4.6622379527277186e-08,
-      "logits/chosen": -1.7367656230926514,
-      "logits/rejected": -1.685956358909607,
-      "logps/chosen": -1.7992569208145142,
-      "logps/rejected": -1.9660634994506836,
-      "loss": 0.5585,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.981104850769043,
-      "rewards/margins": 0.4011806845664978,
-      "rewards/rejected": -2.3822855949401855,
+      "logits/chosen": -1.7480140924453735,
+      "logits/rejected": -1.6941328048706055,
+      "logps/chosen": -1.2107579708099365,
+      "logps/rejected": -1.2955563068389893,
+      "loss": 0.5249,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.0780389308929443,
+      "rewards/margins": 0.7757798433303833,
+      "rewards/rejected": -1.853818655014038,
       "step": 250
     },
     {
       "epoch": 0.59,
-      "grad_norm": 33.27003433225899,
+      "grad_norm": 112.35316117599322,
       "learning_rate": 4.26696226741691e-08,
-      "logits/chosen": -1.7512362003326416,
-      "logits/rejected": -1.6821905374526978,
-      "logps/chosen": -1.897579550743103,
-      "logps/rejected": -2.0977420806884766,
-      "loss": 0.5601,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.9942684173583984,
-      "rewards/margins": 0.5405869483947754,
-      "rewards/rejected": -2.534855365753174,
+      "logits/chosen": -1.7646477222442627,
+      "logits/rejected": -1.6895405054092407,
+      "logps/chosen": -1.3171813488006592,
+      "logps/rejected": -1.3915660381317139,
+      "loss": 0.5203,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.265368103981018,
+      "rewards/margins": 0.8162665367126465,
+      "rewards/rejected": -2.081634521484375,
       "step": 260
     },
     {
       "epoch": 0.61,
-      "grad_norm": 28.394830302289282,
+      "grad_norm": 108.06033286041723,
       "learning_rate": 3.876321082668098e-08,
-      "logits/chosen": -1.8098827600479126,
-      "logits/rejected": -1.7416326999664307,
-      "logps/chosen": -1.8279718160629272,
-      "logps/rejected": -2.0366756916046143,
-      "loss": 0.5516,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.7684434652328491,
-      "rewards/margins": 0.5981845855712891,
-      "rewards/rejected": -2.3666281700134277,
+      "logits/chosen": -1.828253149986267,
+      "logits/rejected": -1.7521867752075195,
+      "logps/chosen": -1.2993474006652832,
+      "logps/rejected": -1.3664964437484741,
+      "loss": 0.5157,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.9128493070602417,
+      "rewards/margins": 0.8676029443740845,
+      "rewards/rejected": -1.7804524898529053,
       "step": 270
     },
     {
       "epoch": 0.64,
-      "grad_norm": 33.9712584613191,
+      "grad_norm": 133.35622623933705,
       "learning_rate": 3.492784157826244e-08,
-      "logits/chosen": -1.7408695220947266,
-      "logits/rejected": -1.6451225280761719,
-      "logps/chosen": -1.844485878944397,
-      "logps/rejected": -2.0920097827911377,
-      "loss": 0.5537,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.864828109741211,
-      "rewards/margins": 0.6364026665687561,
-      "rewards/rejected": -2.5012307167053223,
+      "logits/chosen": -1.7531992197036743,
+      "logits/rejected": -1.6531322002410889,
+      "logps/chosen": -1.2826292514801025,
+      "logps/rejected": -1.3810635805130005,
+      "loss": 0.5156,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.8962936401367188,
+      "rewards/margins": 0.9456663131713867,
+      "rewards/rejected": -1.8419599533081055,
       "step": 280
     },
     {
       "epoch": 0.66,
-      "grad_norm": 26.97002465180278,
+      "grad_norm": 118.30208566104156,
       "learning_rate": 3.118776336817812e-08,
-      "logits/chosen": -1.782135009765625,
-      "logits/rejected": -1.72174870967865,
-      "logps/chosen": -1.8031295537948608,
-      "logps/rejected": -2.0307796001434326,
-      "loss": 0.5443,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.7996422052383423,
-      "rewards/margins": 0.6613016128540039,
-      "rewards/rejected": -2.4609439373016357,
+      "logits/chosen": -1.8004519939422607,
+      "logits/rejected": -1.731174111366272,
+      "logps/chosen": -1.269983172416687,
+      "logps/rejected": -1.337033987045288,
+      "loss": 0.5087,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.0010195970535278,
+      "rewards/margins": 0.89751797914505,
+      "rewards/rejected": -1.8985373973846436,
       "step": 290
     },
     {
       "epoch": 0.68,
-      "grad_norm": 31.17322900966105,
+      "grad_norm": 130.95523326619738,
       "learning_rate": 2.7566622175067443e-08,
-      "logits/chosen": -1.7659027576446533,
-      "logits/rejected": -1.7027981281280518,
-      "logps/chosen": -1.8818680047988892,
-      "logps/rejected": -2.163147449493408,
-      "loss": 0.5502,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -2.0198538303375244,
-      "rewards/margins": 0.6098783612251282,
-      "rewards/rejected": -2.629732131958008,
+      "logits/chosen": -1.7788505554199219,
+      "logits/rejected": -1.709090232849121,
+      "logps/chosen": -1.2863571643829346,
+      "logps/rejected": -1.4172859191894531,
+      "loss": 0.5262,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1666053533554077,
+      "rewards/margins": 0.7941334843635559,
+      "rewards/rejected": -1.9607388973236084,
       "step": 300
     },
     {
       "epoch": 0.71,
-      "grad_norm": 36.262422147908055,
+      "grad_norm": 154.33175524161533,
       "learning_rate": 2.408731201945432e-08,
-      "logits/chosen": -1.761371374130249,
-      "logits/rejected": -1.7142196893692017,
-      "logps/chosen": -1.8247020244598389,
-      "logps/rejected": -2.0095391273498535,
-      "loss": 0.5485,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.8924195766448975,
-      "rewards/margins": 0.49404746294021606,
-      "rewards/rejected": -2.386467218399048,
+      "logits/chosen": -1.7760515213012695,
+      "logits/rejected": -1.727215051651001,
+      "logps/chosen": -1.2580327987670898,
+      "logps/rejected": -1.3251044750213623,
+      "loss": 0.5163,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.962059497833252,
+      "rewards/margins": 0.7037551999092102,
+      "rewards/rejected": -1.6658146381378174,
       "step": 310
     },
     {
       "epoch": 0.73,
-      "grad_norm": 32.89905332615021,
+      "grad_norm": 146.18822564163364,
       "learning_rate": 2.0771830220378112e-08,
-      "logits/chosen": -1.7170169353485107,
-      "logits/rejected": -1.6577152013778687,
-      "logps/chosen": -1.8545100688934326,
-      "logps/rejected": -2.0157198905944824,
-      "loss": 0.5515,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.8948768377304077,
-      "rewards/margins": 0.5071803331375122,
-      "rewards/rejected": -2.40205717086792,
+      "logits/chosen": -1.7305431365966797,
+      "logits/rejected": -1.6727710962295532,
+      "logps/chosen": -1.2899402379989624,
+      "logps/rejected": -1.3281028270721436,
+      "loss": 0.5156,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.0058369636535645,
+      "rewards/margins": 0.6901935935020447,
+      "rewards/rejected": -1.696030616760254,
       "step": 320
     },
     {
       "epoch": 0.75,
-      "grad_norm": 29.752828533537873,
+      "grad_norm": 117.04263934434923,
       "learning_rate": 1.7641138321260257e-08,
-      "logits/chosen": -1.7509727478027344,
-      "logits/rejected": -1.6813408136367798,
-      "logps/chosen": -1.8006902933120728,
-      "logps/rejected": -2.068765878677368,
-      "loss": 0.5463,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.8505061864852905,
-      "rewards/margins": 0.7216218113899231,
-      "rewards/rejected": -2.5721280574798584,
+      "logits/chosen": -1.7670669555664062,
+      "logits/rejected": -1.689979910850525,
+      "logps/chosen": -1.246047019958496,
+      "logps/rejected": -1.3403490781784058,
+      "loss": 0.5069,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.9328843951225281,
+      "rewards/margins": 1.0015037059783936,
+      "rewards/rejected": -1.9343881607055664,
       "step": 330
     },
     {
       "epoch": 0.77,
-      "grad_norm": 29.865199582942665,
+      "grad_norm": 118.21626557678114,
       "learning_rate": 1.4715029564277793e-08,
-      "logits/chosen": -1.8179073333740234,
-      "logits/rejected": -1.771498680114746,
-      "logps/chosen": -1.7910722494125366,
-      "logps/rejected": -2.0462353229522705,
-      "loss": 0.5539,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.759235143661499,
-      "rewards/margins": 0.6542983651161194,
-      "rewards/rejected": -2.4135334491729736,
+      "logits/chosen": -1.838197112083435,
+      "logits/rejected": -1.783423662185669,
+      "logps/chosen": -1.2548508644104004,
+      "logps/rejected": -1.349151849746704,
+      "loss": 0.5246,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.7528579235076904,
+      "rewards/margins": 0.8585602045059204,
+      "rewards/rejected": -1.6114181280136108,
       "step": 340
     },
     {
       "epoch": 0.8,
-      "grad_norm": 33.27484680109776,
+      "grad_norm": 144.7456304205745,
       "learning_rate": 1.2012003751113343e-08,
-      "logits/chosen": -1.806565284729004,
-      "logits/rejected": -1.745749831199646,
-      "logps/chosen": -1.8502833843231201,
-      "logps/rejected": -2.1222567558288574,
-      "loss": 0.5334,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -2.0643410682678223,
-      "rewards/margins": 0.6653475761413574,
-      "rewards/rejected": -2.729688882827759,
+      "logits/chosen": -1.8204278945922852,
+      "logits/rejected": -1.753904938697815,
+      "logps/chosen": -1.2354977130889893,
+      "logps/rejected": -1.3442593812942505,
+      "loss": 0.4987,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.0999176502227783,
+      "rewards/margins": 0.8785642385482788,
+      "rewards/rejected": -1.9784818887710571,
       "step": 350
     },
     {
       "epoch": 0.82,
-      "grad_norm": 35.608600933807864,
+      "grad_norm": 119.65129571028098,
       "learning_rate": 9.549150281252633e-09,
-      "logits/chosen": -1.762521505355835,
-      "logits/rejected": -1.7128925323486328,
-      "logps/chosen": -1.8086223602294922,
-      "logps/rejected": -2.0623652935028076,
-      "loss": 0.5356,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.9094616174697876,
-      "rewards/margins": 0.6060369610786438,
-      "rewards/rejected": -2.515498399734497,
+      "logits/chosen": -1.7847106456756592,
+      "logits/rejected": -1.7283674478530884,
+      "logps/chosen": -1.2320148944854736,
+      "logps/rejected": -1.3409490585327148,
+      "loss": 0.5006,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.8981946110725403,
+      "rewards/margins": 0.858056902885437,
+      "rewards/rejected": -1.756251335144043,
       "step": 360
     },
     {
       "epoch": 0.84,
-      "grad_norm": 31.739203329110374,
+      "grad_norm": 112.87545529277935,
       "learning_rate": 7.3420401072985306e-09,
-      "logits/chosen": -1.7991466522216797,
-      "logits/rejected": -1.746799111366272,
-      "logps/chosen": -1.8659775257110596,
-      "logps/rejected": -2.1334667205810547,
-      "loss": 0.5356,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.9540916681289673,
-      "rewards/margins": 0.6300768256187439,
-      "rewards/rejected": -2.5841686725616455,
+      "logits/chosen": -1.8140869140625,
+      "logits/rejected": -1.7608489990234375,
+      "logps/chosen": -1.282485008239746,
+      "logps/rejected": -1.4008147716522217,
+      "loss": 0.4982,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.018068552017212,
+      "rewards/margins": 0.912992000579834,
+      "rewards/rejected": -1.931060791015625,
       "step": 370
     },
     {
       "epoch": 0.86,
-      "grad_norm": 37.231267463423904,
+      "grad_norm": 144.02889200607225,
       "learning_rate": 5.404627290395369e-09,
-      "logits/chosen": -1.7623279094696045,
-      "logits/rejected": -1.6976515054702759,
-      "logps/chosen": -1.7954782247543335,
-      "logps/rejected": -2.0490074157714844,
-      "loss": 0.5377,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.8176171779632568,
-      "rewards/margins": 0.6405991315841675,
-      "rewards/rejected": -2.4582161903381348,
+      "logits/chosen": -1.779488205909729,
+      "logits/rejected": -1.7096574306488037,
+      "logps/chosen": -1.249098300933838,
+      "logps/rejected": -1.3456512689590454,
+      "loss": 0.5043,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.892386794090271,
+      "rewards/margins": 0.8483527302742004,
+      "rewards/rejected": -1.7407394647598267,
       "step": 380
     },
     {
       "epoch": 0.89,
-      "grad_norm": 48.31129307964873,
+      "grad_norm": 225.16574034992297,
       "learning_rate": 3.74916077816162e-09,
-      "logits/chosen": -1.7699306011199951,
-      "logits/rejected": -1.7098052501678467,
-      "logps/chosen": -1.7985498905181885,
-      "logps/rejected": -1.9994351863861084,
-      "loss": 0.5502,
+      "logits/chosen": -1.7894920110702515,
+      "logits/rejected": -1.7234842777252197,
+      "logps/chosen": -1.222625494003296,
+      "logps/rejected": -1.288031816482544,
+      "loss": 0.5184,
       "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.946265459060669,
-      "rewards/margins": 0.5542693734169006,
-      "rewards/rejected": -2.5005345344543457,
+      "rewards/chosen": -1.092460036277771,
+      "rewards/margins": 0.7391617894172668,
+      "rewards/rejected": -1.831621766090393,
       "step": 390
     },
     {
       "epoch": 0.91,
-      "grad_norm": 36.09144092538641,
+      "grad_norm": 127.97653300507214,
       "learning_rate": 2.386106962899165e-09,
-      "logits/chosen": -1.6945453882217407,
-      "logits/rejected": -1.62551748752594,
-      "logps/chosen": -1.9125568866729736,
-      "logps/rejected": -2.1548168659210205,
-      "loss": 0.5447,
+      "logits/chosen": -1.7130002975463867,
+      "logits/rejected": -1.637550950050354,
+      "logps/chosen": -1.295520544052124,
+      "logps/rejected": -1.403545618057251,
+      "loss": 0.5191,
       "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -2.05840802192688,
-      "rewards/margins": 0.5577607154846191,
-      "rewards/rejected": -2.61616849899292,
+      "rewards/chosen": -1.0364947319030762,
+      "rewards/margins": 0.7752783298492432,
+      "rewards/rejected": -1.8117730617523193,
       "step": 400
     },
     {
       "epoch": 0.93,
-      "grad_norm": 35.37496893135356,
+      "grad_norm": 131.7102880560143,
       "learning_rate": 1.3240835096913706e-09,
-      "logits/chosen": -1.7273473739624023,
-      "logits/rejected": -1.6288312673568726,
-      "logps/chosen": -1.787335753440857,
-      "logps/rejected": -2.1021180152893066,
-      "loss": 0.5546,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.8473552465438843,
-      "rewards/margins": 0.7361037731170654,
-      "rewards/rejected": -2.5834591388702393,
+      "logits/chosen": -1.7465556859970093,
+      "logits/rejected": -1.638227105140686,
+      "logps/chosen": -1.2277872562408447,
+      "logps/rejected": -1.3602231740951538,
+      "loss": 0.5288,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.8435455560684204,
+      "rewards/margins": 0.945324718952179,
+      "rewards/rejected": -1.7888704538345337,
       "step": 410
     },
     {
       "epoch": 0.96,
-      "grad_norm": 32.802494867000924,
+      "grad_norm": 120.72093799584852,
       "learning_rate": 5.698048727497462e-10,
-      "logits/chosen": -1.7570117712020874,
-      "logits/rejected": -1.6864467859268188,
-      "logps/chosen": -1.8803859949111938,
-      "logps/rejected": -2.1749351024627686,
-      "loss": 0.5408,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -2.0672643184661865,
-      "rewards/margins": 0.6827760338783264,
-      "rewards/rejected": -2.7500405311584473,
+      "logits/chosen": -1.7705790996551514,
+      "logits/rejected": -1.6966116428375244,
+      "logps/chosen": -1.259157419204712,
+      "logps/rejected": -1.38863205909729,
+      "loss": 0.5087,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.0178930759429932,
+      "rewards/margins": 0.9377636909484863,
+      "rewards/rejected": -1.9556567668914795,
       "step": 420
     },
     {
       "epoch": 0.98,
-      "grad_norm": 30.44073541133352,
+      "grad_norm": 122.8656564436435,
       "learning_rate": 1.2803984447259387e-10,
-      "logits/chosen": -1.7648996114730835,
-      "logits/rejected": -1.713091254234314,
-      "logps/chosen": -1.8600318431854248,
-      "logps/rejected": -2.127103805541992,
-      "loss": 0.5345,
+      "logits/chosen": -1.7806816101074219,
+      "logits/rejected": -1.719323754310608,
+      "logps/chosen": -1.2518696784973145,
+      "logps/rejected": -1.3653380870819092,
+      "loss": 0.5027,
       "rewards/accuracies": 0.71875,
-      "rewards/chosen": -2.0330777168273926,
-      "rewards/margins": 0.6582567095756531,
-      "rewards/rejected": -2.6913342475891113,
+      "rewards/chosen": -1.0429534912109375,
+      "rewards/margins": 0.987235426902771,
+      "rewards/rejected": -2.030189037322998,
       "step": 430
     },
     {
       "epoch": 1.0,
       "step": 439,
       "total_flos": 0.0,
-      "train_loss": 0.5807303016441016,
-      "train_runtime": 6860.7908,
-      "train_samples_per_second": 8.197,
+      "train_loss": 0.5484831045320202,
+      "train_runtime": 6843.1048,
+      "train_samples_per_second": 8.218,
       "train_steps_per_second": 0.064
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:52fd958028d2a8e77c3db64f0cef52c04c727dbfa54965417b4be2b66b01aefd
+oid sha256:a7e7e48ed484507dc4bf687f99b9afd74522a0ce37982f50ef4c64c14f4f420c
 size 6328