Model save

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.9518976428785737,
-    "train_runtime": 6868.0311,
+    "train_loss": 0.4228534948309896,
+    "train_runtime": 5290.3981,
     "train_samples": 56236,
-    "train_samples_per_second": 8.188,
-    "train_steps_per_second": 0.064
+    "train_samples_per_second": 10.63,
+    "train_steps_per_second": 0.083
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7ca818714ff132e49fe3efaad8395277eee1abf8aa194efda60c3b54df9ba68e
+oid sha256:75d6368d1fdee8107ded739a6b116ec4475e494955ea8e987c3fbe72511666ef
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:120903dde67d78a96fffd5cc95eb23ca4bea822546ee32c239aa2250162c5c51
+oid sha256:9c5b6326f4bbfe2dc14bbb7c67e7fd495feaea85c802948941fd853302b36ac8
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e5b9a3b73d64098d70d2ccb7c88d04b96265090da8b4e12a2a4a554d6bf47cd
+oid sha256:6defafbfd4c91d1663c1f8e99d13d070ad43df11a0d33c2b6e84d278c3d22e01
 size 4540516344

runs/Jul03_00-12-11_n136-100-194/events.out.tfevents.1719936804.n136-100-194.1901075.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:717b17da9043eba81a6debfd3654c81f68dd7c148c557ae69684c0e354261023
-size 25561
+oid sha256:d75b9d5c9ade3cf37815f2e476818445deffeb0ae4b4be2f497cf893dd1f1dc5
+size 27979

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.9518976428785737,
-    "train_runtime": 6868.0311,
+    "train_loss": 0.4228534948309896,
+    "train_runtime": 5290.3981,
     "train_samples": 56236,
-    "train_samples_per_second": 8.188,
-    "train_steps_per_second": 0.064
+    "train_samples_per_second": 10.63,
+    "train_steps_per_second": 0.083
 }

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 1666.4801111075699,
+      "grad_norm": 56.936522397845614,
       "learning_rate": 2.2727272727272727e-09,
       "logits/chosen": -1.6768856048583984,
       "logits/rejected": -1.7259055376052856,
       "logps/chosen": -1.2793102264404297,
       "logps/rejected": -1.2162058353424072,
-      "loss": 1.3182,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,657 +25,657 @@
     },
     {
       "epoch": 0.02,
-      "grad_norm": 1931.392535785282,
+      "grad_norm": 64.38180400614688,
       "learning_rate": 2.2727272727272725e-08,
-      "logits/chosen": -1.7030127048492432,
-      "logits/rejected": -1.6681498289108276,
-      "logps/chosen": -1.2134774923324585,
-      "logps/rejected": -1.2203598022460938,
-      "loss": 1.3237,
-      "rewards/accuracies": 0.3958333432674408,
-      "rewards/chosen": -0.017192421481013298,
-      "rewards/margins": -0.05587056651711464,
-      "rewards/rejected": 0.03867815062403679,
+      "logits/chosen": -1.7024906873703003,
+      "logits/rejected": -1.6682149171829224,
+      "logps/chosen": -1.21290922164917,
+      "logps/rejected": -1.2202513217926025,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.4652777910232544,
+      "rewards/chosen": 0.00198164745233953,
+      "rewards/margins": -0.0004947604611515999,
+      "rewards/rejected": 0.00247640791349113,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 1830.5113687291991,
+      "grad_norm": 64.62406710950098,
       "learning_rate": 4.545454545454545e-08,
-      "logits/chosen": -1.780092477798462,
-      "logits/rejected": -1.7350375652313232,
-      "logps/chosen": -1.1447092294692993,
-      "logps/rejected": -1.1850192546844482,
-      "loss": 1.2938,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.148686021566391,
-      "rewards/margins": 0.007283961866050959,
-      "rewards/rejected": -0.1559699922800064,
+      "logits/chosen": -1.7796684503555298,
+      "logits/rejected": -1.7348577976226807,
+      "logps/chosen": -1.1446455717086792,
+      "logps/rejected": -1.185205340385437,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -0.007116294465959072,
+      "rewards/margins": 0.0016124974936246872,
+      "rewards/rejected": -0.008728791028261185,
       "step": 20
     },
     {
       "epoch": 0.07,
-      "grad_norm": 1936.12548502703,
+      "grad_norm": 65.81836343668654,
       "learning_rate": 6.818181818181817e-08,
-      "logits/chosen": -1.7440731525421143,
-      "logits/rejected": -1.674883246421814,
-      "logps/chosen": -1.1901237964630127,
-      "logps/rejected": -1.2408424615859985,
-      "loss": 1.2335,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.7222227454185486,
-      "rewards/margins": 0.348898708820343,
-      "rewards/rejected": -1.0711214542388916,
+      "logits/chosen": -1.7444534301757812,
+      "logits/rejected": -1.675607442855835,
+      "logps/chosen": -1.1950287818908691,
+      "logps/rejected": -1.2475531101226807,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.06063701584935188,
+      "rewards/margins": 0.0264725498855114,
+      "rewards/rejected": -0.08710956573486328,
       "step": 30
     },
     {
       "epoch": 0.09,
-      "grad_norm": 1274.5224242026777,
+      "grad_norm": 48.424632790373714,
       "learning_rate": 9.09090909090909e-08,
-      "logits/chosen": -1.730063796043396,
-      "logits/rejected": -1.6635162830352783,
-      "logps/chosen": -1.2168413400650024,
-      "logps/rejected": -1.274864912033081,
-      "loss": 1.2163,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.9892686009407043,
-      "rewards/margins": 0.5700188875198364,
-      "rewards/rejected": -1.559287428855896,
+      "logits/chosen": -1.7310537099838257,
+      "logits/rejected": -1.6650664806365967,
+      "logps/chosen": -1.2446069717407227,
+      "logps/rejected": -1.3240689039230347,
+      "loss": 0.6683,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.18829187750816345,
+      "rewards/margins": 0.1356922686100006,
+      "rewards/rejected": -0.3239841163158417,
       "step": 40
     },
     {
       "epoch": 0.11,
-      "grad_norm": 1227.2878182263496,
+      "grad_norm": 50.59306703821365,
       "learning_rate": 9.994307990108962e-08,
-      "logits/chosen": -1.6924018859863281,
-      "logits/rejected": -1.6282413005828857,
-      "logps/chosen": -1.2177562713623047,
-      "logps/rejected": -1.2450090646743774,
-      "loss": 1.1002,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.9669270515441895,
-      "rewards/margins": 0.7160126566886902,
-      "rewards/rejected": -1.6829397678375244,
+      "logits/chosen": -1.695184350013733,
+      "logits/rejected": -1.6308221817016602,
+      "logps/chosen": -1.2791718244552612,
+      "logps/rejected": -1.3329250812530518,
+      "loss": 0.6419,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.35542386770248413,
+      "rewards/margins": 0.1683032065629959,
+      "rewards/rejected": -0.5237270593643188,
       "step": 50
     },
     {
       "epoch": 0.14,
-      "grad_norm": 1544.774039586819,
+      "grad_norm": 70.52284728653416,
       "learning_rate": 9.959570405988094e-08,
-      "logits/chosen": -1.721835732460022,
-      "logits/rejected": -1.6408640146255493,
-      "logps/chosen": -1.1233644485473633,
-      "logps/rejected": -1.1840593814849854,
-      "loss": 1.0709,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.4266197681427002,
-      "rewards/margins": 0.8766496777534485,
-      "rewards/rejected": -2.303269386291504,
+      "logits/chosen": -1.7311099767684937,
+      "logits/rejected": -1.651442527770996,
+      "logps/chosen": -1.2228403091430664,
+      "logps/rejected": -1.3055191040039062,
+      "loss": 0.6443,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.5687106847763062,
+      "rewards/margins": 0.15375135838985443,
+      "rewards/rejected": -0.7224621176719666,
       "step": 60
     },
     {
       "epoch": 0.16,
-      "grad_norm": 1005.439321360097,
+      "grad_norm": 45.679190031541204,
       "learning_rate": 9.893476820924666e-08,
-      "logits/chosen": -1.8021934032440186,
-      "logits/rejected": -1.7167574167251587,
-      "logps/chosen": -1.230804443359375,
-      "logps/rejected": -1.2889682054519653,
-      "loss": 1.0139,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.796240210533142,
-      "rewards/margins": 1.1498826742172241,
-      "rewards/rejected": -2.946122884750366,
+      "logits/chosen": -1.8201316595077515,
+      "logits/rejected": -1.7361564636230469,
+      "logps/chosen": -1.3600715398788452,
+      "logps/rejected": -1.4583489894866943,
+      "loss": 0.6162,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.7361480593681335,
+      "rewards/margins": 0.2580623924732208,
+      "rewards/rejected": -0.994210422039032,
       "step": 70
     },
     {
       "epoch": 0.18,
-      "grad_norm": 1396.6389927727296,
+      "grad_norm": 54.41920848920781,
       "learning_rate": 9.796445099843647e-08,
-      "logits/chosen": -1.7873592376708984,
-      "logits/rejected": -1.6995294094085693,
-      "logps/chosen": -1.2222901582717896,
-      "logps/rejected": -1.2922863960266113,
-      "loss": 1.1257,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.5321768522262573,
-      "rewards/margins": 0.9466292262077332,
-      "rewards/rejected": -2.4788060188293457,
+      "logits/chosen": -1.8112220764160156,
+      "logits/rejected": -1.7294734716415405,
+      "logps/chosen": -1.3612664937973022,
+      "logps/rejected": -1.476828694343567,
+      "loss": 0.6324,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.7714906334877014,
+      "rewards/margins": 0.2751607894897461,
+      "rewards/rejected": -1.0466516017913818,
       "step": 80
     },
     {
       "epoch": 0.2,
-      "grad_norm": 1030.6978947603225,
+      "grad_norm": 55.97475150728081,
       "learning_rate": 9.669088708527066e-08,
-      "logits/chosen": -1.734185814857483,
-      "logits/rejected": -1.6654260158538818,
-      "logps/chosen": -1.2461029291152954,
-      "logps/rejected": -1.265190839767456,
-      "loss": 0.9922,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.0405571460723877,
-      "rewards/margins": 1.204342246055603,
-      "rewards/rejected": -2.244899272918701,
+      "logits/chosen": -1.7606143951416016,
+      "logits/rejected": -1.6983873844146729,
+      "logps/chosen": -1.4084397554397583,
+      "logps/rejected": -1.47458815574646,
+      "loss": 0.6039,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.8637123107910156,
+      "rewards/margins": 0.29551851749420166,
+      "rewards/rejected": -1.1592309474945068,
       "step": 90
     },
     {
       "epoch": 0.23,
-      "grad_norm": 1586.6455210629026,
+      "grad_norm": 59.68871990537901,
       "learning_rate": 9.512212835085849e-08,
-      "logits/chosen": -1.765062689781189,
-      "logits/rejected": -1.6767990589141846,
-      "logps/chosen": -1.1784775257110596,
-      "logps/rejected": -1.233091115951538,
-      "loss": 1.016,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.0022172927856445,
-      "rewards/margins": 1.1903674602508545,
-      "rewards/rejected": -2.192584753036499,
+      "logits/chosen": -1.7991968393325806,
+      "logits/rejected": -1.7159475088119507,
+      "logps/chosen": -1.3749934434890747,
+      "logps/rejected": -1.490278959274292,
+      "loss": 0.5966,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.0326907634735107,
+      "rewards/margins": 0.36287835240364075,
+      "rewards/rejected": -1.3955689668655396,
       "step": 100
     },
     {
       "epoch": 0.25,
-      "grad_norm": 1144.8635215765032,
+      "grad_norm": 51.2753686121636,
       "learning_rate": 9.326809299301306e-08,
-      "logits/chosen": -1.7722240686416626,
-      "logits/rejected": -1.6646277904510498,
-      "logps/chosen": -1.2064539194107056,
-      "logps/rejected": -1.2708475589752197,
-      "loss": 0.9709,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.5811842679977417,
-      "rewards/margins": 1.279047966003418,
-      "rewards/rejected": -1.8602321147918701,
+      "logits/chosen": -1.796499252319336,
+      "logits/rejected": -1.6970577239990234,
+      "logps/chosen": -1.4165159463882446,
+      "logps/rejected": -1.5609779357910156,
+      "loss": 0.5906,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.0793695449829102,
+      "rewards/margins": 0.46429458260536194,
+      "rewards/rejected": -1.5436642169952393,
       "step": 110
     },
     {
       "epoch": 0.27,
-      "grad_norm": 1417.2025435553687,
+      "grad_norm": 55.503246670500936,
       "learning_rate": 9.114050282021158e-08,
-      "logits/chosen": -1.762213110923767,
-      "logits/rejected": -1.6987812519073486,
-      "logps/chosen": -1.1565983295440674,
-      "logps/rejected": -1.2166643142700195,
-      "loss": 0.9277,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.9035682678222656,
-      "rewards/margins": 1.2673728466033936,
-      "rewards/rejected": -2.1709413528442383,
+      "logits/chosen": -1.7899627685546875,
+      "logits/rejected": -1.7314037084579468,
+      "logps/chosen": -1.3753345012664795,
+      "logps/rejected": -1.5070170164108276,
+      "loss": 0.5881,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.1388596296310425,
+      "rewards/margins": 0.4214509427547455,
+      "rewards/rejected": -1.5603104829788208,
       "step": 120
     },
     {
       "epoch": 0.3,
-      "grad_norm": 1150.053731245993,
+      "grad_norm": 53.36720259691284,
       "learning_rate": 8.875280914254802e-08,
-      "logits/chosen": -1.7586562633514404,
-      "logits/rejected": -1.6647939682006836,
-      "logps/chosen": -1.20124089717865,
-      "logps/rejected": -1.2465784549713135,
-      "loss": 0.9452,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.841080367565155,
-      "rewards/margins": 1.6912786960601807,
-      "rewards/rejected": -2.5323591232299805,
+      "logits/chosen": -1.7889665365219116,
+      "logits/rejected": -1.699845552444458,
+      "logps/chosen": -1.4648027420043945,
+      "logps/rejected": -1.600429892539978,
+      "loss": 0.5731,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.3598629236221313,
+      "rewards/margins": 0.5360117554664612,
+      "rewards/rejected": -1.8958743810653687,
       "step": 130
     },
     {
       "epoch": 0.32,
-      "grad_norm": 1256.1179361929435,
+      "grad_norm": 43.48973975700312,
       "learning_rate": 8.612010772821971e-08,
-      "logits/chosen": -1.7858541011810303,
-      "logits/rejected": -1.7347157001495361,
-      "logps/chosen": -1.2216544151306152,
-      "logps/rejected": -1.2101521492004395,
-      "loss": 0.9734,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.7775429487228394,
-      "rewards/margins": 1.340767502784729,
-      "rewards/rejected": -2.1183104515075684,
+      "logits/chosen": -1.8176887035369873,
+      "logits/rejected": -1.7743492126464844,
+      "logps/chosen": -1.4533226490020752,
+      "logps/rejected": -1.5335619449615479,
+      "loss": 0.5749,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1972180604934692,
+      "rewards/margins": 0.5257461667060852,
+      "rewards/rejected": -1.7229640483856201,
       "step": 140
     },
     {
       "epoch": 0.34,
-      "grad_norm": 1137.7394214504618,
+      "grad_norm": 42.61041473166327,
       "learning_rate": 8.325904336322055e-08,
-      "logits/chosen": -1.7536169290542603,
-      "logits/rejected": -1.691907286643982,
-      "logps/chosen": -1.1751422882080078,
-      "logps/rejected": -1.2404186725616455,
-      "loss": 0.9503,
+      "logits/chosen": -1.7937091588974,
+      "logits/rejected": -1.7397197484970093,
+      "logps/chosen": -1.4411404132843018,
+      "logps/rejected": -1.5950322151184082,
+      "loss": 0.5797,
       "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.7777413725852966,
-      "rewards/margins": 1.8102855682373047,
-      "rewards/rejected": -2.588027000427246,
+      "rewards/chosen": -1.3688783645629883,
+      "rewards/margins": 0.5335904359817505,
+      "rewards/rejected": -1.9024689197540283,
       "step": 150
     },
     {
       "epoch": 0.36,
-      "grad_norm": 1056.887388548025,
+      "grad_norm": 43.447112350155855,
       "learning_rate": 8.01877046176447e-08,
-      "logits/chosen": -1.6971683502197266,
-      "logits/rejected": -1.6201127767562866,
-      "logps/chosen": -1.192384958267212,
-      "logps/rejected": -1.2586729526519775,
-      "loss": 0.9572,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.4085007905960083,
-      "rewards/margins": 1.490207314491272,
-      "rewards/rejected": -2.898707866668701,
+      "logits/chosen": -1.7396624088287354,
+      "logits/rejected": -1.6674416065216064,
+      "logps/chosen": -1.4853546619415283,
+      "logps/rejected": -1.6183408498764038,
+      "loss": 0.5754,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.5352730751037598,
+      "rewards/margins": 0.40800127387046814,
+      "rewards/rejected": -1.9432742595672607,
       "step": 160
     },
     {
       "epoch": 0.39,
-      "grad_norm": 991.2349708721418,
+      "grad_norm": 42.81233538750994,
       "learning_rate": 7.692550948392249e-08,
-      "logits/chosen": -1.759788155555725,
-      "logits/rejected": -1.6842018365859985,
-      "logps/chosen": -1.2190964221954346,
-      "logps/rejected": -1.244114637374878,
-      "loss": 0.9699,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.884096622467041,
-      "rewards/margins": 1.4573404788970947,
-      "rewards/rejected": -2.3414371013641357,
+      "logits/chosen": -1.7920860052108765,
+      "logits/rejected": -1.7322765588760376,
+      "logps/chosen": -1.4762592315673828,
+      "logps/rejected": -1.5927846431732178,
+      "loss": 0.5744,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.330018162727356,
+      "rewards/margins": 0.5304046869277954,
+      "rewards/rejected": -1.8604227304458618,
       "step": 170
     },
     {
       "epoch": 0.41,
-      "grad_norm": 1143.0994123075566,
+      "grad_norm": 54.867303914998864,
       "learning_rate": 7.349308261002021e-08,
-      "logits/chosen": -1.7149896621704102,
-      "logits/rejected": -1.649047613143921,
-      "logps/chosen": -1.2045981884002686,
-      "logps/rejected": -1.267183780670166,
-      "loss": 0.9333,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.7512127161026001,
-      "rewards/margins": 1.2935186624526978,
-      "rewards/rejected": -2.044731378555298,
+      "logits/chosen": -1.7482887506484985,
+      "logits/rejected": -1.6929025650024414,
+      "logps/chosen": -1.4677311182022095,
+      "logps/rejected": -1.6114797592163086,
+      "loss": 0.5673,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.3532251119613647,
+      "rewards/margins": 0.47049036622047424,
+      "rewards/rejected": -1.8237155675888062,
       "step": 180
     },
     {
       "epoch": 0.43,
-      "grad_norm": 1220.476164637326,
+      "grad_norm": 47.004742188004336,
       "learning_rate": 6.991212490377531e-08,
-      "logits/chosen": -1.7686491012573242,
-      "logits/rejected": -1.7093932628631592,
-      "logps/chosen": -1.2307132482528687,
-      "logps/rejected": -1.2679650783538818,
-      "loss": 0.9238,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.06102483719587326,
-      "rewards/margins": 1.5709805488586426,
-      "rewards/rejected": -1.6320053339004517,
+      "logits/chosen": -1.8027311563491821,
+      "logits/rejected": -1.751875638961792,
+      "logps/chosen": -1.5003658533096313,
+      "logps/rejected": -1.639650583267212,
+      "loss": 0.5419,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.3513133525848389,
+      "rewards/margins": 0.588715136051178,
+      "rewards/rejected": -1.940028429031372,
       "step": 190
     },
     {
       "epoch": 0.46,
-      "grad_norm": 1207.0013256647494,
+      "grad_norm": 60.076327584157355,
       "learning_rate": 6.620527633276978e-08,
-      "logits/chosen": -1.7065632343292236,
-      "logits/rejected": -1.6239337921142578,
-      "logps/chosen": -1.197272777557373,
-      "logps/rejected": -1.302229642868042,
-      "loss": 0.912,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.28195858001708984,
-      "rewards/margins": 1.8168141841888428,
-      "rewards/rejected": -2.0987727642059326,
+      "logits/chosen": -1.742418646812439,
+      "logits/rejected": -1.66952383518219,
+      "logps/chosen": -1.4815881252288818,
+      "logps/rejected": -1.7071993350982666,
+      "loss": 0.5591,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.4356744289398193,
+      "rewards/margins": 0.6941120028495789,
+      "rewards/rejected": -2.129786491394043,
       "step": 200
     },
     {
       "epoch": 0.48,
-      "grad_norm": 1547.2477497808447,
+      "grad_norm": 47.83824182965551,
       "learning_rate": 6.239597278716581e-08,
-      "logits/chosen": -1.776951551437378,
-      "logits/rejected": -1.708361029624939,
-      "logps/chosen": -1.24917471408844,
-      "logps/rejected": -1.246996283531189,
-      "loss": 0.9177,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.38446491956710815,
-      "rewards/margins": 1.8485082387924194,
-      "rewards/rejected": -2.232973098754883,
+      "logits/chosen": -1.8077137470245361,
+      "logits/rejected": -1.7449321746826172,
+      "logps/chosen": -1.5601786375045776,
+      "logps/rejected": -1.6718772649765015,
+      "loss": 0.5475,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.5742429494857788,
+      "rewards/margins": 0.6618110537528992,
+      "rewards/rejected": -2.236053943634033,
       "step": 210
     },
     {
       "epoch": 0.5,
-      "grad_norm": 986.8221787714062,
+      "grad_norm": 42.258080587167875,
       "learning_rate": 5.8508297910462456e-08,
-      "logits/chosen": -1.727354645729065,
-      "logits/rejected": -1.6401188373565674,
-      "logps/chosen": -1.1687428951263428,
-      "logps/rejected": -1.2474868297576904,
-      "loss": 0.8855,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.5227571725845337,
-      "rewards/margins": 2.097059726715088,
-      "rewards/rejected": -2.619816780090332,
+      "logits/chosen": -1.7566890716552734,
+      "logits/rejected": -1.6750905513763428,
+      "logps/chosen": -1.4759745597839355,
+      "logps/rejected": -1.6640676259994507,
+      "loss": 0.5499,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.562295913696289,
+      "rewards/margins": 0.6515985727310181,
+      "rewards/rejected": -2.2138943672180176,
       "step": 220
     },
     {
       "epoch": 0.52,
-      "grad_norm": 1466.4304397519809,
+      "grad_norm": 52.34767465528489,
       "learning_rate": 5.456683083494731e-08,
-      "logits/chosen": -1.7255643606185913,
-      "logits/rejected": -1.6826807260513306,
-      "logps/chosen": -1.2014678716659546,
-      "logps/rejected": -1.2398700714111328,
-      "loss": 0.9518,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.7659143209457397,
-      "rewards/margins": 1.4660184383392334,
-      "rewards/rejected": -2.231933116912842,
+      "logits/chosen": -1.7469978332519531,
+      "logits/rejected": -1.708335518836975,
+      "logps/chosen": -1.4775021076202393,
+      "logps/rejected": -1.5822871923446655,
+      "loss": 0.5699,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.4184671640396118,
+      "rewards/margins": 0.40521565079689026,
+      "rewards/rejected": -1.8236827850341797,
       "step": 230
     },
     {
       "epoch": 0.55,
-      "grad_norm": 1385.4891697791213,
+      "grad_norm": 61.39373208172248,
       "learning_rate": 5.059649078450834e-08,
-      "logits/chosen": -1.7211616039276123,
-      "logits/rejected": -1.6692482233047485,
-      "logps/chosen": -1.1779934167861938,
-      "logps/rejected": -1.2568769454956055,
-      "loss": 0.8955,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.9639942049980164,
-      "rewards/margins": 1.6844803094863892,
-      "rewards/rejected": -2.6484744548797607,
+      "logits/chosen": -1.73995840549469,
+      "logits/rejected": -1.692831039428711,
+      "logps/chosen": -1.4603643417358398,
+      "logps/rejected": -1.6224209070205688,
+      "loss": 0.5397,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.4600539207458496,
+      "rewards/margins": 0.5000897645950317,
+      "rewards/rejected": -1.9601436853408813,
       "step": 240
     },
     {
       "epoch": 0.57,
-      "grad_norm": 1256.1363696264702,
+      "grad_norm": 46.81475246774513,
       "learning_rate": 4.6622379527277186e-08,
-      "logits/chosen": -1.7240123748779297,
-      "logits/rejected": -1.6668732166290283,
-      "logps/chosen": -1.1512458324432373,
-      "logps/rejected": -1.2001845836639404,
-      "loss": 0.8877,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.2357298135757446,
-      "rewards/margins": 1.5858875513076782,
-      "rewards/rejected": -2.821617364883423,
+      "logits/chosen": -1.7453975677490234,
+      "logits/rejected": -1.6935020685195923,
+      "logps/chosen": -1.460304617881775,
+      "logps/rejected": -1.592510461807251,
+      "loss": 0.5459,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.6070802211761475,
+      "rewards/margins": 0.49562931060791016,
+      "rewards/rejected": -2.1027095317840576,
       "step": 250
     },
     {
       "epoch": 0.59,
-      "grad_norm": 882.5182339158957,
+      "grad_norm": 45.05028544056684,
       "learning_rate": 4.26696226741691e-08,
-      "logits/chosen": -1.736823320388794,
-      "logits/rejected": -1.657954216003418,
-      "logps/chosen": -1.244571328163147,
-      "logps/rejected": -1.2810423374176025,
-      "loss": 0.8757,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.174777626991272,
-      "rewards/margins": 1.6504262685775757,
-      "rewards/rejected": -2.8252038955688477,
+      "logits/chosen": -1.760898232460022,
+      "logits/rejected": -1.6884838342666626,
+      "logps/chosen": -1.5728627443313599,
+      "logps/rejected": -1.7121139764785767,
+      "loss": 0.5508,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.7001960277557373,
+      "rewards/margins": 0.5964212417602539,
+      "rewards/rejected": -2.2966175079345703,
       "step": 260
     },
     {
       "epoch": 0.61,
-      "grad_norm": 927.5282198039703,
+      "grad_norm": 45.62774521822856,
       "learning_rate": 3.876321082668098e-08,
-      "logits/chosen": -1.8001619577407837,
-      "logits/rejected": -1.7206510305404663,
-      "logps/chosen": -1.2467237710952759,
-      "logps/rejected": -1.272040843963623,
-      "loss": 0.8981,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.8233057856559753,
-      "rewards/margins": 1.6008100509643555,
-      "rewards/rejected": -2.4241158962249756,
+      "logits/chosen": -1.8219407796859741,
+      "logits/rejected": -1.7494175434112549,
+      "logps/chosen": -1.5225193500518799,
+      "logps/rejected": -1.6650638580322266,
+      "loss": 0.5391,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.4201425313949585,
+      "rewards/margins": 0.6661779284477234,
+      "rewards/rejected": -2.086320638656616,
       "step": 270
     },
     {
       "epoch": 0.64,
-      "grad_norm": 1414.5359647496766,
+      "grad_norm": 49.38048288534598,
       "learning_rate": 3.492784157826244e-08,
-      "logits/chosen": -1.7251765727996826,
-      "logits/rejected": -1.6225860118865967,
-      "logps/chosen": -1.2284091711044312,
-      "logps/rejected": -1.2839155197143555,
-      "loss": 0.8929,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.5533002018928528,
-      "rewards/margins": 2.0116381645202637,
-      "rewards/rejected": -2.5649383068084717,
+      "logits/chosen": -1.7487083673477173,
+      "logits/rejected": -1.6498956680297852,
+      "logps/chosen": -1.5093038082122803,
+      "logps/rejected": -1.6828804016113281,
+      "loss": 0.5459,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.4321379661560059,
+      "rewards/margins": 0.6909336447715759,
+      "rewards/rejected": -2.1230716705322266,
       "step": 280
     },
     {
       "epoch": 0.66,
-      "grad_norm": 1016.8878069934569,
+      "grad_norm": 39.80249587523834,
       "learning_rate": 3.118776336817812e-08,
-      "logits/chosen": -1.777353048324585,
-      "logits/rejected": -1.704052209854126,
-      "logps/chosen": -1.2131690979003906,
-      "logps/rejected": -1.2390577793121338,
-      "loss": 0.8692,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -0.992035984992981,
-      "rewards/margins": 1.8672508001327515,
-      "rewards/rejected": -2.8592867851257324,
+      "logits/chosen": -1.7935447692871094,
+      "logits/rejected": -1.7284294366836548,
+      "logps/chosen": -1.4926804304122925,
+      "logps/rejected": -1.641869306564331,
+      "loss": 0.5366,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.4471591711044312,
+      "rewards/margins": 0.7098628282546997,
+      "rewards/rejected": -2.157021999359131,
       "step": 290
     },
     {
       "epoch": 0.68,
-      "grad_norm": 1172.1973055026576,
+      "grad_norm": 45.964008321418135,
       "learning_rate": 2.7566622175067443e-08,
-      "logits/chosen": -1.75467050075531,
-      "logits/rejected": -1.6821622848510742,
-      "logps/chosen": -1.2167221307754517,
-      "logps/rejected": -1.312439203262329,
-      "loss": 0.9238,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.8138607144355774,
-      "rewards/margins": 1.7730525732040405,
-      "rewards/rejected": -2.5869133472442627,
+      "logits/chosen": -1.776922583580017,
+      "logits/rejected": -1.7095201015472412,
+      "logps/chosen": -1.5357799530029297,
+      "logps/rejected": -1.7406812906265259,
+      "loss": 0.5452,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.6359821557998657,
+      "rewards/margins": 0.6345733404159546,
+      "rewards/rejected": -2.2705554962158203,
       "step": 300
     },
     {
       "epoch": 0.71,
-      "grad_norm": 1325.0572604746271,
+      "grad_norm": 57.60996565680647,
       "learning_rate": 2.408731201945432e-08,
-      "logits/chosen": -1.7551631927490234,
-      "logits/rejected": -1.7048372030258179,
-      "logps/chosen": -1.1999702453613281,
-      "logps/rejected": -1.2347406148910522,
-      "loss": 0.9048,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.6074885725975037,
-      "rewards/margins": 1.4615652561187744,
-      "rewards/rejected": -2.0690536499023438,
+      "logits/chosen": -1.7720359563827515,
+      "logits/rejected": -1.7233737707138062,
+      "logps/chosen": -1.4881327152252197,
+      "logps/rejected": -1.6220896244049072,
+      "loss": 0.5332,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.4711856842041016,
+      "rewards/margins": 0.5690120458602905,
+      "rewards/rejected": -2.0401976108551025,
       "step": 310
     },
     {
       "epoch": 0.73,
-      "grad_norm": 1307.0038735408154,
+      "grad_norm": 47.81844840381173,
       "learning_rate": 2.0771830220378112e-08,
-      "logits/chosen": -1.7086585760116577,
-      "logits/rejected": -1.6484758853912354,
-      "logps/chosen": -1.2315673828125,
-      "logps/rejected": -1.2379032373428345,
-      "loss": 0.8568,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.8682917356491089,
-      "rewards/margins": 1.418613314628601,
-      "rewards/rejected": -2.28690505027771,
+      "logits/chosen": -1.7288305759429932,
+      "logits/rejected": -1.6700308322906494,
+      "logps/chosen": -1.5263229608535767,
+      "logps/rejected": -1.6329742670059204,
+      "loss": 0.5383,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.5171924829483032,
+      "rewards/margins": 0.5725088715553284,
+      "rewards/rejected": -2.0897011756896973,
       "step": 320
     },
     {
       "epoch": 0.75,
-      "grad_norm": 1081.044775563757,
+      "grad_norm": 42.404801937182704,
       "learning_rate": 1.7641138321260257e-08,
-      "logits/chosen": -1.744683861732483,
-      "logits/rejected": -1.6652017831802368,
-      "logps/chosen": -1.1902309656143188,
-      "logps/rejected": -1.2386529445648193,
-      "loss": 0.8653,
+      "logits/chosen": -1.7647384405136108,
+      "logits/rejected": -1.6919339895248413,
+      "logps/chosen": -1.4824151992797852,
+      "logps/rejected": -1.6646735668182373,
+      "loss": 0.5341,
       "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -0.6376137733459473,
-      "rewards/margins": 2.0886900424957275,
-      "rewards/rejected": -2.726303815841675,
+      "rewards/chosen": -1.4928021430969238,
+      "rewards/margins": 0.7736164927482605,
+      "rewards/rejected": -2.266418695449829,
       "step": 330
     },
     {
       "epoch": 0.77,
-      "grad_norm": 981.6058112333443,
+      "grad_norm": 41.58228712777898,
       "learning_rate": 1.4715029564277793e-08,
-      "logits/chosen": -1.8191375732421875,
-      "logits/rejected": -1.7604516744613647,
-      "logps/chosen": -1.2077248096466064,
-      "logps/rejected": -1.262688398361206,
-      "loss": 0.9084,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.30642908811569214,
-      "rewards/margins": 1.7900102138519287,
-      "rewards/rejected": -2.0964391231536865,
+      "logits/chosen": -1.833791971206665,
+      "logits/rejected": -1.7838408946990967,
+      "logps/chosen": -1.4812287092208862,
+      "logps/rejected": -1.657918930053711,
+      "loss": 0.5469,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.3828407526016235,
+      "rewards/margins": 0.6981338262557983,
+      "rewards/rejected": -2.080974578857422,
       "step": 340
     },
     {
       "epoch": 0.8,
-      "grad_norm": 1453.9865513146688,
+      "grad_norm": 50.79187416231153,
       "learning_rate": 1.2012003751113343e-08,
-      "logits/chosen": -1.7971642017364502,
-      "logits/rejected": -1.7289268970489502,
-      "logps/chosen": -1.171277642250061,
-      "logps/rejected": -1.2399911880493164,
-      "loss": 0.8408,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.9107818603515625,
-      "rewards/margins": 1.8522905111312866,
-      "rewards/rejected": -2.7630724906921387,
+      "logits/chosen": -1.8199939727783203,
+      "logits/rejected": -1.7557872533798218,
+      "logps/chosen": -1.4952068328857422,
+      "logps/rejected": -1.6920936107635498,
+      "loss": 0.5205,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.6651843786239624,
+      "rewards/margins": 0.7334816455841064,
+      "rewards/rejected": -2.3986661434173584,
       "step": 350
     },
     {
       "epoch": 0.82,
-      "grad_norm": 1215.5568858398267,
+      "grad_norm": 46.03762157927165,
       "learning_rate": 9.549150281252633e-09,
-      "logits/chosen": -1.7647504806518555,
-      "logits/rejected": -1.7033681869506836,
-      "logps/chosen": -1.1781424283981323,
-      "logps/rejected": -1.246336579322815,
-      "loss": 0.8691,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.6007339358329773,
-      "rewards/margins": 1.6463441848754883,
-      "rewards/rejected": -2.2470781803131104,
+      "logits/chosen": -1.7812334299087524,
+      "logits/rejected": -1.7282968759536743,
+      "logps/chosen": -1.479620337486267,
+      "logps/rejected": -1.663819670677185,
+      "loss": 0.5227,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5374261140823364,
+      "rewards/margins": 0.6623435020446777,
+      "rewards/rejected": -2.1997694969177246,
       "step": 360
     },
     {
       "epoch": 0.84,
-      "grad_norm": 989.7203519251033,
+      "grad_norm": 42.572961776589516,
       "learning_rate": 7.3420401072985306e-09,
-      "logits/chosen": -1.7938206195831299,
-      "logits/rejected": -1.7368061542510986,
-      "logps/chosen": -1.2212886810302734,
-      "logps/rejected": -1.2962526082992554,
-      "loss": 0.8304,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.6675101518630981,
-      "rewards/margins": 1.7500286102294922,
-      "rewards/rejected": -2.41753888130188,
+      "logits/chosen": -1.8132097721099854,
+      "logits/rejected": -1.7593128681182861,
+      "logps/chosen": -1.5283488035202026,
+      "logps/rejected": -1.730432152748108,
+      "loss": 0.5242,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.5686765909194946,
+      "rewards/margins": 0.7230969667434692,
+      "rewards/rejected": -2.291773557662964,
       "step": 370
     },
     {
       "epoch": 0.86,
-      "grad_norm": 1103.1154426715473,
+      "grad_norm": 52.728611883061056,
       "learning_rate": 5.404627290395369e-09,
-      "logits/chosen": -1.7590833902359009,
-      "logits/rejected": -1.6879241466522217,
-      "logps/chosen": -1.1950905323028564,
-      "logps/rejected": -1.2529264688491821,
-      "loss": 0.8354,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.5484554171562195,
-      "rewards/margins": 1.7839889526367188,
-      "rewards/rejected": -2.332444429397583,
+      "logits/chosen": -1.7789500951766968,
+      "logits/rejected": -1.7105478048324585,
+      "logps/chosen": -1.4771665334701538,
+      "logps/rejected": -1.659226417541504,
+      "loss": 0.5267,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.437803864479065,
+      "rewards/margins": 0.710318386554718,
+      "rewards/rejected": -2.1481223106384277,
       "step": 380
     },
     {
       "epoch": 0.89,
-      "grad_norm": 1735.4553729147015,
+      "grad_norm": 69.05994145197278,
       "learning_rate": 3.74916077816162e-09,
-      "logits/chosen": -1.7692234516143799,
-      "logits/rejected": -1.7004655599594116,
-      "logps/chosen": -1.1599231958389282,
-      "logps/rejected": -1.1907106637954712,
-      "loss": 0.8962,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.0128358602523804,
-      "rewards/margins": 1.4658567905426025,
-      "rewards/rejected": -2.4786927700042725,
+      "logits/chosen": -1.7877365350723267,
+      "logits/rejected": -1.7246090173721313,
+      "logps/chosen": -1.4689042568206787,
+      "logps/rejected": -1.6005665063858032,
+      "loss": 0.5424,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5955479145050049,
+      "rewards/margins": 0.5776661038398743,
+      "rewards/rejected": -2.1732141971588135,
       "step": 390
     },
     {
       "epoch": 0.91,
-      "grad_norm": 1092.0703383397774,
+      "grad_norm": 47.34880971893926,
       "learning_rate": 2.386106962899165e-09,
-      "logits/chosen": -1.6911523342132568,
-      "logits/rejected": -1.6125024557113647,
-      "logps/chosen": -1.233060359954834,
-      "logps/rejected": -1.3061407804489136,
-      "loss": 0.8745,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.6639580130577087,
-      "rewards/margins": 1.6740585565567017,
-      "rewards/rejected": -2.3380167484283447,
+      "logits/chosen": -1.7117360830307007,
+      "logits/rejected": -1.6371567249298096,
+      "logps/chosen": -1.5578153133392334,
+      "logps/rejected": -1.7342666387557983,
+      "loss": 0.5379,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.6569716930389404,
+      "rewards/margins": 0.6005586981773376,
+      "rewards/rejected": -2.257530450820923,
       "step": 400
     },
     {
       "epoch": 0.93,
-      "grad_norm": 1112.8116007670076,
+      "grad_norm": 50.11568741978605,
       "learning_rate": 1.3240835096913706e-09,
-      "logits/chosen": -1.7251644134521484,
-      "logits/rejected": -1.613480567932129,
-      "logps/chosen": -1.1751197576522827,
-      "logps/rejected": -1.2636477947235107,
-      "loss": 0.9325,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.35689860582351685,
-      "rewards/margins": 1.9113714694976807,
-      "rewards/rejected": -2.2682700157165527,
+      "logits/chosen": -1.746082067489624,
+      "logits/rejected": -1.6423866748809814,
+      "logps/chosen": -1.4619499444961548,
+      "logps/rejected": -1.6878178119659424,
+      "loss": 0.5428,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.451996088027954,
+      "rewards/margins": 0.7822669744491577,
+      "rewards/rejected": -2.2342631816864014,
       "step": 410
     },
     {
       "epoch": 0.96,
-      "grad_norm": 1072.5150365604325,
+      "grad_norm": 102.373067174958,
       "learning_rate": 5.698048727497462e-10,
-      "logits/chosen": -1.7486751079559326,
-      "logits/rejected": -1.6716489791870117,
-      "logps/chosen": -1.198484182357788,
-      "logps/rejected": -1.2844183444976807,
-      "loss": 0.8652,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.7186421155929565,
-      "rewards/margins": 1.897702932357788,
-      "rewards/rejected": -2.616345167160034,
+      "logits/chosen": -1.769753098487854,
+      "logits/rejected": -1.696448564529419,
+      "logps/chosen": -1.5222785472869873,
+      "logps/rejected": -1.7370811700820923,
+      "loss": 0.5332,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.6549043655395508,
+      "rewards/margins": 0.7392269968986511,
+      "rewards/rejected": -2.3941311836242676,
       "step": 420
     },
     {
       "epoch": 0.98,
-      "grad_norm": 1007.4274803164712,
+      "grad_norm": 44.58592035367784,
       "learning_rate": 1.2803984447259387e-10,
-      "logits/chosen": -1.756765365600586,
-      "logits/rejected": -1.6918004751205444,
-      "logps/chosen": -1.189513921737671,
-      "logps/rejected": -1.2563507556915283,
-      "loss": 0.8416,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.7174621820449829,
-      "rewards/margins": 1.9183887243270874,
-      "rewards/rejected": -2.635850667953491,
+      "logits/chosen": -1.781581163406372,
+      "logits/rejected": -1.724043846130371,
+      "logps/chosen": -1.5100823640823364,
+      "logps/rejected": -1.7025842666625977,
+      "loss": 0.5259,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.638715386390686,
+      "rewards/margins": 0.7242453694343567,
+      "rewards/rejected": -2.3629605770111084,
       "step": 430
     },
     {
       "epoch": 1.0,
       "step": 439,
       "total_flos": 0.0,
-      "train_loss": 0.9518976428785737,
-      "train_runtime": 6868.0311,
-      "train_samples_per_second": 8.188,
-      "train_steps_per_second": 0.064
+      "train_loss": 0.4228534948309896,
+      "train_runtime": 5290.3981,
+      "train_samples_per_second": 10.63,
+      "train_steps_per_second": 0.083
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0716202a831bf05f44c63f92c313ac4b7bae8886fe6e5852b8e205d38a7569ae
+oid sha256:9a53011c0d49d2ed2a349f4e82d07898fc20dfe797eca2ac6cb079ef749dab9f
 size 6328