Model save

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5309282744935542,
-    "train_runtime": 6847.998,
+    "train_loss": 0.048529281703106095,
+    "train_runtime": 584.5423,
     "train_samples": 56236,
-    "train_samples_per_second": 8.212,
-    "train_steps_per_second": 0.064
+    "train_samples_per_second": 96.205,
+    "train_steps_per_second": 0.751
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8d089e259179f6b43c527a368edf999eec073dc01cfa73c90adde6188d1928d
+oid sha256:0f0917430ce074f3dea083f0ccf860b56e464900291bf59b18d535777662e5e2
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:72900a606cc1d8493faaa0946c473002a0400edf1f17c01905fe2a2232dc1be1
+oid sha256:767ab10c5d7cbffb20427d5ee7b697153038525334a0025d3e17b6ef308adea3
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2cba821d53166885228308afe33150d7a837fce90e03432661986a37f3edccc4
+oid sha256:232f07d779218e892df908602977cfff9f29af01717e280a568b132d80e0d8ca
 size 4540516344

runs/Jul03_22-30-38_n136-100-194/events.out.tfevents.1720017061.n136-100-194.2581213.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:84e374d8af7d1ff5019a418fa1c74da29fe332ec309636b8089659245d802e01
-size 32937
+oid sha256:12eae65fc3ec014f21eb022b7de59188ca9d4cf11fd874531a4aea696d214bd4
+size 35355

runs/Jul04_01-04-27_n136-100-194/events.out.tfevents.1720026338.n136-100-194.2674710.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac4ddd2b80ca04ccda3cebb77135c7a5a86df33683f8c62a32bdc09d8b97f350
+size 7303

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5309282744935542,
-    "train_runtime": 6847.998,
+    "train_loss": 0.048529281703106095,
+    "train_runtime": 584.5423,
     "train_samples": 56236,
-    "train_samples_per_second": 8.212,
-    "train_steps_per_second": 0.064
+    "train_samples_per_second": 96.205,
+    "train_steps_per_second": 0.751
 }

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 120.07926705665244,
+      "grad_norm": 23.98402260612519,
       "learning_rate": 2.2727272727272727e-09,
       "logits/chosen": -1.6768856048583984,
       "logits/rejected": -1.7259055376052856,
       "logps/chosen": -394.9654541015625,
       "logps/rejected": -320.0859069824219,
-      "loss": 0.6931,
+      "loss": 0.693,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,657 +25,657 @@
     },
     {
       "epoch": 0.02,
-      "grad_norm": 160.8036403422587,
+      "grad_norm": 31.95891283237601,
       "learning_rate": 2.2727272727272725e-08,
-      "logits/chosen": -1.703018069267273,
-      "logits/rejected": -1.6685585975646973,
-      "logps/chosen": -429.5360412597656,
-      "logps/rejected": -403.7555236816406,
-      "loss": 0.6934,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.0004270696663297713,
-      "rewards/margins": -0.0031983989756554365,
-      "rewards/rejected": 0.00277132960036397,
+      "logits/chosen": -1.7029528617858887,
+      "logits/rejected": -1.6683764457702637,
+      "logps/chosen": -429.5246887207031,
+      "logps/rejected": -403.747314453125,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4305555522441864,
+      "rewards/chosen": 2.8328322514425963e-05,
+      "rewards/margins": -0.0006085141212679446,
+      "rewards/rejected": 0.0006368425092659891,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 241.66358094434165,
+      "grad_norm": 49.89152060792663,
       "learning_rate": 4.545454545454545e-08,
-      "logits/chosen": -1.780827283859253,
-      "logits/rejected": -1.7355620861053467,
-      "logps/chosen": -442.1951599121094,
-      "logps/rejected": -401.2707214355469,
-      "loss": 0.6889,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.002479883376508951,
-      "rewards/margins": 0.002258532214909792,
-      "rewards/rejected": -0.004738415591418743,
+      "logits/chosen": -1.7806730270385742,
+      "logits/rejected": -1.7358741760253906,
+      "logps/chosen": -442.21636962890625,
+      "logps/rejected": -401.44000244140625,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.0007078830385580659,
+      "rewards/margins": 0.0019328873604536057,
+      "rewards/rejected": -0.0026407705154269934,
       "step": 20
     },
     {
       "epoch": 0.07,
-      "grad_norm": 160.69914583171965,
+      "grad_norm": 35.37044354910807,
       "learning_rate": 6.818181818181817e-08,
-      "logits/chosen": -1.7514013051986694,
-      "logits/rejected": -1.6820430755615234,
-      "logps/chosen": -439.64373779296875,
-      "logps/rejected": -401.2854919433594,
-      "loss": 0.6738,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.0074474625289440155,
-      "rewards/margins": 0.054817844182252884,
-      "rewards/rejected": -0.04737037047743797,
+      "logits/chosen": -1.751228928565979,
+      "logits/rejected": -1.6827186346054077,
+      "logps/chosen": -440.0089416503906,
+      "logps/rejected": -401.7633361816406,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.002162130083888769,
+      "rewards/margins": 0.012089937925338745,
+      "rewards/rejected": -0.014252068474888802,
       "step": 30
     },
     {
       "epoch": 0.09,
-      "grad_norm": 144.27361385062915,
+      "grad_norm": 24.433299552566275,
       "learning_rate": 9.09090909090909e-08,
-      "logits/chosen": -1.7552951574325562,
-      "logits/rejected": -1.6888301372528076,
-      "logps/chosen": -428.416015625,
-      "logps/rejected": -383.068603515625,
-      "loss": 0.6579,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.1923234909772873,
-      "rewards/margins": 0.1899619996547699,
-      "rewards/rejected": 0.0023614875972270966,
+      "logits/chosen": -1.7572921514511108,
+      "logits/rejected": -1.6904194355010986,
+      "logps/chosen": -431.66802978515625,
+      "logps/rejected": -390.01898193359375,
+      "loss": 0.6773,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.00594430323690176,
+      "rewards/margins": 0.07497542351484299,
+      "rewards/rejected": -0.06903111934661865,
       "step": 40
     },
     {
       "epoch": 0.11,
-      "grad_norm": 115.83588190821067,
+      "grad_norm": 31.92529179185726,
       "learning_rate": 9.994307990108962e-08,
-      "logits/chosen": -1.7498763799667358,
-      "logits/rejected": -1.6867773532867432,
-      "logps/chosen": -437.7354431152344,
-      "logps/rejected": -383.8248596191406,
-      "loss": 0.6152,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.5490495562553406,
-      "rewards/margins": 0.33619847893714905,
-      "rewards/rejected": 0.2128511667251587,
+      "logits/chosen": -1.7621084451675415,
+      "logits/rejected": -1.6976591348648071,
+      "logps/chosen": -444.54913330078125,
+      "logps/rejected": -393.5503234863281,
+      "loss": 0.6599,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": 0.04167298600077629,
+      "rewards/margins": 0.09635747224092484,
+      "rewards/rejected": -0.05468447878956795,
       "step": 50
     },
     {
       "epoch": 0.14,
-      "grad_norm": 111.21906155896082,
+      "grad_norm": 28.704756119671675,
       "learning_rate": 9.959570405988094e-08,
-      "logits/chosen": -1.8246532678604126,
-      "logits/rejected": -1.7464463710784912,
-      "logps/chosen": -379.9985046386719,
-      "logps/rejected": -352.2860412597656,
-      "loss": 0.614,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": 0.7562192678451538,
-      "rewards/margins": 0.2702825665473938,
-      "rewards/rejected": 0.4859367311000824,
+      "logits/chosen": -1.8457626104354858,
+      "logits/rejected": -1.7673368453979492,
+      "logps/chosen": -385.0513916015625,
+      "logps/rejected": -358.7567443847656,
+      "loss": 0.6566,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.10071470588445663,
+      "rewards/margins": 0.06823419034481049,
+      "rewards/rejected": 0.032480526715517044,
       "step": 60
     },
     {
       "epoch": 0.16,
-      "grad_norm": 114.01926694263507,
+      "grad_norm": 24.531377945654775,
       "learning_rate": 9.893476820924666e-08,
-      "logits/chosen": -1.9211199283599854,
-      "logits/rejected": -1.8387491703033447,
-      "logps/chosen": -412.0660095214844,
-      "logps/rejected": -374.1966247558594,
-      "loss": 0.5841,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 1.0015352964401245,
-      "rewards/margins": 0.3862503468990326,
-      "rewards/rejected": 0.6152850389480591,
+      "logits/chosen": -1.9651501178741455,
+      "logits/rejected": -1.8813607692718506,
+      "logps/chosen": -403.64105224609375,
+      "logps/rejected": -371.46234130859375,
+      "loss": 0.6395,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.2845568060874939,
+      "rewards/margins": 0.13415710628032684,
+      "rewards/rejected": 0.15039967000484467,
       "step": 70
     },
     {
       "epoch": 0.18,
-      "grad_norm": 143.95597429946028,
+      "grad_norm": 27.08910191775396,
       "learning_rate": 9.796445099843647e-08,
-      "logits/chosen": -1.9127334356307983,
-      "logits/rejected": -1.8274517059326172,
-      "logps/chosen": -424.8004455566406,
-      "logps/rejected": -390.2203063964844,
-      "loss": 0.6144,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.9376031160354614,
-      "rewards/margins": 0.4474514126777649,
-      "rewards/rejected": 0.4901517331600189,
+      "logits/chosen": -2.0368916988372803,
+      "logits/rejected": -1.9531713724136353,
+      "logps/chosen": -407.11553955078125,
+      "logps/rejected": -381.14794921875,
+      "loss": 0.6473,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.3643694519996643,
+      "rewards/margins": 0.1756153404712677,
+      "rewards/rejected": 0.18875406682491302,
       "step": 80
     },
     {
       "epoch": 0.2,
-      "grad_norm": 126.42198014014429,
+      "grad_norm": 38.20992212526602,
       "learning_rate": 9.669088708527066e-08,
-      "logits/chosen": -1.8573029041290283,
-      "logits/rejected": -1.7926852703094482,
-      "logps/chosen": -435.7537536621094,
-      "logps/rejected": -401.1055908203125,
-      "loss": 0.5739,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": 0.8144777417182922,
-      "rewards/margins": 0.4378415048122406,
-      "rewards/rejected": 0.376636266708374,
+      "logits/chosen": -2.025817394256592,
+      "logits/rejected": -1.9624574184417725,
+      "logps/chosen": -415.40020751953125,
+      "logps/rejected": -390.00140380859375,
+      "loss": 0.6291,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": 0.36643102765083313,
+      "rewards/margins": 0.1800619214773178,
+      "rewards/rejected": 0.1863691359758377,
       "step": 90
     },
     {
       "epoch": 0.23,
-      "grad_norm": 132.16216232500156,
+      "grad_norm": 27.439269349288335,
       "learning_rate": 9.512212835085849e-08,
-      "logits/chosen": -1.898048758506775,
-      "logits/rejected": -1.813433051109314,
-      "logps/chosen": -413.4141540527344,
-      "logps/rejected": -398.70526123046875,
-      "loss": 0.5603,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": 0.6169204115867615,
-      "rewards/margins": 0.5242881178855896,
-      "rewards/rejected": 0.09263229370117188,
+      "logits/chosen": -2.1013572216033936,
+      "logits/rejected": -2.0152974128723145,
+      "logps/chosen": -388.7185974121094,
+      "logps/rejected": -383.03546142578125,
+      "loss": 0.6197,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.37033963203430176,
+      "rewards/margins": 0.19511529803276062,
+      "rewards/rejected": 0.17522430419921875,
       "step": 100
     },
     {
       "epoch": 0.25,
-      "grad_norm": 112.13374239362014,
+      "grad_norm": 26.198395783666243,
       "learning_rate": 9.326809299301306e-08,
-      "logits/chosen": -1.89919114112854,
-      "logits/rejected": -1.7970205545425415,
-      "logps/chosen": -454.9283142089844,
-      "logps/rejected": -405.9847717285156,
-      "loss": 0.5581,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": 0.4878689646720886,
-      "rewards/margins": 0.6205599308013916,
-      "rewards/rejected": -0.1326909363269806,
+      "logits/chosen": -2.1095404624938965,
+      "logits/rejected": -2.0083327293395996,
+      "logps/chosen": -427.6131896972656,
+      "logps/rejected": -393.1694030761719,
+      "loss": 0.6112,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.37072473764419556,
+      "rewards/margins": 0.26910915970802307,
+      "rewards/rejected": 0.10161559283733368,
       "step": 110
     },
     {
       "epoch": 0.27,
-      "grad_norm": 113.80543806203703,
+      "grad_norm": 26.3105476660504,
       "learning_rate": 9.114050282021158e-08,
-      "logits/chosen": -1.8961639404296875,
-      "logits/rejected": -1.836488127708435,
-      "logps/chosen": -453.51898193359375,
-      "logps/rejected": -419.9806213378906,
-      "loss": 0.5372,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.4265638291835785,
-      "rewards/margins": 0.6399323344230652,
-      "rewards/rejected": -0.21336853504180908,
+      "logits/chosen": -2.1161999702453613,
+      "logits/rejected": -2.05537748336792,
+      "logps/chosen": -435.20086669921875,
+      "logps/rejected": -412.62078857421875,
+      "loss": 0.607,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.268494188785553,
+      "rewards/margins": 0.23757004737854004,
+      "rewards/rejected": 0.030924171209335327,
       "step": 120
     },
     {
       "epoch": 0.3,
-      "grad_norm": 122.38937390797612,
+      "grad_norm": 27.752798140691322,
       "learning_rate": 8.875280914254802e-08,
-      "logits/chosen": -1.9070549011230469,
-      "logits/rejected": -1.8174508810043335,
-      "logps/chosen": -408.0248107910156,
-      "logps/rejected": -365.46185302734375,
-      "loss": 0.5379,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.505743145942688,
-      "rewards/margins": 0.6783953309059143,
-      "rewards/rejected": -0.17265217006206512,
+      "logits/chosen": -2.1366939544677734,
+      "logits/rejected": -2.0489516258239746,
+      "logps/chosen": -392.9619140625,
+      "logps/rejected": -363.98553466796875,
+      "loss": 0.602,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.2517772316932678,
+      "rewards/margins": 0.2715442478656769,
+      "rewards/rejected": -0.01976701058447361,
       "step": 130
     },
     {
       "epoch": 0.32,
-      "grad_norm": 104.81942476289146,
+      "grad_norm": 24.888665971199188,
       "learning_rate": 8.612010772821971e-08,
-      "logits/chosen": -1.9492871761322021,
-      "logits/rejected": -1.903029203414917,
-      "logps/chosen": -463.1678771972656,
-      "logps/rejected": -408.36138916015625,
-      "loss": 0.5415,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": 0.7421985268592834,
-      "rewards/margins": 0.7196205854415894,
-      "rewards/rejected": 0.02257799357175827,
+      "logits/chosen": -2.180723190307617,
+      "logits/rejected": -2.1342811584472656,
+      "logps/chosen": -445.15472412109375,
+      "logps/rejected": -407.91998291015625,
+      "loss": 0.6048,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.32857105135917664,
+      "rewards/margins": 0.3196411728858948,
+      "rewards/rejected": 0.008929857984185219,
       "step": 140
     },
     {
       "epoch": 0.34,
-      "grad_norm": 130.65997070751789,
+      "grad_norm": 27.895144442719968,
       "learning_rate": 8.325904336322055e-08,
-      "logits/chosen": -1.9297358989715576,
-      "logits/rejected": -1.872240424156189,
-      "logps/chosen": -408.3719482421875,
-      "logps/rejected": -374.2057189941406,
-      "loss": 0.5461,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.6860557198524475,
-      "rewards/margins": 0.6311514973640442,
-      "rewards/rejected": 0.05490417033433914,
+      "logits/chosen": -2.172715663909912,
+      "logits/rejected": -2.112837314605713,
+      "logps/chosen": -402.29217529296875,
+      "logps/rejected": -380.25335693359375,
+      "loss": 0.6071,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.1980087012052536,
+      "rewards/margins": 0.2475043088197708,
+      "rewards/rejected": -0.049495600163936615,
       "step": 150
     },
     {
       "epoch": 0.36,
-      "grad_norm": 100.26257755554322,
+      "grad_norm": 24.760628369907216,
       "learning_rate": 8.01877046176447e-08,
-      "logits/chosen": -1.8793761730194092,
-      "logits/rejected": -1.8072710037231445,
-      "logps/chosen": -399.7383728027344,
-      "logps/rejected": -373.10791015625,
-      "loss": 0.5278,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": 0.4368124008178711,
-      "rewards/margins": 0.6604553461074829,
-      "rewards/rejected": -0.22364301979541779,
+      "logits/chosen": -2.131298303604126,
+      "logits/rejected": -2.059814453125,
+      "logps/chosen": -396.01788330078125,
+      "logps/rejected": -379.63665771484375,
+      "loss": 0.5888,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.12456746399402618,
+      "rewards/margins": 0.23458366096019745,
+      "rewards/rejected": -0.11001620441675186,
       "step": 160
     },
     {
       "epoch": 0.39,
-      "grad_norm": 89.3633959368459,
+      "grad_norm": 22.13569267518615,
       "learning_rate": 7.692550948392249e-08,
-      "logits/chosen": -1.9449889659881592,
-      "logits/rejected": -1.8793401718139648,
-      "logps/chosen": -430.8158264160156,
-      "logps/rejected": -383.00640869140625,
-      "loss": 0.5275,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.5035308599472046,
-      "rewards/margins": 0.5975342988967896,
-      "rewards/rejected": -0.09400341659784317,
+      "logits/chosen": -2.1969549655914307,
+      "logits/rejected": -2.1351304054260254,
+      "logps/chosen": -421.291015625,
+      "logps/rejected": -387.77484130859375,
+      "loss": 0.589,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.19595439732074738,
+      "rewards/margins": 0.26243916153907776,
+      "rewards/rejected": -0.06648479402065277,
       "step": 170
     },
     {
       "epoch": 0.41,
-      "grad_norm": 103.70599270965509,
+      "grad_norm": 29.37383088378205,
       "learning_rate": 7.349308261002021e-08,
-      "logits/chosen": -1.9056031703948975,
-      "logits/rejected": -1.8477048873901367,
-      "logps/chosen": -437.502197265625,
-      "logps/rejected": -403.87762451171875,
-      "loss": 0.526,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.6943656206130981,
-      "rewards/margins": 0.6242468357086182,
-      "rewards/rejected": 0.07011876255273819,
+      "logits/chosen": -2.1651830673217773,
+      "logits/rejected": -2.1070454120635986,
+      "logps/chosen": -426.6806640625,
+      "logps/rejected": -408.07781982421875,
+      "loss": 0.5855,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.2470887452363968,
+      "rewards/margins": 0.2750667631626129,
+      "rewards/rejected": -0.02797803282737732,
       "step": 180
     },
     {
       "epoch": 0.43,
-      "grad_norm": 84.20282962041543,
+      "grad_norm": 29.642029984354075,
       "learning_rate": 6.991212490377531e-08,
-      "logits/chosen": -1.9574388265609741,
-      "logits/rejected": -1.906002402305603,
-      "logps/chosen": -464.6742248535156,
-      "logps/rejected": -417.5992126464844,
-      "loss": 0.5064,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.6871397495269775,
-      "rewards/margins": 0.8371666669845581,
-      "rewards/rejected": -0.15002694725990295,
+      "logits/chosen": -2.239077091217041,
+      "logits/rejected": -2.188472270965576,
+      "logps/chosen": -460.720947265625,
+      "logps/rejected": -433.29608154296875,
+      "loss": 0.58,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.17696049809455872,
+      "rewards/margins": 0.3639344274997711,
+      "rewards/rejected": -0.1869739592075348,
       "step": 190
     },
     {
       "epoch": 0.46,
-      "grad_norm": 166.8394060310626,
+      "grad_norm": 40.17804362794063,
       "learning_rate": 6.620527633276978e-08,
-      "logits/chosen": -1.8844044208526611,
-      "logits/rejected": -1.8095699548721313,
-      "logps/chosen": -425.5460510253906,
-      "logps/rejected": -407.5855712890625,
-      "loss": 0.507,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.5698906779289246,
-      "rewards/margins": 0.8980382680892944,
-      "rewards/rejected": -0.3281475901603699,
+      "logits/chosen": -2.1767477989196777,
+      "logits/rejected": -2.1026599407196045,
+      "logps/chosen": -425.94677734375,
+      "logps/rejected": -428.23846435546875,
+      "loss": 0.5857,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.10997031629085541,
+      "rewards/margins": 0.38212892413139343,
+      "rewards/rejected": -0.2721586227416992,
       "step": 200
     },
     {
       "epoch": 0.48,
-      "grad_norm": 100.6332916667586,
+      "grad_norm": 23.44190889972074,
       "learning_rate": 6.239597278716581e-08,
-      "logits/chosen": -1.9531447887420654,
-      "logits/rejected": -1.8929874897003174,
-      "logps/chosen": -414.9901428222656,
-      "logps/rejected": -375.222412109375,
-      "loss": 0.4997,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 0.5068634748458862,
-      "rewards/margins": 0.9239821434020996,
-      "rewards/rejected": -0.41711869835853577,
+      "logits/chosen": -2.2649803161621094,
+      "logits/rejected": -2.202864646911621,
+      "logps/chosen": -410.3318786621094,
+      "logps/rejected": -393.8056640625,
+      "loss": 0.5721,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.14795458316802979,
+      "rewards/margins": 0.41721048951148987,
+      "rewards/rejected": -0.26925593614578247,
       "step": 210
     },
     {
       "epoch": 0.5,
-      "grad_norm": 109.57190452096494,
+      "grad_norm": 25.725130617472978,
       "learning_rate": 5.8508297910462456e-08,
-      "logits/chosen": -1.8995733261108398,
-      "logits/rejected": -1.818933129310608,
-      "logps/chosen": -414.49176025390625,
-      "logps/rejected": -402.1959228515625,
-      "loss": 0.498,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 0.3541712164878845,
-      "rewards/margins": 0.9701933860778809,
-      "rewards/rejected": -0.6160220503807068,
+      "logits/chosen": -2.2174572944641113,
+      "logits/rejected": -2.1320137977600098,
+      "logps/chosen": -415.51153564453125,
+      "logps/rejected": -422.1241149902344,
+      "loss": 0.5729,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.06063612177968025,
+      "rewards/margins": 0.38312196731567383,
+      "rewards/rejected": -0.32248586416244507,
       "step": 220
     },
     {
       "epoch": 0.52,
-      "grad_norm": 221.2346866193578,
+      "grad_norm": 33.748772620696556,
       "learning_rate": 5.456683083494731e-08,
-      "logits/chosen": -1.892313003540039,
-      "logits/rejected": -1.8525936603546143,
-      "logps/chosen": -472.8267517089844,
-      "logps/rejected": -455.94305419921875,
-      "loss": 0.5217,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.2913890480995178,
-      "rewards/margins": 0.6712461709976196,
-      "rewards/rejected": -0.3798571825027466,
+      "logits/chosen": -2.204589366912842,
+      "logits/rejected": -2.164806842803955,
+      "logps/chosen": -464.0941467285156,
+      "logps/rejected": -462.9925231933594,
+      "loss": 0.5856,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.1456037312746048,
+      "rewards/margins": 0.29207009077072144,
+      "rewards/rejected": -0.14646635949611664,
       "step": 230
     },
     {
       "epoch": 0.55,
-      "grad_norm": 199.8394238496795,
+      "grad_norm": 35.95467752105537,
       "learning_rate": 5.059649078450834e-08,
-      "logits/chosen": -1.8888638019561768,
-      "logits/rejected": -1.8429927825927734,
-      "logps/chosen": -432.7430725097656,
-      "logps/rejected": -422.738037109375,
-      "loss": 0.4908,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 0.274366557598114,
-      "rewards/margins": 0.8101264834403992,
-      "rewards/rejected": -0.5357599854469299,
+      "logits/chosen": -2.2099993228912354,
+      "logits/rejected": -2.1632981300354004,
+      "logps/chosen": -434.889404296875,
+      "logps/rejected": -441.2001037597656,
+      "loss": 0.5675,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.03340945765376091,
+      "rewards/margins": 0.3251820206642151,
+      "rewards/rejected": -0.2917725443840027,
       "step": 240
     },
     {
       "epoch": 0.57,
-      "grad_norm": 80.39074621948711,
+      "grad_norm": 21.692896426665428,
       "learning_rate": 4.6622379527277186e-08,
-      "logits/chosen": -1.8966060876846313,
-      "logits/rejected": -1.8454921245574951,
-      "logps/chosen": -394.818115234375,
-      "logps/rejected": -373.1236877441406,
-      "loss": 0.4952,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.21977496147155762,
-      "rewards/margins": 0.7499412298202515,
-      "rewards/rejected": -0.5301662683486938,
+      "logits/chosen": -2.2246696949005127,
+      "logits/rejected": -2.1740729808807373,
+      "logps/chosen": -406.245361328125,
+      "logps/rejected": -402.9896545410156,
+      "loss": 0.5644,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.07031740248203278,
+      "rewards/margins": 0.33437561988830566,
+      "rewards/rejected": -0.40469303727149963,
       "step": 250
     },
     {
       "epoch": 0.59,
-      "grad_norm": 161.76520518881122,
+      "grad_norm": 39.74076278883575,
       "learning_rate": 4.26696226741691e-08,
-      "logits/chosen": -1.9112564325332642,
-      "logits/rejected": -1.839714765548706,
-      "logps/chosen": -430.2705078125,
-      "logps/rejected": -403.42633056640625,
-      "loss": 0.5027,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.09559208154678345,
-      "rewards/margins": 0.799443244934082,
-      "rewards/rejected": -0.7038511037826538,
+      "logits/chosen": -2.2478280067443848,
+      "logits/rejected": -2.1786162853240967,
+      "logps/chosen": -446.00897216796875,
+      "logps/rejected": -439.8905334472656,
+      "loss": 0.5747,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.13826611638069153,
+      "rewards/margins": 0.3671457767486572,
+      "rewards/rejected": -0.5054119229316711,
       "step": 260
     },
     {
       "epoch": 0.61,
-      "grad_norm": 92.5112436349587,
+      "grad_norm": 24.611550372970484,
       "learning_rate": 3.876321082668098e-08,
-      "logits/chosen": -1.9705326557159424,
-      "logits/rejected": -1.896384596824646,
-      "logps/chosen": -455.70849609375,
-      "logps/rejected": -424.35528564453125,
-      "loss": 0.4925,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": 0.3973539471626282,
-      "rewards/margins": 0.953347384929657,
-      "rewards/rejected": -0.5559934377670288,
+      "logits/chosen": -2.306821346282959,
+      "logits/rejected": -2.2316880226135254,
+      "logps/chosen": -465.56439208984375,
+      "logps/rejected": -459.61700439453125,
+      "loss": 0.564,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.019088072702288628,
+      "rewards/margins": 0.4447278082370758,
+      "rewards/rejected": -0.463815838098526,
       "step": 270
     },
     {
       "epoch": 0.64,
-      "grad_norm": 178.08098991416173,
+      "grad_norm": 39.92086883833645,
       "learning_rate": 3.492784157826244e-08,
-      "logits/chosen": -1.8988901376724243,
-      "logits/rejected": -1.8011884689331055,
-      "logps/chosen": -438.60076904296875,
-      "logps/rejected": -388.46978759765625,
-      "loss": 0.5004,
+      "logits/chosen": -2.236232280731201,
+      "logits/rejected": -2.1331005096435547,
+      "logps/chosen": -452.8758850097656,
+      "logps/rejected": -426.164306640625,
+      "loss": 0.5642,
       "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.43949180841445923,
-      "rewards/margins": 1.0134050846099854,
-      "rewards/rejected": -0.5739132165908813,
+      "rewards/chosen": -0.054852552711963654,
+      "rewards/margins": 0.43687576055526733,
+      "rewards/rejected": -0.49172839522361755,
       "step": 280
     },
     {
       "epoch": 0.66,
-      "grad_norm": 118.63444466442957,
+      "grad_norm": 30.441413413514407,
       "learning_rate": 3.118776336817812e-08,
-      "logits/chosen": -1.9487316608428955,
-      "logits/rejected": -1.8821332454681396,
-      "logps/chosen": -431.9124450683594,
-      "logps/rejected": -391.89208984375,
-      "loss": 0.4867,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.3583192229270935,
-      "rewards/margins": 0.9170917272567749,
-      "rewards/rejected": -0.5587725639343262,
+      "logits/chosen": -2.286245346069336,
+      "logits/rejected": -2.217912435531616,
+      "logps/chosen": -441.1709899902344,
+      "logps/rejected": -427.1979064941406,
+      "loss": 0.5531,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.020920906215906143,
+      "rewards/margins": 0.44389209151268005,
+      "rewards/rejected": -0.4648129940032959,
       "step": 290
     },
     {
       "epoch": 0.68,
-      "grad_norm": 105.93330787956464,
+      "grad_norm": 27.497209882056495,
       "learning_rate": 2.7566622175067443e-08,
-      "logits/chosen": -1.9368865489959717,
-      "logits/rejected": -1.8702398538589478,
-      "logps/chosen": -427.23101806640625,
-      "logps/rejected": -406.6597595214844,
-      "loss": 0.5099,
+      "logits/chosen": -2.2853000164031982,
+      "logits/rejected": -2.2172305583953857,
+      "logps/chosen": -439.1316833496094,
+      "logps/rejected": -443.3128967285156,
+      "loss": 0.5682,
       "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.41136330366134644,
-      "rewards/margins": 0.9053149223327637,
-      "rewards/rejected": -0.4939516484737396,
+      "rewards/chosen": -0.03673394396901131,
+      "rewards/margins": 0.42858797311782837,
+      "rewards/rejected": -0.46532192826271057,
       "step": 300
     },
     {
       "epoch": 0.71,
-      "grad_norm": 94.82101284169875,
+      "grad_norm": 35.65048747036057,
       "learning_rate": 2.408731201945432e-08,
-      "logits/chosen": -1.9327596426010132,
-      "logits/rejected": -1.8846778869628906,
-      "logps/chosen": -422.32061767578125,
-      "logps/rejected": -419.91131591796875,
-      "loss": 0.4923,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.4359460473060608,
-      "rewards/margins": 0.7473770380020142,
-      "rewards/rejected": -0.31143102049827576,
+      "logits/chosen": -2.270371437072754,
+      "logits/rejected": -2.2171905040740967,
+      "logps/chosen": -437.3782653808594,
+      "logps/rejected": -453.547607421875,
+      "loss": 0.5546,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.06338675320148468,
+      "rewards/margins": 0.3352622985839844,
+      "rewards/rejected": -0.39864906668663025,
       "step": 310
     },
     {
       "epoch": 0.73,
-      "grad_norm": 197.8417598175193,
+      "grad_norm": 34.348920131629896,
       "learning_rate": 2.0771830220378112e-08,
-      "logits/chosen": -1.8852752447128296,
-      "logits/rejected": -1.8286488056182861,
-      "logps/chosen": -450.9109802246094,
-      "logps/rejected": -427.3114318847656,
-      "loss": 0.4812,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.44545871019363403,
-      "rewards/margins": 0.8026180267333984,
-      "rewards/rejected": -0.357159286737442,
+      "logits/chosen": -2.223020076751709,
+      "logits/rejected": -2.158247709274292,
+      "logps/chosen": -462.366455078125,
+      "logps/rejected": -461.9310607910156,
+      "loss": 0.5476,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.025463122874498367,
+      "rewards/margins": 0.39216503500938416,
+      "rewards/rejected": -0.41762813925743103,
       "step": 320
     },
     {
       "epoch": 0.75,
-      "grad_norm": 115.40932567329016,
+      "grad_norm": 27.509152301102358,
       "learning_rate": 1.7641138321260257e-08,
-      "logits/chosen": -1.9217668771743774,
-      "logits/rejected": -1.8482532501220703,
-      "logps/chosen": -424.6617736816406,
-      "logps/rejected": -386.41943359375,
-      "loss": 0.4888,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": 0.49162572622299194,
-      "rewards/margins": 1.0169591903686523,
-      "rewards/rejected": -0.5253334045410156,
+      "logits/chosen": -2.2643046379089355,
+      "logits/rejected": -2.188499689102173,
+      "logps/chosen": -440.750732421875,
+      "logps/rejected": -428.7913513183594,
+      "loss": 0.5541,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.06256476789712906,
+      "rewards/margins": 0.46622103452682495,
+      "rewards/rejected": -0.5287858247756958,
       "step": 330
     },
     {
       "epoch": 0.77,
-      "grad_norm": 116.14707152688237,
+      "grad_norm": 27.197721402060907,
       "learning_rate": 1.4715029564277793e-08,
-      "logits/chosen": -1.995141625404358,
-      "logits/rejected": -1.9429680109024048,
-      "logps/chosen": -437.26324462890625,
-      "logps/rejected": -408.69403076171875,
-      "loss": 0.4984,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.6876565217971802,
-      "rewards/margins": 0.9710708856582642,
-      "rewards/rejected": -0.283414363861084,
+      "logits/chosen": -2.340247631072998,
+      "logits/rejected": -2.2862460613250732,
+      "logps/chosen": -450.05523681640625,
+      "logps/rejected": -448.6039123535156,
+      "loss": 0.5553,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.009611198678612709,
+      "rewards/margins": 0.4653921127319336,
+      "rewards/rejected": -0.45578089356422424,
       "step": 340
     },
     {
       "epoch": 0.8,
-      "grad_norm": 96.35236055831872,
+      "grad_norm": 23.708931464533432,
       "learning_rate": 1.2012003751113343e-08,
-      "logits/chosen": -1.9784055948257446,
-      "logits/rejected": -1.913835883140564,
-      "logps/chosen": -427.19921875,
-      "logps/rejected": -406.6058044433594,
-      "loss": 0.4675,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.41034063696861267,
-      "rewards/margins": 0.9592168927192688,
-      "rewards/rejected": -0.5488761067390442,
+      "logits/chosen": -2.325669288635254,
+      "logits/rejected": -2.259887933731079,
+      "logps/chosen": -452.64862060546875,
+      "logps/rejected": -454.2327575683594,
+      "loss": 0.538,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.17242571711540222,
+      "rewards/margins": 0.4136194586753845,
+      "rewards/rejected": -0.5860452055931091,
       "step": 350
     },
     {
       "epoch": 0.82,
-      "grad_norm": 119.40607594359629,
+      "grad_norm": 32.57218388588953,
       "learning_rate": 9.549150281252633e-09,
-      "logits/chosen": -1.9397361278533936,
-      "logits/rejected": -1.884749174118042,
-      "logps/chosen": -451.00531005859375,
-      "logps/rejected": -423.86553955078125,
-      "loss": 0.481,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 0.5500633120536804,
-      "rewards/margins": 0.9735897779464722,
-      "rewards/rejected": -0.4235265851020813,
+      "logits/chosen": -2.284872531890869,
+      "logits/rejected": -2.2292895317077637,
+      "logps/chosen": -473.66009521484375,
+      "logps/rejected": -472.35919189453125,
+      "loss": 0.547,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.11653508991003036,
+      "rewards/margins": 0.4531070590019226,
+      "rewards/rejected": -0.5696421265602112,
       "step": 360
     },
     {
       "epoch": 0.84,
-      "grad_norm": 110.76475497042553,
+      "grad_norm": 36.6888447736333,
       "learning_rate": 7.3420401072985306e-09,
-      "logits/chosen": -1.9737659692764282,
-      "logits/rejected": -1.9217618703842163,
-      "logps/chosen": -427.6600646972656,
-      "logps/rejected": -412.3758850097656,
-      "loss": 0.4918,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 0.44847536087036133,
-      "rewards/margins": 0.9299023747444153,
-      "rewards/rejected": -0.4814269542694092,
+      "logits/chosen": -2.322558641433716,
+      "logits/rejected": -2.267219066619873,
+      "logps/chosen": -451.59295654296875,
+      "logps/rejected": -459.50665283203125,
+      "loss": 0.5582,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.14963454008102417,
+      "rewards/margins": 0.4179585874080658,
+      "rewards/rejected": -0.5675932168960571,
       "step": 370
     },
     {
       "epoch": 0.86,
-      "grad_norm": 95.99625394215789,
+      "grad_norm": 25.485415219281943,
       "learning_rate": 5.404627290395369e-09,
-      "logits/chosen": -1.9365133047103882,
-      "logits/rejected": -1.8697948455810547,
-      "logps/chosen": -429.69219970703125,
-      "logps/rejected": -406.6961975097656,
-      "loss": 0.4841,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.6347268223762512,
-      "rewards/margins": 0.9203673601150513,
-      "rewards/rejected": -0.2856404185295105,
+      "logits/chosen": -2.2841172218322754,
+      "logits/rejected": -2.213308334350586,
+      "logps/chosen": -445.1299743652344,
+      "logps/rejected": -450.1717224121094,
+      "loss": 0.5491,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.027432629838585854,
+      "rewards/margins": 0.4644508957862854,
+      "rewards/rejected": -0.4918835163116455,
       "step": 380
     },
     {
       "epoch": 0.89,
-      "grad_norm": 157.20747053789427,
+      "grad_norm": 37.42685396157706,
       "learning_rate": 3.74916077816162e-09,
-      "logits/chosen": -1.945406198501587,
-      "logits/rejected": -1.8826377391815186,
-      "logps/chosen": -412.59619140625,
-      "logps/rejected": -386.8226623535156,
-      "loss": 0.4932,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": 0.42942380905151367,
-      "rewards/margins": 0.8297268152236938,
-      "rewards/rejected": -0.4003029763698578,
+      "logits/chosen": -2.292269706726074,
+      "logits/rejected": -2.226890802383423,
+      "logps/chosen": -436.8121643066406,
+      "logps/rejected": -433.2718200683594,
+      "loss": 0.5552,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1562749743461609,
+      "rewards/margins": 0.388277530670166,
+      "rewards/rejected": -0.5445524454116821,
       "step": 390
     },
     {
       "epoch": 0.91,
-      "grad_norm": 105.21807814382213,
+      "grad_norm": 24.281602089300215,
       "learning_rate": 2.386106962899165e-09,
-      "logits/chosen": -1.871506690979004,
-      "logits/rejected": -1.7978681325912476,
-      "logps/chosen": -419.5323791503906,
-      "logps/rejected": -388.26239013671875,
-      "loss": 0.4821,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.3208548426628113,
-      "rewards/margins": 0.8715551495552063,
-      "rewards/rejected": -0.5507001876831055,
+      "logits/chosen": -2.218071460723877,
+      "logits/rejected": -2.1384034156799316,
+      "logps/chosen": -447.8563537597656,
+      "logps/rejected": -437.1399841308594,
+      "loss": 0.5478,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.219069242477417,
+      "rewards/margins": 0.37984699010849,
+      "rewards/rejected": -0.598916232585907,
       "step": 400
     },
     {
       "epoch": 0.93,
-      "grad_norm": 114.13246004934948,
+      "grad_norm": 28.633671701703214,
       "learning_rate": 1.3240835096913706e-09,
-      "logits/chosen": -1.9058313369750977,
-      "logits/rejected": -1.8005132675170898,
-      "logps/chosen": -423.19305419921875,
-      "logps/rejected": -383.4271240234375,
-      "loss": 0.4992,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": 0.6571682095527649,
-      "rewards/margins": 0.9992189407348633,
-      "rewards/rejected": -0.3420506417751312,
+      "logits/chosen": -2.2583167552948,
+      "logits/rejected": -2.1519083976745605,
+      "logps/chosen": -440.5953063964844,
+      "logps/rejected": -431.39501953125,
+      "loss": 0.5514,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.04258907586336136,
+      "rewards/margins": 0.5055002570152283,
+      "rewards/rejected": -0.5480893850326538,
       "step": 410
     },
     {
       "epoch": 0.96,
-      "grad_norm": 88.16218181131795,
+      "grad_norm": 25.241002875120977,
       "learning_rate": 5.698048727497462e-10,
-      "logits/chosen": -1.929990530014038,
-      "logits/rejected": -1.8586419820785522,
-      "logps/chosen": -402.3602294921875,
-      "logps/rejected": -381.64605712890625,
-      "loss": 0.4911,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": 0.5359801650047302,
-      "rewards/margins": 1.0312530994415283,
-      "rewards/rejected": -0.4952728748321533,
+      "logits/chosen": -2.2798821926116943,
+      "logits/rejected": -2.2044219970703125,
+      "logps/chosen": -424.9578552246094,
+      "logps/rejected": -435.0152893066406,
+      "loss": 0.5514,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.11878044903278351,
+      "rewards/margins": 0.5139662623405457,
+      "rewards/rejected": -0.632746696472168,
       "step": 420
     },
     {
       "epoch": 0.98,
-      "grad_norm": 104.24331856340095,
+      "grad_norm": 23.361501927411613,
       "learning_rate": 1.2803984447259387e-10,
-      "logits/chosen": -1.9403343200683594,
-      "logits/rejected": -1.881209135055542,
-      "logps/chosen": -453.416259765625,
-      "logps/rejected": -408.7674865722656,
-      "loss": 0.4731,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": 0.4977526068687439,
-      "rewards/margins": 1.0032169818878174,
-      "rewards/rejected": -0.5054643750190735,
+      "logits/chosen": -2.295804977416992,
+      "logits/rejected": -2.235670328140259,
+      "logps/chosen": -475.6675720214844,
+      "logps/rejected": -457.9853515625,
+      "loss": 0.5428,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.12296229600906372,
+      "rewards/margins": 0.4703094959259033,
+      "rewards/rejected": -0.593271791934967,
       "step": 430
     },
     {
       "epoch": 1.0,
       "step": 439,
       "total_flos": 0.0,
-      "train_loss": 0.5309282744935542,
-      "train_runtime": 6847.998,
-      "train_samples_per_second": 8.212,
-      "train_steps_per_second": 0.064
+      "train_loss": 0.048529281703106095,
+      "train_runtime": 584.5423,
+      "train_samples_per_second": 96.205,
+      "train_steps_per_second": 0.751
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:85030f3e8e8a49982a5e76bc0937f8c7c89ad40820c9fa0bff6598e0cf753ecb
+oid sha256:dcdc539826b1dee50f096661ae714e2aafe4a8c16fba498496501f3b2e435739
 size 6264