Model save

README.md

@@ -15,15 +15,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.1420
-- Rewards/chosen: 5.3389
-- Rewards/rejected: -11.5415
-- Rewards/accuracies: 0.9258
-- Rewards/margins: 16.8803
-- Logps/rejected: -131.7123
-- Logps/chosen: -119.9761
-- Logits/rejected: -1.8130
-- Logits/chosen: -1.8134
+- Loss: 0.1390
+- Rewards/chosen: 3.4895
+- Rewards/rejected: -9.2522
+- Rewards/accuracies: 0.9297
+- Rewards/margins: 12.7417
+- Logps/rejected: -139.5015
+- Logps/chosen: -120.3246
+- Logits/rejected: -1.8106
+- Logits/chosen: -1.8098
 
 ## Model description
 
@@ -60,10 +60,10 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.1496        | 0.21  | 100  | 0.1356          | 4.1320         | -11.2810         | 0.9414             | 15.4129         | -130.8439      | -123.9990    | -1.7910         | -1.8010       |
-| 0.1795        | 0.42  | 200  | 0.1364          | 5.2675         | -11.0420         | 0.9336             | 16.3095         | -130.0476      | -120.2140    | -1.8607         | -1.8614       |
-| 0.1585        | 0.63  | 300  | 0.1425          | 5.1387         | -11.7029         | 0.9258             | 16.8416         | -132.2504      | -120.6432    | -1.7960         | -1.7980       |
-| 0.2005        | 0.84  | 400  | 0.1420          | 5.3389         | -11.5415         | 0.9258             | 16.8803         | -131.7123      | -119.9761    | -1.8130         | -1.8134       |
+| 0.1523        | 0.21  | 100  | 0.1399          | 2.5441         | -8.9516          | 0.9375             | 11.4956         | -137.9985      | -125.0519    | -1.8014         | -1.8101       |
+| 0.176         | 0.42  | 200  | 0.1358          | 3.3974         | -8.7531          | 0.9375             | 12.1505         | -137.0064      | -120.7853    | -1.8762         | -1.8764       |
+| 0.1509        | 0.63  | 300  | 0.1403          | 3.3534         | -9.3163          | 0.9336             | 12.6696         | -139.8221      | -121.0054    | -1.7873         | -1.7875       |
+| 0.2009        | 0.84  | 400  | 0.1390          | 3.4895         | -9.2522          | 0.9297             | 12.7417         | -139.5015      | -120.3246    | -1.8106         | -1.8098       |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.17749474911510196,
-    "train_runtime": 7645.2484,
+    "train_loss": 0.17478099153630405,
+    "train_runtime": 7645.1097,
     "train_samples": 61135,
-    "train_samples_per_second": 7.996,
+    "train_samples_per_second": 7.997,
     "train_steps_per_second": 0.063
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5505459ad98890a3203705be3e3b4413c1fdb1bed021f564a6052d2b7286ba53
+oid sha256:c8fe36024d6aef3b53eef0e9ec42db4750fe5b59749cc0d952e83d3dc0c53701
 size 4943178720

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f9cd4bbebcca95695bc07db64291f4364758bb91a1c6301d838900b964109691
+oid sha256:27281dd482564b4b96afafdbf09d408b28d4f3d5fae15c627e97b427e5e9846f
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6603cc6955b683f2147a8107dd2423db37551bb18757806d3089f527c647290d
+oid sha256:3b852db6837b7ef3c2835e6c040e79055a89cbb91a5e93609a246227dc052c44
 size 4540532728

runs/May13_12-29-41_n136-129-074/events.out.tfevents.1715574720.n136-129-074.1898043.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:16d91bcfd42684acb3c691ead1f7c83163f47315ea3bc893779f9fcbf4cdd06c
-size 35913
+oid sha256:73cbd01c36e5321599ce0320582bd802ecd27f095f002acc1abc694f14aa3849
+size 41083

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.17749474911510196,
-    "train_runtime": 7645.2484,
+    "train_loss": 0.17478099153630405,
+    "train_runtime": 7645.1097,
     "train_samples": 61135,
-    "train_samples_per_second": 7.996,
+    "train_samples_per_second": 7.997,
     "train_steps_per_second": 0.063
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 1316.2617480695828,
+      "grad_norm": 877.3754021335358,
       "learning_rate": 1.0416666666666666e-08,
       "logits/chosen": -1.689455509185791,
       "logits/rejected": -1.4794573783874512,
@@ -25,780 +25,780 @@
     },
     {
       "epoch": 0.02,
-      "grad_norm": 1084.7724692148897,
+      "grad_norm": 774.0071219589337,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -1.707624912261963,
-      "logits/rejected": -1.6101186275482178,
-      "logps/chosen": -139.66224670410156,
-      "logps/rejected": -91.32621002197266,
-      "loss": 0.686,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.01818913221359253,
-      "rewards/margins": 0.027222516015172005,
-      "rewards/rejected": -0.009033381938934326,
+      "logits/chosen": -1.7073653936386108,
+      "logits/rejected": -1.610068678855896,
+      "logps/chosen": -139.72607421875,
+      "logps/rejected": -91.37654113769531,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.5138888955116272,
+      "rewards/chosen": -0.0006362733547575772,
+      "rewards/margins": 0.015451871789991856,
+      "rewards/rejected": -0.016088144853711128,
       "step": 10
     },
     {
       "epoch": 0.04,
-      "grad_norm": 372.5890585979663,
+      "grad_norm": 288.7199289028814,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -1.6384038925170898,
-      "logits/rejected": -1.6487312316894531,
-      "logps/chosen": -130.37515258789062,
-      "logps/rejected": -93.99095153808594,
-      "loss": 0.4495,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": 0.6898423433303833,
-      "rewards/margins": 0.9020320177078247,
-      "rewards/rejected": -0.21218962967395782,
+      "logits/chosen": -1.6388952732086182,
+      "logits/rejected": -1.6494001150131226,
+      "logps/chosen": -130.20901489257812,
+      "logps/rejected": -94.0677490234375,
+      "loss": 0.491,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 0.4931251108646393,
+      "rewards/margins": 0.6499455571174622,
+      "rewards/rejected": -0.15682044625282288,
       "step": 20
     },
     {
       "epoch": 0.06,
-      "grad_norm": 363.4016752055454,
+      "grad_norm": 224.69456938109306,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -1.6896642446517944,
-      "logits/rejected": -1.6273959875106812,
-      "logps/chosen": -130.80935668945312,
-      "logps/rejected": -106.37054443359375,
-      "loss": 0.2556,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 2.216484785079956,
-      "rewards/margins": 3.602745771408081,
-      "rewards/rejected": -1.386260986328125,
+      "logits/chosen": -1.6837990283966064,
+      "logits/rejected": -1.6247406005859375,
+      "logps/chosen": -130.29306030273438,
+      "logps/rejected": -107.59431457519531,
+      "loss": 0.2826,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 1.5809106826782227,
+      "rewards/margins": 2.7498364448547363,
+      "rewards/rejected": -1.1689256429672241,
       "step": 30
     },
     {
       "epoch": 0.08,
-      "grad_norm": 254.07955635631413,
+      "grad_norm": 183.79856437398223,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -1.6251739263534546,
-      "logits/rejected": -1.5512189865112305,
-      "logps/chosen": -142.7510528564453,
-      "logps/rejected": -113.31219482421875,
-      "loss": 0.1925,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 2.6909842491149902,
-      "rewards/margins": 7.113295078277588,
-      "rewards/rejected": -4.422310829162598,
+      "logits/chosen": -1.6083917617797852,
+      "logits/rejected": -1.5386857986450195,
+      "logps/chosen": -142.89288330078125,
+      "logps/rejected": -118.29256439208984,
+      "loss": 0.2069,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 1.7656170129776,
+      "rewards/margins": 5.709896087646484,
+      "rewards/rejected": -3.944279432296753,
       "step": 40
     },
     {
       "epoch": 0.1,
-      "grad_norm": 450.1083647442556,
+      "grad_norm": 299.30898027438695,
       "learning_rate": 4.999733114418725e-07,
-      "logits/chosen": -1.5827839374542236,
-      "logits/rejected": -1.6095731258392334,
-      "logps/chosen": -127.47169494628906,
-      "logps/rejected": -124.7472152709961,
-      "loss": 0.187,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 2.2808213233947754,
-      "rewards/margins": 9.9701509475708,
-      "rewards/rejected": -7.6893310546875,
+      "logits/chosen": -1.5618069171905518,
+      "logits/rejected": -1.589414119720459,
+      "logps/chosen": -129.36355590820312,
+      "logps/rejected": -135.2159881591797,
+      "loss": 0.1965,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 1.1421730518341064,
+      "rewards/margins": 8.362150192260742,
+      "rewards/rejected": -7.219976902008057,
       "step": 50
     },
     {
       "epoch": 0.13,
-      "grad_norm": 256.12553340364644,
+      "grad_norm": 245.51415645278104,
       "learning_rate": 4.990398100856366e-07,
-      "logits/chosen": -1.6854372024536133,
-      "logits/rejected": -1.6347172260284424,
-      "logps/chosen": -143.6377410888672,
-      "logps/rejected": -138.22506713867188,
-      "loss": 0.1745,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 2.208740711212158,
-      "rewards/margins": 13.148699760437012,
-      "rewards/rejected": -10.939959526062012,
+      "logits/chosen": -1.6810020208358765,
+      "logits/rejected": -1.6362440586090088,
+      "logps/chosen": -146.06399536132812,
+      "logps/rejected": -148.35720825195312,
+      "loss": 0.1815,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.9872430562973022,
+      "rewards/margins": 10.306981086730957,
+      "rewards/rejected": -9.319738388061523,
       "step": 60
     },
     {
       "epoch": 0.15,
-      "grad_norm": 266.6931182384945,
+      "grad_norm": 184.4357866731278,
       "learning_rate": 4.967775735898179e-07,
-      "logits/chosen": -1.6533622741699219,
-      "logits/rejected": -1.6828343868255615,
-      "logps/chosen": -136.77774047851562,
-      "logps/rejected": -139.76748657226562,
-      "loss": 0.1719,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 2.496314525604248,
-      "rewards/margins": 15.036949157714844,
-      "rewards/rejected": -12.540634155273438,
+      "logits/chosen": -1.6720081567764282,
+      "logits/rejected": -1.706272840499878,
+      "logps/chosen": -138.72039794921875,
+      "logps/rejected": -148.6911163330078,
+      "loss": 0.1754,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 1.2756741046905518,
+      "rewards/margins": 11.420818328857422,
+      "rewards/rejected": -10.14514446258545,
       "step": 70
     },
     {
       "epoch": 0.17,
-      "grad_norm": 311.4093156505733,
+      "grad_norm": 147.5148614226922,
       "learning_rate": 4.931986719649298e-07,
-      "logits/chosen": -1.6512008905410767,
-      "logits/rejected": -1.642218828201294,
-      "logps/chosen": -129.7277374267578,
-      "logps/rejected": -130.1470489501953,
-      "loss": 0.1638,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 3.2196338176727295,
-      "rewards/margins": 14.145663261413574,
-      "rewards/rejected": -10.926031112670898,
+      "logits/chosen": -1.6803735494613647,
+      "logits/rejected": -1.677125334739685,
+      "logps/chosen": -131.1448211669922,
+      "logps/rejected": -137.2668914794922,
+      "loss": 0.1602,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 1.8630034923553467,
+      "rewards/margins": 10.570993423461914,
+      "rewards/rejected": -8.707988739013672,
       "step": 80
     },
     {
       "epoch": 0.19,
-      "grad_norm": 353.08827705610673,
+      "grad_norm": 231.45195159349998,
       "learning_rate": 4.883222001996351e-07,
-      "logits/chosen": -1.7057621479034424,
-      "logits/rejected": -1.7253615856170654,
-      "logps/chosen": -136.57843017578125,
-      "logps/rejected": -141.9429931640625,
-      "loss": 0.1501,
+      "logits/chosen": -1.7273483276367188,
+      "logits/rejected": -1.7491174936294556,
+      "logps/chosen": -137.68093872070312,
+      "logps/rejected": -149.29808044433594,
+      "loss": 0.1529,
       "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 3.7895398139953613,
-      "rewards/margins": 15.804595947265625,
-      "rewards/rejected": -12.015056610107422,
+      "rewards/chosen": 2.305860757827759,
+      "rewards/margins": 11.786917686462402,
+      "rewards/rejected": -9.481058120727539,
       "step": 90
     },
     {
       "epoch": 0.21,
-      "grad_norm": 173.53718581228586,
+      "grad_norm": 125.1418915635804,
       "learning_rate": 4.821741763807186e-07,
-      "logits/chosen": -1.7514231204986572,
-      "logits/rejected": -1.7408148050308228,
-      "logps/chosen": -116.7274169921875,
-      "logps/rejected": -125.6600570678711,
-      "loss": 0.1496,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 3.375108242034912,
-      "rewards/margins": 13.876774787902832,
-      "rewards/rejected": -10.501666069030762,
+      "logits/chosen": -1.7639491558074951,
+      "logits/rejected": -1.7565345764160156,
+      "logps/chosen": -118.08821105957031,
+      "logps/rejected": -132.42239379882812,
+      "loss": 0.1523,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 1.9779144525527954,
+      "rewards/margins": 10.331491470336914,
+      "rewards/rejected": -8.35357666015625,
       "step": 100
     },
     {
       "epoch": 0.21,
-      "eval_logits/chosen": -1.8009788990020752,
-      "eval_logits/rejected": -1.790999174118042,
-      "eval_logps/chosen": -123.99901580810547,
-      "eval_logps/rejected": -130.8439178466797,
-      "eval_loss": 0.13556738197803497,
-      "eval_rewards/accuracies": 0.94140625,
-      "eval_rewards/chosen": 4.131972789764404,
-      "eval_rewards/margins": 15.41292953491211,
-      "eval_rewards/rejected": -11.280956268310547,
-      "eval_runtime": 97.6442,
-      "eval_samples_per_second": 20.483,
+      "eval_logits/chosen": -1.810097336769104,
+      "eval_logits/rejected": -1.8013982772827148,
+      "eval_logps/chosen": -125.05193328857422,
+      "eval_logps/rejected": -137.99850463867188,
+      "eval_loss": 0.13986752927303314,
+      "eval_rewards/accuracies": 0.9375,
+      "eval_rewards/chosen": 2.5440609455108643,
+      "eval_rewards/margins": 11.49561595916748,
+      "eval_rewards/rejected": -8.951555252075195,
+      "eval_runtime": 97.654,
+      "eval_samples_per_second": 20.48,
       "eval_steps_per_second": 0.328,
       "step": 100
     },
     {
       "epoch": 0.23,
-      "grad_norm": 220.3790872760113,
+      "grad_norm": 145.2147024977398,
       "learning_rate": 4.747874028753375e-07,
-      "logits/chosen": -1.6608690023422241,
-      "logits/rejected": -1.7199954986572266,
-      "logps/chosen": -120.6917724609375,
-      "logps/rejected": -133.25762939453125,
-      "loss": 0.1546,
+      "logits/chosen": -1.6646867990493774,
+      "logits/rejected": -1.7233518362045288,
+      "logps/chosen": -122.03011322021484,
+      "logps/rejected": -140.46389770507812,
+      "loss": 0.1545,
       "rewards/accuracies": 0.9375,
-      "rewards/chosen": 3.4064812660217285,
-      "rewards/margins": 12.731483459472656,
-      "rewards/rejected": -9.325002670288086,
+      "rewards/chosen": 2.0033175945281982,
+      "rewards/margins": 9.661236763000488,
+      "rewards/rejected": -7.657918453216553,
       "step": 110
     },
     {
       "epoch": 0.25,
-      "grad_norm": 179.12119430014053,
+      "grad_norm": 129.9317141592677,
       "learning_rate": 4.662012913161997e-07,
-      "logits/chosen": -1.7208821773529053,
-      "logits/rejected": -1.7148889303207397,
-      "logps/chosen": -118.548583984375,
-      "logps/rejected": -133.46463012695312,
-      "loss": 0.1456,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 4.286909580230713,
-      "rewards/margins": 13.92347240447998,
-      "rewards/rejected": -9.636563301086426,
+      "logits/chosen": -1.7326112985610962,
+      "logits/rejected": -1.7273359298706055,
+      "logps/chosen": -119.6495132446289,
+      "logps/rejected": -140.51266479492188,
+      "loss": 0.1492,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 2.637754440307617,
+      "rewards/margins": 10.471738815307617,
+      "rewards/rejected": -7.833985805511475,
       "step": 120
     },
     {
       "epoch": 0.27,
-      "grad_norm": 119.5811123757762,
+      "grad_norm": 81.32565764107251,
       "learning_rate": 4.5646165232345103e-07,
-      "logits/chosen": -1.688997507095337,
-      "logits/rejected": -1.7153244018554688,
-      "logps/chosen": -124.4625244140625,
-      "logps/rejected": -129.4587860107422,
-      "loss": 0.1477,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 4.8756022453308105,
-      "rewards/margins": 15.727206230163574,
-      "rewards/rejected": -10.851605415344238,
+      "logits/chosen": -1.6919190883636475,
+      "logits/rejected": -1.7214174270629883,
+      "logps/chosen": -124.76143646240234,
+      "logps/rejected": -136.91891479492188,
+      "loss": 0.144,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 3.1906180381774902,
+      "rewards/margins": 11.917051315307617,
+      "rewards/rejected": -8.726433753967285,
       "step": 130
     },
     {
       "epoch": 0.29,
-      "grad_norm": 293.67492501764275,
+      "grad_norm": 233.71107343479284,
       "learning_rate": 4.456204510851956e-07,
-      "logits/chosen": -1.6008217334747314,
-      "logits/rejected": -1.5764399766921997,
-      "logps/chosen": -121.84040832519531,
-      "logps/rejected": -125.70499420166016,
-      "loss": 0.1534,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 4.718628406524658,
-      "rewards/margins": 14.510149955749512,
-      "rewards/rejected": -9.791521072387695,
+      "logits/chosen": -1.6101233959197998,
+      "logits/rejected": -1.5833767652511597,
+      "logps/chosen": -121.59381103515625,
+      "logps/rejected": -132.78977966308594,
+      "loss": 0.1501,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 3.1950697898864746,
+      "rewards/margins": 11.13970947265625,
+      "rewards/rejected": -7.944640159606934,
       "step": 140
     },
     {
       "epoch": 0.31,
-      "grad_norm": 250.47040288737202,
+      "grad_norm": 177.51912736620577,
       "learning_rate": 4.337355301007335e-07,
-      "logits/chosen": -1.7394781112670898,
-      "logits/rejected": -1.7636123895645142,
-      "logps/chosen": -119.86863708496094,
-      "logps/rejected": -122.49459075927734,
-      "loss": 0.1403,
+      "logits/chosen": -1.7467788457870483,
+      "logits/rejected": -1.7698566913604736,
+      "logps/chosen": -119.5277328491211,
+      "logps/rejected": -128.22744750976562,
+      "loss": 0.1433,
       "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 4.755049705505371,
-      "rewards/margins": 14.378946304321289,
-      "rewards/rejected": -9.623896598815918,
+      "rewards/chosen": 3.2382194995880127,
+      "rewards/margins": 10.8007173538208,
+      "rewards/rejected": -7.562497615814209,
       "step": 150
     },
     {
       "epoch": 0.33,
-      "grad_norm": 394.54185792168863,
+      "grad_norm": 248.7150598871683,
       "learning_rate": 4.2087030056579986e-07,
-      "logits/chosen": -1.6095482110977173,
-      "logits/rejected": -1.5893223285675049,
-      "logps/chosen": -127.0114517211914,
-      "logps/rejected": -128.70870971679688,
-      "loss": 0.1577,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 4.486311435699463,
-      "rewards/margins": 13.415287971496582,
-      "rewards/rejected": -8.928976058959961,
+      "logits/chosen": -1.6178503036499023,
+      "logits/rejected": -1.5939674377441406,
+      "logps/chosen": -127.01255798339844,
+      "logps/rejected": -134.47132873535156,
+      "loss": 0.1508,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 2.990654468536377,
+      "rewards/margins": 10.095832824707031,
+      "rewards/rejected": -7.105177879333496,
       "step": 160
     },
     {
       "epoch": 0.36,
-      "grad_norm": 336.28954028277553,
+      "grad_norm": 201.08598013628404,
       "learning_rate": 4.070934040463998e-07,
-      "logits/chosen": -1.7846260070800781,
-      "logits/rejected": -1.7679131031036377,
-      "logps/chosen": -121.9030990600586,
-      "logps/rejected": -130.14794921875,
-      "loss": 0.1516,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 5.171528339385986,
-      "rewards/margins": 15.514431953430176,
-      "rewards/rejected": -10.342904090881348,
+      "logits/chosen": -1.7953569889068604,
+      "logits/rejected": -1.7801055908203125,
+      "logps/chosen": -122.10235595703125,
+      "logps/rejected": -136.7244873046875,
+      "loss": 0.1455,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 3.407832622528076,
+      "rewards/margins": 11.618408203125,
+      "rewards/rejected": -8.210575103759766,
       "step": 170
     },
     {
       "epoch": 0.38,
-      "grad_norm": 266.7294156199543,
+      "grad_norm": 169.66216294230117,
       "learning_rate": 3.9247834624635404e-07,
-      "logits/chosen": -1.7311102151870728,
-      "logits/rejected": -1.725064992904663,
-      "logps/chosen": -125.84059143066406,
-      "logps/rejected": -125.44111633300781,
-      "loss": 0.1616,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 5.19378137588501,
-      "rewards/margins": 15.565200805664062,
-      "rewards/rejected": -10.371419906616211,
+      "logits/chosen": -1.7412538528442383,
+      "logits/rejected": -1.734012246131897,
+      "logps/chosen": -126.0450439453125,
+      "logps/rejected": -131.69639587402344,
+      "loss": 0.1565,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 3.421628952026367,
+      "rewards/margins": 11.586966514587402,
+      "rewards/rejected": -8.165337562561035,
       "step": 180
     },
     {
       "epoch": 0.4,
-      "grad_norm": 243.44642169675112,
+      "grad_norm": 165.65601380930994,
       "learning_rate": 3.7710310482256523e-07,
-      "logits/chosen": -1.764722466468811,
-      "logits/rejected": -1.7476263046264648,
-      "logps/chosen": -114.77116394042969,
-      "logps/rejected": -139.33917236328125,
-      "loss": 0.1684,
+      "logits/chosen": -1.7725350856781006,
+      "logits/rejected": -1.7596399784088135,
+      "logps/chosen": -115.76643371582031,
+      "logps/rejected": -146.38294982910156,
+      "loss": 0.1609,
       "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 5.108515739440918,
-      "rewards/margins": 16.28169822692871,
-      "rewards/rejected": -11.173181533813477,
+      "rewards/chosen": 3.206624984741211,
+      "rewards/margins": 12.064164161682129,
+      "rewards/rejected": -8.857539176940918,
       "step": 190
     },
     {
       "epoch": 0.42,
-      "grad_norm": 258.83830985573707,
+      "grad_norm": 164.38352614855714,
       "learning_rate": 3.610497133404795e-07,
-      "logits/chosen": -1.7498699426651,
-      "logits/rejected": -1.7545902729034424,
-      "logps/chosen": -120.42120361328125,
-      "logps/rejected": -123.0103988647461,
-      "loss": 0.1795,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 5.614555835723877,
-      "rewards/margins": 16.115243911743164,
-      "rewards/rejected": -10.500688552856445,
+      "logits/chosen": -1.7693474292755127,
+      "logits/rejected": -1.7811864614486694,
+      "logps/chosen": -121.21810150146484,
+      "logps/rejected": -129.79788208007812,
+      "loss": 0.176,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 3.5836567878723145,
+      "rewards/margins": 11.941611289978027,
+      "rewards/rejected": -8.357954025268555,
       "step": 200
     },
     {
       "epoch": 0.42,
-      "eval_logits/chosen": -1.86138117313385,
-      "eval_logits/rejected": -1.8606913089752197,
-      "eval_logps/chosen": -120.21395111083984,
-      "eval_logps/rejected": -130.0475616455078,
-      "eval_loss": 0.13641399145126343,
-      "eval_rewards/accuracies": 0.93359375,
-      "eval_rewards/chosen": 5.267488479614258,
-      "eval_rewards/margins": 16.30953025817871,
-      "eval_rewards/rejected": -11.042043685913086,
-      "eval_runtime": 97.5652,
-      "eval_samples_per_second": 20.499,
+      "eval_logits/chosen": -1.8763718605041504,
+      "eval_logits/rejected": -1.8761746883392334,
+      "eval_logps/chosen": -120.78527069091797,
+      "eval_logps/rejected": -137.00637817382812,
+      "eval_loss": 0.13582415878772736,
+      "eval_rewards/accuracies": 0.9375,
+      "eval_rewards/chosen": 3.3973963260650635,
+      "eval_rewards/margins": 12.150527954101562,
+      "eval_rewards/rejected": -8.753131866455078,
+      "eval_runtime": 97.4856,
+      "eval_samples_per_second": 20.516,
       "eval_steps_per_second": 0.328,
       "step": 200
     },
     {
       "epoch": 0.44,
-      "grad_norm": 258.76531038458563,
+      "grad_norm": 177.62042791365712,
       "learning_rate": 3.4440382358952115e-07,
-      "logits/chosen": -1.689432144165039,
-      "logits/rejected": -1.632364273071289,
-      "logps/chosen": -117.7408447265625,
-      "logps/rejected": -117.53926086425781,
-      "loss": 0.1494,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 4.45497989654541,
-      "rewards/margins": 12.836206436157227,
-      "rewards/rejected": -8.381224632263184,
+      "logits/chosen": -1.708108901977539,
+      "logits/rejected": -1.654348611831665,
+      "logps/chosen": -118.52095794677734,
+      "logps/rejected": -123.7565689086914,
+      "loss": 0.1417,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 2.8139612674713135,
+      "rewards/margins": 9.644906997680664,
+      "rewards/rejected": -6.830945014953613,
       "step": 210
     },
     {
       "epoch": 0.46,
-      "grad_norm": 163.97448768931915,
+      "grad_norm": 106.72676059840462,
       "learning_rate": 3.272542485937368e-07,
-      "logits/chosen": -1.8162240982055664,
-      "logits/rejected": -1.8604834079742432,
-      "logps/chosen": -118.64457702636719,
-      "logps/rejected": -118.34297180175781,
-      "loss": 0.1524,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 5.34921407699585,
-      "rewards/margins": 14.593510627746582,
-      "rewards/rejected": -9.244296073913574,
+      "logits/chosen": -1.8347289562225342,
+      "logits/rejected": -1.8747615814208984,
+      "logps/chosen": -118.0849609375,
+      "logps/rejected": -125.06263732910156,
+      "loss": 0.1482,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 3.678068161010742,
+      "rewards/margins": 11.184865951538086,
+      "rewards/rejected": -7.506799221038818,
       "step": 220
     },
     {
       "epoch": 0.48,
-      "grad_norm": 122.96583033580265,
+      "grad_norm": 105.28694347115214,
       "learning_rate": 3.096924887558854e-07,
-      "logits/chosen": -1.7572132349014282,
-      "logits/rejected": -1.708581566810608,
-      "logps/chosen": -131.16842651367188,
-      "logps/rejected": -137.84829711914062,
-      "loss": 0.1815,
+      "logits/chosen": -1.7717214822769165,
+      "logits/rejected": -1.720580816268921,
+      "logps/chosen": -131.0685577392578,
+      "logps/rejected": -143.81317138671875,
+      "loss": 0.1684,
       "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 4.962122440338135,
-      "rewards/margins": 16.372663497924805,
-      "rewards/rejected": -11.410540580749512,
+      "rewards/chosen": 3.328054904937744,
+      "rewards/margins": 12.128056526184082,
+      "rewards/rejected": -8.800003051757812,
       "step": 230
     },
     {
       "epoch": 0.5,
-      "grad_norm": 300.9054437339817,
+      "grad_norm": 190.93999332658404,
       "learning_rate": 2.9181224366319943e-07,
-      "logits/chosen": -1.768376111984253,
-      "logits/rejected": -1.7853620052337646,
-      "logps/chosen": -118.08064270019531,
-      "logps/rejected": -129.52337646484375,
-      "loss": 0.1638,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 4.15334415435791,
-      "rewards/margins": 13.721521377563477,
-      "rewards/rejected": -9.568175315856934,
+      "logits/chosen": -1.7859230041503906,
+      "logits/rejected": -1.802541971206665,
+      "logps/chosen": -117.80191802978516,
+      "logps/rejected": -135.9575653076172,
+      "loss": 0.1559,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 2.8246419429779053,
+      "rewards/margins": 10.490262031555176,
+      "rewards/rejected": -7.665619850158691,
       "step": 240
     },
     {
       "epoch": 0.52,
-      "grad_norm": 279.7087004166346,
+      "grad_norm": 200.5301056579919,
       "learning_rate": 2.7370891215954565e-07,
-      "logits/chosen": -1.7569881677627563,
-      "logits/rejected": -1.7871322631835938,
-      "logps/chosen": -121.43367767333984,
-      "logps/rejected": -130.93099975585938,
-      "loss": 0.1737,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 4.7944416999816895,
-      "rewards/margins": 14.978727340698242,
-      "rewards/rejected": -10.184286117553711,
+      "logits/chosen": -1.7719627618789673,
+      "logits/rejected": -1.8007062673568726,
+      "logps/chosen": -120.75521087646484,
+      "logps/rejected": -136.2689971923828,
+      "loss": 0.1635,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 3.3319907188415527,
+      "rewards/margins": 11.189115524291992,
+      "rewards/rejected": -7.857124328613281,
       "step": 250
     },
     {
       "epoch": 0.54,
-      "grad_norm": 163.84527157544522,
+      "grad_norm": 109.42764564797797,
       "learning_rate": 2.55479083351317e-07,
-      "logits/chosen": -1.7968614101409912,
-      "logits/rejected": -1.8222767114639282,
-      "logps/chosen": -125.540283203125,
-      "logps/rejected": -119.97703552246094,
-      "loss": 0.1484,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 5.486014366149902,
-      "rewards/margins": 16.016239166259766,
-      "rewards/rejected": -10.530224800109863,
+      "logits/chosen": -1.8106673955917358,
+      "logits/rejected": -1.8370100259780884,
+      "logps/chosen": -125.1119613647461,
+      "logps/rejected": -127.0175552368164,
+      "loss": 0.1411,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 3.7430100440979004,
+      "rewards/margins": 12.171258926391602,
+      "rewards/rejected": -8.428248405456543,
       "step": 260
     },
     {
       "epoch": 0.56,
-      "grad_norm": 150.71761898213634,
+      "grad_norm": 107.44062116926116,
       "learning_rate": 2.3722002126275822e-07,
-      "logits/chosen": -1.787398099899292,
-      "logits/rejected": -1.723350167274475,
-      "logps/chosen": -115.22535705566406,
-      "logps/rejected": -123.31414794921875,
-      "loss": 0.139,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 5.049933433532715,
-      "rewards/margins": 14.674034118652344,
-      "rewards/rejected": -9.624099731445312,
+      "logits/chosen": -1.795784592628479,
+      "logits/rejected": -1.726580262184143,
+      "logps/chosen": -114.95989990234375,
+      "logps/rejected": -129.06564331054688,
+      "loss": 0.1393,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 3.4197163581848145,
+      "rewards/margins": 10.986078262329102,
+      "rewards/rejected": -7.566361427307129,
       "step": 270
     },
     {
       "epoch": 0.59,
-      "grad_norm": 267.05255152770263,
+      "grad_norm": 191.88979341493172,
       "learning_rate": 2.19029145890313e-07,
-      "logits/chosen": -1.6403396129608154,
-      "logits/rejected": -1.728877067565918,
-      "logps/chosen": -121.6760482788086,
-      "logps/rejected": -137.2602996826172,
-      "loss": 0.1837,
+      "logits/chosen": -1.6350713968276978,
+      "logits/rejected": -1.7275726795196533,
+      "logps/chosen": -121.347412109375,
+      "logps/rejected": -143.2399444580078,
+      "loss": 0.1733,
       "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 4.614184379577637,
-      "rewards/margins": 15.387414932250977,
-      "rewards/rejected": -10.773229598999023,
+      "rewards/chosen": 3.141850233078003,
+      "rewards/margins": 11.51993179321289,
+      "rewards/rejected": -8.378082275390625,
       "step": 280
     },
     {
       "epoch": 0.61,
-      "grad_norm": 355.7773679008215,
+      "grad_norm": 187.2847938998864,
       "learning_rate": 2.0100351342479216e-07,
-      "logits/chosen": -1.7196556329727173,
-      "logits/rejected": -1.7061948776245117,
-      "logps/chosen": -112.95915222167969,
-      "logps/rejected": -124.29833984375,
-      "loss": 0.1648,
+      "logits/chosen": -1.7116390466690063,
+      "logits/rejected": -1.6921733617782593,
+      "logps/chosen": -112.99186706542969,
+      "logps/rejected": -130.45889282226562,
+      "loss": 0.1534,
       "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 4.5611090660095215,
-      "rewards/margins": 15.115681648254395,
-      "rewards/rejected": -10.554571151733398,
+      "rewards/chosen": 3.034196376800537,
+      "rewards/margins": 11.30268669128418,
+      "rewards/rejected": -8.268491744995117,
       "step": 290
     },
     {
       "epoch": 0.63,
-      "grad_norm": 387.9028451533635,
+      "grad_norm": 427.8567531133513,
       "learning_rate": 1.8323929841460178e-07,
-      "logits/chosen": -1.6907150745391846,
-      "logits/rejected": -1.6335220336914062,
-      "logps/chosen": -130.05313110351562,
-      "logps/rejected": -141.36476135253906,
-      "loss": 0.1585,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 4.916111946105957,
-      "rewards/margins": 15.453518867492676,
-      "rewards/rejected": -10.537405967712402,
+      "logits/chosen": -1.6803079843521118,
+      "logits/rejected": -1.6214721202850342,
+      "logps/chosen": -130.09739685058594,
+      "logps/rejected": -148.15475463867188,
+      "loss": 0.1509,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 3.2685546875,
+      "rewards/margins": 11.6514892578125,
+      "rewards/rejected": -8.382935523986816,
       "step": 300
     },
     {
       "epoch": 0.63,
-      "eval_logits/chosen": -1.7980220317840576,
-      "eval_logits/rejected": -1.7959610223770142,
-      "eval_logps/chosen": -120.6431655883789,
-      "eval_logps/rejected": -132.25042724609375,
-      "eval_loss": 0.14247241616249084,
-      "eval_rewards/accuracies": 0.92578125,
-      "eval_rewards/chosen": 5.138728141784668,
-      "eval_rewards/margins": 16.841632843017578,
-      "eval_rewards/rejected": -11.702906608581543,
-      "eval_runtime": 97.7011,
-      "eval_samples_per_second": 20.471,
-      "eval_steps_per_second": 0.328,
+      "eval_logits/chosen": -1.7874743938446045,
+      "eval_logits/rejected": -1.7873148918151855,
+      "eval_logps/chosen": -121.00535583496094,
+      "eval_logps/rejected": -139.82205200195312,
+      "eval_loss": 0.14029568433761597,
+      "eval_rewards/accuracies": 0.93359375,
+      "eval_rewards/chosen": 3.353379249572754,
+      "eval_rewards/margins": 12.669642448425293,
+      "eval_rewards/rejected": -9.316261291503906,
+      "eval_runtime": 97.7473,
+      "eval_samples_per_second": 20.461,
+      "eval_steps_per_second": 0.327,
       "step": 300
     },
     {
       "epoch": 0.65,
-      "grad_norm": 308.7683125090126,
+      "grad_norm": 209.36571305183378,
       "learning_rate": 1.6583128063291573e-07,
-      "logits/chosen": -1.6941922903060913,
-      "logits/rejected": -1.7201515436172485,
-      "logps/chosen": -121.898193359375,
-      "logps/rejected": -132.00962829589844,
-      "loss": 0.1619,
+      "logits/chosen": -1.6824207305908203,
+      "logits/rejected": -1.7118114233016968,
+      "logps/chosen": -122.2557144165039,
+      "logps/rejected": -138.8484344482422,
+      "loss": 0.1544,
       "rewards/accuracies": 0.9375,
-      "rewards/chosen": 5.231846809387207,
-      "rewards/margins": 15.2730131149292,
-      "rewards/rejected": -10.041168212890625,
+      "rewards/chosen": 3.416393280029297,
+      "rewards/margins": 11.478261947631836,
+      "rewards/rejected": -8.061868667602539,
       "step": 310
     },
     {
       "epoch": 0.67,
-      "grad_norm": 208.88097429038612,
+      "grad_norm": 115.39699943542352,
       "learning_rate": 1.488723393865766e-07,
-      "logits/chosen": -1.7199640274047852,
-      "logits/rejected": -1.6923316717147827,
-      "logps/chosen": -111.40040588378906,
-      "logps/rejected": -131.6649627685547,
-      "loss": 0.154,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 5.818942070007324,
-      "rewards/margins": 16.441822052001953,
-      "rewards/rejected": -10.62287712097168,
+      "logits/chosen": -1.7209253311157227,
+      "logits/rejected": -1.6945152282714844,
+      "logps/chosen": -111.72103118896484,
+      "logps/rejected": -138.40509033203125,
+      "loss": 0.144,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 3.8151679039001465,
+      "rewards/margins": 12.245112419128418,
+      "rewards/rejected": -8.42994499206543,
       "step": 320
     },
     {
       "epoch": 0.69,
-      "grad_norm": 167.39144709959334,
+      "grad_norm": 101.70209071929943,
       "learning_rate": 1.3245295796480788e-07,
-      "logits/chosen": -1.65103018283844,
-      "logits/rejected": -1.7277615070343018,
-      "logps/chosen": -121.6876449584961,
-      "logps/rejected": -127.0235366821289,
-      "loss": 0.1507,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 4.948590278625488,
-      "rewards/margins": 14.931228637695312,
-      "rewards/rejected": -9.982640266418457,
+      "logits/chosen": -1.6543442010879517,
+      "logits/rejected": -1.7321891784667969,
+      "logps/chosen": -122.24208068847656,
+      "logps/rejected": -133.8969268798828,
+      "loss": 0.1435,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 3.18817400932312,
+      "rewards/margins": 11.217942237854004,
+      "rewards/rejected": -8.029767990112305,
       "step": 330
     },
     {
       "epoch": 0.71,
-      "grad_norm": 156.90849982717606,
+      "grad_norm": 123.95909153688254,
       "learning_rate": 1.1666074087171627e-07,
-      "logits/chosen": -1.602086067199707,
-      "logits/rejected": -1.6703577041625977,
-      "logps/chosen": -115.71211242675781,
-      "logps/rejected": -116.56599426269531,
-      "loss": 0.1908,
+      "logits/chosen": -1.6039130687713623,
+      "logits/rejected": -1.6741559505462646,
+      "logps/chosen": -116.26409912109375,
+      "logps/rejected": -122.7804946899414,
+      "loss": 0.1746,
       "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 4.433460235595703,
-      "rewards/margins": 13.112295150756836,
-      "rewards/rejected": -8.678834915161133,
+      "rewards/chosen": 2.8452401161193848,
+      "rewards/margins": 9.874032974243164,
+      "rewards/rejected": -7.028792381286621,
       "step": 340
     },
     {
       "epoch": 0.73,
-      "grad_norm": 295.768622683131,
+      "grad_norm": 294.0385926355559,
       "learning_rate": 1.0157994641835734e-07,
-      "logits/chosen": -1.6467043161392212,
-      "logits/rejected": -1.659854531288147,
-      "logps/chosen": -115.77754974365234,
-      "logps/rejected": -137.26531982421875,
-      "loss": 0.1326,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 4.292226314544678,
-      "rewards/margins": 15.004185676574707,
-      "rewards/rejected": -10.711957931518555,
+      "logits/chosen": -1.6423372030258179,
+      "logits/rejected": -1.6570653915405273,
+      "logps/chosen": -116.1936264038086,
+      "logps/rejected": -144.1549072265625,
+      "loss": 0.1291,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 2.7782697677612305,
+      "rewards/margins": 11.297493934631348,
+      "rewards/rejected": -8.519224166870117,
       "step": 350
     },
     {
       "epoch": 0.75,
-      "grad_norm": 201.50715466732544,
+      "grad_norm": 147.62661016854165,
       "learning_rate": 8.729103716819111e-08,
-      "logits/chosen": -1.6420570611953735,
-      "logits/rejected": -1.663745641708374,
-      "logps/chosen": -122.47066497802734,
-      "logps/rejected": -131.9073486328125,
-      "loss": 0.1543,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 4.703906059265137,
-      "rewards/margins": 14.480855941772461,
-      "rewards/rejected": -9.776951789855957,
+      "logits/chosen": -1.6380598545074463,
+      "logits/rejected": -1.6626033782958984,
+      "logps/chosen": -122.89555358886719,
+      "logps/rejected": -138.44113159179688,
+      "loss": 0.1469,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 3.050956964492798,
+      "rewards/margins": 10.875678062438965,
+      "rewards/rejected": -7.824721336364746,
       "step": 360
     },
     {
       "epoch": 0.77,
-      "grad_norm": 398.4959635020424,
+      "grad_norm": 250.3877641186192,
       "learning_rate": 7.387025063449081e-08,
-      "logits/chosen": -1.7505977153778076,
-      "logits/rejected": -1.6930338144302368,
-      "logps/chosen": -112.33251953125,
-      "logps/rejected": -121.57230377197266,
-      "loss": 0.1443,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 4.217165470123291,
-      "rewards/margins": 12.93463134765625,
-      "rewards/rejected": -8.7174654006958,
+      "logits/chosen": -1.749659776687622,
+      "logits/rejected": -1.6942018270492554,
+      "logps/chosen": -113.17866516113281,
+      "logps/rejected": -127.69132232666016,
+      "loss": 0.1488,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 2.642218828201294,
+      "rewards/margins": 9.677666664123535,
+      "rewards/rejected": -7.035447597503662,
       "step": 370
     },
     {
       "epoch": 0.79,
-      "grad_norm": 239.63863129657855,
+      "grad_norm": 179.11776110792954,
       "learning_rate": 6.138919252022435e-08,
-      "logits/chosen": -1.7603282928466797,
-      "logits/rejected": -1.7833961248397827,
-      "logps/chosen": -119.14958190917969,
-      "logps/rejected": -136.85977172851562,
-      "loss": 0.1576,
+      "logits/chosen": -1.7493488788604736,
+      "logits/rejected": -1.777907133102417,
+      "logps/chosen": -119.62767028808594,
+      "logps/rejected": -144.23379516601562,
+      "loss": 0.1547,
       "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 5.251246452331543,
-      "rewards/margins": 16.546478271484375,
-      "rewards/rejected": -11.295232772827148,
+      "rewards/chosen": 3.405210494995117,
+      "rewards/margins": 12.410168647766113,
+      "rewards/rejected": -9.004958152770996,
       "step": 380
     },
     {
       "epoch": 0.82,
-      "grad_norm": 197.58750581294214,
+      "grad_norm": 105.79142677310188,
       "learning_rate": 4.991445467064689e-08,
-      "logits/chosen": -1.656961441040039,
-      "logits/rejected": -1.6608669757843018,
-      "logps/chosen": -115.59519958496094,
-      "logps/rejected": -124.886474609375,
-      "loss": 0.1242,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 5.070186614990234,
-      "rewards/margins": 15.444877624511719,
-      "rewards/rejected": -10.3746919631958,
+      "logits/chosen": -1.6522718667984009,
+      "logits/rejected": -1.657579779624939,
+      "logps/chosen": -115.9373779296875,
+      "logps/rejected": -131.8628387451172,
+      "loss": 0.1191,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": 3.3116908073425293,
+      "rewards/margins": 11.62342643737793,
+      "rewards/rejected": -8.311735153198242,
       "step": 390
     },
     {
       "epoch": 0.84,
-      "grad_norm": 266.2162184394133,
+      "grad_norm": 160.26625520373477,
       "learning_rate": 3.9507259776993954e-08,
-      "logits/chosen": -1.636885643005371,
-      "logits/rejected": -1.713200330734253,
-      "logps/chosen": -117.88565826416016,
-      "logps/rejected": -124.86863708496094,
-      "loss": 0.2005,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 4.46274471282959,
-      "rewards/margins": 14.449310302734375,
-      "rewards/rejected": -9.986566543579102,
+      "logits/chosen": -1.6317713260650635,
+      "logits/rejected": -1.717016577720642,
+      "logps/chosen": -118.45027923583984,
+      "logps/rejected": -131.49853515625,
+      "loss": 0.2009,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 2.862239360809326,
+      "rewards/margins": 10.845926284790039,
+      "rewards/rejected": -7.983686923980713,
       "step": 400
     },
     {
       "epoch": 0.84,
-      "eval_logits/chosen": -1.8133598566055298,
-      "eval_logits/rejected": -1.8130455017089844,
-      "eval_logps/chosen": -119.97606658935547,
-      "eval_logps/rejected": -131.71229553222656,
-      "eval_loss": 0.14202240109443665,
-      "eval_rewards/accuracies": 0.92578125,
-      "eval_rewards/chosen": 5.338851451873779,
-      "eval_rewards/margins": 16.880319595336914,
-      "eval_rewards/rejected": -11.541468620300293,
-      "eval_runtime": 97.6019,
-      "eval_samples_per_second": 20.491,
+      "eval_logits/chosen": -1.8097559213638306,
+      "eval_logits/rejected": -1.8106379508972168,
+      "eval_logps/chosen": -120.3245849609375,
+      "eval_logps/rejected": -139.5015411376953,
+      "eval_loss": 0.1389647275209427,
+      "eval_rewards/accuracies": 0.9296875,
+      "eval_rewards/chosen": 3.489531993865967,
+      "eval_rewards/margins": 12.741693496704102,
+      "eval_rewards/rejected": -9.252161026000977,
+      "eval_runtime": 97.6986,
+      "eval_samples_per_second": 20.471,
       "eval_steps_per_second": 0.328,
       "step": 400
     },
     {
       "epoch": 0.86,
-      "grad_norm": 300.6588760835356,
+      "grad_norm": 192.9545547193168,
       "learning_rate": 3.022313472693447e-08,
-      "logits/chosen": -1.735870599746704,
-      "logits/rejected": -1.7638452053070068,
-      "logps/chosen": -128.670166015625,
-      "logps/rejected": -129.5255126953125,
-      "loss": 0.1508,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 5.593460559844971,
-      "rewards/margins": 16.819358825683594,
-      "rewards/rejected": -11.225897789001465,
+      "logits/chosen": -1.7275054454803467,
+      "logits/rejected": -1.756888747215271,
+      "logps/chosen": -129.18605041503906,
+      "logps/rejected": -136.9790496826172,
+      "loss": 0.1378,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 3.625795841217041,
+      "rewards/margins": 12.600431442260742,
+      "rewards/rejected": -8.974635124206543,
       "step": 410
     },
     {
       "epoch": 0.88,
-      "grad_norm": 184.26802651594184,
+      "grad_norm": 135.89148969120785,
       "learning_rate": 2.2111614344599684e-08,
-      "logits/chosen": -1.7033697366714478,
-      "logits/rejected": -1.704993486404419,
-      "logps/chosen": -113.72819519042969,
-      "logps/rejected": -124.34715270996094,
-      "loss": 0.1419,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 4.3744401931762695,
-      "rewards/margins": 14.479223251342773,
-      "rewards/rejected": -10.104782104492188,
+      "logits/chosen": -1.697636365890503,
+      "logits/rejected": -1.6976314783096313,
+      "logps/chosen": -114.35809326171875,
+      "logps/rejected": -131.75164794921875,
+      "loss": 0.1286,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 2.790315866470337,
+      "rewards/margins": 11.007737159729004,
+      "rewards/rejected": -8.21742057800293,
       "step": 420
     },
     {
       "epoch": 0.9,
-      "grad_norm": 269.33105520831003,
+      "grad_norm": 163.87841082901525,
       "learning_rate": 1.521597710086439e-08,
-      "logits/chosen": -1.6633046865463257,
-      "logits/rejected": -1.6779390573501587,
-      "logps/chosen": -129.4523162841797,
-      "logps/rejected": -124.72953796386719,
-      "loss": 0.154,
+      "logits/chosen": -1.6538774967193604,
+      "logits/rejected": -1.667636513710022,
+      "logps/chosen": -129.87210083007812,
+      "logps/rejected": -131.872802734375,
+      "loss": 0.1437,
       "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 4.997079372406006,
-      "rewards/margins": 14.370283126831055,
-      "rewards/rejected": -9.373201370239258,
+      "rewards/chosen": 3.2474303245544434,
+      "rewards/margins": 10.924890518188477,
+      "rewards/rejected": -7.677459716796875,
       "step": 430
     },
     {
       "epoch": 0.92,
-      "grad_norm": 181.5022378731684,
+      "grad_norm": 117.88131682798563,
       "learning_rate": 9.57301420397924e-09,
-      "logits/chosen": -1.7851531505584717,
-      "logits/rejected": -1.790157675743103,
-      "logps/chosen": -120.8554458618164,
-      "logps/rejected": -131.34410095214844,
-      "loss": 0.144,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": 5.435299873352051,
-      "rewards/margins": 16.81759262084961,
-      "rewards/rejected": -11.382290840148926,
+      "logits/chosen": -1.7787322998046875,
+      "logits/rejected": -1.7886247634887695,
+      "logps/chosen": -121.3507080078125,
+      "logps/rejected": -139.0587921142578,
+      "loss": 0.142,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 3.5244851112365723,
+      "rewards/margins": 12.655617713928223,
+      "rewards/rejected": -9.131133079528809,
       "step": 440
     },
     {
       "epoch": 0.94,
-      "grad_norm": 227.09412853715097,
+      "grad_norm": 179.270368210056,
       "learning_rate": 5.212833302556258e-09,
-      "logits/chosen": -1.7998348474502563,
-      "logits/rejected": -1.7618439197540283,
-      "logps/chosen": -116.46590423583984,
-      "logps/rejected": -127.42796325683594,
-      "loss": 0.159,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 5.652300834655762,
-      "rewards/margins": 17.369140625,
-      "rewards/rejected": -11.716839790344238,
+      "logits/chosen": -1.792822241783142,
+      "logits/rejected": -1.7586778402328491,
+      "logps/chosen": -116.64122009277344,
+      "logps/rejected": -135.26832580566406,
+      "loss": 0.1538,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 3.7331383228302,
+      "rewards/margins": 13.112436294555664,
+      "rewards/rejected": -9.379298210144043,
       "step": 450
     },
     {
       "epoch": 0.96,
-      "grad_norm": 225.69247261851913,
+      "grad_norm": 118.73739536412977,
       "learning_rate": 2.158697848236607e-09,
-      "logits/chosen": -1.6827681064605713,
-      "logits/rejected": -1.7046699523925781,
-      "logps/chosen": -120.79095458984375,
-      "logps/rejected": -130.98043823242188,
-      "loss": 0.1335,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 5.568859100341797,
-      "rewards/margins": 16.174407958984375,
-      "rewards/rejected": -10.605547904968262,
+      "logits/chosen": -1.6758760213851929,
+      "logits/rejected": -1.6996933221817017,
+      "logps/chosen": -120.97281646728516,
+      "logps/rejected": -138.555419921875,
+      "loss": 0.1252,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 3.676201581954956,
+      "rewards/margins": 12.26156234741211,
+      "rewards/rejected": -8.585359573364258,
       "step": 460
     },
     {
       "epoch": 0.98,
-      "grad_norm": 199.48903785359678,
+      "grad_norm": 152.28679005488013,
       "learning_rate": 4.269029751107489e-10,
-      "logits/chosen": -1.6656444072723389,
-      "logits/rejected": -1.6796722412109375,
-      "logps/chosen": -114.7574462890625,
-      "logps/rejected": -139.92117309570312,
-      "loss": 0.1514,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 4.486257553100586,
-      "rewards/margins": 15.330111503601074,
-      "rewards/rejected": -10.843853950500488,
+      "logits/chosen": -1.6590917110443115,
+      "logits/rejected": -1.6743139028549194,
+      "logps/chosen": -115.36566162109375,
+      "logps/rejected": -147.09939575195312,
+      "loss": 0.1469,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 2.869194269180298,
+      "rewards/margins": 11.534073829650879,
+      "rewards/rejected": -8.66487979888916,
       "step": 470
     },
     {
       "epoch": 1.0,
       "step": 478,
       "total_flos": 0.0,
-      "train_loss": 0.17749474911510196,
-      "train_runtime": 7645.2484,
-      "train_samples_per_second": 7.996,
+      "train_loss": 0.17478099153630405,
+      "train_runtime": 7645.1097,
+      "train_samples_per_second": 7.997,
       "train_steps_per_second": 0.063
     }
   ],