Model save

README.md

@@ -15,15 +15,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.1695
-- Rewards/chosen: 8.1916
-- Rewards/rejected: -16.5277
-- Rewards/accuracies: 0.9297
-- Rewards/margins: 24.7193
-- Logps/rejected: -126.2961
-- Logps/chosen: -121.3891
-- Logits/rejected: -1.8336
-- Logits/chosen: -1.8336
+- Loss: 0.1592
+- Rewards/chosen: 8.1420
+- Rewards/rejected: -16.5137
+- Rewards/accuracies: 0.9219
+- Rewards/margins: 24.6558
+- Logps/rejected: -126.2682
+- Logps/chosen: -121.4882
+- Logits/rejected: -1.8539
+- Logits/chosen: -1.8512
 
 ## Model description
 
@@ -60,10 +60,10 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.1474        | 0.21  | 100  | 0.1325          | 6.8739         | -14.1333         | 0.9297             | 21.0072         | -121.5073      | -124.0244    | -1.7724         | -1.7834       |
-| 0.2092        | 0.42  | 200  | 0.1567          | 8.4112         | -15.0418         | 0.9336             | 23.4530         | -123.3244      | -120.9499    | -1.8442         | -1.8477       |
-| 0.1925        | 0.63  | 300  | 0.1715          | 7.7458         | -16.7009         | 0.9258             | 24.4467         | -126.6425      | -122.2807    | -1.8047         | -1.8054       |
-| 0.2762        | 0.84  | 400  | 0.1695          | 8.1916         | -16.5277         | 0.9297             | 24.7193         | -126.2961      | -121.3891    | -1.8336         | -1.8336       |
+| 0.1483        | 0.21  | 100  | 0.1321          | 6.9750         | -14.0592         | 0.9336             | 21.0342         | -121.3591      | -123.8222    | -1.7705         | -1.7813       |
+| 0.2084        | 0.42  | 200  | 0.1510          | 8.5839         | -14.8837         | 0.9375             | 23.4677         | -123.0082      | -120.6044    | -1.8622         | -1.8630       |
+| 0.2072        | 0.63  | 300  | 0.1658          | 7.7715         | -16.5062         | 0.9219             | 24.2778         | -126.2532      | -122.2292    | -1.8451         | -1.8438       |
+| 0.1918        | 0.84  | 400  | 0.1592          | 8.1420         | -16.5137         | 0.9219             | 24.6558         | -126.2682      | -121.4882    | -1.8539         | -1.8512       |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.20260818084413537,
-    "train_runtime": 7642.1158,
+    "train_loss": 0.19963366765856244,
+    "train_runtime": 7635.6428,
     "train_samples": 61135,
-    "train_samples_per_second": 8.0,
+    "train_samples_per_second": 8.007,
     "train_steps_per_second": 0.063
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c236809df2b9a9098eb9f2d34c776fda00f7a9d1bf22382bf81f4f3a61e886d2
+oid sha256:500e19fac2c01478bc3db38253938d7d32fe3a63f7f11d0a98fa9dff084a904e
 size 4943178720

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:01733d5f594df90f00ce7815c23e992b809250faa4a0178be321b9b7ac687a96
+oid sha256:7f98248bb1cf0b17228033770296f1cc3c11b5cb0eaaa4555e57ebcd8e5b29fd
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9bc2f19df9fee5aa320225d07ac06c4aa6645bf7d9d004feb7974633020ea43
+oid sha256:c3c37844ea3f77105b46b9f89e25d911069f6a73e947fd25ea4d94229b5ec676
 size 4540532728

runs/May14_05-53-23_n136-129-074/events.out.tfevents.1715637339.n136-129-074.2416009.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fc57ba32bf47fb9299971c8b52d04a8c670c96f5a3c7f1ec30b9170b7f126b3e
-size 35922
+oid sha256:efa4f90b78a98e4de5fef242fc36ae4907948f8f7289fe00fead4bfc0b88bf28
+size 41092

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.20260818084413537,
-    "train_runtime": 7642.1158,
+    "train_loss": 0.19963366765856244,
+    "train_runtime": 7635.6428,
     "train_samples": 61135,
-    "train_samples_per_second": 8.0,
+    "train_samples_per_second": 8.007,
     "train_steps_per_second": 0.063
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 2193.325330109938,
+      "grad_norm": 2193.4714669768246,
       "learning_rate": 1.0416666666666666e-08,
       "logits/chosen": -1.689455509185791,
       "logits/rejected": -1.4794573783874512,
@@ -25,780 +25,780 @@
     },
     {
       "epoch": 0.02,
-      "grad_norm": 1722.3750513007058,
+      "grad_norm": 1601.7083484310278,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -1.7072248458862305,
-      "logits/rejected": -1.6097939014434814,
-      "logps/chosen": -139.610107421875,
-      "logps/rejected": -91.32671356201172,
-      "loss": 0.6992,
-      "rewards/accuracies": 0.4722222089767456,
-      "rewards/chosen": 0.05638541281223297,
-      "rewards/margins": 0.07169164717197418,
-      "rewards/rejected": -0.015306234359741211,
+      "logits/chosen": -1.7075374126434326,
+      "logits/rejected": -1.610562801361084,
+      "logps/chosen": -139.61968994140625,
+      "logps/rejected": -91.38065338134766,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.051607053726911545,
+      "rewards/margins": 0.09388630837202072,
+      "rewards/rejected": -0.042279258370399475,
       "step": 10
     },
     {
       "epoch": 0.04,
-      "grad_norm": 506.4974128593864,
+      "grad_norm": 542.7830677289153,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -1.6386321783065796,
-      "logits/rejected": -1.6488521099090576,
-      "logps/chosen": -130.5824737548828,
-      "logps/rejected": -94.01884460449219,
-      "loss": 0.3992,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 1.0460855960845947,
-      "rewards/margins": 1.4136831760406494,
-      "rewards/rejected": -0.3675975203514099,
+      "logits/chosen": -1.639533281326294,
+      "logits/rejected": -1.6490185260772705,
+      "logps/chosen": -130.57711791992188,
+      "logps/rejected": -94.0321044921875,
+      "loss": 0.4027,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 1.0487556457519531,
+      "rewards/margins": 1.4229849576950073,
+      "rewards/rejected": -0.3742292523384094,
       "step": 20
     },
     {
       "epoch": 0.06,
-      "grad_norm": 471.66591122190783,
+      "grad_norm": 556.5878829215873,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -1.6968982219696045,
-      "logits/rejected": -1.6314315795898438,
-      "logps/chosen": -131.55714416503906,
-      "logps/rejected": -105.11051177978516,
-      "loss": 0.2288,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 3.3202426433563232,
-      "rewards/margins": 5.000655174255371,
-      "rewards/rejected": -1.680413007736206,
+      "logits/chosen": -1.6960318088531494,
+      "logits/rejected": -1.6311280727386475,
+      "logps/chosen": -131.44830322265625,
+      "logps/rejected": -105.1076431274414,
+      "loss": 0.2285,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 3.3746559619903564,
+      "rewards/margins": 5.053637981414795,
+      "rewards/rejected": -1.6789820194244385,
       "step": 30
     },
     {
       "epoch": 0.08,
-      "grad_norm": 420.8471101103042,
+      "grad_norm": 453.8019698930014,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -1.6393680572509766,
-      "logits/rejected": -1.560689926147461,
-      "logps/chosen": -142.81134033203125,
-      "logps/rejected": -108.94673156738281,
-      "loss": 0.1804,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 4.454810619354248,
-      "rewards/margins": 9.642589569091797,
-      "rewards/rejected": -5.187777996063232,
+      "logits/chosen": -1.6398160457611084,
+      "logits/rejected": -1.5611240863800049,
+      "logps/chosen": -142.69259643554688,
+      "logps/rejected": -108.96995544433594,
+      "loss": 0.1824,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 4.514187335968018,
+      "rewards/margins": 9.713580131530762,
+      "rewards/rejected": -5.199393272399902,
       "step": 40
     },
     {
       "epoch": 0.1,
-      "grad_norm": 759.4656267159737,
+      "grad_norm": 749.5022801510694,
       "learning_rate": 4.999733114418725e-07,
-      "logits/chosen": -1.6069341897964478,
-      "logits/rejected": -1.631784439086914,
-      "logps/chosen": -126.6369400024414,
-      "logps/rejected": -116.30027770996094,
-      "loss": 0.1844,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 4.2187395095825195,
-      "rewards/margins": 12.810827255249023,
-      "rewards/rejected": -8.59208869934082,
+      "logits/chosen": -1.6066747903823853,
+      "logits/rejected": -1.6321827173233032,
+      "logps/chosen": -126.60260009765625,
+      "logps/rejected": -116.45572662353516,
+      "loss": 0.1818,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 4.235908508300781,
+      "rewards/margins": 12.905720710754395,
+      "rewards/rejected": -8.66981029510498,
       "step": 50
     },
     {
       "epoch": 0.13,
-      "grad_norm": 361.53910661789246,
+      "grad_norm": 373.1578806081686,
       "learning_rate": 4.990398100856366e-07,
-      "logits/chosen": -1.7161242961883545,
-      "logits/rejected": -1.661266565322876,
-      "logps/chosen": -141.4559326171875,
-      "logps/rejected": -125.5120620727539,
-      "loss": 0.17,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 4.772133827209473,
-      "rewards/margins": 16.64890480041504,
-      "rewards/rejected": -11.876771926879883,
+      "logits/chosen": -1.713249921798706,
+      "logits/rejected": -1.6583478450775146,
+      "logps/chosen": -141.51318359375,
+      "logps/rejected": -125.83363342285156,
+      "loss": 0.171,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": 4.743514537811279,
+      "rewards/margins": 16.781068801879883,
+      "rewards/rejected": -12.037551879882812,
       "step": 60
     },
     {
       "epoch": 0.15,
-      "grad_norm": 382.2685121568309,
+      "grad_norm": 362.49470680909604,
       "learning_rate": 4.967775735898179e-07,
-      "logits/chosen": -1.670915961265564,
-      "logits/rejected": -1.6990423202514648,
-      "logps/chosen": -134.56948852539062,
-      "logps/rejected": -126.52008056640625,
-      "loss": 0.1751,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 5.264647006988525,
-      "rewards/margins": 19.5419979095459,
-      "rewards/rejected": -14.277349472045898,
+      "logits/chosen": -1.6701009273529053,
+      "logits/rejected": -1.698505163192749,
+      "logps/chosen": -134.6632537841797,
+      "logps/rejected": -126.83392333984375,
+      "loss": 0.1747,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 5.217758655548096,
+      "rewards/margins": 19.652034759521484,
+      "rewards/rejected": -14.434274673461914,
       "step": 70
     },
     {
       "epoch": 0.17,
-      "grad_norm": 337.7204736269515,
+      "grad_norm": 361.796137554066,
       "learning_rate": 4.931986719649298e-07,
-      "logits/chosen": -1.658257246017456,
-      "logits/rejected": -1.6444251537322998,
-      "logps/chosen": -128.6245880126953,
-      "logps/rejected": -119.48954010009766,
-      "loss": 0.1656,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 5.917626857757568,
-      "rewards/margins": 18.79892349243164,
-      "rewards/rejected": -12.88129711151123,
+      "logits/chosen": -1.6567662954330444,
+      "logits/rejected": -1.6445653438568115,
+      "logps/chosen": -128.80026245117188,
+      "logps/rejected": -119.8623046875,
+      "loss": 0.1674,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 5.829800128936768,
+      "rewards/margins": 18.897483825683594,
+      "rewards/rejected": -13.0676851272583,
       "step": 80
     },
     {
       "epoch": 0.19,
-      "grad_norm": 658.102190064697,
+      "grad_norm": 567.4822004779005,
       "learning_rate": 4.883222001996351e-07,
-      "logits/chosen": -1.6909303665161133,
-      "logits/rejected": -1.710211157798767,
-      "logps/chosen": -136.05978393554688,
-      "logps/rejected": -131.3511199951172,
-      "loss": 0.1561,
+      "logits/chosen": -1.689691185951233,
+      "logits/rejected": -1.7095468044281006,
+      "logps/chosen": -136.06716918945312,
+      "logps/rejected": -131.6322784423828,
+      "loss": 0.1532,
       "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 6.575238227844238,
-      "rewards/margins": 21.304401397705078,
-      "rewards/rejected": -14.729164123535156,
+      "rewards/chosen": 6.571531772613525,
+      "rewards/margins": 21.44127655029297,
+      "rewards/rejected": -14.869746208190918,
       "step": 90
     },
     {
       "epoch": 0.21,
-      "grad_norm": 303.0257944851465,
+      "grad_norm": 313.1047143857541,
       "learning_rate": 4.821741763807186e-07,
-      "logits/chosen": -1.739341139793396,
-      "logits/rejected": -1.722266435623169,
-      "logps/chosen": -116.35429382324219,
-      "logps/rejected": -117.0191421508789,
-      "loss": 0.1474,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 5.8117475509643555,
-      "rewards/margins": 18.994068145751953,
-      "rewards/rejected": -13.182321548461914,
+      "logits/chosen": -1.7361831665039062,
+      "logits/rejected": -1.719515085220337,
+      "logps/chosen": -116.3790054321289,
+      "logps/rejected": -117.08357238769531,
+      "loss": 0.1483,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 5.799386501312256,
+      "rewards/margins": 19.013914108276367,
+      "rewards/rejected": -13.21452808380127,
       "step": 100
     },
     {
       "epoch": 0.21,
-      "eval_logits/chosen": -1.783400058746338,
-      "eval_logits/rejected": -1.7724130153656006,
-      "eval_logps/chosen": -124.02442932128906,
-      "eval_logps/rejected": -121.5073013305664,
-      "eval_loss": 0.13251493871212006,
-      "eval_rewards/accuracies": 0.9296875,
-      "eval_rewards/chosen": 6.873911380767822,
-      "eval_rewards/margins": 21.00718879699707,
-      "eval_rewards/rejected": -14.133277893066406,
-      "eval_runtime": 97.5707,
-      "eval_samples_per_second": 20.498,
-      "eval_steps_per_second": 0.328,
+      "eval_logits/chosen": -1.7813222408294678,
+      "eval_logits/rejected": -1.7704529762268066,
+      "eval_logps/chosen": -123.8221664428711,
+      "eval_logps/rejected": -121.35905456542969,
+      "eval_loss": 0.13205325603485107,
+      "eval_rewards/accuracies": 0.93359375,
+      "eval_rewards/chosen": 6.975039482116699,
+      "eval_rewards/margins": 21.034198760986328,
+      "eval_rewards/rejected": -14.059160232543945,
+      "eval_runtime": 97.7776,
+      "eval_samples_per_second": 20.455,
+      "eval_steps_per_second": 0.327,
       "step": 100
     },
     {
       "epoch": 0.23,
-      "grad_norm": 311.0110467993666,
+      "grad_norm": 297.03412259661775,
       "learning_rate": 4.747874028753375e-07,
-      "logits/chosen": -1.6540296077728271,
-      "logits/rejected": -1.7143176794052124,
-      "logps/chosen": -120.4977035522461,
-      "logps/rejected": -124.83198547363281,
-      "loss": 0.1601,
+      "logits/chosen": -1.6531705856323242,
+      "logits/rejected": -1.7122666835784912,
+      "logps/chosen": -120.41741943359375,
+      "logps/rejected": -124.6469497680664,
+      "loss": 0.1606,
       "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 5.774500370025635,
-      "rewards/margins": 17.103343963623047,
-      "rewards/rejected": -11.32884407043457,
+      "rewards/chosen": 5.814640998840332,
+      "rewards/margins": 17.050968170166016,
+      "rewards/rejected": -11.236326217651367,
       "step": 110
     },
     {
       "epoch": 0.25,
-      "grad_norm": 317.1259568395023,
+      "grad_norm": 308.6589257383676,
       "learning_rate": 4.662012913161997e-07,
-      "logits/chosen": -1.6986665725708008,
-      "logits/rejected": -1.6894630193710327,
-      "logps/chosen": -118.8506851196289,
-      "logps/rejected": -125.60331726074219,
-      "loss": 0.1522,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 6.9937944412231445,
-      "rewards/margins": 19.12407875061035,
-      "rewards/rejected": -12.130284309387207,
+      "logits/chosen": -1.7084993124008179,
+      "logits/rejected": -1.699831247329712,
+      "logps/chosen": -118.6161880493164,
+      "logps/rejected": -125.55290222167969,
+      "loss": 0.1532,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 7.1110429763793945,
+      "rewards/margins": 19.216121673583984,
+      "rewards/rejected": -12.105077743530273,
       "step": 120
     },
     {
       "epoch": 0.27,
-      "grad_norm": 183.0416909345485,
+      "grad_norm": 215.2839633523937,
       "learning_rate": 4.5646165232345103e-07,
-      "logits/chosen": -1.6666994094848633,
-      "logits/rejected": -1.6851320266723633,
-      "logps/chosen": -124.86369323730469,
-      "logps/rejected": -121.08885192871094,
-      "loss": 0.1572,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 7.925422668457031,
-      "rewards/margins": 21.82646942138672,
-      "rewards/rejected": -13.901046752929688,
+      "logits/chosen": -1.6880385875701904,
+      "logits/rejected": -1.708009123802185,
+      "logps/chosen": -124.7245101928711,
+      "logps/rejected": -121.17189025878906,
+      "loss": 0.1588,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 7.9950103759765625,
+      "rewards/margins": 21.937570571899414,
+      "rewards/rejected": -13.942561149597168,
       "step": 130
     },
     {
       "epoch": 0.29,
-      "grad_norm": 323.2169573441021,
+      "grad_norm": 353.0707817255302,
       "learning_rate": 4.456204510851956e-07,
-      "logits/chosen": -1.5770366191864014,
-      "logits/rejected": -1.5510694980621338,
-      "logps/chosen": -122.3641128540039,
-      "logps/rejected": -117.8587646484375,
-      "loss": 0.1593,
+      "logits/chosen": -1.6008964776992798,
+      "logits/rejected": -1.578404188156128,
+      "logps/chosen": -122.19905853271484,
+      "logps/rejected": -118.03678131103516,
+      "loss": 0.1592,
       "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 7.602517604827881,
-      "rewards/margins": 19.99860954284668,
-      "rewards/rejected": -12.396090507507324,
+      "rewards/chosen": 7.6850481033325195,
+      "rewards/margins": 20.170154571533203,
+      "rewards/rejected": -12.485105514526367,
       "step": 140
     },
     {
       "epoch": 0.31,
-      "grad_norm": 394.76094620235625,
+      "grad_norm": 387.84134751485317,
       "learning_rate": 4.337355301007335e-07,
-      "logits/chosen": -1.717551827430725,
-      "logits/rejected": -1.7436870336532593,
-      "logps/chosen": -120.7677001953125,
-      "logps/rejected": -116.0407943725586,
-      "loss": 0.135,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 7.475555419921875,
-      "rewards/margins": 20.28847885131836,
-      "rewards/rejected": -12.812922477722168,
+      "logits/chosen": -1.73464035987854,
+      "logits/rejected": -1.7599799633026123,
+      "logps/chosen": -120.38832092285156,
+      "logps/rejected": -115.9221420288086,
+      "loss": 0.1376,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 7.665247917175293,
+      "rewards/margins": 20.418846130371094,
+      "rewards/rejected": -12.753596305847168,
       "step": 150
     },
     {
       "epoch": 0.33,
-      "grad_norm": 696.9618085200943,
+      "grad_norm": 695.9762417445575,
       "learning_rate": 4.2087030056579986e-07,
-      "logits/chosen": -1.5951331853866577,
-      "logits/rejected": -1.5702898502349854,
-      "logps/chosen": -128.13714599609375,
-      "logps/rejected": -122.9395751953125,
-      "loss": 0.1778,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 6.914339542388916,
-      "rewards/margins": 18.91141700744629,
-      "rewards/rejected": -11.997075080871582,
+      "logits/chosen": -1.6107909679412842,
+      "logits/rejected": -1.5863116979599,
+      "logps/chosen": -127.7414779663086,
+      "logps/rejected": -122.50279235839844,
+      "loss": 0.1702,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 7.1121721267700195,
+      "rewards/margins": 18.89084243774414,
+      "rewards/rejected": -11.778670310974121,
       "step": 160
     },
     {
       "epoch": 0.36,
-      "grad_norm": 534.2420908517273,
+      "grad_norm": 501.7172276328359,
       "learning_rate": 4.070934040463998e-07,
-      "logits/chosen": -1.7780072689056396,
-      "logits/rejected": -1.7594010829925537,
-      "logps/chosen": -123.1135025024414,
-      "logps/rejected": -123.6323013305664,
-      "loss": 0.1817,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 8.014005661010742,
-      "rewards/margins": 21.994348526000977,
-      "rewards/rejected": -13.980344772338867,
+      "logits/chosen": -1.7882769107818604,
+      "logits/rejected": -1.7692320346832275,
+      "logps/chosen": -122.8949966430664,
+      "logps/rejected": -123.34075927734375,
+      "loss": 0.1729,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 8.123270034790039,
+      "rewards/margins": 21.957849502563477,
+      "rewards/rejected": -13.834577560424805,
       "step": 170
     },
     {
       "epoch": 0.38,
-      "grad_norm": 515.866307384091,
+      "grad_norm": 517.7468532784978,
       "learning_rate": 3.9247834624635404e-07,
-      "logits/chosen": -1.7234785556793213,
-      "logits/rejected": -1.715887427330017,
-      "logps/chosen": -126.28702545166016,
-      "logps/rejected": -118.8428726196289,
-      "loss": 0.1945,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 8.433076858520508,
-      "rewards/margins": 22.419658660888672,
-      "rewards/rejected": -13.986579895019531,
+      "logits/chosen": -1.7249009609222412,
+      "logits/rejected": -1.7209745645523071,
+      "logps/chosen": -126.01353454589844,
+      "logps/rejected": -118.68614196777344,
+      "loss": 0.1857,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 8.569825172424316,
+      "rewards/margins": 22.478046417236328,
+      "rewards/rejected": -13.908220291137695,
       "step": 180
     },
     {
       "epoch": 0.4,
-      "grad_norm": 442.14688359180957,
+      "grad_norm": 407.33922019202464,
       "learning_rate": 3.7710310482256523e-07,
-      "logits/chosen": -1.7688430547714233,
-      "logits/rejected": -1.7464864253997803,
-      "logps/chosen": -115.27445983886719,
-      "logps/rejected": -132.3951416015625,
-      "loss": 0.1843,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 8.262548446655273,
-      "rewards/margins": 23.41249656677246,
-      "rewards/rejected": -15.149948120117188,
+      "logits/chosen": -1.7666248083114624,
+      "logits/rejected": -1.7464529275894165,
+      "logps/chosen": -114.92606353759766,
+      "logps/rejected": -131.9868927001953,
+      "loss": 0.188,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 8.436745643615723,
+      "rewards/margins": 23.382564544677734,
+      "rewards/rejected": -14.945820808410645,
       "step": 190
     },
     {
       "epoch": 0.42,
-      "grad_norm": 436.2379767277873,
+      "grad_norm": 470.1346443625949,
       "learning_rate": 3.610497133404795e-07,
-      "logits/chosen": -1.7461057901382446,
-      "logits/rejected": -1.7516930103302002,
-      "logps/chosen": -121.12789154052734,
-      "logps/rejected": -116.14985656738281,
-      "loss": 0.2092,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 9.004249572753906,
-      "rewards/margins": 23.075119018554688,
-      "rewards/rejected": -14.070871353149414,
+      "logits/chosen": -1.750284194946289,
+      "logits/rejected": -1.7601133584976196,
+      "logps/chosen": -120.80158996582031,
+      "logps/rejected": -115.9892807006836,
+      "loss": 0.2084,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 9.167404174804688,
+      "rewards/margins": 23.158000946044922,
+      "rewards/rejected": -13.990595817565918,
       "step": 200
     },
     {
       "epoch": 0.42,
-      "eval_logits/chosen": -1.8477407693862915,
-      "eval_logits/rejected": -1.844199538230896,
-      "eval_logps/chosen": -120.94989776611328,
-      "eval_logps/rejected": -123.32437896728516,
-      "eval_loss": 0.15667258203029633,
-      "eval_rewards/accuracies": 0.93359375,
-      "eval_rewards/chosen": 8.411178588867188,
-      "eval_rewards/margins": 23.453001022338867,
-      "eval_rewards/rejected": -15.04181957244873,
-      "eval_runtime": 97.6977,
-      "eval_samples_per_second": 20.471,
-      "eval_steps_per_second": 0.328,
+      "eval_logits/chosen": -1.8629834651947021,
+      "eval_logits/rejected": -1.8622163534164429,
+      "eval_logps/chosen": -120.60441589355469,
+      "eval_logps/rejected": -123.00822448730469,
+      "eval_loss": 0.15096141397953033,
+      "eval_rewards/accuracies": 0.9375,
+      "eval_rewards/chosen": 8.583921432495117,
+      "eval_rewards/margins": 23.467666625976562,
+      "eval_rewards/rejected": -14.883745193481445,
+      "eval_runtime": 97.7447,
+      "eval_samples_per_second": 20.461,
+      "eval_steps_per_second": 0.327,
       "step": 200
     },
     {
       "epoch": 0.44,
-      "grad_norm": 437.460750971005,
+      "grad_norm": 460.33381421539644,
       "learning_rate": 3.4440382358952115e-07,
-      "logits/chosen": -1.6800906658172607,
-      "logits/rejected": -1.6162738800048828,
-      "logps/chosen": -119.17988586425781,
-      "logps/rejected": -112.8397445678711,
-      "loss": 0.1771,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 6.705443382263184,
-      "rewards/margins": 18.32439613342285,
-      "rewards/rejected": -11.618951797485352,
+      "logits/chosen": -1.7009090185165405,
+      "logits/rejected": -1.6389843225479126,
+      "logps/chosen": -119.08918762207031,
+      "logps/rejected": -112.73565673828125,
+      "loss": 0.1747,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 6.750785827636719,
+      "rewards/margins": 18.317689895629883,
+      "rewards/rejected": -11.566904067993164,
       "step": 210
     },
     {
       "epoch": 0.46,
-      "grad_norm": 298.75015782601395,
+      "grad_norm": 293.2833966309726,
       "learning_rate": 3.272542485937368e-07,
-      "logits/chosen": -1.8080447912216187,
-      "logits/rejected": -1.8503248691558838,
-      "logps/chosen": -120.43497467041016,
-      "logps/rejected": -113.6927719116211,
-      "loss": 0.1853,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 8.020153045654297,
-      "rewards/margins": 21.10222053527832,
-      "rewards/rejected": -13.082064628601074,
+      "logits/chosen": -1.8287239074707031,
+      "logits/rejected": -1.8750501871109009,
+      "logps/chosen": -120.6048583984375,
+      "logps/rejected": -113.45402526855469,
+      "loss": 0.1905,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 7.93521785736084,
+      "rewards/margins": 20.89790916442871,
+      "rewards/rejected": -12.962692260742188,
       "step": 220
     },
     {
       "epoch": 0.48,
-      "grad_norm": 290.6447474042267,
+      "grad_norm": 186.44723799481827,
       "learning_rate": 3.096924887558854e-07,
-      "logits/chosen": -1.7483365535736084,
-      "logits/rejected": -1.6977851390838623,
-      "logps/chosen": -132.64036560058594,
-      "logps/rejected": -132.7303009033203,
-      "loss": 0.2348,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 7.534226894378662,
-      "rewards/margins": 23.992801666259766,
-      "rewards/rejected": -16.458572387695312,
+      "logits/chosen": -1.7752487659454346,
+      "logits/rejected": -1.728193998336792,
+      "logps/chosen": -132.70101928710938,
+      "logps/rejected": -132.2133331298828,
+      "loss": 0.2334,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 7.50391149520874,
+      "rewards/margins": 23.704010009765625,
+      "rewards/rejected": -16.200098037719727,
       "step": 230
     },
     {
       "epoch": 0.5,
-      "grad_norm": 435.0583755248922,
+      "grad_norm": 492.6328669399213,
       "learning_rate": 2.9181224366319943e-07,
-      "logits/chosen": -1.7618516683578491,
-      "logits/rejected": -1.7764402627944946,
-      "logps/chosen": -119.7364273071289,
-      "logps/rejected": -124.50996398925781,
-      "loss": 0.2076,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 6.094353675842285,
-      "rewards/margins": 19.534603118896484,
-      "rewards/rejected": -13.440248489379883,
+      "logits/chosen": -1.7902244329452515,
+      "logits/rejected": -1.8099979162216187,
+      "logps/chosen": -119.7879409790039,
+      "logps/rejected": -124.16999816894531,
+      "loss": 0.2101,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 6.068596363067627,
+      "rewards/margins": 19.338857650756836,
+      "rewards/rejected": -13.270261764526367,
       "step": 240
     },
     {
       "epoch": 0.52,
-      "grad_norm": 506.7749182741835,
+      "grad_norm": 497.9717087136447,
       "learning_rate": 2.7370891215954565e-07,
-      "logits/chosen": -1.7425343990325928,
-      "logits/rejected": -1.7735121250152588,
-      "logps/chosen": -123.32588195800781,
-      "logps/rejected": -126.53437805175781,
-      "loss": 0.2561,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 7.04464054107666,
-      "rewards/margins": 21.820133209228516,
-      "rewards/rejected": -14.775495529174805,
+      "logits/chosen": -1.7690420150756836,
+      "logits/rejected": -1.8053967952728271,
+      "logps/chosen": -123.1581039428711,
+      "logps/rejected": -126.19282531738281,
+      "loss": 0.232,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 7.128526210784912,
+      "rewards/margins": 21.73324966430664,
+      "rewards/rejected": -14.604723930358887,
       "step": 250
     },
     {
       "epoch": 0.54,
-      "grad_norm": 278.0299175856851,
+      "grad_norm": 219.15496679415526,
       "learning_rate": 2.55479083351317e-07,
-      "logits/chosen": -1.7782995700836182,
-      "logits/rejected": -1.7973874807357788,
-      "logps/chosen": -127.03205871582031,
-      "logps/rejected": -114.83714294433594,
-      "loss": 0.1749,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 8.397477149963379,
-      "rewards/margins": 23.37790298461914,
-      "rewards/rejected": -14.980427742004395,
+      "logits/chosen": -1.8088661432266235,
+      "logits/rejected": -1.830862045288086,
+      "logps/chosen": -126.8784408569336,
+      "logps/rejected": -114.4140396118164,
+      "loss": 0.172,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 8.474285125732422,
+      "rewards/margins": 23.2431583404541,
+      "rewards/rejected": -14.768875122070312,
       "step": 260
     },
     {
       "epoch": 0.56,
-      "grad_norm": 372.30589353402263,
+      "grad_norm": 339.5863335988586,
       "learning_rate": 2.3722002126275822e-07,
-      "logits/chosen": -1.7808539867401123,
-      "logits/rejected": -1.719679832458496,
-      "logps/chosen": -117.4173355102539,
-      "logps/rejected": -119.30290222167969,
-      "loss": 0.1772,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 7.320569038391113,
-      "rewards/margins": 21.35511589050293,
-      "rewards/rejected": -14.034547805786133,
+      "logits/chosen": -1.8175914287567139,
+      "logits/rejected": -1.7589060068130493,
+      "logps/chosen": -117.0767593383789,
+      "logps/rejected": -118.7080078125,
+      "loss": 0.1731,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 7.490860939025879,
+      "rewards/margins": 21.22795295715332,
+      "rewards/rejected": -13.737092971801758,
       "step": 270
     },
     {
       "epoch": 0.59,
-      "grad_norm": 470.22249240997917,
+      "grad_norm": 442.5487563129449,
       "learning_rate": 2.19029145890313e-07,
-      "logits/chosen": -1.6393606662750244,
-      "logits/rejected": -1.7276159524917603,
-      "logps/chosen": -123.42005920410156,
-      "logps/rejected": -132.44947814941406,
-      "loss": 0.2305,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 6.818305969238281,
-      "rewards/margins": 22.368270874023438,
-      "rewards/rejected": -15.549966812133789,
+      "logits/chosen": -1.678240418434143,
+      "logits/rejected": -1.765446424484253,
+      "logps/chosen": -123.12864685058594,
+      "logps/rejected": -131.90528869628906,
+      "loss": 0.2242,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 6.964009761810303,
+      "rewards/margins": 22.241891860961914,
+      "rewards/rejected": -15.27788257598877,
       "step": 280
     },
     {
       "epoch": 0.61,
-      "grad_norm": 652.1091329463337,
+      "grad_norm": 445.7460142068266,
       "learning_rate": 2.0100351342479216e-07,
-      "logits/chosen": -1.7318487167358398,
-      "logits/rejected": -1.706648826599121,
-      "logps/chosen": -114.55335998535156,
-      "logps/rejected": -119.69996643066406,
+      "logits/chosen": -1.771707534790039,
+      "logits/rejected": -1.7493677139282227,
+      "logps/chosen": -114.3892593383789,
+      "logps/rejected": -119.188720703125,
       "loss": 0.2179,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 6.804747104644775,
-      "rewards/margins": 22.096506118774414,
-      "rewards/rejected": -15.29175853729248,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 6.886797904968262,
+      "rewards/margins": 21.92293930053711,
+      "rewards/rejected": -15.036142349243164,
       "step": 290
     },
     {
       "epoch": 0.63,
-      "grad_norm": 426.10148588025714,
+      "grad_norm": 347.93449539220404,
       "learning_rate": 1.8323929841460178e-07,
-      "logits/chosen": -1.699927568435669,
-      "logits/rejected": -1.6480505466461182,
-      "logps/chosen": -132.07632446289062,
-      "logps/rejected": -136.4704132080078,
-      "loss": 0.1925,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 7.1819257736206055,
-      "rewards/margins": 22.29709243774414,
-      "rewards/rejected": -15.11516284942627,
+      "logits/chosen": -1.7396576404571533,
+      "logits/rejected": -1.6877906322479248,
+      "logps/chosen": -132.26669311523438,
+      "logps/rejected": -135.92764282226562,
+      "loss": 0.2072,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 7.086742401123047,
+      "rewards/margins": 21.930522918701172,
+      "rewards/rejected": -14.843778610229492,
       "step": 300
     },
     {
       "epoch": 0.63,
-      "eval_logits/chosen": -1.8054125308990479,
-      "eval_logits/rejected": -1.804688572883606,
-      "eval_logps/chosen": -122.28070068359375,
-      "eval_logps/rejected": -126.64250946044922,
-      "eval_loss": 0.17152084410190582,
-      "eval_rewards/accuracies": 0.92578125,
-      "eval_rewards/chosen": 7.745772361755371,
-      "eval_rewards/margins": 24.44666290283203,
-      "eval_rewards/rejected": -16.700889587402344,
-      "eval_runtime": 97.5667,
-      "eval_samples_per_second": 20.499,
+      "eval_logits/chosen": -1.84382963180542,
+      "eval_logits/rejected": -1.8451404571533203,
+      "eval_logps/chosen": -122.22917175292969,
+      "eval_logps/rejected": -126.25321960449219,
+      "eval_loss": 0.16582313179969788,
+      "eval_rewards/accuracies": 0.921875,
+      "eval_rewards/chosen": 7.771534442901611,
+      "eval_rewards/margins": 24.27777671813965,
+      "eval_rewards/rejected": -16.506242752075195,
+      "eval_runtime": 97.6572,
+      "eval_samples_per_second": 20.48,
       "eval_steps_per_second": 0.328,
       "step": 300
     },
     {
       "epoch": 0.65,
-      "grad_norm": 499.48262107192954,
+      "grad_norm": 522.5571001977761,
       "learning_rate": 1.6583128063291573e-07,
-      "logits/chosen": -1.7035776376724243,
-      "logits/rejected": -1.73648202419281,
-      "logps/chosen": -123.8221664428711,
-      "logps/rejected": -127.0335464477539,
-      "loss": 0.1993,
+      "logits/chosen": -1.7435277700424194,
+      "logits/rejected": -1.7735698223114014,
+      "logps/chosen": -123.75146484375,
+      "logps/rejected": -126.75709533691406,
+      "loss": 0.2065,
       "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 7.757756233215332,
-      "rewards/margins": 22.004987716674805,
-      "rewards/rejected": -14.247230529785156,
+      "rewards/chosen": 7.7931108474731445,
+      "rewards/margins": 21.902111053466797,
+      "rewards/rejected": -14.109001159667969,
       "step": 310
     },
     {
       "epoch": 0.67,
-      "grad_norm": 351.5168221076795,
+      "grad_norm": 355.22236619382966,
       "learning_rate": 1.488723393865766e-07,
-      "logits/chosen": -1.7380039691925049,
-      "logits/rejected": -1.7079343795776367,
-      "logps/chosen": -113.3768081665039,
-      "logps/rejected": -126.8635482788086,
-      "loss": 0.1891,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 8.71003532409668,
-      "rewards/margins": 24.014118194580078,
-      "rewards/rejected": -15.304081916809082,
+      "logits/chosen": -1.7776333093643188,
+      "logits/rejected": -1.7483892440795898,
+      "logps/chosen": -113.23478698730469,
+      "logps/rejected": -126.3783950805664,
+      "loss": 0.1913,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 8.781044006347656,
+      "rewards/margins": 23.842554092407227,
+      "rewards/rejected": -15.061508178710938,
       "step": 320
     },
     {
       "epoch": 0.69,
-      "grad_norm": 321.010971408508,
+      "grad_norm": 322.18484805793145,
       "learning_rate": 1.3245295796480788e-07,
-      "logits/chosen": -1.664240837097168,
-      "logits/rejected": -1.7476810216903687,
-      "logps/chosen": -123.50030517578125,
-      "logps/rejected": -122.0934066772461,
-      "loss": 0.1955,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 7.341310024261475,
-      "rewards/margins": 21.513980865478516,
-      "rewards/rejected": -14.172670364379883,
+      "logits/chosen": -1.7006194591522217,
+      "logits/rejected": -1.7839641571044922,
+      "logps/chosen": -123.49153137207031,
+      "logps/rejected": -121.6719741821289,
+      "loss": 0.2011,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 7.345705986022949,
+      "rewards/margins": 21.307662963867188,
+      "rewards/rejected": -13.961955070495605,
       "step": 330
     },
     {
       "epoch": 0.71,
-      "grad_norm": 251.55976817411812,
+      "grad_norm": 281.7232240729927,
       "learning_rate": 1.1666074087171627e-07,
-      "logits/chosen": -1.6232049465179443,
-      "logits/rejected": -1.6956592798233032,
-      "logps/chosen": -117.71810150146484,
-      "logps/rejected": -113.10723876953125,
-      "loss": 0.2458,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 6.386099815368652,
-      "rewards/margins": 19.121448516845703,
-      "rewards/rejected": -12.73534870147705,
+      "logits/chosen": -1.6561883687973022,
+      "logits/rejected": -1.7283906936645508,
+      "logps/chosen": -117.68770599365234,
+      "logps/rejected": -112.509765625,
+      "loss": 0.2456,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 6.401297569274902,
+      "rewards/margins": 18.83791732788086,
+      "rewards/rejected": -12.43661880493164,
       "step": 340
     },
     {
       "epoch": 0.73,
-      "grad_norm": 784.7175940461042,
+      "grad_norm": 736.7599817555671,
       "learning_rate": 1.0157994641835734e-07,
-      "logits/chosen": -1.6620721817016602,
-      "logits/rejected": -1.6772327423095703,
-      "logps/chosen": -117.80794525146484,
-      "logps/rejected": -132.93991088867188,
-      "loss": 0.1649,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 6.138510704040527,
-      "rewards/margins": 21.829065322875977,
-      "rewards/rejected": -15.690553665161133,
+      "logits/chosen": -1.6912691593170166,
+      "logits/rejected": -1.7067359685897827,
+      "logps/chosen": -117.70341491699219,
+      "logps/rejected": -132.61856079101562,
+      "loss": 0.1648,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 6.1907782554626465,
+      "rewards/margins": 21.720661163330078,
+      "rewards/rejected": -15.529881477355957,
       "step": 350
     },
     {
       "epoch": 0.75,
-      "grad_norm": 497.1984887279012,
+      "grad_norm": 471.30491456250144,
       "learning_rate": 8.729103716819111e-08,
-      "logits/chosen": -1.6556260585784912,
-      "logits/rejected": -1.6805181503295898,
-      "logps/chosen": -124.04801177978516,
-      "logps/rejected": -128.13685607910156,
-      "loss": 0.1743,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 7.051167964935303,
-      "rewards/margins": 21.460834503173828,
-      "rewards/rejected": -14.40966796875,
+      "logits/chosen": -1.6851584911346436,
+      "logits/rejected": -1.7095420360565186,
+      "logps/chosen": -124.0709228515625,
+      "logps/rejected": -127.85160827636719,
+      "loss": 0.1683,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 7.039712429046631,
+      "rewards/margins": 21.306758880615234,
+      "rewards/rejected": -14.267045974731445,
       "step": 360
     },
     {
       "epoch": 0.77,
-      "grad_norm": 701.6355853905161,
+      "grad_norm": 673.9872436763492,
       "learning_rate": 7.387025063449081e-08,
-      "logits/chosen": -1.7627124786376953,
-      "logits/rejected": -1.7043575048446655,
-      "logps/chosen": -113.8396224975586,
-      "logps/rejected": -117.89778137207031,
-      "loss": 0.1661,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 6.275058269500732,
-      "rewards/margins": 18.96691131591797,
-      "rewards/rejected": -12.691854476928711,
+      "logits/chosen": -1.7928199768066406,
+      "logits/rejected": -1.7365026473999023,
+      "logps/chosen": -113.66243743896484,
+      "logps/rejected": -117.55149841308594,
+      "loss": 0.1713,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 6.36365270614624,
+      "rewards/margins": 18.882360458374023,
+      "rewards/rejected": -12.518709182739258,
       "step": 370
     },
     {
       "epoch": 0.79,
-      "grad_norm": 384.3232766408438,
+      "grad_norm": 424.179983945121,
       "learning_rate": 6.138919252022435e-08,
-      "logits/chosen": -1.7891247272491455,
-      "logits/rejected": -1.8081142902374268,
-      "logps/chosen": -121.1161880493164,
-      "logps/rejected": -131.83599853515625,
-      "loss": 0.1823,
+      "logits/chosen": -1.8132708072662354,
+      "logits/rejected": -1.8341960906982422,
+      "logps/chosen": -121.0456771850586,
+      "logps/rejected": -131.6220703125,
+      "loss": 0.1821,
       "rewards/accuracies": 0.9375,
-      "rewards/chosen": 7.768773078918457,
-      "rewards/margins": 24.082279205322266,
-      "rewards/rejected": -16.31351089477539,
+      "rewards/chosen": 7.8040289878845215,
+      "rewards/margins": 24.01057243347168,
+      "rewards/rejected": -16.20654296875,
       "step": 380
     },
     {
       "epoch": 0.82,
-      "grad_norm": 419.5923423572011,
+      "grad_norm": 480.45370869608473,
       "learning_rate": 4.991445467064689e-08,
-      "logits/chosen": -1.6696172952651978,
-      "logits/rejected": -1.6762924194335938,
-      "logps/chosen": -117.07814025878906,
-      "logps/rejected": -120.15535736083984,
-      "loss": 0.1486,
+      "logits/chosen": -1.704850196838379,
+      "logits/rejected": -1.7101322412490845,
+      "logps/chosen": -116.9380874633789,
+      "logps/rejected": -119.96983337402344,
+      "loss": 0.1493,
       "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 7.70883846282959,
-      "rewards/margins": 22.634441375732422,
-      "rewards/rejected": -14.9256010055542,
+      "rewards/chosen": 7.778868198394775,
+      "rewards/margins": 22.611705780029297,
+      "rewards/rejected": -14.832839965820312,
       "step": 390
     },
     {
       "epoch": 0.84,
-      "grad_norm": 548.5530090451983,
+      "grad_norm": 500.3403823746712,
       "learning_rate": 3.9507259776993954e-08,
-      "logits/chosen": -1.6570144891738892,
-      "logits/rejected": -1.7342147827148438,
-      "logps/chosen": -119.59893798828125,
-      "logps/rejected": -120.49696350097656,
-      "loss": 0.2762,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 6.581268310546875,
-      "rewards/margins": 21.039697647094727,
-      "rewards/rejected": -14.4584321975708,
+      "logits/chosen": -1.6802362203598022,
+      "logits/rejected": -1.7585910558700562,
+      "logps/chosen": -119.65745544433594,
+      "logps/rejected": -120.54292297363281,
+      "loss": 0.1918,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 6.5520124435424805,
+      "rewards/margins": 21.033418655395508,
+      "rewards/rejected": -14.481407165527344,
       "step": 400
     },
     {
       "epoch": 0.84,
-      "eval_logits/chosen": -1.8335998058319092,
-      "eval_logits/rejected": -1.8335609436035156,
-      "eval_logps/chosen": -121.38908386230469,
-      "eval_logps/rejected": -126.29609680175781,
-      "eval_loss": 0.16946831345558167,
-      "eval_rewards/accuracies": 0.9296875,
-      "eval_rewards/chosen": 8.191582679748535,
-      "eval_rewards/margins": 24.719266891479492,
-      "eval_rewards/rejected": -16.527685165405273,
-      "eval_runtime": 97.6978,
-      "eval_samples_per_second": 20.471,
+      "eval_logits/chosen": -1.8512133359909058,
+      "eval_logits/rejected": -1.8539310693740845,
+      "eval_logps/chosen": -121.4881591796875,
+      "eval_logps/rejected": -126.26817321777344,
+      "eval_loss": 0.1591520756483078,
+      "eval_rewards/accuracies": 0.921875,
+      "eval_rewards/chosen": 8.14204216003418,
+      "eval_rewards/margins": 24.65576171875,
+      "eval_rewards/rejected": -16.51371955871582,
+      "eval_runtime": 97.6311,
+      "eval_samples_per_second": 20.485,
       "eval_steps_per_second": 0.328,
       "step": 400
     },
     {
       "epoch": 0.86,
-      "grad_norm": 482.27830673225736,
+      "grad_norm": 464.7237726717384,
       "learning_rate": 3.022313472693447e-08,
-      "logits/chosen": -1.7594387531280518,
-      "logits/rejected": -1.7884505987167358,
-      "logps/chosen": -130.15711975097656,
-      "logps/rejected": -124.42222595214844,
-      "loss": 0.2025,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 8.578957557678223,
-      "rewards/margins": 24.737140655517578,
-      "rewards/rejected": -16.158184051513672,
+      "logits/chosen": -1.7741676568984985,
+      "logits/rejected": -1.8052194118499756,
+      "logps/chosen": -130.3295135498047,
+      "logps/rejected": -124.33296203613281,
+      "loss": 0.1851,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 8.492754936218262,
+      "rewards/margins": 24.606307983398438,
+      "rewards/rejected": -16.113550186157227,
       "step": 410
     },
     {
       "epoch": 0.88,
-      "grad_norm": 378.46967638932284,
+      "grad_norm": 372.69082796807044,
       "learning_rate": 2.2111614344599684e-08,
-      "logits/chosen": -1.7343389987945557,
-      "logits/rejected": -1.731032371520996,
-      "logps/chosen": -115.0967025756836,
-      "logps/rejected": -119.22444915771484,
-      "loss": 0.1669,
+      "logits/chosen": -1.7444789409637451,
+      "logits/rejected": -1.7450637817382812,
+      "logps/chosen": -115.13069915771484,
+      "logps/rejected": -119.38905334472656,
+      "loss": 0.1709,
       "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 6.606478691101074,
-      "rewards/margins": 20.886428833007812,
-      "rewards/rejected": -14.279950141906738,
+      "rewards/chosen": 6.589479923248291,
+      "rewards/margins": 20.951730728149414,
+      "rewards/rejected": -14.362251281738281,
       "step": 420
     },
     {
       "epoch": 0.9,
-      "grad_norm": 500.1474042235916,
+      "grad_norm": 487.96100980731563,
       "learning_rate": 1.521597710086439e-08,
-      "logits/chosen": -1.6852420568466187,
-      "logits/rejected": -1.7072862386703491,
-      "logps/chosen": -131.11505126953125,
-      "logps/rejected": -119.86601257324219,
-      "loss": 0.1861,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 7.49709939956665,
-      "rewards/margins": 20.68734359741211,
-      "rewards/rejected": -13.190244674682617,
+      "logits/chosen": -1.7001349925994873,
+      "logits/rejected": -1.7239036560058594,
+      "logps/chosen": -131.1899871826172,
+      "logps/rejected": -119.99398040771484,
+      "loss": 0.189,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 7.4596357345581055,
+      "rewards/margins": 20.71387481689453,
+      "rewards/rejected": -13.254237174987793,
       "step": 430
     },
     {
       "epoch": 0.92,
-      "grad_norm": 310.90711668513416,
+      "grad_norm": 322.7631079024311,
       "learning_rate": 9.57301420397924e-09,
-      "logits/chosen": -1.8146944046020508,
-      "logits/rejected": -1.8261439800262451,
-      "logps/chosen": -122.62892150878906,
-      "logps/rejected": -125.58287048339844,
-      "loss": 0.1762,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 8.172103881835938,
-      "rewards/margins": 24.26197624206543,
-      "rewards/rejected": -16.089872360229492,
+      "logits/chosen": -1.8264925479888916,
+      "logits/rejected": -1.8369083404541016,
+      "logps/chosen": -122.54289245605469,
+      "logps/rejected": -125.7545394897461,
+      "loss": 0.1707,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 8.215115547180176,
+      "rewards/margins": 24.390819549560547,
+      "rewards/rejected": -16.175704956054688,
       "step": 440
     },
     {
       "epoch": 0.94,
-      "grad_norm": 445.19377861997515,
+      "grad_norm": 358.10893603221916,
       "learning_rate": 5.212833302556258e-09,
-      "logits/chosen": -1.8277533054351807,
-      "logits/rejected": -1.7913105487823486,
-      "logps/chosen": -117.95567321777344,
-      "logps/rejected": -121.9552001953125,
-      "loss": 0.2023,
+      "logits/chosen": -1.836982011795044,
+      "logits/rejected": -1.800283432006836,
+      "logps/chosen": -118.0952377319336,
+      "logps/rejected": -121.89033508300781,
+      "loss": 0.2021,
       "rewards/accuracies": 0.90625,
-      "rewards/chosen": 8.675628662109375,
-      "rewards/margins": 25.467309951782227,
-      "rewards/rejected": -16.79167938232422,
+      "rewards/chosen": 8.605842590332031,
+      "rewards/margins": 25.365093231201172,
+      "rewards/rejected": -16.759248733520508,
       "step": 450
     },
     {
       "epoch": 0.96,
-      "grad_norm": 209.63658036103772,
+      "grad_norm": 246.32063500635945,
       "learning_rate": 2.158697848236607e-09,
-      "logits/chosen": -1.7152612209320068,
-      "logits/rejected": -1.7372353076934814,
-      "logps/chosen": -122.6450424194336,
-      "logps/rejected": -125.43870544433594,
-      "loss": 0.1507,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 8.354387283325195,
-      "rewards/margins": 23.259435653686523,
-      "rewards/rejected": -14.905046463012695,
+      "logits/chosen": -1.7269080877304077,
+      "logits/rejected": -1.749202013015747,
+      "logps/chosen": -122.84416198730469,
+      "logps/rejected": -125.47819519042969,
+      "loss": 0.1556,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 8.254826545715332,
+      "rewards/margins": 23.179615020751953,
+      "rewards/rejected": -14.92479133605957,
       "step": 460
     },
     {
       "epoch": 0.98,
-      "grad_norm": 282.1159277085459,
+      "grad_norm": 311.90558289785304,
       "learning_rate": 4.269029751107489e-10,
-      "logits/chosen": -1.691461205482483,
-      "logits/rejected": -1.706369161605835,
-      "logps/chosen": -116.26470947265625,
-      "logps/rejected": -135.1427459716797,
-      "loss": 0.1833,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 6.723459720611572,
-      "rewards/margins": 22.407346725463867,
-      "rewards/rejected": -15.683886528015137,
+      "logits/chosen": -1.7059637308120728,
+      "logits/rejected": -1.7222188711166382,
+      "logps/chosen": -116.1396255493164,
+      "logps/rejected": -135.0008544921875,
+      "loss": 0.1821,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 6.786002159118652,
+      "rewards/margins": 22.398935317993164,
+      "rewards/rejected": -15.612930297851562,
       "step": 470
     },
     {
       "epoch": 1.0,
       "step": 478,
       "total_flos": 0.0,
-      "train_loss": 0.20260818084413537,
-      "train_runtime": 7642.1158,
-      "train_samples_per_second": 8.0,
+      "train_loss": 0.19963366765856244,
+      "train_runtime": 7635.6428,
+      "train_samples_per_second": 8.007,
       "train_steps_per_second": 0.063
     }
   ],