Model save

README.md

@@ -15,15 +15,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.8652
-- Rewards/chosen: 125.1359
-- Rewards/rejected: -204.8868
+- Loss: 1.1371
+- Rewards/chosen: 66.3125
+- Rewards/rejected: -100.6467
 - Rewards/accuracies: 0.9141
-- Rewards/margins: 330.0227
-- Logps/rejected: -113.7294
-- Logps/chosen: -125.2587
-- Logits/rejected: -1.8708
-- Logits/chosen: -1.8725
+- Rewards/margins: 166.9592
+- Logps/rejected: -113.3701
+- Logps/chosen: -124.5098
+- Logits/rejected: -1.8571
+- Logits/chosen: -1.8597
 
 ## Model description
 
@@ -60,10 +60,10 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 1.1504        | 0.21  | 100  | 0.9784          | 103.9547       | -116.3378        | 0.9102             | 220.2925        | -104.8745      | -127.3768    | -1.6942         | -1.7242       |
-| 2.8553        | 0.42  | 200  | 1.8849          | 118.2614       | -180.4101        | 0.9102             | 298.6714        | -111.2817      | -125.9461    | -1.8462         | -1.8519       |
-| 2.2897        | 0.63  | 300  | 2.1029          | 127.9046       | -196.9116        | 0.9141             | 324.8163        | -112.9319      | -124.9818    | -1.8642         | -1.8683       |
-| 2.2714        | 0.84  | 400  | 1.8652          | 125.1359       | -204.8868        | 0.9141             | 330.0227        | -113.7294      | -125.2587    | -1.8708         | -1.8725       |
+| 0.6626        | 0.21  | 100  | 0.5381          | 56.0335        | -58.2668         | 0.9102             | 114.3003        | -104.8941      | -126.5656    | -1.6835         | -1.7107       |
+| 1.1228        | 0.42  | 200  | 1.1297          | 60.4518        | -88.4985         | 0.9180             | 148.9503        | -110.9404      | -125.6819    | -1.8447         | -1.8541       |
+| 1.2322        | 0.63  | 300  | 1.2523          | 64.1872        | -102.3396        | 0.9062             | 166.5267        | -113.7086      | -124.9348    | -1.8498         | -1.8521       |
+| 1.4879        | 0.84  | 400  | 1.1371          | 66.3125        | -100.6467        | 0.9141             | 166.9592        | -113.3701      | -124.5098    | -1.8571         | -1.8597       |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 2.0128297995323914,
-    "train_runtime": 7588.5519,
+    "train_loss": 1.062665213600861,
+    "train_runtime": 7601.7683,
     "train_samples": 61135,
-    "train_samples_per_second": 8.056,
+    "train_samples_per_second": 8.042,
     "train_steps_per_second": 0.063
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc91be5bc4e64f8cc3c7d6ab30482d81b7e75dcfe0d795c0d2faa6614fb9916a
+oid sha256:fe1a3e7a2a4c6a66a6f58926332daee7422cda7b9e7d7aa07091325ca1e94ae4
 size 4943178720

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:23399768c2f186d099dbf25a818c276f11bd6c402b46b35964c3b41af8d8aed8
+oid sha256:0dd5de183ba55c5f2a8d4dcb509d10948b9ab488df711d7e5cb9b3e390556ab2
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:767c51a4eec8ff384baabbf4abb73f66a3893b96d231485e656f7759af483e06
+oid sha256:d5a971888b2ed0069780a300a1ffbd4cb492968dfdc603b3d6a5133707abbe59
 size 4540532728

runs/May13_02-49-38_n136-129-074/events.out.tfevents.1715539917.n136-129-074.1607405.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:70667af11a055bc1ed4f70f5a06722b94dcf17f3495be0eb4d3791e6be7f44c7
-size 35905
+oid sha256:5493f7856ae817dd98cb6be2eb0be7e39b2558d2c7dc3767589919fa9cc96748
+size 41075

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 2.0128297995323914,
-    "train_runtime": 7588.5519,
+    "train_loss": 1.062665213600861,
+    "train_runtime": 7601.7683,
     "train_samples": 61135,
-    "train_samples_per_second": 8.056,
+    "train_samples_per_second": 8.042,
     "train_steps_per_second": 0.063
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 43894.48099242753,
+      "grad_norm": 21942.443210763005,
       "learning_rate": 1.0416666666666666e-08,
       "logits/chosen": -1.689455509185791,
       "logits/rejected": -1.4794573783874512,
@@ -25,780 +25,780 @@
     },
     {
       "epoch": 0.02,
-      "grad_norm": 32305.118552441847,
+      "grad_norm": 14077.432213077995,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -1.7068803310394287,
-      "logits/rejected": -1.6096948385238647,
-      "logps/chosen": -139.68423461914062,
-      "logps/rejected": -91.41385650634766,
-      "loss": 2.648,
-      "rewards/accuracies": 0.5069444179534912,
-      "rewards/chosen": 0.3865443468093872,
-      "rewards/margins": 1.56412935256958,
-      "rewards/rejected": -1.1775851249694824,
+      "logits/chosen": -1.7074503898620605,
+      "logits/rejected": -1.609968662261963,
+      "logps/chosen": -139.67112731933594,
+      "logps/rejected": -91.38861083984375,
+      "loss": 1.4419,
+      "rewards/accuracies": 0.4791666567325592,
+      "rewards/chosen": 0.2587200403213501,
+      "rewards/margins": 0.7212592959403992,
+      "rewards/rejected": -0.4625391960144043,
       "step": 10
     },
     {
       "epoch": 0.04,
-      "grad_norm": 12815.76079839475,
+      "grad_norm": 5354.815788154259,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -1.6411230564117432,
-      "logits/rejected": -1.6499197483062744,
-      "logps/chosen": -131.1981964111328,
-      "logps/rejected": -93.75257110595703,
-      "loss": 0.8229,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 14.764315605163574,
-      "rewards/margins": 19.453596115112305,
-      "rewards/rejected": -4.68928337097168,
+      "logits/chosen": -1.6398289203643799,
+      "logits/rejected": -1.6472053527832031,
+      "logps/chosen": -131.20123291015625,
+      "logps/rejected": -93.73577880859375,
+      "loss": 0.5235,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 7.366933345794678,
+      "rewards/margins": 9.627552032470703,
+      "rewards/rejected": -2.260619640350342,
       "step": 20
     },
     {
       "epoch": 0.06,
-      "grad_norm": 8663.076785137986,
+      "grad_norm": 7510.038946129227,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -1.7121353149414062,
-      "logits/rejected": -1.6375898122787476,
-      "logps/chosen": -133.71029663085938,
-      "logps/rejected": -103.07096099853516,
-      "loss": 0.5133,
+      "logits/chosen": -1.7087770700454712,
+      "logits/rejected": -1.6332916021347046,
+      "logps/chosen": -133.693115234375,
+      "logps/rejected": -103.26603698730469,
+      "loss": 0.2881,
       "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 44.873260498046875,
-      "rewards/margins": 58.086036682128906,
-      "rewards/rejected": -13.2127685546875,
+      "rewards/chosen": 22.522541046142578,
+      "rewards/margins": 30.10431480407715,
+      "rewards/rejected": -7.581770420074463,
       "step": 30
     },
     {
       "epoch": 0.08,
-      "grad_norm": 9873.577427815002,
+      "grad_norm": 5943.73756122518,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -1.6784517765045166,
-      "logits/rejected": -1.5826914310455322,
-      "logps/chosen": -145.05630493164062,
-      "logps/rejected": -101.44771575927734,
-      "loss": 0.516,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 66.6468734741211,
-      "rewards/margins": 95.41236114501953,
-      "rewards/rejected": -28.765483856201172,
+      "logits/chosen": -1.6724061965942383,
+      "logits/rejected": -1.5752627849578857,
+      "logps/chosen": -145.07150268554688,
+      "logps/rejected": -101.88034057617188,
+      "loss": 0.2954,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 33.247291564941406,
+      "rewards/margins": 49.793121337890625,
+      "rewards/rejected": -16.545827865600586,
       "step": 40
     },
     {
       "epoch": 0.1,
-      "grad_norm": 15484.454406367853,
+      "grad_norm": 7581.752404887514,
       "learning_rate": 4.999733114418725e-07,
-      "logits/chosen": -1.6393781900405884,
-      "logits/rejected": -1.6535584926605225,
-      "logps/chosen": -127.37105560302734,
-      "logps/rejected": -104.55952453613281,
-      "loss": 0.6497,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 77.03364562988281,
-      "rewards/margins": 131.46775817871094,
-      "rewards/rejected": -54.434104919433594,
+      "logits/chosen": -1.6333706378936768,
+      "logits/rejected": -1.6477829217910767,
+      "logps/chosen": -127.62773132324219,
+      "logps/rejected": -105.25482177734375,
+      "loss": 0.3583,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 37.23344039916992,
+      "rewards/margins": 67.92698669433594,
+      "rewards/rejected": -30.693557739257812,
       "step": 50
     },
     {
       "epoch": 0.13,
-      "grad_norm": 4641.588943610254,
+      "grad_norm": 2401.4861603881745,
       "learning_rate": 4.990398100856366e-07,
-      "logits/chosen": -1.7217296361923218,
-      "logits/rejected": -1.651254653930664,
-      "logps/chosen": -141.35108947753906,
-      "logps/rejected": -108.5528793334961,
-      "loss": 0.7574,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 96.49129486083984,
-      "rewards/margins": 164.4349822998047,
-      "rewards/rejected": -67.94366455078125,
+      "logits/chosen": -1.727806806564331,
+      "logits/rejected": -1.6551291942596436,
+      "logps/chosen": -141.4404296875,
+      "logps/rejected": -109.39076232910156,
+      "loss": 0.4201,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 47.79894256591797,
+      "rewards/margins": 85.96011352539062,
+      "rewards/rejected": -38.16118240356445,
       "step": 60
     },
     {
       "epoch": 0.15,
-      "grad_norm": 10223.596716214304,
+      "grad_norm": 5134.999991523168,
       "learning_rate": 4.967775735898179e-07,
-      "logits/chosen": -1.6282870769500732,
-      "logits/rejected": -1.6370842456817627,
-      "logps/chosen": -134.8829803466797,
-      "logps/rejected": -106.41259765625,
-      "loss": 1.0394,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 102.15782928466797,
-      "rewards/margins": 186.63003540039062,
-      "rewards/rejected": -84.47221374511719,
+      "logits/chosen": -1.6500078439712524,
+      "logits/rejected": -1.6597402095794678,
+      "logps/chosen": -134.87258911132812,
+      "logps/rejected": -106.83231353759766,
+      "loss": 0.5503,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 51.130958557128906,
+      "rewards/margins": 95.4656753540039,
+      "rewards/rejected": -44.334712982177734,
       "step": 70
     },
     {
       "epoch": 0.17,
-      "grad_norm": 6360.665892121058,
+      "grad_norm": 3020.747112536719,
       "learning_rate": 4.931986719649298e-07,
-      "logits/chosen": -1.5932952165603638,
-      "logits/rejected": -1.5602772235870361,
-      "logps/chosen": -130.39671325683594,
-      "logps/rejected": -101.85746002197266,
-      "loss": 1.1921,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 100.63119506835938,
-      "rewards/margins": 181.93630981445312,
-      "rewards/rejected": -81.30511474609375,
+      "logits/chosen": -1.6039135456085205,
+      "logits/rejected": -1.5703781843185425,
+      "logps/chosen": -130.39112854003906,
+      "logps/rejected": -102.1457290649414,
+      "loss": 0.6267,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 50.343528747558594,
+      "rewards/margins": 92.43741607666016,
+      "rewards/rejected": -42.09387969970703,
       "step": 80
     },
     {
       "epoch": 0.19,
-      "grad_norm": 12014.555301109034,
+      "grad_norm": 4809.065343882705,
       "learning_rate": 4.883222001996351e-07,
-      "logits/chosen": -1.6406339406967163,
-      "logits/rejected": -1.6412605047225952,
-      "logps/chosen": -138.44619750976562,
-      "logps/rejected": -112.9968032836914,
-      "loss": 0.9751,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 107.64051818847656,
-      "rewards/margins": 218.6805419921875,
-      "rewards/rejected": -111.0400390625,
+      "logits/chosen": -1.6389238834381104,
+      "logits/rejected": -1.6379464864730835,
+      "logps/chosen": -138.47369384765625,
+      "logps/rejected": -113.57975006103516,
+      "loss": 0.5572,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 53.682769775390625,
+      "rewards/margins": 112.1175765991211,
+      "rewards/rejected": -58.43480682373047,
       "step": 90
     },
     {
       "epoch": 0.21,
-      "grad_norm": 11340.576903436586,
+      "grad_norm": 5573.955871135009,
       "learning_rate": 4.821741763807186e-07,
-      "logits/chosen": -1.6779956817626953,
-      "logits/rejected": -1.6324456930160522,
-      "logps/chosen": -118.78487396240234,
-      "logps/rejected": -101.80384826660156,
-      "loss": 1.1504,
+      "logits/chosen": -1.6723031997680664,
+      "logits/rejected": -1.628169059753418,
+      "logps/chosen": -118.47151184082031,
+      "logps/rejected": -102.07991790771484,
+      "loss": 0.6626,
       "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 91.92900848388672,
-      "rewards/margins": 203.4224090576172,
-      "rewards/rejected": -111.493408203125,
+      "rewards/chosen": 47.53132247924805,
+      "rewards/margins": 104.65843200683594,
+      "rewards/rejected": -57.127113342285156,
       "step": 100
     },
     {
       "epoch": 0.21,
-      "eval_logits/chosen": -1.724159836769104,
-      "eval_logits/rejected": -1.6941893100738525,
-      "eval_logps/chosen": -127.37677764892578,
-      "eval_logps/rejected": -104.87450408935547,
-      "eval_loss": 0.9783788323402405,
+      "eval_logits/chosen": -1.7106783390045166,
+      "eval_logits/rejected": -1.6835048198699951,
+      "eval_logps/chosen": -126.56555938720703,
+      "eval_logps/rejected": -104.89409637451172,
+      "eval_loss": 0.5380971431732178,
       "eval_rewards/accuracies": 0.91015625,
-      "eval_rewards/chosen": 103.95471954345703,
-      "eval_rewards/margins": 220.29249572753906,
-      "eval_rewards/rejected": -116.33775329589844,
-      "eval_runtime": 97.7821,
-      "eval_samples_per_second": 20.454,
-      "eval_steps_per_second": 0.327,
+      "eval_rewards/chosen": 56.03348922729492,
+      "eval_rewards/margins": 114.3002700805664,
+      "eval_rewards/rejected": -58.26677703857422,
+      "eval_runtime": 98.2217,
+      "eval_samples_per_second": 20.362,
+      "eval_steps_per_second": 0.326,
       "step": 100
     },
     {
       "epoch": 0.23,
-      "grad_norm": 9309.434410308813,
+      "grad_norm": 5005.917820095319,
       "learning_rate": 4.747874028753375e-07,
-      "logits/chosen": -1.6036758422851562,
-      "logits/rejected": -1.651767373085022,
-      "logps/chosen": -123.1724624633789,
-      "logps/rejected": -111.15580749511719,
-      "loss": 1.3815,
+      "logits/chosen": -1.5906270742416382,
+      "logits/rejected": -1.6415948867797852,
+      "logps/chosen": -122.68074035644531,
+      "logps/rejected": -111.13360595703125,
+      "loss": 0.6247,
       "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 88.74246978759766,
-      "rewards/margins": 178.5576934814453,
-      "rewards/rejected": -89.81523132324219,
+      "rewards/chosen": 46.82985305786133,
+      "rewards/margins": 91.6264419555664,
+      "rewards/rejected": -44.79658126831055,
       "step": 110
     },
     {
       "epoch": 0.25,
-      "grad_norm": 7646.143632789072,
+      "grad_norm": 3655.068297528981,
       "learning_rate": 4.662012913161997e-07,
-      "logits/chosen": -1.6596767902374268,
-      "logits/rejected": -1.639947533607483,
-      "logps/chosen": -122.2258071899414,
-      "logps/rejected": -112.39066314697266,
-      "loss": 1.2948,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 106.12471008300781,
-      "rewards/margins": 216.6038818359375,
-      "rewards/rejected": -110.47917175292969,
+      "logits/chosen": -1.6676746606826782,
+      "logits/rejected": -1.6458972692489624,
+      "logps/chosen": -121.62098693847656,
+      "logps/rejected": -112.0022201538086,
+      "loss": 0.6809,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 56.08646774291992,
+      "rewards/margins": 109.3838119506836,
+      "rewards/rejected": -53.29734420776367,
       "step": 120
     },
     {
       "epoch": 0.27,
-      "grad_norm": 5056.012586834783,
+      "grad_norm": 2616.225265475091,
       "learning_rate": 4.5646165232345103e-07,
-      "logits/chosen": -1.6654844284057617,
-      "logits/rejected": -1.6651356220245361,
-      "logps/chosen": -129.16343688964844,
-      "logps/rejected": -105.808837890625,
-      "loss": 1.5047,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 115.51092529296875,
-      "rewards/margins": 240.7315673828125,
-      "rewards/rejected": -125.22064208984375,
+      "logits/chosen": -1.6679775714874268,
+      "logits/rejected": -1.6719906330108643,
+      "logps/chosen": -128.91368103027344,
+      "logps/rejected": -105.75624084472656,
+      "loss": 0.8587,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 59.0042839050293,
+      "rewards/margins": 121.35162353515625,
+      "rewards/rejected": -62.34734344482422,
       "step": 130
     },
     {
       "epoch": 0.29,
-      "grad_norm": 13712.265823185711,
+      "grad_norm": 5059.433598857849,
       "learning_rate": 4.456204510851956e-07,
-      "logits/chosen": -1.5624234676361084,
-      "logits/rejected": -1.5188586711883545,
-      "logps/chosen": -126.886474609375,
-      "logps/rejected": -106.10212707519531,
-      "loss": 2.4129,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 106.82698822021484,
-      "rewards/margins": 237.18234252929688,
-      "rewards/rejected": -130.3553924560547,
+      "logits/chosen": -1.5762321949005127,
+      "logits/rejected": -1.532997488975525,
+      "logps/chosen": -126.9024887084961,
+      "logps/rejected": -106.51009368896484,
+      "loss": 1.0224,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 53.33324432373047,
+      "rewards/margins": 120.55082702636719,
+      "rewards/rejected": -67.21758270263672,
       "step": 140
     },
     {
       "epoch": 0.31,
-      "grad_norm": 11133.14391408868,
+      "grad_norm": 4855.894994824456,
       "learning_rate": 4.337355301007335e-07,
-      "logits/chosen": -1.688401460647583,
-      "logits/rejected": -1.706859827041626,
-      "logps/chosen": -124.13720703125,
-      "logps/rejected": -103.48664855957031,
-      "loss": 1.5451,
+      "logits/chosen": -1.6975635290145874,
+      "logits/rejected": -1.719987154006958,
+      "logps/chosen": -123.97438049316406,
+      "logps/rejected": -103.92557525634766,
+      "loss": 0.7745,
       "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 115.81596374511719,
-      "rewards/margins": 246.532958984375,
-      "rewards/rejected": -130.7169647216797,
+      "rewards/chosen": 58.722259521484375,
+      "rewards/margins": 126.2754135131836,
+      "rewards/rejected": -67.55316162109375,
       "step": 150
     },
     {
       "epoch": 0.33,
-      "grad_norm": 13437.263938948909,
+      "grad_norm": 7387.191481360179,
       "learning_rate": 4.2087030056579986e-07,
-      "logits/chosen": -1.5865412950515747,
-      "logits/rejected": -1.5408028364181519,
-      "logps/chosen": -131.3970489501953,
-      "logps/rejected": -111.11344909667969,
-      "loss": 1.9447,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 105.68785095214844,
-      "rewards/margins": 227.367919921875,
-      "rewards/rejected": -121.6800308227539,
+      "logits/chosen": -1.591379165649414,
+      "logits/rejected": -1.5474408864974976,
+      "logps/chosen": -131.2600860595703,
+      "logps/rejected": -111.50069427490234,
+      "loss": 0.978,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 53.52874755859375,
+      "rewards/margins": 116.30496978759766,
+      "rewards/rejected": -62.77622604370117,
       "step": 160
     },
     {
       "epoch": 0.36,
-      "grad_norm": 12185.877768787304,
+      "grad_norm": 6058.643602654731,
       "learning_rate": 4.070934040463998e-07,
-      "logits/chosen": -1.8080990314483643,
-      "logits/rejected": -1.7689082622528076,
-      "logps/chosen": -127.97332763671875,
-      "logps/rejected": -110.8963623046875,
-      "loss": 2.3084,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 111.68205261230469,
-      "rewards/margins": 263.9295349121094,
-      "rewards/rejected": -152.2474822998047,
+      "logits/chosen": -1.7858331203460693,
+      "logits/rejected": -1.7473684549331665,
+      "logps/chosen": -127.06109619140625,
+      "logps/rejected": -111.16279602050781,
+      "loss": 1.1302,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 60.402099609375,
+      "rewards/margins": 137.85804748535156,
+      "rewards/rejected": -77.4559326171875,
       "step": 170
     },
     {
       "epoch": 0.38,
-      "grad_norm": 12397.612395894674,
+      "grad_norm": 6167.7476886721415,
       "learning_rate": 3.9247834624635404e-07,
-      "logits/chosen": -1.7653987407684326,
-      "logits/rejected": -1.7487728595733643,
-      "logps/chosen": -130.79266357421875,
-      "logps/rejected": -106.92414855957031,
-      "loss": 2.1426,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 123.60539245605469,
-      "rewards/margins": 284.14984130859375,
-      "rewards/rejected": -160.54443359375,
+      "logits/chosen": -1.7311580181121826,
+      "logits/rejected": -1.7103948593139648,
+      "logps/chosen": -129.98849487304688,
+      "logps/rejected": -106.89299011230469,
+      "loss": 1.1361,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 65.82339477539062,
+      "rewards/margins": 145.93978881835938,
+      "rewards/rejected": -80.11640167236328,
       "step": 180
     },
     {
       "epoch": 0.4,
-      "grad_norm": 8233.63739133568,
+      "grad_norm": 4523.07404036336,
       "learning_rate": 3.7710310482256523e-07,
-      "logits/chosen": -1.803746223449707,
-      "logits/rejected": -1.7734184265136719,
-      "logps/chosen": -120.31190490722656,
-      "logps/rejected": -120.1562271118164,
-      "loss": 1.8983,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 114.87654876708984,
-      "rewards/margins": 295.48638916015625,
-      "rewards/rejected": -180.6098175048828,
+      "logits/chosen": -1.7770191431045532,
+      "logits/rejected": -1.7423677444458008,
+      "logps/chosen": -119.51290130615234,
+      "logps/rejected": -119.39100646972656,
+      "loss": 1.0299,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 61.43328857421875,
+      "rewards/margins": 147.9120635986328,
+      "rewards/rejected": -86.47876739501953,
       "step": 190
     },
     {
       "epoch": 0.42,
-      "grad_norm": 8854.142242075086,
+      "grad_norm": 5215.400997541756,
       "learning_rate": 3.610497133404795e-07,
-      "logits/chosen": -1.748956322669983,
-      "logits/rejected": -1.7438066005706787,
-      "logps/chosen": -126.25050354003906,
-      "logps/rejected": -105.51225280761719,
-      "loss": 2.8553,
+      "logits/chosen": -1.7522026300430298,
+      "logits/rejected": -1.740299940109253,
+      "logps/chosen": -126.15678405761719,
+      "logps/rejected": -104.69581604003906,
+      "loss": 1.1228,
       "rewards/accuracies": 0.9375,
-      "rewards/chosen": 128.85885620117188,
-      "rewards/margins": 303.90032958984375,
-      "rewards/rejected": -175.04144287109375,
+      "rewards/chosen": 64.89796447753906,
+      "rewards/margins": 148.3365020751953,
+      "rewards/rejected": -83.43852996826172,
       "step": 200
     },
     {
       "epoch": 0.42,
-      "eval_logits/chosen": -1.8518784046173096,
-      "eval_logits/rejected": -1.8462214469909668,
-      "eval_logps/chosen": -125.94612121582031,
-      "eval_logps/rejected": -111.28173828125,
-      "eval_loss": 1.8849064111709595,
-      "eval_rewards/accuracies": 0.91015625,
-      "eval_rewards/chosen": 118.2613525390625,
-      "eval_rewards/margins": 298.67144775390625,
-      "eval_rewards/rejected": -180.41009521484375,
-      "eval_runtime": 97.5342,
-      "eval_samples_per_second": 20.506,
+      "eval_logits/chosen": -1.8540858030319214,
+      "eval_logits/rejected": -1.8447176218032837,
+      "eval_logps/chosen": -125.68189239501953,
+      "eval_logps/rejected": -110.9404296875,
+      "eval_loss": 1.1297000646591187,
+      "eval_rewards/accuracies": 0.91796875,
+      "eval_rewards/chosen": 60.45179748535156,
+      "eval_rewards/margins": 148.9502716064453,
+      "eval_rewards/rejected": -88.49847412109375,
+      "eval_runtime": 97.6338,
+      "eval_samples_per_second": 20.485,
       "eval_steps_per_second": 0.328,
       "step": 200
     },
     {
       "epoch": 0.44,
-      "grad_norm": 11055.40272221904,
+      "grad_norm": 5148.841292034743,
       "learning_rate": 3.4440382358952115e-07,
-      "logits/chosen": -1.6907581090927124,
-      "logits/rejected": -1.6238548755645752,
-      "logps/chosen": -122.90483093261719,
-      "logps/rejected": -103.57493591308594,
-      "loss": 2.2063,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 96.85926818847656,
-      "rewards/margins": 236.5902862548828,
-      "rewards/rejected": -139.7310028076172,
+      "logits/chosen": -1.667100191116333,
+      "logits/rejected": -1.5997614860534668,
+      "logps/chosen": -122.68757629394531,
+      "logps/rejected": -103.17578125,
+      "loss": 1.2104,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 49.51594161987305,
+      "rewards/margins": 117.38566589355469,
+      "rewards/rejected": -67.8697280883789,
       "step": 210
     },
     {
       "epoch": 0.46,
-      "grad_norm": 8772.23523058132,
+      "grad_norm": 4256.5034776000675,
       "learning_rate": 3.272542485937368e-07,
-      "logits/chosen": -1.832867980003357,
-      "logits/rejected": -1.8748031854629517,
-      "logps/chosen": -124.8982162475586,
-      "logps/rejected": -103.1186752319336,
-      "loss": 2.1666,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 115.77070617675781,
-      "rewards/margins": 271.67108154296875,
-      "rewards/rejected": -155.9003448486328,
+      "logits/chosen": -1.7727683782577515,
+      "logits/rejected": -1.803052306175232,
+      "logps/chosen": -124.49140930175781,
+      "logps/rejected": -103.07470703125,
+      "loss": 1.1663,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 59.9194221496582,
+      "rewards/margins": 137.64974975585938,
+      "rewards/rejected": -77.73030853271484,
       "step": 220
     },
     {
       "epoch": 0.48,
-      "grad_norm": 7569.137122040314,
+      "grad_norm": 3412.6069657750654,
       "learning_rate": 3.096924887558854e-07,
-      "logits/chosen": -1.7989298105239868,
-      "logits/rejected": -1.737357497215271,
-      "logps/chosen": -135.94210815429688,
-      "logps/rejected": -119.22425842285156,
-      "loss": 2.7122,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 117.66719818115234,
-      "rewards/margins": 311.7781677246094,
-      "rewards/rejected": -194.11097717285156,
+      "logits/chosen": -1.7261736392974854,
+      "logits/rejected": -1.6572097539901733,
+      "logps/chosen": -135.29733276367188,
+      "logps/rejected": -119.34806060791016,
+      "loss": 1.5026,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 62.05744171142578,
+      "rewards/margins": 159.73196411132812,
+      "rewards/rejected": -97.67452239990234,
       "step": 230
     },
     {
       "epoch": 0.5,
-      "grad_norm": 9108.756414029493,
+      "grad_norm": 4045.5773277237267,
       "learning_rate": 2.9181224366319943e-07,
-      "logits/chosen": -1.831538438796997,
-      "logits/rejected": -1.8362337350845337,
-      "logps/chosen": -121.7720947265625,
-      "logps/rejected": -112.72883605957031,
-      "loss": 2.3272,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 101.53041076660156,
-      "rewards/margins": 252.52392578125,
-      "rewards/rejected": -150.99354553222656,
+      "logits/chosen": -1.7580757141113281,
+      "logits/rejected": -1.758367896080017,
+      "logps/chosen": -121.45936584472656,
+      "logps/rejected": -113.35154724121094,
+      "loss": 1.3537,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 52.32881546020508,
+      "rewards/margins": 130.93927001953125,
+      "rewards/rejected": -78.61044311523438,
       "step": 240
     },
     {
       "epoch": 0.52,
-      "grad_norm": 11190.763278546787,
+      "grad_norm": 4905.911393654389,
       "learning_rate": 2.7370891215954565e-07,
-      "logits/chosen": -1.83551824092865,
-      "logits/rejected": -1.8649381399154663,
-      "logps/chosen": -125.136474609375,
-      "logps/rejected": -114.23868560791016,
-      "loss": 2.3585,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 122.786865234375,
-      "rewards/margins": 295.3399658203125,
-      "rewards/rejected": -172.55311584472656,
+      "logits/chosen": -1.764793038368225,
+      "logits/rejected": -1.7861311435699463,
+      "logps/chosen": -125.88441467285156,
+      "logps/rejected": -115.19371032714844,
+      "loss": 1.1734,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 57.653724670410156,
+      "rewards/margins": 148.7053985595703,
+      "rewards/rejected": -91.05168151855469,
       "step": 250
     },
     {
       "epoch": 0.54,
-      "grad_norm": 6158.166356558157,
+      "grad_norm": 3187.091777239148,
       "learning_rate": 2.55479083351317e-07,
-      "logits/chosen": -1.8736432790756226,
-      "logits/rejected": -1.887500524520874,
-      "logps/chosen": -129.7669677734375,
-      "logps/rejected": -101.11165618896484,
-      "loss": 2.615,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 140.60031127929688,
-      "rewards/margins": 302.95391845703125,
-      "rewards/rejected": -162.3535919189453,
+      "logits/chosen": -1.824947714805603,
+      "logits/rejected": -1.8243358135223389,
+      "logps/chosen": -129.93756103515625,
+      "logps/rejected": -101.69322967529297,
+      "loss": 1.2106,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 69.447265625,
+      "rewards/margins": 153.531982421875,
+      "rewards/rejected": -84.08470916748047,
       "step": 260
     },
     {
       "epoch": 0.56,
-      "grad_norm": 5058.294410517059,
+      "grad_norm": 4150.480606577989,
       "learning_rate": 2.3722002126275822e-07,
-      "logits/chosen": -1.8732073307037354,
-      "logits/rejected": -1.817939043045044,
-      "logps/chosen": -120.4271011352539,
-      "logps/rejected": -108.13395690917969,
-      "loss": 2.1432,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 116.31379699707031,
-      "rewards/margins": 285.315185546875,
-      "rewards/rejected": -169.00140380859375,
+      "logits/chosen": -1.8272711038589478,
+      "logits/rejected": -1.761054277420044,
+      "logps/chosen": -119.77864074707031,
+      "logps/rejected": -108.34645080566406,
+      "loss": 1.0651,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 61.39922332763672,
+      "rewards/margins": 146.96238708496094,
+      "rewards/rejected": -85.56315612792969,
       "step": 270
     },
     {
       "epoch": 0.59,
-      "grad_norm": 8591.417699065232,
+      "grad_norm": 4635.565014678977,
       "learning_rate": 2.19029145890313e-07,
-      "logits/chosen": -1.7311344146728516,
-      "logits/rejected": -1.812242865562439,
-      "logps/chosen": -126.07157897949219,
-      "logps/rejected": -120.28532409667969,
-      "loss": 3.1754,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 109.85076904296875,
-      "rewards/margins": 299.208740234375,
-      "rewards/rejected": -189.35797119140625,
+      "logits/chosen": -1.6965610980987549,
+      "logits/rejected": -1.7728793621063232,
+      "logps/chosen": -125.82330322265625,
+      "logps/rejected": -120.9666976928711,
+      "loss": 1.4554,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 56.16676712036133,
+      "rewards/margins": 154.2525634765625,
+      "rewards/rejected": -98.0857925415039,
       "step": 280
     },
     {
       "epoch": 0.61,
-      "grad_norm": 10809.88836686454,
+      "grad_norm": 8154.900158593716,
       "learning_rate": 2.0100351342479216e-07,
-      "logits/chosen": -1.8156871795654297,
-      "logits/rejected": -1.7812505960464478,
-      "logps/chosen": -116.09767150878906,
-      "logps/rejected": -107.4920883178711,
-      "loss": 2.8029,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 120.6517333984375,
-      "rewards/margins": 304.40826416015625,
-      "rewards/rejected": -183.7565460205078,
+      "logits/chosen": -1.7839463949203491,
+      "logits/rejected": -1.746951699256897,
+      "logps/chosen": -116.02726745605469,
+      "logps/rejected": -108.37812805175781,
+      "loss": 1.6888,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 60.677894592285156,
+      "rewards/margins": 156.98635864257812,
+      "rewards/rejected": -96.30845642089844,
       "step": 290
     },
     {
       "epoch": 0.63,
-      "grad_norm": 7050.7886997426285,
+      "grad_norm": 4024.034790970826,
       "learning_rate": 1.8323929841460178e-07,
-      "logits/chosen": -1.7682945728302002,
-      "logits/rejected": -1.7057702541351318,
-      "logps/chosen": -134.6074676513672,
-      "logps/rejected": -124.12396240234375,
-      "loss": 2.2897,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 118.32698059082031,
-      "rewards/margins": 297.16571044921875,
-      "rewards/rejected": -178.83876037597656,
+      "logits/chosen": -1.760197401046753,
+      "logits/rejected": -1.6944879293441772,
+      "logps/chosen": -134.81350708007812,
+      "logps/rejected": -125.29605865478516,
+      "loss": 1.2322,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 58.13340377807617,
+      "rewards/margins": 153.413330078125,
+      "rewards/rejected": -95.2799301147461,
       "step": 300
     },
     {
       "epoch": 0.63,
-      "eval_logits/chosen": -1.868285059928894,
-      "eval_logits/rejected": -1.8641510009765625,
-      "eval_logps/chosen": -124.9817886352539,
-      "eval_logps/rejected": -112.93190002441406,
-      "eval_loss": 2.1029016971588135,
-      "eval_rewards/accuracies": 0.9140625,
-      "eval_rewards/chosen": 127.90460968017578,
-      "eval_rewards/margins": 324.8162536621094,
-      "eval_rewards/rejected": -196.91163635253906,
-      "eval_runtime": 97.6004,
-      "eval_samples_per_second": 20.492,
-      "eval_steps_per_second": 0.328,
+      "eval_logits/chosen": -1.852140188217163,
+      "eval_logits/rejected": -1.8497883081436157,
+      "eval_logps/chosen": -124.934814453125,
+      "eval_logps/rejected": -113.70864868164062,
+      "eval_loss": 1.252324104309082,
+      "eval_rewards/accuracies": 0.90625,
+      "eval_rewards/chosen": 64.18718719482422,
+      "eval_rewards/margins": 166.52674865722656,
+      "eval_rewards/rejected": -102.33955383300781,
+      "eval_runtime": 97.7376,
+      "eval_samples_per_second": 20.463,
+      "eval_steps_per_second": 0.327,
       "step": 300
     },
     {
       "epoch": 0.65,
-      "grad_norm": 11804.0120903992,
+      "grad_norm": 5331.659539146755,
       "learning_rate": 1.6583128063291573e-07,
-      "logits/chosen": -1.7608633041381836,
-      "logits/rejected": -1.7920173406600952,
-      "logps/chosen": -126.25215911865234,
-      "logps/rejected": -115.8753433227539,
-      "loss": 2.7724,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 130.85523986816406,
-      "rewards/margins": 304.2178039550781,
-      "rewards/rejected": -173.3625946044922,
+      "logits/chosen": -1.737621545791626,
+      "logits/rejected": -1.7641786336898804,
+      "logps/chosen": -126.19486236572266,
+      "logps/rejected": -116.2991943359375,
+      "loss": 1.2695,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 65.71412658691406,
+      "rewards/margins": 154.5147247314453,
+      "rewards/rejected": -88.80058288574219,
       "step": 310
     },
     {
       "epoch": 0.67,
-      "grad_norm": 10459.35107314203,
+      "grad_norm": 5116.7147594921935,
       "learning_rate": 1.488723393865766e-07,
-      "logits/chosen": -1.7917076349258423,
-      "logits/rejected": -1.7515103816986084,
-      "logps/chosen": -116.50152587890625,
-      "logps/rejected": -114.2782211303711,
-      "loss": 2.4757,
+      "logits/chosen": -1.7694507837295532,
+      "logits/rejected": -1.7249853610992432,
+      "logps/chosen": -115.9213638305664,
+      "logps/rejected": -114.83390808105469,
+      "loss": 1.0406,
       "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 142.95352172851562,
-      "rewards/margins": 323.181884765625,
-      "rewards/rejected": -180.22837829589844,
+      "rewards/chosen": 74.37757873535156,
+      "rewards/margins": 167.27027893066406,
+      "rewards/rejected": -92.89268493652344,
       "step": 320
     },
     {
       "epoch": 0.69,
-      "grad_norm": 8336.1056366251,
+      "grad_norm": 2983.807184096577,
       "learning_rate": 1.3245295796480788e-07,
-      "logits/chosen": -1.7024962902069092,
-      "logits/rejected": -1.7793302536010742,
-      "logps/chosen": -126.26700592041016,
-      "logps/rejected": -110.26517486572266,
-      "loss": 2.626,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 119.15933990478516,
-      "rewards/margins": 284.33038330078125,
-      "rewards/rejected": -165.17105102539062,
+      "logits/chosen": -1.67508864402771,
+      "logits/rejected": -1.7514469623565674,
+      "logps/chosen": -125.67265319824219,
+      "logps/rejected": -110.57685852050781,
+      "loss": 1.2745,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 62.5514030456543,
+      "rewards/margins": 146.69532775878906,
+      "rewards/rejected": -84.14390563964844,
       "step": 330
     },
     {
       "epoch": 0.71,
-      "grad_norm": 6521.040226026619,
+      "grad_norm": 3107.118077347671,
       "learning_rate": 1.1666074087171627e-07,
-      "logits/chosen": -1.6726289987564087,
-      "logits/rejected": -1.7343635559082031,
-      "logps/chosen": -120.09101867675781,
-      "logps/rejected": -102.69850158691406,
-      "loss": 3.9728,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 103.992919921875,
-      "rewards/margins": 254.6125030517578,
-      "rewards/rejected": -150.61959838867188,
+      "logits/chosen": -1.6563949584960938,
+      "logits/rejected": -1.707965612411499,
+      "logps/chosen": -119.6385269165039,
+      "logps/rejected": -102.9443359375,
+      "loss": 1.9516,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 54.25886917114258,
+      "rewards/margins": 130.79788208007812,
+      "rewards/rejected": -76.53900146484375,
       "step": 340
     },
     {
       "epoch": 0.73,
-      "grad_norm": 9653.282761957253,
+      "grad_norm": 5167.730746109294,
       "learning_rate": 1.0157994641835734e-07,
-      "logits/chosen": -1.7162805795669556,
-      "logits/rejected": -1.7199398279190063,
-      "logps/chosen": -120.1608657836914,
-      "logps/rejected": -121.0823974609375,
-      "loss": 2.2998,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 99.24105072021484,
-      "rewards/margins": 294.47723388671875,
-      "rewards/rejected": -195.23617553710938,
+      "logits/chosen": -1.6847736835479736,
+      "logits/rejected": -1.6919714212417603,
+      "logps/chosen": -119.66569519042969,
+      "logps/rejected": -121.41600036621094,
+      "loss": 1.2195,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 52.09636306762695,
+      "rewards/margins": 151.3823699951172,
+      "rewards/rejected": -99.28602600097656,
       "step": 350
     },
     {
       "epoch": 0.75,
-      "grad_norm": 8483.170816558306,
+      "grad_norm": 2841.989996938151,
       "learning_rate": 8.729103716819111e-08,
-      "logits/chosen": -1.7057559490203857,
-      "logits/rejected": -1.7162882089614868,
-      "logps/chosen": -126.68985748291016,
-      "logps/rejected": -117.80401611328125,
-      "loss": 1.8659,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 114.60487365722656,
-      "rewards/margins": 299.46978759765625,
-      "rewards/rejected": -184.86489868164062,
+      "logits/chosen": -1.669879674911499,
+      "logits/rejected": -1.67971932888031,
+      "logps/chosen": -126.43663024902344,
+      "logps/rejected": -117.80718994140625,
+      "loss": 1.0743,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 58.56859588623047,
+      "rewards/margins": 151.0169677734375,
+      "rewards/rejected": -92.44835662841797,
       "step": 360
     },
     {
       "epoch": 0.77,
-      "grad_norm": 16292.765448877528,
+      "grad_norm": 8325.689181319463,
       "learning_rate": 7.387025063449081e-08,
-      "logits/chosen": -1.8123562335968018,
-      "logits/rejected": -1.7508220672607422,
-      "logps/chosen": -116.35862731933594,
-      "logps/rejected": -107.92704010009766,
-      "loss": 2.4835,
+      "logits/chosen": -1.781070351600647,
+      "logits/rejected": -1.71746027469635,
+      "logps/chosen": -115.61714172363281,
+      "logps/rejected": -107.44537353515625,
+      "loss": 1.373,
       "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 100.31108093261719,
-      "rewards/margins": 254.4406280517578,
-      "rewards/rejected": -154.12954711914062,
+      "rewards/chosen": 53.862998962402344,
+      "rewards/margins": 128.51947021484375,
+      "rewards/rejected": -74.6564712524414,
       "step": 370
     },
     {
       "epoch": 0.79,
-      "grad_norm": 12994.314264805473,
+      "grad_norm": 3407.849063024597,
       "learning_rate": 6.138919252022435e-08,
-      "logits/chosen": -1.84799063205719,
-      "logits/rejected": -1.857129693031311,
-      "logps/chosen": -124.62117004394531,
-      "logps/rejected": -118.90538024902344,
-      "loss": 2.1362,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 120.3254623413086,
-      "rewards/margins": 317.2894592285156,
-      "rewards/rejected": -196.96397399902344,
+      "logits/chosen": -1.8198273181915283,
+      "logits/rejected": -1.8311389684677124,
+      "logps/chosen": -123.67546081542969,
+      "logps/rejected": -118.96287536621094,
+      "loss": 1.1359,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 64.89139556884766,
+      "rewards/margins": 163.66079711914062,
+      "rewards/rejected": -98.76939392089844,
       "step": 380
     },
     {
       "epoch": 0.82,
-      "grad_norm": 11506.166124614792,
+      "grad_norm": 5111.728746290769,
       "learning_rate": 4.991445467064689e-08,
-      "logits/chosen": -1.7219364643096924,
-      "logits/rejected": -1.7155206203460693,
-      "logps/chosen": -119.84449768066406,
-      "logps/rejected": -109.42668151855469,
-      "loss": 2.056,
+      "logits/chosen": -1.6906057596206665,
+      "logits/rejected": -1.679142713546753,
+      "logps/chosen": -119.3856430053711,
+      "logps/rejected": -109.07829284667969,
+      "loss": 1.0033,
       "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 126.5132827758789,
-      "rewards/margins": 317.73858642578125,
-      "rewards/rejected": -191.22531127929688,
+      "rewards/chosen": 65.55087280273438,
+      "rewards/margins": 159.42166137695312,
+      "rewards/rejected": -93.87078094482422,
       "step": 390
     },
     {
       "epoch": 0.84,
-      "grad_norm": 14299.186863928613,
+      "grad_norm": 5225.075784148659,
       "learning_rate": 3.9507259776993954e-08,
-      "logits/chosen": -1.7028141021728516,
-      "logits/rejected": -1.7706788778305054,
-      "logps/chosen": -122.94708251953125,
-      "logps/rejected": -110.153564453125,
-      "loss": 2.2714,
+      "logits/chosen": -1.6872704029083252,
+      "logits/rejected": -1.7490768432617188,
+      "logps/chosen": -122.58219146728516,
+      "logps/rejected": -110.0202865600586,
+      "loss": 1.4879,
       "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 98.14391326904297,
-      "rewards/margins": 283.8785400390625,
-      "rewards/rejected": -185.73460388183594,
+      "rewards/chosen": 50.89643478393555,
+      "rewards/margins": 143.09738159179688,
+      "rewards/rejected": -92.20094299316406,
       "step": 400
     },
     {
       "epoch": 0.84,
-      "eval_logits/chosen": -1.8724907636642456,
-      "eval_logits/rejected": -1.8708370923995972,
-      "eval_logps/chosen": -125.25865936279297,
-      "eval_logps/rejected": -113.72941589355469,
-      "eval_loss": 1.8651787042617798,
+      "eval_logits/chosen": -1.8596594333648682,
+      "eval_logits/rejected": -1.8570736646652222,
+      "eval_logps/chosen": -124.50975799560547,
+      "eval_logps/rejected": -113.37007904052734,
+      "eval_loss": 1.1371028423309326,
       "eval_rewards/accuracies": 0.9140625,
-      "eval_rewards/chosen": 125.13589477539062,
-      "eval_rewards/margins": 330.022705078125,
-      "eval_rewards/rejected": -204.8868408203125,
-      "eval_runtime": 97.6945,
-      "eval_samples_per_second": 20.472,
-      "eval_steps_per_second": 0.328,
+      "eval_rewards/chosen": 66.31253051757812,
+      "eval_rewards/margins": 166.95924377441406,
+      "eval_rewards/rejected": -100.64672088623047,
+      "eval_runtime": 97.767,
+      "eval_samples_per_second": 20.457,
+      "eval_steps_per_second": 0.327,
       "step": 400
     },
     {
       "epoch": 0.86,
-      "grad_norm": 9838.615615925528,
+      "grad_norm": 4568.20589061741,
       "learning_rate": 3.022313472693447e-08,
-      "logits/chosen": -1.7946465015411377,
-      "logits/rejected": -1.8112504482269287,
-      "logps/chosen": -134.29222106933594,
-      "logps/rejected": -112.4157943725586,
-      "loss": 2.3885,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 130.2279815673828,
-      "rewards/margins": 333.3272705078125,
-      "rewards/rejected": -203.0992889404297,
+      "logits/chosen": -1.783117651939392,
+      "logits/rejected": -1.7963669300079346,
+      "logps/chosen": -133.34133911132812,
+      "logps/rejected": -111.79888916015625,
+      "loss": 1.3345,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 69.86833953857422,
+      "rewards/margins": 168.33352661132812,
+      "rewards/rejected": -98.46517181396484,
       "step": 410
     },
     {
       "epoch": 0.88,
-      "grad_norm": 9593.489304491115,
+      "grad_norm": 2004.864553433664,
       "learning_rate": 2.2111614344599684e-08,
-      "logits/chosen": -1.7704830169677734,
-      "logits/rejected": -1.758772611618042,
-      "logps/chosen": -118.04278564453125,
-      "logps/rejected": -109.47645568847656,
-      "loss": 1.872,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 102.66886138916016,
-      "rewards/margins": 290.7878112792969,
-      "rewards/rejected": -188.11898803710938,
+      "logits/chosen": -1.7588926553726196,
+      "logits/rejected": -1.7469125986099243,
+      "logps/chosen": -117.37568664550781,
+      "logps/rejected": -109.0504379272461,
+      "loss": 1.0939,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 54.66992950439453,
+      "rewards/margins": 146.59942626953125,
+      "rewards/rejected": -91.92948913574219,
       "step": 420
     },
     {
       "epoch": 0.9,
-      "grad_norm": 6763.365413342396,
+      "grad_norm": 6002.785034545566,
       "learning_rate": 1.521597710086439e-08,
-      "logits/chosen": -1.710999846458435,
-      "logits/rejected": -1.7392107248306274,
-      "logps/chosen": -134.11866760253906,
-      "logps/rejected": -109.66754150390625,
-      "loss": 2.0497,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 119.90589904785156,
-      "rewards/margins": 281.7261657714844,
-      "rewards/rejected": -161.82025146484375,
+      "logits/chosen": -1.6868712902069092,
+      "logits/rejected": -1.7167789936065674,
+      "logps/chosen": -133.92648315429688,
+      "logps/rejected": -109.4766845703125,
+      "loss": 1.302,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 60.91377639770508,
+      "rewards/margins": 140.86959838867188,
+      "rewards/rejected": -79.955810546875,
       "step": 430
     },
     {
       "epoch": 0.92,
-      "grad_norm": 8451.951191648946,
+      "grad_norm": 4174.567194718766,
       "learning_rate": 9.57301420397924e-09,
-      "logits/chosen": -1.852447748184204,
-      "logits/rejected": -1.8499510288238525,
-      "logps/chosen": -125.65571594238281,
-      "logps/rejected": -113.3338394165039,
-      "loss": 2.2476,
+      "logits/chosen": -1.8478610515594482,
+      "logits/rejected": -1.844618558883667,
+      "logps/chosen": -125.5219497680664,
+      "logps/rejected": -113.06559753417969,
+      "loss": 1.1571,
       "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 133.1741485595703,
-      "rewards/margins": 332.48126220703125,
-      "rewards/rejected": -199.3070831298828,
+      "rewards/chosen": 67.255859375,
+      "rewards/margins": 165.56820678710938,
+      "rewards/rejected": -98.31233978271484,
       "step": 440
     },
     {
       "epoch": 0.94,
-      "grad_norm": 9580.867576787692,
+      "grad_norm": 3138.716824955697,
       "learning_rate": 5.212833302556258e-09,
-      "logits/chosen": -1.8617397546768188,
-      "logits/rejected": -1.8191306591033936,
-      "logps/chosen": -121.5869369506836,
-      "logps/rejected": -108.55745697021484,
-      "loss": 2.5789,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 137.1997528076172,
-      "rewards/margins": 339.0559387207031,
-      "rewards/rejected": -201.85618591308594,
+      "logits/chosen": -1.851266860961914,
+      "logits/rejected": -1.8116323947906494,
+      "logps/chosen": -120.89784240722656,
+      "logps/rejected": -108.1541976928711,
+      "loss": 1.1545,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 72.04536437988281,
+      "rewards/margins": 170.95724487304688,
+      "rewards/rejected": -98.91184997558594,
       "step": 450
     },
     {
       "epoch": 0.96,
-      "grad_norm": 4611.3001759975,
+      "grad_norm": 4135.641329446369,
       "learning_rate": 2.158697848236607e-09,
-      "logits/chosen": -1.7538158893585205,
-      "logits/rejected": -1.7651903629302979,
-      "logps/chosen": -126.25445556640625,
-      "logps/rejected": -113.3001937866211,
-      "loss": 1.6224,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 130.99368286132812,
-      "rewards/margins": 307.7095642089844,
-      "rewards/rejected": -176.7158660888672,
+      "logits/chosen": -1.741848349571228,
+      "logits/rejected": -1.7524465322494507,
+      "logps/chosen": -125.63752746582031,
+      "logps/rejected": -113.10050201416016,
+      "loss": 1.0434,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 68.5815200805664,
+      "rewards/margins": 155.94091796875,
+      "rewards/rejected": -87.35940551757812,
       "step": 460
     },
     {
       "epoch": 0.98,
-      "grad_norm": 8182.843310129901,
+      "grad_norm": 3311.2735765742887,
       "learning_rate": 4.269029751107489e-10,
-      "logits/chosen": -1.728009581565857,
-      "logits/rejected": -1.7379541397094727,
-      "logps/chosen": -119.18067932128906,
-      "logps/rejected": -124.09730529785156,
-      "loss": 2.184,
+      "logits/chosen": -1.7126121520996094,
+      "logits/rejected": -1.7236770391464233,
+      "logps/chosen": -118.24110412597656,
+      "logps/rejected": -123.0606689453125,
+      "loss": 1.3321,
       "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 105.3095703125,
-      "rewards/margins": 308.53289794921875,
-      "rewards/rejected": -203.22329711914062,
+      "rewards/chosen": 57.35273361206055,
+      "rewards/margins": 153.78114318847656,
+      "rewards/rejected": -96.42841339111328,
       "step": 470
     },
     {
       "epoch": 1.0,
       "step": 478,
       "total_flos": 0.0,
-      "train_loss": 2.0128297995323914,
-      "train_runtime": 7588.5519,
-      "train_samples_per_second": 8.056,
+      "train_loss": 1.062665213600861,
+      "train_runtime": 7601.7683,
+      "train_samples_per_second": 8.042,
       "train_steps_per_second": 0.063
     }
   ],