Model save

README.md

@@ -15,15 +15,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.1302
-- Rewards/chosen: -1.0433
-- Rewards/rejected: -0.6633
-- Rewards/accuracies: 0.4102
-- Rewards/margins: -0.3800
-- Logps/rejected: -531.6535
-- Logps/chosen: -411.3397
-- Logits/rejected: -4.8567
-- Logits/chosen: -4.6242
+- Loss: 1.0331
+- Rewards/chosen: -4.8324
+- Rewards/rejected: -5.0569
+- Rewards/accuracies: 0.5664
+- Rewards/margins: 0.2245
+- Logps/rejected: -878.3029
+- Logps/chosen: -862.6211
+- Logits/rejected: -4.1328
+- Logits/chosen: -4.0279
 
 ## Model description
 
@@ -42,7 +42,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 1e-07
+- learning_rate: 5e-07
 - train_batch_size: 8
 - eval_batch_size: 8
 - seed: 42
@@ -60,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.2472        | 0.26  | 100  | 0.9611          | -0.2112        | 0.1236           | 0.3906             | -0.3347         | -515.9166      | -394.6981    | -4.8019         | -4.5851       |
-| 0.2112        | 0.51  | 200  | 1.1025          | -0.7299        | -0.3200          | 0.375              | -0.4098         | -524.7885      | -405.0722    | -4.8388         | -4.6065       |
-| 0.195         | 0.77  | 300  | 1.1302          | -1.0433        | -0.6633          | 0.4102             | -0.3800         | -531.6535      | -411.3397    | -4.8567         | -4.6242       |
+| 0.4239        | 0.26  | 100  | 0.8541          | -2.3650        | -2.4785          | 0.5547             | 0.1135          | -620.4579      | -615.8743    | -2.6916         | -2.6796       |
+| 0.312         | 0.51  | 200  | 1.0052          | -4.1999        | -4.4302          | 0.5547             | 0.2303          | -815.6310      | -799.3696    | -3.7211         | -3.6438       |
+| 0.2922        | 0.77  | 300  | 1.0331          | -4.8324        | -5.0569          | 0.5664             | 0.2245          | -878.3029      | -862.6211    | -4.1328         | -4.0279       |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.256967593336959,
-    "train_runtime": 6146.1986,
-    "train_samples": 50000,
-    "train_samples_per_second": 8.135,
-    "train_steps_per_second": 0.064
+    "train_loss": 0.3765579266164004,
+    "train_runtime": 6025.4327,
+    "train_samples": 49993,
+    "train_samples_per_second": 8.297,
+    "train_steps_per_second": 0.065
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b773863c1c34f71a12f146b8f74ffc662775755518fb796340f28308788aed9b
+oid sha256:c87469429cb0a51638f5a684f27c19d9f9fc026521e0676cc2d3b87e69c7564a
 size 4943178720

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:03e30dfe84f62b6452cd39b1564f484dca8c20abaae2c9fb750f42eb765dad5b
+oid sha256:e006c6731a855bd38a012ff30c7874988b035e27f8d56c0168535fec962669bc
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:146df42653d4b3f0e5289c416cd3c0ea883bd1b3e641c90176feec5bc2b98424
+oid sha256:b00ed7b9e00878b09ff9cd25a08968e2bed9ad1b6bf74dc3bb3c7390f6e5a6f2
 size 4540532728

runs/May28_16-02-58_n136-100-194/events.out.tfevents.1716883727.n136-100-194.819005.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:34d2e9e2606ce8a3886683a0b6a19f00b259e55cbef56b1e6a41bbb8a6eb9954
-size 28265
+oid sha256:0e82fecfacf70cd62ad6dcb954665f3f5c9b107d3ed9c438206c82da33e0a0a2
+size 34811

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.256967593336959,
-    "train_runtime": 6146.1986,
-    "train_samples": 50000,
-    "train_samples_per_second": 8.135,
-    "train_steps_per_second": 0.064
+    "train_loss": 0.3765579266164004,
+    "train_runtime": 6025.4327,
+    "train_samples": 49993,
+    "train_samples_per_second": 8.297,
+    "train_steps_per_second": 0.065
 }

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 210.34713052784278,
-      "learning_rate": 2.5e-09,
-      "logits/chosen": -4.623842239379883,
-      "logits/rejected": -4.85917854309082,
-      "logps/chosen": -239.31422424316406,
-      "logps/rejected": -207.56365966796875,
-      "loss": 0.6927,
+      "grad_norm": 33.77465813806882,
+      "learning_rate": 1.25e-08,
+      "logits/chosen": -2.4028477668762207,
+      "logits/rejected": -2.3522276878356934,
+      "logps/chosen": -210.18783569335938,
+      "logps/rejected": -207.74285888671875,
+      "loss": 0.6932,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,645 +25,645 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 198.95172630432864,
-      "learning_rate": 2.5e-08,
-      "logits/chosen": -4.3338446617126465,
-      "logits/rejected": -4.64424991607666,
-      "logps/chosen": -265.20184326171875,
-      "logps/rejected": -215.72174072265625,
-      "loss": 0.6928,
-      "rewards/accuracies": 0.4236111044883728,
-      "rewards/chosen": -0.004745930898934603,
-      "rewards/margins": -0.004067909903824329,
-      "rewards/rejected": -0.0006780209369026124,
+      "grad_norm": 33.67403058641798,
+      "learning_rate": 1.25e-07,
+      "logits/chosen": -2.2666635513305664,
+      "logits/rejected": -2.262030839920044,
+      "logps/chosen": -281.0534973144531,
+      "logps/rejected": -236.01918029785156,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4930555522441864,
+      "rewards/chosen": 0.002097971737384796,
+      "rewards/margins": 0.0015838216058909893,
+      "rewards/rejected": 0.0005141504807397723,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 204.7891876677461,
-      "learning_rate": 5e-08,
-      "logits/chosen": -4.509727478027344,
-      "logits/rejected": -4.74410343170166,
-      "logps/chosen": -267.73052978515625,
-      "logps/rejected": -216.7478485107422,
-      "loss": 0.6872,
+      "grad_norm": 32.16083287384101,
+      "learning_rate": 2.5e-07,
+      "logits/chosen": -2.3008322715759277,
+      "logits/rejected": -2.2834300994873047,
+      "logps/chosen": -252.7469940185547,
+      "logps/rejected": -213.1288604736328,
+      "loss": 0.6888,
       "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.010470375418663025,
-      "rewards/margins": 0.01739482954144478,
-      "rewards/rejected": -0.006924452725797892,
+      "rewards/chosen": 0.020469963550567627,
+      "rewards/margins": 0.0062884255312383175,
+      "rewards/rejected": 0.014181536622345448,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "grad_norm": 204.94575488992174,
-      "learning_rate": 7.5e-08,
-      "logits/chosen": -4.5970940589904785,
-      "logits/rejected": -4.777865409851074,
-      "logps/chosen": -257.5598449707031,
-      "logps/rejected": -215.4015350341797,
-      "loss": 0.6544,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": 0.04864828661084175,
-      "rewards/margins": 0.09208732843399048,
-      "rewards/rejected": -0.04343904182314873,
+      "grad_norm": 28.777208464642932,
+      "learning_rate": 3.75e-07,
+      "logits/chosen": -2.397601366043091,
+      "logits/rejected": -2.4011425971984863,
+      "logps/chosen": -249.62045288085938,
+      "logps/rejected": -241.3568572998047,
+      "loss": 0.677,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.11921324580907822,
+      "rewards/margins": 0.02860979363322258,
+      "rewards/rejected": 0.09060343354940414,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 163.67699084811588,
-      "learning_rate": 1e-07,
-      "logits/chosen": -4.643096923828125,
-      "logits/rejected": -4.7387237548828125,
-      "logps/chosen": -249.96743774414062,
-      "logps/rejected": -223.3234405517578,
-      "loss": 0.5584,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.18158790469169617,
-      "rewards/margins": 0.36420467495918274,
-      "rewards/rejected": -0.18261677026748657,
+      "grad_norm": 22.90499919324699,
+      "learning_rate": 5e-07,
+      "logits/chosen": -2.554856300354004,
+      "logits/rejected": -2.560594081878662,
+      "logps/chosen": -233.58395385742188,
+      "logps/rejected": -213.80856323242188,
+      "loss": 0.6538,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.224616140127182,
+      "rewards/margins": 0.08350424468517303,
+      "rewards/rejected": 0.14111192524433136,
       "step": 40
     },
     {
       "epoch": 0.13,
-      "grad_norm": 125.1152304775479,
-      "learning_rate": 9.979985922607475e-08,
-      "logits/chosen": -4.558148384094238,
-      "logits/rejected": -4.785082817077637,
-      "logps/chosen": -265.6357727050781,
-      "logps/rejected": -234.0360107421875,
-      "loss": 0.45,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.3166799247264862,
-      "rewards/margins": 0.7249041795730591,
-      "rewards/rejected": -0.4082241952419281,
+      "grad_norm": 24.843813454132814,
+      "learning_rate": 4.989992961303737e-07,
+      "logits/chosen": -2.6003928184509277,
+      "logits/rejected": -2.5902161598205566,
+      "logps/chosen": -231.5245819091797,
+      "logps/rejected": -209.9629364013672,
+      "loss": 0.6231,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.25668689608573914,
+      "rewards/margins": 0.1541016399860382,
+      "rewards/rejected": 0.10258527100086212,
       "step": 50
     },
     {
       "epoch": 0.15,
-      "grad_norm": 110.5697848266263,
-      "learning_rate": 9.92010391574745e-08,
-      "logits/chosen": -4.701218605041504,
-      "logits/rejected": -4.855440139770508,
-      "logps/chosen": -232.1560821533203,
-      "logps/rejected": -235.8180389404297,
-      "loss": 0.3379,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.44831886887550354,
-      "rewards/margins": 1.4881489276885986,
-      "rewards/rejected": -1.039829969406128,
+      "grad_norm": 26.42795303115051,
+      "learning_rate": 4.960051957873725e-07,
+      "logits/chosen": -2.6324307918548584,
+      "logits/rejected": -2.625267505645752,
+      "logps/chosen": -240.960205078125,
+      "logps/rejected": -229.14181518554688,
+      "loss": 0.5931,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.21053731441497803,
+      "rewards/margins": 0.2720930278301239,
+      "rewards/rejected": -0.061555732041597366,
       "step": 60
     },
     {
       "epoch": 0.18,
-      "grad_norm": 103.09926490168155,
-      "learning_rate": 9.820833372667812e-08,
-      "logits/chosen": -4.597586631774902,
-      "logits/rejected": -4.846543312072754,
-      "logps/chosen": -243.5035858154297,
-      "logps/rejected": -245.3424072265625,
-      "loss": 0.3085,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.4819186329841614,
-      "rewards/margins": 1.8609161376953125,
-      "rewards/rejected": -1.378997564315796,
+      "grad_norm": 38.34758607763214,
+      "learning_rate": 4.910416686333906e-07,
+      "logits/chosen": -2.4993252754211426,
+      "logits/rejected": -2.4822261333465576,
+      "logps/chosen": -268.89306640625,
+      "logps/rejected": -274.5641784667969,
+      "loss": 0.5661,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.11898250877857208,
+      "rewards/margins": 0.3820464611053467,
+      "rewards/rejected": -0.5010289549827576,
       "step": 70
     },
     {
       "epoch": 0.2,
-      "grad_norm": 89.87848352821936,
-      "learning_rate": 9.682969016701356e-08,
-      "logits/chosen": -4.592278957366943,
-      "logits/rejected": -4.840281963348389,
-      "logps/chosen": -249.3519744873047,
-      "logps/rejected": -261.445068359375,
-      "loss": 0.2624,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 0.593399703502655,
-      "rewards/margins": 2.1497161388397217,
-      "rewards/rejected": -1.5563163757324219,
+      "grad_norm": 47.717719937508924,
+      "learning_rate": 4.841484508350678e-07,
+      "logits/chosen": -2.4562625885009766,
+      "logits/rejected": -2.419541835784912,
+      "logps/chosen": -298.2601013183594,
+      "logps/rejected": -333.39031982421875,
+      "loss": 0.493,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -0.5006470084190369,
+      "rewards/margins": 0.7320936918258667,
+      "rewards/rejected": -1.2327407598495483,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 98.45898295424381,
-      "learning_rate": 9.507614539004081e-08,
-      "logits/chosen": -4.667254447937012,
-      "logits/rejected": -4.913816928863525,
-      "logps/chosen": -235.763427734375,
-      "logps/rejected": -244.2578582763672,
-      "loss": 0.2462,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 0.7724655866622925,
-      "rewards/margins": 2.8438591957092285,
-      "rewards/rejected": -2.0713934898376465,
+      "grad_norm": 36.45971815259915,
+      "learning_rate": 4.7538072695020405e-07,
+      "logits/chosen": -2.4636752605438232,
+      "logits/rejected": -2.4424967765808105,
+      "logps/chosen": -334.6033020019531,
+      "logps/rejected": -383.89068603515625,
+      "loss": 0.4624,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.8243921995162964,
+      "rewards/margins": 0.7801402807235718,
+      "rewards/rejected": -1.6045325994491577,
       "step": 90
     },
     {
       "epoch": 0.26,
-      "grad_norm": 87.96881533227138,
-      "learning_rate": 9.296173762811083e-08,
-      "logits/chosen": -4.5116472244262695,
-      "logits/rejected": -4.829812049865723,
-      "logps/chosen": -238.08468627929688,
-      "logps/rejected": -269.5484619140625,
-      "loss": 0.2472,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.931675910949707,
-      "rewards/margins": 3.0536458492279053,
-      "rewards/rejected": -2.1219699382781982,
+      "grad_norm": 51.20064211099915,
+      "learning_rate": 4.648086881405542e-07,
+      "logits/chosen": -2.641766309738159,
+      "logits/rejected": -2.629326820373535,
+      "logps/chosen": -372.31072998046875,
+      "logps/rejected": -465.06414794921875,
+      "loss": 0.4239,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.0419604778289795,
+      "rewards/margins": 1.301120638847351,
+      "rewards/rejected": -2.343080997467041,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -4.58513879776001,
-      "eval_logits/rejected": -4.80186128616333,
-      "eval_logps/chosen": -394.6981201171875,
-      "eval_logps/rejected": -515.9166259765625,
-      "eval_loss": 0.9610964059829712,
-      "eval_rewards/accuracies": 0.390625,
-      "eval_rewards/chosen": -0.21118265390396118,
-      "eval_rewards/margins": -0.3347358703613281,
-      "eval_rewards/rejected": 0.12355318665504456,
-      "eval_runtime": 97.8315,
-      "eval_samples_per_second": 20.443,
+      "eval_logits/chosen": -2.679638385772705,
+      "eval_logits/rejected": -2.6916067600250244,
+      "eval_logps/chosen": -615.8743286132812,
+      "eval_logps/rejected": -620.4578857421875,
+      "eval_loss": 0.8541297316551208,
+      "eval_rewards/accuracies": 0.5546875,
+      "eval_rewards/chosen": -2.3649697303771973,
+      "eval_rewards/margins": 0.11349410563707352,
+      "eval_rewards/rejected": -2.478463888168335,
+      "eval_runtime": 97.8441,
+      "eval_samples_per_second": 20.441,
       "eval_steps_per_second": 0.327,
       "step": 100
     },
     {
       "epoch": 0.28,
-      "grad_norm": 84.98735748868098,
-      "learning_rate": 9.050339404945832e-08,
-      "logits/chosen": -4.55401611328125,
-      "logits/rejected": -4.845933437347412,
-      "logps/chosen": -229.4434356689453,
-      "logps/rejected": -257.52984619140625,
-      "loss": 0.2226,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 0.9871166348457336,
-      "rewards/margins": 3.0293149948120117,
-      "rewards/rejected": -2.042198419570923,
+      "grad_norm": 51.01640971210625,
+      "learning_rate": 4.5251697024729164e-07,
+      "logits/chosen": -2.7184574604034424,
+      "logits/rejected": -2.711151599884033,
+      "logps/chosen": -412.488037109375,
+      "logps/rejected": -524.6069946289062,
+      "loss": 0.3925,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.327445149421692,
+      "rewards/margins": 1.4965568780899048,
+      "rewards/rejected": -2.8240020275115967,
       "step": 110
     },
     {
       "epoch": 0.31,
-      "grad_norm": 75.78122724506682,
-      "learning_rate": 8.77207952455395e-08,
-      "logits/chosen": -4.49249792098999,
-      "logits/rejected": -4.787415981292725,
-      "logps/chosen": -252.7578125,
-      "logps/rejected": -273.38555908203125,
-      "loss": 0.2215,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.1615877151489258,
-      "rewards/margins": 3.400435209274292,
-      "rewards/rejected": -2.238847255706787,
+      "grad_norm": 56.65660654626653,
+      "learning_rate": 4.3860397622769754e-07,
+      "logits/chosen": -2.9599475860595703,
+      "logits/rejected": -2.9588942527770996,
+      "logps/chosen": -390.6888122558594,
+      "logps/rejected": -498.27001953125,
+      "loss": 0.3799,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.208303689956665,
+      "rewards/margins": 1.46265709400177,
+      "rewards/rejected": -2.6709606647491455,
       "step": 120
     },
     {
       "epoch": 0.33,
-      "grad_norm": 109.1136183108071,
-      "learning_rate": 8.463621767547997e-08,
-      "logits/chosen": -4.589264869689941,
-      "logits/rejected": -4.87318229675293,
-      "logps/chosen": -239.29531860351562,
-      "logps/rejected": -265.04693603515625,
-      "loss": 0.2169,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 0.9574294090270996,
-      "rewards/margins": 3.4433422088623047,
-      "rewards/rejected": -2.485912799835205,
+      "grad_norm": 43.60997841685666,
+      "learning_rate": 4.2318108837739986e-07,
+      "logits/chosen": -3.1589443683624268,
+      "logits/rejected": -3.2094154357910156,
+      "logps/chosen": -416.260009765625,
+      "logps/rejected": -559.4857788085938,
+      "loss": 0.3681,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.5083014965057373,
+      "rewards/margins": 1.7782371044158936,
+      "rewards/rejected": -3.286539077758789,
       "step": 130
     },
     {
       "epoch": 0.36,
-      "grad_norm": 79.72525878658313,
-      "learning_rate": 8.127435532896387e-08,
-      "logits/chosen": -4.636221885681152,
-      "logits/rejected": -4.9098310470581055,
-      "logps/chosen": -267.59625244140625,
-      "logps/rejected": -288.02349853515625,
-      "loss": 0.2063,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 0.8299416303634644,
-      "rewards/margins": 3.622443675994873,
-      "rewards/rejected": -2.792501926422119,
+      "grad_norm": 48.63627584361642,
+      "learning_rate": 4.063717766448194e-07,
+      "logits/chosen": -3.175791025161743,
+      "logits/rejected": -3.1955463886260986,
+      "logps/chosen": -392.7382507324219,
+      "logps/rejected": -522.8216552734375,
+      "loss": 0.356,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -1.2041221857070923,
+      "rewards/margins": 1.7336938381195068,
+      "rewards/rejected": -2.9378156661987305,
       "step": 140
     },
     {
       "epoch": 0.38,
-      "grad_norm": 94.45112212404622,
-      "learning_rate": 7.766212203526569e-08,
-      "logits/chosen": -4.643942832946777,
-      "logits/rejected": -4.911728382110596,
-      "logps/chosen": -233.4263153076172,
-      "logps/rejected": -277.07818603515625,
-      "loss": 0.2098,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 0.9495984315872192,
-      "rewards/margins": 3.8475449085235596,
-      "rewards/rejected": -2.89794659614563,
+      "grad_norm": 46.068535132889295,
+      "learning_rate": 3.8831061017632844e-07,
+      "logits/chosen": -3.325183868408203,
+      "logits/rejected": -3.3710532188415527,
+      "logps/chosen": -452.86053466796875,
+      "logps/rejected": -651.6317138671875,
+      "loss": 0.3382,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -1.77432119846344,
+      "rewards/margins": 2.3799710273742676,
+      "rewards/rejected": -4.154292106628418,
       "step": 150
     },
     {
       "epoch": 0.41,
-      "grad_norm": 85.35291313866578,
-      "learning_rate": 7.382843600106538e-08,
-      "logits/chosen": -4.690377235412598,
-      "logits/rejected": -4.9024457931518555,
-      "logps/chosen": -233.21981811523438,
-      "logps/rejected": -271.2682189941406,
-      "loss": 0.1861,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.9118326306343079,
-      "rewards/margins": 3.6947906017303467,
-      "rewards/rejected": -2.7829582691192627,
+      "grad_norm": 45.41918990464157,
+      "learning_rate": 3.6914218000532694e-07,
+      "logits/chosen": -3.5561134815216064,
+      "logits/rejected": -3.639812469482422,
+      "logps/chosen": -484.4190979003906,
+      "logps/rejected": -641.7444458007812,
+      "loss": 0.3225,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.7744003534317017,
+      "rewards/margins": 1.9252252578735352,
+      "rewards/rejected": -3.6996254920959473,
       "step": 160
     },
     {
       "epoch": 0.43,
-      "grad_norm": 75.89103255157417,
-      "learning_rate": 6.980398830195784e-08,
-      "logits/chosen": -4.554282188415527,
-      "logits/rejected": -4.874223232269287,
-      "logps/chosen": -236.4412078857422,
-      "logps/rejected": -279.4911804199219,
-      "loss": 0.1833,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 1.2316501140594482,
-      "rewards/margins": 4.21605920791626,
-      "rewards/rejected": -2.9844090938568115,
+      "grad_norm": 51.64709616356377,
+      "learning_rate": 3.490199415097892e-07,
+      "logits/chosen": -3.6620240211486816,
+      "logits/rejected": -3.7261757850646973,
+      "logps/chosen": -405.14630126953125,
+      "logps/rejected": -567.592041015625,
+      "loss": 0.3293,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -1.2989130020141602,
+      "rewards/margins": 1.990605115890503,
+      "rewards/rejected": -3.289518356323242,
       "step": 170
     },
     {
       "epoch": 0.46,
-      "grad_norm": 77.81291773020575,
-      "learning_rate": 6.562099718102787e-08,
-      "logits/chosen": -4.651320934295654,
-      "logits/rejected": -4.9173784255981445,
-      "logps/chosen": -215.70126342773438,
-      "logps/rejected": -251.5159149169922,
-      "loss": 0.2065,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 1.0581190586090088,
-      "rewards/margins": 3.832904815673828,
-      "rewards/rejected": -2.7747855186462402,
+      "grad_norm": 53.26126364908075,
+      "learning_rate": 3.2810498590513937e-07,
+      "logits/chosen": -3.7818820476531982,
+      "logits/rejected": -3.8590400218963623,
+      "logps/chosen": -415.643310546875,
+      "logps/rejected": -616.6235961914062,
+      "loss": 0.3055,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -1.4905955791473389,
+      "rewards/margins": 2.332679033279419,
+      "rewards/rejected": -3.823274612426758,
       "step": 180
     },
     {
       "epoch": 0.49,
-      "grad_norm": 86.461876717381,
-      "learning_rate": 6.131295012148612e-08,
-      "logits/chosen": -4.617634296417236,
-      "logits/rejected": -4.793360233306885,
-      "logps/chosen": -239.32681274414062,
-      "logps/rejected": -286.96124267578125,
-      "loss": 0.2013,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 1.0783030986785889,
-      "rewards/margins": 3.6680614948272705,
-      "rewards/rejected": -2.5897579193115234,
+      "grad_norm": 41.26625259744552,
+      "learning_rate": 3.065647506074306e-07,
+      "logits/chosen": -4.042256832122803,
+      "logits/rejected": -4.141567707061768,
+      "logps/chosen": -510.08819580078125,
+      "logps/rejected": -719.58349609375,
+      "loss": 0.2941,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -2.251588821411133,
+      "rewards/margins": 2.5615081787109375,
+      "rewards/rejected": -4.81309700012207,
       "step": 190
     },
     {
       "epoch": 0.51,
-      "grad_norm": 84.24320751887706,
-      "learning_rate": 5.691433575823665e-08,
-      "logits/chosen": -4.624228477478027,
-      "logits/rejected": -4.830000877380371,
-      "logps/chosen": -233.09713745117188,
-      "logps/rejected": -271.84051513671875,
-      "loss": 0.2112,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.9521042108535767,
-      "rewards/margins": 3.7768027782440186,
-      "rewards/rejected": -2.8246986865997314,
+      "grad_norm": 39.377938580248625,
+      "learning_rate": 2.8457167879118326e-07,
+      "logits/chosen": -3.97863507270813,
+      "logits/rejected": -4.105334281921387,
+      "logps/chosen": -450.7337951660156,
+      "logps/rejected": -627.5828857421875,
+      "loss": 0.312,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.6995036602020264,
+      "rewards/margins": 2.1759800910949707,
+      "rewards/rejected": -3.875483751296997,
       "step": 200
     },
     {
       "epoch": 0.51,
-      "eval_logits/chosen": -4.606511116027832,
-      "eval_logits/rejected": -4.8388166427612305,
-      "eval_logps/chosen": -405.0722351074219,
-      "eval_logps/rejected": -524.7885131835938,
-      "eval_loss": 1.102483868598938,
-      "eval_rewards/accuracies": 0.375,
-      "eval_rewards/chosen": -0.729888916015625,
-      "eval_rewards/margins": -0.40984660387039185,
-      "eval_rewards/rejected": -0.32004231214523315,
-      "eval_runtime": 97.8012,
-      "eval_samples_per_second": 20.45,
+      "eval_logits/chosen": -3.6437933444976807,
+      "eval_logits/rejected": -3.7210965156555176,
+      "eval_logps/chosen": -799.3695678710938,
+      "eval_logps/rejected": -815.6309814453125,
+      "eval_loss": 1.005213975906372,
+      "eval_rewards/accuracies": 0.5546875,
+      "eval_rewards/chosen": -4.199921607971191,
+      "eval_rewards/margins": 0.23027381300926208,
+      "eval_rewards/rejected": -4.430195331573486,
+      "eval_runtime": 97.9651,
+      "eval_samples_per_second": 20.415,
       "eval_steps_per_second": 0.327,
       "step": 200
     },
     {
       "epoch": 0.54,
-      "grad_norm": 70.97621814359026,
-      "learning_rate": 5.2460367774593905e-08,
-      "logits/chosen": -4.6944451332092285,
-      "logits/rejected": -4.962179183959961,
-      "logps/chosen": -243.93307495117188,
-      "logps/rejected": -297.62066650390625,
-      "loss": 0.1723,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.0976004600524902,
-      "rewards/margins": 4.546332836151123,
-      "rewards/rejected": -3.448732376098633,
+      "grad_norm": 46.20285719657136,
+      "learning_rate": 2.6230183887296953e-07,
+      "logits/chosen": -3.944969892501831,
+      "logits/rejected": -4.027307510375977,
+      "logps/chosen": -465.77484130859375,
+      "logps/rejected": -658.12353515625,
+      "loss": 0.3057,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -1.945319414138794,
+      "rewards/margins": 2.280862331390381,
+      "rewards/rejected": -4.2261810302734375,
       "step": 210
     },
     {
       "epoch": 0.56,
-      "grad_norm": 71.88477654183092,
-      "learning_rate": 4.798670299452925e-08,
-      "logits/chosen": -4.529160499572754,
-      "logits/rejected": -4.8643479347229,
-      "logps/chosen": -241.5579833984375,
-      "logps/rejected": -293.224365234375,
-      "loss": 0.1923,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 1.0626676082611084,
-      "rewards/margins": 4.490227699279785,
-      "rewards/rejected": -3.4275600910186768,
+      "grad_norm": 41.79204633037396,
+      "learning_rate": 2.399335149726463e-07,
+      "logits/chosen": -3.8770012855529785,
+      "logits/rejected": -4.010533809661865,
+      "logps/chosen": -460.03900146484375,
+      "logps/rejected": -640.6546630859375,
+      "loss": 0.3023,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.8396174907684326,
+      "rewards/margins": 2.282116651535034,
+      "rewards/rejected": -4.121734142303467,
       "step": 220
     },
     {
       "epoch": 0.59,
-      "grad_norm": 83.09100453064212,
-      "learning_rate": 4.3529155927297226e-08,
-      "logits/chosen": -4.6047258377075195,
-      "logits/rejected": -4.93651008605957,
-      "logps/chosen": -241.11477661132812,
-      "logps/rejected": -293.9808044433594,
-      "loss": 0.2012,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.9927155375480652,
-      "rewards/margins": 4.5062031745910645,
-      "rewards/rejected": -3.5134873390197754,
+      "grad_norm": 45.944454438275905,
+      "learning_rate": 2.176457796364861e-07,
+      "logits/chosen": -3.99033784866333,
+      "logits/rejected": -4.047112464904785,
+      "logps/chosen": -418.53399658203125,
+      "logps/rejected": -645.3985595703125,
+      "loss": 0.3002,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -1.8332922458648682,
+      "rewards/margins": 2.527721881866455,
+      "rewards/rejected": -4.361013889312744,
       "step": 230
     },
     {
       "epoch": 0.61,
-      "grad_norm": 70.11336436391163,
-      "learning_rate": 3.9123412049691636e-08,
-      "logits/chosen": -4.588685035705566,
-      "logits/rejected": -4.866146087646484,
-      "logps/chosen": -252.31533813476562,
-      "logps/rejected": -294.6343688964844,
-      "loss": 0.1875,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 1.0253931283950806,
-      "rewards/margins": 4.710432529449463,
-      "rewards/rejected": -3.6850390434265137,
+      "grad_norm": 53.20015118225104,
+      "learning_rate": 1.9561706024845817e-07,
+      "logits/chosen": -4.0395121574401855,
+      "logits/rejected": -4.144242763519287,
+      "logps/chosen": -509.28173828125,
+      "logps/rejected": -761.5509033203125,
+      "loss": 0.2923,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -2.282113790512085,
+      "rewards/margins": 2.9224889278411865,
+      "rewards/rejected": -5.2046027183532715,
       "step": 240
     },
     {
       "epoch": 0.64,
-      "grad_norm": 77.75874575792918,
-      "learning_rate": 3.480474212128766e-08,
-      "logits/chosen": -4.716187000274658,
-      "logits/rejected": -4.966707229614258,
-      "logps/chosen": -231.89279174804688,
-      "logps/rejected": -266.51666259765625,
-      "loss": 0.1825,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.840434193611145,
-      "rewards/margins": 3.7858078479766846,
-      "rewards/rejected": -2.94537353515625,
+      "grad_norm": 59.96727790651106,
+      "learning_rate": 1.7402371060643827e-07,
+      "logits/chosen": -4.039712429046631,
+      "logits/rejected": -4.1744890213012695,
+      "logps/chosen": -431.595947265625,
+      "logps/rejected": -663.0882568359375,
+      "loss": 0.2714,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -1.820656418800354,
+      "rewards/margins": 2.721750020980835,
+      "rewards/rejected": -4.5424065589904785,
       "step": 250
     },
     {
       "epoch": 0.66,
-      "grad_norm": 97.12524424809816,
-      "learning_rate": 3.060771981975726e-08,
-      "logits/chosen": -4.585513114929199,
-      "logits/rejected": -4.878482341766357,
-      "logps/chosen": -234.92617797851562,
-      "logps/rejected": -297.1214904785156,
-      "loss": 0.1837,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 0.9561206102371216,
-      "rewards/margins": 4.824769973754883,
-      "rewards/rejected": -3.86864972114563,
+      "grad_norm": 40.5236589016037,
+      "learning_rate": 1.530385990987863e-07,
+      "logits/chosen": -4.1159772872924805,
+      "logits/rejected": -4.242281913757324,
+      "logps/chosen": -459.8741149902344,
+      "logps/rejected": -717.7155151367188,
+      "loss": 0.2793,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -1.7888422012329102,
+      "rewards/margins": 2.905510425567627,
+      "rewards/rejected": -4.694352626800537,
       "step": 260
     },
     {
       "epoch": 0.69,
-      "grad_norm": 92.49874438996748,
-      "learning_rate": 2.6565944956764818e-08,
-      "logits/chosen": -4.684746742248535,
-      "logits/rejected": -4.911890983581543,
-      "logps/chosen": -243.29568481445312,
-      "logps/rejected": -288.39111328125,
-      "loss": 0.1961,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.8868792653083801,
-      "rewards/margins": 4.555182456970215,
-      "rewards/rejected": -3.6683037281036377,
+      "grad_norm": 39.42267244893286,
+      "learning_rate": 1.328297247838241e-07,
+      "logits/chosen": -4.21132755279541,
+      "logits/rejected": -4.354168891906738,
+      "logps/chosen": -499.681640625,
+      "logps/rejected": -737.876220703125,
+      "loss": 0.2978,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.2856991291046143,
+      "rewards/margins": 2.7018930912017822,
+      "rewards/rejected": -4.987591743469238,
       "step": 270
     },
     {
       "epoch": 0.72,
-      "grad_norm": 73.7028241699641,
-      "learning_rate": 2.2711774490274766e-08,
-      "logits/chosen": -4.634344577789307,
-      "logits/rejected": -4.873081207275391,
-      "logps/chosen": -245.1703338623047,
-      "logps/rejected": -317.2539978027344,
-      "loss": 0.1644,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.9668266177177429,
-      "rewards/margins": 4.682557582855225,
-      "rewards/rejected": -3.715731143951416,
+      "grad_norm": 41.37992012502987,
+      "learning_rate": 1.1355887245137383e-07,
+      "logits/chosen": -4.326869010925293,
+      "logits/rejected": -4.45259952545166,
+      "logps/chosen": -486.5020446777344,
+      "logps/rejected": -689.6087646484375,
+      "loss": 0.2809,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.0246481895446777,
+      "rewards/margins": 2.378901958465576,
+      "rewards/rejected": -4.403550148010254,
       "step": 280
     },
     {
       "epoch": 0.74,
-      "grad_norm": 142.25337407808868,
-      "learning_rate": 1.9076063486687256e-08,
-      "logits/chosen": -4.503401756286621,
-      "logits/rejected": -4.866554260253906,
-      "logps/chosen": -250.9346160888672,
-      "logps/rejected": -283.41046142578125,
-      "loss": 0.1799,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 1.083687424659729,
-      "rewards/margins": 4.472739219665527,
-      "rewards/rejected": -3.389052152633667,
+      "grad_norm": 45.07540326486855,
+      "learning_rate": 9.538031743343627e-08,
+      "logits/chosen": -4.318695545196533,
+      "logits/rejected": -4.4249982833862305,
+      "logps/chosen": -449.86376953125,
+      "logps/rejected": -631.1079711914062,
+      "loss": 0.2848,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -1.7597681283950806,
+      "rewards/margins": 2.173658609390259,
+      "rewards/rejected": -3.93342661857605,
       "step": 290
     },
     {
       "epoch": 0.77,
-      "grad_norm": 88.66793876665662,
-      "learning_rate": 1.5687918106563324e-08,
-      "logits/chosen": -4.625166416168213,
-      "logits/rejected": -4.831929683685303,
-      "logps/chosen": -232.6981658935547,
-      "logps/rejected": -288.00457763671875,
-      "loss": 0.195,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 1.0354994535446167,
-      "rewards/margins": 4.637454509735107,
-      "rewards/rejected": -3.6019554138183594,
+      "grad_norm": 46.50980155740083,
+      "learning_rate": 7.843959053281663e-08,
+      "logits/chosen": -4.4306511878967285,
+      "logits/rejected": -4.540189266204834,
+      "logps/chosen": -427.296875,
+      "logps/rejected": -649.8468627929688,
+      "loss": 0.2922,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -1.8062429428100586,
+      "rewards/margins": 2.5631582736968994,
+      "rewards/rejected": -4.369400978088379,
       "step": 300
     },
     {
       "epoch": 0.77,
-      "eval_logits/chosen": -4.624210357666016,
-      "eval_logits/rejected": -4.856749057769775,
-      "eval_logps/chosen": -411.3396911621094,
-      "eval_logps/rejected": -531.6535034179688,
-      "eval_loss": 1.1301820278167725,
-      "eval_rewards/accuracies": 0.41015625,
-      "eval_rewards/chosen": -1.0432608127593994,
-      "eval_rewards/margins": -0.3799673318862915,
-      "eval_rewards/rejected": -0.6632934212684631,
-      "eval_runtime": 97.9609,
-      "eval_samples_per_second": 20.416,
-      "eval_steps_per_second": 0.327,
+      "eval_logits/chosen": -4.027931213378906,
+      "eval_logits/rejected": -4.1328206062316895,
+      "eval_logps/chosen": -862.62109375,
+      "eval_logps/rejected": -878.3028564453125,
+      "eval_loss": 1.0330662727355957,
+      "eval_rewards/accuracies": 0.56640625,
+      "eval_rewards/chosen": -4.832437515258789,
+      "eval_rewards/margins": 0.22447684407234192,
+      "eval_rewards/rejected": -5.05691385269165,
+      "eval_runtime": 98.1329,
+      "eval_samples_per_second": 20.381,
+      "eval_steps_per_second": 0.326,
       "step": 300
     },
     {
       "epoch": 0.79,
-      "grad_norm": 90.90394303193246,
-      "learning_rate": 1.257446259144494e-08,
-      "logits/chosen": -4.541079044342041,
-      "logits/rejected": -4.873132228851318,
-      "logps/chosen": -239.60592651367188,
-      "logps/rejected": -298.159423828125,
-      "loss": 0.1847,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.076683759689331,
-      "rewards/margins": 4.874758243560791,
-      "rewards/rejected": -3.7980740070343018,
+      "grad_norm": 41.414441218872,
+      "learning_rate": 6.28723129572247e-08,
+      "logits/chosen": -4.412310600280762,
+      "logits/rejected": -4.562020301818848,
+      "logps/chosen": -466.133056640625,
+      "logps/rejected": -716.8997802734375,
+      "loss": 0.2841,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -1.862836480140686,
+      "rewards/margins": 2.8856778144836426,
+      "rewards/rejected": -4.748514175415039,
       "step": 310
     },
     {
       "epoch": 0.82,
-      "grad_norm": 87.85310576006609,
-      "learning_rate": 9.760622117187234e-09,
-      "logits/chosen": -4.597599029541016,
-      "logits/rejected": -4.9500837326049805,
-      "logps/chosen": -227.94247436523438,
-      "logps/rejected": -279.3809814453125,
-      "loss": 0.1934,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 0.8610901832580566,
-      "rewards/margins": 4.507565498352051,
-      "rewards/rejected": -3.6464743614196777,
+      "grad_norm": 45.33954098782747,
+      "learning_rate": 4.880311058593617e-08,
+      "logits/chosen": -4.565115928649902,
+      "logits/rejected": -4.670710563659668,
+      "logps/chosen": -456.89593505859375,
+      "logps/rejected": -716.8349609375,
+      "loss": 0.2716,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -2.0230953693389893,
+      "rewards/margins": 2.954841136932373,
+      "rewards/rejected": -4.977936744689941,
       "step": 320
     },
     {
       "epoch": 0.84,
-      "grad_norm": 85.81889719468313,
-      "learning_rate": 7.2689232521989885e-09,
-      "logits/chosen": -4.554391860961914,
-      "logits/rejected": -4.864416599273682,
-      "logps/chosen": -249.89169311523438,
-      "logps/rejected": -304.54913330078125,
-      "loss": 0.1773,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 0.9128581881523132,
-      "rewards/margins": 4.5053324699401855,
-      "rewards/rejected": -3.5924744606018066,
+      "grad_norm": 47.474798952013,
+      "learning_rate": 3.634461626099494e-08,
+      "logits/chosen": -4.674079895019531,
+      "logits/rejected": -4.778273105621338,
+      "logps/chosen": -479.1402893066406,
+      "logps/rejected": -746.5988159179688,
+      "loss": 0.2659,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -2.1182336807250977,
+      "rewards/margins": 3.0465946197509766,
+      "rewards/rejected": -5.164828300476074,
       "step": 330
     },
     {
       "epoch": 0.87,
-      "grad_norm": 77.2990699180903,
-      "learning_rate": 5.119313618049309e-09,
-      "logits/chosen": -4.570425987243652,
-      "logits/rejected": -4.913475513458252,
-      "logps/chosen": -250.6792449951172,
-      "logps/rejected": -277.26556396484375,
-      "loss": 0.1723,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 1.1733391284942627,
-      "rewards/margins": 4.697513580322266,
-      "rewards/rejected": -3.524174451828003,
+      "grad_norm": 45.65268574690239,
+      "learning_rate": 2.5596568090246545e-08,
+      "logits/chosen": -4.608619213104248,
+      "logits/rejected": -4.683640956878662,
+      "logps/chosen": -464.8091735839844,
+      "logps/rejected": -704.0225219726562,
+      "loss": 0.274,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.2061734199523926,
+      "rewards/margins": 2.4553611278533936,
+      "rewards/rejected": -4.661534786224365,
       "step": 340
     },
     {
       "epoch": 0.9,
-      "grad_norm": 74.61892537865367,
-      "learning_rate": 3.3290021961708158e-09,
-      "logits/chosen": -4.588479995727539,
-      "logits/rejected": -4.761317253112793,
-      "logps/chosen": -238.91921997070312,
-      "logps/rejected": -291.2458190917969,
-      "loss": 0.1937,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.8275976181030273,
-      "rewards/margins": 4.001389026641846,
-      "rewards/rejected": -3.1737911701202393,
+      "grad_norm": 44.7612300771836,
+      "learning_rate": 1.664501098085408e-08,
+      "logits/chosen": -4.53527307510376,
+      "logits/rejected": -4.685696601867676,
+      "logps/chosen": -467.08319091796875,
+      "logps/rejected": -695.1188354492188,
+      "loss": 0.2829,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.9506038427352905,
+      "rewards/margins": 2.6737771034240723,
+      "rewards/rejected": -4.624380588531494,
       "step": 350
     },
     {
       "epoch": 0.92,
-      "grad_norm": 81.22407668854541,
-      "learning_rate": 1.9123215591052013e-09,
-      "logits/chosen": -4.583038806915283,
-      "logits/rejected": -4.805889129638672,
-      "logps/chosen": -244.8368682861328,
-      "logps/rejected": -294.9869079589844,
-      "loss": 0.1907,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 0.8399018049240112,
-      "rewards/margins": 4.216121673583984,
-      "rewards/rejected": -3.3762192726135254,
+      "grad_norm": 44.963702961151455,
+      "learning_rate": 9.561607795526006e-09,
+      "logits/chosen": -4.5610480308532715,
+      "logits/rejected": -4.635122299194336,
+      "logps/chosen": -472.186767578125,
+      "logps/rejected": -707.2020263671875,
+      "loss": 0.2947,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -2.069366931915283,
+      "rewards/margins": 2.61225962638855,
+      "rewards/rejected": -4.681626796722412,
       "step": 360
     },
     {
       "epoch": 0.95,
-      "grad_norm": 75.17805842008224,
-      "learning_rate": 8.806131292167618e-10,
-      "logits/chosen": -4.595518112182617,
-      "logits/rejected": -4.752079010009766,
-      "logps/chosen": -239.1554412841797,
-      "logps/rejected": -302.4869079589844,
-      "loss": 0.1904,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.8832891583442688,
-      "rewards/margins": 4.165283679962158,
-      "rewards/rejected": -3.281994581222534,
+      "grad_norm": 55.53107755695674,
+      "learning_rate": 4.4030656460838086e-09,
+      "logits/chosen": -4.579834938049316,
+      "logits/rejected": -4.7375922203063965,
+      "logps/chosen": -435.17742919921875,
+      "logps/rejected": -687.3599243164062,
+      "loss": 0.2897,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.8747832775115967,
+      "rewards/margins": 2.8274025917053223,
+      "rewards/rejected": -4.702186107635498,
       "step": 370
     },
     {
       "epoch": 0.97,
-      "grad_norm": 91.00267878372446,
-      "learning_rate": 2.4213638345040867e-10,
-      "logits/chosen": -4.70483922958374,
-      "logits/rejected": -4.97845983505249,
-      "logps/chosen": -242.5469207763672,
-      "logps/rejected": -292.7474670410156,
-      "loss": 0.1788,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.9767888188362122,
-      "rewards/margins": 4.587931156158447,
-      "rewards/rejected": -3.61114239692688,
+      "grad_norm": 59.601168122698574,
+      "learning_rate": 1.2106819172520432e-09,
+      "logits/chosen": -4.654064655303955,
+      "logits/rejected": -4.814013481140137,
+      "logps/chosen": -469.78271484375,
+      "logps/rejected": -723.7748413085938,
+      "loss": 0.2792,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -1.9299259185791016,
+      "rewards/margins": 2.920754909515381,
+      "rewards/rejected": -4.850680351257324,
       "step": 380
     },
     {
       "epoch": 1.0,
-      "grad_norm": 100.5241948062632,
-      "learning_rate": 2.0027310073833516e-12,
-      "logits/chosen": -4.696263313293457,
-      "logits/rejected": -4.96966028213501,
-      "logps/chosen": -238.3385772705078,
-      "logps/rejected": -292.5868835449219,
-      "loss": 0.1773,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 1.066699743270874,
-      "rewards/margins": 4.670289516448975,
-      "rewards/rejected": -3.6035892963409424,
+      "grad_norm": 49.27493205902015,
+      "learning_rate": 1.0013655036916757e-11,
+      "logits/chosen": -4.536452293395996,
+      "logits/rejected": -4.714889049530029,
+      "logps/chosen": -417.2685546875,
+      "logps/rejected": -692.3974609375,
+      "loss": 0.2647,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -1.6718591451644897,
+      "rewards/margins": 2.985755443572998,
+      "rewards/rejected": -4.657614707946777,
       "step": 390
     },
     {
       "epoch": 1.0,
       "step": 391,
       "total_flos": 0.0,
-      "train_loss": 0.256967593336959,
-      "train_runtime": 6146.1986,
-      "train_samples_per_second": 8.135,
-      "train_steps_per_second": 0.064
+      "train_loss": 0.3765579266164004,
+      "train_runtime": 6025.4327,
+      "train_samples_per_second": 8.297,
+      "train_steps_per_second": 0.065
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b25fa3cc8dbf628e10eaf56cc8c09a3c9395753142fdbb66bfb670572424aac
+oid sha256:946f97a778d62178179d703990f714ff6e66ace0391300267631cc87090e86e3
 size 6264