Model save

README.md

@@ -15,15 +15,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.3650
-- Rewards/chosen: -0.6695
-- Rewards/rejected: -0.0468
-- Rewards/accuracies: 0.3828
-- Rewards/margins: -0.6227
-- Logps/rejected: -518.8555
-- Logps/chosen: -397.1692
-- Logits/rejected: -4.7484
-- Logits/chosen: -4.5219
+- Loss: 1.1360
+- Rewards/chosen: -0.1815
+- Rewards/rejected: 0.2981
+- Rewards/accuracies: 0.4102
+- Rewards/margins: -0.4796
+- Logps/rejected: -515.4065
+- Logps/chosen: -392.2896
+- Logits/rejected: -4.7596
+- Logits/chosen: -4.5397
 
 ## Model description
 
@@ -42,7 +42,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 1e-07
+- learning_rate: 5e-08
 - train_batch_size: 8
 - eval_batch_size: 8
 - seed: 42
@@ -60,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.216         | 0.26  | 100  | 1.1906          | -0.1088        | 0.4672           | 0.3984             | -0.5759         | -513.7159      | -391.5621    | -4.7718         | -4.5558       |
-| 0.1925        | 0.51  | 200  | 1.3563          | -0.3996        | 0.2819           | 0.375              | -0.6815         | -515.5683      | -394.4702    | -4.7427         | -4.5166       |
-| 0.179         | 0.77  | 300  | 1.3650          | -0.6695        | -0.0468          | 0.3828             | -0.6227         | -518.8555      | -397.1692    | -4.7484         | -4.5219       |
+| 0.2984        | 0.26  | 100  | 0.9403          | -0.0277        | 0.3050           | 0.4062             | -0.3328         | -515.3372      | -390.7516    | -4.7620         | -4.5507       |
+| 0.2338        | 0.51  | 200  | 1.0997          | -0.0605        | 0.4142           | 0.3867             | -0.4747         | -514.2458      | -391.0792    | -4.7584         | -4.5386       |
+| 0.2158        | 0.77  | 300  | 1.1360          | -0.1815        | 0.2981           | 0.4102             | -0.4796         | -515.4065      | -392.2896    | -4.7596         | -4.5397       |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.23296360492401416,
-    "train_runtime": 6150.4195,
+    "train_loss": 0.29024992444935965,
+    "train_runtime": 6148.7126,
     "train_samples": 50000,
-    "train_samples_per_second": 8.13,
+    "train_samples_per_second": 8.132,
     "train_steps_per_second": 0.064
 }

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "/mnt/bn/xuruijie-llm/checkpoints/hh-rlhf/sft_0521/checkpoint-5500/",
+  "_name_or_path": "/mnt/bn/xuruijie-llm/checkpoints/new_world/v1-ultral",
   "architectures": [
     "MistralForCausalLM"
   ],
@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.41.1",
+  "transformers_version": "4.38.2",
   "use_cache": false,
   "vocab_size": 32002
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e8b81887b85183d4a4fce153e9d2d2fb4c1c12ff26edfc47a967614085e4ba54
+oid sha256:7b1567f82232ae4543a6b8c947587ca02d81b5b1a6d612f6c48505a456bcdaef
 size 4943178720

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:12f930756c5e712e7f34db59925b5831febcdc775f5258f7984274f15dffbba9
+oid sha256:3b63c90bc00eb3c20661922864d3e4196b77a831cb51f070d94bfb2c31046bb6
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab661929946b1e6f5490a343690d65f428f51fd0146f96739d6af9238c402383
+oid sha256:dccab9d7e3cbbc2fe51be697f1fb53d79146c3c708bad129f55934efe50fd566
 size 4540532728

runs/May28_02-16-51_n136-100-194/events.out.tfevents.1716833871.n136-100-194.596156.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2a48c5ff4c5d32d77f5b0be7dcc2fc1e60d7d788f425eedccd2c1c7be4c3d166
-size 28262
+oid sha256:ff847d89aa0668a584824d688412789e4fc2ae8307e332ce41013474c968b365
+size 34808

tokenizer.json

@@ -152,7 +152,6 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
-    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.23296360492401416,
-    "train_runtime": 6150.4195,
+    "train_loss": 0.29024992444935965,
+    "train_runtime": 6148.7126,
     "train_samples": 50000,
-    "train_samples_per_second": 8.13,
+    "train_samples_per_second": 8.132,
     "train_steps_per_second": 0.064
 }

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 421.7748930851862,
-      "learning_rate": 2.5e-09,
+      "grad_norm": 422.289956912934,
+      "learning_rate": 1.25e-09,
       "logits/chosen": -4.623842239379883,
       "logits/rejected": -4.85917854309082,
       "logps/chosen": -239.31422424316406,
       "logps/rejected": -207.56365966796875,
-      "loss": 0.693,
+      "loss": 0.6959,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,644 +25,644 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 395.00460150231794,
-      "learning_rate": 2.5e-08,
-      "logits/chosen": -4.33427619934082,
-      "logits/rejected": -4.64423942565918,
-      "logps/chosen": -265.1573791503906,
-      "logps/rejected": -215.7504425048828,
-      "loss": 0.693,
-      "rewards/accuracies": 0.4444444477558136,
-      "rewards/chosen": -0.0050499276258051395,
-      "rewards/margins": -0.0008213710389100015,
-      "rewards/rejected": -0.004228556528687477,
+      "grad_norm": 397.335507450448,
+      "learning_rate": 1.25e-08,
+      "logits/chosen": -4.334493160247803,
+      "logits/rejected": -4.643917560577393,
+      "logps/chosen": -265.1265563964844,
+      "logps/rejected": -215.76223754882812,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.0019649432506412268,
+      "rewards/margins": 0.0034448718652129173,
+      "rewards/rejected": -0.005409814417362213,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 398.86324560087934,
-      "learning_rate": 5e-08,
-      "logits/chosen": -4.5085954666137695,
-      "logits/rejected": -4.7428812980651855,
-      "logps/chosen": -267.77325439453125,
-      "logps/rejected": -216.7412567138672,
-      "loss": 0.6821,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 0.016667194664478302,
-      "rewards/margins": 0.029861312359571457,
-      "rewards/rejected": -0.01319411676377058,
+      "grad_norm": 411.8624893441942,
+      "learning_rate": 2.5e-08,
+      "logits/chosen": -4.506826400756836,
+      "logits/rejected": -4.740732192993164,
+      "logps/chosen": -267.86932373046875,
+      "logps/rejected": -216.64578247070312,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.007061410695314407,
+      "rewards/margins": 0.010701502673327923,
+      "rewards/rejected": -0.003640091512352228,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "grad_norm": 388.1898348935921,
-      "learning_rate": 7.5e-08,
-      "logits/chosen": -4.594970703125,
-      "logits/rejected": -4.776912689208984,
-      "logps/chosen": -257.5873107910156,
-      "logps/rejected": -215.4453582763672,
-      "loss": 0.6195,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.09455005824565887,
-      "rewards/margins": 0.18580853939056396,
-      "rewards/rejected": -0.09125850349664688,
+      "grad_norm": 426.4548766919431,
+      "learning_rate": 3.75e-08,
+      "logits/chosen": -4.585576057434082,
+      "logits/rejected": -4.762608528137207,
+      "logps/chosen": -258.25,
+      "logps/rejected": -214.71231079101562,
+      "loss": 0.6756,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.028281379491090775,
+      "rewards/margins": 0.04623327776789665,
+      "rewards/rejected": -0.017951902002096176,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 267.0875315068977,
-      "learning_rate": 1e-07,
-      "logits/chosen": -4.637397766113281,
-      "logits/rejected": -4.731802940368652,
-      "logps/chosen": -249.911865234375,
-      "logps/rejected": -222.6023712158203,
-      "loss": 0.4813,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.36873525381088257,
-      "rewards/margins": 0.661862313747406,
-      "rewards/rejected": -0.29312705993652344,
+      "grad_norm": 368.66232685986097,
+      "learning_rate": 5e-08,
+      "logits/chosen": -4.62213659286499,
+      "logits/rejected": -4.706842422485352,
+      "logps/chosen": -252.6122283935547,
+      "logps/rejected": -220.41427612304688,
+      "loss": 0.6177,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.0986957773566246,
+      "rewards/margins": 0.1730131059885025,
+      "rewards/rejected": -0.07431730628013611,
       "step": 40
     },
     {
       "epoch": 0.13,
-      "grad_norm": 193.9339508971275,
-      "learning_rate": 9.979985922607475e-08,
-      "logits/chosen": -4.531577110290527,
-      "logits/rejected": -4.7472710609436035,
-      "logps/chosen": -265.0107116699219,
-      "logps/rejected": -231.0427703857422,
-      "loss": 0.3721,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 0.6958638429641724,
-      "rewards/margins": 1.2129887342453003,
-      "rewards/rejected": -0.5171248316764832,
+      "grad_norm": 294.81806277707193,
+      "learning_rate": 4.989992961303737e-08,
+      "logits/chosen": -4.516692161560059,
+      "logits/rejected": -4.714283466339111,
+      "logps/chosen": -269.63470458984375,
+      "logps/rejected": -227.5962371826172,
+      "loss": 0.5368,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.2334650307893753,
+      "rewards/margins": 0.4059367775917053,
+      "rewards/rejected": -0.17247170209884644,
       "step": 50
     },
     {
       "epoch": 0.15,
-      "grad_norm": 197.66681943180416,
-      "learning_rate": 9.92010391574745e-08,
-      "logits/chosen": -4.6342668533325195,
-      "logits/rejected": -4.783431053161621,
-      "logps/chosen": -230.5199432373047,
-      "logps/rejected": -224.8899383544922,
-      "loss": 0.281,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 1.0602517127990723,
-      "rewards/margins": 2.0471043586730957,
-      "rewards/rejected": -0.9868523478507996,
+      "grad_norm": 263.2732793061953,
+      "learning_rate": 4.960051957873725e-08,
+      "logits/chosen": -4.610293388366699,
+      "logits/rejected": -4.734058856964111,
+      "logps/chosen": -237.3783721923828,
+      "logps/rejected": -218.6456298828125,
+      "loss": 0.4381,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.3744103014469147,
+      "rewards/margins": 0.7368327975273132,
+      "rewards/rejected": -0.3624224364757538,
       "step": 60
     },
     {
       "epoch": 0.18,
-      "grad_norm": 162.90653777538404,
-      "learning_rate": 9.820833372667812e-08,
-      "logits/chosen": -4.519268989562988,
-      "logits/rejected": -4.774823188781738,
-      "logps/chosen": -240.6663818359375,
-      "logps/rejected": -230.677490234375,
-      "loss": 0.2583,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 1.247557520866394,
-      "rewards/margins": 2.539060592651367,
-      "rewards/rejected": -1.2915031909942627,
+      "grad_norm": 243.25478495437125,
+      "learning_rate": 4.910416686333906e-08,
+      "logits/chosen": -4.496267795562744,
+      "logits/rejected": -4.724743843078613,
+      "logps/chosen": -248.19540405273438,
+      "logps/rejected": -222.69140625,
+      "loss": 0.4014,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 0.49465441703796387,
+      "rewards/margins": 0.9875491261482239,
+      "rewards/rejected": -0.49289458990097046,
       "step": 70
     },
     {
       "epoch": 0.2,
-      "grad_norm": 139.86229845766394,
-      "learning_rate": 9.682969016701356e-08,
-      "logits/chosen": -4.514620780944824,
-      "logits/rejected": -4.760144233703613,
-      "logps/chosen": -247.26364135742188,
-      "logps/rejected": -244.28866577148438,
-      "loss": 0.2248,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 1.3956347703933716,
-      "rewards/margins": 2.7926278114318848,
-      "rewards/rejected": -1.3969930410385132,
+      "grad_norm": 203.40516001004536,
+      "learning_rate": 4.841484508350678e-08,
+      "logits/chosen": -4.488529682159424,
+      "logits/rejected": -4.711686611175537,
+      "logps/chosen": -254.6974639892578,
+      "logps/rejected": -235.7099151611328,
+      "loss": 0.3429,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.652252197265625,
+      "rewards/margins": 1.1913691759109497,
+      "rewards/rejected": -0.5391170978546143,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 208.01416582057018,
-      "learning_rate": 9.507614539004081e-08,
-      "logits/chosen": -4.604180812835693,
-      "logits/rejected": -4.849738597869873,
-      "logps/chosen": -236.37899780273438,
-      "logps/rejected": -221.75564575195312,
-      "loss": 0.2139,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 1.4833731651306152,
-      "rewards/margins": 3.3759377002716064,
-      "rewards/rejected": -1.8925644159317017,
+      "grad_norm": 237.79736326938396,
+      "learning_rate": 4.7538072695020406e-08,
+      "logits/chosen": -4.58192777633667,
+      "logits/rejected": -4.797459602355957,
+      "logps/chosen": -243.06143188476562,
+      "logps/rejected": -210.63308715820312,
+      "loss": 0.3085,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 0.8151354789733887,
+      "rewards/margins": 1.595442533493042,
+      "rewards/rejected": -0.7803069353103638,
       "step": 90
     },
     {
       "epoch": 0.26,
-      "grad_norm": 168.83388514254116,
-      "learning_rate": 9.296173762811083e-08,
-      "logits/chosen": -4.470773220062256,
-      "logits/rejected": -4.786902904510498,
-      "logps/chosen": -240.86477661132812,
-      "logps/rejected": -247.6592559814453,
-      "loss": 0.216,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 1.5853464603424072,
-      "rewards/margins": 3.6403656005859375,
-      "rewards/rejected": -2.0550191402435303,
+      "grad_norm": 222.07031689896021,
+      "learning_rate": 4.6480868814055416e-08,
+      "logits/chosen": -4.459914207458496,
+      "logits/rejected": -4.747165679931641,
+      "logps/chosen": -247.2704620361328,
+      "logps/rejected": -236.6487274169922,
+      "loss": 0.2984,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 0.9447771906852722,
+      "rewards/margins": 1.8987438678741455,
+      "rewards/rejected": -0.953966498374939,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -4.555809497833252,
-      "eval_logits/rejected": -4.771756172180176,
-      "eval_logps/chosen": -391.56207275390625,
-      "eval_logps/rejected": -513.7159423828125,
-      "eval_loss": 1.1906155347824097,
-      "eval_rewards/accuracies": 0.3984375,
-      "eval_rewards/chosen": -0.10876215249300003,
-      "eval_rewards/margins": -0.575933575630188,
-      "eval_rewards/rejected": 0.46717149019241333,
-      "eval_runtime": 97.8557,
-      "eval_samples_per_second": 20.438,
+      "eval_logits/chosen": -4.55070686340332,
+      "eval_logits/rejected": -4.762002944946289,
+      "eval_logps/chosen": -390.7516174316406,
+      "eval_logps/rejected": -515.337158203125,
+      "eval_loss": 0.9402573704719543,
+      "eval_rewards/accuracies": 0.40625,
+      "eval_rewards/chosen": -0.027714576572179794,
+      "eval_rewards/margins": -0.3327641487121582,
+      "eval_rewards/rejected": 0.3050495684146881,
+      "eval_runtime": 97.8238,
+      "eval_samples_per_second": 20.445,
       "eval_steps_per_second": 0.327,
       "step": 100
     },
     {
       "epoch": 0.28,
-      "grad_norm": 158.8626687509829,
-      "learning_rate": 9.050339404945832e-08,
-      "logits/chosen": -4.515485763549805,
-      "logits/rejected": -4.795956134796143,
-      "logps/chosen": -232.4200439453125,
-      "logps/rejected": -236.90969848632812,
-      "loss": 0.1985,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.6765697002410889,
-      "rewards/margins": 3.698953628540039,
-      "rewards/rejected": -2.02238392829895,
+      "grad_norm": 179.38114597248955,
+      "learning_rate": 4.525169702472916e-08,
+      "logits/chosen": -4.51773738861084,
+      "logits/rejected": -4.777291297912598,
+      "logps/chosen": -238.6410369873047,
+      "logps/rejected": -227.3874053955078,
+      "loss": 0.2711,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 1.0544707775115967,
+      "rewards/margins": 2.1246237754821777,
+      "rewards/rejected": -1.070152997970581,
       "step": 110
     },
     {
       "epoch": 0.31,
-      "grad_norm": 135.50801896928647,
-      "learning_rate": 8.77207952455395e-08,
-      "logits/chosen": -4.454637050628662,
-      "logits/rejected": -4.722542762756348,
-      "logps/chosen": -256.7890930175781,
-      "logps/rejected": -248.82601928710938,
-      "loss": 0.1975,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.9200481176376343,
-      "rewards/margins": 3.9417896270751953,
-      "rewards/rejected": -2.0217413902282715,
+      "grad_norm": 163.17108435846185,
+      "learning_rate": 4.386039762276975e-08,
+      "logits/chosen": -4.462746620178223,
+      "logits/rejected": -4.7056145668029785,
+      "logps/chosen": -262.86920166015625,
+      "logps/rejected": -238.99801635742188,
+      "loss": 0.267,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 1.3120397329330444,
+      "rewards/margins": 2.3509771823883057,
+      "rewards/rejected": -1.0389372110366821,
       "step": 120
     },
     {
       "epoch": 0.33,
-      "grad_norm": 154.49491996909583,
-      "learning_rate": 8.463621767547997e-08,
-      "logits/chosen": -4.527877330780029,
-      "logits/rejected": -4.81399393081665,
-      "logps/chosen": -241.6123809814453,
-      "logps/rejected": -239.6664276123047,
-      "loss": 0.1993,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 1.683153748512268,
-      "rewards/margins": 4.116929054260254,
-      "rewards/rejected": -2.4337756633758545,
+      "grad_norm": 229.70994586547334,
+      "learning_rate": 4.231810883773999e-08,
+      "logits/chosen": -4.532160758972168,
+      "logits/rejected": -4.803128719329834,
+      "logps/chosen": -246.3385009765625,
+      "logps/rejected": -227.83792114257812,
+      "loss": 0.2577,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 1.2105365991592407,
+      "rewards/margins": 2.461458921432495,
+      "rewards/rejected": -1.2509223222732544,
       "step": 130
     },
     {
       "epoch": 0.36,
-      "grad_norm": 161.00485250742125,
-      "learning_rate": 8.127435532896387e-08,
-      "logits/chosen": -4.5556840896606445,
-      "logits/rejected": -4.849120140075684,
-      "logps/chosen": -268.4659118652344,
-      "logps/rejected": -258.8057556152344,
-      "loss": 0.1874,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.5729143619537354,
-      "rewards/margins": 4.236142158508301,
-      "rewards/rejected": -2.6632275581359863,
+      "grad_norm": 152.87001146329087,
+      "learning_rate": 4.063717766448194e-08,
+      "logits/chosen": -4.559675216674805,
+      "logits/rejected": -4.842075824737549,
+      "logps/chosen": -272.3346252441406,
+      "logps/rejected": -246.54464721679688,
+      "loss": 0.2453,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 1.1860424280166626,
+      "rewards/margins": 2.6231608390808105,
+      "rewards/rejected": -1.4371181726455688,
       "step": 140
     },
     {
       "epoch": 0.38,
-      "grad_norm": 157.59104792116975,
-      "learning_rate": 7.766212203526569e-08,
-      "logits/chosen": -4.545467853546143,
-      "logits/rejected": -4.822617053985596,
-      "logps/chosen": -235.0808563232422,
-      "logps/rejected": -246.62606811523438,
-      "loss": 0.1878,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.7337404489517212,
-      "rewards/margins": 4.484419822692871,
-      "rewards/rejected": -2.7506790161132812,
+      "grad_norm": 176.7311824941399,
+      "learning_rate": 3.8831061017632845e-08,
+      "logits/chosen": -4.557957172393799,
+      "logits/rejected": -4.812293529510498,
+      "logps/chosen": -239.32144165039062,
+      "logps/rejected": -232.82479858398438,
+      "loss": 0.2425,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 1.3096827268600464,
+      "rewards/margins": 2.680234432220459,
+      "rewards/rejected": -1.3705517053604126,
       "step": 150
     },
     {
       "epoch": 0.41,
-      "grad_norm": 153.9528654470227,
-      "learning_rate": 7.382843600106538e-08,
-      "logits/chosen": -4.591027736663818,
-      "logits/rejected": -4.801583290100098,
-      "logps/chosen": -234.0863494873047,
-      "logps/rejected": -241.58761596679688,
-      "loss": 0.1687,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.7370154857635498,
-      "rewards/margins": 4.334869861602783,
-      "rewards/rejected": -2.5978541374206543,
+      "grad_norm": 179.5862213559593,
+      "learning_rate": 3.691421800053269e-08,
+      "logits/chosen": -4.614952564239502,
+      "logits/rejected": -4.799678802490234,
+      "logps/chosen": -238.4506378173828,
+      "logps/rejected": -229.4785614013672,
+      "loss": 0.2216,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 1.300586462020874,
+      "rewards/margins": 2.687537908554077,
+      "rewards/rejected": -1.3869514465332031,
       "step": 160
     },
     {
       "epoch": 0.43,
-      "grad_norm": 134.86505617626943,
-      "learning_rate": 6.980398830195784e-08,
-      "logits/chosen": -4.457674980163574,
-      "logits/rejected": -4.7707014083862305,
-      "logps/chosen": -239.03787231445312,
-      "logps/rejected": -248.50393676757812,
-      "loss": 0.1647,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 2.203636407852173,
-      "rewards/margins": 5.073732852935791,
-      "rewards/rejected": -2.870095729827881,
+      "grad_norm": 160.70849517962517,
+      "learning_rate": 3.490199415097892e-08,
+      "logits/chosen": -4.499081611633301,
+      "logits/rejected": -4.779529571533203,
+      "logps/chosen": -244.9915771484375,
+      "logps/rejected": -235.46743774414062,
+      "loss": 0.2108,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 1.6082652807235718,
+      "rewards/margins": 3.174710988998413,
+      "rewards/rejected": -1.5664453506469727,
       "step": 170
     },
     {
       "epoch": 0.46,
-      "grad_norm": 134.91302963185322,
-      "learning_rate": 6.562099718102787e-08,
-      "logits/chosen": -4.556507110595703,
-      "logits/rejected": -4.814910411834717,
-      "logps/chosen": -217.7527618408203,
-      "logps/rejected": -222.974365234375,
-      "loss": 0.1885,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.9110908508300781,
-      "rewards/margins": 4.606502532958984,
-      "rewards/rejected": -2.695411443710327,
+      "grad_norm": 149.11333079529007,
+      "learning_rate": 3.2810498590513937e-08,
+      "logits/chosen": -4.59390926361084,
+      "logits/rejected": -4.832152366638184,
+      "logps/chosen": -222.95986938476562,
+      "logps/rejected": -211.53585815429688,
+      "loss": 0.2274,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 1.3903782367706299,
+      "rewards/margins": 2.9419426918029785,
+      "rewards/rejected": -1.5515644550323486,
       "step": 180
     },
     {
       "epoch": 0.49,
-      "grad_norm": 164.38739923975325,
-      "learning_rate": 6.131295012148612e-08,
-      "logits/chosen": -4.534274101257324,
-      "logits/rejected": -4.6962666511535645,
-      "logps/chosen": -241.34304809570312,
-      "logps/rejected": -259.50860595703125,
-      "loss": 0.1854,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 1.9549798965454102,
-      "rewards/margins": 4.389232158660889,
-      "rewards/rejected": -2.434252977371216,
+      "grad_norm": 171.77003361632143,
+      "learning_rate": 3.065647506074306e-08,
+      "logits/chosen": -4.56182861328125,
+      "logits/rejected": -4.7075724601745605,
+      "logps/chosen": -245.95556640625,
+      "logps/rejected": -247.3394012451172,
+      "loss": 0.2299,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 1.4937294721603394,
+      "rewards/margins": 2.711061477661133,
+      "rewards/rejected": -1.2173320055007935,
       "step": 190
     },
     {
       "epoch": 0.51,
-      "grad_norm": 145.61398569600198,
-      "learning_rate": 5.691433575823665e-08,
-      "logits/chosen": -4.528763771057129,
-      "logits/rejected": -4.724316596984863,
-      "logps/chosen": -234.06503295898438,
-      "logps/rejected": -241.2750701904297,
-      "loss": 0.1925,
+      "grad_norm": 152.3542212939215,
+      "learning_rate": 2.8457167879118325e-08,
+      "logits/chosen": -4.556639194488525,
+      "logits/rejected": -4.735670566558838,
+      "logps/chosen": -237.8849639892578,
+      "logps/rejected": -229.5240020751953,
+      "loss": 0.2338,
       "rewards/accuracies": 0.875,
-      "rewards/chosen": 1.8074201345443726,
-      "rewards/margins": 4.400271892547607,
-      "rewards/rejected": -2.5928521156311035,
+      "rewards/chosen": 1.4254279136657715,
+      "rewards/margins": 2.843172073364258,
+      "rewards/rejected": -1.4177442789077759,
       "step": 200
     },
     {
       "epoch": 0.51,
-      "eval_logits/chosen": -4.5165863037109375,
-      "eval_logits/rejected": -4.742655277252197,
-      "eval_logps/chosen": -394.47015380859375,
-      "eval_logps/rejected": -515.5682983398438,
-      "eval_loss": 1.3562896251678467,
-      "eval_rewards/accuracies": 0.375,
-      "eval_rewards/chosen": -0.39956870675086975,
-      "eval_rewards/margins": -0.6815085411071777,
-      "eval_rewards/rejected": 0.28193986415863037,
-      "eval_runtime": 97.991,
-      "eval_samples_per_second": 20.41,
-      "eval_steps_per_second": 0.327,
+      "eval_logits/chosen": -4.538640975952148,
+      "eval_logits/rejected": -4.758352279663086,
+      "eval_logps/chosen": -391.07916259765625,
+      "eval_logps/rejected": -514.2457885742188,
+      "eval_loss": 1.0996507406234741,
+      "eval_rewards/accuracies": 0.38671875,
+      "eval_rewards/chosen": -0.06046929210424423,
+      "eval_rewards/margins": -0.4746614694595337,
+      "eval_rewards/rejected": 0.41419219970703125,
+      "eval_runtime": 98.0841,
+      "eval_samples_per_second": 20.391,
+      "eval_steps_per_second": 0.326,
       "step": 200
     },
     {
       "epoch": 0.54,
-      "grad_norm": 126.33823591196861,
-      "learning_rate": 5.2460367774593905e-08,
-      "logits/chosen": -4.592769622802734,
-      "logits/rejected": -4.851358413696289,
-      "logps/chosen": -244.64755249023438,
-      "logps/rejected": -259.65142822265625,
-      "loss": 0.1489,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 2.1237542629241943,
-      "rewards/margins": 5.224299907684326,
-      "rewards/rejected": -3.10054612159729,
+      "grad_norm": 139.65082338502864,
+      "learning_rate": 2.6230183887296952e-08,
+      "logits/chosen": -4.619709014892578,
+      "logits/rejected": -4.859663963317871,
+      "logps/chosen": -249.32113647460938,
+      "logps/rejected": -246.33468627929688,
+      "loss": 0.1967,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 1.6563961505889893,
+      "rewards/margins": 3.425267457962036,
+      "rewards/rejected": -1.768871545791626,
       "step": 210
     },
     {
       "epoch": 0.56,
-      "grad_norm": 139.63020748735394,
-      "learning_rate": 4.798670299452925e-08,
-      "logits/chosen": -4.435277462005615,
-      "logits/rejected": -4.771773338317871,
-      "logps/chosen": -242.7268524169922,
-      "logps/rejected": -258.1542053222656,
-      "loss": 0.1754,
+      "grad_norm": 147.42119588032455,
+      "learning_rate": 2.3993351497264626e-08,
+      "logits/chosen": -4.466108798980713,
+      "logits/rejected": -4.793113708496094,
+      "logps/chosen": -247.28756713867188,
+      "logps/rejected": -244.1797637939453,
+      "loss": 0.2159,
       "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 2.0084481239318848,
-      "rewards/margins": 5.356553077697754,
-      "rewards/rejected": -3.348104953765869,
+      "rewards/chosen": 1.5523773431777954,
+      "rewards/margins": 3.5030410289764404,
+      "rewards/rejected": -1.9506635665893555,
       "step": 220
     },
     {
       "epoch": 0.59,
-      "grad_norm": 146.4580429105822,
-      "learning_rate": 4.3529155927297226e-08,
-      "logits/chosen": -4.511857986450195,
-      "logits/rejected": -4.845773696899414,
-      "logps/chosen": -241.4152374267578,
-      "logps/rejected": -257.61944580078125,
-      "loss": 0.1815,
+      "grad_norm": 158.88183877851495,
+      "learning_rate": 2.1764577963648613e-08,
+      "logits/chosen": -4.541924476623535,
+      "logits/rejected": -4.858447074890137,
+      "logps/chosen": -245.6726531982422,
+      "logps/rejected": -243.28677368164062,
+      "loss": 0.2197,
       "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 1.9553873538970947,
-      "rewards/margins": 5.346222877502441,
-      "rewards/rejected": -3.390836000442505,
+      "rewards/chosen": 1.5296446084976196,
+      "rewards/margins": 3.487210750579834,
+      "rewards/rejected": -1.957566499710083,
       "step": 230
     },
     {
       "epoch": 0.61,
-      "grad_norm": 120.36905618473185,
-      "learning_rate": 3.9123412049691636e-08,
-      "logits/chosen": -4.488260269165039,
-      "logits/rejected": -4.767331123352051,
-      "logps/chosen": -252.67538452148438,
-      "logps/rejected": -255.2183837890625,
-      "loss": 0.1687,
+      "grad_norm": 157.203527489415,
+      "learning_rate": 1.9561706024845818e-08,
+      "logits/chosen": -4.5143561363220215,
+      "logits/rejected": -4.772491455078125,
+      "logps/chosen": -256.7393798828125,
+      "logps/rejected": -240.91226196289062,
+      "loss": 0.2123,
       "rewards/accuracies": 0.9375,
-      "rewards/chosen": 2.014777421951294,
-      "rewards/margins": 5.443254470825195,
-      "rewards/rejected": -3.4284775257110596,
+      "rewards/chosen": 1.6083800792694092,
+      "rewards/margins": 3.60624623298645,
+      "rewards/rejected": -1.9978656768798828,
       "step": 240
     },
     {
       "epoch": 0.64,
-      "grad_norm": 143.11323504040695,
-      "learning_rate": 3.480474212128766e-08,
-      "logits/chosen": -4.618422031402588,
-      "logits/rejected": -4.878515243530273,
-      "logps/chosen": -231.21737670898438,
-      "logps/rejected": -236.189208984375,
-      "loss": 0.1614,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.7484108209609985,
-      "rewards/margins": 4.606410503387451,
-      "rewards/rejected": -2.857999324798584,
+      "grad_norm": 159.9556615524972,
+      "learning_rate": 1.740237106064383e-08,
+      "logits/chosen": -4.641883850097656,
+      "logits/rejected": -4.893360614776611,
+      "logps/chosen": -234.55264282226562,
+      "logps/rejected": -224.25631713867188,
+      "loss": 0.2024,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 1.414884090423584,
+      "rewards/margins": 3.079598903656006,
+      "rewards/rejected": -1.6647151708602905,
       "step": 250
     },
     {
       "epoch": 0.66,
-      "grad_norm": 193.1124896787253,
-      "learning_rate": 3.060771981975726e-08,
-      "logits/chosen": -4.48922872543335,
-      "logits/rejected": -4.790060997009277,
-      "logps/chosen": -234.9071502685547,
-      "logps/rejected": -258.86639404296875,
-      "loss": 0.1653,
+      "grad_norm": 198.60636587673295,
+      "learning_rate": 1.530385990987863e-08,
+      "logits/chosen": -4.517378330230713,
+      "logits/rejected": -4.799233913421631,
+      "logps/chosen": -238.4044189453125,
+      "logps/rejected": -242.4611358642578,
+      "loss": 0.2025,
       "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.9141381978988647,
-      "rewards/margins": 5.825932025909424,
-      "rewards/rejected": -3.9117939472198486,
+      "rewards/chosen": 1.5644124746322632,
+      "rewards/margins": 3.8356785774230957,
+      "rewards/rejected": -2.271266460418701,
       "step": 260
     },
     {
       "epoch": 0.69,
-      "grad_norm": 194.5027032928207,
-      "learning_rate": 2.6565944956764818e-08,
-      "logits/chosen": -4.580440044403076,
-      "logits/rejected": -4.814294815063477,
-      "logps/chosen": -242.62802124023438,
-      "logps/rejected": -251.90444946289062,
-      "loss": 0.1739,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.84052312374115,
-      "rewards/margins": 5.5284624099731445,
-      "rewards/rejected": -3.687939405441284,
+      "grad_norm": 186.10967020286805,
+      "learning_rate": 1.3282972478382409e-08,
+      "logits/chosen": -4.603947639465332,
+      "logits/rejected": -4.826247215270996,
+      "logps/chosen": -245.811279296875,
+      "logps/rejected": -236.3955841064453,
+      "loss": 0.2127,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 1.5221991539001465,
+      "rewards/margins": 3.6592516899108887,
+      "rewards/rejected": -2.137052536010742,
       "step": 270
     },
     {
       "epoch": 0.72,
-      "grad_norm": 134.789590992645,
-      "learning_rate": 2.2711774490274766e-08,
-      "logits/chosen": -4.531357765197754,
-      "logits/rejected": -4.778365135192871,
-      "logps/chosen": -244.4646453857422,
-      "logps/rejected": -277.8075866699219,
-      "loss": 0.148,
+      "grad_norm": 144.59147008918274,
+      "learning_rate": 1.1355887245137383e-08,
+      "logits/chosen": -4.557550430297852,
+      "logits/rejected": -4.780216217041016,
+      "logps/chosen": -248.1402587890625,
+      "logps/rejected": -262.3576354980469,
+      "loss": 0.1884,
       "rewards/accuracies": 0.96875,
-      "rewards/chosen": 2.004220485687256,
-      "rewards/margins": 5.491039276123047,
-      "rewards/rejected": -3.48681902885437,
+      "rewards/chosen": 1.636661171913147,
+      "rewards/margins": 3.578484296798706,
+      "rewards/rejected": -1.9418232440948486,
       "step": 280
     },
     {
       "epoch": 0.74,
-      "grad_norm": 281.0472172318649,
-      "learning_rate": 1.9076063486687256e-08,
-      "logits/chosen": -4.403564453125,
-      "logits/rejected": -4.759608268737793,
-      "logps/chosen": -251.33419799804688,
-      "logps/rejected": -248.35458374023438,
-      "loss": 0.1633,
+      "grad_norm": 331.8489693457681,
+      "learning_rate": 9.538031743343628e-09,
+      "logits/chosen": -4.426544666290283,
+      "logits/rejected": -4.761611461639404,
+      "logps/chosen": -255.4153289794922,
+      "logps/rejected": -234.84487915039062,
+      "loss": 0.1966,
       "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 2.1274170875549316,
-      "rewards/margins": 5.399934768676758,
-      "rewards/rejected": -3.272517681121826,
+      "rewards/chosen": 1.7193052768707275,
+      "rewards/margins": 3.6408512592315674,
+      "rewards/rejected": -1.9215457439422607,
       "step": 290
     },
     {
       "epoch": 0.77,
-      "grad_norm": 153.20971602377742,
-      "learning_rate": 1.5687918106563324e-08,
-      "logits/chosen": -4.5167622566223145,
-      "logits/rejected": -4.735228061676025,
-      "logps/chosen": -232.205322265625,
-      "logps/rejected": -250.81884765625,
-      "loss": 0.179,
+      "grad_norm": 177.15049850318,
+      "learning_rate": 7.843959053281662e-09,
+      "logits/chosen": -4.541173934936523,
+      "logits/rejected": -4.741909503936768,
+      "logps/chosen": -236.41854858398438,
+      "logps/rejected": -236.483642578125,
+      "loss": 0.2158,
       "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 2.1202826499938965,
-      "rewards/margins": 5.605620861053467,
-      "rewards/rejected": -3.4853386878967285,
+      "rewards/chosen": 1.6989631652832031,
+      "rewards/margins": 3.7507786750793457,
+      "rewards/rejected": -2.0518155097961426,
       "step": 300
     },
     {
       "epoch": 0.77,
-      "eval_logits/chosen": -4.521942615509033,
-      "eval_logits/rejected": -4.748414039611816,
-      "eval_logps/chosen": -397.169189453125,
-      "eval_logps/rejected": -518.8555297851562,
-      "eval_loss": 1.3649907112121582,
-      "eval_rewards/accuracies": 0.3828125,
-      "eval_rewards/chosen": -0.6694751977920532,
-      "eval_rewards/margins": -0.6226927638053894,
-      "eval_rewards/rejected": -0.04678243398666382,
-      "eval_runtime": 97.9822,
-      "eval_samples_per_second": 20.412,
+      "eval_logits/chosen": -4.539734840393066,
+      "eval_logits/rejected": -4.759631633758545,
+      "eval_logps/chosen": -392.2895812988281,
+      "eval_logps/rejected": -515.406494140625,
+      "eval_loss": 1.1359957456588745,
+      "eval_rewards/accuracies": 0.41015625,
+      "eval_rewards/chosen": -0.18151262402534485,
+      "eval_rewards/margins": -0.47963014245033264,
+      "eval_rewards/rejected": 0.2981175184249878,
+      "eval_runtime": 97.905,
+      "eval_samples_per_second": 20.428,
       "eval_steps_per_second": 0.327,
       "step": 300
     },
     {
       "epoch": 0.79,
-      "grad_norm": 149.9412754602278,
-      "learning_rate": 1.257446259144494e-08,
-      "logits/chosen": -4.436882972717285,
-      "logits/rejected": -4.769654750823975,
-      "logps/chosen": -239.6269073486328,
-      "logps/rejected": -259.68951416015625,
-      "loss": 0.1671,
+      "grad_norm": 159.8961908197972,
+      "learning_rate": 6.28723129572247e-09,
+      "logits/chosen": -4.461672782897949,
+      "logits/rejected": -4.776505470275879,
+      "logps/chosen": -244.0063934326172,
+      "logps/rejected": -244.4510955810547,
+      "loss": 0.2028,
       "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 2.1512722969055176,
-      "rewards/margins": 5.90042781829834,
-      "rewards/rejected": -3.7491557598114014,
+      "rewards/chosen": 1.7133222818374634,
+      "rewards/margins": 3.9386374950408936,
+      "rewards/rejected": -2.2253153324127197,
       "step": 310
     },
     {
       "epoch": 0.82,
-      "grad_norm": 162.84992879153398,
-      "learning_rate": 9.760622117187234e-09,
-      "logits/chosen": -4.495532512664795,
-      "logits/rejected": -4.839716911315918,
-      "logps/chosen": -226.40927124023438,
-      "logps/rejected": -241.6173553466797,
-      "loss": 0.1706,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 1.8755052089691162,
-      "rewards/margins": 5.392090797424316,
-      "rewards/rejected": -3.5165855884552,
+      "grad_norm": 185.55678167306448,
+      "learning_rate": 4.880311058593617e-09,
+      "logits/chosen": -4.521292209625244,
+      "logits/rejected": -4.848372936248779,
+      "logps/chosen": -230.04397583007812,
+      "logps/rejected": -226.9331817626953,
+      "loss": 0.2095,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 1.5120347738265991,
+      "rewards/margins": 3.560204267501831,
+      "rewards/rejected": -2.0481698513031006,
       "step": 320
     },
     {
       "epoch": 0.84,
-      "grad_norm": 140.15174845446813,
-      "learning_rate": 7.2689232521989885e-09,
-      "logits/chosen": -4.450541019439697,
-      "logits/rejected": -4.755129814147949,
-      "logps/chosen": -248.2396240234375,
-      "logps/rejected": -268.4003601074219,
-      "loss": 0.159,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 1.9909223318099976,
-      "rewards/margins": 5.5609917640686035,
-      "rewards/rejected": -3.5700690746307373,
+      "grad_norm": 155.26912676521727,
+      "learning_rate": 3.6344616260994942e-09,
+      "logits/chosen": -4.473151683807373,
+      "logits/rejected": -4.766911029815674,
+      "logps/chosen": -251.7646484375,
+      "logps/rejected": -254.7379913330078,
+      "loss": 0.1928,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 1.638421654701233,
+      "rewards/margins": 3.8422539234161377,
+      "rewards/rejected": -2.2038321495056152,
       "step": 330
     },
     {
       "epoch": 0.87,
-      "grad_norm": 126.08085826498804,
-      "learning_rate": 5.119313618049309e-09,
-      "logits/chosen": -4.463766574859619,
-      "logits/rejected": -4.806758403778076,
-      "logps/chosen": -251.0575714111328,
-      "logps/rejected": -241.99609375,
-      "loss": 0.1558,
+      "grad_norm": 151.70242269299357,
+      "learning_rate": 2.5596568090246547e-09,
+      "logits/chosen": -4.4894232749938965,
+      "logits/rejected": -4.816695213317871,
+      "logps/chosen": -255.334716796875,
+      "logps/rejected": -228.4678192138672,
+      "loss": 0.1963,
       "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 2.308844804763794,
-      "rewards/margins": 5.830248832702637,
-      "rewards/rejected": -3.5214037895202637,
+      "rewards/chosen": 1.8811309337615967,
+      "rewards/margins": 4.049709320068359,
+      "rewards/rejected": -2.1685783863067627,
       "step": 340
     },
     {
       "epoch": 0.9,
-      "grad_norm": 122.26451804586765,
-      "learning_rate": 3.3290021961708158e-09,
-      "logits/chosen": -4.487333297729492,
-      "logits/rejected": -4.672482967376709,
-      "logps/chosen": -237.46456909179688,
-      "logps/rejected": -258.9417419433594,
-      "loss": 0.1724,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.8006614446640015,
-      "rewards/margins": 4.9178361892700195,
-      "rewards/rejected": -3.117175340652466,
+      "grad_norm": 150.47860807724385,
+      "learning_rate": 1.6645010980854079e-09,
+      "logits/chosen": -4.505433559417725,
+      "logits/rejected": -4.675290584564209,
+      "logps/chosen": -240.54714965820312,
+      "logps/rejected": -246.03665161132812,
+      "loss": 0.2168,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 1.4924026727676392,
+      "rewards/margins": 3.3190674781799316,
+      "rewards/rejected": -1.8266645669937134,
       "step": 350
     },
     {
       "epoch": 0.92,
-      "grad_norm": 144.944818423744,
-      "learning_rate": 1.9123215591052013e-09,
-      "logits/chosen": -4.482793807983398,
-      "logits/rejected": -4.7083001136779785,
-      "logps/chosen": -243.4811248779297,
-      "logps/rejected": -260.84857177734375,
-      "loss": 0.1742,
+      "grad_norm": 165.46679700251414,
+      "learning_rate": 9.561607795526006e-10,
+      "logits/chosen": -4.49678373336792,
+      "logits/rejected": -4.711674690246582,
+      "logps/chosen": -246.95388793945312,
+      "logps/rejected": -247.2928009033203,
+      "loss": 0.2121,
       "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 1.8153793811798096,
-      "rewards/margins": 5.153982162475586,
-      "rewards/rejected": -3.3386025428771973,
+      "rewards/chosen": 1.468100905418396,
+      "rewards/margins": 3.451129913330078,
+      "rewards/rejected": -1.9830286502838135,
       "step": 360
     },
     {
       "epoch": 0.95,
-      "grad_norm": 132.15693545315082,
-      "learning_rate": 8.806131292167618e-10,
-      "logits/chosen": -4.498129367828369,
-      "logits/rejected": -4.671444892883301,
-      "logps/chosen": -237.79995727539062,
-      "logps/rejected": -269.40106201171875,
-      "loss": 0.1751,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 1.9021259546279907,
-      "rewards/margins": 5.157528877258301,
-      "rewards/rejected": -3.2554030418395996,
+      "grad_norm": 149.25359071163066,
+      "learning_rate": 4.403065646083809e-10,
+      "logits/chosen": -4.518364906311035,
+      "logits/rejected": -4.680220603942871,
+      "logps/chosen": -241.2300262451172,
+      "logps/rejected": -255.6038818359375,
+      "loss": 0.2103,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 1.5591168403625488,
+      "rewards/margins": 3.434800386428833,
+      "rewards/rejected": -1.8756835460662842,
       "step": 370
     },
     {
       "epoch": 0.97,
-      "grad_norm": 150.94805422584076,
-      "learning_rate": 2.4213638345040867e-10,
-      "logits/chosen": -4.600647926330566,
-      "logits/rejected": -4.883923053741455,
-      "logps/chosen": -242.57833862304688,
-      "logps/rejected": -258.09075927734375,
-      "loss": 0.1659,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.9504365921020508,
-      "rewards/margins": 5.70705509185791,
-      "rewards/rejected": -3.7566189765930176,
+      "grad_norm": 160.57572024314433,
+      "learning_rate": 1.2106819172520434e-10,
+      "logits/chosen": -4.618407726287842,
+      "logits/rejected": -4.8883843421936035,
+      "logps/chosen": -246.15853881835938,
+      "logps/rejected": -243.1090087890625,
+      "loss": 0.2034,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 1.5924150943756104,
+      "rewards/margins": 3.8508572578430176,
+      "rewards/rejected": -2.2584421634674072,
       "step": 380
     },
     {
       "epoch": 1.0,
-      "grad_norm": 216.65022888736644,
-      "learning_rate": 2.0027310073833516e-12,
-      "logits/chosen": -4.585878372192383,
-      "logits/rejected": -4.851449966430664,
-      "logps/chosen": -237.5348663330078,
-      "logps/rejected": -255.65542602539062,
-      "loss": 0.1632,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 2.213772773742676,
-      "rewards/margins": 5.727804183959961,
-      "rewards/rejected": -3.514031171798706,
+      "grad_norm": 235.40189038757265,
+      "learning_rate": 1.0013655036916758e-12,
+      "logits/chosen": -4.611303329467773,
+      "logits/rejected": -4.857443809509277,
+      "logps/chosen": -242.1800994873047,
+      "logps/rejected": -241.1270294189453,
+      "loss": 0.199,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 1.7492481470108032,
+      "rewards/margins": 3.8104407787323,
+      "rewards/rejected": -2.0611929893493652,
       "step": 390
     },
     {
       "epoch": 1.0,
       "step": 391,
       "total_flos": 0.0,
-      "train_loss": 0.23296360492401416,
-      "train_runtime": 6150.4195,
-      "train_samples_per_second": 8.13,
+      "train_loss": 0.29024992444935965,
+      "train_runtime": 6148.7126,
+      "train_samples_per_second": 8.132,
       "train_steps_per_second": 0.064
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cf525e556a4e72ad76dc3263558be495a00b73c02de0b6ea713d4bfeb6a07eb0
-size 6456
+oid sha256:2ba26fafd03dc99e7d3db95db6c6e4c6d32ca5324c807136077205b8734c2180
+size 6264