Model save

README.md

@@ -3,15 +3,9 @@ library_name: transformers
 license: apache-2.0
 base_model: mistralai/Mistral-7B-Instruct-v0.2
 tags:
-- alignment-handbook
 - trl
 - simpo
 - generated_from_trainer
-- trl
-- simpo
-- generated_from_trainer
-datasets:
-- princeton-nlp/mistral-instruct-ultrafeedback
 model-index:
 - name: mistral-7b-instruct-v0.2-simpo-full
   results: []
@@ -22,17 +16,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # mistral-7b-instruct-v0.2-simpo-full
 
-This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2) on the princeton-nlp/mistral-instruct-ultrafeedback dataset.
-It achieves the following results on the evaluation set:
-- Loss: 0.7703
-- Rewards/chosen: -3.0312
-- Rewards/rejected: -3.3438
-- Rewards/accuracies: 0.6330
-- Rewards/margins: 0.2988
-- Logps/rejected: -1.3359
-- Logps/chosen: -1.2188
-- Logits/rejected: -2.5156
-- Logits/chosen: -2.5156
+This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2) on an unknown dataset.
 
 ## Model description
 
@@ -51,7 +35,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 1e-06
+- learning_rate: 3e-07
 - train_batch_size: 2
 - eval_batch_size: 4
 - seed: 42

all_results.json

@@ -1,22 +1,9 @@
 {
     "epoch": 0.998660594695955,
-    "eval_logits/chosen": -2.515625,
-    "eval_logits/rejected": -2.515625,
-    "eval_logps/chosen": -1.21875,
-    "eval_logps/rejected": -1.3359375,
-    "eval_loss": 0.7703349590301514,
-    "eval_rewards/accuracies": 0.6329787373542786,
-    "eval_rewards/chosen": -3.03125,
-    "eval_rewards/margins": 0.298828125,
-    "eval_rewards/rejected": -3.34375,
-    "eval_runtime": 63.5185,
-    "eval_samples": 2994,
-    "eval_samples_per_second": 47.136,
-    "eval_steps_per_second": 1.48,
     "total_flos": 0.0,
-    "train_loss": 0.7728924853607309,
-    "train_runtime": 5369.4124,
+    "train_loss": 0.8197165558778166,
+    "train_runtime": 5383.1312,
     "train_samples": 59720,
-    "train_samples_per_second": 11.122,
+    "train_samples_per_second": 11.094,
     "train_steps_per_second": 0.043
 }

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.998660594695955,
     "total_flos": 0.0,
-    "train_loss": 0.7728924853607309,
-    "train_runtime": 5369.4124,
+    "train_loss": 0.8197165558778166,
+    "train_runtime": 5383.1312,
     "train_samples": 59720,
-    "train_samples_per_second": 11.122,
+    "train_samples_per_second": 11.094,
     "train_steps_per_second": 0.043
 }

trainer_state.json

@@ -10,701 +10,701 @@
   "log_history": [
     {
       "epoch": 0.021430484864720063,
-      "grad_norm": 20.883335931995553,
-      "learning_rate": 2.0833333333333333e-07,
+      "grad_norm": 20.796442088035388,
+      "learning_rate": 6.25e-08,
       "logits/chosen": -2.265625,
       "logits/rejected": -2.28125,
       "logps/chosen": -0.59765625,
       "logps/rejected": -0.58984375,
-      "loss": 0.8615,
-      "rewards/accuracies": 0.4937500059604645,
+      "loss": 0.8617,
+      "rewards/accuracies": 0.48750001192092896,
       "rewards/chosen": -1.4921875,
-      "rewards/margins": -0.0164794921875,
+      "rewards/margins": -0.01556396484375,
       "rewards/rejected": -1.4765625,
       "step": 5
     },
     {
       "epoch": 0.042860969729440125,
-      "grad_norm": 19.90222092204048,
-      "learning_rate": 4.1666666666666667e-07,
+      "grad_norm": 19.928934559506764,
+      "learning_rate": 1.25e-07,
       "logits/chosen": -2.3125,
       "logits/rejected": -2.296875,
-      "logps/chosen": -0.59765625,
-      "logps/rejected": -0.609375,
-      "loss": 0.8473,
+      "logps/chosen": -0.6015625,
+      "logps/rejected": -0.61328125,
+      "loss": 0.8487,
       "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": -1.4921875,
-      "rewards/margins": 0.034423828125,
-      "rewards/rejected": -1.5234375,
+      "rewards/chosen": -1.5,
+      "rewards/margins": 0.0341796875,
+      "rewards/rejected": -1.53125,
       "step": 10
     },
     {
       "epoch": 0.06429145459416019,
-      "grad_norm": 14.62873307673371,
-      "learning_rate": 6.249999999999999e-07,
+      "grad_norm": 15.726558554798403,
+      "learning_rate": 1.875e-07,
       "logits/chosen": -2.359375,
       "logits/rejected": -2.34375,
-      "logps/chosen": -0.5078125,
-      "logps/rejected": -0.5390625,
-      "loss": 0.8339,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -1.265625,
-      "rewards/margins": 0.07568359375,
-      "rewards/rejected": -1.34375,
+      "logps/chosen": -0.5234375,
+      "logps/rejected": -0.5546875,
+      "loss": 0.84,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.3046875,
+      "rewards/margins": 0.07763671875,
+      "rewards/rejected": -1.3828125,
       "step": 15
     },
     {
       "epoch": 0.08572193945888025,
-      "grad_norm": 19.932354741180678,
-      "learning_rate": 8.333333333333333e-07,
-      "logits/chosen": -2.3125,
-      "logits/rejected": -2.328125,
-      "logps/chosen": -0.50390625,
-      "logps/rejected": -0.52734375,
-      "loss": 0.839,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -1.2578125,
-      "rewards/margins": 0.053955078125,
-      "rewards/rejected": -1.3125,
+      "grad_norm": 27.238077932917754,
+      "learning_rate": 2.5e-07,
+      "logits/chosen": -2.296875,
+      "logits/rejected": -2.3125,
+      "logps/chosen": -0.5546875,
+      "logps/rejected": -0.5625,
+      "loss": 0.8593,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.390625,
+      "rewards/margins": 0.0166015625,
+      "rewards/rejected": -1.40625,
       "step": 20
     },
     {
       "epoch": 0.10715242432360032,
-      "grad_norm": 14.148190772420111,
-      "learning_rate": 9.999435142363483e-07,
-      "logits/chosen": -2.28125,
-      "logits/rejected": -2.28125,
-      "logps/chosen": -0.494140625,
-      "logps/rejected": -0.498046875,
-      "loss": 0.8281,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -1.234375,
-      "rewards/margins": 0.01104736328125,
-      "rewards/rejected": -1.2421875,
+      "grad_norm": 17.44470846346788,
+      "learning_rate": 2.9998305427090446e-07,
+      "logits/chosen": -2.25,
+      "logits/rejected": -2.25,
+      "logps/chosen": -0.5546875,
+      "logps/rejected": -0.55078125,
+      "loss": 0.8443,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.390625,
+      "rewards/margins": -0.01483154296875,
+      "rewards/rejected": -1.375,
       "step": 25
     },
     {
       "epoch": 0.12858290918832038,
-      "grad_norm": 13.826503591038982,
-      "learning_rate": 9.97967852255038e-07,
-      "logits/chosen": -2.296875,
-      "logits/rejected": -2.3125,
-      "logps/chosen": -0.47265625,
-      "logps/rejected": -0.48046875,
-      "loss": 0.8253,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -1.1796875,
-      "rewards/margins": 0.018798828125,
-      "rewards/rejected": -1.203125,
+      "grad_norm": 16.214353030839522,
+      "learning_rate": 2.9939035567651145e-07,
+      "logits/chosen": -2.25,
+      "logits/rejected": -2.265625,
+      "logps/chosen": -0.51953125,
+      "logps/rejected": -0.5234375,
+      "loss": 0.8395,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.3046875,
+      "rewards/margins": 0.00341796875,
+      "rewards/rejected": -1.3046875,
       "step": 30
     },
     {
       "epoch": 0.15001339405304046,
-      "grad_norm": 14.367978791172906,
-      "learning_rate": 9.931806517013612e-07,
-      "logits/chosen": -2.359375,
-      "logits/rejected": -2.390625,
-      "logps/chosen": -0.462890625,
-      "logps/rejected": -0.462890625,
-      "loss": 0.8095,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -1.15625,
-      "rewards/margins": 0.002044677734375,
-      "rewards/rejected": -1.15625,
+      "grad_norm": 16.013043669294714,
+      "learning_rate": 2.9795419551040836e-07,
+      "logits/chosen": -2.3125,
+      "logits/rejected": -2.328125,
+      "logps/chosen": -0.490234375,
+      "logps/rejected": -0.486328125,
+      "loss": 0.8206,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.2265625,
+      "rewards/margins": -0.0128173828125,
+      "rewards/rejected": -1.21875,
       "step": 35
     },
     {
       "epoch": 0.1714438789177605,
-      "grad_norm": 23.918114882300664,
-      "learning_rate": 9.856089412257604e-07,
-      "logits/chosen": -2.421875,
-      "logits/rejected": -2.40625,
-      "logps/chosen": -0.486328125,
-      "logps/rejected": -0.5234375,
-      "loss": 0.8151,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -1.21875,
-      "rewards/margins": 0.095703125,
-      "rewards/rejected": -1.3125,
+      "grad_norm": 28.754058339786674,
+      "learning_rate": 2.9568268236772813e-07,
+      "logits/chosen": -2.359375,
+      "logits/rejected": -2.328125,
+      "logps/chosen": -0.50390625,
+      "logps/rejected": -0.53125,
+      "loss": 0.8365,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.265625,
+      "rewards/margins": 0.060546875,
+      "rewards/rejected": -1.328125,
       "step": 40
     },
     {
       "epoch": 0.19287436378248057,
-      "grad_norm": 21.40358486661956,
-      "learning_rate": 9.752954708892377e-07,
-      "logits/chosen": -2.34375,
-      "logits/rejected": -2.359375,
-      "logps/chosen": -0.5,
-      "logps/rejected": -0.5234375,
-      "loss": 0.8191,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -1.25,
-      "rewards/margins": 0.054443359375,
-      "rewards/rejected": -1.3046875,
+      "grad_norm": 20.18445351417994,
+      "learning_rate": 2.9258864126677133e-07,
+      "logits/chosen": -2.28125,
+      "logits/rejected": -2.296875,
+      "logps/chosen": -0.5078125,
+      "logps/rejected": -0.51953125,
+      "loss": 0.8341,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.265625,
+      "rewards/margins": 0.0341796875,
+      "rewards/rejected": -1.296875,
       "step": 45
     },
     {
       "epoch": 0.21430484864720065,
-      "grad_norm": 13.919989253092268,
-      "learning_rate": 9.62298470795473e-07,
-      "logits/chosen": -2.34375,
-      "logits/rejected": -2.328125,
-      "logps/chosen": -0.490234375,
-      "logps/rejected": -0.53515625,
-      "loss": 0.801,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -1.2265625,
-      "rewards/margins": 0.11083984375,
-      "rewards/rejected": -1.3359375,
+      "grad_norm": 12.573906954523073,
+      "learning_rate": 2.8868954123864195e-07,
+      "logits/chosen": -2.296875,
+      "logits/rejected": -2.28125,
+      "logps/chosen": -0.4765625,
+      "logps/rejected": -0.5078125,
+      "loss": 0.8167,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.1875,
+      "rewards/margins": 0.080078125,
+      "rewards/rejected": -1.2734375,
       "step": 50
     },
     {
       "epoch": 0.23573533351192072,
-      "grad_norm": 15.390687236299986,
-      "learning_rate": 9.466913223222465e-07,
-      "logits/chosen": -2.28125,
-      "logits/rejected": -2.25,
-      "logps/chosen": -0.51171875,
-      "logps/rejected": -0.51953125,
-      "loss": 0.8121,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -1.28125,
-      "rewards/margins": 0.0235595703125,
-      "rewards/rejected": -1.3046875,
+      "grad_norm": 14.110236274827258,
+      "learning_rate": 2.8400739669667395e-07,
+      "logits/chosen": -2.25,
+      "logits/rejected": -2.21875,
+      "logps/chosen": -0.462890625,
+      "logps/rejected": -0.466796875,
+      "loss": 0.8284,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.15625,
+      "rewards/margins": 0.0125732421875,
+      "rewards/rejected": -1.171875,
       "step": 55
     },
     {
       "epoch": 0.25716581837664076,
-      "grad_norm": 15.533470825408608,
-      "learning_rate": 9.285621438083997e-07,
-      "logits/chosen": -2.34375,
-      "logits/rejected": -2.34375,
-      "logps/chosen": -0.5234375,
-      "logps/rejected": -0.5703125,
-      "loss": 0.8084,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -1.3125,
-      "rewards/margins": 0.12060546875,
-      "rewards/rejected": -1.4296875,
+      "grad_norm": 12.596646572796107,
+      "learning_rate": 2.785686431425199e-07,
+      "logits/chosen": -2.28125,
+      "logits/rejected": -2.28125,
+      "logps/chosen": -0.474609375,
+      "logps/rejected": -0.49609375,
+      "loss": 0.8239,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.1875,
+      "rewards/margins": 0.055419921875,
+      "rewards/rejected": -1.2421875,
       "step": 60
     },
     {
       "epoch": 0.2785963032413608,
-      "grad_norm": 20.15952117472032,
-      "learning_rate": 9.080132930355566e-07,
-      "logits/chosen": -2.328125,
-      "logits/rejected": -2.34375,
-      "logps/chosen": -0.53515625,
-      "logps/rejected": -0.6015625,
-      "loss": 0.7938,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.34375,
-      "rewards/margins": 0.1640625,
-      "rewards/rejected": -1.5078125,
+      "grad_norm": 14.485013434243887,
+      "learning_rate": 2.7240398791066697e-07,
+      "logits/chosen": -2.234375,
+      "logits/rejected": -2.25,
+      "logps/chosen": -0.462890625,
+      "logps/rejected": -0.50390625,
+      "loss": 0.8186,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.15625,
+      "rewards/margins": 0.1015625,
+      "rewards/rejected": -1.2578125,
       "step": 65
     },
     {
       "epoch": 0.3000267881060809,
-      "grad_norm": 20.257235368327233,
-      "learning_rate": 8.851607893136064e-07,
-      "logits/chosen": -2.390625,
-      "logits/rejected": -2.40625,
-      "logps/chosen": -0.55078125,
-      "logps/rejected": -0.625,
-      "loss": 0.7934,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -1.3828125,
-      "rewards/margins": 0.18359375,
-      "rewards/rejected": -1.5625,
+      "grad_norm": 19.32002245091787,
+      "learning_rate": 2.655482367940819e-07,
+      "logits/chosen": -2.234375,
+      "logits/rejected": -2.265625,
+      "logps/chosen": -0.453125,
+      "logps/rejected": -0.49609375,
+      "loss": 0.8104,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.1328125,
+      "rewards/margins": 0.10888671875,
+      "rewards/rejected": -1.2421875,
       "step": 70
     },
     {
       "epoch": 0.32145727297080096,
-      "grad_norm": 20.838334942567865,
-      "learning_rate": 8.601336584328658e-07,
-      "logits/chosen": -2.5625,
-      "logits/rejected": -2.53125,
-      "logps/chosen": -0.578125,
-      "logps/rejected": -0.6328125,
-      "loss": 0.7834,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -1.453125,
-      "rewards/margins": 0.1337890625,
-      "rewards/rejected": -1.5859375,
+      "grad_norm": 16.753935764702977,
+      "learning_rate": 2.5804009752985975e-07,
+      "logits/chosen": -2.359375,
+      "logits/rejected": -2.328125,
+      "logps/chosen": -0.474609375,
+      "logps/rejected": -0.49609375,
+      "loss": 0.8104,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.1875,
+      "rewards/margins": 0.052490234375,
+      "rewards/rejected": -1.2421875,
       "step": 75
     },
     {
       "epoch": 0.342887757835521,
-      "grad_norm": 20.73361765612086,
-      "learning_rate": 8.330732041813366e-07,
-      "logits/chosen": -2.515625,
-      "logits/rejected": -2.515625,
-      "logps/chosen": -0.5859375,
-      "logps/rejected": -0.63671875,
-      "loss": 0.7917,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -1.4609375,
-      "rewards/margins": 0.130859375,
-      "rewards/rejected": -1.59375,
+      "grad_norm": 15.362054280986376,
+      "learning_rate": 2.4992196125440097e-07,
+      "logits/chosen": -2.296875,
+      "logits/rejected": -2.296875,
+      "logps/chosen": -0.451171875,
+      "logps/rejected": -0.4765625,
+      "loss": 0.8219,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.1328125,
+      "rewards/margins": 0.061767578125,
+      "rewards/rejected": -1.1875,
       "step": 80
     },
     {
       "epoch": 0.3643182427002411,
-      "grad_norm": 19.794702155542176,
-      "learning_rate": 8.041322105400921e-07,
-      "logits/chosen": -2.53125,
-      "logits/rejected": -2.546875,
-      "logps/chosen": -0.6171875,
-      "logps/rejected": -0.625,
-      "loss": 0.793,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -1.546875,
-      "rewards/margins": 0.01544189453125,
-      "rewards/rejected": -1.5625,
+      "grad_norm": 15.930888067676664,
+      "learning_rate": 2.4123966316202767e-07,
+      "logits/chosen": -2.34375,
+      "logits/rejected": -2.359375,
+      "logps/chosen": -0.482421875,
+      "logps/rejected": -0.466796875,
+      "loss": 0.8178,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.203125,
+      "rewards/margins": -0.036376953125,
+      "rewards/rejected": -1.171875,
       "step": 85
     },
     {
       "epoch": 0.38574872756496115,
-      "grad_norm": 19.137565727745777,
-      "learning_rate": 7.734740790612136e-07,
-      "logits/chosen": -2.5,
-      "logits/rejected": -2.5,
-      "logps/chosen": -0.58203125,
-      "logps/rejected": -0.6328125,
-      "loss": 0.7773,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -1.453125,
-      "rewards/margins": 0.1240234375,
-      "rewards/rejected": -1.578125,
+      "grad_norm": 16.334947210441346,
+      "learning_rate": 2.3204222371836406e-07,
+      "logits/chosen": -2.328125,
+      "logits/rejected": -2.3125,
+      "logps/chosen": -0.435546875,
+      "logps/rejected": -0.462890625,
+      "loss": 0.8079,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.0859375,
+      "rewards/margins": 0.0693359375,
+      "rewards/rejected": -1.15625,
       "step": 90
     },
     {
       "epoch": 0.40717921242968125,
-      "grad_norm": 18.51359957650475,
-      "learning_rate": 7.412719062986631e-07,
-      "logits/chosen": -2.53125,
-      "logits/rejected": -2.5,
-      "logps/chosen": -0.6171875,
-      "logps/rejected": -0.6640625,
-      "loss": 0.7898,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -1.5390625,
-      "rewards/margins": 0.12109375,
-      "rewards/rejected": -1.6640625,
+      "grad_norm": 13.696150578405154,
+      "learning_rate": 2.2238157188959894e-07,
+      "logits/chosen": -2.3125,
+      "logits/rejected": -2.28125,
+      "logps/chosen": -0.431640625,
+      "logps/rejected": -0.447265625,
+      "loss": 0.8187,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.078125,
+      "rewards/margins": 0.040283203125,
+      "rewards/rejected": -1.1171875,
       "step": 95
     },
     {
       "epoch": 0.4286096972944013,
-      "grad_norm": 20.516762220606736,
-      "learning_rate": 7.077075065009433e-07,
-      "logits/chosen": -2.640625,
-      "logits/rejected": -2.625,
-      "logps/chosen": -0.65625,
-      "logps/rejected": -0.7265625,
-      "loss": 0.7906,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -1.6484375,
-      "rewards/margins": 0.1708984375,
-      "rewards/rejected": -1.8125,
+      "grad_norm": 15.348826675752841,
+      "learning_rate": 2.1231225195028297e-07,
+      "logits/chosen": -2.390625,
+      "logits/rejected": -2.375,
+      "logps/chosen": -0.447265625,
+      "logps/rejected": -0.4609375,
+      "loss": 0.8178,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.1171875,
+      "rewards/margins": 0.033447265625,
+      "rewards/rejected": -1.1484375,
       "step": 100
     },
     {
       "epoch": 0.45004018215912134,
-      "grad_norm": 23.39610952973476,
-      "learning_rate": 6.72970385083438e-07,
-      "logits/chosen": -2.65625,
-      "logits/rejected": -2.625,
-      "logps/chosen": -0.671875,
-      "logps/rejected": -0.734375,
-      "loss": 0.7652,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -1.6796875,
-      "rewards/margins": 0.1611328125,
-      "rewards/rejected": -1.8359375,
+      "grad_norm": 17.82603034406786,
+      "learning_rate": 2.0189111552503142e-07,
+      "logits/chosen": -2.390625,
+      "logits/rejected": -2.359375,
+      "logps/chosen": -0.435546875,
+      "logps/rejected": -0.458984375,
+      "loss": 0.8039,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.0859375,
+      "rewards/margins": 0.059814453125,
+      "rewards/rejected": -1.1484375,
       "step": 105
     },
     {
       "epoch": 0.47147066702384144,
-      "grad_norm": 21.60030365380359,
-      "learning_rate": 6.372566686762426e-07,
-      "logits/chosen": -2.671875,
-      "logits/rejected": -2.703125,
-      "logps/chosen": -0.68359375,
-      "logps/rejected": -0.75,
-      "loss": 0.7744,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -1.7109375,
-      "rewards/margins": 0.1708984375,
-      "rewards/rejected": -1.875,
+      "grad_norm": 17.177167162469342,
+      "learning_rate": 1.9117700060287278e-07,
+      "logits/chosen": -2.40625,
+      "logits/rejected": -2.4375,
+      "logps/chosen": -0.462890625,
+      "logps/rejected": -0.474609375,
+      "loss": 0.8207,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.15625,
+      "rewards/margins": 0.0311279296875,
+      "rewards/rejected": -1.1875,
       "step": 110
     },
     {
       "epoch": 0.4929011518885615,
-      "grad_norm": 27.735737738499445,
-      "learning_rate": 6.00767997788451e-07,
-      "logits/chosen": -2.609375,
-      "logits/rejected": -2.625,
-      "logps/chosen": -0.76171875,
-      "logps/rejected": -0.80859375,
-      "loss": 0.7672,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -1.90625,
-      "rewards/margins": 0.11083984375,
-      "rewards/rejected": -2.015625,
+      "grad_norm": 14.511281560002834,
+      "learning_rate": 1.802303993365353e-07,
+      "logits/chosen": -2.390625,
+      "logits/rejected": -2.40625,
+      "logps/chosen": -0.47265625,
+      "logps/rejected": -0.466796875,
+      "loss": 0.8176,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -1.1796875,
+      "rewards/margins": -0.01416015625,
+      "rewards/rejected": -1.1640625,
       "step": 115
     },
     {
       "epoch": 0.5143316367532815,
-      "grad_norm": 24.358513667557904,
-      "learning_rate": 5.637103883409525e-07,
-      "logits/chosen": -2.59375,
-      "logits/rejected": -2.59375,
-      "logps/chosen": -0.81640625,
-      "logps/rejected": -0.87890625,
-      "loss": 0.7625,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -2.03125,
-      "rewards/margins": 0.1640625,
-      "rewards/rejected": -2.203125,
+      "grad_norm": 16.683865452732594,
+      "learning_rate": 1.6911311650228573e-07,
+      "logits/chosen": -2.390625,
+      "logits/rejected": -2.40625,
+      "logps/chosen": -0.47265625,
+      "logps/rejected": -0.46875,
+      "loss": 0.8171,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.1796875,
+      "rewards/margins": -0.00848388671875,
+      "rewards/rejected": -1.171875,
       "step": 120
     },
     {
       "epoch": 0.5357621216180016,
-      "grad_norm": 25.06386819321774,
-      "learning_rate": 5.262930684955438e-07,
-      "logits/chosen": -2.46875,
-      "logits/rejected": -2.453125,
-      "logps/chosen": -0.90234375,
-      "logps/rejected": -1.0390625,
-      "loss": 0.7596,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -2.25,
-      "rewards/margins": 0.333984375,
-      "rewards/rejected": -2.59375,
+      "grad_norm": 15.929826049707975,
+      "learning_rate": 1.5788792054866312e-07,
+      "logits/chosen": -2.3125,
+      "logits/rejected": -2.296875,
+      "logps/chosen": -0.46484375,
+      "logps/rejected": -0.4765625,
+      "loss": 0.8197,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.15625,
+      "rewards/margins": 0.033203125,
+      "rewards/rejected": -1.1953125,
       "step": 125
     },
     {
       "epoch": 0.5571926064827216,
-      "grad_norm": 30.74239972927362,
-      "learning_rate": 4.88727297347654e-07,
-      "logits/chosen": -2.5625,
-      "logits/rejected": -2.546875,
-      "logps/chosen": -0.859375,
-      "logps/rejected": -0.9375,
-      "loss": 0.7626,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -2.140625,
-      "rewards/margins": 0.1953125,
-      "rewards/rejected": -2.34375,
+      "grad_norm": 17.045138294854596,
+      "learning_rate": 1.466181892042962e-07,
+      "logits/chosen": -2.46875,
+      "logits/rejected": -2.453125,
+      "logps/chosen": -0.421875,
+      "logps/rejected": -0.44140625,
+      "loss": 0.8148,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.0546875,
+      "rewards/margins": 0.048095703125,
+      "rewards/rejected": -1.1015625,
       "step": 130
     },
     {
       "epoch": 0.5786230913474417,
-      "grad_norm": 28.54486073469317,
-      "learning_rate": 4.512251721523659e-07,
-      "logits/chosen": -2.53125,
-      "logits/rejected": -2.546875,
-      "logps/chosen": -0.984375,
-      "logps/rejected": -1.078125,
-      "loss": 0.7557,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -2.46875,
-      "rewards/margins": 0.2333984375,
-      "rewards/rejected": -2.6875,
+      "grad_norm": 17.18768140682746,
+      "learning_rate": 1.3536755164570977e-07,
+      "logits/chosen": -2.4375,
+      "logits/rejected": -2.453125,
+      "logps/chosen": -0.443359375,
+      "logps/rejected": -0.46484375,
+      "loss": 0.8208,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.109375,
+      "rewards/margins": 0.054931640625,
+      "rewards/rejected": -1.1640625,
       "step": 135
     },
     {
       "epoch": 0.6000535762121618,
-      "grad_norm": 34.81427967339666,
-      "learning_rate": 4.139984308181708e-07,
-      "logits/chosen": -2.5,
-      "logits/rejected": -2.5,
-      "logps/chosen": -1.0078125,
-      "logps/rejected": -1.1640625,
-      "loss": 0.7408,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -2.515625,
-      "rewards/margins": 0.3984375,
-      "rewards/rejected": -2.90625,
+      "grad_norm": 15.016518702295423,
+      "learning_rate": 1.2419952924545125e-07,
+      "logits/chosen": -2.453125,
+      "logits/rejected": -2.453125,
+      "logps/chosen": -0.44921875,
+      "logps/rejected": -0.482421875,
+      "loss": 0.803,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.125,
+      "rewards/margins": 0.08642578125,
+      "rewards/rejected": -1.2109375,
       "step": 140
     },
     {
       "epoch": 0.6214840610768818,
-      "grad_norm": 36.00644023249742,
-      "learning_rate": 3.772572564296004e-07,
-      "logits/chosen": -2.5,
-      "logits/rejected": -2.484375,
-      "logps/chosen": -1.0859375,
-      "logps/rejected": -1.2109375,
-      "loss": 0.7467,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -2.703125,
-      "rewards/margins": 0.31640625,
-      "rewards/rejected": -3.015625,
+      "grad_norm": 16.50690025565912,
+      "learning_rate": 1.1317717692888012e-07,
+      "logits/chosen": -2.453125,
+      "logits/rejected": -2.4375,
+      "logps/chosen": -0.462890625,
+      "logps/rejected": -0.474609375,
+      "loss": 0.8098,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.15625,
+      "rewards/margins": 0.02880859375,
+      "rewards/rejected": -1.1875,
       "step": 145
     },
     {
       "epoch": 0.6429145459416019,
-      "grad_norm": 32.41309299717272,
-      "learning_rate": 3.412090905484337e-07,
-      "logits/chosen": -2.453125,
-      "logits/rejected": -2.453125,
-      "logps/chosen": -1.109375,
-      "logps/rejected": -1.3046875,
-      "loss": 0.7513,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -2.78125,
-      "rewards/margins": 0.4765625,
-      "rewards/rejected": -3.25,
+      "grad_norm": 15.476929291026718,
+      "learning_rate": 1.0236272716453012e-07,
+      "logits/chosen": -2.390625,
+      "logits/rejected": -2.390625,
+      "logps/chosen": -0.431640625,
+      "logps/rejected": -0.451171875,
+      "loss": 0.8101,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.078125,
+      "rewards/margins": 0.048583984375,
+      "rewards/rejected": -1.125,
       "step": 150
     },
     {
       "epoch": 0.664345030806322,
-      "grad_norm": 36.106876811155786,
-      "learning_rate": 3.060574619936075e-07,
-      "logits/chosen": -2.5,
-      "logits/rejected": -2.484375,
-      "logps/chosen": -1.1171875,
-      "logps/rejected": -1.2421875,
-      "loss": 0.7541,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -2.796875,
-      "rewards/margins": 0.31640625,
-      "rewards/rejected": -3.109375,
+      "grad_norm": 18.32917177722921,
+      "learning_rate": 9.181723859808225e-08,
+      "logits/chosen": -2.390625,
+      "logits/rejected": -2.390625,
+      "logps/chosen": -0.4609375,
+      "logps/rejected": -0.51171875,
+      "loss": 0.8092,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.15625,
+      "rewards/margins": 0.123046875,
+      "rewards/rejected": -1.2734375,
       "step": 155
     },
     {
       "epoch": 0.685775515671042,
-      "grad_norm": 29.82009232975657,
-      "learning_rate": 2.720008377125682e-07,
-      "logits/chosen": -2.515625,
-      "logits/rejected": -2.53125,
-      "logps/chosen": -1.0859375,
-      "logps/rejected": -1.1796875,
-      "loss": 0.7298,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -2.71875,
-      "rewards/margins": 0.2412109375,
-      "rewards/rejected": -2.953125,
+      "grad_norm": 15.797837592657558,
+      "learning_rate": 8.160025131377045e-08,
+      "logits/chosen": -2.390625,
+      "logits/rejected": -2.40625,
+      "logps/chosen": -0.453125,
+      "logps/rejected": -0.466796875,
+      "loss": 0.8041,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.1328125,
+      "rewards/margins": 0.0341796875,
+      "rewards/rejected": -1.1640625,
       "step": 160
     },
     {
       "epoch": 0.7072060005357621,
-      "grad_norm": 33.878683796382994,
-      "learning_rate": 2.3923150223207173e-07,
-      "logits/chosen": -2.5625,
-      "logits/rejected": -2.578125,
-      "logps/chosen": -1.0625,
-      "logps/rejected": -1.1875,
-      "loss": 0.7428,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -2.640625,
-      "rewards/margins": 0.318359375,
-      "rewards/rejected": -2.96875,
+      "grad_norm": 15.83987492665544,
+      "learning_rate": 7.176945066962152e-08,
+      "logits/chosen": -2.4375,
+      "logits/rejected": -2.453125,
+      "logps/chosen": -0.4375,
+      "logps/rejected": -0.453125,
+      "loss": 0.8179,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.09375,
+      "rewards/margins": 0.04248046875,
+      "rewards/rejected": -1.1328125,
       "step": 165
     },
     {
       "epoch": 0.7286364854004822,
-      "grad_norm": 28.349316658677726,
-      "learning_rate": 2.0793447201508286e-07,
-      "logits/chosen": -2.53125,
-      "logits/rejected": -2.546875,
-      "logps/chosen": -1.140625,
-      "logps/rejected": -1.2734375,
-      "loss": 0.7299,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -2.859375,
-      "rewards/margins": 0.333984375,
-      "rewards/rejected": -3.1875,
+      "grad_norm": 12.716591707890245,
+      "learning_rate": 6.238034160452486e-08,
+      "logits/chosen": -2.390625,
+      "logits/rejected": -2.40625,
+      "logps/chosen": -0.48828125,
+      "logps/rejected": -0.4921875,
+      "loss": 0.809,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.21875,
+      "rewards/margins": 0.01385498046875,
+      "rewards/rejected": -1.234375,
       "step": 170
     },
     {
       "epoch": 0.7500669702652023,
-      "grad_norm": 29.876505430358502,
-      "learning_rate": 1.7828645085333644e-07,
-      "logits/chosen": -2.609375,
-      "logits/rejected": -2.59375,
-      "logps/chosen": -1.1015625,
-      "logps/rejected": -1.25,
-      "loss": 0.7457,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -2.75,
-      "rewards/margins": 0.3671875,
-      "rewards/rejected": -3.125,
+      "grad_norm": 17.320925898231533,
+      "learning_rate": 5.348593525600093e-08,
+      "logits/chosen": -2.484375,
+      "logits/rejected": -2.46875,
+      "logps/chosen": -0.44140625,
+      "logps/rejected": -0.48046875,
+      "loss": 0.8046,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.1015625,
+      "rewards/margins": 0.09814453125,
+      "rewards/rejected": -1.203125,
       "step": 175
     },
     {
       "epoch": 0.7714974551299223,
-      "grad_norm": 27.71938507760261,
-      "learning_rate": 1.5045483219344385e-07,
-      "logits/chosen": -2.546875,
-      "logits/rejected": -2.546875,
-      "logps/chosen": -1.125,
-      "logps/rejected": -1.28125,
-      "loss": 0.7361,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -2.8125,
-      "rewards/margins": 0.388671875,
-      "rewards/rejected": -3.203125,
+      "grad_norm": 14.848170648613623,
+      "learning_rate": 4.513644965803315e-08,
+      "logits/chosen": -2.375,
+      "logits/rejected": -2.375,
+      "logps/chosen": -0.46875,
+      "logps/rejected": -0.4921875,
+      "loss": 0.8063,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.171875,
+      "rewards/margins": 0.057861328125,
+      "rewards/rejected": -1.234375,
       "step": 180
     },
     {
       "epoch": 0.7929279399946424,
-      "grad_norm": 27.85983890657696,
-      "learning_rate": 1.2459675402943288e-07,
-      "logits/chosen": -2.578125,
-      "logits/rejected": -2.5625,
-      "logps/chosen": -1.1328125,
-      "logps/rejected": -1.21875,
-      "loss": 0.7261,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -2.828125,
-      "rewards/margins": 0.21484375,
-      "rewards/rejected": -3.046875,
+      "grad_norm": 14.724586156423719,
+      "learning_rate": 3.737902620882986e-08,
+      "logits/chosen": -2.4375,
+      "logits/rejected": -2.421875,
+      "logps/chosen": -0.447265625,
+      "logps/rejected": -0.45703125,
+      "loss": 0.8113,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.1171875,
+      "rewards/margins": 0.0247802734375,
+      "rewards/rejected": -1.140625,
       "step": 185
     },
     {
       "epoch": 0.8143584248593625,
-      "grad_norm": 37.85316797275618,
-      "learning_rate": 1.0085821169782199e-07,
-      "logits/chosen": -2.578125,
-      "logits/rejected": -2.59375,
-      "logps/chosen": -1.1796875,
-      "logps/rejected": -1.265625,
-      "loss": 0.7512,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -2.953125,
-      "rewards/margins": 0.2255859375,
-      "rewards/rejected": -3.171875,
+      "grad_norm": 19.697387000299695,
+      "learning_rate": 3.0257463509346596e-08,
+      "logits/chosen": -2.4375,
+      "logits/rejected": -2.453125,
+      "logps/chosen": -0.443359375,
+      "logps/rejected": -0.447265625,
+      "loss": 0.8197,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.109375,
+      "rewards/margins": 0.01226806640625,
+      "rewards/rejected": -1.1171875,
       "step": 190
     },
     {
       "epoch": 0.8357889097240825,
-      "grad_norm": 36.34792913013898,
-      "learning_rate": 7.937323358440934e-08,
-      "logits/chosen": -2.5625,
-      "logits/rejected": -2.5625,
-      "logps/chosen": -1.15625,
-      "logps/rejected": -1.265625,
-      "loss": 0.7437,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -2.890625,
-      "rewards/margins": 0.27734375,
-      "rewards/rejected": -3.171875,
+      "grad_norm": 15.448382101784729,
+      "learning_rate": 2.38119700753228e-08,
+      "logits/chosen": -2.421875,
+      "logits/rejected": -2.421875,
+      "logps/chosen": -0.4296875,
+      "logps/rejected": -0.44140625,
+      "loss": 0.8157,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.078125,
+      "rewards/margins": 0.02783203125,
+      "rewards/rejected": -1.1015625,
       "step": 195
     },
     {
       "epoch": 0.8572193945888026,
-      "grad_norm": 38.47006137486786,
-      "learning_rate": 6.026312439675551e-08,
-      "logits/chosen": -2.609375,
-      "logits/rejected": -2.609375,
-      "logps/chosen": -1.1328125,
-      "logps/rejected": -1.3359375,
-      "loss": 0.7203,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -2.828125,
-      "rewards/margins": 0.5078125,
-      "rewards/rejected": -3.34375,
+      "grad_norm": 15.451526845141785,
+      "learning_rate": 1.8078937319026654e-08,
+      "logits/chosen": -2.453125,
+      "logits/rejected": -2.46875,
+      "logps/chosen": -0.43359375,
+      "logps/rejected": -0.474609375,
+      "loss": 0.8013,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.078125,
+      "rewards/margins": 0.1015625,
+      "rewards/rejected": -1.1875,
       "step": 200
     },
     {
       "epoch": 0.8786498794535227,
-      "grad_norm": 32.23204631755238,
-      "learning_rate": 4.3635780274861864e-08,
-      "logits/chosen": -2.59375,
-      "logits/rejected": -2.59375,
-      "logps/chosen": -1.1328125,
-      "logps/rejected": -1.2890625,
-      "loss": 0.7247,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -2.828125,
-      "rewards/margins": 0.40625,
-      "rewards/rejected": -3.234375,
+      "grad_norm": 16.717520641902464,
+      "learning_rate": 1.309073408245856e-08,
+      "logits/chosen": -2.46875,
+      "logits/rejected": -2.46875,
+      "logps/chosen": -0.466796875,
+      "logps/rejected": -0.478515625,
+      "loss": 0.8215,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.1640625,
+      "rewards/margins": 0.0311279296875,
+      "rewards/rejected": -1.1953125,
       "step": 205
     },
     {
       "epoch": 0.9000803643182427,
-      "grad_norm": 34.213864254754874,
-      "learning_rate": 2.958507960694784e-08,
-      "logits/chosen": -2.546875,
-      "logits/rejected": -2.5625,
-      "logps/chosen": -1.2265625,
-      "logps/rejected": -1.453125,
-      "loss": 0.7235,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -3.078125,
-      "rewards/margins": 0.55078125,
-      "rewards/rejected": -3.625,
+      "grad_norm": 15.003508275093822,
+      "learning_rate": 8.875523882084352e-09,
+      "logits/chosen": -2.421875,
+      "logits/rejected": -2.4375,
+      "logps/chosen": -0.451171875,
+      "logps/rejected": -0.48828125,
+      "loss": 0.806,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.125,
+      "rewards/margins": 0.0947265625,
+      "rewards/rejected": -1.21875,
       "step": 210
     },
     {
       "epoch": 0.9215108491829628,
-      "grad_norm": 31.906095133000242,
-      "learning_rate": 1.8190352989793322e-08,
-      "logits/chosen": -2.578125,
-      "logits/rejected": -2.578125,
-      "logps/chosen": -1.1640625,
-      "logps/rejected": -1.296875,
-      "loss": 0.7316,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -2.90625,
-      "rewards/margins": 0.333984375,
-      "rewards/rejected": -3.25,
+      "grad_norm": 17.19996096094844,
+      "learning_rate": 5.457105896937997e-09,
+      "logits/chosen": -2.453125,
+      "logits/rejected": -2.453125,
+      "logps/chosen": -0.435546875,
+      "logps/rejected": -0.458984375,
+      "loss": 0.8106,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.09375,
+      "rewards/margins": 0.055419921875,
+      "rewards/rejected": -1.1484375,
       "step": 215
     },
     {
       "epoch": 0.9429413340476829,
-      "grad_norm": 35.354867146517954,
-      "learning_rate": 9.515935326265378e-09,
-      "logits/chosen": -2.5,
-      "logits/rejected": -2.5,
-      "logps/chosen": -1.25,
-      "logps/rejected": -1.421875,
-      "loss": 0.729,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -3.125,
-      "rewards/margins": 0.42578125,
-      "rewards/rejected": -3.546875,
+      "grad_norm": 15.295942668590707,
+      "learning_rate": 2.8547805978796136e-09,
+      "logits/chosen": -2.359375,
+      "logits/rejected": -2.359375,
+      "logps/chosen": -0.462890625,
+      "logps/rejected": -0.4765625,
+      "loss": 0.8135,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.15625,
+      "rewards/margins": 0.033447265625,
+      "rewards/rejected": -1.1875,
       "step": 220
     },
     {
       "epoch": 0.9643718189124029,
-      "grad_norm": 32.177844585654924,
-      "learning_rate": 3.6108025888958447e-09,
-      "logits/chosen": -2.578125,
-      "logits/rejected": -2.59375,
-      "logps/chosen": -1.140625,
-      "logps/rejected": -1.3203125,
-      "loss": 0.7181,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -2.859375,
-      "rewards/margins": 0.443359375,
-      "rewards/rejected": -3.3125,
+      "grad_norm": 15.258286128142098,
+      "learning_rate": 1.0832407766687535e-09,
+      "logits/chosen": -2.453125,
+      "logits/rejected": -2.46875,
+      "logps/chosen": -0.435546875,
+      "logps/rejected": -0.4609375,
+      "loss": 0.8146,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.09375,
+      "rewards/margins": 0.0634765625,
+      "rewards/rejected": -1.15625,
       "step": 225
     },
     {
       "epoch": 0.985802303777123,
-      "grad_norm": 36.438384324409185,
-      "learning_rate": 5.082953003528456e-10,
-      "logits/chosen": -2.546875,
-      "logits/rejected": -2.546875,
-      "logps/chosen": -1.2109375,
-      "logps/rejected": -1.3828125,
-      "loss": 0.7353,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -3.015625,
-      "rewards/margins": 0.435546875,
-      "rewards/rejected": -3.453125,
+      "grad_norm": 17.342572860172456,
+      "learning_rate": 1.524885901058537e-10,
+      "logits/chosen": -2.421875,
+      "logits/rejected": -2.421875,
+      "logps/chosen": -0.462890625,
+      "logps/rejected": -0.490234375,
+      "loss": 0.8179,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.15625,
+      "rewards/margins": 0.0703125,
+      "rewards/rejected": -1.2265625,
       "step": 230
     },
     {
       "epoch": 0.998660594695955,
       "step": 233,
       "total_flos": 0.0,
-      "train_loss": 0.7728924853607309,
-      "train_runtime": 5369.4124,
-      "train_samples_per_second": 11.122,
+      "train_loss": 0.8197165558778166,
+      "train_runtime": 5383.1312,
+      "train_samples_per_second": 11.094,
       "train_steps_per_second": 0.043
     }
   ],