Model save

README.md

@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/yifanwang/huggingface/runs/mh37c4f6)
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/yifanwang/huggingface/runs/uf1l7s7c)
 
 This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
 

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 3.0,
     "total_flos": 0.0,
-    "train_loss": 0.3019624336686715,
-    "train_runtime": 1725.269,
-    "train_samples": 2000,
-    "train_samples_per_second": 3.478,
-    "train_steps_per_second": 0.11
+    "train_loss": 0.2713436476084879,
+    "train_runtime": 5870.1586,
+    "train_samples": 9995,
+    "train_samples_per_second": 5.108,
+    "train_steps_per_second": 0.16
 }

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 3.0,
     "total_flos": 0.0,
-    "train_loss": 0.3019624336686715,
-    "train_runtime": 1725.269,
-    "train_samples": 2000,
-    "train_samples_per_second": 3.478,
-    "train_steps_per_second": 0.11
+    "train_loss": 0.2713436476084879,
+    "train_runtime": 5870.1586,
+    "train_samples": 9995,
+    "train_samples_per_second": 5.108,
+    "train_steps_per_second": 0.16
 }

trainer_state.json

@@ -3,19 +3,19 @@
   "best_model_checkpoint": null,
   "epoch": 3.0,
   "eval_steps": 500,
-  "global_step": 189,
+  "global_step": 939,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.015873015873015872,
-      "grad_norm": 71.32411896966381,
-      "learning_rate": 2.6315789473684208e-08,
-      "logits/chosen": -3.375,
-      "logits/rejected": -3.6875,
-      "logps/chosen": -61.5,
-      "logps/rejected": -89.5,
+      "epoch": 0.003194888178913738,
+      "grad_norm": 112.59852093560752,
+      "learning_rate": 5.3191489361702125e-09,
+      "logits/chosen": -3.453125,
+      "logits/rejected": -3.4375,
+      "logps/chosen": -241.0,
+      "logps/rejected": -306.0,
       "loss": 0.6914,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -24,335 +24,1460 @@
       "step": 1
     },
     {
-      "epoch": 0.15873015873015872,
-      "grad_norm": 58.40627921944624,
-      "learning_rate": 2.631578947368421e-07,
-      "logits/chosen": -3.5625,
-      "logits/rejected": -3.25,
-      "logps/chosen": -88.0,
-      "logps/rejected": -77.5,
-      "loss": 0.6859,
-      "rewards/accuracies": 0.4027777910232544,
-      "rewards/chosen": -0.033203125,
-      "rewards/margins": 0.01806640625,
-      "rewards/rejected": -0.051025390625,
+      "epoch": 0.03194888178913738,
+      "grad_norm": 98.2234007319421,
+      "learning_rate": 5.3191489361702123e-08,
+      "logits/chosen": -3.46875,
+      "logits/rejected": -3.359375,
+      "logps/chosen": -264.0,
+      "logps/rejected": -290.0,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.1388888955116272,
+      "rewards/chosen": -0.0013885498046875,
+      "rewards/margins": 0.00139617919921875,
+      "rewards/rejected": -0.002777099609375,
       "step": 10
     },
     {
-      "epoch": 0.31746031746031744,
-      "grad_norm": 55.91063294187816,
-      "learning_rate": 4.970588235294118e-07,
-      "logits/chosen": -3.453125,
-      "logits/rejected": -3.515625,
-      "logps/chosen": -75.0,
-      "logps/rejected": -90.0,
-      "loss": 0.8452,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.01055908203125,
-      "rewards/margins": 0.15625,
-      "rewards/rejected": -0.1669921875,
+      "epoch": 0.06389776357827476,
+      "grad_norm": 86.4602762267518,
+      "learning_rate": 1.0638297872340425e-07,
+      "logits/chosen": -3.484375,
+      "logits/rejected": -3.421875,
+      "logps/chosen": -264.0,
+      "logps/rejected": -292.0,
+      "loss": 0.6658,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.03515625,
+      "rewards/margins": 0.09033203125,
+      "rewards/rejected": -0.05517578125,
       "step": 20
     },
     {
-      "epoch": 0.47619047619047616,
-      "grad_norm": 70.43265591663462,
-      "learning_rate": 4.676470588235294e-07,
+      "epoch": 0.09584664536741214,
+      "grad_norm": 94.85823054466248,
+      "learning_rate": 1.5957446808510638e-07,
+      "logits/chosen": -3.46875,
+      "logits/rejected": -3.40625,
+      "logps/chosen": -266.0,
+      "logps/rejected": -288.0,
+      "loss": 0.6385,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.06884765625,
+      "rewards/margins": 0.1474609375,
+      "rewards/rejected": -0.07861328125,
+      "step": 30
+    },
+    {
+      "epoch": 0.12779552715654952,
+      "grad_norm": 100.31095539911314,
+      "learning_rate": 2.127659574468085e-07,
+      "logits/chosen": -3.421875,
+      "logits/rejected": -3.328125,
+      "logps/chosen": -270.0,
+      "logps/rejected": -288.0,
+      "loss": 0.6431,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.10546875,
+      "rewards/margins": 0.275390625,
+      "rewards/rejected": -0.169921875,
+      "step": 40
+    },
+    {
+      "epoch": 0.1597444089456869,
+      "grad_norm": 78.36165692613308,
+      "learning_rate": 2.659574468085106e-07,
+      "logits/chosen": -3.421875,
+      "logits/rejected": -3.390625,
+      "logps/chosen": -264.0,
+      "logps/rejected": -292.0,
+      "loss": 0.6141,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.12255859375,
+      "rewards/margins": 0.396484375,
+      "rewards/rejected": -0.2734375,
+      "step": 50
+    },
+    {
+      "epoch": 0.19169329073482427,
+      "grad_norm": 94.58575900723498,
+      "learning_rate": 3.1914893617021275e-07,
+      "logits/chosen": -3.484375,
+      "logits/rejected": -3.3125,
+      "logps/chosen": -274.0,
+      "logps/rejected": -290.0,
+      "loss": 0.605,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.13671875,
+      "rewards/margins": 0.291015625,
+      "rewards/rejected": -0.42578125,
+      "step": 60
+    },
+    {
+      "epoch": 0.22364217252396165,
+      "grad_norm": 78.25280245432886,
+      "learning_rate": 3.7234042553191484e-07,
+      "logits/chosen": -3.40625,
+      "logits/rejected": -3.328125,
+      "logps/chosen": -268.0,
+      "logps/rejected": -290.0,
+      "loss": 0.6421,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0113525390625,
+      "rewards/margins": 0.2333984375,
+      "rewards/rejected": -0.2216796875,
+      "step": 70
+    },
+    {
+      "epoch": 0.25559105431309903,
+      "grad_norm": 106.17272918226905,
+      "learning_rate": 4.25531914893617e-07,
       "logits/chosen": -3.390625,
-      "logits/rejected": -3.234375,
-      "logps/chosen": -85.5,
-      "logps/rejected": -83.5,
-      "loss": 0.6521,
+      "logits/rejected": -3.296875,
+      "logps/chosen": -272.0,
+      "logps/rejected": -290.0,
+      "loss": 0.6067,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.06494140625,
+      "rewards/margins": 0.421875,
+      "rewards/rejected": -0.357421875,
+      "step": 80
+    },
+    {
+      "epoch": 0.28753993610223644,
+      "grad_norm": 99.99161702083042,
+      "learning_rate": 4.787234042553192e-07,
+      "logits/chosen": -3.453125,
+      "logits/rejected": -3.390625,
+      "logps/chosen": -266.0,
+      "logps/rejected": -292.0,
+      "loss": 0.5363,
       "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.000293731689453125,
-      "rewards/margins": 0.48046875,
-      "rewards/rejected": -0.48046875,
-      "step": 30
+      "rewards/chosen": 0.109375,
+      "rewards/margins": 0.81640625,
+      "rewards/rejected": -0.703125,
+      "step": 90
+    },
+    {
+      "epoch": 0.3194888178913738,
+      "grad_norm": 160.86915301272222,
+      "learning_rate": 4.964497041420119e-07,
+      "logits/chosen": -3.28125,
+      "logits/rejected": -3.203125,
+      "logps/chosen": -276.0,
+      "logps/rejected": -302.0,
+      "loss": 0.5781,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.2373046875,
+      "rewards/margins": 0.83203125,
+      "rewards/rejected": -1.0703125,
+      "step": 100
+    },
+    {
+      "epoch": 0.3514376996805112,
+      "grad_norm": 86.76992795560113,
+      "learning_rate": 4.905325443786982e-07,
+      "logits/chosen": -3.265625,
+      "logits/rejected": -3.34375,
+      "logps/chosen": -282.0,
+      "logps/rejected": -288.0,
+      "loss": 0.5998,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.212890625,
+      "rewards/margins": 0.5,
+      "rewards/rejected": -0.71484375,
+      "step": 110
+    },
+    {
+      "epoch": 0.38338658146964855,
+      "grad_norm": 82.75956530263679,
+      "learning_rate": 4.846153846153846e-07,
+      "logits/chosen": -3.3125,
+      "logits/rejected": -3.265625,
+      "logps/chosen": -262.0,
+      "logps/rejected": -288.0,
+      "loss": 0.5585,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.0235595703125,
+      "rewards/margins": 0.90625,
+      "rewards/rejected": -0.9296875,
+      "step": 120
     },
     {
-      "epoch": 0.6349206349206349,
-      "grad_norm": 65.79372778748639,
-      "learning_rate": 4.38235294117647e-07,
+      "epoch": 0.41533546325878595,
+      "grad_norm": 89.68766146971834,
+      "learning_rate": 4.78698224852071e-07,
+      "logits/chosen": -3.328125,
+      "logits/rejected": -3.140625,
+      "logps/chosen": -262.0,
+      "logps/rejected": -296.0,
+      "loss": 0.625,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.015869140625,
+      "rewards/margins": 0.95703125,
+      "rewards/rejected": -0.97265625,
+      "step": 130
+    },
+    {
+      "epoch": 0.4472843450479233,
+      "grad_norm": 88.73932966641566,
+      "learning_rate": 4.727810650887574e-07,
+      "logits/chosen": -3.3125,
+      "logits/rejected": -3.265625,
+      "logps/chosen": -266.0,
+      "logps/rejected": -296.0,
+      "loss": 0.5814,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.146484375,
+      "rewards/margins": 0.8671875,
+      "rewards/rejected": -0.72265625,
+      "step": 140
+    },
+    {
+      "epoch": 0.4792332268370607,
+      "grad_norm": 80.22077137308958,
+      "learning_rate": 4.668639053254438e-07,
+      "logits/chosen": -3.40625,
+      "logits/rejected": -3.3125,
+      "logps/chosen": -264.0,
+      "logps/rejected": -292.0,
+      "loss": 0.6617,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.11865234375,
+      "rewards/margins": 0.78125,
+      "rewards/rejected": -0.6640625,
+      "step": 150
+    },
+    {
+      "epoch": 0.5111821086261981,
+      "grad_norm": 69.46647513532135,
+      "learning_rate": 4.6094674556213014e-07,
+      "logits/chosen": -3.453125,
+      "logits/rejected": -3.3125,
+      "logps/chosen": -260.0,
+      "logps/rejected": -304.0,
+      "loss": 0.6105,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.1748046875,
+      "rewards/margins": 1.0546875,
+      "rewards/rejected": -0.87890625,
+      "step": 160
+    },
+    {
+      "epoch": 0.5431309904153354,
+      "grad_norm": 65.64663477268456,
+      "learning_rate": 4.5502958579881655e-07,
       "logits/chosen": -3.390625,
-      "logits/rejected": -3.1875,
-      "logps/chosen": -72.5,
-      "logps/rejected": -113.5,
-      "loss": 0.5335,
+      "logits/rejected": -3.40625,
+      "logps/chosen": -266.0,
+      "logps/rejected": -304.0,
+      "loss": 0.5167,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.05126953125,
+      "rewards/margins": 1.015625,
+      "rewards/rejected": -0.96875,
+      "step": 170
+    },
+    {
+      "epoch": 0.5750798722044729,
+      "grad_norm": 83.30187342990341,
+      "learning_rate": 4.491124260355029e-07,
+      "logits/chosen": -3.40625,
+      "logits/rejected": -3.328125,
+      "logps/chosen": -266.0,
+      "logps/rejected": -300.0,
+      "loss": 0.6977,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.095703125,
+      "rewards/margins": 0.80859375,
+      "rewards/rejected": -0.7109375,
+      "step": 180
+    },
+    {
+      "epoch": 0.6070287539936102,
+      "grad_norm": 85.93385883193163,
+      "learning_rate": 4.4319526627218936e-07,
+      "logits/chosen": -3.390625,
+      "logits/rejected": -3.359375,
+      "logps/chosen": -268.0,
+      "logps/rejected": -296.0,
+      "loss": 0.6405,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.014892578125,
+      "rewards/margins": 0.734375,
+      "rewards/rejected": -0.71875,
+      "step": 190
+    },
+    {
+      "epoch": 0.6389776357827476,
+      "grad_norm": 65.79581249570145,
+      "learning_rate": 4.372781065088757e-07,
+      "logits/chosen": -3.421875,
+      "logits/rejected": -3.34375,
+      "logps/chosen": -276.0,
+      "logps/rejected": -302.0,
+      "loss": 0.5838,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1455078125,
+      "rewards/margins": 0.86328125,
+      "rewards/rejected": -1.0078125,
+      "step": 200
+    },
+    {
+      "epoch": 0.670926517571885,
+      "grad_norm": 73.87414280006479,
+      "learning_rate": 4.313609467455621e-07,
+      "logits/chosen": -3.40625,
+      "logits/rejected": -3.296875,
+      "logps/chosen": -266.0,
+      "logps/rejected": -300.0,
+      "loss": 0.6263,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.0262451171875,
+      "rewards/margins": 1.1640625,
+      "rewards/rejected": -1.1328125,
+      "step": 210
+    },
+    {
+      "epoch": 0.7028753993610224,
+      "grad_norm": 66.74225798577231,
+      "learning_rate": 4.2544378698224847e-07,
+      "logits/chosen": -3.421875,
+      "logits/rejected": -3.296875,
+      "logps/chosen": -276.0,
+      "logps/rejected": -304.0,
+      "loss": 0.612,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.57421875,
+      "rewards/margins": 0.51953125,
+      "rewards/rejected": -1.09375,
+      "step": 220
+    },
+    {
+      "epoch": 0.7348242811501597,
+      "grad_norm": 74.66898120877077,
+      "learning_rate": 4.195266272189349e-07,
+      "logits/chosen": -3.375,
+      "logits/rejected": -3.3125,
+      "logps/chosen": -268.0,
+      "logps/rejected": -308.0,
+      "loss": 0.5992,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.062255859375,
-      "rewards/margins": 1.703125,
-      "rewards/rejected": -1.6328125,
-      "step": 40
+      "rewards/chosen": -0.2333984375,
+      "rewards/margins": 1.0078125,
+      "rewards/rejected": -1.2421875,
+      "step": 230
+    },
+    {
+      "epoch": 0.7667731629392971,
+      "grad_norm": 67.29982785595878,
+      "learning_rate": 4.1360946745562133e-07,
+      "logits/chosen": -3.390625,
+      "logits/rejected": -3.3125,
+      "logps/chosen": -272.0,
+      "logps/rejected": -304.0,
+      "loss": 0.6371,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.228515625,
+      "rewards/margins": 0.7890625,
+      "rewards/rejected": -1.015625,
+      "step": 240
     },
     {
-      "epoch": 0.7936507936507936,
-      "grad_norm": 49.98022579534793,
-      "learning_rate": 4.0882352941176465e-07,
+      "epoch": 0.7987220447284346,
+      "grad_norm": 67.77775124966749,
+      "learning_rate": 4.076923076923077e-07,
+      "logits/chosen": -3.359375,
+      "logits/rejected": -3.3125,
+      "logps/chosen": -272.0,
+      "logps/rejected": -304.0,
+      "loss": 0.5125,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.080078125,
+      "rewards/margins": 1.1796875,
+      "rewards/rejected": -1.1015625,
+      "step": 250
+    },
+    {
+      "epoch": 0.8306709265175719,
+      "grad_norm": 63.62382192709015,
+      "learning_rate": 4.017751479289941e-07,
+      "logits/chosen": -3.40625,
+      "logits/rejected": -3.265625,
+      "logps/chosen": -266.0,
+      "logps/rejected": -304.0,
+      "loss": 0.5929,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.1484375,
+      "rewards/margins": 1.1796875,
+      "rewards/rejected": -1.328125,
+      "step": 260
+    },
+    {
+      "epoch": 0.8626198083067093,
+      "grad_norm": 68.55660834034994,
+      "learning_rate": 3.9585798816568044e-07,
+      "logits/chosen": -3.359375,
+      "logits/rejected": -3.3125,
+      "logps/chosen": -262.0,
+      "logps/rejected": -294.0,
+      "loss": 0.5444,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.265625,
+      "rewards/margins": 1.2734375,
+      "rewards/rejected": -1.0078125,
+      "step": 270
+    },
+    {
+      "epoch": 0.8945686900958466,
+      "grad_norm": 68.28143704514723,
+      "learning_rate": 3.8994082840236685e-07,
+      "logits/chosen": -3.375,
+      "logits/rejected": -3.296875,
+      "logps/chosen": -258.0,
+      "logps/rejected": -294.0,
+      "loss": 0.5463,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.01080322265625,
+      "rewards/margins": 1.2109375,
+      "rewards/rejected": -1.203125,
+      "step": 280
+    },
+    {
+      "epoch": 0.9265175718849841,
+      "grad_norm": 68.03000461758734,
+      "learning_rate": 3.840236686390532e-07,
       "logits/chosen": -3.328125,
+      "logits/rejected": -3.21875,
+      "logps/chosen": -272.0,
+      "logps/rejected": -296.0,
+      "loss": 0.5104,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2080078125,
+      "rewards/margins": 1.1328125,
+      "rewards/rejected": -1.34375,
+      "step": 290
+    },
+    {
+      "epoch": 0.9584664536741214,
+      "grad_norm": 52.521207376602675,
+      "learning_rate": 3.7810650887573966e-07,
+      "logits/chosen": -3.3125,
       "logits/rejected": -3.296875,
-      "logps/chosen": -85.5,
-      "logps/rejected": -97.5,
-      "loss": 0.5936,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.09521484375,
-      "rewards/margins": 0.67578125,
-      "rewards/rejected": -0.76953125,
-      "step": 50
+      "logps/chosen": -266.0,
+      "logps/rejected": -296.0,
+      "loss": 0.5458,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.2734375,
+      "rewards/margins": 1.078125,
+      "rewards/rejected": -1.3515625,
+      "step": 300
     },
     {
-      "epoch": 0.9523809523809523,
-      "grad_norm": 43.186638836771685,
-      "learning_rate": 3.7941176470588235e-07,
-      "logits/chosen": -3.421875,
+      "epoch": 0.9904153354632588,
+      "grad_norm": 70.79142810582293,
+      "learning_rate": 3.72189349112426e-07,
+      "logits/chosen": -3.265625,
       "logits/rejected": -3.21875,
-      "logps/chosen": -85.0,
-      "logps/rejected": -108.0,
-      "loss": 0.5225,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.1064453125,
-      "rewards/margins": 1.609375,
-      "rewards/rejected": -1.7109375,
-      "step": 60
+      "logps/chosen": -278.0,
+      "logps/rejected": -310.0,
+      "loss": 0.6083,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.326171875,
+      "rewards/margins": 0.984375,
+      "rewards/rejected": -1.3125,
+      "step": 310
     },
     {
       "epoch": 1.0,
       "eval_logits/chosen": -3.296875,
-      "eval_logits/rejected": -3.359375,
-      "eval_logps/chosen": -71.5,
-      "eval_logps/rejected": -109.0,
-      "eval_loss": 0.5614062547683716,
-      "eval_rewards/accuracies": 0.5892857313156128,
-      "eval_rewards/chosen": -0.05126953125,
-      "eval_rewards/margins": 1.703125,
-      "eval_rewards/rejected": -1.75,
-      "eval_runtime": 12.5419,
-      "eval_samples_per_second": 15.947,
-      "eval_steps_per_second": 0.558,
-      "step": 63
-    },
-    {
-      "epoch": 1.1111111111111112,
-      "grad_norm": 22.42352075128014,
-      "learning_rate": 3.5e-07,
-      "logits/chosen": -3.40625,
-      "logits/rejected": -3.34375,
-      "logps/chosen": -76.5,
-      "logps/rejected": -97.0,
-      "loss": 0.3137,
+      "eval_logits/rejected": -3.328125,
+      "eval_logps/chosen": -274.0,
+      "eval_logps/rejected": -292.0,
+      "eval_loss": 0.6047656536102295,
+      "eval_rewards/accuracies": 0.6428571343421936,
+      "eval_rewards/chosen": -0.369140625,
+      "eval_rewards/margins": 0.39453125,
+      "eval_rewards/rejected": -0.765625,
+      "eval_runtime": 12.471,
+      "eval_samples_per_second": 16.037,
+      "eval_steps_per_second": 0.561,
+      "step": 313
+    },
+    {
+      "epoch": 1.0223642172523961,
+      "grad_norm": 36.05905954774972,
+      "learning_rate": 3.662721893491124e-07,
+      "logits/chosen": -3.34375,
+      "logits/rejected": -3.296875,
+      "logps/chosen": -262.0,
+      "logps/rejected": -296.0,
+      "loss": 0.2583,
       "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.55078125,
-      "rewards/margins": 1.71875,
-      "rewards/rejected": -1.171875,
-      "step": 70
+      "rewards/chosen": 0.54296875,
+      "rewards/margins": 2.3125,
+      "rewards/rejected": -1.78125,
+      "step": 320
     },
     {
-      "epoch": 1.2698412698412698,
-      "grad_norm": 22.203010949677886,
-      "learning_rate": 3.205882352941177e-07,
-      "logits/chosen": -3.234375,
-      "logits/rejected": -3.140625,
-      "logps/chosen": -62.0,
-      "logps/rejected": -134.0,
-      "loss": 0.2116,
+      "epoch": 1.0543130990415335,
+      "grad_norm": 21.429827842699556,
+      "learning_rate": 3.603550295857988e-07,
+      "logits/chosen": -3.3125,
+      "logits/rejected": -3.3125,
+      "logps/chosen": -268.0,
+      "logps/rejected": -310.0,
+      "loss": 0.1608,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 0.7109375,
+      "rewards/margins": 2.640625,
+      "rewards/rejected": -1.921875,
+      "step": 330
+    },
+    {
+      "epoch": 1.0862619808306708,
+      "grad_norm": 16.0209244650132,
+      "learning_rate": 3.544378698224852e-07,
+      "logits/chosen": -3.359375,
+      "logits/rejected": -3.296875,
+      "logps/chosen": -270.0,
+      "logps/rejected": -312.0,
+      "loss": 0.1589,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.349609375,
+      "rewards/margins": 2.578125,
+      "rewards/rejected": -2.234375,
+      "step": 340
+    },
+    {
+      "epoch": 1.1182108626198084,
+      "grad_norm": 34.62320829485204,
+      "learning_rate": 3.485207100591716e-07,
+      "logits/chosen": -3.375,
+      "logits/rejected": -3.25,
+      "logps/chosen": -258.0,
+      "logps/rejected": -318.0,
+      "loss": 0.1545,
       "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.59375,
-      "rewards/margins": 4.625,
-      "rewards/rejected": -4.03125,
-      "step": 80
+      "rewards/chosen": 0.6015625,
+      "rewards/margins": 3.234375,
+      "rewards/rejected": -2.625,
+      "step": 350
+    },
+    {
+      "epoch": 1.1501597444089458,
+      "grad_norm": 16.69479811098465,
+      "learning_rate": 3.42603550295858e-07,
+      "logits/chosen": -3.390625,
+      "logits/rejected": -3.3125,
+      "logps/chosen": -258.0,
+      "logps/rejected": -312.0,
+      "loss": 0.1368,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.451171875,
+      "rewards/margins": 3.1875,
+      "rewards/rejected": -2.734375,
+      "step": 360
+    },
+    {
+      "epoch": 1.182108626198083,
+      "grad_norm": 40.088441212920145,
+      "learning_rate": 3.366863905325444e-07,
+      "logits/chosen": -3.359375,
+      "logits/rejected": -3.21875,
+      "logps/chosen": -262.0,
+      "logps/rejected": -316.0,
+      "loss": 0.1428,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 0.357421875,
+      "rewards/margins": 3.140625,
+      "rewards/rejected": -2.78125,
+      "step": 370
     },
     {
-      "epoch": 1.4285714285714286,
-      "grad_norm": 11.358846253272562,
-      "learning_rate": 2.911764705882353e-07,
+      "epoch": 1.2140575079872205,
+      "grad_norm": 31.12940704536787,
+      "learning_rate": 3.3076923076923075e-07,
       "logits/chosen": -3.328125,
-      "logits/rejected": -3.171875,
-      "logps/chosen": -76.0,
-      "logps/rejected": -105.5,
-      "loss": 0.1863,
+      "logits/rejected": -3.203125,
+      "logps/chosen": -260.0,
+      "logps/rejected": -314.0,
+      "loss": 0.1738,
       "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.0234375,
-      "rewards/margins": 3.46875,
-      "rewards/rejected": -2.453125,
-      "step": 90
+      "rewards/chosen": 0.447265625,
+      "rewards/margins": 3.25,
+      "rewards/rejected": -2.796875,
+      "step": 380
     },
     {
-      "epoch": 1.5873015873015874,
-      "grad_norm": 13.818344885174145,
-      "learning_rate": 2.6176470588235295e-07,
-      "logits/chosen": -3.21875,
-      "logits/rejected": -3.28125,
-      "logps/chosen": -70.5,
-      "logps/rejected": -123.0,
-      "loss": 0.1332,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.57421875,
-      "rewards/margins": 3.796875,
-      "rewards/rejected": -3.21875,
-      "step": 100
+      "epoch": 1.2460063897763578,
+      "grad_norm": 26.512380471589744,
+      "learning_rate": 3.2485207100591716e-07,
+      "logits/chosen": -3.28125,
+      "logits/rejected": -3.21875,
+      "logps/chosen": -258.0,
+      "logps/rejected": -310.0,
+      "loss": 0.1643,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 0.65625,
+      "rewards/margins": 3.125,
+      "rewards/rejected": -2.46875,
+      "step": 390
     },
     {
-      "epoch": 1.746031746031746,
-      "grad_norm": 29.078644987873066,
-      "learning_rate": 2.323529411764706e-07,
-      "logits/chosen": -3.234375,
-      "logits/rejected": -3.328125,
-      "logps/chosen": -78.5,
-      "logps/rejected": -114.5,
-      "loss": 0.3217,
+      "epoch": 1.2779552715654952,
+      "grad_norm": 57.22559983737887,
+      "learning_rate": 3.189349112426035e-07,
+      "logits/chosen": -3.3125,
+      "logits/rejected": -3.25,
+      "logps/chosen": -262.0,
+      "logps/rejected": -318.0,
+      "loss": 0.175,
       "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.21875,
+      "rewards/chosen": 0.5625,
+      "rewards/margins": 3.5,
+      "rewards/rejected": -2.9375,
+      "step": 400
+    },
+    {
+      "epoch": 1.3099041533546325,
+      "grad_norm": 45.20149131017795,
+      "learning_rate": 3.130177514792899e-07,
+      "logits/chosen": -3.296875,
+      "logits/rejected": -3.203125,
+      "logps/chosen": -264.0,
+      "logps/rejected": -324.0,
+      "loss": 0.1905,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.484375,
+      "rewards/margins": 3.359375,
+      "rewards/rejected": -2.875,
+      "step": 410
+    },
+    {
+      "epoch": 1.34185303514377,
+      "grad_norm": 58.08393838019337,
+      "learning_rate": 3.071005917159763e-07,
+      "logits/chosen": -3.3125,
+      "logits/rejected": -3.25,
+      "logps/chosen": -262.0,
+      "logps/rejected": -320.0,
+      "loss": 0.211,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 0.23828125,
       "rewards/margins": 2.65625,
       "rewards/rejected": -2.421875,
-      "step": 110
+      "step": 420
     },
     {
-      "epoch": 1.9047619047619047,
-      "grad_norm": 50.92732053835927,
-      "learning_rate": 2.0294117647058823e-07,
-      "logits/chosen": -3.265625,
+      "epoch": 1.3738019169329074,
+      "grad_norm": 43.490436155175196,
+      "learning_rate": 3.011834319526627e-07,
+      "logits/chosen": -3.3125,
+      "logits/rejected": -3.234375,
+      "logps/chosen": -268.0,
+      "logps/rejected": -314.0,
+      "loss": 0.2061,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 0.51953125,
+      "rewards/margins": 3.078125,
+      "rewards/rejected": -2.5625,
+      "step": 430
+    },
+    {
+      "epoch": 1.4057507987220448,
+      "grad_norm": 42.92731422365154,
+      "learning_rate": 2.952662721893491e-07,
+      "logits/chosen": -3.28125,
+      "logits/rejected": -3.1875,
+      "logps/chosen": -260.0,
+      "logps/rejected": -318.0,
+      "loss": 0.1799,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.796875,
+      "rewards/margins": 3.265625,
+      "rewards/rejected": -2.46875,
+      "step": 440
+    },
+    {
+      "epoch": 1.4376996805111821,
+      "grad_norm": 48.43681339755949,
+      "learning_rate": 2.893491124260355e-07,
+      "logits/chosen": -3.359375,
+      "logits/rejected": -3.28125,
+      "logps/chosen": -262.0,
+      "logps/rejected": -312.0,
+      "loss": 0.1597,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.66796875,
+      "rewards/margins": 3.328125,
+      "rewards/rejected": -2.65625,
+      "step": 450
+    },
+    {
+      "epoch": 1.4696485623003195,
+      "grad_norm": 42.804143642804064,
+      "learning_rate": 2.834319526627219e-07,
+      "logits/chosen": -3.28125,
+      "logits/rejected": -3.21875,
+      "logps/chosen": -260.0,
+      "logps/rejected": -316.0,
+      "loss": 0.1899,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 0.56640625,
+      "rewards/margins": 3.4375,
+      "rewards/rejected": -2.875,
+      "step": 460
+    },
+    {
+      "epoch": 1.5015974440894568,
+      "grad_norm": 22.44845309353451,
+      "learning_rate": 2.7751479289940824e-07,
+      "logits/chosen": -3.328125,
+      "logits/rejected": -3.1875,
+      "logps/chosen": -255.0,
+      "logps/rejected": -314.0,
+      "loss": 0.1494,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 0.76171875,
+      "rewards/margins": 3.09375,
+      "rewards/rejected": -2.34375,
+      "step": 470
+    },
+    {
+      "epoch": 1.5335463258785942,
+      "grad_norm": 61.057682334430304,
+      "learning_rate": 2.715976331360947e-07,
+      "logits/chosen": -3.34375,
+      "logits/rejected": -3.3125,
+      "logps/chosen": -256.0,
+      "logps/rejected": -310.0,
+      "loss": 0.1934,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 0.63671875,
+      "rewards/margins": 2.953125,
+      "rewards/rejected": -2.3125,
+      "step": 480
+    },
+    {
+      "epoch": 1.5654952076677318,
+      "grad_norm": 82.10445588433767,
+      "learning_rate": 2.6568047337278106e-07,
+      "logits/chosen": -3.375,
+      "logits/rejected": -3.203125,
+      "logps/chosen": -262.0,
+      "logps/rejected": -332.0,
+      "loss": 0.1835,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.35546875,
+      "rewards/margins": 3.546875,
+      "rewards/rejected": -3.1875,
+      "step": 490
+    },
+    {
+      "epoch": 1.5974440894568689,
+      "grad_norm": 28.388214795776417,
+      "learning_rate": 2.5976331360946746e-07,
+      "logits/chosen": -3.28125,
+      "logits/rejected": -3.203125,
+      "logps/chosen": -256.0,
+      "logps/rejected": -328.0,
+      "loss": 0.157,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.68359375,
+      "rewards/margins": 3.609375,
+      "rewards/rejected": -2.921875,
+      "step": 500
+    },
+    {
+      "epoch": 1.6293929712460065,
+      "grad_norm": 54.4825408072865,
+      "learning_rate": 2.538461538461538e-07,
+      "logits/chosen": -3.28125,
+      "logits/rejected": -3.140625,
+      "logps/chosen": -252.0,
+      "logps/rejected": -316.0,
+      "loss": 0.1892,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 0.8359375,
+      "rewards/margins": 3.328125,
+      "rewards/rejected": -2.484375,
+      "step": 510
+    },
+    {
+      "epoch": 1.6613418530351438,
+      "grad_norm": 35.97307970653537,
+      "learning_rate": 2.479289940828402e-07,
+      "logits/chosen": -3.28125,
       "logits/rejected": -3.25,
-      "logps/chosen": -69.5,
-      "logps/rejected": -125.0,
-      "loss": 0.2144,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 0.30859375,
+      "logps/chosen": -264.0,
+      "logps/rejected": -314.0,
+      "loss": 0.1822,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.75390625,
       "rewards/margins": 3.359375,
-      "rewards/rejected": -3.046875,
-      "step": 120
+      "rewards/rejected": -2.609375,
+      "step": 520
+    },
+    {
+      "epoch": 1.6932907348242812,
+      "grad_norm": 30.66622502292917,
+      "learning_rate": 2.420118343195266e-07,
+      "logits/chosen": -3.28125,
+      "logits/rejected": -3.1875,
+      "logps/chosen": -262.0,
+      "logps/rejected": -324.0,
+      "loss": 0.1306,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.6640625,
+      "rewards/margins": 3.734375,
+      "rewards/rejected": -3.0625,
+      "step": 530
+    },
+    {
+      "epoch": 1.7252396166134185,
+      "grad_norm": 40.98944151272041,
+      "learning_rate": 2.36094674556213e-07,
+      "logits/chosen": -3.359375,
+      "logits/rejected": -3.21875,
+      "logps/chosen": -268.0,
+      "logps/rejected": -318.0,
+      "loss": 0.2227,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.408203125,
+      "rewards/margins": 3.296875,
+      "rewards/rejected": -2.890625,
+      "step": 540
+    },
+    {
+      "epoch": 1.7571884984025559,
+      "grad_norm": 17.791388184091243,
+      "learning_rate": 2.301775147928994e-07,
+      "logits/chosen": -3.328125,
+      "logits/rejected": -3.1875,
+      "logps/chosen": -254.0,
+      "logps/rejected": -328.0,
+      "loss": 0.1407,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.5,
+      "rewards/margins": 3.4375,
+      "rewards/rejected": -2.9375,
+      "step": 550
+    },
+    {
+      "epoch": 1.7891373801916934,
+      "grad_norm": 29.64126287075603,
+      "learning_rate": 2.242603550295858e-07,
+      "logits/chosen": -3.296875,
+      "logits/rejected": -3.1875,
+      "logps/chosen": -262.0,
+      "logps/rejected": -324.0,
+      "loss": 0.1812,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 0.56640625,
+      "rewards/margins": 3.25,
+      "rewards/rejected": -2.671875,
+      "step": 560
+    },
+    {
+      "epoch": 1.8210862619808306,
+      "grad_norm": 29.29526060603243,
+      "learning_rate": 2.1834319526627217e-07,
+      "logits/chosen": -3.296875,
+      "logits/rejected": -3.234375,
+      "logps/chosen": -262.0,
+      "logps/rejected": -310.0,
+      "loss": 0.1546,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.56640625,
+      "rewards/margins": 3.3125,
+      "rewards/rejected": -2.734375,
+      "step": 570
+    },
+    {
+      "epoch": 1.8530351437699681,
+      "grad_norm": 39.821532940530886,
+      "learning_rate": 2.1242603550295858e-07,
+      "logits/chosen": -3.21875,
+      "logits/rejected": -3.21875,
+      "logps/chosen": -266.0,
+      "logps/rejected": -314.0,
+      "loss": 0.168,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.376953125,
+      "rewards/margins": 3.234375,
+      "rewards/rejected": -2.859375,
+      "step": 580
+    },
+    {
+      "epoch": 1.8849840255591053,
+      "grad_norm": 33.00369541627336,
+      "learning_rate": 2.0650887573964496e-07,
+      "logits/chosen": -3.28125,
+      "logits/rejected": -3.234375,
+      "logps/chosen": -258.0,
+      "logps/rejected": -316.0,
+      "loss": 0.1539,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 0.365234375,
+      "rewards/margins": 3.265625,
+      "rewards/rejected": -2.90625,
+      "step": 590
+    },
+    {
+      "epoch": 1.9169329073482428,
+      "grad_norm": 19.25778286261607,
+      "learning_rate": 2.0059171597633133e-07,
+      "logits/chosen": -3.265625,
+      "logits/rejected": -3.21875,
+      "logps/chosen": -262.0,
+      "logps/rejected": -320.0,
+      "loss": 0.2083,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 0.74609375,
+      "rewards/margins": 3.640625,
+      "rewards/rejected": -2.890625,
+      "step": 600
+    },
+    {
+      "epoch": 1.9488817891373802,
+      "grad_norm": 18.303275461840887,
+      "learning_rate": 1.9467455621301774e-07,
+      "logits/chosen": -3.265625,
+      "logits/rejected": -3.21875,
+      "logps/chosen": -264.0,
+      "logps/rejected": -312.0,
+      "loss": 0.1597,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.53125,
+      "rewards/margins": 3.265625,
+      "rewards/rejected": -2.734375,
+      "step": 610
+    },
+    {
+      "epoch": 1.9808306709265175,
+      "grad_norm": 27.512963046909476,
+      "learning_rate": 1.8875739644970412e-07,
+      "logits/chosen": -3.375,
+      "logits/rejected": -3.234375,
+      "logps/chosen": -256.0,
+      "logps/rejected": -310.0,
+      "loss": 0.1655,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 0.734375,
+      "rewards/margins": 3.25,
+      "rewards/rejected": -2.515625,
+      "step": 620
     },
     {
       "epoch": 2.0,
-      "eval_logits/chosen": -3.265625,
-      "eval_logits/rejected": -3.328125,
-      "eval_logps/chosen": -72.5,
-      "eval_logps/rejected": -103.0,
-      "eval_loss": 0.5692187547683716,
-      "eval_rewards/accuracies": 0.6071428656578064,
-      "eval_rewards/chosen": -0.1494140625,
-      "eval_rewards/margins": 1.0,
-      "eval_rewards/rejected": -1.15625,
-      "eval_runtime": 15.8783,
-      "eval_samples_per_second": 12.596,
-      "eval_steps_per_second": 0.441,
-      "step": 126
-    },
-    {
-      "epoch": 2.0634920634920633,
-      "grad_norm": 6.721932196448545,
-      "learning_rate": 1.7352941176470587e-07,
-      "logits/chosen": -3.5,
-      "logits/rejected": -3.140625,
-      "logps/chosen": -77.5,
-      "logps/rejected": -116.0,
-      "loss": 0.1135,
+      "eval_logits/chosen": -3.25,
+      "eval_logits/rejected": -3.28125,
+      "eval_logps/chosen": -276.0,
+      "eval_logps/rejected": -296.0,
+      "eval_loss": 0.6567968726158142,
+      "eval_rewards/accuracies": 0.7142857313156128,
+      "eval_rewards/chosen": -0.58203125,
+      "eval_rewards/margins": 0.6875,
+      "eval_rewards/rejected": -1.2734375,
+      "eval_runtime": 15.5734,
+      "eval_samples_per_second": 12.842,
+      "eval_steps_per_second": 0.449,
+      "step": 626
+    },
+    {
+      "epoch": 2.012779552715655,
+      "grad_norm": 6.827739890857729,
+      "learning_rate": 1.8284023668639053e-07,
+      "logits/chosen": -3.421875,
+      "logits/rejected": -3.28125,
+      "logps/chosen": -255.0,
+      "logps/rejected": -318.0,
+      "loss": 0.1178,
       "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.984375,
+      "rewards/margins": 3.96875,
+      "rewards/rejected": -2.984375,
+      "step": 630
+    },
+    {
+      "epoch": 2.0447284345047922,
+      "grad_norm": 6.879698584336087,
+      "learning_rate": 1.7692307692307693e-07,
+      "logits/chosen": -3.234375,
+      "logits/rejected": -3.140625,
+      "logps/chosen": -264.0,
+      "logps/rejected": -326.0,
+      "loss": 0.0414,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.9609375,
+      "rewards/margins": 4.375,
+      "rewards/rejected": -3.40625,
+      "step": 640
+    },
+    {
+      "epoch": 2.07667731629393,
+      "grad_norm": 8.04324601097532,
+      "learning_rate": 1.710059171597633e-07,
+      "logits/chosen": -3.21875,
+      "logits/rejected": -3.1875,
+      "logps/chosen": -260.0,
+      "logps/rejected": -322.0,
+      "loss": 0.0431,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.490234375,
+      "rewards/margins": 4.3125,
+      "rewards/rejected": -3.828125,
+      "step": 650
+    },
+    {
+      "epoch": 2.108626198083067,
+      "grad_norm": 15.814349381846583,
+      "learning_rate": 1.650887573964497e-07,
+      "logits/chosen": -3.265625,
+      "logits/rejected": -3.234375,
+      "logps/chosen": -254.0,
+      "logps/rejected": -308.0,
+      "loss": 0.044,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 1.0859375,
+      "rewards/margins": 4.09375,
+      "rewards/rejected": -3.015625,
+      "step": 660
+    },
+    {
+      "epoch": 2.1405750798722045,
+      "grad_norm": 14.992951799544796,
+      "learning_rate": 1.591715976331361e-07,
+      "logits/chosen": -3.296875,
+      "logits/rejected": -3.171875,
+      "logps/chosen": -256.0,
+      "logps/rejected": -326.0,
+      "loss": 0.0335,
+      "rewards/accuracies": 1.0,
       "rewards/chosen": 0.953125,
-      "rewards/margins": 3.640625,
-      "rewards/rejected": -2.6875,
-      "step": 130
+      "rewards/margins": 4.625,
+      "rewards/rejected": -3.671875,
+      "step": 670
     },
     {
-      "epoch": 2.2222222222222223,
-      "grad_norm": 7.153434605779754,
-      "learning_rate": 1.441176470588235e-07,
-      "logits/chosen": -3.390625,
+      "epoch": 2.1725239616613417,
+      "grad_norm": 9.820616605992774,
+      "learning_rate": 1.5325443786982248e-07,
+      "logits/chosen": -3.265625,
+      "logits/rejected": -3.078125,
+      "logps/chosen": -260.0,
+      "logps/rejected": -340.0,
+      "loss": 0.0384,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.55859375,
+      "rewards/margins": 4.53125,
+      "rewards/rejected": -3.96875,
+      "step": 680
+    },
+    {
+      "epoch": 2.2044728434504792,
+      "grad_norm": 7.442615753050879,
+      "learning_rate": 1.4733727810650885e-07,
+      "logits/chosen": -3.28125,
+      "logits/rejected": -3.171875,
+      "logps/chosen": -256.0,
+      "logps/rejected": -326.0,
+      "loss": 0.0482,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.703125,
+      "rewards/margins": 4.59375,
+      "rewards/rejected": -3.890625,
+      "step": 690
+    },
+    {
+      "epoch": 2.236421725239617,
+      "grad_norm": 21.80087276569692,
+      "learning_rate": 1.4142011834319526e-07,
+      "logits/chosen": -3.21875,
       "logits/rejected": -3.171875,
-      "logps/chosen": -76.5,
-      "logps/rejected": -118.5,
-      "loss": 0.0585,
+      "logps/chosen": -274.0,
+      "logps/rejected": -310.0,
+      "loss": 0.0385,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.3203125,
-      "rewards/margins": 4.65625,
-      "rewards/rejected": -3.328125,
-      "step": 140
+      "rewards/chosen": 0.921875,
+      "rewards/margins": 4.625,
+      "rewards/rejected": -3.703125,
+      "step": 700
+    },
+    {
+      "epoch": 2.268370607028754,
+      "grad_norm": 10.78604645718658,
+      "learning_rate": 1.3550295857988164e-07,
+      "logits/chosen": -3.1875,
+      "logits/rejected": -3.171875,
+      "logps/chosen": -262.0,
+      "logps/rejected": -334.0,
+      "loss": 0.0356,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.90625,
+      "rewards/margins": 4.90625,
+      "rewards/rejected": -3.984375,
+      "step": 710
+    },
+    {
+      "epoch": 2.3003194888178915,
+      "grad_norm": 150.17573957812536,
+      "learning_rate": 1.2958579881656802e-07,
+      "logits/chosen": -3.15625,
+      "logits/rejected": -3.125,
+      "logps/chosen": -260.0,
+      "logps/rejected": -318.0,
+      "loss": 0.0495,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 1.0234375,
+      "rewards/margins": 4.71875,
+      "rewards/rejected": -3.6875,
+      "step": 720
+    },
+    {
+      "epoch": 2.3322683706070286,
+      "grad_norm": 11.203760912993355,
+      "learning_rate": 1.2366863905325443e-07,
+      "logits/chosen": -3.265625,
+      "logits/rejected": -3.109375,
+      "logps/chosen": -258.0,
+      "logps/rejected": -336.0,
+      "loss": 0.0455,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.57421875,
+      "rewards/margins": 4.90625,
+      "rewards/rejected": -4.34375,
+      "step": 730
+    },
+    {
+      "epoch": 2.364217252396166,
+      "grad_norm": 14.380421624149415,
+      "learning_rate": 1.1775147928994082e-07,
+      "logits/chosen": -3.25,
+      "logits/rejected": -3.21875,
+      "logps/chosen": -266.0,
+      "logps/rejected": -324.0,
+      "loss": 0.045,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.95703125,
+      "rewards/margins": 4.46875,
+      "rewards/rejected": -3.515625,
+      "step": 740
+    },
+    {
+      "epoch": 2.3961661341853033,
+      "grad_norm": 44.53105889705061,
+      "learning_rate": 1.1183431952662721e-07,
+      "logits/chosen": -3.3125,
+      "logits/rejected": -3.109375,
+      "logps/chosen": -250.0,
+      "logps/rejected": -328.0,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 1.0234375,
+      "rewards/margins": 4.6875,
+      "rewards/rejected": -3.671875,
+      "step": 750
+    },
+    {
+      "epoch": 2.428115015974441,
+      "grad_norm": 19.712244388286937,
+      "learning_rate": 1.059171597633136e-07,
+      "logits/chosen": -3.28125,
+      "logits/rejected": -3.15625,
+      "logps/chosen": -258.0,
+      "logps/rejected": -330.0,
+      "loss": 0.0426,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 1.3984375,
+      "rewards/margins": 5.0625,
+      "rewards/rejected": -3.671875,
+      "step": 760
+    },
+    {
+      "epoch": 2.460063897763578,
+      "grad_norm": 7.2319176168084285,
+      "learning_rate": 1e-07,
+      "logits/chosen": -3.234375,
+      "logits/rejected": -3.015625,
+      "logps/chosen": -243.0,
+      "logps/rejected": -326.0,
+      "loss": 0.0304,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.1953125,
+      "rewards/margins": 4.71875,
+      "rewards/rejected": -3.53125,
+      "step": 770
     },
     {
-      "epoch": 2.380952380952381,
-      "grad_norm": 12.932848674845916,
-      "learning_rate": 1.1470588235294116e-07,
+      "epoch": 2.4920127795527156,
+      "grad_norm": 46.435193449762224,
+      "learning_rate": 9.408284023668639e-08,
       "logits/chosen": -3.25,
+      "logits/rejected": -3.15625,
+      "logps/chosen": -255.0,
+      "logps/rejected": -326.0,
+      "loss": 0.0441,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.9453125,
+      "rewards/margins": 4.9375,
+      "rewards/rejected": -3.984375,
+      "step": 780
+    },
+    {
+      "epoch": 2.523961661341853,
+      "grad_norm": 60.95677927620741,
+      "learning_rate": 8.816568047337278e-08,
+      "logits/chosen": -3.28125,
+      "logits/rejected": -3.109375,
+      "logps/chosen": -254.0,
+      "logps/rejected": -334.0,
+      "loss": 0.0471,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.91796875,
+      "rewards/margins": 5.0,
+      "rewards/rejected": -4.09375,
+      "step": 790
+    },
+    {
+      "epoch": 2.5559105431309903,
+      "grad_norm": 7.063701140397027,
+      "learning_rate": 8.224852071005916e-08,
+      "logits/chosen": -3.21875,
+      "logits/rejected": -3.15625,
+      "logps/chosen": -258.0,
+      "logps/rejected": -320.0,
+      "loss": 0.035,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 1.171875,
+      "rewards/margins": 5.03125,
+      "rewards/rejected": -3.859375,
+      "step": 800
+    },
+    {
+      "epoch": 2.587859424920128,
+      "grad_norm": 33.326919159113785,
+      "learning_rate": 7.633136094674555e-08,
+      "logits/chosen": -3.234375,
       "logits/rejected": -3.171875,
-      "logps/chosen": -70.0,
-      "logps/rejected": -115.5,
-      "loss": 0.0819,
+      "logps/chosen": -260.0,
+      "logps/rejected": -322.0,
+      "loss": 0.0395,
       "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.15625,
-      "rewards/margins": 4.09375,
-      "rewards/rejected": -2.9375,
-      "step": 150
+      "rewards/chosen": 0.75,
+      "rewards/margins": 4.8125,
+      "rewards/rejected": -4.0625,
+      "step": 810
     },
     {
-      "epoch": 2.5396825396825395,
-      "grad_norm": 8.625375656141378,
-      "learning_rate": 8.529411764705883e-08,
-      "logits/chosen": -3.140625,
-      "logits/rejected": -3.359375,
-      "logps/chosen": -66.0,
-      "logps/rejected": -114.5,
-      "loss": 0.0774,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 1.03125,
-      "rewards/margins": 4.15625,
-      "rewards/rejected": -3.140625,
-      "step": 160
+      "epoch": 2.619808306709265,
+      "grad_norm": 12.594057219447679,
+      "learning_rate": 7.041420118343195e-08,
+      "logits/chosen": -3.171875,
+      "logits/rejected": -3.0625,
+      "logps/chosen": -264.0,
+      "logps/rejected": -332.0,
+      "loss": 0.0493,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.7421875,
+      "rewards/margins": 5.0,
+      "rewards/rejected": -4.25,
+      "step": 820
+    },
+    {
+      "epoch": 2.6517571884984026,
+      "grad_norm": 33.029055960344195,
+      "learning_rate": 6.449704142011835e-08,
+      "logits/chosen": -3.21875,
+      "logits/rejected": -3.125,
+      "logps/chosen": -254.0,
+      "logps/rejected": -326.0,
+      "loss": 0.0629,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 1.0859375,
+      "rewards/margins": 5.0,
+      "rewards/rejected": -3.921875,
+      "step": 830
     },
     {
-      "epoch": 2.6984126984126986,
-      "grad_norm": 5.566169213675054,
-      "learning_rate": 5.5882352941176474e-08,
+      "epoch": 2.68370607028754,
+      "grad_norm": 30.32400613634307,
+      "learning_rate": 5.857988165680473e-08,
       "logits/chosen": -3.28125,
-      "logits/rejected": -3.078125,
-      "logps/chosen": -71.5,
-      "logps/rejected": -125.0,
-      "loss": 0.0638,
+      "logits/rejected": -3.171875,
+      "logps/chosen": -260.0,
+      "logps/rejected": -326.0,
+      "loss": 0.0373,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.92578125,
+      "rewards/margins": 5.0,
+      "rewards/rejected": -4.0625,
+      "step": 840
+    },
+    {
+      "epoch": 2.7156549520766773,
+      "grad_norm": 22.574215074429016,
+      "learning_rate": 5.266272189349112e-08,
+      "logits/chosen": -3.296875,
+      "logits/rejected": -3.15625,
+      "logps/chosen": -258.0,
+      "logps/rejected": -320.0,
+      "loss": 0.0632,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 1.1015625,
+      "rewards/margins": 4.84375,
+      "rewards/rejected": -3.734375,
+      "step": 850
+    },
+    {
+      "epoch": 2.747603833865815,
+      "grad_norm": 29.217230381621665,
+      "learning_rate": 4.674556213017751e-08,
+      "logits/chosen": -3.234375,
+      "logits/rejected": -3.109375,
+      "logps/chosen": -250.0,
+      "logps/rejected": -322.0,
+      "loss": 0.0454,
       "rewards/accuracies": 1.0,
-      "rewards/chosen": 1.0625,
+      "rewards/chosen": 0.86328125,
       "rewards/margins": 4.96875,
-      "rewards/rejected": -3.890625,
-      "step": 170
+      "rewards/rejected": -4.09375,
+      "step": 860
     },
     {
-      "epoch": 2.857142857142857,
-      "grad_norm": 7.068133297920158,
-      "learning_rate": 2.6470588235294116e-08,
-      "logits/chosen": -3.1875,
-      "logits/rejected": -3.203125,
-      "logps/chosen": -74.5,
-      "logps/rejected": -116.0,
-      "loss": 0.0605,
+      "epoch": 2.779552715654952,
+      "grad_norm": 24.47409244596035,
+      "learning_rate": 4.082840236686391e-08,
+      "logits/chosen": -3.265625,
+      "logits/rejected": -3.015625,
+      "logps/chosen": -246.0,
+      "logps/rejected": -332.0,
+      "loss": 0.0336,
       "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.9765625,
-      "rewards/margins": 4.21875,
-      "rewards/rejected": -3.234375,
-      "step": 180
+      "rewards/chosen": 1.078125,
+      "rewards/margins": 5.3125,
+      "rewards/rejected": -4.21875,
+      "step": 870
+    },
+    {
+      "epoch": 2.8115015974440896,
+      "grad_norm": 23.40322050713798,
+      "learning_rate": 3.4911242603550294e-08,
+      "logits/chosen": -3.25,
+      "logits/rejected": -3.046875,
+      "logps/chosen": -256.0,
+      "logps/rejected": -324.0,
+      "loss": 0.0395,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.9140625,
+      "rewards/margins": 4.71875,
+      "rewards/rejected": -3.796875,
+      "step": 880
+    },
+    {
+      "epoch": 2.8434504792332267,
+      "grad_norm": 15.570214261598547,
+      "learning_rate": 2.8994082840236687e-08,
+      "logits/chosen": -3.171875,
+      "logits/rejected": -3.09375,
+      "logps/chosen": -260.0,
+      "logps/rejected": -326.0,
+      "loss": 0.048,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.83203125,
+      "rewards/margins": 4.875,
+      "rewards/rejected": -4.03125,
+      "step": 890
+    },
+    {
+      "epoch": 2.8753993610223643,
+      "grad_norm": 17.232233333414978,
+      "learning_rate": 2.3076923076923076e-08,
+      "logits/chosen": -3.171875,
+      "logits/rejected": -3.0625,
+      "logps/chosen": -258.0,
+      "logps/rejected": -340.0,
+      "loss": 0.0385,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.9609375,
+      "rewards/margins": 5.40625,
+      "rewards/rejected": -4.46875,
+      "step": 900
+    },
+    {
+      "epoch": 2.9073482428115014,
+      "grad_norm": 14.094432941834466,
+      "learning_rate": 1.7159763313609465e-08,
+      "logits/chosen": -3.25,
+      "logits/rejected": -3.03125,
+      "logps/chosen": -251.0,
+      "logps/rejected": -322.0,
+      "loss": 0.0462,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 1.1015625,
+      "rewards/margins": 5.0625,
+      "rewards/rejected": -3.96875,
+      "step": 910
+    },
+    {
+      "epoch": 2.939297124600639,
+      "grad_norm": 23.651349751352807,
+      "learning_rate": 1.1242603550295858e-08,
+      "logits/chosen": -3.25,
+      "logits/rejected": -3.109375,
+      "logps/chosen": -251.0,
+      "logps/rejected": -328.0,
+      "loss": 0.0386,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.98046875,
+      "rewards/margins": 4.9375,
+      "rewards/rejected": -3.953125,
+      "step": 920
+    },
+    {
+      "epoch": 2.9712460063897765,
+      "grad_norm": 11.670176750652393,
+      "learning_rate": 5.325443786982248e-09,
+      "logits/chosen": -3.234375,
+      "logits/rejected": -3.234375,
+      "logps/chosen": -262.0,
+      "logps/rejected": -318.0,
+      "loss": 0.0413,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.765625,
+      "rewards/margins": 4.875,
+      "rewards/rejected": -4.125,
+      "step": 930
     },
     {
       "epoch": 3.0,
-      "eval_logits/chosen": -3.171875,
-      "eval_logits/rejected": -3.234375,
-      "eval_logps/chosen": -75.0,
-      "eval_logps/rejected": -108.0,
-      "eval_loss": 0.6133202910423279,
-      "eval_rewards/accuracies": 0.625,
-      "eval_rewards/chosen": -0.369140625,
-      "eval_rewards/margins": 1.2421875,
-      "eval_rewards/rejected": -1.609375,
-      "eval_runtime": 15.6062,
-      "eval_samples_per_second": 12.815,
-      "eval_steps_per_second": 0.449,
-      "step": 189
+      "eval_logits/chosen": -3.15625,
+      "eval_logits/rejected": -3.171875,
+      "eval_logps/chosen": -280.0,
+      "eval_logps/rejected": -304.0,
+      "eval_loss": 0.7325195074081421,
+      "eval_rewards/accuracies": 0.6964285969734192,
+      "eval_rewards/chosen": -1.0625,
+      "eval_rewards/margins": 0.875,
+      "eval_rewards/rejected": -1.9375,
+      "eval_runtime": 18.8072,
+      "eval_samples_per_second": 10.634,
+      "eval_steps_per_second": 0.372,
+      "step": 939
     },
     {
       "epoch": 3.0,
-      "step": 189,
+      "step": 939,
       "total_flos": 0.0,
-      "train_loss": 0.3019624336686715,
-      "train_runtime": 1725.269,
-      "train_samples_per_second": 3.478,
-      "train_steps_per_second": 0.11
+      "train_loss": 0.2713436476084879,
+      "train_runtime": 5870.1586,
+      "train_samples_per_second": 5.108,
+      "train_steps_per_second": 0.16
     }
   ],
   "logging_steps": 10,
-  "max_steps": 189,
+  "max_steps": 939,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 3,
   "save_steps": 500,