Model save

README.md

@@ -2,16 +2,11 @@
 license: apache-2.0
 library_name: peft
 tags:
-- alignment-handbook
-- trl
-- dpo
-- generated_from_trainer
 - trl
 - dpo
+- alignment-handbook
 - generated_from_trainer
 base_model: mistralai/Mistral-7B-v0.1
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: zephyr-7b
   results: []
@@ -22,17 +17,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-qlora](https://huggingface.co/alignment-handbook/zephyr-7b-sft-qlora) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6171
-- Rewards/chosen: -0.4648
-- Rewards/rejected: -0.8388
+- Loss: 0.6157
+- Rewards/chosen: -0.4865
+- Rewards/rejected: -0.8500
 - Rewards/accuracies: 0.3711
-- Rewards/margins: 0.3740
-- Logps/rejected: -161.0705
-- Logps/chosen: -110.3948
-- Logits/rejected: 1.0411
-- Logits/chosen: 0.9868
+- Rewards/margins: 0.3636
+- Logps/rejected: -162.1976
+- Logps/chosen: -112.5605
+- Logits/rejected: 1.5453
+- Logits/chosen: 1.4533
 - Use Label: 0.0
 - Pred Label: 0.0
 
@@ -71,10 +66,10 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Use Label | Pred Label |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:---------:|:----------:|
-| 0.6553        | 0.21  | 100  | 0.6557          | -0.1267        | -0.2685          | 0.3633             | 0.1419          | -104.0477      | -76.5787     | -2.0726         | -2.0833       | 0.0       | 0.0        |
-| 0.6446        | 0.42  | 200  | 0.6343          | -0.2873        | -0.5376          | 0.3828             | 0.2503          | -130.9503      | -92.6377     | -0.6864         | -0.7124       | 0.0       | 0.0        |
-| 0.6273        | 0.63  | 300  | 0.6204          | -0.4623        | -0.7994          | 0.3672             | 0.3371          | -157.1332      | -110.1469    | 0.6726          | 0.6280        | 0.0       | 0.0        |
-| 0.6165        | 0.84  | 400  | 0.6182          | -0.4457        | -0.8122          | 0.3672             | 0.3666          | -158.4149      | -108.4784    | 0.9580          | 0.9035        | 0.0       | 0.0        |
+| 0.6551        | 0.21  | 100  | 0.6526          | -0.2364        | -0.3728          | 0.3359             | 0.1364          | -114.4721      | -87.5525     | -1.7460         | -1.7620       | 0.0       | 0.0        |
+| 0.6376        | 0.42  | 200  | 0.6289          | -0.3405        | -0.6072          | 0.3672             | 0.2667          | -137.9142      | -97.9614     | 0.0432          | -0.0238       | 0.0       | 0.0        |
+| 0.6196        | 0.63  | 300  | 0.6189          | -0.3871        | -0.7293          | 0.375              | 0.3422          | -150.1250      | -102.6218    | 1.1831          | 1.0945        | 0.0       | 0.0        |
+| 0.6139        | 0.84  | 400  | 0.6157          | -0.4865        | -0.8500          | 0.3711             | 0.3636          | -162.1976      | -112.5605    | 1.5453          | 1.4533        | 0.0       | 0.0        |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d2e1f0bfb0d758cdedb2cd45f7662f8bc813716b3404510234c43038042c9886
+oid sha256:28859559ead1b89c8a8bb9f09b1b677412e14c5c17bfd4811e08be08821f0e55
 size 671150064

all_results.json

@@ -15,9 +15,9 @@
     "eval_samples_per_second": 15.947,
     "eval_steps_per_second": 0.255,
     "eval_use_label": 0.0,
-    "train_loss": 0.6389844682481554,
-    "train_runtime": 9615.2592,
+    "train_loss": 0.6357159084743924,
+    "train_runtime": 9601.7268,
     "train_samples": 61135,
-    "train_samples_per_second": 6.358,
+    "train_samples_per_second": 6.367,
     "train_steps_per_second": 0.05
 }

runs/Mar20_15-13-30_uclaml04.cs.ucla.edu/events.out.tfevents.1710972859.uclaml04.cs.ucla.edu.3989465.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:888fe7d2c19e3333a49dacd02fa3417075bbcd270b0229a62979815675a95051
-size 45086
+oid sha256:e73d6b9e5ad9da8999138eb89142dd7eb3d77188a523892441b841aab3d42a2f
+size 47034

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6389844682481554,
-    "train_runtime": 9615.2592,
+    "train_loss": 0.6357159084743924,
+    "train_runtime": 9601.7268,
     "train_samples": 61135,
-    "train_samples_per_second": 6.358,
+    "train_samples_per_second": 6.367,
     "train_steps_per_second": 0.05
 }

trainer_state.json

@@ -29,870 +29,870 @@
       "epoch": 0.02,
       "grad_norm": 0.4609375,
       "learning_rate": 1.0416666666666667e-06,
-      "logits/chosen": -2.242556571960449,
-      "logits/rejected": -2.277317762374878,
-      "logps/chosen": -51.96327209472656,
-      "logps/rejected": -64.98894500732422,
+      "logits/chosen": -2.2421462535858154,
+      "logits/rejected": -2.2770614624023438,
+      "logps/chosen": -51.98179626464844,
+      "logps/rejected": -64.9604263305664,
       "loss": 0.6929,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.2361111044883728,
-      "rewards/chosen": 0.002160965697839856,
-      "rewards/margins": 0.0009470728691667318,
-      "rewards/rejected": 0.0012138929450884461,
+      "rewards/accuracies": 0.2222222238779068,
+      "rewards/chosen": 0.001975727966055274,
+      "rewards/margins": 0.00047667179023846984,
+      "rewards/rejected": 0.001499056350439787,
       "step": 10,
       "use_label": 0.0
     },
     {
       "epoch": 0.04,
-      "grad_norm": 0.396484375,
+      "grad_norm": 0.39453125,
       "learning_rate": 2.0833333333333334e-06,
-      "logits/chosen": -2.252474784851074,
-      "logits/rejected": -2.256141185760498,
-      "logps/chosen": -62.50165557861328,
-      "logps/rejected": -72.6328125,
+      "logits/chosen": -2.2520272731781006,
+      "logits/rejected": -2.255510091781616,
+      "logps/chosen": -62.492515563964844,
+      "logps/rejected": -72.63607788085938,
       "loss": 0.6919,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.28125,
-      "rewards/chosen": 0.01592240110039711,
-      "rewards/margins": 0.001004441175609827,
-      "rewards/rejected": 0.014917959459125996,
+      "rewards/accuracies": 0.2874999940395355,
+      "rewards/chosen": 0.01601376011967659,
+      "rewards/margins": 0.0011284304782748222,
+      "rewards/rejected": 0.014885328710079193,
       "step": 20,
       "use_label": 0.0
     },
     {
       "epoch": 0.06,
-      "grad_norm": 0.51171875,
+      "grad_norm": 0.5078125,
       "learning_rate": 3.125e-06,
-      "logits/chosen": -2.342515468597412,
-      "logits/rejected": -2.3552591800689697,
-      "logps/chosen": -79.15455627441406,
-      "logps/rejected": -98.8229751586914,
+      "logits/chosen": -2.3422012329101562,
+      "logits/rejected": -2.3548905849456787,
+      "logps/chosen": -79.14694213867188,
+      "logps/rejected": -98.82722473144531,
       "loss": 0.6898,
       "pred_label": 0.0,
       "rewards/accuracies": 0.2874999940395355,
-      "rewards/chosen": 0.030873581767082214,
-      "rewards/margins": 0.002844910603016615,
-      "rewards/rejected": 0.02802867256104946,
+      "rewards/chosen": 0.030949687585234642,
+      "rewards/margins": 0.0029636542312800884,
+      "rewards/rejected": 0.027986034750938416,
       "step": 30,
       "use_label": 0.0
     },
     {
       "epoch": 0.08,
-      "grad_norm": 0.51953125,
+      "grad_norm": 0.515625,
       "learning_rate": 4.166666666666667e-06,
-      "logits/chosen": -2.323695421218872,
-      "logits/rejected": -2.3019304275512695,
-      "logps/chosen": -82.8508071899414,
-      "logps/rejected": -82.39540100097656,
+      "logits/chosen": -2.322833776473999,
+      "logits/rejected": -2.3010501861572266,
+      "logps/chosen": -82.85880279541016,
+      "logps/rejected": -82.40392303466797,
       "loss": 0.6866,
       "pred_label": 0.0,
       "rewards/accuracies": 0.2874999940395355,
-      "rewards/chosen": 0.033413294702768326,
-      "rewards/margins": 0.011912978254258633,
-      "rewards/rejected": 0.021500317379832268,
+      "rewards/chosen": 0.033333443105220795,
+      "rewards/margins": 0.011918319389224052,
+      "rewards/rejected": 0.021415119990706444,
       "step": 40,
       "use_label": 0.0
     },
     {
       "epoch": 0.1,
-      "grad_norm": 0.6640625,
+      "grad_norm": 0.67578125,
       "learning_rate": 4.999731868769027e-06,
-      "logits/chosen": -2.2408015727996826,
-      "logits/rejected": -2.2638282775878906,
-      "logps/chosen": -67.89698028564453,
-      "logps/rejected": -81.84117126464844,
+      "logits/chosen": -2.241189956665039,
+      "logits/rejected": -2.263849973678589,
+      "logps/chosen": -67.93062591552734,
+      "logps/rejected": -81.85546875,
       "loss": 0.6805,
       "pred_label": 0.0,
       "rewards/accuracies": 0.32499998807907104,
-      "rewards/chosen": 0.009338948875665665,
-      "rewards/margins": 0.030354563146829605,
-      "rewards/rejected": -0.02101561427116394,
+      "rewards/chosen": 0.009002490900456905,
+      "rewards/margins": 0.03016103245317936,
+      "rewards/rejected": -0.02115854248404503,
       "step": 50,
       "use_label": 0.0
     },
     {
       "epoch": 0.13,
-      "grad_norm": 1.53125,
+      "grad_norm": 1.09375,
       "learning_rate": 4.9903533134293035e-06,
-      "logits/chosen": -2.2194154262542725,
-      "logits/rejected": -2.1603574752807617,
-      "logps/chosen": -62.444313049316406,
-      "logps/rejected": -72.18606567382812,
-      "loss": 0.6753,
+      "logits/chosen": -2.218756914138794,
+      "logits/rejected": -2.1594481468200684,
+      "logps/chosen": -62.0407600402832,
+      "logps/rejected": -71.9369888305664,
+      "loss": 0.6748,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.30000001192092896,
-      "rewards/chosen": -0.027180707082152367,
-      "rewards/margins": 0.044989973306655884,
-      "rewards/rejected": -0.072170689702034,
+      "rewards/accuracies": 0.3062500059604645,
+      "rewards/chosen": -0.0231451578438282,
+      "rewards/margins": 0.04653460532426834,
+      "rewards/rejected": -0.06967976689338684,
       "step": 60,
       "use_label": 0.0
     },
     {
       "epoch": 0.15,
-      "grad_norm": 1.84375,
+      "grad_norm": 0.8984375,
       "learning_rate": 4.967625656594782e-06,
-      "logits/chosen": -2.1111249923706055,
-      "logits/rejected": -2.109537124633789,
-      "logps/chosen": -62.041603088378906,
-      "logps/rejected": -75.64030456542969,
-      "loss": 0.666,
+      "logits/chosen": -2.08909273147583,
+      "logits/rejected": -2.088801383972168,
+      "logps/chosen": -68.09326171875,
+      "logps/rejected": -81.9454116821289,
+      "loss": 0.6684,
       "pred_label": 0.0,
       "rewards/accuracies": 0.25,
-      "rewards/chosen": -0.06330498307943344,
-      "rewards/margins": 0.03508424013853073,
-      "rewards/rejected": -0.09838922321796417,
+      "rewards/chosen": -0.12382155656814575,
+      "rewards/margins": 0.03761869668960571,
+      "rewards/rejected": -0.16144026815891266,
       "step": 70,
       "use_label": 0.0
     },
     {
       "epoch": 0.17,
-      "grad_norm": 1.03125,
+      "grad_norm": 1.15625,
       "learning_rate": 4.93167072587771e-06,
-      "logits/chosen": -2.21980881690979,
-      "logits/rejected": -2.1616053581237793,
-      "logps/chosen": -60.844932556152344,
-      "logps/rejected": -74.95368957519531,
-      "loss": 0.66,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.26249998807907104,
-      "rewards/chosen": -0.12314031273126602,
-      "rewards/margins": 0.0946219339966774,
-      "rewards/rejected": -0.21776223182678223,
+      "logits/chosen": -2.20400071144104,
+      "logits/rejected": -2.1452622413635254,
+      "logps/chosen": -55.867881774902344,
+      "logps/rejected": -70.91771697998047,
+      "loss": 0.6588,
+      "pred_label": 0.0,
+      "rewards/accuracies": 0.26875001192092896,
+      "rewards/chosen": -0.0733698159456253,
+      "rewards/margins": 0.10403277724981308,
+      "rewards/rejected": -0.17740261554718018,
       "step": 80,
       "use_label": 0.0
     },
     {
       "epoch": 0.19,
-      "grad_norm": 1.5390625,
+      "grad_norm": 1.0546875,
       "learning_rate": 4.882681251368549e-06,
-      "logits/chosen": -2.109405279159546,
-      "logits/rejected": -2.1181578636169434,
-      "logps/chosen": -77.24811553955078,
-      "logps/rejected": -95.32093811035156,
-      "loss": 0.6621,
+      "logits/chosen": -1.991231918334961,
+      "logits/rejected": -1.9964717626571655,
+      "logps/chosen": -72.28443908691406,
+      "logps/rejected": -90.79218292236328,
+      "loss": 0.6587,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.3062500059604645,
-      "rewards/chosen": -0.1886606067419052,
-      "rewards/margins": 0.07690713554620743,
-      "rewards/rejected": -0.26556771993637085,
+      "rewards/accuracies": 0.30000001192092896,
+      "rewards/chosen": -0.13902384042739868,
+      "rewards/margins": 0.08125626295804977,
+      "rewards/rejected": -0.22028008103370667,
       "step": 90,
       "use_label": 0.0
     },
     {
       "epoch": 0.21,
-      "grad_norm": 1.1640625,
+      "grad_norm": 2.359375,
       "learning_rate": 4.8209198325401815e-06,
-      "logits/chosen": -2.1972146034240723,
-      "logits/rejected": -2.169661283493042,
-      "logps/chosen": -92.16123962402344,
-      "logps/rejected": -84.31734466552734,
-      "loss": 0.6553,
+      "logits/chosen": -1.9231764078140259,
+      "logits/rejected": -1.9043807983398438,
+      "logps/chosen": -103.5636978149414,
+      "logps/rejected": -96.08602142333984,
+      "loss": 0.6551,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.3499999940395355,
-      "rewards/chosen": -0.12131345272064209,
-      "rewards/margins": 0.08319222182035446,
-      "rewards/rejected": -0.20450565218925476,
+      "rewards/accuracies": 0.35624998807907104,
+      "rewards/chosen": -0.2353379726409912,
+      "rewards/margins": 0.08685441315174103,
+      "rewards/rejected": -0.32219237089157104,
       "step": 100,
       "use_label": 0.0
     },
     {
       "epoch": 0.21,
-      "eval_logits/chosen": -2.0832693576812744,
-      "eval_logits/rejected": -2.0725808143615723,
-      "eval_logps/chosen": -76.57865905761719,
-      "eval_logps/rejected": -104.04773712158203,
-      "eval_loss": 0.6557236313819885,
+      "eval_logits/chosen": -1.762041687965393,
+      "eval_logits/rejected": -1.7460479736328125,
+      "eval_logps/chosen": -87.55253601074219,
+      "eval_logps/rejected": -114.47212219238281,
+      "eval_loss": 0.652633547782898,
       "eval_pred_label": 0.0,
-      "eval_rewards/accuracies": 0.36328125,
-      "eval_rewards/chosen": -0.12666408717632294,
-      "eval_rewards/margins": 0.14188387989997864,
-      "eval_rewards/rejected": -0.26854798197746277,
-      "eval_runtime": 125.5075,
-      "eval_samples_per_second": 15.935,
+      "eval_rewards/accuracies": 0.3359375,
+      "eval_rewards/chosen": -0.23640292882919312,
+      "eval_rewards/margins": 0.136388897895813,
+      "eval_rewards/rejected": -0.3727918267250061,
+      "eval_runtime": 125.4491,
+      "eval_samples_per_second": 15.943,
       "eval_steps_per_second": 0.255,
       "eval_use_label": 0.0,
       "step": 100
     },
     {
       "epoch": 0.23,
-      "grad_norm": 1.1484375,
+      "grad_norm": 1.59375,
       "learning_rate": 4.746717530629565e-06,
-      "logits/chosen": -2.125093460083008,
-      "logits/rejected": -2.108320713043213,
-      "logps/chosen": -86.47650146484375,
-      "logps/rejected": -108.77266693115234,
-      "loss": 0.6536,
+      "logits/chosen": -1.7847106456756592,
+      "logits/rejected": -1.7590484619140625,
+      "logps/chosen": -85.73925018310547,
+      "logps/rejected": -106.20509338378906,
+      "loss": 0.6557,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": -0.1537572741508484,
-      "rewards/margins": 0.14806225895881653,
-      "rewards/rejected": -0.3018195331096649,
+      "rewards/accuracies": 0.3499999940395355,
+      "rewards/chosen": -0.14638465642929077,
+      "rewards/margins": 0.12975916266441345,
+      "rewards/rejected": -0.2761438190937042,
       "step": 110,
       "use_label": 0.0
     },
     {
       "epoch": 0.25,
-      "grad_norm": 1.5390625,
+      "grad_norm": 1.828125,
       "learning_rate": 4.660472094042121e-06,
-      "logits/chosen": -1.9497900009155273,
-      "logits/rejected": -1.8884683847427368,
-      "logps/chosen": -95.01170349121094,
-      "logps/rejected": -114.40583801269531,
-      "loss": 0.652,
+      "logits/chosen": -1.1902318000793457,
+      "logits/rejected": -1.0542975664138794,
+      "logps/chosen": -108.4779052734375,
+      "logps/rejected": -127.95109558105469,
+      "loss": 0.6493,
       "pred_label": 0.0,
       "rewards/accuracies": 0.36250001192092896,
-      "rewards/chosen": -0.2506612241268158,
-      "rewards/margins": 0.16420678794384003,
-      "rewards/rejected": -0.414868026971817,
+      "rewards/chosen": -0.38532325625419617,
+      "rewards/margins": 0.1649974286556244,
+      "rewards/rejected": -0.5503206849098206,
       "step": 120,
       "use_label": 0.0
     },
     {
       "epoch": 0.27,
-      "grad_norm": 1.9296875,
+      "grad_norm": 1.9375,
       "learning_rate": 4.5626458262912745e-06,
-      "logits/chosen": -1.7961517572402954,
-      "logits/rejected": -1.7706302404403687,
-      "logps/chosen": -90.99502563476562,
-      "logps/rejected": -112.71142578125,
-      "loss": 0.654,
+      "logits/chosen": -0.818010687828064,
+      "logits/rejected": -0.7847374081611633,
+      "logps/chosen": -109.61775207519531,
+      "logps/rejected": -133.42086791992188,
+      "loss": 0.6524,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.34375,
-      "rewards/chosen": -0.2521664500236511,
-      "rewards/margins": 0.1464831829071045,
-      "rewards/rejected": -0.3986496329307556,
+      "rewards/accuracies": 0.33125001192092896,
+      "rewards/chosen": -0.43839359283447266,
+      "rewards/margins": 0.16735044121742249,
+      "rewards/rejected": -0.6057440638542175,
       "step": 130,
       "use_label": 0.0
     },
     {
       "epoch": 0.29,
-      "grad_norm": 1.9921875,
+      "grad_norm": 1.71875,
       "learning_rate": 4.453763107901676e-06,
-      "logits/chosen": -1.7561969757080078,
-      "logits/rejected": -1.796431541442871,
-      "logps/chosen": -96.94844818115234,
-      "logps/rejected": -107.52276611328125,
-      "loss": 0.6488,
+      "logits/chosen": -0.7395650148391724,
+      "logits/rejected": -0.8444339036941528,
+      "logps/chosen": -116.97528076171875,
+      "logps/rejected": -130.2399139404297,
+      "loss": 0.6381,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.26875001192092896,
-      "rewards/chosen": -0.1620088815689087,
-      "rewards/margins": 0.12216176092624664,
-      "rewards/rejected": -0.28417062759399414,
+      "rewards/accuracies": 0.33125001192092896,
+      "rewards/chosen": -0.3622770607471466,
+      "rewards/margins": 0.1490650475025177,
+      "rewards/rejected": -0.5113420486450195,
       "step": 140,
       "use_label": 0.0
     },
     {
       "epoch": 0.31,
-      "grad_norm": 1.7578125,
+      "grad_norm": 2.125,
       "learning_rate": 4.33440758555951e-06,
-      "logits/chosen": -1.7516326904296875,
-      "logits/rejected": -1.7187411785125732,
-      "logps/chosen": -78.70259857177734,
-      "logps/rejected": -104.34063720703125,
-      "loss": 0.6451,
+      "logits/chosen": -0.6497868299484253,
+      "logits/rejected": -0.6378159523010254,
+      "logps/chosen": -89.60552978515625,
+      "logps/rejected": -115.42192077636719,
+      "loss": 0.6379,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.32499998807907104,
-      "rewards/chosen": -0.13555890321731567,
-      "rewards/margins": 0.22945857048034668,
-      "rewards/rejected": -0.36501747369766235,
+      "rewards/accuracies": 0.3187499940395355,
+      "rewards/chosen": -0.2445882111787796,
+      "rewards/margins": 0.23124215006828308,
+      "rewards/rejected": -0.4758303761482239,
       "step": 150,
       "use_label": 0.0
     },
     {
       "epoch": 0.33,
-      "grad_norm": 2.640625,
+      "grad_norm": 2.15625,
       "learning_rate": 4.205219043576955e-06,
-      "logits/chosen": -1.481575608253479,
-      "logits/rejected": -1.468014121055603,
-      "logps/chosen": -100.68672180175781,
-      "logps/rejected": -127.04164123535156,
-      "loss": 0.6442,
+      "logits/chosen": -0.3159053921699524,
+      "logits/rejected": -0.33064812421798706,
+      "logps/chosen": -99.68696594238281,
+      "logps/rejected": -129.45729064941406,
+      "loss": 0.6317,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.29374998807907104,
-      "rewards/chosen": -0.36356669664382935,
-      "rewards/margins": 0.1327240914106369,
-      "rewards/rejected": -0.49629077315330505,
+      "rewards/accuracies": 0.2874999940395355,
+      "rewards/chosen": -0.35356926918029785,
+      "rewards/margins": 0.16687795519828796,
+      "rewards/rejected": -0.5204472541809082,
       "step": 160,
       "use_label": 0.0
     },
     {
       "epoch": 0.36,
-      "grad_norm": 2.390625,
+      "grad_norm": 2.4375,
       "learning_rate": 4.066889974440757e-06,
-      "logits/chosen": -0.9005377888679504,
-      "logits/rejected": -0.8864371180534363,
-      "logps/chosen": -85.81999206542969,
-      "logps/rejected": -110.4801254272461,
-      "loss": 0.6339,
+      "logits/chosen": 0.14531800150871277,
+      "logits/rejected": 0.18166163563728333,
+      "logps/chosen": -95.45491027832031,
+      "logps/rejected": -125.1463623046875,
+      "loss": 0.6291,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.30000001192092896,
-      "rewards/chosen": -0.3031192421913147,
-      "rewards/margins": 0.1594724804162979,
-      "rewards/rejected": -0.4625917375087738,
+      "rewards/accuracies": 0.29374998807907104,
+      "rewards/chosen": -0.39946848154067993,
+      "rewards/margins": 0.20978550612926483,
+      "rewards/rejected": -0.609254002571106,
       "step": 170,
       "use_label": 0.0
     },
     {
       "epoch": 0.38,
-      "grad_norm": 2.78125,
+      "grad_norm": 2.453125,
       "learning_rate": 3.92016186682789e-06,
-      "logits/chosen": -0.591436505317688,
-      "logits/rejected": -0.5489451885223389,
-      "logps/chosen": -103.7041015625,
-      "logps/rejected": -123.32816314697266,
-      "loss": 0.6554,
+      "logits/chosen": -0.3282355070114136,
+      "logits/rejected": -0.21966704726219177,
+      "logps/chosen": -108.00712585449219,
+      "logps/rejected": -128.67587280273438,
+      "loss": 0.649,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.3375000059604645,
-      "rewards/chosen": -0.40916457772254944,
-      "rewards/margins": 0.2612735629081726,
-      "rewards/rejected": -0.6704381108283997,
+      "rewards/accuracies": 0.35624998807907104,
+      "rewards/chosen": -0.4521949887275696,
+      "rewards/margins": 0.27172034978866577,
+      "rewards/rejected": -0.7239152789115906,
       "step": 180,
       "use_label": 0.0
     },
     {
       "epoch": 0.4,
-      "grad_norm": 2.09375,
+      "grad_norm": 1.84375,
       "learning_rate": 3.7658212309857576e-06,
-      "logits/chosen": -0.801749587059021,
-      "logits/rejected": -0.588916003704071,
-      "logps/chosen": -96.86283874511719,
-      "logps/rejected": -123.17811584472656,
-      "loss": 0.6508,
+      "logits/chosen": -0.889633297920227,
+      "logits/rejected": -0.6851574778556824,
+      "logps/chosen": -91.25111389160156,
+      "logps/rejected": -118.9649887084961,
+      "loss": 0.6461,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.3062500059604645,
-      "rewards/chosen": -0.37751203775405884,
-      "rewards/margins": 0.21026258170604706,
-      "rewards/rejected": -0.5877746343612671,
+      "rewards/accuracies": 0.33125001192092896,
+      "rewards/chosen": -0.32139474153518677,
+      "rewards/margins": 0.22424864768981934,
+      "rewards/rejected": -0.5456433892250061,
       "step": 190,
       "use_label": 0.0
     },
     {
       "epoch": 0.42,
-      "grad_norm": 1.59375,
+      "grad_norm": 1.9453125,
       "learning_rate": 3.604695382782159e-06,
-      "logits/chosen": -1.114527940750122,
-      "logits/rejected": -1.0130901336669922,
-      "logps/chosen": -111.54571533203125,
-      "logps/rejected": -115.97926330566406,
-      "loss": 0.6446,
+      "logits/chosen": -0.8204952478408813,
+      "logits/rejected": -0.7186430096626282,
+      "logps/chosen": -112.41142272949219,
+      "logps/rejected": -120.7835693359375,
+      "loss": 0.6376,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.3187499940395355,
-      "rewards/chosen": -0.2986941933631897,
-      "rewards/margins": 0.1296522319316864,
-      "rewards/rejected": -0.4283464550971985,
+      "rewards/accuracies": 0.3125,
+      "rewards/chosen": -0.30735117197036743,
+      "rewards/margins": 0.169038325548172,
+      "rewards/rejected": -0.47638946771621704,
       "step": 200,
       "use_label": 0.0
     },
     {
       "epoch": 0.42,
-      "eval_logits/chosen": -0.7123901844024658,
-      "eval_logits/rejected": -0.6864092350006104,
-      "eval_logps/chosen": -92.6377182006836,
-      "eval_logps/rejected": -130.9503173828125,
-      "eval_loss": 0.6342783570289612,
+      "eval_logits/chosen": -0.023804781958460808,
+      "eval_logits/rejected": 0.04317883029580116,
+      "eval_logps/chosen": -97.96138000488281,
+      "eval_logps/rejected": -137.9141845703125,
+      "eval_loss": 0.6288520693778992,
       "eval_pred_label": 0.0,
-      "eval_rewards/accuracies": 0.3828125,
-      "eval_rewards/chosen": -0.28725457191467285,
-      "eval_rewards/margins": 0.250319242477417,
-      "eval_rewards/rejected": -0.5375738143920898,
-      "eval_runtime": 125.6586,
-      "eval_samples_per_second": 15.916,
+      "eval_rewards/accuracies": 0.3671875,
+      "eval_rewards/chosen": -0.34049129486083984,
+      "eval_rewards/margins": 0.26672109961509705,
+      "eval_rewards/rejected": -0.6072123646736145,
+      "eval_runtime": 125.433,
+      "eval_samples_per_second": 15.945,
       "eval_steps_per_second": 0.255,
       "eval_use_label": 0.0,
       "step": 200
     },
     {
       "epoch": 0.44,
-      "grad_norm": 2.140625,
+      "grad_norm": 2.265625,
       "learning_rate": 3.437648009023905e-06,
-      "logits/chosen": -0.6364002227783203,
-      "logits/rejected": -0.629191517829895,
-      "logps/chosen": -79.12034606933594,
-      "logps/rejected": -109.35395812988281,
-      "loss": 0.6319,
+      "logits/chosen": -0.05805685371160507,
+      "logits/rejected": -0.06056814268231392,
+      "logps/chosen": -88.78871154785156,
+      "logps/rejected": -124.3318862915039,
+      "loss": 0.6218,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.35624998807907104,
-      "rewards/chosen": -0.23145589232444763,
-      "rewards/margins": 0.2322908192873001,
-      "rewards/rejected": -0.46374672651290894,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.3281395435333252,
+      "rewards/margins": 0.28538644313812256,
+      "rewards/rejected": -0.613525927066803,
       "step": 210,
       "use_label": 0.0
     },
     {
       "epoch": 0.46,
-      "grad_norm": 2.453125,
+      "grad_norm": 2.21875,
       "learning_rate": 3.265574537815398e-06,
-      "logits/chosen": -0.24914255738258362,
-      "logits/rejected": -0.12895795702934265,
-      "logps/chosen": -123.09925842285156,
-      "logps/rejected": -127.96968078613281,
-      "loss": 0.633,
+      "logits/chosen": -0.1400775909423828,
+      "logits/rejected": -0.005620801355689764,
+      "logps/chosen": -133.7158660888672,
+      "logps/rejected": -136.84619140625,
+      "loss": 0.627,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.3187499940395355,
-      "rewards/chosen": -0.43470579385757446,
-      "rewards/margins": 0.1813107430934906,
-      "rewards/rejected": -0.6160165071487427,
+      "rewards/accuracies": 0.30000001192092896,
+      "rewards/chosen": -0.5408719778060913,
+      "rewards/margins": 0.16390959918498993,
+      "rewards/rejected": -0.7047815918922424,
       "step": 220,
       "use_label": 0.0
     },
     {
       "epoch": 0.48,
-      "grad_norm": 2.734375,
+      "grad_norm": 1.8515625,
       "learning_rate": 3.089397338773569e-06,
-      "logits/chosen": 0.08423249423503876,
-      "logits/rejected": 0.1725344955921173,
-      "logps/chosen": -98.91605377197266,
-      "logps/rejected": -125.9875259399414,
-      "loss": 0.6278,
+      "logits/chosen": 0.16266627609729767,
+      "logits/rejected": 0.2626825273036957,
+      "logps/chosen": -93.3644027709961,
+      "logps/rejected": -119.67996978759766,
+      "loss": 0.6261,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.33125001192092896,
-      "rewards/chosen": -0.3448147773742676,
-      "rewards/margins": 0.287472665309906,
-      "rewards/rejected": -0.6322874426841736,
+      "rewards/accuracies": 0.3187499940395355,
+      "rewards/chosen": -0.28929832577705383,
+      "rewards/margins": 0.27991363406181335,
+      "rewards/rejected": -0.5692119598388672,
       "step": 230,
       "use_label": 0.0
     },
     {
       "epoch": 0.5,
-      "grad_norm": 2.015625,
+      "grad_norm": 1.8984375,
       "learning_rate": 2.9100607788275547e-06,
-      "logits/chosen": 0.48232460021972656,
-      "logits/rejected": 0.39376580715179443,
-      "logps/chosen": -108.98759460449219,
-      "logps/rejected": -142.29344177246094,
-      "loss": 0.6294,
+      "logits/chosen": 0.854693591594696,
+      "logits/rejected": 0.7261193990707397,
+      "logps/chosen": -99.00528717041016,
+      "logps/rejected": -135.73580932617188,
+      "loss": 0.6295,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.3812499940395355,
-      "rewards/chosen": -0.39955058693885803,
-      "rewards/margins": 0.28114694356918335,
-      "rewards/rejected": -0.680697500705719,
+      "rewards/accuracies": 0.3687500059604645,
+      "rewards/chosen": -0.2997274696826935,
+      "rewards/margins": 0.3153937757015228,
+      "rewards/rejected": -0.6151211857795715,
       "step": 240,
       "use_label": 0.0
     },
     {
       "epoch": 0.52,
-      "grad_norm": 2.25,
+      "grad_norm": 2.03125,
       "learning_rate": 2.72852616010567e-06,
-      "logits/chosen": 0.35806649923324585,
-      "logits/rejected": 0.41671887040138245,
-      "logps/chosen": -126.65348052978516,
-      "logps/rejected": -151.3179168701172,
-      "loss": 0.6419,
+      "logits/chosen": 0.6816203594207764,
+      "logits/rejected": 0.7033491134643555,
+      "logps/chosen": -119.7255859375,
+      "logps/rejected": -144.8857421875,
+      "loss": 0.6376,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.36250001192092896,
-      "rewards/chosen": -0.5325437784194946,
-      "rewards/margins": 0.28831106424331665,
-      "rewards/rejected": -0.8208548426628113,
+      "rewards/accuracies": 0.3812499940395355,
+      "rewards/chosen": -0.4632648825645447,
+      "rewards/margins": 0.2932681143283844,
+      "rewards/rejected": -0.7565330266952515,
       "step": 250,
       "use_label": 0.0
     },
     {
       "epoch": 0.54,
-      "grad_norm": 2.46875,
+      "grad_norm": 1.8984375,
       "learning_rate": 2.5457665670441937e-06,
-      "logits/chosen": 0.4644729197025299,
-      "logits/rejected": 0.45051756501197815,
-      "logps/chosen": -110.62007904052734,
-      "logps/rejected": -142.76722717285156,
-      "loss": 0.6232,
+      "logits/chosen": 0.5938165187835693,
+      "logits/rejected": 0.5592354536056519,
+      "logps/chosen": -110.32804870605469,
+      "logps/rejected": -146.76275634765625,
+      "loss": 0.6162,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.32499998807907104,
-      "rewards/chosen": -0.4451447129249573,
-      "rewards/margins": 0.2380482256412506,
-      "rewards/rejected": -0.6831929087638855,
+      "rewards/accuracies": 0.34375,
+      "rewards/chosen": -0.44222426414489746,
+      "rewards/margins": 0.2809238135814667,
+      "rewards/rejected": -0.7231480479240417,
       "step": 260,
       "use_label": 0.0
     },
     {
       "epoch": 0.57,
-      "grad_norm": 2.4375,
+      "grad_norm": 2.90625,
       "learning_rate": 2.3627616503391813e-06,
-      "logits/chosen": 0.6336380839347839,
-      "logits/rejected": 0.5556719303131104,
-      "logps/chosen": -116.7416000366211,
-      "logps/rejected": -135.33096313476562,
-      "loss": 0.6174,
+      "logits/chosen": 0.6390979290008545,
+      "logits/rejected": 0.5789315700531006,
+      "logps/chosen": -123.83528137207031,
+      "logps/rejected": -144.61489868164062,
+      "loss": 0.6162,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.3375000059604645,
-      "rewards/chosen": -0.43825817108154297,
-      "rewards/margins": 0.22129836678504944,
-      "rewards/rejected": -0.65955650806427,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": -0.5091949701309204,
+      "rewards/margins": 0.24320097267627716,
+      "rewards/rejected": -0.7523959279060364,
       "step": 270,
       "use_label": 0.0
     },
     {
       "epoch": 0.59,
-      "grad_norm": 3.0625,
+      "grad_norm": 2.34375,
       "learning_rate": 2.1804923757009885e-06,
-      "logits/chosen": 0.6383472681045532,
-      "logits/rejected": 0.7697634100914001,
-      "logps/chosen": -106.45858001708984,
-      "logps/rejected": -125.5028305053711,
-      "loss": 0.6353,
+      "logits/chosen": 0.8771865963935852,
+      "logits/rejected": 1.0158352851867676,
+      "logps/chosen": -118.5296859741211,
+      "logps/rejected": -138.31729125976562,
+      "loss": 0.6357,
       "pred_label": 0.0,
       "rewards/accuracies": 0.30000001192092896,
-      "rewards/chosen": -0.4095306992530823,
-      "rewards/margins": 0.21630148589611053,
-      "rewards/rejected": -0.625832200050354,
+      "rewards/chosen": -0.5302416086196899,
+      "rewards/margins": 0.2237352430820465,
+      "rewards/rejected": -0.7539768218994141,
       "step": 280,
       "use_label": 0.0
     },
     {
       "epoch": 0.61,
-      "grad_norm": 3.328125,
+      "grad_norm": 2.59375,
       "learning_rate": 1.9999357655598894e-06,
-      "logits/chosen": 0.1407470554113388,
-      "logits/rejected": 0.12877413630485535,
-      "logps/chosen": -108.0340805053711,
-      "logps/rejected": -136.49562072753906,
-      "loss": 0.6265,
+      "logits/chosen": 0.44083184003829956,
+      "logits/rejected": 0.41123947501182556,
+      "logps/chosen": -112.27372741699219,
+      "logps/rejected": -146.95498657226562,
+      "loss": 0.6228,
       "pred_label": 0.0,
       "rewards/accuracies": 0.30000001192092896,
-      "rewards/chosen": -0.41485634446144104,
-      "rewards/margins": 0.18648667633533478,
-      "rewards/rejected": -0.601343035697937,
+      "rewards/chosen": -0.4572528004646301,
+      "rewards/margins": 0.24868395924568176,
+      "rewards/rejected": -0.7059367299079895,
       "step": 290,
       "use_label": 0.0
     },
     {
       "epoch": 0.63,
-      "grad_norm": 3.03125,
+      "grad_norm": 2.34375,
       "learning_rate": 1.8220596619089576e-06,
-      "logits/chosen": 0.4002162516117096,
-      "logits/rejected": 0.25351682305336,
-      "logps/chosen": -127.95108795166016,
-      "logps/rejected": -172.98793029785156,
-      "loss": 0.6273,
-      "pred_label": 0.0,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": -0.5035675168037415,
-      "rewards/margins": 0.2851078510284424,
-      "rewards/rejected": -0.7886753678321838,
+      "logits/chosen": 0.6273639798164368,
+      "logits/rejected": 0.5140804052352905,
+      "logps/chosen": -123.02046966552734,
+      "logps/rejected": -168.80987548828125,
+      "loss": 0.6196,
+      "pred_label": 0.0,
+      "rewards/accuracies": 0.40625,
+      "rewards/chosen": -0.4542613625526428,
+      "rewards/margins": 0.2926333546638489,
+      "rewards/rejected": -0.7468947172164917,
       "step": 300,
       "use_label": 0.0
     },
     {
       "epoch": 0.63,
-      "eval_logits/chosen": 0.6280341148376465,
-      "eval_logits/rejected": 0.6725929379463196,
-      "eval_logps/chosen": -110.14692687988281,
-      "eval_logps/rejected": -157.1332244873047,
-      "eval_loss": 0.620426595211029,
+      "eval_logits/chosen": 1.0944873094558716,
+      "eval_logits/rejected": 1.1831356287002563,
+      "eval_logps/chosen": -102.62176513671875,
+      "eval_logps/rejected": -150.12503051757812,
+      "eval_loss": 0.618873655796051,
       "eval_pred_label": 0.0,
-      "eval_rewards/accuracies": 0.3671875,
-      "eval_rewards/chosen": -0.46234679222106934,
-      "eval_rewards/margins": 0.33705610036849976,
-      "eval_rewards/rejected": -0.7994028329849243,
-      "eval_runtime": 125.7299,
-      "eval_samples_per_second": 15.907,
+      "eval_rewards/accuracies": 0.375,
+      "eval_rewards/chosen": -0.3870951533317566,
+      "eval_rewards/margins": 0.34222573041915894,
+      "eval_rewards/rejected": -0.7293209433555603,
+      "eval_runtime": 125.4362,
+      "eval_samples_per_second": 15.944,
       "eval_steps_per_second": 0.255,
       "eval_use_label": 0.0,
       "step": 300
     },
     {
       "epoch": 0.65,
-      "grad_norm": 2.390625,
+      "grad_norm": 1.8515625,
       "learning_rate": 1.647817538357072e-06,
-      "logits/chosen": 0.33872538805007935,
-      "logits/rejected": 0.3415250778198242,
-      "logps/chosen": -95.08795166015625,
-      "logps/rejected": -142.95713806152344,
-      "loss": 0.6014,
+      "logits/chosen": 0.8131985664367676,
+      "logits/rejected": 0.8752232789993286,
+      "logps/chosen": -91.52378845214844,
+      "logps/rejected": -139.95840454101562,
+      "loss": 0.5999,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": -0.39491352438926697,
-      "rewards/margins": 0.35215410590171814,
-      "rewards/rejected": -0.7470676302909851,
+      "rewards/accuracies": 0.3687500059604645,
+      "rewards/chosen": -0.3592718541622162,
+      "rewards/margins": 0.3578081727027893,
+      "rewards/rejected": -0.7170799970626831,
       "step": 310,
       "use_label": 0.0
     },
     {
       "epoch": 0.67,
-      "grad_norm": 2.546875,
+      "grad_norm": 2.40625,
       "learning_rate": 1.4781433892011132e-06,
-      "logits/chosen": 0.2642754018306732,
-      "logits/rejected": 0.4063233435153961,
-      "logps/chosen": -131.07791137695312,
-      "logps/rejected": -164.12667846679688,
-      "loss": 0.6133,
+      "logits/chosen": 0.9751952886581421,
+      "logits/rejected": 1.1630818843841553,
+      "logps/chosen": -135.82566833496094,
+      "logps/rejected": -168.11805725097656,
+      "loss": 0.6109,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.38749998807907104,
-      "rewards/chosen": -0.580074667930603,
-      "rewards/margins": 0.38923436403274536,
-      "rewards/rejected": -0.9693089723587036,
+      "rewards/accuracies": 0.3687500059604645,
+      "rewards/chosen": -0.6275521516799927,
+      "rewards/margins": 0.3816707730293274,
+      "rewards/rejected": -1.0092228651046753,
       "step": 320,
       "use_label": 0.0
     },
     {
       "epoch": 0.69,
-      "grad_norm": 3.15625,
+      "grad_norm": 1.984375,
       "learning_rate": 1.3139467229135999e-06,
-      "logits/chosen": 0.5224499106407166,
-      "logits/rejected": 0.5213581919670105,
-      "logps/chosen": -130.00186157226562,
-      "logps/rejected": -156.6516876220703,
-      "loss": 0.6387,
+      "logits/chosen": 1.3293979167938232,
+      "logits/rejected": 1.3260401487350464,
+      "logps/chosen": -135.96664428710938,
+      "logps/rejected": -166.52359008789062,
+      "loss": 0.6295,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.3375000059604645,
-      "rewards/chosen": -0.5989372134208679,
-      "rewards/margins": 0.2814994752407074,
-      "rewards/rejected": -0.8804367184638977,
+      "rewards/accuracies": 0.33125001192092896,
+      "rewards/chosen": -0.6585850715637207,
+      "rewards/margins": 0.3205706775188446,
+      "rewards/rejected": -0.9791557192802429,
       "step": 330,
       "use_label": 0.0
     },
     {
       "epoch": 0.71,
-      "grad_norm": 2.28125,
+      "grad_norm": 2.09375,
       "learning_rate": 1.1561076868822756e-06,
-      "logits/chosen": 0.1671726554632187,
-      "logits/rejected": 0.0974355936050415,
-      "logps/chosen": -140.3222198486328,
-      "logps/rejected": -155.46217346191406,
-      "loss": 0.6252,
+      "logits/chosen": 0.7383319139480591,
+      "logits/rejected": 0.6407849192619324,
+      "logps/chosen": -150.60504150390625,
+      "logps/rejected": -166.74940490722656,
+      "loss": 0.6247,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.3375000059604645,
-      "rewards/chosen": -0.5558302998542786,
-      "rewards/margins": 0.23368898034095764,
-      "rewards/rejected": -0.7895193099975586,
+      "rewards/accuracies": 0.3499999940395355,
+      "rewards/chosen": -0.658658504486084,
+      "rewards/margins": 0.24373307824134827,
+      "rewards/rejected": -0.9023915529251099,
       "step": 340,
       "use_label": 0.0
     },
     {
       "epoch": 0.73,
-      "grad_norm": 3.328125,
+      "grad_norm": 2.21875,
       "learning_rate": 1.0054723495346484e-06,
-      "logits/chosen": 0.081739641726017,
-      "logits/rejected": 0.08175826817750931,
-      "logps/chosen": -150.41506958007812,
-      "logps/rejected": -178.51565551757812,
-      "loss": 0.6231,
+      "logits/chosen": 0.6359546184539795,
+      "logits/rejected": 0.7167641520500183,
+      "logps/chosen": -163.8385772705078,
+      "logps/rejected": -195.6297607421875,
+      "loss": 0.6138,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": -0.6099845170974731,
-      "rewards/margins": 0.322490930557251,
-      "rewards/rejected": -0.9324753880500793,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": -0.7442194819450378,
+      "rewards/margins": 0.3593973219394684,
+      "rewards/rejected": -1.103616714477539,
       "step": 350,
       "use_label": 0.0
     },
     {
       "epoch": 0.75,
-      "grad_norm": 1.875,
+      "grad_norm": 1.859375,
       "learning_rate": 8.628481651367876e-07,
-      "logits/chosen": 0.12279005348682404,
-      "logits/rejected": 0.20824797451496124,
-      "logps/chosen": -110.51042175292969,
-      "logps/rejected": -153.92698669433594,
-      "loss": 0.6186,
+      "logits/chosen": 0.7298086881637573,
+      "logits/rejected": 0.8517257571220398,
+      "logps/chosen": -119.41548156738281,
+      "logps/rejected": -165.3460235595703,
+      "loss": 0.6137,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.3687500059604645,
-      "rewards/chosen": -0.46872806549072266,
-      "rewards/margins": 0.3482593894004822,
-      "rewards/rejected": -0.8169875144958496,
+      "rewards/accuracies": 0.3812499940395355,
+      "rewards/chosen": -0.5577787160873413,
+      "rewards/margins": 0.37339919805526733,
+      "rewards/rejected": -0.9311779141426086,
       "step": 360,
       "use_label": 0.0
     },
     {
       "epoch": 0.77,
-      "grad_norm": 2.15625,
+      "grad_norm": 2.421875,
       "learning_rate": 7.289996455765749e-07,
-      "logits/chosen": 0.19759848713874817,
-      "logits/rejected": 0.29472407698631287,
-      "logps/chosen": -103.1863021850586,
-      "logps/rejected": -143.578125,
-      "loss": 0.6166,
+      "logits/chosen": 0.8383787274360657,
+      "logits/rejected": 0.9305205345153809,
+      "logps/chosen": -111.84449768066406,
+      "logps/rejected": -153.93136596679688,
+      "loss": 0.6125,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.3687500059604645,
-      "rewards/chosen": -0.37751519680023193,
-      "rewards/margins": 0.37911203503608704,
-      "rewards/rejected": -0.7566272020339966,
+      "rewards/accuracies": 0.3499999940395355,
+      "rewards/chosen": -0.46409696340560913,
+      "rewards/margins": 0.39606258273124695,
+      "rewards/rejected": -0.8601595759391785,
       "step": 370,
       "use_label": 0.0
     },
     {
       "epoch": 0.8,
-      "grad_norm": 1.96875,
+      "grad_norm": 1.8984375,
       "learning_rate": 6.046442623320145e-07,
-      "logits/chosen": 0.03893072158098221,
-      "logits/rejected": 0.019468214362859726,
-      "logps/chosen": -108.17799377441406,
-      "logps/rejected": -158.08056640625,
-      "loss": 0.6183,
+      "logits/chosen": 0.5329448580741882,
+      "logits/rejected": 0.513522744178772,
+      "logps/chosen": -116.62841796875,
+      "logps/rejected": -165.17893981933594,
+      "loss": 0.6191,
       "pred_label": 0.0,
       "rewards/accuracies": 0.3187499940395355,
-      "rewards/chosen": -0.42342591285705566,
-      "rewards/margins": 0.2937392592430115,
-      "rewards/rejected": -0.7171651124954224,
+      "rewards/chosen": -0.5079302787780762,
+      "rewards/margins": 0.2802185118198395,
+      "rewards/rejected": -0.7881487607955933,
       "step": 380,
       "use_label": 0.0
     },
     {
       "epoch": 0.82,
-      "grad_norm": 2.59375,
+      "grad_norm": 2.4375,
       "learning_rate": 4.904486005914027e-07,
-      "logits/chosen": 0.33429718017578125,
-      "logits/rejected": 0.08158789575099945,
-      "logps/chosen": -151.29055786132812,
-      "logps/rejected": -180.48861694335938,
-      "loss": 0.6114,
+      "logits/chosen": 0.8266662359237671,
+      "logits/rejected": 0.5234752893447876,
+      "logps/chosen": -159.83407592773438,
+      "logps/rejected": -186.96768188476562,
+      "loss": 0.6085,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.39375001192092896,
-      "rewards/chosen": -0.5847219824790955,
-      "rewards/margins": 0.3904651999473572,
-      "rewards/rejected": -0.9751871824264526,
+      "rewards/accuracies": 0.38749998807907104,
+      "rewards/chosen": -0.6701575517654419,
+      "rewards/margins": 0.36982032656669617,
+      "rewards/rejected": -1.039977788925171,
       "step": 390,
       "use_label": 0.0
     },
     {
       "epoch": 0.84,
-      "grad_norm": 2.015625,
+      "grad_norm": 2.46875,
       "learning_rate": 3.8702478614051353e-07,
-      "logits/chosen": 0.126608207821846,
-      "logits/rejected": 0.2576550841331482,
-      "logps/chosen": -109.39167785644531,
-      "logps/rejected": -134.27053833007812,
-      "loss": 0.6165,
+      "logits/chosen": 0.511390745639801,
+      "logits/rejected": 0.6720080971717834,
+      "logps/chosen": -116.7987060546875,
+      "logps/rejected": -141.3931884765625,
+      "loss": 0.6139,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.38749998807907104,
-      "rewards/chosen": -0.36900678277015686,
-      "rewards/margins": 0.3390708863735199,
-      "rewards/rejected": -0.708077609539032,
+      "rewards/accuracies": 0.3812499940395355,
+      "rewards/chosen": -0.4430771768093109,
+      "rewards/margins": 0.3362268805503845,
+      "rewards/rejected": -0.779304027557373,
       "step": 400,
       "use_label": 0.0
     },
     {
       "epoch": 0.84,
-      "eval_logits/chosen": 0.903490424156189,
-      "eval_logits/rejected": 0.958048939704895,
-      "eval_logps/chosen": -108.47840881347656,
-      "eval_logps/rejected": -158.4149169921875,
-      "eval_loss": 0.6182093620300293,
+      "eval_logits/chosen": 1.4532994031906128,
+      "eval_logits/rejected": 1.5453113317489624,
+      "eval_logps/chosen": -112.56050109863281,
+      "eval_logps/rejected": -162.19764709472656,
+      "eval_loss": 0.6157013177871704,
       "eval_pred_label": 0.0,
-      "eval_rewards/accuracies": 0.3671875,
-      "eval_rewards/chosen": -0.4456615447998047,
-      "eval_rewards/margins": 0.3665582537651062,
-      "eval_rewards/rejected": -0.8122197389602661,
-      "eval_runtime": 125.7278,
-      "eval_samples_per_second": 15.907,
+      "eval_rewards/accuracies": 0.37109375,
+      "eval_rewards/chosen": -0.4864824414253235,
+      "eval_rewards/margins": 0.36356455087661743,
+      "eval_rewards/rejected": -0.8500469923019409,
+      "eval_runtime": 125.4203,
+      "eval_samples_per_second": 15.946,
       "eval_steps_per_second": 0.255,
       "eval_use_label": 0.0,
       "step": 400
     },
     {
       "epoch": 0.86,
-      "grad_norm": 2.046875,
+      "grad_norm": 2.203125,
       "learning_rate": 2.9492720416985004e-07,
-      "logits/chosen": 0.39335688948631287,
-      "logits/rejected": 0.41703349351882935,
-      "logps/chosen": -106.9058837890625,
-      "logps/rejected": -138.57296752929688,
-      "loss": 0.6272,
+      "logits/chosen": 0.8359997868537903,
+      "logits/rejected": 0.8144146800041199,
+      "logps/chosen": -110.30177307128906,
+      "logps/rejected": -143.6800079345703,
+      "loss": 0.6222,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.36250001192092896,
-      "rewards/chosen": -0.45482879877090454,
-      "rewards/margins": 0.3337084650993347,
-      "rewards/rejected": -0.788537323474884,
+      "rewards/accuracies": 0.39375001192092896,
+      "rewards/chosen": -0.4887877404689789,
+      "rewards/margins": 0.3508199453353882,
+      "rewards/rejected": -0.8396075963973999,
       "step": 410,
       "use_label": 0.0
     },
     {
       "epoch": 0.88,
-      "grad_norm": 2.078125,
+      "grad_norm": 1.984375,
       "learning_rate": 2.1464952759020857e-07,
-      "logits/chosen": 0.5264393091201782,
-      "logits/rejected": 0.4952784478664398,
-      "logps/chosen": -104.27522277832031,
-      "logps/rejected": -112.507080078125,
-      "loss": 0.6235,
+      "logits/chosen": 1.027252435684204,
+      "logits/rejected": 0.9827619791030884,
+      "logps/chosen": -106.49784851074219,
+      "logps/rejected": -116.97566223144531,
+      "loss": 0.6216,
       "pred_label": 0.0,
       "rewards/accuracies": 0.2750000059604645,
-      "rewards/chosen": -0.4333609640598297,
-      "rewards/margins": 0.1778794825077057,
-      "rewards/rejected": -0.6112405061721802,
+      "rewards/chosen": -0.4555872976779938,
+      "rewards/margins": 0.20033884048461914,
+      "rewards/rejected": -0.6559261083602905,
       "step": 420,
       "use_label": 0.0
     },
     {
       "epoch": 0.9,
-      "grad_norm": 1.734375,
+      "grad_norm": 1.96875,
       "learning_rate": 1.4662207078575685e-07,
-      "logits/chosen": 0.47332754731178284,
-      "logits/rejected": 0.4613571763038635,
-      "logps/chosen": -144.65744018554688,
-      "logps/rejected": -170.08921813964844,
-      "loss": 0.5988,
+      "logits/chosen": 0.9206047058105469,
+      "logits/rejected": 0.8673297166824341,
+      "logps/chosen": -151.376220703125,
+      "logps/rejected": -178.04725646972656,
+      "loss": 0.5986,
       "pred_label": 0.0,
       "rewards/accuracies": 0.45625001192092896,
-      "rewards/chosen": -0.4539059102535248,
-      "rewards/margins": 0.4534150958061218,
-      "rewards/rejected": -0.9073210954666138,
+      "rewards/chosen": -0.5210937261581421,
+      "rewards/margins": 0.46580758690834045,
+      "rewards/rejected": -0.9869012832641602,
       "step": 430,
       "use_label": 0.0
     },
     {
       "epoch": 0.92,
-      "grad_norm": 1.9609375,
+      "grad_norm": 2.125,
       "learning_rate": 9.120948298936422e-08,
-      "logits/chosen": 0.48202329874038696,
-      "logits/rejected": 0.6259401440620422,
-      "logps/chosen": -114.15118408203125,
-      "logps/rejected": -161.5361785888672,
-      "loss": 0.6098,
+      "logits/chosen": 0.9004503488540649,
+      "logits/rejected": 1.0573413372039795,
+      "logps/chosen": -119.21500396728516,
+      "logps/rejected": -165.19241333007812,
+      "loss": 0.6064,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.36250001192092896,
-      "rewards/chosen": -0.4724721908569336,
-      "rewards/margins": 0.39225998520851135,
-      "rewards/rejected": -0.8647321462631226,
+      "rewards/accuracies": 0.35624998807907104,
+      "rewards/chosen": -0.5231102705001831,
+      "rewards/margins": 0.37818416953086853,
+      "rewards/rejected": -0.9012944102287292,
       "step": 440,
       "use_label": 0.0
     },
     {
       "epoch": 0.94,
-      "grad_norm": 2.265625,
+      "grad_norm": 2.46875,
       "learning_rate": 4.870879364444109e-08,
-      "logits/chosen": 0.8100695610046387,
-      "logits/rejected": 0.5903851389884949,
-      "logps/chosen": -126.81998443603516,
-      "logps/rejected": -174.6106719970703,
-      "loss": 0.6122,
+      "logits/chosen": 1.300728440284729,
+      "logits/rejected": 1.0580918788909912,
+      "logps/chosen": -129.29281616210938,
+      "logps/rejected": -178.3690948486328,
+      "loss": 0.6111,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.36250001192092896,
-      "rewards/chosen": -0.5456215739250183,
-      "rewards/margins": 0.3175886273384094,
-      "rewards/rejected": -0.8632103204727173,
+      "rewards/accuracies": 0.3499999940395355,
+      "rewards/chosen": -0.570349931716919,
+      "rewards/margins": 0.3304445147514343,
+      "rewards/rejected": -0.9007943868637085,
       "step": 450,
       "use_label": 0.0
     },
     {
       "epoch": 0.96,
-      "grad_norm": 2.140625,
+      "grad_norm": 1.8359375,
       "learning_rate": 1.93478202307823e-08,
-      "logits/chosen": 0.7001665830612183,
-      "logits/rejected": 0.7000536322593689,
-      "logps/chosen": -80.71357727050781,
-      "logps/rejected": -126.110595703125,
-      "loss": 0.6182,
+      "logits/chosen": 1.1906068325042725,
+      "logits/rejected": 1.2149587869644165,
+      "logps/chosen": -83.74864196777344,
+      "logps/rejected": -130.91348266601562,
+      "loss": 0.6154,
       "pred_label": 0.0,
-      "rewards/accuracies": 0.32499998807907104,
-      "rewards/chosen": -0.3459371328353882,
-      "rewards/margins": 0.2817174792289734,
-      "rewards/rejected": -0.6276546716690063,
+      "rewards/accuracies": 0.3375000059604645,
+      "rewards/chosen": -0.3762877583503723,
+      "rewards/margins": 0.2993956208229065,
+      "rewards/rejected": -0.6756833791732788,
       "step": 460,
       "use_label": 0.0
     },
     {
       "epoch": 0.98,
-      "grad_norm": 2.78125,
+      "grad_norm": 2.375,
       "learning_rate": 3.283947088983663e-09,
-      "logits/chosen": 0.7130995392799377,
-      "logits/rejected": 0.5145190954208374,
-      "logps/chosen": -110.40830993652344,
-      "logps/rejected": -137.49429321289062,
-      "loss": 0.6251,
+      "logits/chosen": 1.1844379901885986,
+      "logits/rejected": 0.9474547505378723,
+      "logps/chosen": -113.1079330444336,
+      "logps/rejected": -141.49147033691406,
+      "loss": 0.6213,
       "pred_label": 0.0,
       "rewards/accuracies": 0.3125,
-      "rewards/chosen": -0.43079155683517456,
-      "rewards/margins": 0.25358152389526367,
-      "rewards/rejected": -0.6843730211257935,
+      "rewards/chosen": -0.4577876627445221,
+      "rewards/margins": 0.26655709743499756,
+      "rewards/rejected": -0.7243447303771973,
       "step": 470,
       "use_label": 0.0
     },
@@ -900,9 +900,9 @@
       "epoch": 1.0,
       "step": 477,
       "total_flos": 0.0,
-      "train_loss": 0.6389844682481554,
-      "train_runtime": 9615.2592,
-      "train_samples_per_second": 6.358,
+      "train_loss": 0.6357159084743924,
+      "train_runtime": 9601.7268,
+      "train_samples_per_second": 6.367,
       "train_steps_per_second": 0.05
     }
   ],