Model save

README.md

@@ -17,15 +17,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.0660
-- Rewards/chosen: -2.5606
-- Rewards/rejected: -2.9549
-- Rewards/accuracies: 0.625
-- Rewards/margins: 0.3944
-- Logps/rejected: -552.8470
-- Logps/chosen: -513.0960
-- Logits/rejected: -2.2459
-- Logits/chosen: -2.2708
+- Loss: 0.4920
+- Rewards/chosen: -2.3074
+- Rewards/rejected: -3.5196
+- Rewards/accuracies: 0.7734
+- Rewards/margins: 1.2122
+- Logps/rejected: -609.3139
+- Logps/chosen: -487.7755
+- Logits/rejected: -0.7242
+- Logits/chosen: -0.9597
 
 ## Model description
 
@@ -47,7 +47,7 @@ The following hyperparameters were used during training:
 - learning_rate: 5e-07
 - train_batch_size: 8
 - eval_batch_size: 8
-- seed: 4
+- seed: 2
 - distributed_type: multi-GPU
 - num_devices: 8
 - gradient_accumulation_steps: 2
@@ -62,10 +62,14 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.0437        | 0.25  | 100  | 0.0824          | -2.2538        | -2.4741          | 0.5859             | 0.2203          | -504.7590      | -482.4154    | -2.3143         | -2.3260       |
-| 0.0258        | 0.49  | 200  | 0.0581          | -2.8677        | -3.2192          | 0.5977             | 0.3515          | -579.2755      | -543.8072    | -2.1155         | -2.1394       |
-| 0.0402        | 0.74  | 300  | 0.0837          | -2.0997        | -2.5006          | 0.6289             | 0.4009          | -507.4115      | -467.0057    | -2.2751         | -2.2980       |
-| 0.0288        | 0.99  | 400  | 0.0660          | -2.5606        | -2.9549          | 0.625              | 0.3944          | -552.8470      | -513.0960    | -2.2459         | -2.2708       |
+| 0.5392        | 0.11  | 100  | 0.6286          | -0.6554        | -0.9418          | 0.6523             | 0.2865          | -351.5352      | -322.5750    | -2.5756         | -2.5908       |
+| 0.4524        | 0.23  | 200  | 0.5475          | -1.4831        | -2.1698          | 0.7227             | 0.6867          | -474.3327      | -405.3454    | -1.9678         | -1.9878       |
+| 0.3976        | 0.34  | 300  | 0.5194          | -1.8541        | -2.8790          | 0.7617             | 1.0249          | -545.2501      | -442.4474    | -0.9783         | -1.1841       |
+| 0.3892        | 0.45  | 400  | 0.5160          | -2.0795        | -3.1766          | 0.7773             | 1.0971          | -575.0087      | -464.9888    | -0.6002         | -0.8579       |
+| 0.3964        | 0.57  | 500  | 0.4992          | -2.1896        | -3.3081          | 0.7656             | 1.1185          | -588.1666      | -476.0038    | -0.8012         | -1.0189       |
+| 0.4149        | 0.68  | 600  | 0.4948          | -2.2061        | -3.3241          | 0.7461             | 1.1179          | -589.7601      | -477.6525    | -1.0527         | -1.2398       |
+| 0.4004        | 0.79  | 700  | 0.4905          | -2.1723        | -3.3652          | 0.7695             | 1.1929          | -593.8731      | -474.2662    | -0.8519         | -1.0643       |
+| 0.3887        | 0.91  | 800  | 0.4920          | -2.3074        | -3.5196          | 0.7734             | 1.2122          | -609.3139      | -487.7755    | -0.7242         | -0.9597       |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.07151281171374851,
-    "train_runtime": 3738.25,
-    "train_samples": 51894,
-    "train_samples_per_second": 13.882,
-    "train_steps_per_second": 0.108
+    "train_loss": 0.43981607611020046,
+    "train_runtime": 8273.4147,
+    "train_samples": 113028,
+    "train_samples_per_second": 13.662,
+    "train_steps_per_second": 0.107
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fb2741318d25fa010663fe61ed02f4f293fa8ad301934c24bbabaf6e60633fb3
+oid sha256:3706298fa431e1f5810589ffba965a8702ccfbe931becf39115a25cf32b0500d
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0f559ba79771ca5e5cdbae085a55b0de304927c43a4793b3f8234d1f33152354
+oid sha256:ce0bba006f75edd300be523b9ddf4224846bca628f24428e1354ecbf84bda2f1
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:901f2833dc2e0f3adb4f4bd18d3a372877da1018c193c779ed31f78d98f1f0a4
+oid sha256:3bdc684b29544e54449b1407ec61244d68a0f37ef83f91adbb77ec167e63be6c
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.07151281171374851,
-    "train_runtime": 3738.25,
-    "train_samples": 51894,
-    "train_samples_per_second": 13.882,
-    "train_steps_per_second": 0.108
+    "train_loss": 0.43981607611020046,
+    "train_runtime": 8273.4147,
+    "train_samples": 113028,
+    "train_samples_per_second": 13.662,
+    "train_steps_per_second": 0.107
 }

trainer_state.json

@@ -1,21 +1,21 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.998766954377312,
+  "epoch": 0.9994340690435767,
   "eval_steps": 100,
-  "global_step": 405,
+  "global_step": 883,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.0,
-      "learning_rate": 1.2195121951219512e-08,
-      "logits/chosen": -2.8681135177612305,
-      "logits/rejected": -2.8858838081359863,
-      "logps/chosen": -518.1907958984375,
-      "logps/rejected": -109.31971740722656,
-      "loss": 0.3475,
+      "learning_rate": 5.617977528089887e-09,
+      "logits/chosen": -2.7943434715270996,
+      "logits/rejected": -2.817823886871338,
+      "logps/chosen": -334.107666015625,
+      "logps/rejected": -197.05621337890625,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -23,641 +23,1377 @@
       "step": 1
     },
     {
-      "epoch": 0.02,
-      "learning_rate": 1.219512195121951e-07,
-      "logits/chosen": -2.7986178398132324,
-      "logits/rejected": -2.752176284790039,
-      "logps/chosen": -434.208251953125,
-      "logps/rejected": -114.19618225097656,
-      "loss": 0.3394,
-      "rewards/accuracies": 0.4930555522441864,
-      "rewards/chosen": 0.0002649651141837239,
-      "rewards/margins": 0.0009347840095870197,
-      "rewards/rejected": -0.0006698188371956348,
+      "epoch": 0.01,
+      "learning_rate": 5.617977528089887e-08,
+      "logits/chosen": -2.833451271057129,
+      "logits/rejected": -2.7827768325805664,
+      "logps/chosen": -323.80584716796875,
+      "logps/rejected": -189.39964294433594,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4722222089767456,
+      "rewards/chosen": 0.0005755923339165747,
+      "rewards/margins": 0.0003566421801224351,
+      "rewards/rejected": 0.00021895011013839394,
       "step": 10
     },
     {
-      "epoch": 0.05,
-      "learning_rate": 2.439024390243902e-07,
-      "logits/chosen": -2.8215415477752686,
-      "logits/rejected": -2.7983882427215576,
-      "logps/chosen": -417.2633361816406,
-      "logps/rejected": -118.0062026977539,
-      "loss": 0.3373,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.019945567473769188,
-      "rewards/margins": 0.03575458750128746,
-      "rewards/rejected": -0.015809018164873123,
+      "epoch": 0.02,
+      "learning_rate": 1.1235955056179774e-07,
+      "logits/chosen": -2.778655767440796,
+      "logits/rejected": -2.7627151012420654,
+      "logps/chosen": -323.3365783691406,
+      "logps/rejected": -168.40744018554688,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.0010369193041697145,
+      "rewards/margins": 0.0018870027270168066,
+      "rewards/rejected": -0.0008500836556777358,
       "step": 20
     },
     {
-      "epoch": 0.07,
-      "learning_rate": 3.6585365853658536e-07,
-      "logits/chosen": -2.6574862003326416,
-      "logits/rejected": -2.6451315879821777,
-      "logps/chosen": -398.87353515625,
-      "logps/rejected": -125.69970703125,
-      "loss": 0.3045,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 0.07569055259227753,
-      "rewards/margins": 0.19884702563285828,
-      "rewards/rejected": -0.12315647304058075,
+      "epoch": 0.03,
+      "learning_rate": 1.6853932584269663e-07,
+      "logits/chosen": -2.7871737480163574,
+      "logits/rejected": -2.7326064109802246,
+      "logps/chosen": -305.997314453125,
+      "logps/rejected": -180.06800842285156,
+      "loss": 0.683,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.009164368733763695,
+      "rewards/margins": 0.015919247642159462,
+      "rewards/rejected": -0.006754877977073193,
       "step": 30
     },
     {
-      "epoch": 0.1,
-      "learning_rate": 4.878048780487804e-07,
-      "logits/chosen": -2.54256272315979,
-      "logits/rejected": -2.5281729698181152,
-      "logps/chosen": -384.5321044921875,
-      "logps/rejected": -168.55758666992188,
-      "loss": 0.2564,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.024631643667817116,
-      "rewards/margins": 0.41851943731307983,
-      "rewards/rejected": -0.39388787746429443,
+      "epoch": 0.05,
+      "learning_rate": 2.2471910112359549e-07,
+      "logits/chosen": -2.7199320793151855,
+      "logits/rejected": -2.711822032928467,
+      "logps/chosen": -314.8984680175781,
+      "logps/rejected": -178.45077514648438,
+      "loss": 0.6653,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.034292496740818024,
+      "rewards/margins": 0.06667112559080124,
+      "rewards/rejected": -0.03237862139940262,
       "step": 40
     },
     {
-      "epoch": 0.12,
-      "learning_rate": 4.992461696250783e-07,
-      "logits/chosen": -2.4257261753082275,
-      "logits/rejected": -2.3928446769714355,
-      "logps/chosen": -436.45330810546875,
-      "logps/rejected": -219.0617218017578,
-      "loss": 0.1809,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.1671580970287323,
-      "rewards/margins": 0.7879143953323364,
-      "rewards/rejected": -0.9550724029541016,
+      "epoch": 0.06,
+      "learning_rate": 2.8089887640449437e-07,
+      "logits/chosen": -2.6660404205322266,
+      "logits/rejected": -2.6608872413635254,
+      "logps/chosen": -340.89056396484375,
+      "logps/rejected": -192.22543334960938,
+      "loss": 0.6387,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.0421828031539917,
+      "rewards/margins": 0.14488723874092102,
+      "rewards/rejected": -0.10270445048809052,
       "step": 50
     },
     {
-      "epoch": 0.15,
-      "learning_rate": 4.966461721767899e-07,
-      "logits/chosen": -2.3805835247039795,
-      "logits/rejected": -2.3364853858947754,
-      "logps/chosen": -437.4466857910156,
-      "logps/rejected": -240.6685791015625,
-      "loss": 0.1377,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.32454290986061096,
-      "rewards/margins": 0.9316140413284302,
-      "rewards/rejected": -1.2561569213867188,
+      "epoch": 0.07,
+      "learning_rate": 3.3707865168539325e-07,
+      "logits/chosen": -2.6621761322021484,
+      "logits/rejected": -2.6332790851593018,
+      "logps/chosen": -290.0724182128906,
+      "logps/rejected": -199.76377868652344,
+      "loss": 0.6192,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.03800051286816597,
+      "rewards/margins": 0.1342642456293106,
+      "rewards/rejected": -0.17226476967334747,
       "step": 60
     },
     {
-      "epoch": 0.17,
-      "learning_rate": 4.922100518015975e-07,
-      "logits/chosen": -2.3752458095550537,
-      "logits/rejected": -2.3281030654907227,
-      "logps/chosen": -419.6747131347656,
-      "logps/rejected": -264.75787353515625,
-      "loss": 0.103,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.3937300443649292,
-      "rewards/margins": 1.1917842626571655,
-      "rewards/rejected": -1.5855143070220947,
+      "epoch": 0.08,
+      "learning_rate": 3.9325842696629214e-07,
+      "logits/chosen": -2.5926709175109863,
+      "logits/rejected": -2.5758044719696045,
+      "logps/chosen": -318.26446533203125,
+      "logps/rejected": -217.8231964111328,
+      "loss": 0.5872,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.061497531831264496,
+      "rewards/margins": 0.23234911262989044,
+      "rewards/rejected": -0.29384663701057434,
       "step": 70
     },
     {
-      "epoch": 0.2,
-      "learning_rate": 4.859708325770919e-07,
-      "logits/chosen": -2.4320530891418457,
-      "logits/rejected": -2.3738579750061035,
-      "logps/chosen": -472.10479736328125,
-      "logps/rejected": -330.32403564453125,
-      "loss": 0.0674,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -0.5765678286552429,
-      "rewards/margins": 1.5421369075775146,
-      "rewards/rejected": -2.1187047958374023,
+      "epoch": 0.09,
+      "learning_rate": 4.4943820224719097e-07,
+      "logits/chosen": -2.561908483505249,
+      "logits/rejected": -2.5379791259765625,
+      "logps/chosen": -396.86993408203125,
+      "logps/rejected": -253.50143432617188,
+      "loss": 0.5579,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.01939094439148903,
+      "rewards/margins": 0.49067315459251404,
+      "rewards/rejected": -0.5100641250610352,
       "step": 80
     },
     {
-      "epoch": 0.22,
-      "learning_rate": 4.779749614980225e-07,
-      "logits/chosen": -2.3991949558258057,
-      "logits/rejected": -2.357053279876709,
-      "logps/chosen": -487.83074951171875,
-      "logps/rejected": -349.1925354003906,
-      "loss": 0.0553,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.666537880897522,
-      "rewards/margins": 1.7182201147079468,
-      "rewards/rejected": -2.3847577571868896,
+      "epoch": 0.1,
+      "learning_rate": 4.999980431020109e-07,
+      "logits/chosen": -2.5810797214508057,
+      "logits/rejected": -2.5567336082458496,
+      "logps/chosen": -380.4464416503906,
+      "logps/rejected": -262.82904052734375,
+      "loss": 0.5455,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.24769897758960724,
+      "rewards/margins": 0.5812320709228516,
+      "rewards/rejected": -0.8289310336112976,
       "step": 90
     },
     {
-      "epoch": 0.25,
-      "learning_rate": 4.682819627081427e-07,
-      "logits/chosen": -2.3752927780151367,
-      "logits/rejected": -2.326216220855713,
-      "logps/chosen": -515.1549682617188,
-      "logps/rejected": -378.8877868652344,
-      "loss": 0.0437,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -0.6667075157165527,
-      "rewards/margins": 2.000246524810791,
-      "rewards/rejected": -2.666954278945923,
+      "epoch": 0.11,
+      "learning_rate": 4.997632524101301e-07,
+      "logits/chosen": -2.6055984497070312,
+      "logits/rejected": -2.5864219665527344,
+      "logps/chosen": -367.29071044921875,
+      "logps/rejected": -280.4869079589844,
+      "loss": 0.5392,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.2197243869304657,
+      "rewards/margins": 0.5160216689109802,
+      "rewards/rejected": -0.7357459664344788,
       "step": 100
     },
     {
-      "epoch": 0.25,
-      "eval_logits/chosen": -2.3259778022766113,
-      "eval_logits/rejected": -2.314302682876587,
-      "eval_logps/chosen": -482.4153747558594,
-      "eval_logps/rejected": -504.759033203125,
-      "eval_loss": 0.08243285864591599,
-      "eval_rewards/accuracies": 0.5859375,
-      "eval_rewards/chosen": -2.2537574768066406,
-      "eval_rewards/margins": 0.22029951214790344,
-      "eval_rewards/rejected": -2.4740567207336426,
-      "eval_runtime": 53.3582,
-      "eval_samples_per_second": 37.483,
-      "eval_steps_per_second": 0.6,
+      "epoch": 0.11,
+      "eval_logits/chosen": -2.590785264968872,
+      "eval_logits/rejected": -2.5756187438964844,
+      "eval_logps/chosen": -322.57501220703125,
+      "eval_logps/rejected": -351.5352478027344,
+      "eval_loss": 0.6285832524299622,
+      "eval_rewards/accuracies": 0.65234375,
+      "eval_rewards/chosen": -0.6553537845611572,
+      "eval_rewards/margins": 0.2864663004875183,
+      "eval_rewards/rejected": -0.9418200850486755,
+      "eval_runtime": 53.1932,
+      "eval_samples_per_second": 37.599,
+      "eval_steps_per_second": 0.602,
       "step": 100
     },
     {
-      "epoch": 0.27,
-      "learning_rate": 4.569639943810477e-07,
-      "logits/chosen": -2.3144371509552,
-      "logits/rejected": -2.2340025901794434,
-      "logps/chosen": -490.12921142578125,
-      "logps/rejected": -419.07867431640625,
-      "loss": 0.0359,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.3260681629180908,
-      "rewards/margins": 1.7610738277435303,
-      "rewards/rejected": -3.0871422290802,
+      "epoch": 0.12,
+      "learning_rate": 4.991375032514749e-07,
+      "logits/chosen": -2.5533313751220703,
+      "logits/rejected": -2.5264110565185547,
+      "logps/chosen": -363.4510498046875,
+      "logps/rejected": -284.7992248535156,
+      "loss": 0.5232,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3205306828022003,
+      "rewards/margins": 0.6818863749504089,
+      "rewards/rejected": -1.002416968345642,
       "step": 110
     },
     {
-      "epoch": 0.3,
-      "learning_rate": 4.4410531154874543e-07,
-      "logits/chosen": -2.3716444969177246,
-      "logits/rejected": -2.3235533237457275,
-      "logps/chosen": -466.01702880859375,
-      "logps/rejected": -356.8735046386719,
-      "loss": 0.0575,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.7180399298667908,
-      "rewards/margins": 1.6505486965179443,
-      "rewards/rejected": -2.368588924407959,
+      "epoch": 0.14,
+      "learning_rate": 4.98121775121344e-07,
+      "logits/chosen": -2.6315197944641113,
+      "logits/rejected": -2.5978212356567383,
+      "logps/chosen": -410.644775390625,
+      "logps/rejected": -323.01190185546875,
+      "loss": 0.4994,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.27390944957733154,
+      "rewards/margins": 0.8327864408493042,
+      "rewards/rejected": -1.1066958904266357,
       "step": 120
     },
     {
-      "epoch": 0.32,
-      "learning_rate": 4.298016388768561e-07,
-      "logits/chosen": -2.3074584007263184,
-      "logits/rejected": -2.257930040359497,
-      "logps/chosen": -472.1845703125,
-      "logps/rejected": -373.66522216796875,
-      "loss": 0.0498,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.0088322162628174,
-      "rewards/margins": 1.6445964574813843,
-      "rewards/rejected": -2.653428792953491,
+      "epoch": 0.15,
+      "learning_rate": 4.96717657955441e-07,
+      "logits/chosen": -2.59904408454895,
+      "logits/rejected": -2.5410983562469482,
+      "logps/chosen": -416.3720703125,
+      "logps/rejected": -325.9648132324219,
+      "loss": 0.5013,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.40370965003967285,
+      "rewards/margins": 0.9006286859512329,
+      "rewards/rejected": -1.3043382167816162,
       "step": 130
     },
     {
-      "epoch": 0.35,
-      "learning_rate": 4.1415945805573005e-07,
-      "logits/chosen": -2.225804328918457,
-      "logits/rejected": -2.15400767326355,
-      "logps/chosen": -534.1700439453125,
-      "logps/rejected": -430.3104553222656,
-      "loss": 0.0361,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.042690634727478,
-      "rewards/margins": 2.092653751373291,
-      "rewards/rejected": -3.1353445053100586,
+      "epoch": 0.16,
+      "learning_rate": 4.949273496411216e-07,
+      "logits/chosen": -2.545508861541748,
+      "logits/rejected": -2.5205612182617188,
+      "logps/chosen": -379.17767333984375,
+      "logps/rejected": -337.29962158203125,
+      "loss": 0.4954,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.451716810464859,
+      "rewards/margins": 0.8486088514328003,
+      "rewards/rejected": -1.300325632095337,
       "step": 140
     },
     {
-      "epoch": 0.37,
-      "learning_rate": 3.972952151123984e-07,
-      "logits/chosen": -2.2562787532806396,
-      "logits/rejected": -2.164506673812866,
-      "logps/chosen": -522.7659912109375,
-      "logps/rejected": -425.18109130859375,
-      "loss": 0.0344,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.1226718425750732,
-      "rewards/margins": 2.0036892890930176,
-      "rewards/rejected": -3.12636137008667,
+      "epoch": 0.17,
+      "learning_rate": 4.927536525770046e-07,
+      "logits/chosen": -2.5130438804626465,
+      "logits/rejected": -2.487233877182007,
+      "logps/chosen": -423.2710876464844,
+      "logps/rejected": -352.5829772949219,
+      "loss": 0.4976,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.5199152231216431,
+      "rewards/margins": 1.0131314992904663,
+      "rewards/rejected": -1.5330466032028198,
       "step": 150
     },
     {
-      "epoch": 0.39,
-      "learning_rate": 3.793344535444142e-07,
-      "logits/chosen": -2.267565965652466,
-      "logits/rejected": -2.1969974040985107,
-      "logps/chosen": -530.3189086914062,
-      "logps/rejected": -426.72332763671875,
-      "loss": 0.0393,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.2129985094070435,
-      "rewards/margins": 1.8231168985366821,
-      "rewards/rejected": -3.0361156463623047,
+      "epoch": 0.18,
+      "learning_rate": 4.901999692863326e-07,
+      "logits/chosen": -2.520357847213745,
+      "logits/rejected": -2.4684462547302246,
+      "logps/chosen": -498.07098388671875,
+      "logps/rejected": -388.2645263671875,
+      "loss": 0.463,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.5807570219039917,
+      "rewards/margins": 1.1767116785049438,
+      "rewards/rejected": -1.757468581199646,
       "step": 160
     },
     {
-      "epoch": 0.42,
-      "learning_rate": 3.604108797288461e-07,
-      "logits/chosen": -2.237342119216919,
-      "logits/rejected": -2.1961898803710938,
-      "logps/chosen": -448.13812255859375,
-      "logps/rejected": -372.9068298339844,
-      "loss": 0.0465,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.092370629310608,
-      "rewards/margins": 1.5985119342803955,
-      "rewards/rejected": -2.690882444381714,
+      "epoch": 0.19,
+      "learning_rate": 4.872702970909464e-07,
+      "logits/chosen": -2.345059633255005,
+      "logits/rejected": -2.281158924102783,
+      "logps/chosen": -455.2555236816406,
+      "logps/rejected": -373.2399597167969,
+      "loss": 0.4471,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -0.8019993901252747,
+      "rewards/margins": 1.065147042274475,
+      "rewards/rejected": -1.8671462535858154,
       "step": 170
     },
     {
-      "epoch": 0.44,
-      "learning_rate": 3.40665367563858e-07,
-      "logits/chosen": -2.2571911811828613,
-      "logits/rejected": -2.140353202819824,
-      "logps/chosen": -548.1529541015625,
-      "logps/rejected": -449.4532165527344,
-      "loss": 0.035,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -0.8807584643363953,
-      "rewards/margins": 2.355053663253784,
-      "rewards/rejected": -3.2358124256134033,
+      "epoch": 0.2,
+      "learning_rate": 4.839692218542131e-07,
+      "logits/chosen": -2.167600631713867,
+      "logits/rejected": -2.1524620056152344,
+      "logps/chosen": -445.18963623046875,
+      "logps/rejected": -420.07354736328125,
+      "loss": 0.4607,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.5789515972137451,
+      "rewards/margins": 0.9403783082962036,
+      "rewards/rejected": -2.5193300247192383,
       "step": 180
     },
     {
-      "epoch": 0.47,
-      "learning_rate": 3.202449097526798e-07,
-      "logits/chosen": -2.1954236030578613,
-      "logits/rejected": -2.113832950592041,
-      "logps/chosen": -545.7277221679688,
-      "logps/rejected": -466.76580810546875,
-      "loss": 0.029,
+      "epoch": 0.22,
+      "learning_rate": 4.803019108026997e-07,
+      "logits/chosen": -2.0659067630767822,
+      "logits/rejected": -2.0179924964904785,
+      "logps/chosen": -446.5098571777344,
+      "logps/rejected": -408.96685791015625,
+      "loss": 0.4605,
       "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.4056795835494995,
-      "rewards/margins": 2.1022555828094482,
-      "rewards/rejected": -3.5079357624053955,
+      "rewards/chosen": -1.1547313928604126,
+      "rewards/margins": 1.099097490310669,
+      "rewards/rejected": -2.253828525543213,
       "step": 190
     },
     {
-      "epoch": 0.49,
-      "learning_rate": 2.993015235369905e-07,
-      "logits/chosen": -2.1386027336120605,
-      "logits/rejected": -2.0572166442871094,
-      "logps/chosen": -560.2534790039062,
-      "logps/rejected": -491.8816833496094,
-      "loss": 0.0258,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.810624361038208,
-      "rewards/margins": 1.9691530466079712,
-      "rewards/rejected": -3.7797775268554688,
+      "epoch": 0.23,
+      "learning_rate": 4.7627410443782887e-07,
+      "logits/chosen": -1.9613704681396484,
+      "logits/rejected": -1.9336235523223877,
+      "logps/chosen": -434.38311767578125,
+      "logps/rejected": -421.72308349609375,
+      "loss": 0.4524,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.2233312129974365,
+      "rewards/margins": 1.0944594144821167,
+      "rewards/rejected": -2.3177905082702637,
       "step": 200
     },
     {
-      "epoch": 0.49,
-      "eval_logits/chosen": -2.1394448280334473,
-      "eval_logits/rejected": -2.1155476570129395,
-      "eval_logps/chosen": -543.8071899414062,
-      "eval_logps/rejected": -579.2755126953125,
-      "eval_loss": 0.0581156425178051,
-      "eval_rewards/accuracies": 0.59765625,
-      "eval_rewards/chosen": -2.86767578125,
-      "eval_rewards/margins": 0.35154610872268677,
-      "eval_rewards/rejected": -3.219222068786621,
-      "eval_runtime": 53.2701,
-      "eval_samples_per_second": 37.545,
-      "eval_steps_per_second": 0.601,
+      "epoch": 0.23,
+      "eval_logits/chosen": -1.9878398180007935,
+      "eval_logits/rejected": -1.9677612781524658,
+      "eval_logps/chosen": -405.3453674316406,
+      "eval_logps/rejected": -474.3326721191406,
+      "eval_loss": 0.5474696755409241,
+      "eval_rewards/accuracies": 0.72265625,
+      "eval_rewards/chosen": -1.4830571413040161,
+      "eval_rewards/margins": 0.6867368221282959,
+      "eval_rewards/rejected": -2.1697940826416016,
+      "eval_runtime": 53.0465,
+      "eval_samples_per_second": 37.703,
+      "eval_steps_per_second": 0.603,
       "step": 200
     },
     {
-      "epoch": 0.52,
-      "learning_rate": 2.7799111902582693e-07,
-      "logits/chosen": -2.1782305240631104,
-      "logits/rejected": -2.044674873352051,
-      "logps/chosen": -579.908935546875,
-      "logps/rejected": -500.6641540527344,
-      "loss": 0.0219,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -1.408044695854187,
-      "rewards/margins": 2.4992563724517822,
-      "rewards/rejected": -3.9073009490966797,
+      "epoch": 0.24,
+      "learning_rate": 4.7189210755018034e-07,
+      "logits/chosen": -1.916168212890625,
+      "logits/rejected": -1.849001169204712,
+      "logps/chosen": -497.56134033203125,
+      "logps/rejected": -451.7841796875,
+      "loss": 0.4423,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.2633593082427979,
+      "rewards/margins": 1.236897587776184,
+      "rewards/rejected": -2.5002567768096924,
       "step": 210
     },
     {
-      "epoch": 0.54,
-      "learning_rate": 2.564723385445869e-07,
-      "logits/chosen": -2.2589755058288574,
-      "logits/rejected": -2.156228542327881,
-      "logps/chosen": -563.1976318359375,
-      "logps/rejected": -475.75030517578125,
-      "loss": 0.038,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.3078866004943848,
-      "rewards/margins": 2.1681323051452637,
-      "rewards/rejected": -3.4760184288024902,
+      "epoch": 0.25,
+      "learning_rate": 4.671627793504988e-07,
+      "logits/chosen": -1.965778112411499,
+      "logits/rejected": -1.8829681873321533,
+      "logps/chosen": -516.19921875,
+      "logps/rejected": -489.0526428222656,
+      "loss": 0.4306,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.368606686592102,
+      "rewards/margins": 1.4575475454330444,
+      "rewards/rejected": -2.8261542320251465,
       "step": 220
     },
     {
-      "epoch": 0.57,
-      "learning_rate": 2.3490537564442845e-07,
-      "logits/chosen": -2.2288191318511963,
-      "logits/rejected": -2.136579751968384,
-      "logps/chosen": -507.54632568359375,
-      "logps/rejected": -419.88470458984375,
-      "loss": 0.0432,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.2000774145126343,
-      "rewards/margins": 1.7510545253753662,
-      "rewards/rejected": -2.951131820678711,
+      "epoch": 0.26,
+      "learning_rate": 4.6209352273286095e-07,
+      "logits/chosen": -1.8527837991714478,
+      "logits/rejected": -1.7781047821044922,
+      "logps/chosen": -492.2167053222656,
+      "logps/rejected": -515.4146728515625,
+      "loss": 0.4315,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.5558216571807861,
+      "rewards/margins": 1.2494769096374512,
+      "rewards/rejected": -2.8052985668182373,
       "step": 230
     },
     {
-      "epoch": 0.59,
-      "learning_rate": 2.1345078256378801e-07,
-      "logits/chosen": -2.282217264175415,
-      "logits/rejected": -2.1927459239959717,
-      "logps/chosen": -539.92822265625,
-      "logps/rejected": -433.8241271972656,
-      "loss": 0.0373,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.2784963846206665,
-      "rewards/margins": 1.8950881958007812,
-      "rewards/rejected": -3.1735846996307373,
+      "epoch": 0.27,
+      "learning_rate": 4.56692272686805e-07,
+      "logits/chosen": -1.8593418598175049,
+      "logits/rejected": -1.7763780355453491,
+      "logps/chosen": -473.20245361328125,
+      "logps/rejected": -463.26849365234375,
+      "loss": 0.4462,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.5299947261810303,
+      "rewards/margins": 1.291903018951416,
+      "rewards/rejected": -2.8218979835510254,
       "step": 240
     },
     {
-      "epoch": 0.62,
-      "learning_rate": 1.9226827501969865e-07,
-      "logits/chosen": -2.2803092002868652,
-      "logits/rejected": -2.1990160942077637,
-      "logps/chosen": -537.9136962890625,
-      "logps/rejected": -442.28350830078125,
-      "loss": 0.04,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -1.2320274114608765,
-      "rewards/margins": 2.1027939319610596,
-      "rewards/rejected": -3.3348212242126465,
+      "epoch": 0.28,
+      "learning_rate": 4.5096748387656326e-07,
+      "logits/chosen": -1.6604913473129272,
+      "logits/rejected": -1.5300872325897217,
+      "logps/chosen": -527.0318603515625,
+      "logps/rejected": -502.64129638671875,
+      "loss": 0.4618,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.062129497528076,
+      "rewards/margins": 1.1275193691253662,
+      "rewards/rejected": -3.1896486282348633,
       "step": 250
     },
     {
-      "epoch": 0.64,
-      "learning_rate": 1.715155432264775e-07,
-      "logits/chosen": -2.2646355628967285,
-      "logits/rejected": -2.14613676071167,
-      "logps/chosen": -502.49664306640625,
-      "logps/rejected": -420.11004638671875,
-      "loss": 0.0396,
+      "epoch": 0.29,
+      "learning_rate": 4.4492811740683877e-07,
+      "logits/chosen": -1.5592234134674072,
+      "logits/rejected": -1.3744081258773804,
+      "logps/chosen": -491.737548828125,
+      "logps/rejected": -486.6441345214844,
+      "loss": 0.4473,
       "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -1.1264328956604004,
-      "rewards/margins": 2.006878614425659,
-      "rewards/rejected": -3.1333117485046387,
+      "rewards/chosen": -1.907570481300354,
+      "rewards/margins": 1.1632691621780396,
+      "rewards/rejected": -3.0708391666412354,
       "step": 260
     },
     {
-      "epoch": 0.67,
-      "learning_rate": 1.51347077992983e-07,
-      "logits/chosen": -2.3088698387145996,
-      "logits/rejected": -2.2018628120422363,
-      "logps/chosen": -554.0256958007812,
-      "logps/rejected": -421.2101135253906,
-      "loss": 0.0375,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.10258948802948,
-      "rewards/margins": 1.9626919031143188,
-      "rewards/rejected": -3.065281391143799,
+      "epoch": 0.31,
+      "learning_rate": 4.3858362679584354e-07,
+      "logits/chosen": -1.5746996402740479,
+      "logits/rejected": -1.2380870580673218,
+      "logps/chosen": -457.90753173828125,
+      "logps/rejected": -446.56683349609375,
+      "loss": 0.4103,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -1.1260545253753662,
+      "rewards/margins": 1.6088136434555054,
+      "rewards/rejected": -2.734868288040161,
       "step": 270
     },
     {
-      "epoch": 0.69,
-      "learning_rate": 1.3191302063739906e-07,
-      "logits/chosen": -2.310133457183838,
-      "logits/rejected": -2.216827392578125,
-      "logps/chosen": -522.3606567382812,
-      "logps/rejected": -438.058349609375,
-      "loss": 0.043,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -1.1809624433517456,
-      "rewards/margins": 1.9291051626205444,
-      "rewards/rejected": -3.110067844390869,
+      "epoch": 0.32,
+      "learning_rate": 4.3194394317755245e-07,
+      "logits/chosen": -1.3573920726776123,
+      "logits/rejected": -1.0481122732162476,
+      "logps/chosen": -512.2153930664062,
+      "logps/rejected": -469.4527893066406,
+      "loss": 0.4381,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.6660184860229492,
+      "rewards/margins": 1.3182036876678467,
+      "rewards/rejected": -2.984222173690796,
       "step": 280
     },
     {
-      "epoch": 0.72,
-      "learning_rate": 1.1335804528119475e-07,
-      "logits/chosen": -2.3108785152435303,
-      "logits/rejected": -2.2141172885894775,
-      "logps/chosen": -544.7510986328125,
-      "logps/rejected": -427.60150146484375,
-      "loss": 0.044,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.00501549243927,
-      "rewards/margins": 2.146829605102539,
-      "rewards/rejected": -3.1518452167510986,
+      "epoch": 0.33,
+      "learning_rate": 4.2501945975633914e-07,
+      "logits/chosen": -1.5231261253356934,
+      "logits/rejected": -1.2471725940704346,
+      "logps/chosen": -508.29248046875,
+      "logps/rejected": -447.50689697265625,
+      "loss": 0.4364,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.4383156299591064,
+      "rewards/margins": 1.2735927104949951,
+      "rewards/rejected": -2.7119078636169434,
       "step": 290
     },
     {
-      "epoch": 0.74,
-      "learning_rate": 9.582028184286423e-08,
-      "logits/chosen": -2.350487470626831,
-      "logits/rejected": -2.307096481323242,
-      "logps/chosen": -554.42529296875,
-      "logps/rejected": -470.14434814453125,
-      "loss": 0.0402,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.1662867069244385,
-      "rewards/margins": 2.156501531600952,
-      "rewards/rejected": -3.3227882385253906,
+      "epoch": 0.34,
+      "learning_rate": 4.1782101553832405e-07,
+      "logits/chosen": -1.4166069030761719,
+      "logits/rejected": -1.1425375938415527,
+      "logps/chosen": -467.41717529296875,
+      "logps/rejected": -439.3959045410156,
+      "loss": 0.3976,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.5179073810577393,
+      "rewards/margins": 1.1478455066680908,
+      "rewards/rejected": -2.665753126144409,
       "step": 300
     },
     {
-      "epoch": 0.74,
-      "eval_logits/chosen": -2.2979543209075928,
-      "eval_logits/rejected": -2.2751243114471436,
-      "eval_logps/chosen": -467.0057067871094,
-      "eval_logps/rejected": -507.4114685058594,
-      "eval_loss": 0.08367828279733658,
-      "eval_rewards/accuracies": 0.62890625,
-      "eval_rewards/chosen": -2.099660634994507,
-      "eval_rewards/margins": 0.4009218215942383,
-      "eval_rewards/rejected": -2.500582218170166,
-      "eval_runtime": 53.3734,
-      "eval_samples_per_second": 37.472,
-      "eval_steps_per_second": 0.6,
+      "epoch": 0.34,
+      "eval_logits/chosen": -1.18406081199646,
+      "eval_logits/rejected": -0.9782991409301758,
+      "eval_logps/chosen": -442.4473571777344,
+      "eval_logps/rejected": -545.2501220703125,
+      "eval_loss": 0.5194380879402161,
+      "eval_rewards/accuracies": 0.76171875,
+      "eval_rewards/chosen": -1.8540773391723633,
+      "eval_rewards/margins": 1.0248912572860718,
+      "eval_rewards/rejected": -2.8789682388305664,
+      "eval_runtime": 53.0005,
+      "eval_samples_per_second": 37.736,
+      "eval_steps_per_second": 0.604,
       "step": 300
     },
     {
-      "epoch": 0.76,
-      "learning_rate": 7.943028774907065e-08,
-      "logits/chosen": -2.316253185272217,
-      "logits/rejected": -2.209606170654297,
-      "logps/chosen": -524.6145629882812,
-      "logps/rejected": -420.94671630859375,
-      "loss": 0.0324,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.0568145513534546,
-      "rewards/margins": 2.0644707679748535,
-      "rewards/rejected": -3.1212852001190186,
+      "epoch": 0.35,
+      "learning_rate": 4.103598783649029e-07,
+      "logits/chosen": -1.0781385898590088,
+      "logits/rejected": -0.6068095564842224,
+      "logps/chosen": -542.6256713867188,
+      "logps/rejected": -505.87078857421875,
+      "loss": 0.4248,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.779624342918396,
+      "rewards/margins": 1.5406283140182495,
+      "rewards/rejected": -3.3202528953552246,
       "step": 310
     },
     {
-      "epoch": 0.79,
-      "learning_rate": 6.431007601814637e-08,
-      "logits/chosen": -2.2733869552612305,
-      "logits/rejected": -2.169506549835205,
-      "logps/chosen": -532.5906982421875,
-      "logps/rejected": -450.932373046875,
-      "loss": 0.0316,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.5221502780914307,
-      "rewards/margins": 1.9199508428573608,
-      "rewards/rejected": -3.442101001739502,
+      "epoch": 0.36,
+      "learning_rate": 4.026477272750119e-07,
+      "logits/chosen": -0.7725287079811096,
+      "logits/rejected": -0.2756701111793518,
+      "logps/chosen": -545.5137329101562,
+      "logps/rejected": -528.4269409179688,
+      "loss": 0.4226,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -2.142789602279663,
+      "rewards/margins": 1.3576524257659912,
+      "rewards/rejected": -3.500441789627075,
       "step": 320
     },
     {
-      "epoch": 0.81,
-      "learning_rate": 5.0572206951246e-08,
-      "logits/chosen": -2.290539503097534,
-      "logits/rejected": -2.193920850753784,
-      "logps/chosen": -562.043701171875,
-      "logps/rejected": -479.5208435058594,
-      "loss": 0.0278,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.3767458200454712,
-      "rewards/margins": 2.1372973918914795,
-      "rewards/rejected": -3.514043092727661,
+      "epoch": 0.37,
+      "learning_rate": 3.9469663422373864e-07,
+      "logits/chosen": -0.9761560559272766,
+      "logits/rejected": -0.6311030983924866,
+      "logps/chosen": -517.2960205078125,
+      "logps/rejected": -506.86328125,
+      "loss": 0.4432,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.8108386993408203,
+      "rewards/margins": 1.37090265750885,
+      "rewards/rejected": -3.181741237640381,
       "step": 330
     },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.865190451858954e-07,
+      "logits/chosen": -0.865078330039978,
+      "logits/rejected": -0.3488244414329529,
+      "logps/chosen": -540.340087890625,
+      "logps/rejected": -525.5319213867188,
+      "loss": 0.43,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.597611904144287,
+      "rewards/margins": 1.6154896020889282,
+      "rewards/rejected": -3.213101625442505,
+      "step": 340
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.781277606741327e-07,
+      "logits/chosen": -1.0114878416061401,
+      "logits/rejected": -0.7175018191337585,
+      "logps/chosen": -450.4183654785156,
+      "logps/rejected": -459.7533264160156,
+      "loss": 0.4271,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.427380919456482,
+      "rewards/margins": 1.2692419290542603,
+      "rewards/rejected": -2.696622610092163,
+      "step": 350
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6953591570208996e-07,
+      "logits/chosen": -1.2963850498199463,
+      "logits/rejected": -0.8218928575515747,
+      "logps/chosen": -540.1664428710938,
+      "logps/rejected": -555.89306640625,
+      "loss": 0.4147,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.701040506362915,
+      "rewards/margins": 1.8265488147735596,
+      "rewards/rejected": -3.5275893211364746,
+      "step": 360
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.607569592239452e-07,
+      "logits/chosen": -1.0880775451660156,
+      "logits/rejected": -0.6546664237976074,
+      "logps/chosen": -559.7450561523438,
+      "logps/rejected": -538.1546630859375,
+      "loss": 0.4192,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.7286078929901123,
+      "rewards/margins": 1.7532542943954468,
+      "rewards/rejected": -3.4818618297576904,
+      "step": 370
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.518046330825494e-07,
+      "logits/chosen": -1.1186842918395996,
+      "logits/rejected": -0.6067591905593872,
+      "logps/chosen": -560.1696166992188,
+      "logps/rejected": -522.5840454101562,
+      "loss": 0.4349,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -1.8213374614715576,
+      "rewards/margins": 1.5133308172225952,
+      "rewards/rejected": -3.3346683979034424,
+      "step": 380
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4269295049909713e-07,
+      "logits/chosen": -1.1209189891815186,
+      "logits/rejected": -0.7713836431503296,
+      "logps/chosen": -473.28759765625,
+      "logps/rejected": -484.11065673828125,
+      "loss": 0.3979,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.7088820934295654,
+      "rewards/margins": 1.425378441810608,
+      "rewards/rejected": -3.1342601776123047,
+      "step": 390
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3343617413800453e-07,
+      "logits/chosen": -1.1869983673095703,
+      "logits/rejected": -0.6728812456130981,
+      "logps/chosen": -529.2347412109375,
+      "logps/rejected": -498.1748962402344,
+      "loss": 0.3892,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.649171233177185,
+      "rewards/margins": 1.6641887426376343,
+      "rewards/rejected": -3.3133597373962402,
+      "step": 400
+    },
+    {
+      "epoch": 0.45,
+      "eval_logits/chosen": -0.8579260110855103,
+      "eval_logits/rejected": -0.6001935601234436,
+      "eval_logps/chosen": -464.98876953125,
+      "eval_logps/rejected": -575.0087280273438,
+      "eval_loss": 0.5159767866134644,
+      "eval_rewards/accuracies": 0.77734375,
+      "eval_rewards/chosen": -2.079491376876831,
+      "eval_rewards/margins": 1.0970630645751953,
+      "eval_rewards/rejected": -3.1765542030334473,
+      "eval_runtime": 53.0852,
+      "eval_samples_per_second": 37.675,
+      "eval_steps_per_second": 0.603,
+      "step": 400
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2404879378132893e-07,
+      "logits/chosen": -0.8699030876159668,
+      "logits/rejected": -0.48875007033348083,
+      "logps/chosen": -468.9755859375,
+      "logps/rejected": -495.357421875,
+      "loss": 0.4084,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.6207454204559326,
+      "rewards/margins": 1.6607239246368408,
+      "rewards/rejected": -3.2814698219299316,
+      "step": 410
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1454550364767894e-07,
+      "logits/chosen": -1.098257303237915,
+      "logits/rejected": -0.709359347820282,
+      "logps/chosen": -512.3826904296875,
+      "logps/rejected": -523.95458984375,
+      "loss": 0.4354,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.7999498844146729,
+      "rewards/margins": 1.4373884201049805,
+      "rewards/rejected": -3.2373383045196533,
+      "step": 420
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.049411793911154e-07,
+      "logits/chosen": -0.9810858964920044,
+      "logits/rejected": -0.6282259821891785,
+      "logps/chosen": -509.84368896484375,
+      "logps/rejected": -517.60107421875,
+      "loss": 0.3974,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.8548433780670166,
+      "rewards/margins": 1.386967420578003,
+      "rewards/rejected": -3.2418110370635986,
+      "step": 430
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9525085481604914e-07,
+      "logits/chosen": -0.6511877775192261,
+      "logits/rejected": -0.07081355899572372,
+      "logps/chosen": -509.661376953125,
+      "logps/rejected": -524.3201904296875,
+      "loss": 0.4151,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.8540523052215576,
+      "rewards/margins": 1.5340583324432373,
+      "rewards/rejected": -3.388110399246216,
+      "step": 440
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.854896983445833e-07,
+      "logits/chosen": -0.5572197437286377,
+      "logits/rejected": 0.0708194151520729,
+      "logps/chosen": -562.8184814453125,
+      "logps/rejected": -528.6909790039062,
+      "loss": 0.4329,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.886749267578125,
+      "rewards/margins": 1.5286136865615845,
+      "rewards/rejected": -3.415362596511841,
+      "step": 450
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7567298927313654e-07,
+      "logits/chosen": -0.8817178606987,
+      "logits/rejected": -0.6781443357467651,
+      "logps/chosen": -470.3621520996094,
+      "logps/rejected": -495.33111572265625,
+      "loss": 0.4137,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.4939197301864624,
+      "rewards/margins": 1.3965364694595337,
+      "rewards/rejected": -2.890456199645996,
+      "step": 460
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.658160938555123e-07,
+      "logits/chosen": -0.900059700012207,
+      "logits/rejected": -0.38037875294685364,
+      "logps/chosen": -530.0759887695312,
+      "logps/rejected": -549.6453857421875,
+      "loss": 0.3727,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.6465427875518799,
+      "rewards/margins": 1.722328543663025,
+      "rewards/rejected": -3.3688716888427734,
+      "step": 470
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.559344412498532e-07,
+      "logits/chosen": -0.5834644436836243,
+      "logits/rejected": -0.024540895596146584,
+      "logps/chosen": -526.4287719726562,
+      "logps/rejected": -525.1407470703125,
+      "loss": 0.4301,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.8217204809188843,
+      "rewards/margins": 1.5024107694625854,
+      "rewards/rejected": -3.3241310119628906,
+      "step": 480
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.460434993671294e-07,
+      "logits/chosen": -0.999637246131897,
+      "logits/rejected": -0.7088354825973511,
+      "logps/chosen": -467.12353515625,
+      "logps/rejected": -472.5782775878906,
+      "loss": 0.3968,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.5620094537734985,
+      "rewards/margins": 1.4622641801834106,
+      "rewards/rejected": -3.02427339553833,
+      "step": 490
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.361587506589672e-07,
+      "logits/chosen": -1.169862151145935,
+      "logits/rejected": -0.6735583543777466,
+      "logps/chosen": -547.8793334960938,
+      "logps/rejected": -530.5782470703125,
+      "loss": 0.3964,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.728243112564087,
+      "rewards/margins": 1.6642783880233765,
+      "rewards/rejected": -3.392521381378174,
+      "step": 500
+    },
+    {
+      "epoch": 0.57,
+      "eval_logits/chosen": -1.018913984298706,
+      "eval_logits/rejected": -0.8011811375617981,
+      "eval_logps/chosen": -476.00384521484375,
+      "eval_logps/rejected": -588.1665649414062,
+      "eval_loss": 0.49919986724853516,
+      "eval_rewards/accuracies": 0.765625,
+      "eval_rewards/chosen": -2.1896419525146484,
+      "eval_rewards/margins": 1.11849045753479,
+      "eval_rewards/rejected": -3.3081324100494385,
+      "eval_runtime": 73.4341,
+      "eval_samples_per_second": 27.235,
+      "eval_steps_per_second": 0.436,
+      "step": 500
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2629566788271613e-07,
+      "logits/chosen": -1.1643812656402588,
+      "logits/rejected": -0.6770884394645691,
+      "logps/chosen": -498.718994140625,
+      "logps/rejected": -513.8646240234375,
+      "loss": 0.4072,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.7343899011611938,
+      "rewards/margins": 1.711033582687378,
+      "rewards/rejected": -3.4454236030578613,
+      "step": 510
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1646968988169135e-07,
+      "logits/chosen": -1.2519400119781494,
+      "logits/rejected": -0.7656970620155334,
+      "logps/chosen": -552.4429931640625,
+      "logps/rejected": -580.3065185546875,
+      "loss": 0.3859,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.9031795263290405,
+      "rewards/margins": 1.720510721206665,
+      "rewards/rejected": -3.623690366744995,
+      "step": 520
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0669619741850232e-07,
+      "logits/chosen": -1.166473388671875,
+      "logits/rejected": -0.5304248929023743,
+      "logps/chosen": -543.8204345703125,
+      "logps/rejected": -517.4561767578125,
+      "loss": 0.4265,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.9481357336044312,
+      "rewards/margins": 1.5012562274932861,
+      "rewards/rejected": -3.4493918418884277,
+      "step": 530
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9699048909929518e-07,
+      "logits/chosen": -1.3502863645553589,
+      "logits/rejected": -0.972245991230011,
+      "logps/chosen": -513.7689819335938,
+      "logps/rejected": -506.95953369140625,
+      "loss": 0.3917,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -1.8054568767547607,
+      "rewards/margins": 1.3885786533355713,
+      "rewards/rejected": -3.194035291671753,
+      "step": 540
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8736775742659732e-07,
+      "logits/chosen": -1.1914501190185547,
+      "logits/rejected": -0.8519012331962585,
+      "logps/chosen": -489.40234375,
+      "logps/rejected": -517.2066040039062,
+      "loss": 0.3892,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -1.6697397232055664,
+      "rewards/margins": 1.5511436462402344,
+      "rewards/rejected": -3.2208831310272217,
+      "step": 550
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7784306501824616e-07,
+      "logits/chosen": -1.1693607568740845,
+      "logits/rejected": -0.501569926738739,
+      "logps/chosen": -549.26220703125,
+      "logps/rejected": -523.9554443359375,
+      "loss": 0.4249,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.8511940240859985,
+      "rewards/margins": 1.4909955263137817,
+      "rewards/rejected": -3.342189311981201,
+      "step": 560
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6843132102963025e-07,
+      "logits/chosen": -1.1927831172943115,
+      "logits/rejected": -0.8532694578170776,
+      "logps/chosen": -539.3836669921875,
+      "logps/rejected": -510.72637939453125,
+      "loss": 0.3897,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.6676031351089478,
+      "rewards/margins": 1.5155996084213257,
+      "rewards/rejected": -3.1832027435302734,
+      "step": 570
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.591472578161458e-07,
+      "logits/chosen": -1.3109443187713623,
+      "logits/rejected": -0.9485646486282349,
+      "logps/chosen": -494.5043029785156,
+      "logps/rejected": -489.75537109375,
+      "loss": 0.4009,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.5513614416122437,
+      "rewards/margins": 1.510770559310913,
+      "rewards/rejected": -3.062131881713867,
+      "step": 580
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.5000540787240274e-07,
+      "logits/chosen": -1.2452589273452759,
+      "logits/rejected": -0.857632040977478,
+      "logps/chosen": -504.5924377441406,
+      "logps/rejected": -519.4934692382812,
+      "loss": 0.3993,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.7807199954986572,
+      "rewards/margins": 1.571396827697754,
+      "rewards/rejected": -3.352116823196411,
+      "step": 590
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.410200810842749e-07,
+      "logits/chosen": -1.2575485706329346,
+      "logits/rejected": -0.8479830622673035,
+      "logps/chosen": -503.79388427734375,
+      "logps/rejected": -516.9588012695312,
+      "loss": 0.4149,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.7127326726913452,
+      "rewards/margins": 1.6169878244400024,
+      "rewards/rejected": -3.3297202587127686,
+      "step": 600
+    },
+    {
+      "epoch": 0.68,
+      "eval_logits/chosen": -1.2397898435592651,
+      "eval_logits/rejected": -1.0526514053344727,
+      "eval_logps/chosen": -477.6524658203125,
+      "eval_logps/rejected": -589.7600708007812,
+      "eval_loss": 0.4948367774486542,
+      "eval_rewards/accuracies": 0.74609375,
+      "eval_rewards/chosen": -2.2061285972595215,
+      "eval_rewards/margins": 1.1179393529891968,
+      "eval_rewards/rejected": -3.3240678310394287,
+      "eval_runtime": 53.1159,
+      "eval_samples_per_second": 37.654,
+      "eval_steps_per_second": 0.602,
+      "step": 600
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.322053423294041e-07,
+      "logits/chosen": -1.256247639656067,
+      "logits/rejected": -0.9272082448005676,
+      "logps/chosen": -501.60675048828125,
+      "logps/rejected": -531.3302001953125,
+      "loss": 0.4028,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -1.7432218790054321,
+      "rewards/margins": 1.7136541604995728,
+      "rewards/rejected": -3.456876039505005,
+      "step": 610
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2357498946121905e-07,
+      "logits/chosen": -1.3026126623153687,
+      "logits/rejected": -0.9675828218460083,
+      "logps/chosen": -534.3182373046875,
+      "logps/rejected": -527.3935546875,
+      "loss": 0.4187,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.8490867614746094,
+      "rewards/margins": 1.5810914039611816,
+      "rewards/rejected": -3.430178165435791,
+      "step": 620
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1514253171093161e-07,
+      "logits/chosen": -1.2203739881515503,
+      "logits/rejected": -0.7822047472000122,
+      "logps/chosen": -493.72821044921875,
+      "logps/rejected": -506.88690185546875,
+      "loss": 0.4051,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.6152998208999634,
+      "rewards/margins": 1.5858867168426514,
+      "rewards/rejected": -3.201186418533325,
+      "step": 630
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0692116854131883e-07,
+      "logits/chosen": -1.000585913658142,
+      "logits/rejected": -0.7414053678512573,
+      "logps/chosen": -493.9928283691406,
+      "logps/rejected": -534.2704467773438,
+      "loss": 0.3866,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.7848412990570068,
+      "rewards/margins": 1.534330129623413,
+      "rewards/rejected": -3.319171905517578,
+      "step": 640
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.89237689853889e-08,
+      "logits/chosen": -0.9636529684066772,
+      "logits/rejected": -0.6193439364433289,
+      "logps/chosen": -499.71234130859375,
+      "logps/rejected": -517.5823974609375,
+      "loss": 0.394,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -1.829134225845337,
+      "rewards/margins": 1.6988353729248047,
+      "rewards/rejected": -3.5279693603515625,
+      "step": 650
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.11628515022765e-08,
+      "logits/chosen": -1.0789777040481567,
+      "logits/rejected": -0.6399408578872681,
+      "logps/chosen": -513.3380737304688,
+      "logps/rejected": -544.7978515625,
+      "loss": 0.3623,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.7524398565292358,
+      "rewards/margins": 1.8130983114242554,
+      "rewards/rejected": -3.565537929534912,
+      "step": 660
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.365056438189486e-08,
+      "logits/chosen": -1.0069674253463745,
+      "logits/rejected": -0.5994616746902466,
+      "logps/chosen": -542.05712890625,
+      "logps/rejected": -564.6227416992188,
+      "loss": 0.4122,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.9871015548706055,
+      "rewards/margins": 1.6281112432479858,
+      "rewards/rejected": -3.6152126789093018,
+      "step": 670
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.639866672902101e-08,
+      "logits/chosen": -1.0949068069458008,
+      "logits/rejected": -0.7090824246406555,
+      "logps/chosen": -549.8911743164062,
+      "logps/rejected": -559.15771484375,
+      "loss": 0.4132,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.8535239696502686,
+      "rewards/margins": 1.733758568763733,
+      "rewards/rejected": -3.587282657623291,
+      "step": 680
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.941851005657851e-08,
+      "logits/chosen": -1.1339385509490967,
+      "logits/rejected": -0.738599419593811,
+      "logps/chosen": -494.1913146972656,
+      "logps/rejected": -504.7791442871094,
+      "loss": 0.3813,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -1.752722978591919,
+      "rewards/margins": 1.4443397521972656,
+      "rewards/rejected": -3.1970624923706055,
+      "step": 690
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.272102051693051e-08,
+      "logits/chosen": -1.2199567556381226,
+      "logits/rejected": -0.9412355422973633,
+      "logps/chosen": -552.1275024414062,
+      "logps/rejected": -515.4296264648438,
+      "loss": 0.4004,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.7129371166229248,
+      "rewards/margins": 1.4609147310256958,
+      "rewards/rejected": -3.1738522052764893,
+      "step": 700
+    },
+    {
+      "epoch": 0.79,
+      "eval_logits/chosen": -1.0643391609191895,
+      "eval_logits/rejected": -0.8519161343574524,
+      "eval_logps/chosen": -474.26617431640625,
+      "eval_logps/rejected": -593.8731079101562,
+      "eval_loss": 0.49052032828330994,
+      "eval_rewards/accuracies": 0.76953125,
+      "eval_rewards/chosen": -2.1722652912139893,
+      "eval_rewards/margins": 1.192933201789856,
+      "eval_rewards/rejected": -3.3651983737945557,
+      "eval_runtime": 53.0717,
+      "eval_samples_per_second": 37.685,
+      "eval_steps_per_second": 0.603,
+      "step": 700
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.6316681798995844e-08,
+      "logits/chosen": -1.0180628299713135,
+      "logits/rejected": -0.7236673831939697,
+      "logps/chosen": -491.35565185546875,
+      "logps/rejected": -525.1397705078125,
+      "loss": 0.3851,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.7499040365219116,
+      "rewards/margins": 1.7332220077514648,
+      "rewards/rejected": -3.483126163482666,
+      "step": 710
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.0215518717961256e-08,
+      "logits/chosen": -1.0655405521392822,
+      "logits/rejected": -0.6608148813247681,
+      "logps/chosen": -525.560302734375,
+      "logps/rejected": -529.1053466796875,
+      "loss": 0.3984,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.8151214122772217,
+      "rewards/margins": 1.7203428745269775,
+      "rewards/rejected": -3.53546404838562,
+      "step": 720
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.4427081523275925e-08,
+      "logits/chosen": -1.0117073059082031,
+      "logits/rejected": -0.715721607208252,
+      "logps/chosen": -504.0994567871094,
+      "logps/rejected": -539.0814208984375,
+      "loss": 0.3756,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.8760732412338257,
+      "rewards/margins": 1.5654491186141968,
+      "rewards/rejected": -3.4415221214294434,
+      "step": 730
+    },
     {
       "epoch": 0.84,
-      "learning_rate": 3.831895019292897e-08,
-      "logits/chosen": -2.3263535499572754,
-      "logits/rejected": -2.207899570465088,
-      "logps/chosen": -619.2625122070312,
-      "logps/rejected": -520.6148071289062,
-      "loss": 0.0305,
+      "learning_rate": 3.896043094949061e-08,
+      "logits/chosen": -1.1520367860794067,
+      "logits/rejected": -0.5986729860305786,
+      "logps/chosen": -532.8388061523438,
+      "logps/rejected": -558.7303466796875,
+      "loss": 0.4003,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -1.8816320896148682,
+      "rewards/margins": 1.7765041589736938,
+      "rewards/rejected": -3.6581363677978516,
+      "step": 740
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.3824124033343557e-08,
+      "logits/chosen": -0.8991321325302124,
+      "logits/rejected": -0.6385317444801331,
+      "logps/chosen": -567.1549072265625,
+      "logps/rejected": -579.142578125,
+      "loss": 0.3994,
       "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.5666420459747314,
-      "rewards/margins": 2.300938844680786,
-      "rewards/rejected": -3.8675804138183594,
-      "step": 340
+      "rewards/chosen": -2.1505260467529297,
+      "rewards/margins": 1.6301181316375732,
+      "rewards/rejected": -3.780644178390503,
+      "step": 750
     },
     {
       "epoch": 0.86,
-      "learning_rate": 2.764152339909756e-08,
-      "logits/chosen": -2.305875539779663,
-      "logits/rejected": -2.1924188137054443,
-      "logps/chosen": -568.1319580078125,
-      "logps/rejected": -475.6539611816406,
-      "loss": 0.0245,
+      "learning_rate": 2.9026200719291904e-08,
+      "logits/chosen": -1.001379370689392,
+      "logits/rejected": -0.6102081537246704,
+      "logps/chosen": -508.70147705078125,
+      "logps/rejected": -536.8555297851562,
+      "loss": 0.4286,
       "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.341552972793579,
-      "rewards/margins": 2.3198726177215576,
-      "rewards/rejected": -3.661425828933716,
-      "step": 350
+      "rewards/chosen": -1.9450082778930664,
+      "rewards/margins": 1.5907418727874756,
+      "rewards/rejected": -3.535750150680542,
+      "step": 760
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.4574171274456433e-08,
+      "logits/chosen": -1.0912601947784424,
+      "logits/rejected": -0.6700750589370728,
+      "logps/chosen": -519.89208984375,
+      "logps/rejected": -525.0479736328125,
+      "loss": 0.3678,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.884690523147583,
+      "rewards/margins": 1.6182489395141602,
+      "rewards/rejected": -3.502938747406006,
+      "step": 770
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.047500453267881e-08,
+      "logits/chosen": -1.0198689699172974,
+      "logits/rejected": -0.5865429043769836,
+      "logps/chosen": -526.2073974609375,
+      "logps/rejected": -554.8323974609375,
+      "loss": 0.3887,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.9122520685195923,
+      "rewards/margins": 1.7002776861190796,
+      "rewards/rejected": -3.612529754638672,
+      "step": 780
     },
     {
       "epoch": 0.89,
-      "learning_rate": 1.861941317991664e-08,
-      "logits/chosen": -2.31453800201416,
-      "logits/rejected": -2.209552049636841,
-      "logps/chosen": -574.0198974609375,
-      "logps/rejected": -498.809326171875,
-      "loss": 0.0246,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -1.3858083486557007,
-      "rewards/margins": 2.422987699508667,
-      "rewards/rejected": -3.80879545211792,
-      "step": 360
+      "learning_rate": 1.673511698609292e-08,
+      "logits/chosen": -0.9797528982162476,
+      "logits/rejected": -0.5832753777503967,
+      "logps/chosen": -553.0879516601562,
+      "logps/rejected": -561.693603515625,
+      "loss": 0.3901,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.9004099369049072,
+      "rewards/margins": 1.7125294208526611,
+      "rewards/rejected": -3.6129393577575684,
+      "step": 790
     },
     {
       "epoch": 0.91,
-      "learning_rate": 1.13197833728636e-08,
-      "logits/chosen": -2.2876980304718018,
-      "logits/rejected": -2.1881823539733887,
-      "logps/chosen": -583.4609985351562,
-      "logps/rejected": -515.4216918945312,
-      "loss": 0.0274,
+      "learning_rate": 1.3360362741285769e-08,
+      "logits/chosen": -1.0027343034744263,
+      "logits/rejected": -0.7030217051506042,
+      "logps/chosen": -501.4288024902344,
+      "logps/rejected": -526.0103759765625,
+      "loss": 0.3887,
       "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.2713569402694702,
-      "rewards/margins": 2.548645496368408,
-      "rewards/rejected": -3.820002317428589,
-      "step": 370
+      "rewards/chosen": -1.869350790977478,
+      "rewards/margins": 1.5730822086334229,
+      "rewards/rejected": -3.4424331188201904,
+      "step": 800
+    },
+    {
+      "epoch": 0.91,
+      "eval_logits/chosen": -0.9597108364105225,
+      "eval_logits/rejected": -0.7242004871368408,
+      "eval_logps/chosen": -487.77545166015625,
+      "eval_logps/rejected": -609.3139038085938,
+      "eval_loss": 0.4919503927230835,
+      "eval_rewards/accuracies": 0.7734375,
+      "eval_rewards/chosen": -2.3073582649230957,
+      "eval_rewards/margins": 1.21224844455719,
+      "eval_rewards/rejected": -3.519606590270996,
+      "eval_runtime": 53.0285,
+      "eval_samples_per_second": 37.716,
+      "eval_steps_per_second": 0.603,
+      "step": 800
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.0356024355769433e-08,
+      "logits/chosen": -1.0092878341674805,
+      "logits/rejected": -0.7896069884300232,
+      "logps/chosen": -532.9703369140625,
+      "logps/rejected": -526.88330078125,
+      "loss": 0.3767,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.830877661705017,
+      "rewards/margins": 1.5391663312911987,
+      "rewards/rejected": -3.370044231414795,
+      "step": 810
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.726804569108597e-09,
+      "logits/chosen": -1.1117920875549316,
+      "logits/rejected": -0.6487603187561035,
+      "logps/chosen": -553.5621337890625,
+      "logps/rejected": -571.2651977539062,
+      "loss": 0.4191,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.9834985733032227,
+      "rewards/margins": 1.6336091756820679,
+      "rewards/rejected": -3.61710786819458,
+      "step": 820
     },
     {
       "epoch": 0.94,
-      "learning_rate": 5.79697505093521e-09,
-      "logits/chosen": -2.2938995361328125,
-      "logits/rejected": -2.161371946334839,
-      "logps/chosen": -567.2229614257812,
-      "logps/rejected": -493.6429138183594,
-      "loss": 0.0339,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -1.495012879371643,
-      "rewards/margins": 2.173337697982788,
-      "rewards/rejected": -3.6683506965637207,
-      "step": 380
+      "learning_rate": 5.476818941645561e-09,
+      "logits/chosen": -1.1343705654144287,
+      "logits/rejected": -0.5430102348327637,
+      "logps/chosen": -569.5715942382812,
+      "logps/rejected": -541.0032958984375,
+      "loss": 0.3755,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.8141847848892212,
+      "rewards/margins": 1.6208727359771729,
+      "rewards/rejected": -3.4350574016571045,
+      "step": 830
     },
     {
-      "epoch": 0.96,
-      "learning_rate": 2.092101988131256e-09,
-      "logits/chosen": -2.3137059211730957,
-      "logits/rejected": -2.1986048221588135,
-      "logps/chosen": -600.1227416992188,
-      "logps/rejected": -496.6559143066406,
-      "loss": 0.0258,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -1.368238925933838,
-      "rewards/margins": 2.388805389404297,
-      "rewards/rejected": -3.7570443153381348,
-      "step": 390
+      "epoch": 0.95,
+      "learning_rate": 3.609589412347347e-09,
+      "logits/chosen": -1.0426546335220337,
+      "logits/rejected": -0.6130795478820801,
+      "logps/chosen": -517.9793701171875,
+      "logps/rejected": -554.4876098632812,
+      "loss": 0.3741,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.7485253810882568,
+      "rewards/margins": 1.8532413244247437,
+      "rewards/rejected": -3.601766586303711,
+      "step": 840
     },
     {
-      "epoch": 0.99,
-      "learning_rate": 2.327445937151673e-10,
-      "logits/chosen": -2.316849708557129,
-      "logits/rejected": -2.1959729194641113,
-      "logps/chosen": -559.0263061523438,
-      "logps/rejected": -482.11773681640625,
-      "loss": 0.0288,
+      "epoch": 0.96,
+      "learning_rate": 2.1280387858572667e-09,
+      "logits/chosen": -0.9971386194229126,
+      "logits/rejected": -0.6877419352531433,
+      "logps/chosen": -496.14208984375,
+      "logps/rejected": -510.2310485839844,
+      "loss": 0.3879,
       "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.659148931503296,
-      "rewards/margins": 2.0086750984191895,
-      "rewards/rejected": -3.6678237915039062,
-      "step": 400
+      "rewards/chosen": -1.8203623294830322,
+      "rewards/margins": 1.555535078048706,
+      "rewards/rejected": -3.3758976459503174,
+      "step": 850
     },
     {
-      "epoch": 0.99,
-      "eval_logits/chosen": -2.2708253860473633,
-      "eval_logits/rejected": -2.245922565460205,
-      "eval_logps/chosen": -513.0960083007812,
-      "eval_logps/rejected": -552.8470458984375,
-      "eval_loss": 0.06599809229373932,
-      "eval_rewards/accuracies": 0.625,
-      "eval_rewards/chosen": -2.560563564300537,
-      "eval_rewards/margins": 0.3943747282028198,
-      "eval_rewards/rejected": -2.9549384117126465,
-      "eval_runtime": 53.3482,
-      "eval_samples_per_second": 37.49,
-      "eval_steps_per_second": 0.6,
-      "step": 400
+      "epoch": 0.97,
+      "learning_rate": 1.03448615738172e-09,
+      "logits/chosen": -1.004620909690857,
+      "logits/rejected": -0.6449930667877197,
+      "logps/chosen": -515.4451904296875,
+      "logps/rejected": -543.5704956054688,
+      "loss": 0.3946,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.7996273040771484,
+      "rewards/margins": 1.7972816228866577,
+      "rewards/rejected": -3.5969085693359375,
+      "step": 860
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 3.3064328257259575e-10,
+      "logits/chosen": -1.0538240671157837,
+      "logits/rejected": -0.67207270860672,
+      "logps/chosen": -510.56414794921875,
+      "logps/rejected": -538.0349731445312,
+      "loss": 0.3821,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.884305715560913,
+      "rewards/margins": 1.7009254693984985,
+      "rewards/rejected": -3.585231065750122,
+      "step": 870
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.7611898088715216e-11,
+      "logits/chosen": -1.063408613204956,
+      "logits/rejected": -0.8093023300170898,
+      "logps/chosen": -536.09716796875,
+      "logps/rejected": -549.3472290039062,
+      "loss": 0.4077,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.7760584354400635,
+      "rewards/margins": 1.5979098081588745,
+      "rewards/rejected": -3.3739686012268066,
+      "step": 880
     },
     {
       "epoch": 1.0,
-      "step": 405,
+      "step": 883,
       "total_flos": 0.0,
-      "train_loss": 0.07151281171374851,
-      "train_runtime": 3738.25,
-      "train_samples_per_second": 13.882,
-      "train_steps_per_second": 0.108
+      "train_loss": 0.43981607611020046,
+      "train_runtime": 8273.4147,
+      "train_samples_per_second": 13.662,
+      "train_steps_per_second": 0.107
     }
   ],
   "logging_steps": 10,
-  "max_steps": 405,
+  "max_steps": 883,
   "num_train_epochs": 1,
   "save_steps": 100,
   "total_flos": 0.0,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8c44f53c9792f03eceacb06a2800acd74827f6b7f87a069e588eb192870cb597
+oid sha256:5b78f99d6c1a4671754549b6b5e97ff7d0343ff96a70b1d1ddf812b2a5bc88bf
 size 5944