Model save

README.md

@@ -15,15 +15,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.0417
-- Rewards/chosen: -1.1011
-- Rewards/rejected: -1.9879
-- Rewards/accuracies: 0.7266
-- Rewards/margins: 0.8868
-- Logps/rejected: -528.9260
-- Logps/chosen: -433.9420
-- Logits/rejected: 0.8669
-- Logits/chosen: 0.7479
+- Loss: 0.0415
+- Rewards/chosen: -1.1176
+- Rewards/rejected: -2.0114
+- Rewards/accuracies: 0.7070
+- Rewards/margins: 0.8938
+- Logps/rejected: -531.2747
+- Logps/chosen: -435.5875
+- Logits/rejected: 0.8196
+- Logits/chosen: 0.7291
 
 ## Model description
 
@@ -45,7 +45,7 @@ The following hyperparameters were used during training:
 - learning_rate: 3e-06
 - train_batch_size: 4
 - eval_batch_size: 8
-- seed: 4
+- seed: 5
 - distributed_type: multi-GPU
 - num_devices: 8
 - gradient_accumulation_steps: 4
@@ -60,10 +60,10 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.0521        | 0.21  | 100  | 0.0521          | -0.8681        | -1.4945          | 0.7109             | 0.6264          | -479.5833      | -410.6368    | 0.9218          | 0.7855        |
-| 0.0475        | 0.42  | 200  | 0.0601          | -0.7216        | -1.4421          | 0.7383             | 0.7205          | -474.3407      | -395.9902    | 0.5575          | 0.4386        |
-| 0.0476        | 0.63  | 300  | 0.0584          | -0.8286        | -1.5875          | 0.7227             | 0.7589          | -488.8869      | -406.6899    | 0.5327          | 0.4048        |
-| 0.0392        | 0.84  | 400  | 0.0417          | -1.1011        | -1.9879          | 0.7266             | 0.8868          | -528.9260      | -433.9420    | 0.8669          | 0.7479        |
+| 0.066         | 0.21  | 100  | 0.0702          | -0.4714        | -1.0800          | 0.7266             | 0.6086          | -438.1371      | -370.9747    | 0.7687          | 0.6183        |
+| 0.0477        | 0.42  | 200  | 0.0505          | -1.0382        | -1.8566          | 0.7461             | 0.8184          | -515.7967      | -427.6501    | 0.5198          | 0.4181        |
+| 0.0313        | 0.63  | 300  | 0.0344          | -1.3029        | -2.2224          | 0.7227             | 0.9195          | -552.3698      | -454.1193    | 1.0434          | 0.9401        |
+| 0.0359        | 0.84  | 400  | 0.0415          | -1.1176        | -2.0114          | 0.7070             | 0.8938          | -531.2747      | -435.5875    | 0.8196          | 0.7291        |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.05423464074699634,
-    "train_runtime": 4545.6697,
+    "train_loss": 0.055112330793584664,
+    "train_runtime": 4571.3444,
     "train_samples": 61134,
-    "train_samples_per_second": 13.449,
-    "train_steps_per_second": 0.105
+    "train_samples_per_second": 13.373,
+    "train_steps_per_second": 0.104
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bba28a181d97753031ca76691f7e70603aeddef1cd3970f974728b30188d52fe
+oid sha256:77c78f44ae927b8c5f876cba766716862c391ff327d777f630df2273dc608ad2
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a3634a8c506846675a8a7101c40445f227bb7af96b151c0ebfe45b1497d3a7ac
+oid sha256:52890ec8e3b01c2a425c75a5fe8026fad3760550ffe4ecc542adabcb6547e556
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:97c9de524013be6975b5843fe7fd0bde5a216581cbe0bb10c068b4dc17cffc0b
+oid sha256:7c4e28b526b64115f67f1a7d9ceb1156546b14ddfbf6c799c751ac2c949af93b
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c12a0bb4b9560bc67876bae213267e33acdf73e22432c92dd70443ce0038244e
+oid sha256:331daeef21c9b60a293872df524529661446efaf2f056cc336b124cce438e3cb
 size 1168138808

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.05423464074699634,
-    "train_runtime": 4545.6697,
+    "train_loss": 0.055112330793584664,
+    "train_runtime": 4571.3444,
     "train_samples": 61134,
-    "train_samples_per_second": 13.449,
-    "train_steps_per_second": 0.105
+    "train_samples_per_second": 13.373,
+    "train_steps_per_second": 0.104
 }

trainer_state.json

@@ -11,11 +11,11 @@
     {
       "epoch": 0.0,
       "learning_rate": 6.25e-08,
-      "logits/chosen": 0.09203790873289108,
-      "logits/rejected": 0.2914758026599884,
-      "logps/chosen": -315.45611572265625,
-      "logps/rejected": -241.00250244140625,
-      "loss": 0.1409,
+      "logits/chosen": 0.10802720487117767,
+      "logits/rejected": 0.30745893716812134,
+      "logps/chosen": -475.5745544433594,
+      "logps/rejected": -317.21234130859375,
+      "loss": 0.1378,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,733 +25,733 @@
     {
       "epoch": 0.02,
       "learning_rate": 6.25e-07,
-      "logits/chosen": 0.2606273889541626,
-      "logits/rejected": 0.33430540561676025,
-      "logps/chosen": -323.0765686035156,
-      "logps/rejected": -307.2400817871094,
-      "loss": 0.1441,
-      "rewards/accuracies": 0.4236111044883728,
-      "rewards/chosen": -0.0016678691608831286,
-      "rewards/margins": -7.814847049303353e-05,
-      "rewards/rejected": -0.0015897207194939256,
+      "logits/chosen": 0.21480141580104828,
+      "logits/rejected": 0.3137889802455902,
+      "logps/chosen": -308.09619140625,
+      "logps/rejected": -308.85736083984375,
+      "loss": 0.1432,
+      "rewards/accuracies": 0.4791666567325592,
+      "rewards/chosen": -0.0008134886738844216,
+      "rewards/margins": 0.0006454013055190444,
+      "rewards/rejected": -0.001458889921195805,
       "step": 10
     },
     {
       "epoch": 0.04,
       "learning_rate": 1.25e-06,
-      "logits/chosen": 0.3121325671672821,
-      "logits/rejected": 0.33961328864097595,
-      "logps/chosen": -304.62060546875,
-      "logps/rejected": -280.9560546875,
-      "loss": 0.1425,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.0012181587517261505,
-      "rewards/margins": 0.008560886606574059,
-      "rewards/rejected": -0.007342727389186621,
+      "logits/chosen": 0.249754399061203,
+      "logits/rejected": 0.2825905978679657,
+      "logps/chosen": -304.75286865234375,
+      "logps/rejected": -317.61688232421875,
+      "loss": 0.1418,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.0010095896432176232,
+      "rewards/margins": 0.010475357994437218,
+      "rewards/rejected": -0.009465768001973629,
       "step": 20
     },
     {
       "epoch": 0.06,
       "learning_rate": 1.875e-06,
-      "logits/chosen": 0.21220548450946808,
-      "logits/rejected": 0.3127239942550659,
-      "logps/chosen": -373.3235778808594,
-      "logps/rejected": -329.6069030761719,
-      "loss": 0.1491,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.031112518161535263,
-      "rewards/margins": 0.08273597061634064,
-      "rewards/rejected": -0.05162344500422478,
+      "logits/chosen": 0.24968624114990234,
+      "logits/rejected": 0.2685222029685974,
+      "logps/chosen": -366.27813720703125,
+      "logps/rejected": -365.3521728515625,
+      "loss": 0.1431,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.014242827892303467,
+      "rewards/margins": 0.06069143861532211,
+      "rewards/rejected": -0.046448610723018646,
       "step": 30
     },
     {
       "epoch": 0.08,
       "learning_rate": 2.5e-06,
-      "logits/chosen": 0.3977668881416321,
-      "logits/rejected": 0.4907824397087097,
-      "logps/chosen": -347.14422607421875,
-      "logps/rejected": -337.64599609375,
-      "loss": 0.1246,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.08021022379398346,
-      "rewards/margins": 0.17033424973487854,
-      "rewards/rejected": -0.2505444586277008,
+      "logits/chosen": 0.5138859748840332,
+      "logits/rejected": 0.6031057238578796,
+      "logps/chosen": -333.85650634765625,
+      "logps/rejected": -331.0009765625,
+      "loss": 0.1181,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.17486190795898438,
+      "rewards/margins": 0.1082921177148819,
+      "rewards/rejected": -0.2831540107727051,
       "step": 40
     },
     {
       "epoch": 0.1,
       "learning_rate": 2.999839121261416e-06,
-      "logits/chosen": 0.7560127973556519,
-      "logits/rejected": 0.8282445073127747,
-      "logps/chosen": -400.455322265625,
-      "logps/rejected": -433.1502380371094,
-      "loss": 0.0829,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.5089501142501831,
-      "rewards/margins": 0.3683263659477234,
-      "rewards/rejected": -0.8772764205932617,
+      "logits/chosen": 0.7348484992980957,
+      "logits/rejected": 0.8855365514755249,
+      "logps/chosen": -370.4933776855469,
+      "logps/rejected": -411.83404541015625,
+      "loss": 0.0741,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.4408305287361145,
+      "rewards/margins": 0.4697234034538269,
+      "rewards/rejected": -0.9105539321899414,
       "step": 50
     },
     {
       "epoch": 0.13,
       "learning_rate": 2.994211988057582e-06,
-      "logits/chosen": 0.7812812328338623,
-      "logits/rejected": 1.0137856006622314,
-      "logps/chosen": -386.59674072265625,
-      "logps/rejected": -442.959716796875,
-      "loss": 0.0611,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.805428683757782,
-      "rewards/margins": 0.5730851292610168,
-      "rewards/rejected": -1.3785139322280884,
+      "logits/chosen": 0.7168207764625549,
+      "logits/rejected": 0.8200086355209351,
+      "logps/chosen": -341.53277587890625,
+      "logps/rejected": -380.68243408203125,
+      "loss": 0.0819,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.4272558093070984,
+      "rewards/margins": 0.4549214839935303,
+      "rewards/rejected": -0.8821773529052734,
       "step": 60
     },
     {
       "epoch": 0.15,
       "learning_rate": 2.9805753939568693e-06,
-      "logits/chosen": 0.810570240020752,
-      "logits/rejected": 0.9147614240646362,
-      "logps/chosen": -360.248291015625,
-      "logps/rejected": -423.7164611816406,
-      "loss": 0.0539,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.7509704828262329,
-      "rewards/margins": 0.4445374608039856,
-      "rewards/rejected": -1.1955080032348633,
+      "logits/chosen": 0.5615164041519165,
+      "logits/rejected": 0.7741672396659851,
+      "logps/chosen": -347.7218017578125,
+      "logps/rejected": -330.172607421875,
+      "loss": 0.0929,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.3559855818748474,
+      "rewards/margins": 0.2854944169521332,
+      "rewards/rejected": -0.6414799690246582,
       "step": 70
     },
     {
       "epoch": 0.17,
       "learning_rate": 2.959002435526626e-06,
-      "logits/chosen": 0.46960344910621643,
-      "logits/rejected": 0.6186197996139526,
-      "logps/chosen": -387.56683349609375,
-      "logps/rejected": -440.8048400878906,
-      "loss": 0.058,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.7455588579177856,
-      "rewards/margins": 0.6929988265037537,
-      "rewards/rejected": -1.438557744026184,
+      "logits/chosen": 0.5198915004730225,
+      "logits/rejected": 0.725387454032898,
+      "logps/chosen": -389.0698547363281,
+      "logps/rejected": -371.3795471191406,
+      "loss": 0.0736,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.4846402108669281,
+      "rewards/margins": 0.395100474357605,
+      "rewards/rejected": -0.8797407150268555,
       "step": 80
     },
     {
       "epoch": 0.19,
       "learning_rate": 2.929608750821129e-06,
-      "logits/chosen": 0.49714404344558716,
-      "logits/rejected": 0.7643166780471802,
-      "logps/chosen": -387.7370300292969,
-      "logps/rejected": -411.63250732421875,
-      "loss": 0.051,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.8395982980728149,
-      "rewards/margins": 0.5891604423522949,
-      "rewards/rejected": -1.4287587404251099,
+      "logits/chosen": 0.3736918568611145,
+      "logits/rejected": 0.5658319592475891,
+      "logps/chosen": -444.59234619140625,
+      "logps/rejected": -464.6935119628906,
+      "loss": 0.0491,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.8689848184585571,
+      "rewards/margins": 0.6035453081130981,
+      "rewards/rejected": -1.4725301265716553,
       "step": 90
     },
     {
       "epoch": 0.21,
       "learning_rate": 2.892551899524109e-06,
-      "logits/chosen": 0.6043367981910706,
-      "logits/rejected": 0.8246415853500366,
-      "logps/chosen": -417.3805236816406,
-      "logps/rejected": -413.0718688964844,
-      "loss": 0.0521,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.8979790806770325,
-      "rewards/margins": 0.3839009404182434,
-      "rewards/rejected": -1.2818800210952759,
+      "logits/chosen": 0.3380030393600464,
+      "logits/rejected": 0.443446546792984,
+      "logps/chosen": -408.71551513671875,
+      "logps/rejected": -431.513671875,
+      "loss": 0.066,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.8214343786239624,
+      "rewards/margins": 0.3987075388431549,
+      "rewards/rejected": -1.2201420068740845,
       "step": 100
     },
     {
       "epoch": 0.21,
-      "eval_logits/chosen": 0.7855331897735596,
-      "eval_logits/rejected": 0.9217743873596191,
-      "eval_logps/chosen": -410.63677978515625,
-      "eval_logps/rejected": -479.5832824707031,
-      "eval_loss": 0.05205187946557999,
-      "eval_rewards/accuracies": 0.7109375,
-      "eval_rewards/chosen": -0.8680679798126221,
-      "eval_rewards/margins": 0.6264181137084961,
-      "eval_rewards/rejected": -1.4944860935211182,
-      "eval_runtime": 73.9092,
-      "eval_samples_per_second": 27.06,
-      "eval_steps_per_second": 0.433,
+      "eval_logits/chosen": 0.6183323860168457,
+      "eval_logits/rejected": 0.7686768174171448,
+      "eval_logps/chosen": -370.9747009277344,
+      "eval_logps/rejected": -438.13714599609375,
+      "eval_loss": 0.07016688585281372,
+      "eval_rewards/accuracies": 0.7265625,
+      "eval_rewards/chosen": -0.47144782543182373,
+      "eval_rewards/margins": 0.6085766553878784,
+      "eval_rewards/rejected": -1.0800243616104126,
+      "eval_runtime": 74.3034,
+      "eval_samples_per_second": 26.917,
+      "eval_steps_per_second": 0.431,
       "step": 100
     },
     {
       "epoch": 0.23,
       "learning_rate": 2.848030518377739e-06,
-      "logits/chosen": 0.4907767176628113,
-      "logits/rejected": 0.6656876802444458,
-      "logps/chosen": -397.6432189941406,
-      "logps/rejected": -422.59735107421875,
-      "loss": 0.0651,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.6896931529045105,
-      "rewards/margins": 0.4766135811805725,
-      "rewards/rejected": -1.166306734085083,
+      "logits/chosen": 0.48754867911338806,
+      "logits/rejected": 0.6056569814682007,
+      "logps/chosen": -394.04449462890625,
+      "logps/rejected": -424.449951171875,
+      "loss": 0.06,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.6658821105957031,
+      "rewards/margins": 0.43674975633621216,
+      "rewards/rejected": -1.1026318073272705,
       "step": 110
     },
     {
       "epoch": 0.25,
       "learning_rate": 2.7962832564252724e-06,
-      "logits/chosen": 0.39288032054901123,
-      "logits/rejected": 0.555514931678772,
-      "logps/chosen": -435.5804138183594,
-      "logps/rejected": -464.86651611328125,
-      "loss": 0.0483,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.8776789903640747,
-      "rewards/margins": 0.542883038520813,
-      "rewards/rejected": -1.4205620288848877,
+      "logits/chosen": 0.5436107516288757,
+      "logits/rejected": 0.6737319231033325,
+      "logps/chosen": -429.415283203125,
+      "logps/rejected": -469.0088806152344,
+      "loss": 0.0627,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.7700729370117188,
+      "rewards/margins": 0.48356789350509644,
+      "rewards/rejected": -1.2536407709121704,
       "step": 120
     },
     {
       "epoch": 0.27,
       "learning_rate": 2.7375874957747644e-06,
-      "logits/chosen": 0.7341902256011963,
-      "logits/rejected": 0.9476861953735352,
-      "logps/chosen": -487.042236328125,
-      "logps/rejected": -509.67510986328125,
-      "loss": 0.0303,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.6147487163543701,
-      "rewards/margins": 0.5281225442886353,
-      "rewards/rejected": -2.142871379852295,
+      "logits/chosen": 0.5728715062141418,
+      "logits/rejected": 0.7463508248329163,
+      "logps/chosen": -441.0868225097656,
+      "logps/rejected": -454.98748779296875,
+      "loss": 0.0621,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.8495699763298035,
+      "rewards/margins": 0.5289269685745239,
+      "rewards/rejected": -1.3784968852996826,
       "step": 130
     },
     {
       "epoch": 0.29,
       "learning_rate": 2.672257864741005e-06,
-      "logits/chosen": 0.7316943407058716,
-      "logits/rejected": 0.9424102902412415,
-      "logps/chosen": -451.3330078125,
-      "logps/rejected": -464.85601806640625,
-      "loss": 0.0309,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.3325563669204712,
-      "rewards/margins": 0.5245406031608582,
-      "rewards/rejected": -1.8570966720581055,
+      "logits/chosen": 0.6253047585487366,
+      "logits/rejected": 0.786455512046814,
+      "logps/chosen": -433.4244079589844,
+      "logps/rejected": -461.5254821777344,
+      "loss": 0.0435,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.083187460899353,
+      "rewards/margins": 0.5303990840911865,
+      "rewards/rejected": -1.61358642578125,
       "step": 140
     },
     {
       "epoch": 0.31,
       "learning_rate": 2.600644551335706e-06,
-      "logits/chosen": 0.6526888608932495,
-      "logits/rejected": 0.8180145025253296,
-      "logps/chosen": -443.1895446777344,
-      "logps/rejected": -462.8304748535156,
-      "loss": 0.0465,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.9286714792251587,
-      "rewards/margins": 0.5423834919929504,
-      "rewards/rejected": -1.471055030822754,
+      "logits/chosen": 0.7765518426895142,
+      "logits/rejected": 0.984174907207489,
+      "logps/chosen": -419.31109619140625,
+      "logps/rejected": -431.96795654296875,
+      "loss": 0.0444,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.0778591632843018,
+      "rewards/margins": 0.4318017363548279,
+      "rewards/rejected": -1.5096609592437744,
       "step": 150
     },
     {
       "epoch": 0.33,
       "learning_rate": 2.5231314261461732e-06,
-      "logits/chosen": 0.5461139678955078,
-      "logits/rejected": 0.7980540990829468,
-      "logps/chosen": -441.15704345703125,
-      "logps/rejected": -453.66033935546875,
-      "loss": 0.0528,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.9410704374313354,
-      "rewards/margins": 0.6425323486328125,
-      "rewards/rejected": -1.5836029052734375,
+      "logits/chosen": 0.513221025466919,
+      "logits/rejected": 0.7459092140197754,
+      "logps/chosen": -418.07421875,
+      "logps/rejected": -463.25408935546875,
+      "loss": 0.0586,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.6770002245903015,
+      "rewards/margins": 0.5035561323165894,
+      "rewards/rejected": -1.1805565357208252,
       "step": 160
     },
     {
       "epoch": 0.36,
       "learning_rate": 2.440133984664454e-06,
-      "logits/chosen": 0.315818727016449,
-      "logits/rejected": 0.576252281665802,
-      "logps/chosen": -468.5994567871094,
-      "logps/rejected": -481.32818603515625,
-      "loss": 0.0531,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.8597052693367004,
-      "rewards/margins": 0.47268643975257874,
-      "rewards/rejected": -1.3323917388916016,
+      "logits/chosen": 0.5670315027236938,
+      "logits/rejected": 0.8073333501815796,
+      "logps/chosen": -390.5821228027344,
+      "logps/rejected": -419.92626953125,
+      "loss": 0.0562,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.8130921125411987,
+      "rewards/margins": 0.4765067994594574,
+      "rewards/rejected": -1.289598822593689,
       "step": 170
     },
     {
       "epoch": 0.38,
       "learning_rate": 2.3520971200967337e-06,
-      "logits/chosen": 0.44164711236953735,
-      "logits/rejected": 0.6109380125999451,
-      "logps/chosen": -452.79473876953125,
-      "logps/rejected": -522.9495849609375,
-      "loss": 0.04,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -1.0779365301132202,
-      "rewards/margins": 0.8940714597702026,
-      "rewards/rejected": -1.9720081090927124,
+      "logits/chosen": 0.39020082354545593,
+      "logits/rejected": 0.4927116334438324,
+      "logps/chosen": -379.1041259765625,
+      "logps/rejected": -440.0082092285156,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.8111687898635864,
+      "rewards/margins": 0.5016359090805054,
+      "rewards/rejected": -1.3128045797348022,
       "step": 180
     },
     {
       "epoch": 0.4,
       "learning_rate": 2.2594927385914546e-06,
-      "logits/chosen": 0.341614305973053,
-      "logits/rejected": 0.5234003067016602,
-      "logps/chosen": -462.4978942871094,
-      "logps/rejected": -528.377685546875,
-      "loss": 0.0362,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.309615135192871,
-      "rewards/margins": 0.7517553567886353,
-      "rewards/rejected": -2.061370372772217,
+      "logits/chosen": 0.32924190163612366,
+      "logits/rejected": 0.46087831258773804,
+      "logps/chosen": -382.1633605957031,
+      "logps/rejected": -444.0999450683594,
+      "loss": 0.0495,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.7652384042739868,
+      "rewards/margins": 0.6649683117866516,
+      "rewards/rejected": -1.4302066564559937,
       "step": 190
     },
     {
       "epoch": 0.42,
       "learning_rate": 2.1628172296692954e-06,
-      "logits/chosen": 0.40094342827796936,
-      "logits/rejected": 0.5750107169151306,
-      "logps/chosen": -412.12445068359375,
-      "logps/rejected": -439.3114318847656,
-      "loss": 0.0475,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.915791392326355,
-      "rewards/margins": 0.6095176935195923,
-      "rewards/rejected": -1.5253090858459473,
+      "logits/chosen": 0.21413707733154297,
+      "logits/rejected": 0.302509069442749,
+      "logps/chosen": -465.3833923339844,
+      "logps/rejected": -511.8447265625,
+      "loss": 0.0477,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.172499179840088,
+      "rewards/margins": 0.573866069316864,
+      "rewards/rejected": -1.7463653087615967,
       "step": 200
     },
     {
       "epoch": 0.42,
-      "eval_logits/chosen": 0.4385632872581482,
-      "eval_logits/rejected": 0.5574513077735901,
-      "eval_logps/chosen": -395.9902038574219,
-      "eval_logps/rejected": -474.3407287597656,
-      "eval_loss": 0.06009303405880928,
-      "eval_rewards/accuracies": 0.73828125,
-      "eval_rewards/chosen": -0.7216026782989502,
-      "eval_rewards/margins": 0.7204576134681702,
-      "eval_rewards/rejected": -1.442060112953186,
-      "eval_runtime": 73.4554,
-      "eval_samples_per_second": 27.227,
-      "eval_steps_per_second": 0.436,
+      "eval_logits/chosen": 0.41806796193122864,
+      "eval_logits/rejected": 0.5197638273239136,
+      "eval_logps/chosen": -427.650146484375,
+      "eval_logps/rejected": -515.7966918945312,
+      "eval_loss": 0.050458863377571106,
+      "eval_rewards/accuracies": 0.74609375,
+      "eval_rewards/chosen": -1.038202166557312,
+      "eval_rewards/margins": 0.8184179663658142,
+      "eval_rewards/rejected": -1.856619954109192,
+      "eval_runtime": 75.1858,
+      "eval_samples_per_second": 26.601,
+      "eval_steps_per_second": 0.426,
       "step": 200
     },
     {
       "epoch": 0.44,
       "learning_rate": 2.062588805414343e-06,
-      "logits/chosen": 0.45482128858566284,
-      "logits/rejected": 0.6150248050689697,
-      "logps/chosen": -403.6705322265625,
-      "logps/rejected": -426.13140869140625,
-      "loss": 0.0621,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.8358365297317505,
-      "rewards/margins": 0.547526478767395,
-      "rewards/rejected": -1.3833630084991455,
+      "logits/chosen": 0.29592061042785645,
+      "logits/rejected": 0.39124542474746704,
+      "logps/chosen": -458.99554443359375,
+      "logps/rejected": -476.7998046875,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.0953991413116455,
+      "rewards/margins": 0.6356866955757141,
+      "rewards/rejected": -1.731086015701294,
       "step": 210
     },
     {
       "epoch": 0.46,
       "learning_rate": 1.9593447226892386e-06,
-      "logits/chosen": 0.604540228843689,
-      "logits/rejected": 0.8555408716201782,
-      "logps/chosen": -436.47479248046875,
-      "logps/rejected": -434.32855224609375,
-      "loss": 0.0508,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.9561892747879028,
-      "rewards/margins": 0.5492093563079834,
-      "rewards/rejected": -1.5053986310958862,
+      "logits/chosen": 0.23310557007789612,
+      "logits/rejected": 0.4742186963558197,
+      "logps/chosen": -441.21649169921875,
+      "logps/rejected": -468.25286865234375,
+      "loss": 0.0599,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.9769255518913269,
+      "rewards/margins": 0.7468104362487793,
+      "rewards/rejected": -1.723736047744751,
       "step": 220
     },
     {
       "epoch": 0.48,
       "learning_rate": 1.853638403264141e-06,
-      "logits/chosen": 0.6232072114944458,
-      "logits/rejected": 0.8155434727668762,
-      "logps/chosen": -421.7066955566406,
-      "logps/rejected": -488.530517578125,
-      "loss": 0.0455,
+      "logits/chosen": 0.4100280702114105,
+      "logits/rejected": 0.5993035435676575,
+      "logps/chosen": -494.64324951171875,
+      "logps/rejected": -490.0165100097656,
+      "loss": 0.0578,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.9787136912345886,
-      "rewards/margins": 0.787739098072052,
-      "rewards/rejected": -1.7664527893066406,
+      "rewards/chosen": -1.2230786085128784,
+      "rewards/margins": 0.6530172824859619,
+      "rewards/rejected": -1.8760957717895508,
       "step": 230
     },
     {
       "epoch": 0.5,
       "learning_rate": 1.7460364672965328e-06,
-      "logits/chosen": 0.45760011672973633,
-      "logits/rejected": 0.7767106890678406,
-      "logps/chosen": -487.28802490234375,
-      "logps/rejected": -456.274658203125,
-      "loss": 0.0492,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.8502359390258789,
-      "rewards/margins": 0.6182385683059692,
-      "rewards/rejected": -1.4684743881225586,
+      "logits/chosen": 0.6504024267196655,
+      "logits/rejected": 0.7802666425704956,
+      "logps/chosen": -466.16973876953125,
+      "logps/rejected": -511.08502197265625,
+      "loss": 0.0549,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.191239595413208,
+      "rewards/margins": 0.7851654291152954,
+      "rewards/rejected": -1.976405143737793,
       "step": 240
     },
     {
       "epoch": 0.52,
       "learning_rate": 1.637115696063402e-06,
-      "logits/chosen": 0.4637879431247711,
-      "logits/rejected": 0.684526801109314,
-      "logps/chosen": -422.9723205566406,
-      "logps/rejected": -474.6338806152344,
-      "loss": 0.0418,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.8930182456970215,
-      "rewards/margins": 0.8561725616455078,
-      "rewards/rejected": -1.7491906881332397,
+      "logits/chosen": 0.7357971668243408,
+      "logits/rejected": 0.8341084718704224,
+      "logps/chosen": -462.93048095703125,
+      "logps/rejected": -550.9013671875,
+      "loss": 0.0342,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.577097773551941,
+      "rewards/margins": 0.7956889271736145,
+      "rewards/rejected": -2.3727867603302,
       "step": 250
     },
     {
       "epoch": 0.54,
       "learning_rate": 1.5274599402265162e-06,
-      "logits/chosen": 0.455331027507782,
-      "logits/rejected": 0.6168816089630127,
-      "logps/chosen": -440.8057556152344,
-      "logps/rejected": -504.82940673828125,
-      "loss": 0.0426,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.1558371782302856,
-      "rewards/margins": 0.8892456889152527,
-      "rewards/rejected": -2.0450828075408936,
+      "logits/chosen": 0.7676488757133484,
+      "logits/rejected": 0.9279497861862183,
+      "logps/chosen": -490.0227966308594,
+      "logps/rejected": -543.2033081054688,
+      "loss": 0.0336,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.5475876331329346,
+      "rewards/margins": 0.6384353041648865,
+      "rewards/rejected": -2.186022996902466,
       "step": 260
     },
     {
       "epoch": 0.57,
       "learning_rate": 1.4176569902035088e-06,
-      "logits/chosen": 0.4234936833381653,
-      "logits/rejected": 0.5202213525772095,
-      "logps/chosen": -428.471923828125,
-      "logps/rejected": -493.4828186035156,
-      "loss": 0.0394,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.2287781238555908,
-      "rewards/margins": 0.7696730494499207,
-      "rewards/rejected": -1.9984509944915771,
+      "logits/chosen": 0.7670334577560425,
+      "logits/rejected": 0.927658200263977,
+      "logps/chosen": -455.6305236816406,
+      "logps/rejected": -507.54913330078125,
+      "loss": 0.0334,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.38298761844635,
+      "rewards/margins": 0.6534308195114136,
+      "rewards/rejected": -2.0364184379577637,
       "step": 270
     },
     {
       "epoch": 0.59,
       "learning_rate": 1.308295425420593e-06,
-      "logits/chosen": 0.3574947118759155,
-      "logits/rejected": 0.5266100168228149,
-      "logps/chosen": -484.1373596191406,
-      "logps/rejected": -514.14794921875,
-      "loss": 0.0387,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.3580681085586548,
-      "rewards/margins": 0.7606993317604065,
-      "rewards/rejected": -2.118767261505127,
+      "logits/chosen": 0.7235329151153564,
+      "logits/rejected": 0.8158149719238281,
+      "logps/chosen": -491.1328125,
+      "logps/rejected": -560.6801147460938,
+      "loss": 0.0301,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.4047319889068604,
+      "rewards/margins": 0.7390089631080627,
+      "rewards/rejected": -2.1437408924102783,
       "step": 280
     },
     {
       "epoch": 0.61,
       "learning_rate": 1.1999614593359337e-06,
-      "logits/chosen": 0.3510825037956238,
-      "logits/rejected": 0.566794753074646,
-      "logps/chosen": -445.3075256347656,
-      "logps/rejected": -521.5965576171875,
-      "loss": 0.0467,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.9900503158569336,
-      "rewards/margins": 0.8154880404472351,
-      "rewards/rejected": -1.8055381774902344,
+      "logits/chosen": 0.7884746789932251,
+      "logits/rejected": 1.0120609998703003,
+      "logps/chosen": -492.41693115234375,
+      "logps/rejected": -518.9060668945312,
+      "loss": 0.03,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.4595239162445068,
+      "rewards/margins": 0.7071082592010498,
+      "rewards/rejected": -2.1666321754455566,
       "step": 290
     },
     {
       "epoch": 0.63,
       "learning_rate": 1.0932357971453745e-06,
-      "logits/chosen": 0.32849544286727905,
-      "logits/rejected": 0.4735318720340729,
-      "logps/chosen": -399.39764404296875,
-      "logps/rejected": -470.449951171875,
-      "loss": 0.0476,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.8132196664810181,
-      "rewards/margins": 0.7410646080970764,
-      "rewards/rejected": -1.5542842149734497,
+      "logits/chosen": 0.8025213479995728,
+      "logits/rejected": 0.9630680084228516,
+      "logps/chosen": -472.7798767089844,
+      "logps/rejected": -523.0516967773438,
+      "loss": 0.0313,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.4041074514389038,
+      "rewards/margins": 0.6285351514816284,
+      "rewards/rejected": -2.0326426029205322,
       "step": 300
     },
     {
       "epoch": 0.63,
-      "eval_logits/chosen": 0.4048309922218323,
-      "eval_logits/rejected": 0.5326845049858093,
-      "eval_logps/chosen": -406.68994140625,
-      "eval_logps/rejected": -488.8869323730469,
-      "eval_loss": 0.05840897932648659,
+      "eval_logits/chosen": 0.9400739669799805,
+      "eval_logits/rejected": 1.0433921813964844,
+      "eval_logps/chosen": -454.1192932128906,
+      "eval_logps/rejected": -552.3697509765625,
+      "eval_loss": 0.03436482325196266,
       "eval_rewards/accuracies": 0.72265625,
-      "eval_rewards/chosen": -0.8285996913909912,
-      "eval_rewards/margins": 0.7589226961135864,
-      "eval_rewards/rejected": -1.5875223875045776,
-      "eval_runtime": 74.4017,
-      "eval_samples_per_second": 26.881,
-      "eval_steps_per_second": 0.43,
+      "eval_rewards/chosen": -1.3028936386108398,
+      "eval_rewards/margins": 0.9194571375846863,
+      "eval_rewards/rejected": -2.222350835800171,
+      "eval_runtime": 75.6069,
+      "eval_samples_per_second": 26.453,
+      "eval_steps_per_second": 0.423,
       "step": 300
     },
     {
       "epoch": 0.65,
       "learning_rate": 9.886905230142433e-07,
-      "logits/chosen": 0.36471518874168396,
-      "logits/rejected": 0.5999516844749451,
-      "logps/chosen": -382.37371826171875,
-      "logps/rejected": -438.2478942871094,
-      "loss": 0.0526,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.8080012202262878,
-      "rewards/margins": 0.687478244304657,
-      "rewards/rejected": -1.4954793453216553,
+      "logits/chosen": 0.7544746398925781,
+      "logits/rejected": 0.9142723083496094,
+      "logps/chosen": -462.0435485839844,
+      "logps/rejected": -525.331298828125,
+      "loss": 0.0346,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.3456170558929443,
+      "rewards/margins": 0.749636709690094,
+      "rewards/rejected": -2.0952537059783936,
       "step": 310
     },
     {
       "epoch": 0.67,
       "learning_rate": 8.868860335206678e-07,
-      "logits/chosen": 0.4309239387512207,
-      "logits/rejected": 0.6108436584472656,
-      "logps/chosen": -416.4072265625,
-      "logps/rejected": -505.565185546875,
-      "loss": 0.0425,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -1.0292515754699707,
-      "rewards/margins": 1.0127887725830078,
-      "rewards/rejected": -2.0420401096343994,
+      "logits/chosen": 0.9283370971679688,
+      "logits/rejected": 1.136993169784546,
+      "logps/chosen": -478.44976806640625,
+      "logps/rejected": -530.1534423828125,
+      "loss": 0.0338,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.244257926940918,
+      "rewards/margins": 0.6402724385261536,
+      "rewards/rejected": -1.8845303058624268,
       "step": 320
     },
     {
       "epoch": 0.69,
       "learning_rate": 7.883680337481599e-07,
-      "logits/chosen": 0.5084182620048523,
-      "logits/rejected": 0.5882959961891174,
-      "logps/chosen": -483.5926818847656,
-      "logps/rejected": -550.4625854492188,
-      "loss": 0.0389,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.4037799835205078,
-      "rewards/margins": 0.7988616228103638,
-      "rewards/rejected": -2.202641725540161,
+      "logits/chosen": 0.7307278513908386,
+      "logits/rejected": 0.8725861310958862,
+      "logps/chosen": -448.43280029296875,
+      "logps/rejected": -533.6476440429688,
+      "loss": 0.0375,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.2363145351409912,
+      "rewards/margins": 0.7372487187385559,
+      "rewards/rejected": -1.9735629558563232,
       "step": 330
     },
     {
       "epoch": 0.71,
       "learning_rate": 6.936646121293654e-07,
-      "logits/chosen": 0.5944998860359192,
-      "logits/rejected": 0.7185046076774597,
-      "logps/chosen": -417.82891845703125,
-      "logps/rejected": -470.05816650390625,
-      "loss": 0.0373,
+      "logits/chosen": 0.5649510622024536,
+      "logits/rejected": 0.7639907598495483,
+      "logps/chosen": -466.2808532714844,
+      "logps/rejected": -526.1297607421875,
+      "loss": 0.0435,
       "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.1171300411224365,
-      "rewards/margins": 0.692410945892334,
-      "rewards/rejected": -1.80954110622406,
+      "rewards/chosen": -1.0562084913253784,
+      "rewards/margins": 0.7370297312736511,
+      "rewards/rejected": -1.7932384014129639,
       "step": 340
     },
     {
       "epoch": 0.73,
       "learning_rate": 6.032834097207889e-07,
-      "logits/chosen": 0.5707942247390747,
-      "logits/rejected": 0.6327140927314758,
-      "logps/chosen": -440.57830810546875,
-      "logps/rejected": -524.21337890625,
-      "loss": 0.0426,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.2543996572494507,
-      "rewards/margins": 0.7487698793411255,
-      "rewards/rejected": -2.003169536590576,
+      "logits/chosen": 0.7209309935569763,
+      "logits/rejected": 0.7828409671783447,
+      "logps/chosen": -401.2094421386719,
+      "logps/rejected": -480.31671142578125,
+      "loss": 0.0403,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.098332405090332,
+      "rewards/margins": 0.6962517499923706,
+      "rewards/rejected": -1.7945845127105713,
       "step": 350
     },
     {
       "epoch": 0.75,
       "learning_rate": 5.177088990820725e-07,
-      "logits/chosen": 0.5283172130584717,
-      "logits/rejected": 0.70453280210495,
-      "logps/chosen": -436.8048400878906,
-      "logps/rejected": -522.0552978515625,
-      "loss": 0.0438,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.0285941362380981,
-      "rewards/margins": 0.8370414972305298,
-      "rewards/rejected": -1.865635633468628,
+      "logits/chosen": 0.6787894368171692,
+      "logits/rejected": 0.8372275233268738,
+      "logps/chosen": -446.56317138671875,
+      "logps/rejected": -465.1809997558594,
+      "loss": 0.0453,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.136115550994873,
+      "rewards/margins": 0.6250497698783875,
+      "rewards/rejected": -1.7611652612686157,
       "step": 360
     },
     {
       "epoch": 0.77,
       "learning_rate": 4.3739978734594494e-07,
-      "logits/chosen": 0.5157877206802368,
-      "logits/rejected": 0.7344454526901245,
-      "logps/chosen": -460.63690185546875,
-      "logps/rejected": -486.64312744140625,
-      "loss": 0.0446,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.1092673540115356,
-      "rewards/margins": 0.6609092950820923,
-      "rewards/rejected": -1.7701762914657593,
+      "logits/chosen": 0.6346519589424133,
+      "logits/rejected": 0.867949366569519,
+      "logps/chosen": -439.4676208496094,
+      "logps/rejected": -468.6329040527344,
+      "loss": 0.0364,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.016570806503296,
+      "rewards/margins": 0.8048780560493469,
+      "rewards/rejected": -1.8214489221572876,
       "step": 370
     },
     {
       "epoch": 0.8,
       "learning_rate": 3.627865573992087e-07,
-      "logits/chosen": 0.5912496447563171,
-      "logits/rejected": 0.6915227770805359,
-      "logps/chosen": -406.13079833984375,
-      "logps/rejected": -485.49761962890625,
-      "loss": 0.0426,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.1371818780899048,
-      "rewards/margins": 0.7529508471488953,
-      "rewards/rejected": -1.8901325464248657,
+      "logits/chosen": 0.6531890630722046,
+      "logits/rejected": 0.6925245523452759,
+      "logps/chosen": -437.359375,
+      "logps/rejected": -492.814453125,
+      "loss": 0.0425,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.09610116481781,
+      "rewards/margins": 0.6472191214561462,
+      "rewards/rejected": -1.7433204650878906,
       "step": 380
     },
     {
       "epoch": 0.82,
       "learning_rate": 2.9426916035484166e-07,
-      "logits/chosen": 0.6277132034301758,
-      "logits/rejected": 0.8218109011650085,
-      "logps/chosen": -414.68621826171875,
-      "logps/rejected": -475.742919921875,
-      "loss": 0.0455,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.0436302423477173,
-      "rewards/margins": 0.6562752723693848,
-      "rewards/rejected": -1.6999053955078125,
+      "logits/chosen": 0.4887206554412842,
+      "logits/rejected": 0.7168077230453491,
+      "logps/chosen": -490.0777893066406,
+      "logps/rejected": -530.9293212890625,
+      "loss": 0.038,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.2254283428192139,
+      "rewards/margins": 0.8675802946090698,
+      "rewards/rejected": -2.0930087566375732,
       "step": 390
     },
     {
       "epoch": 0.84,
       "learning_rate": 2.322148716843081e-07,
-      "logits/chosen": 0.7061805725097656,
-      "logits/rejected": 0.7324530482292175,
-      "logps/chosen": -435.91534423828125,
-      "logps/rejected": -503.77325439453125,
-      "loss": 0.0392,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.2680333852767944,
-      "rewards/margins": 0.7200425267219543,
-      "rewards/rejected": -1.9880759716033936,
+      "logits/chosen": 0.6055541038513184,
+      "logits/rejected": 0.687682032585144,
+      "logps/chosen": -429.68603515625,
+      "logps/rejected": -461.8595275878906,
+      "loss": 0.0359,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.1895955801010132,
+      "rewards/margins": 0.5694113373756409,
+      "rewards/rejected": -1.7590070962905884,
       "step": 400
     },
     {
       "epoch": 0.84,
-      "eval_logits/chosen": 0.7478917241096497,
-      "eval_logits/rejected": 0.8668873310089111,
-      "eval_logps/chosen": -433.94195556640625,
-      "eval_logps/rejected": -528.926025390625,
-      "eval_loss": 0.04172799736261368,
-      "eval_rewards/accuracies": 0.7265625,
-      "eval_rewards/chosen": -1.1011202335357666,
-      "eval_rewards/margins": 0.8867928981781006,
-      "eval_rewards/rejected": -1.9879131317138672,
-      "eval_runtime": 72.7813,
-      "eval_samples_per_second": 27.48,
-      "eval_steps_per_second": 0.44,
+      "eval_logits/chosen": 0.7290832996368408,
+      "eval_logits/rejected": 0.8196390867233276,
+      "eval_logps/chosen": -435.5875244140625,
+      "eval_logps/rejected": -531.2747192382812,
+      "eval_loss": 0.04154704138636589,
+      "eval_rewards/accuracies": 0.70703125,
+      "eval_rewards/chosen": -1.1175758838653564,
+      "eval_rewards/margins": 0.8938245177268982,
+      "eval_rewards/rejected": -2.0114002227783203,
+      "eval_runtime": 75.1852,
+      "eval_samples_per_second": 26.601,
+      "eval_steps_per_second": 0.426,
       "step": 400
     },
     {
       "epoch": 0.86,
       "learning_rate": 1.7695632250191002e-07,
-      "logits/chosen": 0.5754364728927612,
-      "logits/rejected": 0.834900975227356,
-      "logps/chosen": -481.41033935546875,
-      "logps/rejected": -517.2847290039062,
-      "loss": 0.0392,
+      "logits/chosen": 0.5428584814071655,
+      "logits/rejected": 0.6822582483291626,
+      "logps/chosen": -435.78680419921875,
+      "logps/rejected": -452.6622009277344,
+      "loss": 0.0367,
       "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.2066195011138916,
-      "rewards/margins": 0.7020702958106995,
-      "rewards/rejected": -1.9086897373199463,
+      "rewards/chosen": -1.1751288175582886,
+      "rewards/margins": 0.5176131129264832,
+      "rewards/rejected": -1.6927419900894165,
       "step": 410
     },
     {
       "epoch": 0.88,
       "learning_rate": 1.2878971655412515e-07,
-      "logits/chosen": 0.5507728457450867,
-      "logits/rejected": 0.6630374193191528,
-      "logps/chosen": -465.51812744140625,
-      "logps/rejected": -536.48974609375,
-      "loss": 0.0414,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.2821763753890991,
-      "rewards/margins": 0.6950958967208862,
-      "rewards/rejected": -1.977272391319275,
+      "logits/chosen": 0.5744162797927856,
+      "logits/rejected": 0.6994149088859558,
+      "logps/chosen": -474.30908203125,
+      "logps/rejected": -495.92852783203125,
+      "loss": 0.0394,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.3165512084960938,
+      "rewards/margins": 0.6040414571762085,
+      "rewards/rejected": -1.9205926656723022,
       "step": 420
     },
     {
       "epoch": 0.9,
       "learning_rate": 8.797324247145411e-08,
-      "logits/chosen": 0.5740979313850403,
-      "logits/rejected": 0.8506999015808105,
-      "logps/chosen": -438.9664611816406,
-      "logps/rejected": -447.83563232421875,
-      "loss": 0.0393,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.1305776834487915,
-      "rewards/margins": 0.7222962975502014,
-      "rewards/rejected": -1.8528740406036377,
+      "logits/chosen": 0.6493648290634155,
+      "logits/rejected": 0.6758213043212891,
+      "logps/chosen": -426.60223388671875,
+      "logps/rejected": -521.1129150390625,
+      "loss": 0.0365,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1816965341567993,
+      "rewards/margins": 0.7749902009963989,
+      "rewards/rejected": -1.9566866159439087,
       "step": 430
     },
     {
       "epoch": 0.92,
       "learning_rate": 5.472568979361853e-08,
-      "logits/chosen": 0.45752525329589844,
-      "logits/rejected": 0.7472774386405945,
-      "logps/chosen": -464.6128845214844,
-      "logps/rejected": -493.8684997558594,
-      "loss": 0.0451,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.2262476682662964,
-      "rewards/margins": 0.7051091194152832,
-      "rewards/rejected": -1.9313567876815796,
+      "logits/chosen": 0.7012882232666016,
+      "logits/rejected": 0.7845873832702637,
+      "logps/chosen": -459.6414489746094,
+      "logps/rejected": -518.3292846679688,
+      "loss": 0.0412,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.2379354238510132,
+      "rewards/margins": 0.7565950155258179,
+      "rewards/rejected": -1.994530439376831,
       "step": 440
     },
     {
       "epoch": 0.94,
       "learning_rate": 2.922527618666465e-08,
-      "logits/chosen": 0.5190210938453674,
-      "logits/rejected": 0.6143854856491089,
-      "logps/chosen": -426.7154235839844,
-      "logps/rejected": -538.7984619140625,
-      "loss": 0.0438,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.0999343395233154,
-      "rewards/margins": 0.9706279039382935,
-      "rewards/rejected": -2.0705626010894775,
+      "logits/chosen": 0.6378465294837952,
+      "logits/rejected": 0.8079195022583008,
+      "logps/chosen": -484.46197509765625,
+      "logps/rejected": -520.6287841796875,
+      "loss": 0.0404,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.2168313264846802,
+      "rewards/margins": 0.676922082901001,
+      "rewards/rejected": -1.8937534093856812,
       "step": 450
     },
     {
       "epoch": 0.96,
       "learning_rate": 1.1608692138469379e-08,
-      "logits/chosen": 0.548923134803772,
-      "logits/rejected": 0.786422610282898,
-      "logps/chosen": -473.91448974609375,
-      "logps/rejected": -503.9942321777344,
-      "loss": 0.0436,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.1051746606826782,
-      "rewards/margins": 0.7683829069137573,
-      "rewards/rejected": -1.873557686805725,
+      "logits/chosen": 0.7224764227867126,
+      "logits/rejected": 0.8670576214790344,
+      "logps/chosen": -398.640869140625,
+      "logps/rejected": -444.4422912597656,
+      "loss": 0.039,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.110528826713562,
+      "rewards/margins": 0.6203423738479614,
+      "rewards/rejected": -1.7308712005615234,
       "step": 460
     },
     {
       "epoch": 0.98,
       "learning_rate": 1.970368253390198e-09,
-      "logits/chosen": 0.5904209017753601,
-      "logits/rejected": 0.7317419052124023,
-      "logps/chosen": -472.50244140625,
-      "logps/rejected": -537.5592041015625,
-      "loss": 0.046,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.2209278345108032,
-      "rewards/margins": 0.8498128056526184,
-      "rewards/rejected": -2.0707404613494873,
+      "logits/chosen": 0.6133291125297546,
+      "logits/rejected": 0.744029700756073,
+      "logps/chosen": -413.31732177734375,
+      "logps/rejected": -497.829345703125,
+      "loss": 0.0407,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.0763300657272339,
+      "rewards/margins": 0.8329319953918457,
+      "rewards/rejected": -1.9092620611190796,
       "step": 470
     },
     {
       "epoch": 1.0,
       "step": 477,
       "total_flos": 0.0,
-      "train_loss": 0.05423464074699634,
-      "train_runtime": 4545.6697,
-      "train_samples_per_second": 13.449,
-      "train_steps_per_second": 0.105
+      "train_loss": 0.055112330793584664,
+      "train_runtime": 4571.3444,
+      "train_samples_per_second": 13.373,
+      "train_steps_per_second": 0.104
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:849bf876d39c1ecc4d413b77ba74c1ef1c656105fcb277b563b6359ad5dfa298
+oid sha256:200542098b43881df0df6dc0ff3056ca0236db5763f486bb392f305292932d2f
 size 5944