Model save

README.md

@@ -15,15 +15,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4801
-- Rewards/chosen: -0.7670
-- Rewards/rejected: -1.9120
-- Rewards/accuracies: 0.7617
-- Rewards/margins: 1.1450
-- Logps/rejected: -508.7245
-- Logps/chosen: -388.4799
-- Logits/rejected: 0.7285
-- Logits/chosen: 0.2273
+- Loss: 0.4905
+- Rewards/chosen: -0.7983
+- Rewards/rejected: -1.8319
+- Rewards/accuracies: 0.7539
+- Rewards/margins: 1.0337
+- Logps/rejected: -470.9522
+- Logps/chosen: -361.4586
+- Logits/rejected: -0.7601
+- Logits/chosen: -0.9126
 
 ## Model description
 
@@ -60,10 +60,10 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.539         | 0.21  | 100  | 0.5452          | -0.5988        | -1.3188          | 0.7422             | 0.7200          | -449.4053      | -371.6547    | -0.7240         | -0.8812       |
-| 0.5294        | 0.42  | 200  | 0.5000          | -0.6654        | -1.5376          | 0.7617             | 0.8722          | -471.2849      | -378.3161    | 0.8080          | 0.4502        |
-| 0.4704        | 0.63  | 300  | 0.4878          | -0.8441        | -1.9900          | 0.7539             | 1.1459          | -516.5240      | -396.1895    | 0.8998          | 0.3977        |
-| 0.4856        | 0.84  | 400  | 0.4801          | -0.7670        | -1.9120          | 0.7617             | 1.1450          | -508.7245      | -388.4799    | 0.7285          | 0.2273        |
+| 0.5759        | 0.21  | 100  | 0.5723          | -0.5418        | -1.0793          | 0.7148             | 0.5375          | -395.6910      | -335.8161    | -1.8988         | -1.9151       |
+| 0.5391        | 0.42  | 200  | 0.5138          | -0.3953        | -1.1724          | 0.7461             | 0.7771          | -405.0056      | -321.1682    | -1.5774         | -1.6570       |
+| 0.4788        | 0.63  | 300  | 0.4939          | -0.8609        | -1.8779          | 0.7578             | 1.0170          | -475.5532      | -367.7277    | -0.8826         | -0.9864       |
+| 0.4937        | 0.84  | 400  | 0.4905          | -0.7983        | -1.8319          | 0.7539             | 1.0337          | -470.9522      | -361.4586    | -0.7601         | -0.9126       |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5266154430900158,
-    "train_runtime": 7640.9818,
+    "train_loss": 0.5340310439903866,
+    "train_runtime": 7643.694,
     "train_samples": 61134,
-    "train_samples_per_second": 8.001,
+    "train_samples_per_second": 7.998,
     "train_steps_per_second": 0.063
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a94cc29971becb2892b676d823d5547997ac3dc41aaaff95ce9ce4444477dd87
+oid sha256:c417537cd9f72d3b9982a49e9bf3f998bddf15d14938d80a6e36ab72526f9fd7
 size 4943178720

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:275d99b6466c9a99ee5fecec6e3ddc6b5751a621eeb8e231d9e43b98c4fc6abe
+oid sha256:766ffc1a5c2c0b6be99d80867a370060fefa862662878603dd1d216696fd1474
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:98350e5344a6aafa12632e3c15efc90db214d41fed645b8823cfc54c914e6c79
+oid sha256:9aa9bc9c26ab1a0cb4a22d35a2a6665a5a7aa428cec1da824b817152339ab070
 size 4540532728

runs/May11_00-51-53_n136-101-036/events.out.tfevents.1715359981.n136-101-036.30703.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:566819b915837cc25e94ba6b69cc3afe65fced55f7d19b3ca62d486106ffe9d9
-size 35844
+oid sha256:59f2651dac6673037c550ce2d4f5c8bc31f9f6ccc4a5026b18b0338b5260539e
+size 41014

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5266154430900158,
-    "train_runtime": 7640.9818,
+    "train_loss": 0.5340310439903866,
+    "train_runtime": 7643.694,
     "train_samples": 61134,
-    "train_samples_per_second": 8.001,
+    "train_samples_per_second": 7.998,
     "train_steps_per_second": 0.063
 }

trainer_state.json

@@ -10,12 +10,12 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 27.516530702491266,
+      "grad_norm": 27.376848082151543,
       "learning_rate": 1.0416666666666666e-08,
-      "logits/chosen": -1.6311802864074707,
-      "logits/rejected": -1.6300675868988037,
-      "logps/chosen": -327.34149169921875,
-      "logps/rejected": -429.85833740234375,
+      "logits/chosen": -1.8783892393112183,
+      "logits/rejected": -1.8756425380706787,
+      "logps/chosen": -298.4870300292969,
+      "logps/rejected": -398.0157165527344,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,780 +25,780 @@
     },
     {
       "epoch": 0.02,
-      "grad_norm": 25.454136109809017,
+      "grad_norm": 25.183613892194092,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -1.5537606477737427,
-      "logits/rejected": -1.506614089012146,
-      "logps/chosen": -312.7512512207031,
-      "logps/rejected": -303.0074462890625,
-      "loss": 0.6929,
-      "rewards/accuracies": 0.4513888955116272,
-      "rewards/chosen": 7.373202970484272e-05,
-      "rewards/margins": -5.077762034488842e-05,
-      "rewards/rejected": 0.000124509577290155,
+      "logits/chosen": -1.7508937120437622,
+      "logits/rejected": -1.7071995735168457,
+      "logps/chosen": -280.5123596191406,
+      "logps/rejected": -271.9100341796875,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4791666567325592,
+      "rewards/chosen": -0.00033082481240853667,
+      "rewards/margins": 0.0001291988301090896,
+      "rewards/rejected": -0.00046002367162145674,
       "step": 10
     },
     {
       "epoch": 0.04,
-      "grad_norm": 23.434204335413668,
+      "grad_norm": 23.230240384482265,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -1.6461513042449951,
-      "logits/rejected": -1.5507893562316895,
-      "logps/chosen": -329.1868896484375,
-      "logps/rejected": -351.44891357421875,
-      "loss": 0.6915,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.01134030893445015,
-      "rewards/margins": 0.00160890631377697,
-      "rewards/rejected": 0.009731403551995754,
+      "logits/chosen": -1.832267165184021,
+      "logits/rejected": -1.7261158227920532,
+      "logps/chosen": -298.87811279296875,
+      "logps/rejected": -320.8003845214844,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.009286092594265938,
+      "rewards/margins": 0.0007404378848150373,
+      "rewards/rejected": 0.008545654825866222,
       "step": 20
     },
     {
       "epoch": 0.06,
-      "grad_norm": 23.393322169409338,
+      "grad_norm": 22.82438334979188,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -1.6500400304794312,
-      "logits/rejected": -1.5703976154327393,
-      "logps/chosen": -346.0126953125,
-      "logps/rejected": -312.0833435058594,
-      "loss": 0.6837,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.07147429138422012,
-      "rewards/margins": 0.022869160398840904,
-      "rewards/rejected": 0.04860512539744377,
+      "logits/chosen": -1.8619463443756104,
+      "logits/rejected": -1.8105783462524414,
+      "logps/chosen": -315.0911865234375,
+      "logps/rejected": -281.7936096191406,
+      "loss": 0.6843,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.06353694945573807,
+      "rewards/margins": 0.020365100353956223,
+      "rewards/rejected": 0.043171852827072144,
       "step": 30
     },
     {
       "epoch": 0.08,
-      "grad_norm": 20.10353412052848,
+      "grad_norm": 20.30578423378571,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -1.5655517578125,
-      "logits/rejected": -1.5073188543319702,
-      "logps/chosen": -298.72418212890625,
-      "logps/rejected": -287.02362060546875,
-      "loss": 0.67,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.16840842366218567,
-      "rewards/margins": 0.05939362570643425,
-      "rewards/rejected": 0.10901479423046112,
+      "logits/chosen": -1.7271950244903564,
+      "logits/rejected": -1.6917632818222046,
+      "logps/chosen": -269.0830078125,
+      "logps/rejected": -258.13482666015625,
+      "loss": 0.6707,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.15124677121639252,
+      "rewards/margins": 0.058850765228271484,
+      "rewards/rejected": 0.09239600598812103,
       "step": 40
     },
     {
       "epoch": 0.1,
-      "grad_norm": 18.756434278176222,
+      "grad_norm": 18.991767469871117,
       "learning_rate": 4.999733114418725e-07,
-      "logits/chosen": -1.57842218875885,
-      "logits/rejected": -1.529872179031372,
-      "logps/chosen": -299.90679931640625,
-      "logps/rejected": -323.1398010253906,
-      "loss": 0.6556,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.2638908326625824,
-      "rewards/margins": 0.05764131620526314,
-      "rewards/rejected": 0.20624952018260956,
+      "logits/chosen": -1.7609294652938843,
+      "logits/rejected": -1.749925971031189,
+      "logps/chosen": -274.7580261230469,
+      "logps/rejected": -298.2468566894531,
+      "loss": 0.6568,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.22399310767650604,
+      "rewards/margins": 0.05384649708867073,
+      "rewards/rejected": 0.1701466143131256,
       "step": 50
     },
     {
       "epoch": 0.13,
-      "grad_norm": 22.017328959957382,
+      "grad_norm": 22.512008467267577,
       "learning_rate": 4.990398100856366e-07,
-      "logits/chosen": -1.6750396490097046,
-      "logits/rejected": -1.6164966821670532,
-      "logps/chosen": -288.38311767578125,
-      "logps/rejected": -338.00555419921875,
-      "loss": 0.6408,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.3388586640357971,
-      "rewards/margins": 0.11625377833843231,
-      "rewards/rejected": 0.222604900598526,
+      "logits/chosen": -1.8478819131851196,
+      "logits/rejected": -1.8104110956192017,
+      "logps/chosen": -268.5568542480469,
+      "logps/rejected": -318.2205505371094,
+      "loss": 0.6419,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.22708478569984436,
+      "rewards/margins": 0.11861655861139297,
+      "rewards/rejected": 0.108468197286129,
       "step": 60
     },
     {
       "epoch": 0.15,
-      "grad_norm": 25.053484688153212,
+      "grad_norm": 24.33517976511266,
       "learning_rate": 4.967775735898179e-07,
-      "logits/chosen": -1.492963433265686,
-      "logits/rejected": -1.4768620729446411,
-      "logps/chosen": -287.0970764160156,
-      "logps/rejected": -302.3998718261719,
-      "loss": 0.6124,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.2931686043739319,
-      "rewards/margins": 0.22670602798461914,
-      "rewards/rejected": 0.06646257638931274,
+      "logits/chosen": -1.6735728979110718,
+      "logits/rejected": -1.6890596151351929,
+      "logps/chosen": -274.67156982421875,
+      "logps/rejected": -289.94256591796875,
+      "loss": 0.6121,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 0.11328981816768646,
+      "rewards/margins": 0.22474534809589386,
+      "rewards/rejected": -0.11145554482936859,
       "step": 70
     },
     {
       "epoch": 0.17,
-      "grad_norm": 35.26897936392246,
+      "grad_norm": 32.81066991350388,
       "learning_rate": 4.931986719649298e-07,
-      "logits/chosen": -1.5673052072525024,
-      "logits/rejected": -1.4728682041168213,
-      "logps/chosen": -367.7672119140625,
-      "logps/rejected": -335.300048828125,
-      "loss": 0.5894,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.1240416020154953,
-      "rewards/margins": 0.28898105025291443,
-      "rewards/rejected": -0.16493944823741913,
+      "logits/chosen": -1.9151760339736938,
+      "logits/rejected": -1.802167534828186,
+      "logps/chosen": -356.9640808105469,
+      "logps/rejected": -325.5661315917969,
+      "loss": 0.5878,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.08058764040470123,
+      "rewards/margins": 0.30372124910354614,
+      "rewards/rejected": -0.38430893421173096,
       "step": 80
     },
     {
       "epoch": 0.19,
-      "grad_norm": 40.64595674335777,
+      "grad_norm": 31.216011823878958,
       "learning_rate": 4.883222001996351e-07,
-      "logits/chosen": -1.1538174152374268,
-      "logits/rejected": -1.0762898921966553,
-      "logps/chosen": -330.572265625,
-      "logps/rejected": -377.3353576660156,
-      "loss": 0.5545,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.18641509115695953,
-      "rewards/margins": 0.5493143200874329,
-      "rewards/rejected": -0.7357293963432312,
+      "logits/chosen": -1.7752355337142944,
+      "logits/rejected": -1.7595351934432983,
+      "logps/chosen": -325.2732849121094,
+      "logps/rejected": -370.2122497558594,
+      "loss": 0.5628,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.44604864716529846,
+      "rewards/margins": 0.5209277272224426,
+      "rewards/rejected": -0.9669763445854187,
       "step": 90
     },
     {
       "epoch": 0.21,
-      "grad_norm": 48.97381689745216,
+      "grad_norm": 30.757968765158964,
       "learning_rate": 4.821741763807186e-07,
-      "logits/chosen": -0.7731859683990479,
-      "logits/rejected": -0.7151339054107666,
-      "logps/chosen": -357.76898193359375,
-      "logps/rejected": -418.14154052734375,
-      "loss": 0.539,
+      "logits/chosen": -1.8407386541366577,
+      "logits/rejected": -1.818633794784546,
+      "logps/chosen": -356.88604736328125,
+      "logps/rejected": -402.81597900390625,
+      "loss": 0.5759,
       "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.4816914498806,
-      "rewards/margins": 0.6462317705154419,
-      "rewards/rejected": -1.1279233694076538,
+      "rewards/chosen": -0.7781068682670593,
+      "rewards/margins": 0.501887321472168,
+      "rewards/rejected": -1.279994249343872,
       "step": 100
     },
     {
       "epoch": 0.21,
-      "eval_logits/chosen": -0.8811883330345154,
-      "eval_logits/rejected": -0.7240222096443176,
-      "eval_logps/chosen": -371.65472412109375,
-      "eval_logps/rejected": -449.4052734375,
-      "eval_loss": 0.5451700091362,
-      "eval_rewards/accuracies": 0.7421875,
-      "eval_rewards/chosen": -0.5987821817398071,
-      "eval_rewards/margins": 0.7200188040733337,
-      "eval_rewards/rejected": -1.3188011646270752,
-      "eval_runtime": 97.5928,
-      "eval_samples_per_second": 20.493,
-      "eval_steps_per_second": 0.328,
+      "eval_logits/chosen": -1.9150919914245605,
+      "eval_logits/rejected": -1.8988227844238281,
+      "eval_logps/chosen": -335.8161315917969,
+      "eval_logps/rejected": -395.6910095214844,
+      "eval_loss": 0.5723333954811096,
+      "eval_rewards/accuracies": 0.71484375,
+      "eval_rewards/chosen": -0.5418255925178528,
+      "eval_rewards/margins": 0.5374687910079956,
+      "eval_rewards/rejected": -1.0792944431304932,
+      "eval_runtime": 97.7859,
+      "eval_samples_per_second": 20.453,
+      "eval_steps_per_second": 0.327,
       "step": 100
     },
     {
       "epoch": 0.23,
-      "grad_norm": 40.67779253000725,
+      "grad_norm": 31.68583269080582,
       "learning_rate": 4.747874028753375e-07,
-      "logits/chosen": -0.9343360066413879,
-      "logits/rejected": -0.7007087469100952,
-      "logps/chosen": -409.1253967285156,
-      "logps/rejected": -429.0912170410156,
-      "loss": 0.571,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.4740072786808014,
-      "rewards/margins": 0.6539417505264282,
-      "rewards/rejected": -1.1279489994049072,
+      "logits/chosen": -1.8968604803085327,
+      "logits/rejected": -1.7719805240631104,
+      "logps/chosen": -367.7148742675781,
+      "logps/rejected": -371.1282653808594,
+      "loss": 0.5798,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.36616355180740356,
+      "rewards/margins": 0.49689429998397827,
+      "rewards/rejected": -0.8630577921867371,
       "step": 110
     },
     {
       "epoch": 0.25,
-      "grad_norm": 52.47159513458736,
+      "grad_norm": 30.73893119347065,
       "learning_rate": 4.662012913161997e-07,
-      "logits/chosen": -0.5735512971878052,
-      "logits/rejected": -0.4024372696876526,
-      "logps/chosen": -359.32373046875,
-      "logps/rejected": -393.42974853515625,
-      "loss": 0.5452,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.2522381842136383,
-      "rewards/margins": 0.5496727228164673,
-      "rewards/rejected": -0.8019108772277832,
+      "logits/chosen": -1.7683579921722412,
+      "logits/rejected": -1.7642805576324463,
+      "logps/chosen": -332.87030029296875,
+      "logps/rejected": -358.53521728515625,
+      "loss": 0.547,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.2855382263660431,
+      "rewards/margins": 0.4636038839817047,
+      "rewards/rejected": -0.7491421103477478,
       "step": 120
     },
     {
       "epoch": 0.27,
-      "grad_norm": 41.93944182107726,
+      "grad_norm": 36.07936832382646,
       "learning_rate": 4.5646165232345103e-07,
-      "logits/chosen": -0.06039993837475777,
-      "logits/rejected": 0.15637175738811493,
-      "logps/chosen": -362.6398010253906,
-      "logps/rejected": -421.77099609375,
-      "loss": 0.5241,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.3607412576675415,
-      "rewards/margins": 0.6238666772842407,
-      "rewards/rejected": -0.9846078753471375,
+      "logits/chosen": -1.8693767786026,
+      "logits/rejected": -1.794420599937439,
+      "logps/chosen": -345.6004638671875,
+      "logps/rejected": -402.54608154296875,
+      "loss": 0.5353,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.4933016300201416,
+      "rewards/margins": 0.5930127501487732,
+      "rewards/rejected": -1.0863142013549805,
       "step": 130
     },
     {
       "epoch": 0.29,
-      "grad_norm": 58.39454249885024,
+      "grad_norm": 39.43015574676247,
       "learning_rate": 4.456204510851956e-07,
-      "logits/chosen": 0.14134523272514343,
-      "logits/rejected": 0.4324416220188141,
-      "logps/chosen": -426.36358642578125,
-      "logps/rejected": -497.6366271972656,
-      "loss": 0.5256,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.5936006903648376,
-      "rewards/margins": 0.8088639378547668,
-      "rewards/rejected": -1.4024646282196045,
+      "logits/chosen": -1.7766962051391602,
+      "logits/rejected": -1.7008291482925415,
+      "logps/chosen": -394.8687744140625,
+      "logps/rejected": -457.8002014160156,
+      "loss": 0.5255,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.5899434089660645,
+      "rewards/margins": 0.7221861481666565,
+      "rewards/rejected": -1.3121296167373657,
       "step": 140
     },
     {
       "epoch": 0.31,
-      "grad_norm": 47.848586407040784,
+      "grad_norm": 48.649979456976475,
       "learning_rate": 4.337355301007335e-07,
-      "logits/chosen": 0.6952211260795593,
-      "logits/rejected": 0.8886941075325012,
-      "logps/chosen": -412.7098083496094,
-      "logps/rejected": -467.67486572265625,
-      "loss": 0.5424,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.8109712600708008,
-      "rewards/margins": 0.5735460519790649,
-      "rewards/rejected": -1.3845174312591553,
+      "logits/chosen": -1.7205531597137451,
+      "logits/rejected": -1.6475614309310913,
+      "logps/chosen": -382.6226806640625,
+      "logps/rejected": -433.13409423828125,
+      "loss": 0.5514,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.8077861666679382,
+      "rewards/margins": 0.5227850675582886,
+      "rewards/rejected": -1.3305714130401611,
       "step": 150
     },
     {
       "epoch": 0.33,
-      "grad_norm": 41.0369792560692,
+      "grad_norm": 44.8404423652529,
       "learning_rate": 4.2087030056579986e-07,
-      "logits/chosen": 0.18771931529045105,
-      "logits/rejected": 0.7211147546768188,
-      "logps/chosen": -363.5400695800781,
-      "logps/rejected": -417.8056640625,
-      "loss": 0.5424,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.3788861632347107,
-      "rewards/margins": 0.7999863028526306,
-      "rewards/rejected": -1.1788724660873413,
+      "logits/chosen": -1.698359727859497,
+      "logits/rejected": -1.583348035812378,
+      "logps/chosen": -338.5320129394531,
+      "logps/rejected": -390.748291015625,
+      "loss": 0.5328,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.4389611780643463,
+      "rewards/margins": 0.7732844352722168,
+      "rewards/rejected": -1.2122455835342407,
       "step": 160
     },
     {
       "epoch": 0.36,
-      "grad_norm": 40.70764257859768,
+      "grad_norm": 44.50813022956455,
       "learning_rate": 4.070934040463998e-07,
-      "logits/chosen": 0.3842553198337555,
-      "logits/rejected": 0.7558060884475708,
-      "logps/chosen": -360.5339660644531,
-      "logps/rejected": -410.0292053222656,
-      "loss": 0.5243,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.6288506388664246,
-      "rewards/margins": 0.626385509967804,
-      "rewards/rejected": -1.2552361488342285,
+      "logits/chosen": -1.607373833656311,
+      "logits/rejected": -1.4848835468292236,
+      "logps/chosen": -323.30230712890625,
+      "logps/rejected": -363.1103515625,
+      "loss": 0.5341,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.5660237669944763,
+      "rewards/margins": 0.5307204723358154,
+      "rewards/rejected": -1.0967442989349365,
       "step": 170
     },
     {
       "epoch": 0.38,
-      "grad_norm": 42.009965736488304,
+      "grad_norm": 36.42011775174975,
       "learning_rate": 3.9247834624635404e-07,
-      "logits/chosen": 0.8420892953872681,
-      "logits/rejected": 1.2455089092254639,
-      "logps/chosen": -370.99090576171875,
-      "logps/rejected": -420.73358154296875,
-      "loss": 0.4907,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.6862328052520752,
-      "rewards/margins": 0.7635400891304016,
-      "rewards/rejected": -1.449772834777832,
+      "logits/chosen": -1.4740314483642578,
+      "logits/rejected": -1.3969428539276123,
+      "logps/chosen": -355.79022216796875,
+      "logps/rejected": -386.48846435546875,
+      "loss": 0.5082,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.8436854481697083,
+      "rewards/margins": 0.5756739377975464,
+      "rewards/rejected": -1.4193594455718994,
       "step": 180
     },
     {
       "epoch": 0.4,
-      "grad_norm": 40.59427137175583,
+      "grad_norm": 37.5415107419379,
       "learning_rate": 3.7710310482256523e-07,
-      "logits/chosen": 0.35120826959609985,
-      "logits/rejected": 0.5841476321220398,
-      "logps/chosen": -369.50762939453125,
-      "logps/rejected": -431.60430908203125,
-      "loss": 0.5135,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.45533379912376404,
-      "rewards/margins": 0.7292474508285522,
-      "rewards/rejected": -1.1845813989639282,
+      "logits/chosen": -1.5718328952789307,
+      "logits/rejected": -1.524204134941101,
+      "logps/chosen": -348.835693359375,
+      "logps/rejected": -401.84857177734375,
+      "loss": 0.5283,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.5492793321609497,
+      "rewards/margins": 0.6370285153388977,
+      "rewards/rejected": -1.186307668685913,
       "step": 190
     },
     {
       "epoch": 0.42,
-      "grad_norm": 74.46574358755885,
+      "grad_norm": 86.8216257987796,
       "learning_rate": 3.610497133404795e-07,
-      "logits/chosen": 0.5723182559013367,
-      "logits/rejected": 0.8850580453872681,
-      "logps/chosen": -384.32122802734375,
-      "logps/rejected": -460.3968200683594,
-      "loss": 0.5294,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.7563287019729614,
-      "rewards/margins": 0.8355164527893066,
-      "rewards/rejected": -1.591845154762268,
+      "logits/chosen": -1.508467674255371,
+      "logits/rejected": -1.409771203994751,
+      "logps/chosen": -339.861328125,
+      "logps/rejected": -404.37200927734375,
+      "loss": 0.5391,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.6251331567764282,
+      "rewards/margins": 0.7106647491455078,
+      "rewards/rejected": -1.335797905921936,
       "step": 200
     },
     {
       "epoch": 0.42,
-      "eval_logits/chosen": 0.45017001032829285,
-      "eval_logits/rejected": 0.8080278038978577,
-      "eval_logps/chosen": -378.31610107421875,
-      "eval_logps/rejected": -471.284912109375,
-      "eval_loss": 0.49995097517967224,
-      "eval_rewards/accuracies": 0.76171875,
-      "eval_rewards/chosen": -0.6653960347175598,
-      "eval_rewards/margins": 0.8722014427185059,
-      "eval_rewards/rejected": -1.537597417831421,
-      "eval_runtime": 97.5595,
-      "eval_samples_per_second": 20.5,
+      "eval_logits/chosen": -1.6569650173187256,
+      "eval_logits/rejected": -1.577364206314087,
+      "eval_logps/chosen": -321.1681823730469,
+      "eval_logps/rejected": -405.005615234375,
+      "eval_loss": 0.5137878060340881,
+      "eval_rewards/accuracies": 0.74609375,
+      "eval_rewards/chosen": -0.3953460156917572,
+      "eval_rewards/margins": 0.777094841003418,
+      "eval_rewards/rejected": -1.172440767288208,
+      "eval_runtime": 97.4997,
+      "eval_samples_per_second": 20.513,
       "eval_steps_per_second": 0.328,
       "step": 200
     },
     {
       "epoch": 0.44,
-      "grad_norm": 46.031984990497406,
+      "grad_norm": 38.50117908255001,
       "learning_rate": 3.4440382358952115e-07,
-      "logits/chosen": 0.48276287317276,
-      "logits/rejected": 0.8357312083244324,
-      "logps/chosen": -404.5564880371094,
-      "logps/rejected": -446.3399963378906,
-      "loss": 0.5221,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.5977016687393188,
-      "rewards/margins": 0.8030574917793274,
-      "rewards/rejected": -1.400759220123291,
+      "logits/chosen": -1.443800687789917,
+      "logits/rejected": -1.3134807348251343,
+      "logps/chosen": -354.0191345214844,
+      "logps/rejected": -393.02264404296875,
+      "loss": 0.5328,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.40636110305786133,
+      "rewards/margins": 0.759956419467926,
+      "rewards/rejected": -1.1663174629211426,
       "step": 210
     },
     {
       "epoch": 0.46,
-      "grad_norm": 48.160697637910886,
+      "grad_norm": 50.76318175549852,
       "learning_rate": 3.272542485937368e-07,
-      "logits/chosen": 0.5655020475387573,
-      "logits/rejected": 0.9708032608032227,
-      "logps/chosen": -374.70733642578125,
-      "logps/rejected": -441.2498474121094,
-      "loss": 0.5028,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.6240397691726685,
-      "rewards/margins": 0.8947107195854187,
-      "rewards/rejected": -1.5187504291534424,
+      "logits/chosen": -1.2096138000488281,
+      "logits/rejected": -1.064009189605713,
+      "logps/chosen": -339.54339599609375,
+      "logps/rejected": -401.81622314453125,
+      "loss": 0.5149,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.570379376411438,
+      "rewards/margins": 0.8607443571090698,
+      "rewards/rejected": -1.4311238527297974,
       "step": 220
     },
     {
       "epoch": 0.48,
-      "grad_norm": 48.6209922138632,
+      "grad_norm": 56.74639801937906,
       "learning_rate": 3.096924887558854e-07,
-      "logits/chosen": 0.6631933450698853,
-      "logits/rejected": 1.059995412826538,
-      "logps/chosen": -387.43206787109375,
-      "logps/rejected": -495.94976806640625,
-      "loss": 0.5111,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.9049599766731262,
-      "rewards/margins": 1.0336183309555054,
-      "rewards/rejected": -1.9385783672332764,
+      "logits/chosen": -1.1264355182647705,
+      "logits/rejected": -1.0245306491851807,
+      "logps/chosen": -354.9466857910156,
+      "logps/rejected": -463.1206970214844,
+      "loss": 0.5149,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.8869966268539429,
+      "rewards/margins": 1.0312917232513428,
+      "rewards/rejected": -1.9182884693145752,
       "step": 230
     },
     {
       "epoch": 0.5,
-      "grad_norm": 50.906502494422604,
+      "grad_norm": 42.18851613816063,
       "learning_rate": 2.9181224366319943e-07,
-      "logits/chosen": 0.07225779443979263,
-      "logits/rejected": 0.43359050154685974,
-      "logps/chosen": -382.9250183105469,
-      "logps/rejected": -458.04705810546875,
-      "loss": 0.4847,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.6258467435836792,
-      "rewards/margins": 0.9445594549179077,
-      "rewards/rejected": -1.570406198501587,
+      "logits/chosen": -1.2307146787643433,
+      "logits/rejected": -1.139776349067688,
+      "logps/chosen": -370.49853515625,
+      "logps/rejected": -442.11114501953125,
+      "loss": 0.487,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.8160694241523743,
+      "rewards/margins": 0.8950576782226562,
+      "rewards/rejected": -1.7111270427703857,
       "step": 240
     },
     {
       "epoch": 0.52,
-      "grad_norm": 51.78937629264524,
+      "grad_norm": 42.578136485563924,
       "learning_rate": 2.7370891215954565e-07,
-      "logits/chosen": 0.30628976225852966,
-      "logits/rejected": 0.6650404930114746,
-      "logps/chosen": -413.821044921875,
-      "logps/rejected": -486.84588623046875,
-      "loss": 0.5071,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.6004015207290649,
-      "rewards/margins": 1.1020289659500122,
-      "rewards/rejected": -1.7024303674697876,
+      "logits/chosen": -1.104014277458191,
+      "logits/rejected": -0.9260801076889038,
+      "logps/chosen": -386.5050964355469,
+      "logps/rejected": -449.59722900390625,
+      "loss": 0.5122,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.6448307633399963,
+      "rewards/margins": 0.9931491017341614,
+      "rewards/rejected": -1.6379798650741577,
       "step": 250
     },
     {
       "epoch": 0.54,
-      "grad_norm": 45.99084165109373,
+      "grad_norm": 45.2823901088297,
       "learning_rate": 2.55479083351317e-07,
-      "logits/chosen": 0.2199174165725708,
-      "logits/rejected": 0.5914181470870972,
-      "logps/chosen": -416.1441955566406,
-      "logps/rejected": -481.27020263671875,
-      "loss": 0.4928,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.6798352003097534,
-      "rewards/margins": 1.0327410697937012,
-      "rewards/rejected": -1.7125762701034546,
+      "logits/chosen": -0.9789173007011414,
+      "logits/rejected": -0.7768954634666443,
+      "logps/chosen": -395.30548095703125,
+      "logps/rejected": -449.7933044433594,
+      "loss": 0.505,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.7873316407203674,
+      "rewards/margins": 0.9275019764900208,
+      "rewards/rejected": -1.7148334980010986,
       "step": 260
     },
     {
       "epoch": 0.56,
-      "grad_norm": 44.6265768332988,
+      "grad_norm": 44.49352654305102,
       "learning_rate": 2.3722002126275822e-07,
-      "logits/chosen": 0.2704251706600189,
-      "logits/rejected": 0.6909293532371521,
-      "logps/chosen": -405.48089599609375,
-      "logps/rejected": -471.8568420410156,
-      "loss": 0.5062,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.7410668134689331,
-      "rewards/margins": 0.8480156064033508,
-      "rewards/rejected": -1.5890824794769287,
+      "logits/chosen": -0.9071099162101746,
+      "logits/rejected": -0.8342393636703491,
+      "logps/chosen": -374.5643310546875,
+      "logps/rejected": -433.95098876953125,
+      "loss": 0.5205,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.729364275932312,
+      "rewards/margins": 0.7837658524513245,
+      "rewards/rejected": -1.5131301879882812,
       "step": 270
     },
     {
       "epoch": 0.59,
-      "grad_norm": 45.745689932530695,
+      "grad_norm": 42.661081149871585,
       "learning_rate": 2.19029145890313e-07,
-      "logits/chosen": 0.5122145414352417,
-      "logits/rejected": 0.9128357172012329,
-      "logps/chosen": -386.9629821777344,
-      "logps/rejected": -474.33697509765625,
-      "loss": 0.5119,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.7393491864204407,
-      "rewards/margins": 1.0442432165145874,
-      "rewards/rejected": -1.7835925817489624,
+      "logits/chosen": -0.9833901524543762,
+      "logits/rejected": -0.7881035804748535,
+      "logps/chosen": -336.73388671875,
+      "logps/rejected": -407.16326904296875,
+      "loss": 0.5186,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.5388863682746887,
+      "rewards/margins": 0.8700063824653625,
+      "rewards/rejected": -1.4088926315307617,
       "step": 280
     },
     {
       "epoch": 0.61,
-      "grad_norm": 39.82940708039207,
+      "grad_norm": 42.8181212203347,
       "learning_rate": 2.0100351342479216e-07,
-      "logits/chosen": 0.5138941407203674,
-      "logits/rejected": 0.7629462480545044,
-      "logps/chosen": -382.9958801269531,
-      "logps/rejected": -461.4434509277344,
-      "loss": 0.4913,
+      "logits/chosen": -1.1338340044021606,
+      "logits/rejected": -1.0056653022766113,
+      "logps/chosen": -327.16217041015625,
+      "logps/rejected": -406.1835021972656,
+      "loss": 0.5062,
       "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.8513978719711304,
-      "rewards/margins": 0.8561092615127563,
-      "rewards/rejected": -1.7075071334838867,
+      "rewards/chosen": -0.5975332260131836,
+      "rewards/margins": 0.854636549949646,
+      "rewards/rejected": -1.4521698951721191,
       "step": 290
     },
     {
       "epoch": 0.63,
-      "grad_norm": 46.49918219263532,
+      "grad_norm": 43.557515582129994,
       "learning_rate": 1.8323929841460178e-07,
-      "logits/chosen": 0.3830031752586365,
-      "logits/rejected": 0.9853779077529907,
-      "logps/chosen": -448.693359375,
-      "logps/rejected": -515.8469848632812,
-      "loss": 0.4704,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.9706672430038452,
-      "rewards/margins": 1.0561556816101074,
-      "rewards/rejected": -2.026822805404663,
+      "logits/chosen": -1.1290686130523682,
+      "logits/rejected": -0.9314834475517273,
+      "logps/chosen": -405.9638671875,
+      "logps/rejected": -460.133544921875,
+      "loss": 0.4788,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8333964347839355,
+      "rewards/margins": 0.9296092987060547,
+      "rewards/rejected": -1.7630058526992798,
       "step": 300
     },
     {
       "epoch": 0.63,
-      "eval_logits/chosen": 0.3976738452911377,
-      "eval_logits/rejected": 0.8997932076454163,
-      "eval_logps/chosen": -396.1895446777344,
-      "eval_logps/rejected": -516.5240478515625,
-      "eval_loss": 0.48780035972595215,
-      "eval_rewards/accuracies": 0.75390625,
-      "eval_rewards/chosen": -0.8441304564476013,
-      "eval_rewards/margins": 1.1458582878112793,
-      "eval_rewards/rejected": -1.9899888038635254,
-      "eval_runtime": 97.6501,
-      "eval_samples_per_second": 20.481,
-      "eval_steps_per_second": 0.328,
+      "eval_logits/chosen": -0.986372709274292,
+      "eval_logits/rejected": -0.8825947046279907,
+      "eval_logps/chosen": -367.72772216796875,
+      "eval_logps/rejected": -475.55322265625,
+      "eval_loss": 0.49388447403907776,
+      "eval_rewards/accuracies": 0.7578125,
+      "eval_rewards/chosen": -0.8609416484832764,
+      "eval_rewards/margins": 1.016974687576294,
+      "eval_rewards/rejected": -1.8779162168502808,
+      "eval_runtime": 97.7186,
+      "eval_samples_per_second": 20.467,
+      "eval_steps_per_second": 0.327,
       "step": 300
     },
     {
       "epoch": 0.65,
-      "grad_norm": 52.076181120314466,
+      "grad_norm": 50.71884572569721,
       "learning_rate": 1.6583128063291573e-07,
-      "logits/chosen": 0.43108898401260376,
-      "logits/rejected": 0.7893391251564026,
-      "logps/chosen": -438.8291931152344,
-      "logps/rejected": -502.6856994628906,
-      "loss": 0.485,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.9125974774360657,
-      "rewards/margins": 0.9991793632507324,
-      "rewards/rejected": -1.9117769002914429,
+      "logits/chosen": -0.8109232783317566,
+      "logits/rejected": -0.6804934144020081,
+      "logps/chosen": -421.7939453125,
+      "logps/rejected": -471.0542907714844,
+      "loss": 0.4943,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.045866847038269,
+      "rewards/margins": 0.8514853715896606,
+      "rewards/rejected": -1.8973522186279297,
       "step": 310
     },
     {
       "epoch": 0.67,
-      "grad_norm": 53.82916961985562,
+      "grad_norm": 50.530325658635704,
       "learning_rate": 1.488723393865766e-07,
-      "logits/chosen": 0.5213574171066284,
-      "logits/rejected": 1.0497839450836182,
-      "logps/chosen": -437.28936767578125,
-      "logps/rejected": -490.869873046875,
-      "loss": 0.4654,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.8886297941207886,
-      "rewards/margins": 1.0424896478652954,
-      "rewards/rejected": -1.9311193227767944,
+      "logits/chosen": -0.697489321231842,
+      "logits/rejected": -0.5368015170097351,
+      "logps/chosen": -416.4817810058594,
+      "logps/rejected": -457.8753356933594,
+      "loss": 0.4844,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.9900579452514648,
+      "rewards/margins": 0.9124795198440552,
+      "rewards/rejected": -1.9025375843048096,
       "step": 320
     },
     {
       "epoch": 0.69,
-      "grad_norm": 52.0979391510305,
+      "grad_norm": 51.29954417938159,
       "learning_rate": 1.3245295796480788e-07,
-      "logits/chosen": 0.27822378277778625,
-      "logits/rejected": 0.6966886520385742,
-      "logps/chosen": -394.94219970703125,
-      "logps/rejected": -488.16571044921875,
-      "loss": 0.4721,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.8261836767196655,
-      "rewards/margins": 0.8932548761367798,
-      "rewards/rejected": -1.7194385528564453,
+      "logits/chosen": -0.9539875984191895,
+      "logits/rejected": -0.7579910159111023,
+      "logps/chosen": -368.8915710449219,
+      "logps/rejected": -450.0838317871094,
+      "loss": 0.4783,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.8678079843521118,
+      "rewards/margins": 0.7714605331420898,
+      "rewards/rejected": -1.6392685174942017,
       "step": 330
     },
     {
       "epoch": 0.71,
-      "grad_norm": 53.006317641552194,
+      "grad_norm": 47.146951845139505,
       "learning_rate": 1.1666074087171627e-07,
-      "logits/chosen": 0.24821150302886963,
-      "logits/rejected": 0.725691556930542,
-      "logps/chosen": -424.24761962890625,
-      "logps/rejected": -520.8505249023438,
-      "loss": 0.485,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.772143542766571,
-      "rewards/margins": 1.1264902353286743,
-      "rewards/rejected": -1.8986337184906006,
+      "logits/chosen": -0.9075170755386353,
+      "logits/rejected": -0.7373852133750916,
+      "logps/chosen": -388.6153259277344,
+      "logps/rejected": -477.04217529296875,
+      "loss": 0.4968,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.7279259562492371,
+      "rewards/margins": 1.0357873439788818,
+      "rewards/rejected": -1.7637134790420532,
       "step": 340
     },
     {
       "epoch": 0.73,
-      "grad_norm": 50.07894063945848,
+      "grad_norm": 40.51519007652658,
       "learning_rate": 1.0157994641835734e-07,
-      "logits/chosen": 0.5090032815933228,
-      "logits/rejected": 1.0217373371124268,
-      "logps/chosen": -399.2255859375,
-      "logps/rejected": -497.14910888671875,
-      "loss": 0.4596,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.9144848585128784,
-      "rewards/margins": 1.1523492336273193,
-      "rewards/rejected": -2.0668342113494873,
+      "logits/chosen": -0.9085396528244019,
+      "logits/rejected": -0.7681713104248047,
+      "logps/chosen": -352.57073974609375,
+      "logps/rejected": -430.44903564453125,
+      "loss": 0.4725,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.7491118311882019,
+      "rewards/margins": 0.9513088464736938,
+      "rewards/rejected": -1.7004207372665405,
       "step": 350
     },
     {
       "epoch": 0.75,
-      "grad_norm": 46.12071728187378,
+      "grad_norm": 44.22500120445586,
       "learning_rate": 8.729103716819111e-08,
-      "logits/chosen": 0.29714930057525635,
-      "logits/rejected": 0.8836923837661743,
-      "logps/chosen": -449.24822998046875,
-      "logps/rejected": -515.9562377929688,
-      "loss": 0.507,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.9280809164047241,
-      "rewards/margins": 1.0685356855392456,
-      "rewards/rejected": -1.9966167211532593,
+      "logits/chosen": -1.0195329189300537,
+      "logits/rejected": -0.7282712459564209,
+      "logps/chosen": -405.8896179199219,
+      "logps/rejected": -458.53826904296875,
+      "loss": 0.5133,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.7919691801071167,
+      "rewards/margins": 0.9254425168037415,
+      "rewards/rejected": -1.7174116373062134,
       "step": 360
     },
     {
       "epoch": 0.77,
-      "grad_norm": 44.466114694620565,
+      "grad_norm": 42.97964211979383,
       "learning_rate": 7.387025063449081e-08,
-      "logits/chosen": 0.3406144082546234,
-      "logits/rejected": 0.8408617973327637,
-      "logps/chosen": -423.6982421875,
-      "logps/rejected": -472.16302490234375,
-      "loss": 0.4937,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.973863959312439,
-      "rewards/margins": 0.8518773317337036,
-      "rewards/rejected": -1.825741171836853,
+      "logits/chosen": -0.7996746301651001,
+      "logits/rejected": -0.5788342356681824,
+      "logps/chosen": -381.4356689453125,
+      "logps/rejected": -423.28570556640625,
+      "loss": 0.5002,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8598394393920898,
+      "rewards/margins": 0.7896124124526978,
+      "rewards/rejected": -1.6494518518447876,
       "step": 370
     },
     {
       "epoch": 0.79,
-      "grad_norm": 40.2546545345207,
+      "grad_norm": 40.60703994205003,
       "learning_rate": 6.138919252022435e-08,
-      "logits/chosen": 0.5687306523323059,
-      "logits/rejected": 0.8843731880187988,
-      "logps/chosen": -388.6229248046875,
-      "logps/rejected": -532.6536865234375,
-      "loss": 0.4777,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.018700361251831,
-      "rewards/margins": 1.1921350955963135,
-      "rewards/rejected": -2.2108356952667236,
+      "logits/chosen": -0.7264416217803955,
+      "logits/rejected": -0.689493715763092,
+      "logps/chosen": -355.46893310546875,
+      "logps/rejected": -487.8539123535156,
+      "loss": 0.485,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.9955714344978333,
+      "rewards/margins": 1.0758744478225708,
+      "rewards/rejected": -2.071445941925049,
       "step": 380
     },
     {
       "epoch": 0.82,
-      "grad_norm": 50.26373841675365,
+      "grad_norm": 46.93787360588045,
       "learning_rate": 4.991445467064689e-08,
-      "logits/chosen": 0.3668118119239807,
-      "logits/rejected": 0.6758331060409546,
-      "logps/chosen": -445.70294189453125,
-      "logps/rejected": -524.3595581054688,
-      "loss": 0.4756,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.906339168548584,
-      "rewards/margins": 0.9413349032402039,
-      "rewards/rejected": -1.8476741313934326,
+      "logits/chosen": -0.8655691146850586,
+      "logits/rejected": -0.7559862732887268,
+      "logps/chosen": -414.95660400390625,
+      "logps/rejected": -487.86029052734375,
+      "loss": 0.4776,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.9033805727958679,
+      "rewards/margins": 0.8835199475288391,
+      "rewards/rejected": -1.786900281906128,
       "step": 390
     },
     {
       "epoch": 0.84,
-      "grad_norm": 45.03150381268163,
+      "grad_norm": 44.554590531311376,
       "learning_rate": 3.9507259776993954e-08,
-      "logits/chosen": 0.47453969717025757,
-      "logits/rejected": 0.8979718089103699,
-      "logps/chosen": -417.6993713378906,
-      "logps/rejected": -514.3675537109375,
-      "loss": 0.4856,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.9480649828910828,
-      "rewards/margins": 1.0335150957107544,
-      "rewards/rejected": -1.981580138206482,
+      "logits/chosen": -0.6984928846359253,
+      "logits/rejected": -0.5499908924102783,
+      "logps/chosen": -390.16436767578125,
+      "logps/rejected": -478.6742248535156,
+      "loss": 0.4937,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.9803212285041809,
+      "rewards/margins": 0.9358729124069214,
+      "rewards/rejected": -1.916194200515747,
       "step": 400
     },
     {
       "epoch": 0.84,
-      "eval_logits/chosen": 0.22729139029979706,
-      "eval_logits/rejected": 0.7285040616989136,
-      "eval_logps/chosen": -388.4799499511719,
-      "eval_logps/rejected": -508.7244873046875,
-      "eval_loss": 0.4800879657268524,
-      "eval_rewards/accuracies": 0.76171875,
-      "eval_rewards/chosen": -0.7670345306396484,
-      "eval_rewards/margins": 1.1449588537216187,
-      "eval_rewards/rejected": -1.911993384361267,
-      "eval_runtime": 97.4275,
-      "eval_samples_per_second": 20.528,
-      "eval_steps_per_second": 0.328,
+      "eval_logits/chosen": -0.9126077890396118,
+      "eval_logits/rejected": -0.760056734085083,
+      "eval_logps/chosen": -361.4586486816406,
+      "eval_logps/rejected": -470.95220947265625,
+      "eval_loss": 0.4905379116535187,
+      "eval_rewards/accuracies": 0.75390625,
+      "eval_rewards/chosen": -0.7982508540153503,
+      "eval_rewards/margins": 1.0336554050445557,
+      "eval_rewards/rejected": -1.8319063186645508,
+      "eval_runtime": 97.7293,
+      "eval_samples_per_second": 20.465,
+      "eval_steps_per_second": 0.327,
       "step": 400
     },
     {
       "epoch": 0.86,
-      "grad_norm": 43.3283841726019,
+      "grad_norm": 39.592549123818905,
       "learning_rate": 3.022313472693447e-08,
-      "logits/chosen": 0.2033652514219284,
-      "logits/rejected": 0.8105279803276062,
-      "logps/chosen": -432.13153076171875,
-      "logps/rejected": -513.7069702148438,
-      "loss": 0.4936,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.8029797673225403,
-      "rewards/margins": 1.0931544303894043,
-      "rewards/rejected": -1.8961341381072998,
+      "logits/chosen": -0.9394437670707703,
+      "logits/rejected": -0.6239842176437378,
+      "logps/chosen": -400.7373352050781,
+      "logps/rejected": -476.02685546875,
+      "loss": 0.4864,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.8022397756576538,
+      "rewards/margins": 1.024395227432251,
+      "rewards/rejected": -1.8266351222991943,
       "step": 410
     },
     {
       "epoch": 0.88,
-      "grad_norm": 43.3329326100997,
+      "grad_norm": 42.81869810726971,
       "learning_rate": 2.2111614344599684e-08,
-      "logits/chosen": 0.23255594074726105,
-      "logits/rejected": 0.6442626118659973,
-      "logps/chosen": -432.7608337402344,
-      "logps/rejected": -520.2620239257812,
-      "loss": 0.4641,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.8288372159004211,
-      "rewards/margins": 1.0977327823638916,
-      "rewards/rejected": -1.926569938659668,
+      "logits/chosen": -0.8887699246406555,
+      "logits/rejected": -0.7618988752365112,
+      "logps/chosen": -404.9107971191406,
+      "logps/rejected": -487.3875427246094,
+      "loss": 0.4799,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.8359407186508179,
+      "rewards/margins": 1.0478320121765137,
+      "rewards/rejected": -1.883772611618042,
       "step": 420
     },
     {
       "epoch": 0.9,
-      "grad_norm": 49.02509873372083,
+      "grad_norm": 52.25677628091801,
       "learning_rate": 1.521597710086439e-08,
-      "logits/chosen": 0.39820823073387146,
-      "logits/rejected": 0.7836225628852844,
-      "logps/chosen": -431.13739013671875,
-      "logps/rejected": -512.7986450195312,
-      "loss": 0.471,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.9906813502311707,
-      "rewards/margins": 1.0531766414642334,
-      "rewards/rejected": -2.043858051300049,
+      "logits/chosen": -0.7199336290359497,
+      "logits/rejected": -0.5324488282203674,
+      "logps/chosen": -401.64862060546875,
+      "logps/rejected": -478.2361755371094,
+      "loss": 0.4734,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.9977178573608398,
+      "rewards/margins": 0.997569739818573,
+      "rewards/rejected": -1.9952876567840576,
       "step": 430
     },
     {
       "epoch": 0.92,
-      "grad_norm": 46.801343844231866,
+      "grad_norm": 45.91250203680497,
       "learning_rate": 9.57301420397924e-09,
-      "logits/chosen": 0.2471725195646286,
-      "logits/rejected": 0.7467008829116821,
-      "logps/chosen": -417.58258056640625,
-      "logps/rejected": -499.89215087890625,
-      "loss": 0.4942,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.8245165944099426,
-      "rewards/margins": 0.9129160642623901,
-      "rewards/rejected": -1.737432837486267,
+      "logits/chosen": -0.801552951335907,
+      "logits/rejected": -0.615414023399353,
+      "logps/chosen": -388.91595458984375,
+      "logps/rejected": -466.5445861816406,
+      "loss": 0.4956,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.8414070010185242,
+      "rewards/margins": 0.872752845287323,
+      "rewards/rejected": -1.7141599655151367,
       "step": 440
     },
     {
       "epoch": 0.94,
-      "grad_norm": 46.37037120468359,
+      "grad_norm": 42.65993118046237,
       "learning_rate": 5.212833302556258e-09,
-      "logits/chosen": 0.4000117778778076,
-      "logits/rejected": 0.6783515810966492,
-      "logps/chosen": -436.942138671875,
-      "logps/rejected": -551.7266845703125,
-      "loss": 0.4786,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.9379842877388,
-      "rewards/margins": 0.9772690534591675,
-      "rewards/rejected": -1.9152534008026123,
+      "logits/chosen": -0.7094382047653198,
+      "logits/rejected": -0.6163294911384583,
+      "logps/chosen": -411.10333251953125,
+      "logps/rejected": -515.189208984375,
+      "loss": 0.4986,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.9838272929191589,
+      "rewards/margins": 0.8583157658576965,
+      "rewards/rejected": -1.8421430587768555,
       "step": 450
     },
     {
       "epoch": 0.96,
-      "grad_norm": 46.5176855624682,
+      "grad_norm": 61.47323172824489,
       "learning_rate": 2.158697848236607e-09,
-      "logits/chosen": 0.3254472017288208,
-      "logits/rejected": 0.7334408164024353,
-      "logps/chosen": -410.4520568847656,
-      "logps/rejected": -479.54351806640625,
-      "loss": 0.4804,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.8168198466300964,
-      "rewards/margins": 0.9972711801528931,
-      "rewards/rejected": -1.8140909671783447,
+      "logits/chosen": -0.750347912311554,
+      "logits/rejected": -0.5575055480003357,
+      "logps/chosen": -384.8453369140625,
+      "logps/rejected": -444.24237060546875,
+      "loss": 0.4922,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.8604202270507812,
+      "rewards/margins": 0.9049084782600403,
+      "rewards/rejected": -1.7653284072875977,
       "step": 460
     },
     {
       "epoch": 0.98,
-      "grad_norm": 42.92566705814915,
+      "grad_norm": 44.32569694586999,
       "learning_rate": 4.269029751107489e-10,
-      "logits/chosen": 0.2615105211734772,
-      "logits/rejected": 0.6600261926651001,
-      "logps/chosen": -408.7879638671875,
-      "logps/rejected": -514.4189453125,
-      "loss": 0.4762,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.8471711874008179,
-      "rewards/margins": 1.000536322593689,
-      "rewards/rejected": -1.8477073907852173,
+      "logits/chosen": -0.783599853515625,
+      "logits/rejected": -0.5731066465377808,
+      "logps/chosen": -376.4079895019531,
+      "logps/rejected": -473.00457763671875,
+      "loss": 0.4797,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.8160070180892944,
+      "rewards/margins": 0.9131675958633423,
+      "rewards/rejected": -1.7291743755340576,
       "step": 470
     },
     {
       "epoch": 1.0,
       "step": 478,
       "total_flos": 0.0,
-      "train_loss": 0.5266154430900158,
-      "train_runtime": 7640.9818,
-      "train_samples_per_second": 8.001,
+      "train_loss": 0.5340310439903866,
+      "train_runtime": 7643.694,
+      "train_samples_per_second": 7.998,
       "train_steps_per_second": 0.063
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c0584d705e1910899a2cefd1070bfd070f2e8f13e53448281c5e88cfb5c6a342
-size 6264
+oid sha256:436e163b2561b71ea89af4f478739ae2d75ffcde7a9b6ef219294fa6b53a7806
+size 6200