Model save

README.md

@@ -16,20 +16,20 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.9861
-- Rewards/chosen: -0.7020
-- Rewards/rejected: -0.9079
-- Rewards/gen: -2.6099
-- Rewards/accuracies: 0.6020
-- Rewards/margins: 0.2059
-- Logps/rejected: -280.7023
-- Logps/chosen: -315.6724
-- Logps/response: -196.7418
-- Logits/rejected: 0.8408
-- Logits/chosen: 0.8746
-- Logits/response: 0.9388
-- Improvement: 0.3049
-- Penalty: 0.6913
+- Loss: 1.3955
+- Rewards/chosen: -0.7655
+- Rewards/rejected: -1.1037
+- Rewards/gen: -0.9813
+- Rewards/accuracies: 0.6480
+- Rewards/margins: 0.3382
+- Logps/rejected: -291.7406
+- Logps/chosen: -323.3212
+- Logps/response: -206.5588
+- Logits/rejected: 1.0075
+- Logits/chosen: 1.0518
+- Logits/response: 1.1119
+- Improvement: 0.7646
+- Penalty: 0.6340
 
 ## Model description
 
@@ -66,9 +66,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/gen | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logps/response | Logits/rejected | Logits/chosen | Logits/response | Improvement | Penalty |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:-----------:|:------------------:|:---------------:|:--------------:|:------------:|:--------------:|:---------------:|:-------------:|:---------------:|:-----------:|:-------:|
-| 1.0506        | 0.3140 | 100  | 1.0429          | -0.3597        | -0.4629          | -1.7879     | 0.5620             | 0.1032          | -276.2524      | -312.2498    | -188.5220      | 0.8938          | 0.9341        | 0.9707          | 0.3487      | 0.7045  |
-| 0.9432        | 0.6281 | 200  | 0.9939          | -0.6203        | -0.8053          | -2.4531     | 0.5920             | 0.1850          | -279.6760      | -314.8556    | -195.1740      | 0.8562          | 0.8910        | 0.9488          | 0.3102      | 0.6945  |
-| 0.9766        | 0.9421 | 300  | 0.9861          | -0.7020        | -0.9079          | -2.6099     | 0.6020             | 0.2059          | -280.7023      | -315.6724    | -196.7418      | 0.8408          | 0.8746        | 0.9388          | 0.3049      | 0.6913  |
+| 0.6875        | 0.3140 | 100  | 1.3506          | -0.4754        | -0.6595          | -0.7565     | 0.6060             | 0.1841          | -287.2987      | -320.4206    | -204.3113      | 1.0762          | 1.1203        | 1.1703          | 0.6764      | 0.6691  |
+| 0.6367        | 0.6281 | 200  | 1.4008          | -0.6776        | -1.0030          | -0.8686     | 0.6420             | 0.3254          | -290.7334      | -322.4421    | -205.4324      | 1.0335          | 1.0745        | 1.1297          | 0.7678      | 0.6372  |
+| 0.6335        | 0.9421 | 300  | 1.3955          | -0.7655        | -1.1037          | -0.9813     | 0.6480             | 0.3382          | -291.7406      | -323.3212    | -206.5588      | 1.0075          | 1.0518        | 1.1119          | 0.7646      | 0.6340  |
 
 
 ### Framework versions

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9986261040235525,
     "total_flos": 0.0,
-    "train_loss": 1.0325740605780163,
-    "train_runtime": 3343.8277,
+    "train_loss": 0.7174485007172111,
+    "train_runtime": 3896.582,
     "train_samples": 20378,
-    "train_samples_per_second": 6.094,
-    "train_steps_per_second": 0.095
+    "train_samples_per_second": 5.23,
+    "train_steps_per_second": 0.082
 }

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9986261040235525,
     "total_flos": 0.0,
-    "train_loss": 1.0325740605780163,
-    "train_runtime": 3343.8277,
+    "train_loss": 0.7174485007172111,
+    "train_runtime": 3896.582,
     "train_samples": 20378,
-    "train_samples_per_second": 6.094,
-    "train_steps_per_second": 0.095
+    "train_samples_per_second": 5.23,
+    "train_steps_per_second": 0.082
 }

trainer_state.json

@@ -10,15 +10,15 @@
   "log_history": [
     {
       "epoch": 0.003140333660451423,
-      "grad_norm": 90.99359893798828,
+      "grad_norm": 200.00250244140625,
       "improvement": 0.6931471824645996,
       "learning_rate": 1.5625e-08,
-      "logits/chosen": 1.3514697551727295,
-      "logits/rejected": 1.4558440446853638,
-      "logits/response": 1.3213673830032349,
-      "logps/chosen": -256.549560546875,
-      "logps/rejected": -246.02056884765625,
-      "logps/response": -111.43739318847656,
+      "logits/chosen": 1.1002808809280396,
+      "logits/rejected": 1.2185591459274292,
+      "logits/response": 1.4746066331863403,
+      "logps/chosen": -266.4275207519531,
+      "logps/rejected": -258.69293212890625,
+      "logps/response": -758.1343994140625,
       "loss": 1.3863,
       "penalty": 0.6931471824645996,
       "rewards/accuracies": 0.0,
@@ -30,695 +30,695 @@
     },
     {
       "epoch": 0.03140333660451423,
-      "grad_norm": 88.48505401611328,
-      "improvement": 0.6908075213432312,
+      "grad_norm": 167.06924438476562,
+      "improvement": 0.6249845623970032,
       "learning_rate": 1.5624999999999999e-07,
-      "logits/chosen": 1.0925401449203491,
-      "logits/rejected": 1.2437928915023804,
-      "logits/response": 1.1619499921798706,
-      "logps/chosen": -286.2481994628906,
-      "logps/rejected": -304.1895751953125,
-      "logps/response": -179.84832763671875,
-      "loss": 1.3863,
-      "penalty": 0.6960046291351318,
-      "rewards/accuracies": 0.4097222089767456,
-      "rewards/chosen": 0.0026971669867634773,
-      "rewards/gen": -0.0007864322979003191,
-      "rewards/margins": -0.004067544359713793,
-      "rewards/rejected": 0.006764710880815983,
+      "logits/chosen": 1.2528102397918701,
+      "logits/rejected": 1.1278939247131348,
+      "logits/response": 1.543837308883667,
+      "logps/chosen": -338.7834167480469,
+      "logps/rejected": -262.37567138671875,
+      "logps/response": -920.6803588867188,
+      "loss": 1.3201,
+      "penalty": 0.6959547996520996,
+      "rewards/accuracies": 0.4305555522441864,
+      "rewards/chosen": 0.010056542232632637,
+      "rewards/gen": -0.14015476405620575,
+      "rewards/margins": -0.004349019844084978,
+      "rewards/rejected": 0.014405561611056328,
       "step": 10
     },
     {
       "epoch": 0.06280667320902845,
-      "grad_norm": 81.83534240722656,
-      "improvement": 0.6806281805038452,
+      "grad_norm": 74.37409973144531,
+      "improvement": 0.2921519875526428,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": 1.1330327987670898,
-      "logits/rejected": 1.1094038486480713,
-      "logits/response": 1.12299382686615,
-      "logps/chosen": -265.14605712890625,
-      "logps/rejected": -268.95941162109375,
-      "logps/response": -156.56362915039062,
-      "loss": 1.3741,
-      "penalty": 0.6888696551322937,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.004303407855331898,
-      "rewards/gen": -0.027122128754854202,
-      "rewards/margins": 0.01006249524652958,
-      "rewards/rejected": -0.005759088322520256,
+      "logits/chosen": 1.2214809656143188,
+      "logits/rejected": 1.1795613765716553,
+      "logits/response": 1.5969406366348267,
+      "logps/chosen": -302.14825439453125,
+      "logps/rejected": -276.88629150390625,
+      "logps/response": -1001.9197387695312,
+      "loss": 1.0297,
+      "penalty": 0.7252423763275146,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": 0.050227295607328415,
+      "rewards/gen": -1.1749426126480103,
+      "rewards/margins": -0.05469644069671631,
+      "rewards/rejected": 0.10492374747991562,
       "step": 20
     },
     {
       "epoch": 0.09421000981354269,
-      "grad_norm": 77.37152099609375,
-      "improvement": 0.6238707304000854,
+      "grad_norm": 52.853057861328125,
+      "improvement": 0.08411481231451035,
       "learning_rate": 4.6874999999999996e-07,
-      "logits/chosen": 1.2423532009124756,
-      "logits/rejected": 1.0369462966918945,
-      "logits/response": 1.284911870956421,
-      "logps/chosen": -366.21832275390625,
-      "logps/rejected": -271.5616149902344,
-      "logps/response": -191.85791015625,
-      "loss": 1.3185,
-      "penalty": 0.6943666934967041,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.013009263202548027,
-      "rewards/gen": -0.16623075306415558,
-      "rewards/margins": 0.0020996746607124805,
-      "rewards/rejected": -0.015108938328921795,
+      "logits/chosen": 1.488523244857788,
+      "logits/rejected": 1.3755172491073608,
+      "logits/response": 1.854111909866333,
+      "logps/chosen": -329.28521728515625,
+      "logps/rejected": -283.4335021972656,
+      "logps/response": -953.9349365234375,
+      "loss": 0.8522,
+      "penalty": 0.7766379714012146,
+      "rewards/accuracies": 0.34375,
+      "rewards/chosen": 0.03275767341256142,
+      "rewards/gen": -3.0305302143096924,
+      "rewards/margins": -0.12535743415355682,
+      "rewards/rejected": 0.15811510384082794,
       "step": 30
     },
     {
       "epoch": 0.1256133464180569,
-      "grad_norm": 61.32085037231445,
-      "improvement": 0.5091749429702759,
+      "grad_norm": 55.16160583496094,
+      "improvement": 0.044371988624334335,
       "learning_rate": 4.990353313429303e-07,
-      "logits/chosen": 1.388942837715149,
-      "logits/rejected": 1.2175312042236328,
-      "logits/response": 1.2681037187576294,
-      "logps/chosen": -350.94537353515625,
-      "logps/rejected": -293.6703186035156,
-      "logps/response": -179.65731811523438,
-      "loss": 1.2024,
-      "penalty": 0.6836713552474976,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.07153712213039398,
-      "rewards/gen": -0.587524950504303,
-      "rewards/margins": 0.04365935176610947,
-      "rewards/rejected": -0.11519646644592285,
+      "logits/chosen": 1.4546756744384766,
+      "logits/rejected": 1.3696863651275635,
+      "logits/response": 2.0380258560180664,
+      "logps/chosen": -291.39862060546875,
+      "logps/rejected": -258.83160400390625,
+      "logps/response": -974.2811279296875,
+      "loss": 0.8206,
+      "penalty": 0.794792890548706,
+      "rewards/accuracies": 0.41874998807907104,
+      "rewards/chosen": -0.042534660547971725,
+      "rewards/gen": -4.516351222991943,
+      "rewards/margins": -0.1319345235824585,
+      "rewards/rejected": 0.08939988166093826,
       "step": 40
     },
     {
       "epoch": 0.15701668302257116,
-      "grad_norm": 52.960693359375,
-      "improvement": 0.43595728278160095,
+      "grad_norm": 52.997276306152344,
+      "improvement": 0.039832912385463715,
       "learning_rate": 4.951291206355559e-07,
-      "logits/chosen": 1.4330917596817017,
-      "logits/rejected": 1.3490939140319824,
-      "logits/response": 1.453476905822754,
-      "logps/chosen": -284.6172180175781,
-      "logps/rejected": -279.71728515625,
-      "logps/response": -189.18643188476562,
-      "loss": 1.1356,
-      "penalty": 0.7028869986534119,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.20936663448810577,
-      "rewards/gen": -1.1179193258285522,
-      "rewards/margins": 0.05230969190597534,
-      "rewards/rejected": -0.2616763114929199,
+      "logits/chosen": 1.4147427082061768,
+      "logits/rejected": 1.3478193283081055,
+      "logits/response": 1.897972822189331,
+      "logps/chosen": -316.64971923828125,
+      "logps/rejected": -287.1830749511719,
+      "logps/response": -962.357421875,
+      "loss": 0.8111,
+      "penalty": 0.75470370054245,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.02022326923906803,
+      "rewards/gen": -4.572790145874023,
+      "rewards/margins": -0.057704925537109375,
+      "rewards/rejected": 0.03748166561126709,
       "step": 50
     },
     {
       "epoch": 0.18842001962708538,
-      "grad_norm": 59.61778259277344,
-      "improvement": 0.3726009428501129,
+      "grad_norm": 51.460330963134766,
+      "improvement": 0.04238835722208023,
       "learning_rate": 4.882681251368548e-07,
-      "logits/chosen": 1.404496192932129,
-      "logits/rejected": 1.3673858642578125,
-      "logits/response": 1.3688445091247559,
-      "logps/chosen": -329.80340576171875,
-      "logps/rejected": -319.18133544921875,
-      "logps/response": -194.2289581298828,
-      "loss": 1.1305,
-      "penalty": 0.7743828892707825,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.4070376753807068,
-      "rewards/gen": -1.5880019664764404,
-      "rewards/margins": -0.050781238824129105,
-      "rewards/rejected": -0.3562564253807068,
+      "logits/chosen": 1.493253469467163,
+      "logits/rejected": 1.5756161212921143,
+      "logits/response": 2.1205339431762695,
+      "logps/chosen": -271.7294616699219,
+      "logps/rejected": -284.16998291015625,
+      "logps/response": -965.3502807617188,
+      "loss": 0.7701,
+      "penalty": 0.6884164214134216,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": 0.01879013516008854,
+      "rewards/gen": -4.314782619476318,
+      "rewards/margins": 0.05694611743092537,
+      "rewards/rejected": -0.03815598413348198,
       "step": 60
     },
     {
       "epoch": 0.2198233562315996,
-      "grad_norm": 50.95313262939453,
-      "improvement": 0.3971942067146301,
+      "grad_norm": 47.554962158203125,
+      "improvement": 0.05140012502670288,
       "learning_rate": 4.785350472409791e-07,
-      "logits/chosen": 1.4829801321029663,
-      "logits/rejected": 1.3746827840805054,
-      "logits/response": 1.4223880767822266,
-      "logps/chosen": -297.8169860839844,
-      "logps/rejected": -259.929443359375,
-      "logps/response": -165.90725708007812,
-      "loss": 1.0959,
-      "penalty": 0.6936275959014893,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.3267964720726013,
-      "rewards/gen": -1.5778385400772095,
-      "rewards/margins": 0.11241116374731064,
-      "rewards/rejected": -0.43920764327049255,
+      "logits/chosen": 1.4707852602005005,
+      "logits/rejected": 1.3398946523666382,
+      "logits/response": 1.9091663360595703,
+      "logps/chosen": -293.58685302734375,
+      "logps/rejected": -277.17047119140625,
+      "logps/response": -1005.49560546875,
+      "loss": 0.7433,
+      "penalty": 0.6779423952102661,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.21125507354736328,
+      "rewards/gen": -4.562561511993408,
+      "rewards/margins": 0.09027546644210815,
+      "rewards/rejected": -0.30153053998947144,
       "step": 70
     },
     {
       "epoch": 0.2512266928361138,
-      "grad_norm": 50.096981048583984,
-      "improvement": 0.3729608952999115,
+      "grad_norm": 52.48830795288086,
+      "improvement": 0.05445639044046402,
       "learning_rate": 4.6604720940421207e-07,
-      "logits/chosen": 1.3744746446609497,
-      "logits/rejected": 1.2923038005828857,
-      "logits/response": 1.3090846538543701,
-      "logps/chosen": -313.1800537109375,
-      "logps/rejected": -258.7066955566406,
-      "logps/response": -175.9040985107422,
-      "loss": 1.1115,
-      "penalty": 0.7144418358802795,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.3523169457912445,
-      "rewards/gen": -1.8431284427642822,
-      "rewards/margins": 0.10072679817676544,
-      "rewards/rejected": -0.45304378867149353,
+      "logits/chosen": 1.3364737033843994,
+      "logits/rejected": 1.348964810371399,
+      "logits/response": 1.8606315851211548,
+      "logps/chosen": -330.0252685546875,
+      "logps/rejected": -295.9020080566406,
+      "logps/response": -994.1389770507812,
+      "loss": 0.7334,
+      "penalty": 0.6729768514633179,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.43410032987594604,
+      "rewards/gen": -4.849533557891846,
+      "rewards/margins": 0.1275758445262909,
+      "rewards/rejected": -0.5616761445999146,
       "step": 80
     },
     {
       "epoch": 0.2826300294406281,
-      "grad_norm": 51.301666259765625,
-      "improvement": 0.3327198624610901,
+      "grad_norm": 48.90186309814453,
+      "improvement": 0.03790447860956192,
       "learning_rate": 4.5095513994085974e-07,
-      "logits/chosen": 1.3864705562591553,
-      "logits/rejected": 1.2353299856185913,
-      "logits/response": 1.3959097862243652,
-      "logps/chosen": -324.7561340332031,
-      "logps/rejected": -244.14535522460938,
-      "logps/response": -200.8162078857422,
-      "loss": 1.0443,
-      "penalty": 0.6970937252044678,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.30312663316726685,
-      "rewards/gen": -1.87189519405365,
-      "rewards/margins": 0.10124621540307999,
-      "rewards/rejected": -0.4043728709220886,
+      "logits/chosen": 1.3569543361663818,
+      "logits/rejected": 1.298161506652832,
+      "logits/response": 1.9093761444091797,
+      "logps/chosen": -333.34332275390625,
+      "logps/rejected": -314.8207092285156,
+      "logps/response": -995.0445556640625,
+      "loss": 0.6955,
+      "penalty": 0.6896320581436157,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.5641836524009705,
+      "rewards/gen": -5.513597011566162,
+      "rewards/margins": 0.11063194274902344,
+      "rewards/rejected": -0.6748155355453491,
       "step": 90
     },
     {
       "epoch": 0.3140333660451423,
-      "grad_norm": 46.86961364746094,
-      "improvement": 0.3321753144264221,
+      "grad_norm": 43.70199966430664,
+      "improvement": 0.045615434646606445,
       "learning_rate": 4.3344075855595097e-07,
-      "logits/chosen": 1.1061931848526,
-      "logits/rejected": 1.1866940259933472,
-      "logits/response": 1.2408634424209595,
-      "logps/chosen": -272.04541015625,
-      "logps/rejected": -263.6874084472656,
-      "logps/response": -188.56777954101562,
-      "loss": 1.0506,
-      "penalty": 0.6661034226417542,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.30777376890182495,
-      "rewards/gen": -1.7470309734344482,
-      "rewards/margins": 0.17530158162117004,
-      "rewards/rejected": -0.4830753207206726,
+      "logits/chosen": 1.3917421102523804,
+      "logits/rejected": 1.3425309658050537,
+      "logits/response": 1.9128587245941162,
+      "logps/chosen": -337.1028747558594,
+      "logps/rejected": -292.86566162109375,
+      "logps/response": -978.150390625,
+      "loss": 0.6875,
+      "penalty": 0.6265669465065002,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.48066097497940063,
+      "rewards/gen": -5.6218461990356445,
+      "rewards/margins": 0.2668846547603607,
+      "rewards/rejected": -0.747545599937439,
       "step": 100
     },
     {
       "epoch": 0.3140333660451423,
-      "eval_improvement": 0.34868791699409485,
-      "eval_logits/chosen": 0.9340688586235046,
-      "eval_logits/rejected": 0.8938046097755432,
-      "eval_logits/response": 0.9706884026527405,
-      "eval_logps/chosen": -312.2497863769531,
-      "eval_logps/rejected": -276.25238037109375,
-      "eval_logps/response": -188.52197265625,
-      "eval_loss": 1.042933702468872,
-      "eval_penalty": 0.704525351524353,
-      "eval_rewards/accuracies": 0.5619999766349792,
-      "eval_rewards/chosen": -0.35974380373954773,
-      "eval_rewards/gen": -1.787913203239441,
-      "eval_rewards/margins": 0.10319552570581436,
-      "eval_rewards/rejected": -0.4629393219947815,
-      "eval_runtime": 133.7047,
-      "eval_samples_per_second": 14.958,
-      "eval_steps_per_second": 1.87,
+      "eval_improvement": 0.6763742566108704,
+      "eval_logits/chosen": 1.120324730873108,
+      "eval_logits/rejected": 1.0761666297912598,
+      "eval_logits/response": 1.170290231704712,
+      "eval_logps/chosen": -320.4205627441406,
+      "eval_logps/rejected": -287.2986755371094,
+      "eval_logps/response": -204.3113250732422,
+      "eval_loss": 1.3505975008010864,
+      "eval_penalty": 0.6691088080406189,
+      "eval_rewards/accuracies": 0.6060000061988831,
+      "eval_rewards/chosen": -0.47539979219436646,
+      "eval_rewards/gen": -0.756511926651001,
+      "eval_rewards/margins": 0.18411880731582642,
+      "eval_rewards/rejected": -0.6595185995101929,
+      "eval_runtime": 141.2797,
+      "eval_samples_per_second": 14.156,
+      "eval_steps_per_second": 1.77,
       "step": 100
     },
     {
       "epoch": 0.34543670264965654,
-      "grad_norm": 47.566768646240234,
-      "improvement": 0.310733437538147,
+      "grad_norm": 50.130496978759766,
+      "improvement": 0.03334973007440567,
       "learning_rate": 4.137151834863213e-07,
-      "logits/chosen": 1.0738614797592163,
-      "logits/rejected": 1.086809515953064,
-      "logits/response": 1.1392853260040283,
-      "logps/chosen": -295.4719543457031,
-      "logps/rejected": -279.11383056640625,
-      "logps/response": -192.47640991210938,
-      "loss": 1.0474,
-      "penalty": 0.6999384760856628,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.3630922734737396,
-      "rewards/gen": -1.9901072978973389,
-      "rewards/margins": 0.14921404421329498,
-      "rewards/rejected": -0.5123063325881958,
+      "logits/chosen": 1.4784291982650757,
+      "logits/rejected": 1.4504770040512085,
+      "logits/response": 2.0669291019439697,
+      "logps/chosen": -282.5090026855469,
+      "logps/rejected": -292.196533203125,
+      "logps/response": -916.5537109375,
+      "loss": 0.6779,
+      "penalty": 0.6117620468139648,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.41435232758522034,
+      "rewards/gen": -6.181846618652344,
+      "rewards/margins": 0.31640559434890747,
+      "rewards/rejected": -0.730758011341095,
       "step": 110
     },
     {
       "epoch": 0.37684003925417076,
-      "grad_norm": 47.566810607910156,
-      "improvement": 0.2977697253227234,
+      "grad_norm": 52.74955749511719,
+      "improvement": 0.058611877262592316,
       "learning_rate": 3.920161866827889e-07,
-      "logits/chosen": 1.3284032344818115,
-      "logits/rejected": 1.2398704290390015,
-      "logits/response": 1.488197922706604,
-      "logps/chosen": -347.5028991699219,
-      "logps/rejected": -263.88165283203125,
-      "logps/response": -211.24081420898438,
-      "loss": 0.9844,
-      "penalty": 0.6048867702484131,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.2518952190876007,
-      "rewards/gen": -2.0901896953582764,
-      "rewards/margins": 0.3502174913883209,
-      "rewards/rejected": -0.6021127104759216,
+      "logits/chosen": 1.3459668159484863,
+      "logits/rejected": 1.2685140371322632,
+      "logits/response": 1.8824383020401,
+      "logps/chosen": -362.947509765625,
+      "logps/rejected": -320.85797119140625,
+      "logps/response": -1031.55517578125,
+      "loss": 0.7052,
+      "penalty": 0.6450746655464172,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.8527549505233765,
+      "rewards/gen": -6.363945007324219,
+      "rewards/margins": 0.3166798949241638,
+      "rewards/rejected": -1.169434905052185,
       "step": 120
     },
     {
       "epoch": 0.408243375858685,
-      "grad_norm": 45.997779846191406,
-      "improvement": 0.3221287727355957,
+      "grad_norm": 44.321163177490234,
+      "improvement": 0.036691777408123016,
       "learning_rate": 3.6860532770864005e-07,
-      "logits/chosen": 1.2325729131698608,
-      "logits/rejected": 1.1537472009658813,
-      "logits/response": 1.307138204574585,
-      "logps/chosen": -286.8653564453125,
-      "logps/rejected": -271.20159912109375,
-      "logps/response": -182.6356658935547,
-      "loss": 0.9735,
-      "penalty": 0.6481137275695801,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.3536524772644043,
-      "rewards/gen": -1.9381176233291626,
-      "rewards/margins": 0.21084347367286682,
-      "rewards/rejected": -0.5644959211349487,
+      "logits/chosen": 1.4267325401306152,
+      "logits/rejected": 1.4107747077941895,
+      "logits/response": 1.9945790767669678,
+      "logps/chosen": -290.82928466796875,
+      "logps/rejected": -272.4383239746094,
+      "logps/response": -988.95263671875,
+      "loss": 0.6601,
+      "penalty": 0.6543839573860168,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.7737542390823364,
+      "rewards/gen": -6.266350269317627,
+      "rewards/margins": 0.24219664931297302,
+      "rewards/rejected": -1.0159507989883423,
       "step": 130
     },
     {
       "epoch": 0.4396467124631992,
-      "grad_norm": 42.91578674316406,
-      "improvement": 0.3166593611240387,
+      "grad_norm": 46.13864517211914,
+      "improvement": 0.03532712906599045,
       "learning_rate": 3.4376480090239047e-07,
-      "logits/chosen": 1.259896993637085,
-      "logits/rejected": 1.2792214155197144,
-      "logits/response": 1.3045755624771118,
-      "logps/chosen": -300.57342529296875,
-      "logps/rejected": -275.1106262207031,
-      "logps/response": -190.06996154785156,
-      "loss": 0.9986,
-      "penalty": 0.6845930814743042,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.44455188512802124,
-      "rewards/gen": -2.133059501647949,
-      "rewards/margins": 0.1889132261276245,
-      "rewards/rejected": -0.6334651708602905,
+      "logits/chosen": 1.4152085781097412,
+      "logits/rejected": 1.2808418273925781,
+      "logits/response": 1.9436925649642944,
+      "logps/chosen": -276.03021240234375,
+      "logps/rejected": -248.2073211669922,
+      "logps/response": -1000.24462890625,
+      "loss": 0.6838,
+      "penalty": 0.6066700220108032,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.6766573190689087,
+      "rewards/gen": -6.422157287597656,
+      "rewards/margins": 0.367643803358078,
+      "rewards/rejected": -1.044301152229309,
       "step": 140
     },
     {
       "epoch": 0.47105004906771347,
-      "grad_norm": 43.6589241027832,
-      "improvement": 0.3184241056442261,
+      "grad_norm": 63.85268783569336,
+      "improvement": 0.03399471566081047,
       "learning_rate": 3.1779403380910425e-07,
-      "logits/chosen": 1.1337382793426514,
-      "logits/rejected": 1.1580231189727783,
-      "logits/response": 1.1197118759155273,
-      "logps/chosen": -277.9029846191406,
-      "logps/rejected": -250.3749542236328,
-      "logps/response": -191.55584716796875,
-      "loss": 0.9929,
-      "penalty": 0.6434761881828308,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.38797393441200256,
-      "rewards/gen": -2.0863261222839355,
-      "rewards/margins": 0.23583289980888367,
-      "rewards/rejected": -0.6238068342208862,
+      "logits/chosen": 1.3651916980743408,
+      "logits/rejected": 1.3227938413619995,
+      "logits/response": 1.9420855045318604,
+      "logps/chosen": -297.1623840332031,
+      "logps/rejected": -276.270751953125,
+      "logps/response": -1037.3096923828125,
+      "loss": 0.668,
+      "penalty": 0.6184068918228149,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.6357800960540771,
+      "rewards/gen": -7.107823371887207,
+      "rewards/margins": 0.3691752851009369,
+      "rewards/rejected": -1.0049554109573364,
       "step": 150
     },
     {
       "epoch": 0.5024533856722276,
-      "grad_norm": 49.80767059326172,
-      "improvement": 0.3020181953907013,
+      "grad_norm": 55.31007766723633,
+      "improvement": 0.04934271052479744,
       "learning_rate": 2.910060778827554e-07,
-      "logits/chosen": 1.080392599105835,
-      "logits/rejected": 1.1482455730438232,
-      "logits/response": 1.1770050525665283,
-      "logps/chosen": -302.5500183105469,
-      "logps/rejected": -283.81341552734375,
-      "logps/response": -196.9333953857422,
-      "loss": 0.9816,
-      "penalty": 0.6278713345527649,
+      "logits/chosen": 1.3888250589370728,
+      "logits/rejected": 1.2889330387115479,
+      "logits/response": 1.8870728015899658,
+      "logps/chosen": -378.60809326171875,
+      "logps/rejected": -333.920654296875,
+      "logps/response": -1007.1419067382812,
+      "loss": 0.6667,
+      "penalty": 0.6215599775314331,
       "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.4150875210762024,
-      "rewards/gen": -2.412015199661255,
-      "rewards/margins": 0.31777530908584595,
-      "rewards/rejected": -0.7328628301620483,
+      "rewards/chosen": -0.671469509601593,
+      "rewards/gen": -6.274459362030029,
+      "rewards/margins": 0.35334306955337524,
+      "rewards/rejected": -1.0248124599456787,
       "step": 160
     },
     {
       "epoch": 0.5338567222767419,
-      "grad_norm": 62.087135314941406,
-      "improvement": 0.3313751220703125,
+      "grad_norm": 52.735923767089844,
+      "improvement": 0.03530705347657204,
       "learning_rate": 2.6372383496608186e-07,
-      "logits/chosen": 1.2298156023025513,
-      "logits/rejected": 1.187656044960022,
-      "logits/response": 1.1628153324127197,
-      "logps/chosen": -319.54022216796875,
-      "logps/rejected": -319.6458740234375,
-      "logps/response": -189.41993713378906,
-      "loss": 0.9651,
-      "penalty": 0.7312244176864624,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -0.5790542364120483,
-      "rewards/gen": -2.1839423179626465,
-      "rewards/margins": 0.11392831802368164,
-      "rewards/rejected": -0.69298255443573,
+      "logits/chosen": 1.3521907329559326,
+      "logits/rejected": 1.3136357069015503,
+      "logits/response": 1.958385705947876,
+      "logps/chosen": -309.96441650390625,
+      "logps/rejected": -290.8580627441406,
+      "logps/response": -1000.8353271484375,
+      "loss": 0.6716,
+      "penalty": 0.6742448210716248,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.5993863940238953,
+      "rewards/gen": -6.6541900634765625,
+      "rewards/margins": 0.18541964888572693,
+      "rewards/rejected": -0.7848061323165894,
       "step": 170
     },
     {
       "epoch": 0.5652600588812562,
-      "grad_norm": 48.49706268310547,
-      "improvement": 0.34473639726638794,
+      "grad_norm": 59.980445861816406,
+      "improvement": 0.047461725771427155,
       "learning_rate": 2.3627616503391812e-07,
-      "logits/chosen": 1.1173150539398193,
-      "logits/rejected": 1.1408215761184692,
-      "logits/response": 1.1199573278427124,
-      "logps/chosen": -283.539306640625,
-      "logps/rejected": -250.66598510742188,
-      "logps/response": -179.87173461914062,
-      "loss": 0.9908,
-      "penalty": 0.6365185379981995,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.39007699489593506,
-      "rewards/gen": -2.085050106048584,
-      "rewards/margins": 0.3132508099079132,
-      "rewards/rejected": -0.7033277750015259,
+      "logits/chosen": 1.546133041381836,
+      "logits/rejected": 1.4397121667861938,
+      "logits/response": 2.062786340713501,
+      "logps/chosen": -307.1769104003906,
+      "logps/rejected": -260.46331787109375,
+      "logps/response": -957.2349853515625,
+      "loss": 0.6516,
+      "penalty": 0.5987246632575989,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.5737658739089966,
+      "rewards/gen": -6.288596153259277,
+      "rewards/margins": 0.36477407813072205,
+      "rewards/rejected": -0.938539981842041,
       "step": 180
     },
     {
       "epoch": 0.5966633954857704,
-      "grad_norm": 52.51746368408203,
-      "improvement": 0.3269854485988617,
+      "grad_norm": 48.051780700683594,
+      "improvement": 0.03724605590105057,
       "learning_rate": 2.089939221172446e-07,
-      "logits/chosen": 1.2556055784225464,
-      "logits/rejected": 1.26613450050354,
-      "logits/response": 1.2274242639541626,
-      "logps/chosen": -280.4549255371094,
-      "logps/rejected": -285.5497741699219,
-      "logps/response": -159.73428344726562,
-      "loss": 0.9521,
-      "penalty": 0.6330380439758301,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.3872259557247162,
-      "rewards/gen": -2.0674312114715576,
-      "rewards/margins": 0.30792227387428284,
-      "rewards/rejected": -0.6951482892036438,
+      "logits/chosen": 1.3898411989212036,
+      "logits/rejected": 1.3639791011810303,
+      "logits/response": 1.9668989181518555,
+      "logps/chosen": -292.2279052734375,
+      "logps/rejected": -282.55889892578125,
+      "logps/response": -946.9928588867188,
+      "loss": 0.6857,
+      "penalty": 0.660628616809845,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.6309083700180054,
+      "rewards/gen": -6.227112770080566,
+      "rewards/margins": 0.27697476744651794,
+      "rewards/rejected": -0.9078830480575562,
       "step": 190
     },
     {
       "epoch": 0.6280667320902846,
-      "grad_norm": 50.308650970458984,
-      "improvement": 0.29781386256217957,
+      "grad_norm": 50.111351013183594,
+      "improvement": 0.04991893097758293,
       "learning_rate": 1.8220596619089573e-07,
-      "logits/chosen": 1.2265857458114624,
-      "logits/rejected": 1.2468065023422241,
-      "logits/response": 1.3728151321411133,
-      "logps/chosen": -271.88836669921875,
-      "logps/rejected": -263.320068359375,
-      "logps/response": -174.91575622558594,
-      "loss": 0.9432,
-      "penalty": 0.6788309812545776,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.5131560564041138,
-      "rewards/gen": -2.364034652709961,
-      "rewards/margins": 0.21325425803661346,
-      "rewards/rejected": -0.7264103293418884,
+      "logits/chosen": 1.3693121671676636,
+      "logits/rejected": 1.301695466041565,
+      "logits/response": 1.932735800743103,
+      "logps/chosen": -292.2115478515625,
+      "logps/rejected": -260.68133544921875,
+      "logps/response": -974.4039916992188,
+      "loss": 0.6367,
+      "penalty": 0.6076933741569519,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.6337991952896118,
+      "rewards/gen": -6.308970928192139,
+      "rewards/margins": 0.385838121175766,
+      "rewards/rejected": -1.0196373462677002,
       "step": 200
     },
     {
       "epoch": 0.6280667320902846,
-      "eval_improvement": 0.31024378538131714,
-      "eval_logits/chosen": 0.890977680683136,
-      "eval_logits/rejected": 0.8562321662902832,
-      "eval_logits/response": 0.9488180875778198,
-      "eval_logps/chosen": -314.8556213378906,
-      "eval_logps/rejected": -279.6759948730469,
-      "eval_logps/response": -195.17404174804688,
-      "eval_loss": 0.9939214587211609,
-      "eval_penalty": 0.6945350170135498,
-      "eval_rewards/accuracies": 0.5920000076293945,
-      "eval_rewards/chosen": -0.6203244924545288,
-      "eval_rewards/gen": -2.453124523162842,
-      "eval_rewards/margins": 0.18497607111930847,
-      "eval_rewards/rejected": -0.8053005337715149,
-      "eval_runtime": 131.2858,
-      "eval_samples_per_second": 15.234,
-      "eval_steps_per_second": 1.904,
+      "eval_improvement": 0.7678172588348389,
+      "eval_logits/chosen": 1.0745381116867065,
+      "eval_logits/rejected": 1.0335055589675903,
+      "eval_logits/response": 1.1297274827957153,
+      "eval_logps/chosen": -322.44207763671875,
+      "eval_logps/rejected": -290.7333679199219,
+      "eval_logps/response": -205.4324493408203,
+      "eval_loss": 1.400818109512329,
+      "eval_penalty": 0.637177050113678,
+      "eval_rewards/accuracies": 0.6420000195503235,
+      "eval_rewards/chosen": -0.6775563359260559,
+      "eval_rewards/gen": -0.8686242699623108,
+      "eval_rewards/margins": 0.32543402910232544,
+      "eval_rewards/rejected": -1.0029902458190918,
+      "eval_runtime": 131.9039,
+      "eval_samples_per_second": 15.163,
+      "eval_steps_per_second": 1.895,
       "step": 200
     },
     {
       "epoch": 0.6594700686947988,
-      "grad_norm": 46.08613967895508,
-      "improvement": 0.3185407519340515,
+      "grad_norm": 50.02403259277344,
+      "improvement": 0.03354328125715256,
       "learning_rate": 1.562351990976095e-07,
-      "logits/chosen": 1.124887228012085,
-      "logits/rejected": 1.2165838479995728,
-      "logits/response": 1.2256819009780884,
-      "logps/chosen": -318.11102294921875,
-      "logps/rejected": -313.8572082519531,
-      "logps/response": -191.56932067871094,
-      "loss": 0.9183,
-      "penalty": 0.6234660148620605,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.4947032034397125,
-      "rewards/gen": -2.4716310501098633,
-      "rewards/margins": 0.3273914158344269,
-      "rewards/rejected": -0.8220946192741394,
+      "logits/chosen": 1.250815749168396,
+      "logits/rejected": 1.2162940502166748,
+      "logits/response": 1.792128562927246,
+      "logps/chosen": -315.5356750488281,
+      "logps/rejected": -285.4361267089844,
+      "logps/response": -1000.24658203125,
+      "loss": 0.643,
+      "penalty": 0.6080543994903564,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.6579871773719788,
+      "rewards/gen": -6.532958984375,
+      "rewards/margins": 0.366300493478775,
+      "rewards/rejected": -1.0242877006530762,
       "step": 210
     },
     {
       "epoch": 0.6908734052993131,
-      "grad_norm": 49.8296012878418,
-      "improvement": 0.2562139630317688,
+      "grad_norm": 48.42168045043945,
+      "improvement": 0.03473372012376785,
       "learning_rate": 1.3139467229135998e-07,
-      "logits/chosen": 1.0995049476623535,
-      "logits/rejected": 1.1437008380889893,
-      "logits/response": 1.1871978044509888,
-      "logps/chosen": -310.8740539550781,
-      "logps/rejected": -332.66162109375,
-      "logps/response": -202.8819580078125,
-      "loss": 0.9309,
-      "penalty": 0.6312650442123413,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.5009514093399048,
-      "rewards/gen": -2.697756052017212,
-      "rewards/margins": 0.3737282156944275,
-      "rewards/rejected": -0.874679684638977,
+      "logits/chosen": 1.3695895671844482,
+      "logits/rejected": 1.2674177885055542,
+      "logits/response": 1.913102149963379,
+      "logps/chosen": -317.5673522949219,
+      "logps/rejected": -282.95709228515625,
+      "logps/response": -1023.9091796875,
+      "loss": 0.6433,
+      "penalty": 0.580784797668457,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.6436624526977539,
+      "rewards/gen": -6.871123313903809,
+      "rewards/margins": 0.46090683341026306,
+      "rewards/rejected": -1.1045693159103394,
       "step": 220
     },
     {
       "epoch": 0.7222767419038273,
-      "grad_norm": 43.834163665771484,
-      "improvement": 0.30367541313171387,
+      "grad_norm": 48.59343338012695,
+      "improvement": 0.047753967344760895,
       "learning_rate": 1.0798381331721107e-07,
-      "logits/chosen": 1.1779167652130127,
-      "logits/rejected": 1.0963248014450073,
-      "logits/response": 1.1921260356903076,
-      "logps/chosen": -292.7645568847656,
-      "logps/rejected": -265.25018310546875,
-      "logps/response": -173.6116485595703,
-      "loss": 0.9371,
-      "penalty": 0.6615164279937744,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.5470173358917236,
-      "rewards/gen": -2.3808140754699707,
-      "rewards/margins": 0.3109579086303711,
-      "rewards/rejected": -0.85797518491745,
+      "logits/chosen": 1.3397281169891357,
+      "logits/rejected": 1.1965497732162476,
+      "logits/response": 1.9877132177352905,
+      "logps/chosen": -329.905029296875,
+      "logps/rejected": -280.7567138671875,
+      "logps/response": -942.7415771484375,
+      "loss": 0.6473,
+      "penalty": 0.6103171110153198,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.839415431022644,
+      "rewards/gen": -6.383160591125488,
+      "rewards/margins": 0.3692413568496704,
+      "rewards/rejected": -1.2086567878723145,
       "step": 230
     },
     {
       "epoch": 0.7536800785083415,
-      "grad_norm": 48.935142517089844,
-      "improvement": 0.2968369424343109,
+      "grad_norm": 50.30048751831055,
+      "improvement": 0.03364395350217819,
       "learning_rate": 8.628481651367875e-08,
-      "logits/chosen": 1.1252264976501465,
-      "logits/rejected": 1.0409650802612305,
-      "logits/response": 1.1756417751312256,
-      "logps/chosen": -312.1551208496094,
-      "logps/rejected": -272.1668395996094,
-      "logps/response": -187.1859893798828,
-      "loss": 0.9424,
-      "penalty": 0.6408424377441406,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.6089781522750854,
-      "rewards/gen": -2.672689437866211,
-      "rewards/margins": 0.27286165952682495,
-      "rewards/rejected": -0.8818397521972656,
+      "logits/chosen": 1.3906304836273193,
+      "logits/rejected": 1.307106614112854,
+      "logits/response": 1.8935962915420532,
+      "logps/chosen": -333.33880615234375,
+      "logps/rejected": -319.1541748046875,
+      "logps/response": -1009.0065307617188,
+      "loss": 0.6684,
+      "penalty": 0.6743585467338562,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.7209497094154358,
+      "rewards/gen": -6.688617706298828,
+      "rewards/margins": 0.20695701241493225,
+      "rewards/rejected": -0.9279066920280457,
       "step": 240
     },
     {
       "epoch": 0.7850834151128557,
-      "grad_norm": 48.9501838684082,
-      "improvement": 0.29138535261154175,
+      "grad_norm": 51.985145568847656,
+      "improvement": 0.025273319333791733,
       "learning_rate": 6.655924144404906e-08,
-      "logits/chosen": 1.2226974964141846,
-      "logits/rejected": 1.2076431512832642,
-      "logits/response": 1.2697948217391968,
-      "logps/chosen": -296.0802917480469,
-      "logps/rejected": -281.9087829589844,
-      "logps/response": -191.81581115722656,
-      "loss": 0.9396,
-      "penalty": 0.6523431539535522,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.5890125036239624,
-      "rewards/gen": -2.5938172340393066,
-      "rewards/margins": 0.2846723198890686,
-      "rewards/rejected": -0.8736848831176758,
+      "logits/chosen": 1.3345959186553955,
+      "logits/rejected": 1.3350740671157837,
+      "logits/response": 1.926537275314331,
+      "logps/chosen": -320.46905517578125,
+      "logps/rejected": -313.0986633300781,
+      "logps/response": -1047.0450439453125,
+      "loss": 0.6355,
+      "penalty": 0.5920716524124146,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.5942948460578918,
+      "rewards/gen": -7.248889923095703,
+      "rewards/margins": 0.43543609976768494,
+      "rewards/rejected": -1.029731035232544,
       "step": 250
     },
     {
       "epoch": 0.81648675171737,
-      "grad_norm": 50.87124252319336,
-      "improvement": 0.2962267994880676,
+      "grad_norm": 52.067352294921875,
+      "improvement": 0.0384710431098938,
       "learning_rate": 4.904486005914027e-08,
-      "logits/chosen": 1.247997760772705,
-      "logits/rejected": 1.2229816913604736,
-      "logits/response": 1.220820426940918,
-      "logps/chosen": -325.2000427246094,
-      "logps/rejected": -324.00360107421875,
-      "logps/response": -192.4630126953125,
-      "loss": 0.9512,
-      "penalty": 0.6405457258224487,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.5452840924263,
-      "rewards/gen": -2.5521068572998047,
-      "rewards/margins": 0.31265154480934143,
-      "rewards/rejected": -0.8579355478286743,
+      "logits/chosen": 1.3885962963104248,
+      "logits/rejected": 1.286679983139038,
+      "logits/response": 2.0341262817382812,
+      "logps/chosen": -302.25225830078125,
+      "logps/rejected": -276.5771484375,
+      "logps/response": -1033.217529296875,
+      "loss": 0.6409,
+      "penalty": 0.6347585916519165,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.6630799174308777,
+      "rewards/gen": -7.127025604248047,
+      "rewards/margins": 0.33273279666900635,
+      "rewards/rejected": -0.9958127737045288,
       "step": 260
     },
     {
       "epoch": 0.8478900883218842,
-      "grad_norm": 49.64020538330078,
-      "improvement": 0.2779327929019928,
+      "grad_norm": 52.939659118652344,
+      "improvement": 0.032857220619916916,
       "learning_rate": 3.3952790595787986e-08,
-      "logits/chosen": 1.181114912033081,
-      "logits/rejected": 1.0998259782791138,
-      "logits/response": 1.2433074712753296,
-      "logps/chosen": -349.3531494140625,
-      "logps/rejected": -303.54290771484375,
-      "logps/response": -204.0261688232422,
-      "loss": 0.9736,
-      "penalty": 0.6398892402648926,
+      "logits/chosen": 1.3275368213653564,
+      "logits/rejected": 1.3410176038742065,
+      "logits/response": 1.9151852130889893,
+      "logps/chosen": -300.0262451171875,
+      "logps/rejected": -301.26348876953125,
+      "logps/response": -984.4091796875,
+      "loss": 0.6556,
+      "penalty": 0.616500973701477,
       "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.5694975256919861,
-      "rewards/gen": -2.798640489578247,
-      "rewards/margins": 0.3327513337135315,
-      "rewards/rejected": -0.9022488594055176,
+      "rewards/chosen": -0.6747496724128723,
+      "rewards/gen": -7.032576560974121,
+      "rewards/margins": 0.3772509694099426,
+      "rewards/rejected": -1.052000641822815,
       "step": 270
     },
     {
       "epoch": 0.8792934249263984,
-      "grad_norm": 41.62852096557617,
-      "improvement": 0.2895236015319824,
+      "grad_norm": 56.18733215332031,
+      "improvement": 0.04124368727207184,
       "learning_rate": 2.1464952759020856e-08,
-      "logits/chosen": 1.3301640748977661,
-      "logits/rejected": 1.3078454732894897,
-      "logits/response": 1.366155743598938,
-      "logps/chosen": -290.0614013671875,
-      "logps/rejected": -289.795654296875,
-      "logps/response": -183.56405639648438,
-      "loss": 0.9523,
-      "penalty": 0.6176464557647705,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.517002284526825,
-      "rewards/gen": -2.5910143852233887,
-      "rewards/margins": 0.3448910117149353,
-      "rewards/rejected": -0.861893355846405,
+      "logits/chosen": 1.3870338201522827,
+      "logits/rejected": 1.2385810613632202,
+      "logits/response": 1.9627625942230225,
+      "logps/chosen": -339.856201171875,
+      "logps/rejected": -290.2395935058594,
+      "logps/response": -957.0828247070312,
+      "loss": 0.6484,
+      "penalty": 0.5773279666900635,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.6688198447227478,
+      "rewards/gen": -6.297931671142578,
+      "rewards/margins": 0.4349055290222168,
+      "rewards/rejected": -1.103725552558899,
       "step": 280
     },
     {
       "epoch": 0.9106967615309126,
-      "grad_norm": 44.526798248291016,
-      "improvement": 0.27915069460868835,
+      "grad_norm": 50.07227325439453,
+      "improvement": 0.0390530601143837,
       "learning_rate": 1.1731874863145142e-08,
-      "logits/chosen": 1.107360601425171,
-      "logits/rejected": 1.115980863571167,
-      "logits/response": 1.1596436500549316,
-      "logps/chosen": -331.88482666015625,
-      "logps/rejected": -284.38092041015625,
-      "logps/response": -202.94888305664062,
-      "loss": 0.9388,
-      "penalty": 0.5815633535385132,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.5069407224655151,
-      "rewards/gen": -2.5722098350524902,
-      "rewards/margins": 0.4680548310279846,
-      "rewards/rejected": -0.974995493888855,
+      "logits/chosen": 1.4286466836929321,
+      "logits/rejected": 1.3211020231246948,
+      "logits/response": 1.9781084060668945,
+      "logps/chosen": -288.7481384277344,
+      "logps/rejected": -284.15789794921875,
+      "logps/response": -978.9161987304688,
+      "loss": 0.6264,
+      "penalty": 0.5874598026275635,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.5895588994026184,
+      "rewards/gen": -6.995156764984131,
+      "rewards/margins": 0.39368805289268494,
+      "rewards/rejected": -0.983246922492981,
       "step": 290
     },
     {
       "epoch": 0.9421000981354269,
-      "grad_norm": 48.79948425292969,
-      "improvement": 0.276977002620697,
+      "grad_norm": 47.96150207519531,
+      "improvement": 0.026903603225946426,
       "learning_rate": 4.8708793644441086e-09,
-      "logits/chosen": 1.1860829591751099,
-      "logits/rejected": 1.1684296131134033,
-      "logits/response": 1.3256675004959106,
-      "logps/chosen": -323.58575439453125,
-      "logps/rejected": -287.38873291015625,
-      "logps/response": -207.91500854492188,
-      "loss": 0.9766,
-      "penalty": 0.7038360834121704,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.6609238982200623,
-      "rewards/gen": -2.6817288398742676,
-      "rewards/margins": 0.17689001560211182,
-      "rewards/rejected": -0.8378139734268188,
+      "logits/chosen": 1.3613035678863525,
+      "logits/rejected": 1.3257195949554443,
+      "logits/response": 1.956418752670288,
+      "logps/chosen": -295.4048156738281,
+      "logps/rejected": -294.7374267578125,
+      "logps/response": -1010.1788940429688,
+      "loss": 0.6335,
+      "penalty": 0.5789179801940918,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.6945083737373352,
+      "rewards/gen": -7.097268581390381,
+      "rewards/margins": 0.49477386474609375,
+      "rewards/rejected": -1.1892822980880737,
       "step": 300
     },
     {
       "epoch": 0.9421000981354269,
-      "eval_improvement": 0.30491918325424194,
-      "eval_logits/chosen": 0.8745693564414978,
-      "eval_logits/rejected": 0.8407858610153198,
-      "eval_logits/response": 0.9387850165367126,
-      "eval_logps/chosen": -315.67236328125,
-      "eval_logps/rejected": -280.70233154296875,
-      "eval_logps/response": -196.74180603027344,
-      "eval_loss": 0.9861029982566833,
-      "eval_penalty": 0.6912589073181152,
-      "eval_rewards/accuracies": 0.6019999980926514,
-      "eval_rewards/chosen": -0.7020009160041809,
-      "eval_rewards/gen": -2.609898328781128,
-      "eval_rewards/margins": 0.20593461394309998,
-      "eval_rewards/rejected": -0.9079356789588928,
-      "eval_runtime": 130.1224,
-      "eval_samples_per_second": 15.37,
-      "eval_steps_per_second": 1.921,
+      "eval_improvement": 0.7646235227584839,
+      "eval_logits/chosen": 1.051769733428955,
+      "eval_logits/rejected": 1.007547378540039,
+      "eval_logits/response": 1.1118723154067993,
+      "eval_logps/chosen": -323.3211975097656,
+      "eval_logps/rejected": -291.7405700683594,
+      "eval_logps/response": -206.5587921142578,
+      "eval_loss": 1.3955390453338623,
+      "eval_penalty": 0.6340250372886658,
+      "eval_rewards/accuracies": 0.6480000019073486,
+      "eval_rewards/chosen": -0.7654658555984497,
+      "eval_rewards/gen": -0.9812589287757874,
+      "eval_rewards/margins": 0.3382430076599121,
+      "eval_rewards/rejected": -1.1037088632583618,
+      "eval_runtime": 130.0193,
+      "eval_samples_per_second": 15.382,
+      "eval_steps_per_second": 1.923,
       "step": 300
     },
     {
       "epoch": 0.9735034347399412,
-      "grad_norm": 48.872318267822266,
-      "improvement": 0.27313023805618286,
+      "grad_norm": 46.95528030395508,
+      "improvement": 0.053758006542921066,
       "learning_rate": 9.64668657069706e-10,
-      "logits/chosen": 1.1439440250396729,
-      "logits/rejected": 1.109357237815857,
-      "logits/response": 1.206132173538208,
-      "logps/chosen": -325.14862060546875,
-      "logps/rejected": -304.11956787109375,
-      "logps/response": -201.95692443847656,
-      "loss": 0.9496,
-      "penalty": 0.6938046216964722,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.654919445514679,
-      "rewards/gen": -2.729240894317627,
-      "rewards/margins": 0.16051390767097473,
-      "rewards/rejected": -0.8154333233833313,
+      "logits/chosen": 1.3695099353790283,
+      "logits/rejected": 1.280415415763855,
+      "logits/response": 1.937442421913147,
+      "logps/chosen": -311.23272705078125,
+      "logps/rejected": -276.55255126953125,
+      "logps/response": -1002.0380859375,
+      "loss": 0.6576,
+      "penalty": 0.6296704411506653,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.7500702738761902,
+      "rewards/gen": -6.900748252868652,
+      "rewards/margins": 0.36686232686042786,
+      "rewards/rejected": -1.1169326305389404,
       "step": 310
     },
     {
       "epoch": 0.9986261040235525,
       "step": 318,
       "total_flos": 0.0,
-      "train_loss": 1.0325740605780163,
-      "train_runtime": 3343.8277,
-      "train_samples_per_second": 6.094,
-      "train_steps_per_second": 0.095
+      "train_loss": 0.7174485007172111,
+      "train_runtime": 3896.582,
+      "train_samples_per_second": 5.23,
+      "train_steps_per_second": 0.082
     }
   ],
   "logging_steps": 10,