Model save

README.md

@@ -32,7 +32,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 1e-07
+- learning_rate: 4e-07
 - train_batch_size: 4
 - eval_batch_size: 4
 - seed: 42

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5277958785073232,
-    "train_runtime": 6372.8883,
+    "train_loss": 3.8421780889894426,
+    "train_runtime": 6381.4933,
     "train_samples": 52922,
-    "train_samples_per_second": 8.304,
+    "train_samples_per_second": 8.293,
     "train_steps_per_second": 0.065
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ba15de61ecf1fab38692965bea81fb4fbc15bdea9730cfaafee7a941062fd99a
+oid sha256:4e88c6ac9b4127cd43c95069301c9c933118b1525797db81fc6d80f544cf7cec
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a86b9d593b597cc2ecc5f43476a07ebe0e278ba57f94d4f7643b216560e89d16
+oid sha256:bd64f1edcaa32df3d1b32cd35af92f119cd07ea2ba7ebf00b6828eed52da98f0
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b9c8ca0b8509b351f3013c55b40a6f892b4fec7ad730321d44ee3a7d0c8d4bf4
+oid sha256:87c91c34961d0019bf71d071d06cfd82742211fd4d3379b61e5bc9cad550096c
 size 4540516344

runs/Jun30_16-42-14_n136-100-194/events.out.tfevents.1719737283.n136-100-194.302982.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:824e28b5cb20671cfc59b0993e4e6babc44a148db8e00053f715d99ea18124f3
-size 32943
+oid sha256:ae7c4645315dc9db21552bd2b3b636781f6717efc0255ff69d2ebf3333897606
+size 33985

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5277958785073232,
-    "train_runtime": 6372.8883,
+    "train_loss": 3.8421780889894426,
+    "train_runtime": 6381.4933,
     "train_samples": 52922,
-    "train_samples_per_second": 8.304,
+    "train_samples_per_second": 8.293,
     "train_steps_per_second": 0.065
 }

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 1308.2122296641476,
-      "learning_rate": 2.3809523809523806e-09,
+      "grad_norm": 6401.270603874373,
+      "learning_rate": 9.523809523809522e-09,
       "logits/chosen": -2.7005977630615234,
       "logits/rejected": -2.6288318634033203,
       "logps/chosen": -1.1158788204193115,
       "logps/rejected": -1.1333446502685547,
-      "loss": 0.6931,
+      "loss": 0.7544,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,626 +25,626 @@
     },
     {
       "epoch": 0.02,
-      "grad_norm": 1049.0503356036236,
-      "learning_rate": 2.3809523809523807e-08,
-      "logits/chosen": -2.762399435043335,
-      "logits/rejected": -2.6968984603881836,
-      "logps/chosen": -0.8372963070869446,
-      "logps/rejected": -0.8176102638244629,
-      "loss": 0.7125,
-      "rewards/accuracies": 0.4236111044883728,
-      "rewards/chosen": 0.03224152699112892,
-      "rewards/margins": 0.04410284012556076,
-      "rewards/rejected": -0.011861314065754414,
+      "grad_norm": 9369.590990783972,
+      "learning_rate": 9.523809523809523e-08,
+      "logits/chosen": -2.76228666305542,
+      "logits/rejected": -2.6970374584198,
+      "logps/chosen": -0.837486743927002,
+      "logps/rejected": -0.8182350993156433,
+      "loss": 0.9695,
+      "rewards/accuracies": 0.4305555522441864,
+      "rewards/chosen": 0.06597563624382019,
+      "rewards/margins": 0.437710702419281,
+      "rewards/rejected": -0.3717350959777832,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 1236.016019303981,
-      "learning_rate": 4.7619047619047613e-08,
-      "logits/chosen": -2.689128875732422,
-      "logits/rejected": -2.64937686920166,
-      "logps/chosen": -0.9927361607551575,
-      "logps/rejected": -1.03745436668396,
-      "loss": 0.7127,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -0.09930654615163803,
-      "rewards/margins": 0.010919012129306793,
-      "rewards/rejected": -0.11022555828094482,
+      "grad_norm": 5966.657402243146,
+      "learning_rate": 1.9047619047619045e-07,
+      "logits/chosen": -2.6901049613952637,
+      "logits/rejected": -2.6502909660339355,
+      "logps/chosen": -0.9933319091796875,
+      "logps/rejected": -1.0394352674484253,
+      "loss": 1.0318,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.794396698474884,
+      "rewards/margins": 0.7471516728401184,
+      "rewards/rejected": -1.5415483713150024,
       "step": 20
     },
     {
       "epoch": 0.07,
-      "grad_norm": 1468.1565781012905,
-      "learning_rate": 7.142857142857142e-08,
-      "logits/chosen": -2.7302658557891846,
-      "logits/rejected": -2.6768224239349365,
-      "logps/chosen": -0.967939555644989,
-      "logps/rejected": -0.9760215878486633,
-      "loss": 0.6942,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.1426212042570114,
-      "rewards/margins": 0.09910523146390915,
-      "rewards/rejected": -0.24172644317150116,
+      "grad_norm": 8820.198504372876,
+      "learning_rate": 2.857142857142857e-07,
+      "logits/chosen": -2.7333264350891113,
+      "logits/rejected": -2.6793360710144043,
+      "logps/chosen": -0.9710652232170105,
+      "logps/rejected": -0.9799602627754211,
+      "loss": 1.3198,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.275942325592041,
+      "rewards/margins": 0.9020620584487915,
+      "rewards/rejected": -3.178004264831543,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 1718.8019373304546,
-      "learning_rate": 9.523809523809523e-08,
-      "logits/chosen": -2.6734580993652344,
-      "logits/rejected": -2.6297881603240967,
-      "logps/chosen": -0.9952117204666138,
-      "logps/rejected": -0.920923113822937,
-      "loss": 0.6688,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.6706060171127319,
-      "rewards/margins": 0.4042028486728668,
-      "rewards/rejected": -1.0748088359832764,
+      "grad_norm": 8453.783513094899,
+      "learning_rate": 3.809523809523809e-07,
+      "logits/chosen": -2.6771621704101562,
+      "logits/rejected": -2.6321842670440674,
+      "logps/chosen": -0.989823043346405,
+      "logps/rejected": -0.9216930270195007,
+      "loss": 2.0555,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.6586966514587402,
+      "rewards/margins": 5.100310325622559,
+      "rewards/rejected": -5.759006500244141,
       "step": 40
     },
     {
       "epoch": 0.12,
-      "grad_norm": 886.1855666160817,
-      "learning_rate": 9.988531521210217e-08,
-      "logits/chosen": -2.739663600921631,
-      "logits/rejected": -2.6859798431396484,
-      "logps/chosen": -0.9715211987495422,
-      "logps/rejected": -0.9160677790641785,
-      "loss": 0.6298,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.4754611551761627,
-      "rewards/margins": 0.44252967834472656,
-      "rewards/rejected": -0.9179908633232117,
+      "grad_norm": 4192.139232222726,
+      "learning_rate": 3.995412608484087e-07,
+      "logits/chosen": -2.743403911590576,
+      "logits/rejected": -2.6878693103790283,
+      "logps/chosen": -0.9671042561531067,
+      "logps/rejected": -0.917597770690918,
+      "loss": 2.6495,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.16885781288146973,
+      "rewards/margins": 5.186079978942871,
+      "rewards/rejected": -5.35493803024292,
       "step": 50
     },
     {
       "epoch": 0.15,
-      "grad_norm": 1659.7392402989283,
-      "learning_rate": 9.94203097871474e-08,
-      "logits/chosen": -2.743115186691284,
-      "logits/rejected": -2.676964521408081,
-      "logps/chosen": -0.9205001592636108,
-      "logps/rejected": -0.9147119522094727,
-      "loss": 0.6278,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.4151291251182556,
-      "rewards/margins": 0.4165617823600769,
-      "rewards/rejected": -0.8316909074783325,
+      "grad_norm": 5643.860863524967,
+      "learning_rate": 3.976812391485896e-07,
+      "logits/chosen": -2.7438769340515137,
+      "logits/rejected": -2.676765203475952,
+      "logps/chosen": -0.911353588104248,
+      "logps/rejected": -0.9122518301010132,
+      "loss": 3.8047,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 2.4976494312286377,
+      "rewards/margins": 5.426072120666504,
+      "rewards/rejected": -2.928422212600708,
       "step": 60
     },
     {
       "epoch": 0.17,
-      "grad_norm": 1314.7754621298307,
-      "learning_rate": 9.860114570402053e-08,
-      "logits/chosen": -2.744157314300537,
-      "logits/rejected": -2.7204043865203857,
-      "logps/chosen": -0.9306680560112,
-      "logps/rejected": -0.8743250966072083,
-      "loss": 0.5734,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.3868916630744934,
-      "rewards/margins": 0.6596783399581909,
-      "rewards/rejected": -1.046570062637329,
+      "grad_norm": 4497.230754903385,
+      "learning_rate": 3.9440458281608213e-07,
+      "logits/chosen": -2.740940570831299,
+      "logits/rejected": -2.7162723541259766,
+      "logps/chosen": -0.9154363870620728,
+      "logps/rejected": -0.868497371673584,
+      "loss": 3.6432,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 5.681364059448242,
+      "rewards/margins": 8.000432968139648,
+      "rewards/rejected": -2.319068431854248,
       "step": 70
     },
     {
       "epoch": 0.19,
-      "grad_norm": 1220.0578782834486,
-      "learning_rate": 9.743369330335185e-08,
-      "logits/chosen": -2.6871607303619385,
-      "logits/rejected": -2.6331558227539062,
-      "logps/chosen": -0.901233971118927,
-      "logps/rejected": -0.9974308013916016,
-      "loss": 0.5749,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.6774497032165527,
-      "rewards/margins": 0.5885565876960754,
-      "rewards/rejected": -1.266006350517273,
+      "grad_norm": 9843.974275847575,
+      "learning_rate": 3.897347732134074e-07,
+      "logits/chosen": -2.679215908050537,
+      "logits/rejected": -2.625516891479492,
+      "logps/chosen": -0.9146322011947632,
+      "logps/rejected": -1.0181081295013428,
+      "loss": 5.767,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -10.08639907836914,
+      "rewards/margins": 6.582289695739746,
+      "rewards/rejected": -16.668689727783203,
       "step": 80
     },
     {
       "epoch": 0.22,
-      "grad_norm": 906.8663763416491,
-      "learning_rate": 9.592631884948653e-08,
-      "logits/chosen": -2.6903910636901855,
-      "logits/rejected": -2.6411349773406982,
-      "logps/chosen": -1.0175859928131104,
-      "logps/rejected": -1.0327794551849365,
-      "loss": 0.5569,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.40117961168289185,
-      "rewards/margins": 0.8878555297851562,
-      "rewards/rejected": -1.2890350818634033,
+      "grad_norm": 4773.013380320505,
+      "learning_rate": 3.8370527539794614e-07,
+      "logits/chosen": -2.6771388053894043,
+      "logits/rejected": -2.6291418075561523,
+      "logps/chosen": -1.003847360610962,
+      "logps/rejected": -1.0297266244888306,
+      "loss": 4.6354,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 4.863407611846924,
+      "rewards/margins": 9.78220272064209,
+      "rewards/rejected": -4.918795585632324,
       "step": 90
     },
     {
       "epoch": 0.24,
-      "grad_norm": 866.6073604256766,
-      "learning_rate": 9.408982457568138e-08,
-      "logits/chosen": -2.7176403999328613,
-      "logits/rejected": -2.671095371246338,
-      "logps/chosen": -0.8981878161430359,
-      "logps/rejected": -0.9157611131668091,
-      "loss": 0.5818,
+      "grad_norm": 3074.8663144850243,
+      "learning_rate": 3.763592983027255e-07,
+      "logits/chosen": -2.705735683441162,
+      "logits/rejected": -2.6605448722839355,
+      "logps/chosen": -0.9163268804550171,
+      "logps/rejected": -0.9396775960922241,
+      "loss": 5.8585,
       "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.6816717386245728,
-      "rewards/margins": 0.9627196192741394,
-      "rewards/rejected": -1.6443912982940674,
+      "rewards/chosen": -12.477940559387207,
+      "rewards/margins": 7.702305793762207,
+      "rewards/rejected": -20.180248260498047,
       "step": 100
     },
     {
       "epoch": 0.27,
-      "grad_norm": 1160.110812326109,
-      "learning_rate": 9.193737127252132e-08,
-      "logits/chosen": -2.7276268005371094,
-      "logits/rejected": -2.6847987174987793,
-      "logps/chosen": -0.9827289581298828,
-      "logps/rejected": -0.94146728515625,
-      "loss": 0.5535,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.4507879614830017,
-      "rewards/margins": 0.8122564554214478,
-      "rewards/rejected": -1.2630443572998047,
+      "grad_norm": 4099.610429119441,
+      "learning_rate": 3.6774948509008527e-07,
+      "logits/chosen": -2.714970111846924,
+      "logits/rejected": -2.6705470085144043,
+      "logps/chosen": -0.9598251581192017,
+      "logps/rejected": -0.9319995641708374,
+      "loss": 5.1529,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 9.19798755645752,
+      "rewards/margins": 10.779365539550781,
+      "rewards/rejected": -1.5813770294189453,
       "step": 110
     },
     {
       "epoch": 0.29,
-      "grad_norm": 1118.8296262026665,
-      "learning_rate": 8.94843839743072e-08,
-      "logits/chosen": -2.684906244277954,
-      "logits/rejected": -2.6362223625183105,
-      "logps/chosen": -0.8857740163803101,
-      "logps/rejected": -0.9783684611320496,
-      "loss": 0.5057,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -0.4952009320259094,
-      "rewards/margins": 1.4160592555999756,
-      "rewards/rejected": -1.9112603664398193,
+      "grad_norm": 4599.711217449366,
+      "learning_rate": 3.579375358972288e-07,
+      "logits/chosen": -2.678779125213623,
+      "logits/rejected": -2.6315762996673584,
+      "logps/chosen": -0.9081487655639648,
+      "logps/rejected": -1.0060938596725464,
+      "loss": 4.0915,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -13.663342475891113,
+      "rewards/margins": 9.755656242370605,
+      "rewards/rejected": -23.418994903564453,
       "step": 120
     },
     {
       "epoch": 0.31,
-      "grad_norm": 816.4913832702339,
-      "learning_rate": 8.674844141929039e-08,
-      "logits/chosen": -2.7162153720855713,
-      "logits/rejected": -2.680973529815674,
-      "logps/chosen": -0.8731144666671753,
-      "logps/rejected": -0.8701594471931458,
-      "loss": 0.5054,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -0.34728819131851196,
-      "rewards/margins": 1.2909767627716064,
-      "rewards/rejected": -1.6382650136947632,
+      "grad_norm": 4010.334966061441,
+      "learning_rate": 3.4699376567716156e-07,
+      "logits/chosen": -2.7230353355407715,
+      "logits/rejected": -2.684389591217041,
+      "logps/chosen": -0.8652521967887878,
+      "logps/rejected": -0.8799147605895996,
+      "loss": 4.4027,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 2.19469952583313,
+      "rewards/margins": 15.263641357421875,
+      "rewards/rejected": -13.068939208984375,
       "step": 130
     },
     {
       "epoch": 0.34,
-      "grad_norm": 1469.7710854903712,
-      "learning_rate": 8.374915007591053e-08,
-      "logits/chosen": -2.691988945007324,
-      "logits/rejected": -2.652631998062134,
-      "logps/chosen": -0.9311792254447937,
-      "logps/rejected": -0.9303830862045288,
-      "loss": 0.534,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.5356858968734741,
-      "rewards/margins": 1.0560630559921265,
-      "rewards/rejected": -1.5917489528656006,
+      "grad_norm": 5239.11146834966,
+      "learning_rate": 3.349966003036421e-07,
+      "logits/chosen": -2.689558506011963,
+      "logits/rejected": -2.649766445159912,
+      "logps/chosen": -0.9352903366088867,
+      "logps/rejected": -0.9416161775588989,
+      "loss": 4.7953,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.734063148498535,
+      "rewards/margins": 8.841203689575195,
+      "rewards/rejected": -13.575268745422363,
       "step": 140
     },
     {
       "epoch": 0.36,
-      "grad_norm": 1688.4287131310427,
-      "learning_rate": 8.05080036377971e-08,
-      "logits/chosen": -2.7248551845550537,
-      "logits/rejected": -2.6649651527404785,
-      "logps/chosen": -0.9584500193595886,
-      "logps/rejected": -0.9874745607376099,
-      "loss": 0.5552,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.24187500774860382,
-      "rewards/margins": 1.0612785816192627,
-      "rewards/rejected": -1.303153395652771,
+      "grad_norm": 5394.35498681908,
+      "learning_rate": 3.220320145511884e-07,
+      "logits/chosen": -2.7070841789245605,
+      "logits/rejected": -2.647737979888916,
+      "logps/chosen": -0.9441506266593933,
+      "logps/rejected": -0.9885166883468628,
+      "loss": 4.2219,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 5.9402689933776855,
+      "rewards/margins": 12.97706413269043,
+      "rewards/rejected": -7.036795139312744,
       "step": 150
     },
     {
       "epoch": 0.39,
-      "grad_norm": 883.2476388726114,
-      "learning_rate": 7.704822899442949e-08,
-      "logits/chosen": -2.717927932739258,
-      "logits/rejected": -2.6812281608581543,
-      "logps/chosen": -0.9134844541549683,
-      "logps/rejected": -0.9353663325309753,
-      "loss": 0.5424,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -0.3312217593193054,
-      "rewards/margins": 1.2677682638168335,
-      "rewards/rejected": -1.5989899635314941,
+      "grad_norm": 5022.189692479379,
+      "learning_rate": 3.0819291597771795e-07,
+      "logits/chosen": -2.7051825523376465,
+      "logits/rejected": -2.667494297027588,
+      "logps/chosen": -0.911395251750946,
+      "logps/rejected": -0.939487099647522,
+      "loss": 4.7963,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -0.6114660501480103,
+      "rewards/margins": 9.443866729736328,
+      "rewards/rejected": -10.055331230163574,
       "step": 160
     },
     {
       "epoch": 0.41,
-      "grad_norm": 1741.9681865719251,
-      "learning_rate": 7.339461978126947e-08,
-      "logits/chosen": -2.696554660797119,
-      "logits/rejected": -2.628129482269287,
-      "logps/chosen": -0.896826446056366,
-      "logps/rejected": -0.8772991895675659,
-      "loss": 0.5503,
+      "grad_norm": 5428.944545727042,
+      "learning_rate": 2.9357847912507786e-07,
+      "logits/chosen": -2.6787288188934326,
+      "logits/rejected": -2.609421968460083,
+      "logps/chosen": -0.8976411819458008,
+      "logps/rejected": -0.8857674598693848,
+      "loss": 4.6262,
       "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.48312321305274963,
-      "rewards/margins": 1.3627839088439941,
-      "rewards/rejected": -1.845907211303711,
+      "rewards/chosen": -2.82297945022583,
+      "rewards/margins": 10.640687942504883,
+      "rewards/rejected": -13.463666915893555,
       "step": 170
     },
     {
       "epoch": 0.44,
-      "grad_norm": 1946.5851641629447,
-      "learning_rate": 6.957335870218904e-08,
-      "logits/chosen": -2.673733949661255,
-      "logits/rejected": -2.6041712760925293,
-      "logps/chosen": -0.939963698387146,
-      "logps/rejected": -0.956584095954895,
-      "loss": 0.4957,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.2670658528804779,
-      "rewards/margins": 1.5085264444351196,
-      "rewards/rejected": -1.2414608001708984,
+      "grad_norm": 7317.882582449178,
+      "learning_rate": 2.7829343480875617e-07,
+      "logits/chosen": -2.6716930866241455,
+      "logits/rejected": -2.6018152236938477,
+      "logps/chosen": -0.9342878460884094,
+      "logps/rejected": -0.9536906480789185,
+      "loss": 4.5209,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 4.173262596130371,
+      "rewards/margins": 8.933877944946289,
+      "rewards/rejected": -4.760615348815918,
       "step": 180
     },
     {
       "epoch": 0.46,
-      "grad_norm": 2276.5142714905005,
-      "learning_rate": 6.56118298974763e-08,
-      "logits/chosen": -2.732027292251587,
-      "logits/rejected": -2.7048563957214355,
-      "logps/chosen": -0.9618963003158569,
-      "logps/rejected": -1.0246574878692627,
-      "loss": 0.5708,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.570631742477417,
-      "rewards/margins": 1.4420315027236938,
-      "rewards/rejected": -2.012662887573242,
+      "grad_norm": 5046.2946182405685,
+      "learning_rate": 2.624473195899052e-07,
+      "logits/chosen": -2.737992763519287,
+      "logits/rejected": -2.7089955806732178,
+      "logps/chosen": -0.9629039764404297,
+      "logps/rejected": -1.039236307144165,
+      "loss": 4.5521,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.3569388389587402,
+      "rewards/margins": 13.995905876159668,
+      "rewards/rejected": -17.352848052978516,
       "step": 190
     },
     {
       "epoch": 0.48,
-      "grad_norm": 1178.2661546943384,
-      "learning_rate": 6.153842270203887e-08,
-      "logits/chosen": -2.6935534477233887,
-      "logits/rejected": -2.6572394371032715,
-      "logps/chosen": -0.8579891920089722,
-      "logps/rejected": -0.931664764881134,
-      "loss": 0.5316,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.0793720930814743,
-      "rewards/margins": 1.138346552848816,
-      "rewards/rejected": -1.217718482017517,
+      "grad_norm": 4268.8163809344915,
+      "learning_rate": 2.4615369080815547e-07,
+      "logits/chosen": -2.6982626914978027,
+      "logits/rejected": -2.6629488468170166,
+      "logps/chosen": -0.8523995280265808,
+      "logps/rejected": -0.9246847033500671,
+      "loss": 3.8184,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 2.3979854583740234,
+      "rewards/margins": 4.996596336364746,
+      "rewards/rejected": -2.5986106395721436,
       "step": 200
     },
     {
       "epoch": 0.51,
-      "grad_norm": 913.5130412259963,
-      "learning_rate": 5.738232820012406e-08,
-      "logits/chosen": -2.7250068187713623,
-      "logits/rejected": -2.6632461547851562,
-      "logps/chosen": -1.0246375799179077,
-      "logps/rejected": -0.9633069038391113,
-      "loss": 0.5393,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.25627315044403076,
-      "rewards/margins": 1.4161301851272583,
-      "rewards/rejected": -1.672403335571289,
+      "grad_norm": 4211.242306423206,
+      "learning_rate": 2.2952931280049625e-07,
+      "logits/chosen": -2.7346115112304688,
+      "logits/rejected": -2.6734609603881836,
+      "logps/chosen": -1.0063531398773193,
+      "logps/rejected": -0.9570119976997375,
+      "loss": 4.9954,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 7.86081600189209,
+      "rewards/margins": 13.075413703918457,
+      "rewards/rejected": -5.214597225189209,
       "step": 210
     },
     {
       "epoch": 0.53,
-      "grad_norm": 1025.54215312441,
-      "learning_rate": 5.317333003449687e-08,
-      "logits/chosen": -2.733330011367798,
-      "logits/rejected": -2.715299606323242,
-      "logps/chosen": -0.8904998898506165,
-      "logps/rejected": -0.8720332980155945,
-      "loss": 0.5228,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": 0.2184334546327591,
-      "rewards/margins": 1.4306997060775757,
-      "rewards/rejected": -1.212266206741333,
+      "grad_norm": 4906.2448320907815,
+      "learning_rate": 2.1269332013798747e-07,
+      "logits/chosen": -2.7431142330169678,
+      "logits/rejected": -2.7241249084472656,
+      "logps/chosen": -0.8835189938545227,
+      "logps/rejected": -0.8670462369918823,
+      "loss": 4.3795,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 4.582579135894775,
+      "rewards/margins": 8.150335311889648,
+      "rewards/rejected": -3.567755937576294,
       "step": 220
     },
     {
       "epoch": 0.56,
-      "grad_norm": 932.0109937677821,
-      "learning_rate": 4.894159096919109e-08,
-      "logits/chosen": -2.6878042221069336,
-      "logits/rejected": -2.65002179145813,
-      "logps/chosen": -0.9498124122619629,
-      "logps/rejected": -0.9380944967269897,
-      "loss": 0.4868,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.5880553722381592,
-      "rewards/margins": 1.365509271621704,
-      "rewards/rejected": -1.9535646438598633,
+      "grad_norm": 2995.5119741253625,
+      "learning_rate": 1.9576636387676436e-07,
+      "logits/chosen": -2.690732955932617,
+      "logits/rejected": -2.653067111968994,
+      "logps/chosen": -0.960831344127655,
+      "logps/rejected": -0.9556485414505005,
+      "loss": 4.0487,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -8.449748039245605,
+      "rewards/margins": 10.095115661621094,
+      "rewards/rejected": -18.544864654541016,
       "step": 230
     },
     {
       "epoch": 0.58,
-      "grad_norm": 936.3493959909762,
-      "learning_rate": 4.471743673537994e-08,
-      "logits/chosen": -2.7069993019104004,
-      "logits/rejected": -2.6865086555480957,
-      "logps/chosen": -0.9886058568954468,
-      "logps/rejected": -0.9898191690444946,
-      "loss": 0.5347,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.04917572811245918,
-      "rewards/margins": 1.6429868936538696,
-      "rewards/rejected": -1.5938111543655396,
+      "grad_norm": 3504.414329050279,
+      "learning_rate": 1.7886974694151976e-07,
+      "logits/chosen": -2.7119805812835693,
+      "logits/rejected": -2.6879172325134277,
+      "logps/chosen": -0.990290641784668,
+      "logps/rejected": -0.9934972524642944,
+      "loss": 4.3644,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.5965616106987,
+      "rewards/margins": 9.211602210998535,
+      "rewards/rejected": -9.808164596557617,
       "step": 240
     },
     {
       "epoch": 0.6,
-      "grad_norm": 984.7410594483609,
-      "learning_rate": 4.053113870938224e-08,
-      "logits/chosen": -2.758028507232666,
-      "logits/rejected": -2.679352283477783,
-      "logps/chosen": -0.8968666791915894,
-      "logps/rejected": -0.9234074354171753,
-      "loss": 0.5007,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.0006614074227400124,
-      "rewards/margins": 1.3911110162734985,
-      "rewards/rejected": -1.3904496431350708,
+      "grad_norm": 3400.5848210057216,
+      "learning_rate": 1.6212455483752895e-07,
+      "logits/chosen": -2.756906270980835,
+      "logits/rejected": -2.6796135902404785,
+      "logps/chosen": -0.8838168978691101,
+      "logps/rejected": -0.9137406349182129,
+      "loss": 4.5034,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 6.5281982421875,
+      "rewards/margins": 8.64702033996582,
+      "rewards/rejected": -2.1188230514526367,
       "step": 250
     },
     {
       "epoch": 0.63,
-      "grad_norm": 2539.435048877582,
-      "learning_rate": 3.641269698018933e-08,
-      "logits/chosen": -2.713007926940918,
-      "logits/rejected": -2.6481966972351074,
-      "logps/chosen": -1.007727861404419,
-      "logps/rejected": -0.9894694089889526,
-      "loss": 0.497,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.06799235194921494,
-      "rewards/margins": 1.8265388011932373,
-      "rewards/rejected": -1.7585465908050537,
+      "grad_norm": 6194.117841583386,
+      "learning_rate": 1.4565078792075733e-07,
+      "logits/chosen": -2.7132773399353027,
+      "logits/rejected": -2.6494650840759277,
+      "logps/chosen": -1.002362847328186,
+      "logps/rejected": -0.9982520341873169,
+      "loss": 4.8134,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 3.0224878787994385,
+      "rewards/margins": 16.206506729125977,
+      "rewards/rejected": -13.1840181350708,
       "step": 260
     },
     {
       "epoch": 0.65,
-      "grad_norm": 1438.0528773750598,
-      "learning_rate": 3.2391625361107026e-08,
-      "logits/chosen": -2.7407853603363037,
-      "logits/rejected": -2.6641056537628174,
-      "logps/chosen": -0.9492539167404175,
-      "logps/rejected": -0.9392199516296387,
-      "loss": 0.5391,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -0.02730640210211277,
-      "rewards/margins": 1.7186416387557983,
-      "rewards/rejected": -1.745948076248169,
+      "grad_norm": 4565.495892627232,
+      "learning_rate": 1.295665014444281e-07,
+      "logits/chosen": -2.7381529808044434,
+      "logits/rejected": -2.6608738899230957,
+      "logps/chosen": -0.9501218795776367,
+      "logps/rejected": -0.9476363062858582,
+      "loss": 5.3754,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.570526123046875,
+      "rewards/margins": 12.367398262023926,
+      "rewards/rejected": -12.9379243850708,
       "step": 270
     },
     {
       "epoch": 0.68,
-      "grad_norm": 1220.7276779461947,
-      "learning_rate": 2.8496739886173992e-08,
-      "logits/chosen": -2.697509288787842,
-      "logits/rejected": -2.640756607055664,
-      "logps/chosen": -0.9139662981033325,
-      "logps/rejected": -0.9015041589736938,
-      "loss": 0.4723,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": 0.023266727104783058,
-      "rewards/margins": 1.4177985191345215,
-      "rewards/rejected": -1.3945319652557373,
+      "grad_norm": 5337.153187944306,
+      "learning_rate": 1.1398695954469597e-07,
+      "logits/chosen": -2.6872425079345703,
+      "logits/rejected": -2.630267381668091,
+      "logps/chosen": -0.9056104421615601,
+      "logps/rejected": -0.8939152956008911,
+      "loss": 4.1053,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 4.294297218322754,
+      "rewards/margins": 7.472552299499512,
+      "rewards/rejected": -3.1782548427581787,
       "step": 280
     },
     {
       "epoch": 0.7,
-      "grad_norm": 1542.4514800663226,
-      "learning_rate": 2.4755952307046063e-08,
-      "logits/chosen": -2.742187023162842,
-      "logits/rejected": -2.699744701385498,
-      "logps/chosen": -1.0006037950515747,
-      "logps/rejected": -0.9843395352363586,
-      "loss": 0.448,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": 0.14072290062904358,
-      "rewards/margins": 1.5138235092163086,
-      "rewards/rejected": -1.3731005191802979,
+      "grad_norm": 3582.07962645892,
+      "learning_rate": 9.902380922818425e-08,
+      "logits/chosen": -2.7334370613098145,
+      "logits/rejected": -2.6919913291931152,
+      "logps/chosen": -0.9840775728225708,
+      "logps/rejected": -0.9756690263748169,
+      "loss": 3.2759,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 8.966680526733398,
+      "rewards/margins": 11.496904373168945,
+      "rewards/rejected": -2.5302233695983887,
       "step": 290
     },
     {
       "epoch": 0.73,
-      "grad_norm": 1452.2302527315665,
-      "learning_rate": 2.1196070070200995e-08,
-      "logits/chosen": -2.7377541065216064,
-      "logits/rejected": -2.6858716011047363,
-      "logps/chosen": -0.900943398475647,
-      "logps/rejected": -0.9453694224357605,
-      "loss": 0.484,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -0.028856370598077774,
-      "rewards/margins": 1.6425704956054688,
-      "rewards/rejected": -1.671426773071289,
+      "grad_norm": 4767.591882910886,
+      "learning_rate": 8.478428028080398e-08,
+      "logits/chosen": -2.7305169105529785,
+      "logits/rejected": -2.6773815155029297,
+      "logps/chosen": -0.8988749384880066,
+      "logps/rejected": -0.9437707662582397,
+      "loss": 4.3175,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.8898951411247253,
+      "rewards/margins": 8.447718620300293,
+      "rewards/rejected": -7.55782413482666,
       "step": 300
     },
     {
       "epoch": 0.75,
-      "grad_norm": 966.038328895732,
-      "learning_rate": 1.7842604207878005e-08,
-      "logits/chosen": -2.7250287532806396,
-      "logits/rejected": -2.6620233058929443,
-      "logps/chosen": -1.0749655961990356,
-      "logps/rejected": -1.015625238418579,
-      "loss": 0.4403,
+      "grad_norm": 4819.380329592898,
+      "learning_rate": 7.137041683151202e-08,
+      "logits/chosen": -2.7228643894195557,
+      "logits/rejected": -2.6581058502197266,
+      "logps/chosen": -1.0781683921813965,
+      "logps/rejected": -1.028840184211731,
+      "loss": 2.9744,
       "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.16425299644470215,
-      "rewards/margins": 1.6934306621551514,
-      "rewards/rejected": -1.8576834201812744,
+      "rewards/chosen": -2.4226202964782715,
+      "rewards/margins": 13.473236083984375,
+      "rewards/rejected": -15.895855903625488,
       "step": 310
     },
     {
       "epoch": 0.77,
-      "grad_norm": 1686.580221181831,
-      "learning_rate": 1.4719586519455534e-08,
-      "logits/chosen": -2.728663444519043,
-      "logits/rejected": -2.64876389503479,
-      "logps/chosen": -0.9033769369125366,
-      "logps/rejected": -0.9375128746032715,
-      "loss": 0.4301,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.11283926665782928,
-      "rewards/margins": 2.1201956272125244,
-      "rewards/rejected": -2.0073564052581787,
+      "grad_norm": 7840.551721640683,
+      "learning_rate": 5.8878346077822135e-08,
+      "logits/chosen": -2.7280871868133545,
+      "logits/rejected": -2.649958848953247,
+      "logps/chosen": -0.9020591974258423,
+      "logps/rejected": -0.9361578822135925,
+      "loss": 2.7082,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 1.223115086555481,
+      "rewards/margins": 10.582406997680664,
+      "rewards/rejected": -9.35929012298584,
       "step": 320
     },
     {
       "epoch": 0.8,
-      "grad_norm": 1227.1950699118374,
-      "learning_rate": 1.18493973533924e-08,
-      "logits/chosen": -2.679917097091675,
-      "logits/rejected": -2.611525297164917,
-      "logps/chosen": -0.9286376237869263,
-      "logps/rejected": -0.9542851448059082,
-      "loss": 0.4554,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.18441525101661682,
-      "rewards/margins": 1.7495098114013672,
-      "rewards/rejected": -1.9339250326156616,
+      "grad_norm": 4662.77535052248,
+      "learning_rate": 4.73975894135696e-08,
+      "logits/chosen": -2.6770853996276855,
+      "logits/rejected": -2.6099040508270264,
+      "logps/chosen": -0.9263202548027039,
+      "logps/rejected": -0.9608638882637024,
+      "loss": 3.1985,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.2365754395723343,
+      "rewards/margins": 13.195585250854492,
+      "rewards/rejected": -12.959010124206543,
       "step": 330
     },
     {
       "epoch": 0.82,
-      "grad_norm": 1294.7256357947958,
-      "learning_rate": 9.252605223891208e-09,
-      "logits/chosen": -2.759120464324951,
-      "logits/rejected": -2.6892759799957275,
-      "logps/chosen": -0.8945956230163574,
-      "logps/rejected": -0.9347489476203918,
-      "loss": 0.5031,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.42182081937789917,
-      "rewards/margins": 1.480148196220398,
-      "rewards/rejected": -1.9019691944122314,
+      "grad_norm": 4550.588002339864,
+      "learning_rate": 3.701042089556483e-08,
+      "logits/chosen": -2.756493330001831,
+      "logits/rejected": -2.687851667404175,
+      "logps/chosen": -0.8901381492614746,
+      "logps/rejected": -0.9301478266716003,
+      "loss": 3.841,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.11963929980993271,
+      "rewards/margins": 7.3289618492126465,
+      "rewards/rejected": -7.209322929382324,
       "step": 340
     },
     {
       "epoch": 0.85,
-      "grad_norm": 914.832611398372,
-      "learning_rate": 6.947819411632222e-09,
-      "logits/chosen": -2.6895413398742676,
-      "logits/rejected": -2.635143995285034,
-      "logps/chosen": -0.9759384393692017,
-      "logps/rejected": -0.96312016248703,
-      "loss": 0.4451,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -0.2653306722640991,
-      "rewards/margins": 1.628769874572754,
-      "rewards/rejected": -1.894100546836853,
+      "grad_norm": 5464.471487236709,
+      "learning_rate": 2.779127764652889e-08,
+      "logits/chosen": -2.689107656478882,
+      "logits/rejected": -2.6330015659332275,
+      "logps/chosen": -0.9756801724433899,
+      "logps/rejected": -0.9646003842353821,
+      "loss": 3.6421,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.1974527835845947,
+      "rewards/margins": 9.013090133666992,
+      "rewards/rejected": -10.210542678833008,
       "step": 350
     },
     {
       "epoch": 0.87,
-      "grad_norm": 1785.1282881305524,
-      "learning_rate": 4.951556604879048e-09,
-      "logits/chosen": -2.7073302268981934,
-      "logits/rejected": -2.661701202392578,
-      "logps/chosen": -0.9704357385635376,
-      "logps/rejected": -1.0066497325897217,
-      "loss": 0.4535,
+      "grad_norm": 5949.708940984834,
+      "learning_rate": 1.9806226419516193e-08,
+      "logits/chosen": -2.704460620880127,
+      "logits/rejected": -2.656071186065674,
+      "logps/chosen": -0.9623576402664185,
+      "logps/rejected": -1.0082406997680664,
+      "loss": 3.5231,
       "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.30233365297317505,
-      "rewards/margins": 1.4092557430267334,
-      "rewards/rejected": -1.7115894556045532,
+      "rewards/chosen": 2.5273587703704834,
+      "rewards/margins": 11.88086223602295,
+      "rewards/rejected": -9.35350227355957,
       "step": 360
     },
     {
       "epoch": 0.89,
-      "grad_norm": 821.6518799466353,
-      "learning_rate": 3.278122536639888e-09,
-      "logits/chosen": -2.7481675148010254,
-      "logits/rejected": -2.692375898361206,
-      "logps/chosen": -0.8963810205459595,
-      "logps/rejected": -0.8914927244186401,
-      "loss": 0.4351,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.013069706968963146,
-      "rewards/margins": 1.6356351375579834,
-      "rewards/rejected": -1.622565507888794,
+      "grad_norm": 4320.933402478669,
+      "learning_rate": 1.3112490146559552e-08,
+      "logits/chosen": -2.7451281547546387,
+      "logits/rejected": -2.686728000640869,
+      "logps/chosen": -0.8951610326766968,
+      "logps/rejected": -0.89850914478302,
+      "loss": 3.0053,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 0.6753175258636475,
+      "rewards/margins": 12.29626750946045,
+      "rewards/rejected": -11.620949745178223,
       "step": 370
     },
     {
       "epoch": 0.92,
-      "grad_norm": 820.7523726208289,
-      "learning_rate": 1.9395094661033118e-09,
-      "logits/chosen": -2.717128276824951,
-      "logits/rejected": -2.693026542663574,
-      "logps/chosen": -0.9069439172744751,
-      "logps/rejected": -0.9684022068977356,
-      "loss": 0.4381,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -0.17060108482837677,
-      "rewards/margins": 1.5492388010025024,
-      "rewards/rejected": -1.7198398113250732,
+      "grad_norm": 2514.940389992379,
+      "learning_rate": 7.758037864413247e-09,
+      "logits/chosen": -2.7158432006835938,
+      "logits/rejected": -2.6906635761260986,
+      "logps/chosen": -0.9033122062683105,
+      "logps/rejected": -0.9709407091140747,
+      "loss": 2.8751,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 0.9628832936286926,
+      "rewards/margins": 10.83133316040039,
+      "rewards/rejected": -9.868449211120605,
       "step": 380
     },
     {
       "epoch": 0.94,
-      "grad_norm": 791.5724729924565,
-      "learning_rate": 9.453102390227175e-10,
-      "logits/chosen": -2.694126605987549,
-      "logits/rejected": -2.655355453491211,
-      "logps/chosen": -0.9297587275505066,
-      "logps/rejected": -0.9920668601989746,
-      "loss": 0.4482,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -0.0569925419986248,
-      "rewards/margins": 1.6660388708114624,
-      "rewards/rejected": -1.7230314016342163,
+      "grad_norm": 3504.225752431698,
+      "learning_rate": 3.78124095609087e-09,
+      "logits/chosen": -2.6947999000549316,
+      "logits/rejected": -2.6553878784179688,
+      "logps/chosen": -0.9263744354248047,
+      "logps/rejected": -0.9935058355331421,
+      "loss": 3.019,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 1.4072116613388062,
+      "rewards/margins": 10.741894721984863,
+      "rewards/rejected": -9.334683418273926,
       "step": 390
     },
     {
       "epoch": 0.97,
-      "grad_norm": 1054.605495809102,
-      "learning_rate": 3.0264954291494007e-10,
-      "logits/chosen": -2.7315666675567627,
-      "logits/rejected": -2.6392362117767334,
-      "logps/chosen": -0.970044732093811,
-      "logps/rejected": -0.9285378456115723,
-      "loss": 0.4633,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -0.0074133919551968575,
-      "rewards/margins": 1.79477858543396,
-      "rewards/rejected": -1.802191972732544,
+      "grad_norm": 4987.634749508018,
+      "learning_rate": 1.2105981716597603e-09,
+      "logits/chosen": -2.7300946712493896,
+      "logits/rejected": -2.6389007568359375,
+      "logps/chosen": -0.9686774015426636,
+      "logps/rejected": -0.9328421354293823,
+      "loss": 3.7864,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 0.6465551853179932,
+      "rewards/margins": 11.809611320495605,
+      "rewards/rejected": -11.163057327270508,
       "step": 400
     },
     {
       "epoch": 0.99,
-      "grad_norm": 1541.6219713183218,
-      "learning_rate": 1.6132849715988494e-11,
-      "logits/chosen": -2.7301852703094482,
-      "logits/rejected": -2.6915435791015625,
-      "logps/chosen": -0.9321584701538086,
-      "logps/rejected": -0.9507132768630981,
-      "loss": 0.4578,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -0.2989009618759155,
-      "rewards/margins": 1.854230523109436,
-      "rewards/rejected": -2.1531314849853516,
+      "grad_norm": 5473.226219590305,
+      "learning_rate": 6.453139886395398e-11,
+      "logits/chosen": -2.7284317016601562,
+      "logits/rejected": -2.6886637210845947,
+      "logps/chosen": -0.9334842562675476,
+      "logps/rejected": -0.9600637555122375,
+      "loss": 3.6391,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -2.157397747039795,
+      "rewards/margins": 13.2835054397583,
+      "rewards/rejected": -15.440902709960938,
       "step": 410
     },
     {
       "epoch": 1.0,
       "step": 413,
       "total_flos": 0.0,
-      "train_loss": 0.5277958785073232,
-      "train_runtime": 6372.8883,
-      "train_samples_per_second": 8.304,
+      "train_loss": 3.8421780889894426,
+      "train_runtime": 6381.4933,
+      "train_samples_per_second": 8.293,
       "train_steps_per_second": 0.065
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:93c9cdc5d47cf48041862ca767083718bee40c0c9d421521bddf246b76bcabfa
+oid sha256:d13f3fc8759aca2ff57e3efc81099fc07a2315dc63120d75fb63db001a764cfe
 size 6264