Model save

README.md

@@ -32,7 +32,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 4e-07
+- learning_rate: 1e-07
 - train_batch_size: 4
 - eval_batch_size: 4
 - seed: 42
@@ -52,7 +52,7 @@ The following hyperparameters were used during training:
 
 ### Framework versions
 
-- Transformers 4.41.1
+- Transformers 4.39.3
 - Pytorch 2.1.2+cu118
 - Datasets 2.16.1
-- Tokenizers 0.19.1
+- Tokenizers 0.15.2

all_results.json

@@ -1,9 +1,8 @@
 {
-    "epoch": 0.9987908101571947,
-    "total_flos": 0.0,
-    "train_loss": 0.9438655665076674,
-    "train_runtime": 7020.7263,
+    "epoch": 1.0,
+    "train_loss": 0.5277958785073232,
+    "train_runtime": 6372.8883,
     "train_samples": 52922,
-    "train_samples_per_second": 7.538,
-    "train_steps_per_second": 0.059
+    "train_samples_per_second": 8.304,
+    "train_steps_per_second": 0.065
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "transformers_version": "4.41.1"
+  "transformers_version": "4.39.3"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:999f423f854e13fc506453d3f2abf8f89fa836065069fe6dc0288669f4e509e9
+oid sha256:ba15de61ecf1fab38692965bea81fb4fbc15bdea9730cfaafee7a941062fd99a
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b06e8c9d65bf6da14fcfbf182b621f376d131e989115189e52c8989e9918d89e
+oid sha256:a86b9d593b597cc2ecc5f43476a07ebe0e278ba57f94d4f7643b216560e89d16
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f238dd9434885914394ed70efc6f48253db2b43b50dee3f8e6d21d12e952102a
+oid sha256:b9c8ca0b8509b351f3013c55b40a6f892b4fec7ad730321d44ee3a7d0c8d4bf4
 size 4540516344

runs/Jun30_16-43-35_n136-129-074/events.out.tfevents.1719737035.n136-129-074.1271954.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b9c13cd1652226849ff02b0c201eab6dd29006aad4b3e6b9a23666758fbbdc65
-size 32943
+oid sha256:23af12085174b96f7419c292bede08c913f4deed18456f1554b5e4498ef1e928
+size 33985

train_results.json

@@ -1,9 +1,8 @@
 {
-    "epoch": 0.9987908101571947,
-    "total_flos": 0.0,
-    "train_loss": 0.9438655665076674,
-    "train_runtime": 7020.7263,
+    "epoch": 1.0,
+    "train_loss": 0.5277958785073232,
+    "train_runtime": 6372.8883,
     "train_samples": 52922,
-    "train_samples_per_second": 7.538,
-    "train_steps_per_second": 0.059
+    "train_samples_per_second": 8.304,
+    "train_steps_per_second": 0.065
 }

trainer_state.json

@@ -9,9 +9,9 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0024183796856106408,
-      "grad_norm": 1308.2534984366307,
-      "learning_rate": 9.523809523809522e-09,
+      "epoch": 0.0,
+      "grad_norm": 1308.2122296641476,
+      "learning_rate": 2.3809523809523806e-09,
       "logits/chosen": -2.7005977630615234,
       "logits/rejected": -2.6288318634033203,
       "logps/chosen": -1.1158788204193115,
@@ -24,628 +24,628 @@
       "step": 1
     },
     {
-      "epoch": 0.02418379685610641,
-      "grad_norm": 1018.5726644948129,
-      "learning_rate": 9.523809523809523e-08,
-      "logits/chosen": -2.762432098388672,
-      "logits/rejected": -2.697216033935547,
-      "logps/chosen": -0.83743816614151,
-      "logps/rejected": -0.8184080123901367,
-      "loss": 0.7126,
-      "rewards/accuracies": 0.4652777910232544,
-      "rewards/chosen": 0.018050068989396095,
-      "rewards/margins": 0.10968472808599472,
-      "rewards/rejected": -0.09163466095924377,
+      "epoch": 0.02,
+      "grad_norm": 1049.0503356036236,
+      "learning_rate": 2.3809523809523807e-08,
+      "logits/chosen": -2.762399435043335,
+      "logits/rejected": -2.6968984603881836,
+      "logps/chosen": -0.8372963070869446,
+      "logps/rejected": -0.8176102638244629,
+      "loss": 0.7125,
+      "rewards/accuracies": 0.4236111044883728,
+      "rewards/chosen": 0.03224152699112892,
+      "rewards/margins": 0.04410284012556076,
+      "rewards/rejected": -0.011861314065754414,
       "step": 10
     },
     {
-      "epoch": 0.04836759371221282,
-      "grad_norm": 1180.2159938574362,
-      "learning_rate": 1.9047619047619045e-07,
-      "logits/chosen": -2.6905813217163086,
-      "logits/rejected": -2.6509311199188232,
-      "logps/chosen": -0.9959298968315125,
-      "logps/rejected": -1.0417280197143555,
-      "loss": 0.6914,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.41868314146995544,
-      "rewards/margins": 0.11889855563640594,
-      "rewards/rejected": -0.5375816822052002,
+      "epoch": 0.05,
+      "grad_norm": 1236.016019303981,
+      "learning_rate": 4.7619047619047613e-08,
+      "logits/chosen": -2.689128875732422,
+      "logits/rejected": -2.64937686920166,
+      "logps/chosen": -0.9927361607551575,
+      "logps/rejected": -1.03745436668396,
+      "loss": 0.7127,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.09930654615163803,
+      "rewards/margins": 0.010919012129306793,
+      "rewards/rejected": -0.11022555828094482,
       "step": 20
     },
     {
-      "epoch": 0.07255139056831923,
-      "grad_norm": 2875.4669192965785,
-      "learning_rate": 2.857142857142857e-07,
-      "logits/chosen": -2.7360830307006836,
-      "logits/rejected": -2.68190336227417,
-      "logps/chosen": -0.9741474986076355,
-      "logps/rejected": -0.9867057800292969,
-      "loss": 0.7598,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.7634207606315613,
-      "rewards/margins": 0.5467337965965271,
-      "rewards/rejected": -1.3101545572280884,
+      "epoch": 0.07,
+      "grad_norm": 1468.1565781012905,
+      "learning_rate": 7.142857142857142e-08,
+      "logits/chosen": -2.7302658557891846,
+      "logits/rejected": -2.6768224239349365,
+      "logps/chosen": -0.967939555644989,
+      "logps/rejected": -0.9760215878486633,
+      "loss": 0.6942,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.1426212042570114,
+      "rewards/margins": 0.09910523146390915,
+      "rewards/rejected": -0.24172644317150116,
       "step": 30
     },
     {
-      "epoch": 0.09673518742442563,
-      "grad_norm": 2637.428065513591,
-      "learning_rate": 3.809523809523809e-07,
-      "logits/chosen": -2.686359405517578,
-      "logits/rejected": -2.641474485397339,
-      "logps/chosen": -0.9992626309394836,
-      "logps/rejected": -0.9355602264404297,
-      "loss": 0.9431,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.0757026672363281,
-      "rewards/margins": 1.4628146886825562,
-      "rewards/rejected": -2.538517475128174,
+      "epoch": 0.1,
+      "grad_norm": 1718.8019373304546,
+      "learning_rate": 9.523809523809523e-08,
+      "logits/chosen": -2.6734580993652344,
+      "logits/rejected": -2.6297881603240967,
+      "logps/chosen": -0.9952117204666138,
+      "logps/rejected": -0.920923113822937,
+      "loss": 0.6688,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.6706060171127319,
+      "rewards/margins": 0.4042028486728668,
+      "rewards/rejected": -1.0748088359832764,
       "step": 40
     },
     {
-      "epoch": 0.12091898428053205,
-      "grad_norm": 975.4162480488375,
-      "learning_rate": 3.995412608484087e-07,
-      "logits/chosen": -2.759364366531372,
-      "logits/rejected": -2.7036304473876953,
-      "logps/chosen": -0.9589918255805969,
-      "logps/rejected": -0.9145007133483887,
-      "loss": 0.9551,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 0.7774861454963684,
-      "rewards/margins": 1.538771629333496,
-      "rewards/rejected": -0.7612855434417725,
+      "epoch": 0.12,
+      "grad_norm": 886.1855666160817,
+      "learning_rate": 9.988531521210217e-08,
+      "logits/chosen": -2.739663600921631,
+      "logits/rejected": -2.6859798431396484,
+      "logps/chosen": -0.9715211987495422,
+      "logps/rejected": -0.9160677790641785,
+      "loss": 0.6298,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.4754611551761627,
+      "rewards/margins": 0.44252967834472656,
+      "rewards/rejected": -0.9179908633232117,
       "step": 50
     },
     {
-      "epoch": 0.14510278113663846,
-      "grad_norm": 876.8403861103245,
-      "learning_rate": 3.976812391485896e-07,
-      "logits/chosen": -2.762204647064209,
-      "logits/rejected": -2.693788528442383,
-      "logps/chosen": -0.9397481083869934,
-      "logps/rejected": -0.9560295939445496,
-      "loss": 1.0176,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.3399124145507812,
-      "rewards/margins": 2.62353777885437,
-      "rewards/rejected": -4.9634504318237305,
+      "epoch": 0.15,
+      "grad_norm": 1659.7392402989283,
+      "learning_rate": 9.94203097871474e-08,
+      "logits/chosen": -2.743115186691284,
+      "logits/rejected": -2.676964521408081,
+      "logps/chosen": -0.9205001592636108,
+      "logps/rejected": -0.9147119522094727,
+      "loss": 0.6278,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.4151291251182556,
+      "rewards/margins": 0.4165617823600769,
+      "rewards/rejected": -0.8316909074783325,
       "step": 60
     },
     {
-      "epoch": 0.16928657799274485,
-      "grad_norm": 851.542671730339,
-      "learning_rate": 3.9440458281608213e-07,
-      "logits/chosen": -2.753873109817505,
-      "logits/rejected": -2.727172613143921,
-      "logps/chosen": -0.9090434312820435,
-      "logps/rejected": -0.8764857053756714,
-      "loss": 0.9609,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": 1.77558171749115,
-      "rewards/margins": 3.0382332801818848,
-      "rewards/rejected": -1.2626516819000244,
+      "epoch": 0.17,
+      "grad_norm": 1314.7754621298307,
+      "learning_rate": 9.860114570402053e-08,
+      "logits/chosen": -2.744157314300537,
+      "logits/rejected": -2.7204043865203857,
+      "logps/chosen": -0.9306680560112,
+      "logps/rejected": -0.8743250966072083,
+      "loss": 0.5734,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.3868916630744934,
+      "rewards/margins": 0.6596783399581909,
+      "rewards/rejected": -1.046570062637329,
       "step": 70
     },
     {
-      "epoch": 0.19347037484885127,
-      "grad_norm": 1072.4102961713095,
-      "learning_rate": 3.897347732134074e-07,
-      "logits/chosen": -2.6825203895568848,
-      "logits/rejected": -2.6251988410949707,
-      "logps/chosen": -0.8969934582710266,
-      "logps/rejected": -1.0139261484146118,
-      "loss": 1.0729,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -0.25338679552078247,
-      "rewards/margins": 2.662139415740967,
-      "rewards/rejected": -2.9155266284942627,
+      "epoch": 0.19,
+      "grad_norm": 1220.0578782834486,
+      "learning_rate": 9.743369330335185e-08,
+      "logits/chosen": -2.6871607303619385,
+      "logits/rejected": -2.6331558227539062,
+      "logps/chosen": -0.901233971118927,
+      "logps/rejected": -0.9974308013916016,
+      "loss": 0.5749,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.6774497032165527,
+      "rewards/margins": 0.5885565876960754,
+      "rewards/rejected": -1.266006350517273,
       "step": 80
     },
     {
-      "epoch": 0.21765417170495768,
-      "grad_norm": 1045.2736360081997,
-      "learning_rate": 3.8370527539794614e-07,
-      "logits/chosen": -2.6782150268554688,
-      "logits/rejected": -2.6257784366607666,
-      "logps/chosen": -1.0229610204696655,
-      "logps/rejected": -1.0610243082046509,
-      "loss": 1.0661,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.9386919736862183,
-      "rewards/margins": 3.1748452186584473,
-      "rewards/rejected": -4.113537311553955,
+      "epoch": 0.22,
+      "grad_norm": 906.8663763416491,
+      "learning_rate": 9.592631884948653e-08,
+      "logits/chosen": -2.6903910636901855,
+      "logits/rejected": -2.6411349773406982,
+      "logps/chosen": -1.0175859928131104,
+      "logps/rejected": -1.0327794551849365,
+      "loss": 0.5569,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.40117961168289185,
+      "rewards/margins": 0.8878555297851562,
+      "rewards/rejected": -1.2890350818634033,
       "step": 90
     },
     {
-      "epoch": 0.2418379685610641,
-      "grad_norm": 717.8046840728214,
-      "learning_rate": 3.763592983027255e-07,
-      "logits/chosen": -2.7070720195770264,
-      "logits/rejected": -2.657769203186035,
-      "logps/chosen": -0.8991826772689819,
-      "logps/rejected": -0.9345502853393555,
-      "loss": 1.1057,
+      "epoch": 0.24,
+      "grad_norm": 866.6073604256766,
+      "learning_rate": 9.408982457568138e-08,
+      "logits/chosen": -2.7176403999328613,
+      "logits/rejected": -2.671095371246338,
+      "logps/chosen": -0.8981878161430359,
+      "logps/rejected": -0.9157611131668091,
+      "loss": 0.5818,
       "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.7811681032180786,
-      "rewards/margins": 2.7421443462371826,
-      "rewards/rejected": -3.52331280708313,
+      "rewards/chosen": -0.6816717386245728,
+      "rewards/margins": 0.9627196192741394,
+      "rewards/rejected": -1.6443912982940674,
       "step": 100
     },
     {
-      "epoch": 0.2660217654171705,
-      "grad_norm": 573.0671194598642,
-      "learning_rate": 3.6774948509008527e-07,
-      "logits/chosen": -2.715163469314575,
-      "logits/rejected": -2.6663975715637207,
-      "logps/chosen": -0.9820922613143921,
-      "logps/rejected": -0.9612447023391724,
-      "loss": 0.9831,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -0.3871256709098816,
-      "rewards/margins": 2.8536558151245117,
-      "rewards/rejected": -3.240781784057617,
+      "epoch": 0.27,
+      "grad_norm": 1160.110812326109,
+      "learning_rate": 9.193737127252132e-08,
+      "logits/chosen": -2.7276268005371094,
+      "logits/rejected": -2.6847987174987793,
+      "logps/chosen": -0.9827289581298828,
+      "logps/rejected": -0.94146728515625,
+      "loss": 0.5535,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.4507879614830017,
+      "rewards/margins": 0.8122564554214478,
+      "rewards/rejected": -1.2630443572998047,
       "step": 110
     },
     {
-      "epoch": 0.29020556227327693,
-      "grad_norm": 813.0340299259154,
-      "learning_rate": 3.579375358972288e-07,
-      "logits/chosen": -2.667712688446045,
-      "logits/rejected": -2.6149513721466064,
-      "logps/chosen": -0.9040060043334961,
-      "logps/rejected": -1.0196150541305542,
-      "loss": 0.8569,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.3184046745300293,
-      "rewards/margins": 3.717514753341675,
-      "rewards/rejected": -6.035919189453125,
+      "epoch": 0.29,
+      "grad_norm": 1118.8296262026665,
+      "learning_rate": 8.94843839743072e-08,
+      "logits/chosen": -2.684906244277954,
+      "logits/rejected": -2.6362223625183105,
+      "logps/chosen": -0.8857740163803101,
+      "logps/rejected": -0.9783684611320496,
+      "loss": 0.5057,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.4952009320259094,
+      "rewards/margins": 1.4160592555999756,
+      "rewards/rejected": -1.9112603664398193,
       "step": 120
     },
     {
-      "epoch": 0.3143893591293833,
-      "grad_norm": 803.5906659383959,
-      "learning_rate": 3.4699376567716156e-07,
-      "logits/chosen": -2.7040085792541504,
-      "logits/rejected": -2.662682056427002,
-      "logps/chosen": -0.8633000254631042,
-      "logps/rejected": -0.8938501477241516,
-      "loss": 1.0665,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": 0.6341629028320312,
-      "rewards/margins": 4.641495704650879,
-      "rewards/rejected": -4.007332801818848,
+      "epoch": 0.31,
+      "grad_norm": 816.4913832702339,
+      "learning_rate": 8.674844141929039e-08,
+      "logits/chosen": -2.7162153720855713,
+      "logits/rejected": -2.680973529815674,
+      "logps/chosen": -0.8731144666671753,
+      "logps/rejected": -0.8701594471931458,
+      "loss": 0.5054,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.34728819131851196,
+      "rewards/margins": 1.2909767627716064,
+      "rewards/rejected": -1.6382650136947632,
       "step": 130
     },
     {
-      "epoch": 0.3385731559854897,
-      "grad_norm": 894.1357604383927,
-      "learning_rate": 3.349966003036421e-07,
-      "logits/chosen": -2.678401470184326,
-      "logits/rejected": -2.6329421997070312,
-      "logps/chosen": -0.9393678903579712,
-      "logps/rejected": -0.9559372663497925,
-      "loss": 0.9368,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.3545489311218262,
-      "rewards/margins": 2.7926135063171387,
-      "rewards/rejected": -4.147162437438965,
+      "epoch": 0.34,
+      "grad_norm": 1469.7710854903712,
+      "learning_rate": 8.374915007591053e-08,
+      "logits/chosen": -2.691988945007324,
+      "logits/rejected": -2.652631998062134,
+      "logps/chosen": -0.9311792254447937,
+      "logps/rejected": -0.9303830862045288,
+      "loss": 0.534,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.5356858968734741,
+      "rewards/margins": 1.0560630559921265,
+      "rewards/rejected": -1.5917489528656006,
       "step": 140
     },
     {
-      "epoch": 0.36275695284159615,
-      "grad_norm": 1218.2734701044658,
-      "learning_rate": 3.220320145511884e-07,
-      "logits/chosen": -2.691074848175049,
-      "logits/rejected": -2.6277267932891846,
-      "logps/chosen": -0.9556415677070618,
-      "logps/rejected": -1.0070703029632568,
-      "loss": 1.0655,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": 0.038965918123722076,
-      "rewards/margins": 3.301687717437744,
-      "rewards/rejected": -3.262721538543701,
+      "epoch": 0.36,
+      "grad_norm": 1688.4287131310427,
+      "learning_rate": 8.05080036377971e-08,
+      "logits/chosen": -2.7248551845550537,
+      "logits/rejected": -2.6649651527404785,
+      "logps/chosen": -0.9584500193595886,
+      "logps/rejected": -0.9874745607376099,
+      "loss": 0.5552,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.24187500774860382,
+      "rewards/margins": 1.0612785816192627,
+      "rewards/rejected": -1.303153395652771,
       "step": 150
     },
     {
-      "epoch": 0.38694074969770254,
-      "grad_norm": 933.333059653217,
-      "learning_rate": 3.0819291597771795e-07,
-      "logits/chosen": -2.6850905418395996,
-      "logits/rejected": -2.641737461090088,
-      "logps/chosen": -0.9150887727737427,
-      "logps/rejected": -0.9567450284957886,
-      "loss": 1.1619,
+      "epoch": 0.39,
+      "grad_norm": 883.2476388726114,
+      "learning_rate": 7.704822899442949e-08,
+      "logits/chosen": -2.717927932739258,
+      "logits/rejected": -2.6812281608581543,
+      "logps/chosen": -0.9134844541549683,
+      "logps/rejected": -0.9353663325309753,
+      "loss": 0.5424,
       "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -0.4916467070579529,
-      "rewards/margins": 3.2452170848846436,
-      "rewards/rejected": -3.7368640899658203,
+      "rewards/chosen": -0.3312217593193054,
+      "rewards/margins": 1.2677682638168335,
+      "rewards/rejected": -1.5989899635314941,
       "step": 160
     },
     {
-      "epoch": 0.4111245465538089,
-      "grad_norm": 1195.530681047137,
-      "learning_rate": 2.9357847912507786e-07,
-      "logits/chosen": -2.664726734161377,
-      "logits/rejected": -2.5912623405456543,
-      "logps/chosen": -0.908007800579071,
-      "logps/rejected": -0.9083712697029114,
-      "loss": 1.0815,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.6012508869171143,
-      "rewards/margins": 3.351865768432617,
-      "rewards/rejected": -4.9531168937683105,
+      "epoch": 0.41,
+      "grad_norm": 1741.9681865719251,
+      "learning_rate": 7.339461978126947e-08,
+      "logits/chosen": -2.696554660797119,
+      "logits/rejected": -2.628129482269287,
+      "logps/chosen": -0.896826446056366,
+      "logps/rejected": -0.8772991895675659,
+      "loss": 0.5503,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.48312321305274963,
+      "rewards/margins": 1.3627839088439941,
+      "rewards/rejected": -1.845907211303711,
       "step": 170
     },
     {
-      "epoch": 0.43530834340991537,
-      "grad_norm": 1688.331550337793,
-      "learning_rate": 2.7829343480875617e-07,
-      "logits/chosen": -2.662369966506958,
-      "logits/rejected": -2.5877506732940674,
-      "logps/chosen": -0.9355185627937317,
-      "logps/rejected": -0.9674497842788696,
-      "loss": 1.0193,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": 0.7115722894668579,
-      "rewards/margins": 3.03961181640625,
-      "rewards/rejected": -2.3280396461486816,
+      "epoch": 0.44,
+      "grad_norm": 1946.5851641629447,
+      "learning_rate": 6.957335870218904e-08,
+      "logits/chosen": -2.673733949661255,
+      "logits/rejected": -2.6041712760925293,
+      "logps/chosen": -0.939963698387146,
+      "logps/rejected": -0.956584095954895,
+      "loss": 0.4957,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.2670658528804779,
+      "rewards/margins": 1.5085264444351196,
+      "rewards/rejected": -1.2414608001708984,
       "step": 180
     },
     {
-      "epoch": 0.45949214026602175,
-      "grad_norm": 1025.3772351835823,
-      "learning_rate": 2.624473195899052e-07,
-      "logits/chosen": -2.7287068367004395,
-      "logits/rejected": -2.696254253387451,
-      "logps/chosen": -0.9740760922431946,
-      "logps/rejected": -1.0570310354232788,
-      "loss": 1.0517,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.7886015176773071,
-      "rewards/margins": 3.46142578125,
-      "rewards/rejected": -5.250028133392334,
+      "epoch": 0.46,
+      "grad_norm": 2276.5142714905005,
+      "learning_rate": 6.56118298974763e-08,
+      "logits/chosen": -2.732027292251587,
+      "logits/rejected": -2.7048563957214355,
+      "logps/chosen": -0.9618963003158569,
+      "logps/rejected": -1.0246574878692627,
+      "loss": 0.5708,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.570631742477417,
+      "rewards/margins": 1.4420315027236938,
+      "rewards/rejected": -2.012662887573242,
       "step": 190
     },
     {
-      "epoch": 0.4836759371221282,
-      "grad_norm": 781.90565238033,
-      "learning_rate": 2.4615369080815547e-07,
-      "logits/chosen": -2.6937649250030518,
-      "logits/rejected": -2.651794195175171,
-      "logps/chosen": -0.8561639785766602,
-      "logps/rejected": -0.9404775500297546,
-      "loss": 1.0482,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": 0.10315074771642685,
-      "rewards/margins": 2.2021610736846924,
-      "rewards/rejected": -2.0990102291107178,
+      "epoch": 0.48,
+      "grad_norm": 1178.2661546943384,
+      "learning_rate": 6.153842270203887e-08,
+      "logits/chosen": -2.6935534477233887,
+      "logits/rejected": -2.6572394371032715,
+      "logps/chosen": -0.8579891920089722,
+      "logps/rejected": -0.931664764881134,
+      "loss": 0.5316,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.0793720930814743,
+      "rewards/margins": 1.138346552848816,
+      "rewards/rejected": -1.217718482017517,
       "step": 200
     },
     {
-      "epoch": 0.5078597339782346,
-      "grad_norm": 718.5486788904052,
-      "learning_rate": 2.2952931280049625e-07,
-      "logits/chosen": -2.7320737838745117,
-      "logits/rejected": -2.666625499725342,
-      "logps/chosen": -1.0195786952972412,
-      "logps/rejected": -0.985085129737854,
-      "loss": 1.2414,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": 0.24961701035499573,
-      "rewards/margins": 4.099853038787842,
-      "rewards/rejected": -3.850236415863037,
+      "epoch": 0.51,
+      "grad_norm": 913.5130412259963,
+      "learning_rate": 5.738232820012406e-08,
+      "logits/chosen": -2.7250068187713623,
+      "logits/rejected": -2.6632461547851562,
+      "logps/chosen": -1.0246375799179077,
+      "logps/rejected": -0.9633069038391113,
+      "loss": 0.5393,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.25627315044403076,
+      "rewards/margins": 1.4161301851272583,
+      "rewards/rejected": -1.672403335571289,
       "step": 210
     },
     {
-      "epoch": 0.532043530834341,
-      "grad_norm": 896.7965591252768,
-      "learning_rate": 2.1269332013798747e-07,
-      "logits/chosen": -2.7412266731262207,
-      "logits/rejected": -2.719102144241333,
-      "logps/chosen": -0.887231171131134,
-      "logps/rejected": -0.8848485946655273,
-      "loss": 1.1268,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.5452920198440552,
-      "rewards/margins": 3.039072036743164,
-      "rewards/rejected": -2.4937801361083984,
+      "epoch": 0.53,
+      "grad_norm": 1025.54215312441,
+      "learning_rate": 5.317333003449687e-08,
+      "logits/chosen": -2.733330011367798,
+      "logits/rejected": -2.715299606323242,
+      "logps/chosen": -0.8904998898506165,
+      "logps/rejected": -0.8720332980155945,
+      "loss": 0.5228,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 0.2184334546327591,
+      "rewards/margins": 1.4306997060775757,
+      "rewards/rejected": -1.212266206741333,
       "step": 220
     },
     {
-      "epoch": 0.5562273276904474,
-      "grad_norm": 606.0417693874598,
-      "learning_rate": 1.9576636387676436e-07,
-      "logits/chosen": -2.687635898590088,
-      "logits/rejected": -2.6457619667053223,
-      "logps/chosen": -0.9592903852462769,
-      "logps/rejected": -0.967265784740448,
-      "loss": 0.8874,
+      "epoch": 0.56,
+      "grad_norm": 932.0109937677821,
+      "learning_rate": 4.894159096919109e-08,
+      "logits/chosen": -2.6878042221069336,
+      "logits/rejected": -2.65002179145813,
+      "logps/chosen": -0.9498124122619629,
+      "logps/rejected": -0.9380944967269897,
+      "loss": 0.4868,
       "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.5358507633209229,
-      "rewards/margins": 3.3348402976989746,
-      "rewards/rejected": -4.870690822601318,
+      "rewards/chosen": -0.5880553722381592,
+      "rewards/margins": 1.365509271621704,
+      "rewards/rejected": -1.9535646438598633,
       "step": 230
     },
     {
-      "epoch": 0.5804111245465539,
-      "grad_norm": 794.9088518264721,
-      "learning_rate": 1.7886974694151976e-07,
-      "logits/chosen": -2.701521158218384,
-      "logits/rejected": -2.6726996898651123,
-      "logps/chosen": -0.9976873397827148,
-      "logps/rejected": -1.01997709274292,
-      "loss": 1.0599,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -0.8589717745780945,
-      "rewards/margins": 3.750645875930786,
-      "rewards/rejected": -4.609617710113525,
+      "epoch": 0.58,
+      "grad_norm": 936.3493959909762,
+      "learning_rate": 4.471743673537994e-08,
+      "logits/chosen": -2.7069993019104004,
+      "logits/rejected": -2.6865086555480957,
+      "logps/chosen": -0.9886058568954468,
+      "logps/rejected": -0.9898191690444946,
+      "loss": 0.5347,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.04917572811245918,
+      "rewards/margins": 1.6429868936538696,
+      "rewards/rejected": -1.5938111543655396,
       "step": 240
     },
     {
-      "epoch": 0.6045949214026602,
-      "grad_norm": 706.1920920085257,
-      "learning_rate": 1.6212455483752895e-07,
-      "logits/chosen": -2.7483222484588623,
-      "logits/rejected": -2.66450834274292,
-      "logps/chosen": -0.8861182928085327,
-      "logps/rejected": -0.9307917356491089,
-      "loss": 1.0161,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 1.0754988193511963,
-      "rewards/margins": 3.2043755054473877,
-      "rewards/rejected": -2.1288766860961914,
+      "epoch": 0.6,
+      "grad_norm": 984.7410594483609,
+      "learning_rate": 4.053113870938224e-08,
+      "logits/chosen": -2.758028507232666,
+      "logits/rejected": -2.679352283477783,
+      "logps/chosen": -0.8968666791915894,
+      "logps/rejected": -0.9234074354171753,
+      "loss": 0.5007,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 0.0006614074227400124,
+      "rewards/margins": 1.3911110162734985,
+      "rewards/rejected": -1.3904496431350708,
       "step": 250
     },
     {
-      "epoch": 0.6287787182587666,
-      "grad_norm": 1160.850163008235,
-      "learning_rate": 1.4565078792075733e-07,
-      "logits/chosen": -2.699370861053467,
-      "logits/rejected": -2.6291871070861816,
-      "logps/chosen": -1.0056778192520142,
-      "logps/rejected": -1.0176211595535278,
-      "loss": 1.0258,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.27300015091896057,
-      "rewards/margins": 4.846710681915283,
-      "rewards/rejected": -4.5737104415893555,
+      "epoch": 0.63,
+      "grad_norm": 2539.435048877582,
+      "learning_rate": 3.641269698018933e-08,
+      "logits/chosen": -2.713007926940918,
+      "logits/rejected": -2.6481966972351074,
+      "logps/chosen": -1.007727861404419,
+      "logps/rejected": -0.9894694089889526,
+      "loss": 0.497,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.06799235194921494,
+      "rewards/margins": 1.8265388011932373,
+      "rewards/rejected": -1.7585465908050537,
       "step": 260
     },
     {
-      "epoch": 0.652962515114873,
-      "grad_norm": 882.9319448662282,
-      "learning_rate": 1.295665014444281e-07,
-      "logits/chosen": -2.726219654083252,
-      "logits/rejected": -2.6420578956604004,
-      "logps/chosen": -0.9637600183486938,
-      "logps/rejected": -0.975050151348114,
-      "loss": 1.1212,
+      "epoch": 0.65,
+      "grad_norm": 1438.0528773750598,
+      "learning_rate": 3.2391625361107026e-08,
+      "logits/chosen": -2.7407853603363037,
+      "logits/rejected": -2.6641056537628174,
+      "logps/chosen": -0.9492539167404175,
+      "logps/rejected": -0.9392199516296387,
+      "loss": 0.5391,
       "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -1.4779157638549805,
-      "rewards/margins": 3.851034641265869,
-      "rewards/rejected": -5.328949928283691,
+      "rewards/chosen": -0.02730640210211277,
+      "rewards/margins": 1.7186416387557983,
+      "rewards/rejected": -1.745948076248169,
       "step": 270
     },
     {
-      "epoch": 0.6771463119709794,
-      "grad_norm": 914.3545207404874,
-      "learning_rate": 1.1398695954469597e-07,
-      "logits/chosen": -2.673750162124634,
-      "logits/rejected": -2.611456871032715,
-      "logps/chosen": -0.9046699404716492,
-      "logps/rejected": -0.9061228036880493,
-      "loss": 0.9744,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.9529012441635132,
-      "rewards/margins": 2.8092963695526123,
-      "rewards/rejected": -1.8563950061798096,
+      "epoch": 0.68,
+      "grad_norm": 1220.7276779461947,
+      "learning_rate": 2.8496739886173992e-08,
+      "logits/chosen": -2.697509288787842,
+      "logits/rejected": -2.640756607055664,
+      "logps/chosen": -0.9139662981033325,
+      "logps/rejected": -0.9015041589736938,
+      "loss": 0.4723,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.023266727104783058,
+      "rewards/margins": 1.4177985191345215,
+      "rewards/rejected": -1.3945319652557373,
       "step": 280
     },
     {
-      "epoch": 0.7013301088270859,
-      "grad_norm": 1001.1455181062619,
-      "learning_rate": 9.902380922818425e-08,
-      "logits/chosen": -2.7217631340026855,
-      "logits/rejected": -2.676079273223877,
-      "logps/chosen": -0.986601710319519,
-      "logps/rejected": -0.992447555065155,
-      "loss": 0.7804,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 1.5409294366836548,
-      "rewards/margins": 3.72483491897583,
-      "rewards/rejected": -2.1839053630828857,
+      "epoch": 0.7,
+      "grad_norm": 1542.4514800663226,
+      "learning_rate": 2.4755952307046063e-08,
+      "logits/chosen": -2.742187023162842,
+      "logits/rejected": -2.699744701385498,
+      "logps/chosen": -1.0006037950515747,
+      "logps/rejected": -0.9843395352363586,
+      "loss": 0.448,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 0.14072290062904358,
+      "rewards/margins": 1.5138235092163086,
+      "rewards/rejected": -1.3731005191802979,
       "step": 290
     },
     {
-      "epoch": 0.7255139056831923,
-      "grad_norm": 929.8360933885888,
-      "learning_rate": 8.478428028080398e-08,
-      "logits/chosen": -2.7178378105163574,
-      "logits/rejected": -2.6601595878601074,
-      "logps/chosen": -0.9026691317558289,
-      "logps/rejected": -0.9635857343673706,
-      "loss": 0.963,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.20143906772136688,
-      "rewards/margins": 3.291618824005127,
-      "rewards/rejected": -3.4930579662323,
+      "epoch": 0.73,
+      "grad_norm": 1452.2302527315665,
+      "learning_rate": 2.1196070070200995e-08,
+      "logits/chosen": -2.7377541065216064,
+      "logits/rejected": -2.6858716011047363,
+      "logps/chosen": -0.900943398475647,
+      "logps/rejected": -0.9453694224357605,
+      "loss": 0.484,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.028856370598077774,
+      "rewards/margins": 1.6425704956054688,
+      "rewards/rejected": -1.671426773071289,
       "step": 300
     },
     {
-      "epoch": 0.7496977025392987,
-      "grad_norm": 918.556437892916,
-      "learning_rate": 7.137041683151202e-08,
-      "logits/chosen": -2.7046775817871094,
-      "logits/rejected": -2.6347708702087402,
-      "logps/chosen": -1.0870015621185303,
-      "logps/rejected": -1.049997091293335,
-      "loss": 0.7071,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.3678367137908936,
-      "rewards/margins": 3.927046298980713,
-      "rewards/rejected": -5.294882774353027,
+      "epoch": 0.75,
+      "grad_norm": 966.038328895732,
+      "learning_rate": 1.7842604207878005e-08,
+      "logits/chosen": -2.7250287532806396,
+      "logits/rejected": -2.6620233058929443,
+      "logps/chosen": -1.0749655961990356,
+      "logps/rejected": -1.015625238418579,
+      "loss": 0.4403,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.16425299644470215,
+      "rewards/margins": 1.6934306621551514,
+      "rewards/rejected": -1.8576834201812744,
       "step": 310
     },
     {
-      "epoch": 0.7738814993954051,
-      "grad_norm": 1444.2173436767062,
-      "learning_rate": 5.8878346077822135e-08,
-      "logits/chosen": -2.7104134559631348,
-      "logits/rejected": -2.6237189769744873,
-      "logps/chosen": -0.8994497060775757,
-      "logps/rejected": -0.9579949378967285,
-      "loss": 0.7542,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 0.5055674910545349,
-      "rewards/margins": 4.5611371994018555,
-      "rewards/rejected": -4.055570602416992,
+      "epoch": 0.77,
+      "grad_norm": 1686.580221181831,
+      "learning_rate": 1.4719586519455534e-08,
+      "logits/chosen": -2.728663444519043,
+      "logits/rejected": -2.64876389503479,
+      "logps/chosen": -0.9033769369125366,
+      "logps/rejected": -0.9375128746032715,
+      "loss": 0.4301,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 0.11283926665782928,
+      "rewards/margins": 2.1201956272125244,
+      "rewards/rejected": -2.0073564052581787,
       "step": 320
     },
     {
-      "epoch": 0.7980652962515115,
-      "grad_norm": 758.7885560560964,
-      "learning_rate": 4.73975894135696e-08,
-      "logits/chosen": -2.6587905883789062,
-      "logits/rejected": -2.584770917892456,
-      "logps/chosen": -0.9313557744026184,
-      "logps/rejected": -0.9829689860343933,
-      "loss": 0.7798,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -0.45623573660850525,
-      "rewards/margins": 4.346068859100342,
-      "rewards/rejected": -4.802304744720459,
+      "epoch": 0.8,
+      "grad_norm": 1227.1950699118374,
+      "learning_rate": 1.18493973533924e-08,
+      "logits/chosen": -2.679917097091675,
+      "logits/rejected": -2.611525297164917,
+      "logps/chosen": -0.9286376237869263,
+      "logps/rejected": -0.9542851448059082,
+      "loss": 0.4554,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.18441525101661682,
+      "rewards/margins": 1.7495098114013672,
+      "rewards/rejected": -1.9339250326156616,
       "step": 330
     },
     {
-      "epoch": 0.8222490931076178,
-      "grad_norm": 818.3776763189691,
-      "learning_rate": 3.701042089556483e-08,
-      "logits/chosen": -2.7390825748443604,
-      "logits/rejected": -2.6632418632507324,
-      "logps/chosen": -0.9014676809310913,
-      "logps/rejected": -0.9521511197090149,
-      "loss": 0.8929,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.1090164184570312,
-      "rewards/margins": 2.5331737995147705,
-      "rewards/rejected": -3.642190456390381,
+      "epoch": 0.82,
+      "grad_norm": 1294.7256357947958,
+      "learning_rate": 9.252605223891208e-09,
+      "logits/chosen": -2.759120464324951,
+      "logits/rejected": -2.6892759799957275,
+      "logps/chosen": -0.8945956230163574,
+      "logps/rejected": -0.9347489476203918,
+      "loss": 0.5031,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.42182081937789917,
+      "rewards/margins": 1.480148196220398,
+      "rewards/rejected": -1.9019691944122314,
       "step": 340
     },
     {
-      "epoch": 0.8464328899637243,
-      "grad_norm": 633.4468460505269,
-      "learning_rate": 2.779127764652889e-08,
-      "logits/chosen": -2.671653985977173,
-      "logits/rejected": -2.6091012954711914,
-      "logps/chosen": -0.9844328761100769,
-      "logps/rejected": -0.9899514317512512,
-      "loss": 0.8681,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.1147739887237549,
-      "rewards/margins": 3.46244478225708,
-      "rewards/rejected": -4.577218532562256,
+      "epoch": 0.85,
+      "grad_norm": 914.832611398372,
+      "learning_rate": 6.947819411632222e-09,
+      "logits/chosen": -2.6895413398742676,
+      "logits/rejected": -2.635143995285034,
+      "logps/chosen": -0.9759384393692017,
+      "logps/rejected": -0.96312016248703,
+      "loss": 0.4451,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.2653306722640991,
+      "rewards/margins": 1.628769874572754,
+      "rewards/rejected": -1.894100546836853,
       "step": 350
     },
     {
-      "epoch": 0.8706166868198307,
-      "grad_norm": 790.3321831454036,
-      "learning_rate": 1.9806226419516193e-08,
-      "logits/chosen": -2.682281494140625,
-      "logits/rejected": -2.6280040740966797,
-      "logps/chosen": -0.9713428616523743,
-      "logps/rejected": -1.028427004814148,
-      "loss": 0.8939,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.3930521309375763,
-      "rewards/margins": 3.496274948120117,
-      "rewards/rejected": -3.889326810836792,
+      "epoch": 0.87,
+      "grad_norm": 1785.1282881305524,
+      "learning_rate": 4.951556604879048e-09,
+      "logits/chosen": -2.7073302268981934,
+      "logits/rejected": -2.661701202392578,
+      "logps/chosen": -0.9704357385635376,
+      "logps/rejected": -1.0066497325897217,
+      "loss": 0.4535,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.30233365297317505,
+      "rewards/margins": 1.4092557430267334,
+      "rewards/rejected": -1.7115894556045532,
       "step": 360
     },
     {
-      "epoch": 0.8948004836759371,
-      "grad_norm": 758.0022509460326,
-      "learning_rate": 1.3112490146559552e-08,
-      "logits/chosen": -2.7291171550750732,
-      "logits/rejected": -2.6654868125915527,
-      "logps/chosen": -0.8997148275375366,
-      "logps/rejected": -0.9173600077629089,
-      "loss": 0.6798,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": -0.3203060030937195,
-      "rewards/margins": 3.888981580734253,
-      "rewards/rejected": -4.209287166595459,
+      "epoch": 0.89,
+      "grad_norm": 821.6518799466353,
+      "learning_rate": 3.278122536639888e-09,
+      "logits/chosen": -2.7481675148010254,
+      "logits/rejected": -2.692375898361206,
+      "logps/chosen": -0.8963810205459595,
+      "logps/rejected": -0.8914927244186401,
+      "loss": 0.4351,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 0.013069706968963146,
+      "rewards/margins": 1.6356351375579834,
+      "rewards/rejected": -1.622565507888794,
       "step": 370
     },
     {
-      "epoch": 0.9189842805320435,
-      "grad_norm": 580.6775247669598,
-      "learning_rate": 7.758037864413247e-09,
-      "logits/chosen": -2.6995849609375,
-      "logits/rejected": -2.6686558723449707,
-      "logps/chosen": -0.906518280506134,
-      "logps/rejected": -0.9883183240890503,
-      "loss": 0.7423,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -0.12804336845874786,
-      "rewards/margins": 3.583407163619995,
-      "rewards/rejected": -3.7114500999450684,
+      "epoch": 0.92,
+      "grad_norm": 820.7523726208289,
+      "learning_rate": 1.9395094661033118e-09,
+      "logits/chosen": -2.717128276824951,
+      "logits/rejected": -2.693026542663574,
+      "logps/chosen": -0.9069439172744751,
+      "logps/rejected": -0.9684022068977356,
+      "loss": 0.4381,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.17060108482837677,
+      "rewards/margins": 1.5492388010025024,
+      "rewards/rejected": -1.7198398113250732,
       "step": 380
     },
     {
-      "epoch": 0.9431680773881499,
-      "grad_norm": 740.0766659575787,
-      "learning_rate": 3.78124095609087e-09,
-      "logits/chosen": -2.677032947540283,
-      "logits/rejected": -2.63029408454895,
-      "logps/chosen": -0.9278643727302551,
-      "logps/rejected": -1.0108085870742798,
-      "loss": 0.7435,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": 0.13243435323238373,
-      "rewards/margins": 3.7296395301818848,
-      "rewards/rejected": -3.597205400466919,
+      "epoch": 0.94,
+      "grad_norm": 791.5724729924565,
+      "learning_rate": 9.453102390227175e-10,
+      "logits/chosen": -2.694126605987549,
+      "logits/rejected": -2.655355453491211,
+      "logps/chosen": -0.9297587275505066,
+      "logps/rejected": -0.9920668601989746,
+      "loss": 0.4482,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -0.0569925419986248,
+      "rewards/margins": 1.6660388708114624,
+      "rewards/rejected": -1.7230314016342163,
       "step": 390
     },
     {
-      "epoch": 0.9673518742442564,
-      "grad_norm": 664.330995388725,
-      "learning_rate": 1.2105981716597603e-09,
-      "logits/chosen": -2.711874485015869,
-      "logits/rejected": -2.614854574203491,
-      "logps/chosen": -0.9732038378715515,
-      "logps/rejected": -0.9536064863204956,
-      "loss": 0.955,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -0.32332465052604675,
-      "rewards/margins": 3.985724925994873,
-      "rewards/rejected": -4.309049129486084,
+      "epoch": 0.97,
+      "grad_norm": 1054.605495809102,
+      "learning_rate": 3.0264954291494007e-10,
+      "logits/chosen": -2.7315666675567627,
+      "logits/rejected": -2.6392362117767334,
+      "logps/chosen": -0.970044732093811,
+      "logps/rejected": -0.9285378456115723,
+      "loss": 0.4633,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.0074133919551968575,
+      "rewards/margins": 1.79477858543396,
+      "rewards/rejected": -1.802191972732544,
       "step": 400
     },
     {
-      "epoch": 0.9915356711003628,
-      "grad_norm": 1022.9589946634649,
-      "learning_rate": 6.453139886395398e-11,
-      "logits/chosen": -2.710538864135742,
-      "logits/rejected": -2.6645543575286865,
-      "logps/chosen": -0.9343970417976379,
-      "logps/rejected": -0.9787089228630066,
-      "loss": 0.8541,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -0.5227594375610352,
-      "rewards/margins": 4.429939270019531,
-      "rewards/rejected": -4.952698707580566,
+      "epoch": 0.99,
+      "grad_norm": 1541.6219713183218,
+      "learning_rate": 1.6132849715988494e-11,
+      "logits/chosen": -2.7301852703094482,
+      "logits/rejected": -2.6915435791015625,
+      "logps/chosen": -0.9321584701538086,
+      "logps/rejected": -0.9507132768630981,
+      "loss": 0.4578,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -0.2989009618759155,
+      "rewards/margins": 1.854230523109436,
+      "rewards/rejected": -2.1531314849853516,
       "step": 410
     },
     {
-      "epoch": 0.9987908101571947,
+      "epoch": 1.0,
       "step": 413,
       "total_flos": 0.0,
-      "train_loss": 0.9438655665076674,
-      "train_runtime": 7020.7263,
-      "train_samples_per_second": 7.538,
-      "train_steps_per_second": 0.059
+      "train_loss": 0.5277958785073232,
+      "train_runtime": 6372.8883,
+      "train_samples_per_second": 8.304,
+      "train_steps_per_second": 0.065
     }
   ],
   "logging_steps": 10,
@@ -653,18 +653,6 @@
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
-  "stateful_callbacks": {
-    "TrainerControl": {
-      "args": {
-        "should_epoch_stop": false,
-        "should_evaluate": false,
-        "should_log": false,
-        "should_save": true,
-        "should_training_stop": false
-      },
-      "attributes": {}
-    }
-  },
   "total_flos": 0.0,
   "train_batch_size": 4,
   "trial_name": null,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d13f3fc8759aca2ff57e3efc81099fc07a2315dc63120d75fb63db001a764cfe
+oid sha256:93c9cdc5d47cf48041862ca767083718bee40c0c9d421521bddf246b76bcabfa
 size 6264