Model save

README.md

@@ -52,7 +52,7 @@ The following hyperparameters were used during training:
 
 ### Framework versions
 
-- Transformers 4.39.3
+- Transformers 4.41.1
 - Pytorch 2.1.2+cu118
 - Datasets 2.16.1
-- Tokenizers 0.15.2
+- Tokenizers 0.19.1

all_results.json

@@ -1,8 +1,9 @@
 {
-    "epoch": 1.0,
-    "train_loss": 0.47487434232494735,
-    "train_runtime": 6395.0149,
+    "epoch": 0.9987908101571947,
+    "total_flos": 0.0,
+    "train_loss": 0.9438655665076674,
+    "train_runtime": 7020.7263,
     "train_samples": 52922,
-    "train_samples_per_second": 8.276,
-    "train_steps_per_second": 0.065
+    "train_samples_per_second": 7.538,
+    "train_steps_per_second": 0.059
 }

config.json

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.39.3",
+  "transformers_version": "4.41.1",
   "use_cache": false,
   "vocab_size": 32000
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "transformers_version": "4.39.3"
+  "transformers_version": "4.41.1"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:64aa9daab10538c7a172f1ba6ce926958b4649375c9adf71f318ee88b08f1695
+oid sha256:999f423f854e13fc506453d3f2abf8f89fa836065069fe6dc0288669f4e509e9
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a65d81bdb0b9cf53f6d0a2ebd9398d3d840ff94e873a58c1f76500522e9c680a
+oid sha256:b06e8c9d65bf6da14fcfbf182b621f376d131e989115189e52c8989e9918d89e
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7dbdf8663fe04d3fe1a9263d68cb9b9a4ebb1d6eb0064e77e9f75e381536a669
+oid sha256:f238dd9434885914394ed70efc6f48253db2b43b50dee3f8e6d21d12e952102a
 size 4540516344

runs/Jun30_15-54-56_n136-082-130/events.out.tfevents.1719734116.n136-082-130.3539205.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d9ac714f454659e7abdfd0a9a94b14ab0a3dbb0cb9076b9c0a0b2bf9c439cd96
-size 33157
+oid sha256:9d65198d6efd08eae5bf1772fbfb49100799bd12e4569bb1c2a51a76f12d0b99
+size 34199

tokenizer.json

@@ -134,6 +134,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
+    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

train_results.json

@@ -1,8 +1,9 @@
 {
-    "epoch": 1.0,
-    "train_loss": 0.47487434232494735,
-    "train_runtime": 6395.0149,
+    "epoch": 0.9987908101571947,
+    "total_flos": 0.0,
+    "train_loss": 0.9438655665076674,
+    "train_runtime": 7020.7263,
     "train_samples": 52922,
-    "train_samples_per_second": 8.276,
-    "train_steps_per_second": 0.065
+    "train_samples_per_second": 7.538,
+    "train_steps_per_second": 0.059
 }

trainer_state.json

@@ -9,8 +9,8 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0,
-      "grad_norm": 39.29752927174121,
+      "epoch": 0.0024183796856106408,
+      "grad_norm": 1308.2534984366307,
       "learning_rate": 9.523809523809522e-09,
       "logits/chosen": -2.7005977630615234,
       "logits/rejected": -2.6288318634033203,
@@ -24,628 +24,628 @@
       "step": 1
     },
     {
-      "epoch": 0.02,
-      "grad_norm": 30.185149110893168,
+      "epoch": 0.02418379685610641,
+      "grad_norm": 1018.5726644948129,
       "learning_rate": 9.523809523809523e-08,
-      "logits/chosen": -2.762653350830078,
-      "logits/rejected": -2.697390556335449,
-      "logps/chosen": -0.8379318118095398,
-      "logps/rejected": -0.8183739185333252,
-      "loss": 0.6931,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.0009392754291184247,
-      "rewards/margins": 0.0017076488584280014,
-      "rewards/rejected": -0.002646924229338765,
+      "logits/chosen": -2.762432098388672,
+      "logits/rejected": -2.697216033935547,
+      "logps/chosen": -0.83743816614151,
+      "logps/rejected": -0.8184080123901367,
+      "loss": 0.7126,
+      "rewards/accuracies": 0.4652777910232544,
+      "rewards/chosen": 0.018050068989396095,
+      "rewards/margins": 0.10968472808599472,
+      "rewards/rejected": -0.09163466095924377,
       "step": 10
     },
     {
-      "epoch": 0.05,
-      "grad_norm": 35.53290334119399,
+      "epoch": 0.04836759371221282,
+      "grad_norm": 1180.2159938574362,
       "learning_rate": 1.9047619047619045e-07,
-      "logits/chosen": -2.6904428005218506,
-      "logits/rejected": -2.6507983207702637,
-      "logps/chosen": -1.00472092628479,
-      "logps/rejected": -1.0533316135406494,
-      "loss": 0.6909,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.038933612406253815,
-      "rewards/margins": 0.012004666030406952,
-      "rewards/rejected": -0.05093827843666077,
+      "logits/chosen": -2.6905813217163086,
+      "logits/rejected": -2.6509311199188232,
+      "logps/chosen": -0.9959298968315125,
+      "logps/rejected": -1.0417280197143555,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.41868314146995544,
+      "rewards/margins": 0.11889855563640594,
+      "rewards/rejected": -0.5375816822052002,
       "step": 20
     },
     {
-      "epoch": 0.07,
-      "grad_norm": 45.663947909799106,
+      "epoch": 0.07255139056831923,
+      "grad_norm": 2875.4669192965785,
       "learning_rate": 2.857142857142857e-07,
-      "logits/chosen": -2.740260601043701,
-      "logits/rejected": -2.6853420734405518,
-      "logps/chosen": -1.003110647201538,
-      "logps/rejected": -1.0300285816192627,
-      "loss": 0.6766,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.10979163646697998,
-      "rewards/margins": 0.059481240808963776,
-      "rewards/rejected": -0.16927286982536316,
+      "logits/chosen": -2.7360830307006836,
+      "logits/rejected": -2.68190336227417,
+      "logps/chosen": -0.9741474986076355,
+      "logps/rejected": -0.9867057800292969,
+      "loss": 0.7598,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.7634207606315613,
+      "rewards/margins": 0.5467337965965271,
+      "rewards/rejected": -1.3101545572280884,
       "step": 30
     },
     {
-      "epoch": 0.1,
-      "grad_norm": 50.39344860503985,
+      "epoch": 0.09673518742442563,
+      "grad_norm": 2637.428065513591,
       "learning_rate": 3.809523809523809e-07,
-      "logits/chosen": -2.70363187789917,
-      "logits/rejected": -2.6573939323425293,
-      "logps/chosen": -1.0712021589279175,
-      "logps/rejected": -1.028562307357788,
-      "loss": 0.6766,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.24808958172798157,
-      "rewards/margins": 0.1070721298456192,
-      "rewards/rejected": -0.35516172647476196,
+      "logits/chosen": -2.686359405517578,
+      "logits/rejected": -2.641474485397339,
+      "logps/chosen": -0.9992626309394836,
+      "logps/rejected": -0.9355602264404297,
+      "loss": 0.9431,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.0757026672363281,
+      "rewards/margins": 1.4628146886825562,
+      "rewards/rejected": -2.538517475128174,
       "step": 40
     },
     {
-      "epoch": 0.12,
-      "grad_norm": 20.60693623284319,
+      "epoch": 0.12091898428053205,
+      "grad_norm": 975.4162480488375,
       "learning_rate": 3.995412608484087e-07,
-      "logits/chosen": -2.8126511573791504,
-      "logits/rejected": -2.751605749130249,
-      "logps/chosen": -1.0192598104476929,
-      "logps/rejected": -1.0224714279174805,
-      "loss": 0.6433,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -0.15747961401939392,
-      "rewards/margins": 0.18927089869976044,
-      "rewards/rejected": -0.34675049781799316,
+      "logits/chosen": -2.759364366531372,
+      "logits/rejected": -2.7036304473876953,
+      "logps/chosen": -0.9589918255805969,
+      "logps/rejected": -0.9145007133483887,
+      "loss": 0.9551,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 0.7774861454963684,
+      "rewards/margins": 1.538771629333496,
+      "rewards/rejected": -0.7612855434417725,
       "step": 50
     },
     {
-      "epoch": 0.15,
-      "grad_norm": 23.99667278112726,
+      "epoch": 0.14510278113663846,
+      "grad_norm": 876.8403861103245,
       "learning_rate": 3.976812391485896e-07,
-      "logits/chosen": -2.8627750873565674,
-      "logits/rejected": -2.7865381240844727,
-      "logps/chosen": -1.0689642429351807,
-      "logps/rejected": -1.1306763887405396,
-      "loss": 0.6136,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.4578463137149811,
-      "rewards/margins": 0.2149978131055832,
-      "rewards/rejected": -0.6728441119194031,
+      "logits/chosen": -2.762204647064209,
+      "logits/rejected": -2.693788528442383,
+      "logps/chosen": -0.9397481083869934,
+      "logps/rejected": -0.9560295939445496,
+      "loss": 1.0176,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.3399124145507812,
+      "rewards/margins": 2.62353777885437,
+      "rewards/rejected": -4.9634504318237305,
       "step": 60
     },
     {
-      "epoch": 0.17,
-      "grad_norm": 19.879421280729744,
+      "epoch": 0.16928657799274485,
+      "grad_norm": 851.542671730339,
       "learning_rate": 3.9440458281608213e-07,
-      "logits/chosen": -2.923736572265625,
-      "logits/rejected": -2.8855957984924316,
-      "logps/chosen": -1.1652107238769531,
-      "logps/rejected": -1.2260390520095825,
-      "loss": 0.5821,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.7152349948883057,
-      "rewards/margins": 0.3713044226169586,
-      "rewards/rejected": -1.0865395069122314,
+      "logits/chosen": -2.753873109817505,
+      "logits/rejected": -2.727172613143921,
+      "logps/chosen": -0.9090434312820435,
+      "logps/rejected": -0.8764857053756714,
+      "loss": 0.9609,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": 1.77558171749115,
+      "rewards/margins": 3.0382332801818848,
+      "rewards/rejected": -1.2626516819000244,
       "step": 70
     },
     {
-      "epoch": 0.19,
-      "grad_norm": 24.793244665974544,
+      "epoch": 0.19347037484885127,
+      "grad_norm": 1072.4102961713095,
       "learning_rate": 3.897347732134074e-07,
-      "logits/chosen": -2.9123826026916504,
-      "logits/rejected": -2.848158597946167,
-      "logps/chosen": -1.2571847438812256,
-      "logps/rejected": -1.4717210531234741,
-      "loss": 0.5658,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.088175654411316,
-      "rewards/margins": 0.37267494201660156,
-      "rewards/rejected": -1.460850477218628,
+      "logits/chosen": -2.6825203895568848,
+      "logits/rejected": -2.6251988410949707,
+      "logps/chosen": -0.8969934582710266,
+      "logps/rejected": -1.0139261484146118,
+      "loss": 1.0729,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.25338679552078247,
+      "rewards/margins": 2.662139415740967,
+      "rewards/rejected": -2.9155266284942627,
       "step": 80
     },
     {
-      "epoch": 0.22,
-      "grad_norm": 31.25766570385382,
+      "epoch": 0.21765417170495768,
+      "grad_norm": 1045.2736360081997,
       "learning_rate": 3.8370527539794614e-07,
-      "logits/chosen": -2.944538116455078,
-      "logits/rejected": -2.8904216289520264,
-      "logps/chosen": -1.4984596967697144,
-      "logps/rejected": -1.692941665649414,
-      "loss": 0.5469,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.4546568393707275,
-      "rewards/margins": 0.5645014643669128,
-      "rewards/rejected": -2.019158363342285,
+      "logits/chosen": -2.6782150268554688,
+      "logits/rejected": -2.6257784366607666,
+      "logps/chosen": -1.0229610204696655,
+      "logps/rejected": -1.0610243082046509,
+      "loss": 1.0661,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.9386919736862183,
+      "rewards/margins": 3.1748452186584473,
+      "rewards/rejected": -4.113537311553955,
       "step": 90
     },
     {
-      "epoch": 0.24,
-      "grad_norm": 20.71509399425638,
+      "epoch": 0.2418379685610641,
+      "grad_norm": 717.8046840728214,
       "learning_rate": 3.763592983027255e-07,
-      "logits/chosen": -2.998934268951416,
-      "logits/rejected": -2.938382148742676,
-      "logps/chosen": -1.3874661922454834,
-      "logps/rejected": -1.5309767723083496,
-      "loss": 0.5419,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.4882850646972656,
-      "rewards/margins": 0.4066939353942871,
-      "rewards/rejected": -1.8949787616729736,
+      "logits/chosen": -2.7070720195770264,
+      "logits/rejected": -2.657769203186035,
+      "logps/chosen": -0.8991826772689819,
+      "logps/rejected": -0.9345502853393555,
+      "loss": 1.1057,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.7811681032180786,
+      "rewards/margins": 2.7421443462371826,
+      "rewards/rejected": -3.52331280708313,
       "step": 100
     },
     {
-      "epoch": 0.27,
-      "grad_norm": 29.801311977572134,
+      "epoch": 0.2660217654171705,
+      "grad_norm": 573.0671194598642,
       "learning_rate": 3.6774948509008527e-07,
-      "logits/chosen": -3.0467724800109863,
-      "logits/rejected": -2.9940078258514404,
-      "logps/chosen": -1.5098116397857666,
-      "logps/rejected": -1.6806358098983765,
-      "loss": 0.5002,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.5947716236114502,
-      "rewards/margins": 0.6606251001358032,
-      "rewards/rejected": -2.255396604537964,
+      "logits/chosen": -2.715163469314575,
+      "logits/rejected": -2.6663975715637207,
+      "logps/chosen": -0.9820922613143921,
+      "logps/rejected": -0.9612447023391724,
+      "loss": 0.9831,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -0.3871256709098816,
+      "rewards/margins": 2.8536558151245117,
+      "rewards/rejected": -3.240781784057617,
       "step": 110
     },
     {
-      "epoch": 0.29,
-      "grad_norm": 42.6845336844709,
+      "epoch": 0.29020556227327693,
+      "grad_norm": 813.0340299259154,
       "learning_rate": 3.579375358972288e-07,
-      "logits/chosen": -3.008394718170166,
-      "logits/rejected": -2.95668363571167,
-      "logps/chosen": -1.6003528833389282,
-      "logps/rejected": -1.9440205097198486,
-      "loss": 0.4873,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -2.158592700958252,
-      "rewards/margins": 0.7957009077072144,
-      "rewards/rejected": -2.954293966293335,
+      "logits/chosen": -2.667712688446045,
+      "logits/rejected": -2.6149513721466064,
+      "logps/chosen": -0.9040060043334961,
+      "logps/rejected": -1.0196150541305542,
+      "loss": 0.8569,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.3184046745300293,
+      "rewards/margins": 3.717514753341675,
+      "rewards/rejected": -6.035919189453125,
       "step": 120
     },
     {
-      "epoch": 0.31,
-      "grad_norm": 30.10506083188768,
+      "epoch": 0.3143893591293833,
+      "grad_norm": 803.5906659383959,
       "learning_rate": 3.4699376567716156e-07,
-      "logits/chosen": -3.0275816917419434,
-      "logits/rejected": -2.9870047569274902,
-      "logps/chosen": -1.5803468227386475,
-      "logps/rejected": -1.8975141048431396,
-      "loss": 0.4953,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -2.132115125656128,
-      "rewards/margins": 0.9990972280502319,
-      "rewards/rejected": -3.1312127113342285,
+      "logits/chosen": -2.7040085792541504,
+      "logits/rejected": -2.662682056427002,
+      "logps/chosen": -0.8633000254631042,
+      "logps/rejected": -0.8938501477241516,
+      "loss": 1.0665,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 0.6341629028320312,
+      "rewards/margins": 4.641495704650879,
+      "rewards/rejected": -4.007332801818848,
       "step": 130
     },
     {
-      "epoch": 0.34,
-      "grad_norm": 35.330331142732994,
+      "epoch": 0.3385731559854897,
+      "grad_norm": 894.1357604383927,
       "learning_rate": 3.349966003036421e-07,
-      "logits/chosen": -2.95058012008667,
-      "logits/rejected": -2.9124808311462402,
-      "logps/chosen": -1.5571823120117188,
-      "logps/rejected": -1.8096323013305664,
-      "loss": 0.495,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.8940801620483398,
-      "rewards/margins": 0.7914196848869324,
-      "rewards/rejected": -2.685499906539917,
+      "logits/chosen": -2.678401470184326,
+      "logits/rejected": -2.6329421997070312,
+      "logps/chosen": -0.9393678903579712,
+      "logps/rejected": -0.9559372663497925,
+      "loss": 0.9368,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.3545489311218262,
+      "rewards/margins": 2.7926135063171387,
+      "rewards/rejected": -4.147162437438965,
       "step": 140
     },
     {
-      "epoch": 0.36,
-      "grad_norm": 33.986511474196,
+      "epoch": 0.36275695284159615,
+      "grad_norm": 1218.2734701044658,
       "learning_rate": 3.220320145511884e-07,
-      "logits/chosen": -2.9779887199401855,
-      "logits/rejected": -2.9252359867095947,
-      "logps/chosen": -1.6742808818817139,
-      "logps/rejected": -2.0122931003570557,
-      "loss": 0.4417,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -2.1547489166259766,
-      "rewards/margins": 0.95880126953125,
-      "rewards/rejected": -3.1135504245758057,
+      "logits/chosen": -2.691074848175049,
+      "logits/rejected": -2.6277267932891846,
+      "logps/chosen": -0.9556415677070618,
+      "logps/rejected": -1.0070703029632568,
+      "loss": 1.0655,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": 0.038965918123722076,
+      "rewards/margins": 3.301687717437744,
+      "rewards/rejected": -3.262721538543701,
       "step": 150
     },
     {
-      "epoch": 0.39,
-      "grad_norm": 74.32560370403728,
+      "epoch": 0.38694074969770254,
+      "grad_norm": 933.333059653217,
       "learning_rate": 3.0819291597771795e-07,
-      "logits/chosen": -2.953810691833496,
-      "logits/rejected": -2.92661714553833,
-      "logps/chosen": -1.8501989841461182,
-      "logps/rejected": -2.310631513595581,
-      "loss": 0.474,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -2.820080280303955,
-      "rewards/margins": 1.353685736656189,
-      "rewards/rejected": -4.173765659332275,
+      "logits/chosen": -2.6850905418395996,
+      "logits/rejected": -2.641737461090088,
+      "logps/chosen": -0.9150887727737427,
+      "logps/rejected": -0.9567450284957886,
+      "loss": 1.1619,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.4916467070579529,
+      "rewards/margins": 3.2452170848846436,
+      "rewards/rejected": -3.7368640899658203,
       "step": 160
     },
     {
-      "epoch": 0.41,
-      "grad_norm": 35.80288567238183,
+      "epoch": 0.4111245465538089,
+      "grad_norm": 1195.530681047137,
       "learning_rate": 2.9357847912507786e-07,
-      "logits/chosen": -2.8709921836853027,
-      "logits/rejected": -2.8073086738586426,
-      "logps/chosen": -1.7988742589950562,
-      "logps/rejected": -2.1855826377868652,
-      "loss": 0.4533,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -2.720637321472168,
-      "rewards/margins": 1.2595900297164917,
-      "rewards/rejected": -3.98022723197937,
+      "logits/chosen": -2.664726734161377,
+      "logits/rejected": -2.5912623405456543,
+      "logps/chosen": -0.908007800579071,
+      "logps/rejected": -0.9083712697029114,
+      "loss": 1.0815,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -1.6012508869171143,
+      "rewards/margins": 3.351865768432617,
+      "rewards/rejected": -4.9531168937683105,
       "step": 170
     },
     {
-      "epoch": 0.44,
-      "grad_norm": 46.84058838823037,
+      "epoch": 0.43530834340991537,
+      "grad_norm": 1688.331550337793,
       "learning_rate": 2.7829343480875617e-07,
-      "logits/chosen": -2.851166009902954,
-      "logits/rejected": -2.78760027885437,
-      "logps/chosen": -1.7197291851043701,
-      "logps/rejected": -2.1098084449768066,
-      "loss": 0.4477,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -2.331284523010254,
-      "rewards/margins": 1.165631651878357,
-      "rewards/rejected": -3.4969162940979004,
+      "logits/chosen": -2.662369966506958,
+      "logits/rejected": -2.5877506732940674,
+      "logps/chosen": -0.9355185627937317,
+      "logps/rejected": -0.9674497842788696,
+      "loss": 1.0193,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.7115722894668579,
+      "rewards/margins": 3.03961181640625,
+      "rewards/rejected": -2.3280396461486816,
       "step": 180
     },
     {
-      "epoch": 0.46,
-      "grad_norm": 37.81790656131401,
+      "epoch": 0.45949214026602175,
+      "grad_norm": 1025.3772351835823,
       "learning_rate": 2.624473195899052e-07,
-      "logits/chosen": -2.891326427459717,
-      "logits/rejected": -2.8765931129455566,
-      "logps/chosen": -1.8199418783187866,
-      "logps/rejected": -2.3341312408447266,
-      "loss": 0.4242,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -2.5912556648254395,
-      "rewards/margins": 1.397545576095581,
-      "rewards/rejected": -3.9888012409210205,
+      "logits/chosen": -2.7287068367004395,
+      "logits/rejected": -2.696254253387451,
+      "logps/chosen": -0.9740760922431946,
+      "logps/rejected": -1.0570310354232788,
+      "loss": 1.0517,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.7886015176773071,
+      "rewards/margins": 3.46142578125,
+      "rewards/rejected": -5.250028133392334,
       "step": 190
     },
     {
-      "epoch": 0.48,
-      "grad_norm": 35.221702704198776,
+      "epoch": 0.4836759371221282,
+      "grad_norm": 781.90565238033,
       "learning_rate": 2.4615369080815547e-07,
-      "logits/chosen": -2.812504529953003,
-      "logits/rejected": -2.7768425941467285,
-      "logps/chosen": -1.8649930953979492,
-      "logps/rejected": -2.4729537963867188,
-      "loss": 0.4123,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -3.02339243888855,
-      "rewards/margins": 1.6370065212249756,
-      "rewards/rejected": -4.660399436950684,
+      "logits/chosen": -2.6937649250030518,
+      "logits/rejected": -2.651794195175171,
+      "logps/chosen": -0.8561639785766602,
+      "logps/rejected": -0.9404775500297546,
+      "loss": 1.0482,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 0.10315074771642685,
+      "rewards/margins": 2.2021610736846924,
+      "rewards/rejected": -2.0990102291107178,
       "step": 200
     },
     {
-      "epoch": 0.51,
-      "grad_norm": 32.300318557435396,
+      "epoch": 0.5078597339782346,
+      "grad_norm": 718.5486788904052,
       "learning_rate": 2.2952931280049625e-07,
-      "logits/chosen": -2.821171760559082,
-      "logits/rejected": -2.767702579498291,
-      "logps/chosen": -1.9325168132781982,
-      "logps/rejected": -2.349825382232666,
-      "loss": 0.4405,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -2.7313263416290283,
-      "rewards/margins": 1.4784008264541626,
-      "rewards/rejected": -4.2097272872924805,
+      "logits/chosen": -2.7320737838745117,
+      "logits/rejected": -2.666625499725342,
+      "logps/chosen": -1.0195786952972412,
+      "logps/rejected": -0.985085129737854,
+      "loss": 1.2414,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 0.24961701035499573,
+      "rewards/margins": 4.099853038787842,
+      "rewards/rejected": -3.850236415863037,
       "step": 210
     },
     {
-      "epoch": 0.53,
-      "grad_norm": 35.749077115079686,
+      "epoch": 0.532043530834341,
+      "grad_norm": 896.7965591252768,
       "learning_rate": 2.1269332013798747e-07,
-      "logits/chosen": -2.7949957847595215,
-      "logits/rejected": -2.7779276371002197,
-      "logps/chosen": -1.7968380451202393,
-      "logps/rejected": -2.2258944511413574,
-      "loss": 0.453,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -2.712461471557617,
-      "rewards/margins": 1.3854889869689941,
-      "rewards/rejected": -4.097950458526611,
+      "logits/chosen": -2.7412266731262207,
+      "logits/rejected": -2.719102144241333,
+      "logps/chosen": -0.887231171131134,
+      "logps/rejected": -0.8848485946655273,
+      "loss": 1.1268,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.5452920198440552,
+      "rewards/margins": 3.039072036743164,
+      "rewards/rejected": -2.4937801361083984,
       "step": 220
     },
     {
-      "epoch": 0.56,
-      "grad_norm": 31.776735030395933,
+      "epoch": 0.5562273276904474,
+      "grad_norm": 606.0417693874598,
       "learning_rate": 1.9576636387676436e-07,
-      "logits/chosen": -2.751516580581665,
-      "logits/rejected": -2.718752861022949,
-      "logps/chosen": -1.593168020248413,
-      "logps/rejected": -1.954358458518982,
-      "loss": 0.4259,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.9477083683013916,
-      "rewards/margins": 1.1596901416778564,
-      "rewards/rejected": -3.107398271560669,
+      "logits/chosen": -2.687635898590088,
+      "logits/rejected": -2.6457619667053223,
+      "logps/chosen": -0.9592903852462769,
+      "logps/rejected": -0.967265784740448,
+      "loss": 0.8874,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.5358507633209229,
+      "rewards/margins": 3.3348402976989746,
+      "rewards/rejected": -4.870690822601318,
       "step": 230
     },
     {
-      "epoch": 0.58,
-      "grad_norm": 32.77337576878758,
+      "epoch": 0.5804111245465539,
+      "grad_norm": 794.9088518264721,
       "learning_rate": 1.7886974694151976e-07,
-      "logits/chosen": -2.7686049938201904,
-      "logits/rejected": -2.7549407482147217,
-      "logps/chosen": -1.7974746227264404,
-      "logps/rejected": -2.256289005279541,
-      "loss": 0.4095,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.42513108253479,
-      "rewards/margins": 1.4220924377441406,
-      "rewards/rejected": -3.8472237586975098,
+      "logits/chosen": -2.701521158218384,
+      "logits/rejected": -2.6726996898651123,
+      "logps/chosen": -0.9976873397827148,
+      "logps/rejected": -1.01997709274292,
+      "loss": 1.0599,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.8589717745780945,
+      "rewards/margins": 3.750645875930786,
+      "rewards/rejected": -4.609617710113525,
       "step": 240
     },
     {
-      "epoch": 0.6,
-      "grad_norm": 36.95311129615954,
+      "epoch": 0.6045949214026602,
+      "grad_norm": 706.1920920085257,
       "learning_rate": 1.6212455483752895e-07,
-      "logits/chosen": -2.767482280731201,
-      "logits/rejected": -2.6901721954345703,
-      "logps/chosen": -2.1267056465148926,
-      "logps/rejected": -2.841970682144165,
-      "loss": 0.4046,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -3.689495801925659,
-      "rewards/margins": 2.1079065799713135,
-      "rewards/rejected": -5.797402381896973,
+      "logits/chosen": -2.7483222484588623,
+      "logits/rejected": -2.66450834274292,
+      "logps/chosen": -0.8861182928085327,
+      "logps/rejected": -0.9307917356491089,
+      "loss": 1.0161,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 1.0754988193511963,
+      "rewards/margins": 3.2043755054473877,
+      "rewards/rejected": -2.1288766860961914,
       "step": 250
     },
     {
-      "epoch": 0.63,
-      "grad_norm": 55.383812837143964,
+      "epoch": 0.6287787182587666,
+      "grad_norm": 1160.850163008235,
       "learning_rate": 1.4565078792075733e-07,
-      "logits/chosen": -2.673295259475708,
-      "logits/rejected": -2.607630729675293,
-      "logps/chosen": -2.1603939533233643,
-      "logps/rejected": -2.8367295265197754,
-      "loss": 0.4201,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -3.455958127975464,
-      "rewards/margins": 2.1385786533355713,
-      "rewards/rejected": -5.594536304473877,
+      "logits/chosen": -2.699370861053467,
+      "logits/rejected": -2.6291871070861816,
+      "logps/chosen": -1.0056778192520142,
+      "logps/rejected": -1.0176211595535278,
+      "loss": 1.0258,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 0.27300015091896057,
+      "rewards/margins": 4.846710681915283,
+      "rewards/rejected": -4.5737104415893555,
       "step": 260
     },
     {
-      "epoch": 0.65,
-      "grad_norm": 42.80678372159452,
+      "epoch": 0.652962515114873,
+      "grad_norm": 882.9319448662282,
       "learning_rate": 1.295665014444281e-07,
-      "logits/chosen": -2.688502788543701,
-      "logits/rejected": -2.606804609298706,
-      "logps/chosen": -1.99863600730896,
-      "logps/rejected": -2.58821439743042,
-      "loss": 0.4165,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -3.148965358734131,
-      "rewards/margins": 1.850395917892456,
-      "rewards/rejected": -4.999361038208008,
+      "logits/chosen": -2.726219654083252,
+      "logits/rejected": -2.6420578956604004,
+      "logps/chosen": -0.9637600183486938,
+      "logps/rejected": -0.975050151348114,
+      "loss": 1.1212,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -1.4779157638549805,
+      "rewards/margins": 3.851034641265869,
+      "rewards/rejected": -5.328949928283691,
       "step": 270
     },
     {
-      "epoch": 0.68,
-      "grad_norm": 37.58898607243965,
+      "epoch": 0.6771463119709794,
+      "grad_norm": 914.3545207404874,
       "learning_rate": 1.1398695954469597e-07,
-      "logits/chosen": -2.64621901512146,
-      "logits/rejected": -2.5769548416137695,
-      "logps/chosen": -1.7739193439483643,
-      "logps/rejected": -2.2734031677246094,
-      "loss": 0.4303,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -2.5791609287261963,
-      "rewards/margins": 1.5783722400665283,
-      "rewards/rejected": -4.157532691955566,
+      "logits/chosen": -2.673750162124634,
+      "logits/rejected": -2.611456871032715,
+      "logps/chosen": -0.9046699404716492,
+      "logps/rejected": -0.9061228036880493,
+      "loss": 0.9744,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.9529012441635132,
+      "rewards/margins": 2.8092963695526123,
+      "rewards/rejected": -1.8563950061798096,
       "step": 280
     },
     {
-      "epoch": 0.7,
-      "grad_norm": 40.75962534734924,
+      "epoch": 0.7013301088270859,
+      "grad_norm": 1001.1455181062619,
       "learning_rate": 9.902380922818425e-08,
-      "logits/chosen": -2.694524049758911,
-      "logits/rejected": -2.652637004852295,
-      "logps/chosen": -1.975266695022583,
-      "logps/rejected": -2.4491236209869385,
-      "loss": 0.3784,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -2.919767141342163,
-      "rewards/margins": 1.515777826309204,
-      "rewards/rejected": -4.435544490814209,
+      "logits/chosen": -2.7217631340026855,
+      "logits/rejected": -2.676079273223877,
+      "logps/chosen": -0.986601710319519,
+      "logps/rejected": -0.992447555065155,
+      "loss": 0.7804,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 1.5409294366836548,
+      "rewards/margins": 3.72483491897583,
+      "rewards/rejected": -2.1839053630828857,
       "step": 290
     },
     {
-      "epoch": 0.73,
-      "grad_norm": 36.10958020829023,
+      "epoch": 0.7255139056831923,
+      "grad_norm": 929.8360933885888,
       "learning_rate": 8.478428028080398e-08,
-      "logits/chosen": -2.664501667022705,
-      "logits/rejected": -2.6047203540802,
-      "logps/chosen": -1.8609319925308228,
-      "logps/rejected": -2.4197893142700195,
-      "loss": 0.4087,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -2.880831718444824,
-      "rewards/margins": 1.5925710201263428,
-      "rewards/rejected": -4.473402976989746,
+      "logits/chosen": -2.7178378105163574,
+      "logits/rejected": -2.6601595878601074,
+      "logps/chosen": -0.9026691317558289,
+      "logps/rejected": -0.9635857343673706,
+      "loss": 0.963,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.20143906772136688,
+      "rewards/margins": 3.291618824005127,
+      "rewards/rejected": -3.4930579662323,
       "step": 300
     },
     {
-      "epoch": 0.75,
-      "grad_norm": 47.402040633811865,
+      "epoch": 0.7496977025392987,
+      "grad_norm": 918.556437892916,
       "learning_rate": 7.137041683151202e-08,
-      "logits/chosen": -2.639944553375244,
-      "logits/rejected": -2.569493293762207,
-      "logps/chosen": -2.0725433826446533,
-      "logps/rejected": -2.7214386463165283,
-      "loss": 0.4035,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -2.9976601600646973,
-      "rewards/margins": 2.1755104064941406,
-      "rewards/rejected": -5.173171043395996,
+      "logits/chosen": -2.7046775817871094,
+      "logits/rejected": -2.6347708702087402,
+      "logps/chosen": -1.0870015621185303,
+      "logps/rejected": -1.049997091293335,
+      "loss": 0.7071,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.3678367137908936,
+      "rewards/margins": 3.927046298980713,
+      "rewards/rejected": -5.294882774353027,
       "step": 310
     },
     {
-      "epoch": 0.77,
-      "grad_norm": 44.820441736884675,
+      "epoch": 0.7738814993954051,
+      "grad_norm": 1444.2173436767062,
       "learning_rate": 5.8878346077822135e-08,
-      "logits/chosen": -2.6515159606933594,
-      "logits/rejected": -2.5604031085968018,
-      "logps/chosen": -1.9098122119903564,
-      "logps/rejected": -2.612426280975342,
-      "loss": 0.39,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -3.0159199237823486,
-      "rewards/margins": 2.0690410137176514,
-      "rewards/rejected": -5.084961414337158,
+      "logits/chosen": -2.7104134559631348,
+      "logits/rejected": -2.6237189769744873,
+      "logps/chosen": -0.8994497060775757,
+      "logps/rejected": -0.9579949378967285,
+      "loss": 0.7542,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.5055674910545349,
+      "rewards/margins": 4.5611371994018555,
+      "rewards/rejected": -4.055570602416992,
       "step": 320
     },
     {
-      "epoch": 0.8,
-      "grad_norm": 33.77844042473463,
+      "epoch": 0.7980652962515115,
+      "grad_norm": 758.7885560560964,
       "learning_rate": 4.73975894135696e-08,
-      "logits/chosen": -2.5840868949890137,
-      "logits/rejected": -2.512350559234619,
-      "logps/chosen": -1.8187984228134155,
-      "logps/rejected": -2.377983570098877,
-      "loss": 0.4242,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.6760153770446777,
-      "rewards/margins": 1.6530983448028564,
-      "rewards/rejected": -4.329113483428955,
+      "logits/chosen": -2.6587905883789062,
+      "logits/rejected": -2.584770917892456,
+      "logps/chosen": -0.9313557744026184,
+      "logps/rejected": -0.9829689860343933,
+      "loss": 0.7798,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.45623573660850525,
+      "rewards/margins": 4.346068859100342,
+      "rewards/rejected": -4.802304744720459,
       "step": 330
     },
     {
-      "epoch": 0.82,
-      "grad_norm": 39.87319974223161,
+      "epoch": 0.8222490931076178,
+      "grad_norm": 818.3776763189691,
       "learning_rate": 3.701042089556483e-08,
-      "logits/chosen": -2.666865825653076,
-      "logits/rejected": -2.5898385047912598,
-      "logps/chosen": -1.8051681518554688,
-      "logps/rejected": -2.415076494216919,
-      "loss": 0.393,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -2.7443721294403076,
-      "rewards/margins": 1.7536697387695312,
-      "rewards/rejected": -4.498042106628418,
+      "logits/chosen": -2.7390825748443604,
+      "logits/rejected": -2.6632418632507324,
+      "logps/chosen": -0.9014676809310913,
+      "logps/rejected": -0.9521511197090149,
+      "loss": 0.8929,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.1090164184570312,
+      "rewards/margins": 2.5331737995147705,
+      "rewards/rejected": -3.642190456390381,
       "step": 340
     },
     {
-      "epoch": 0.85,
-      "grad_norm": 34.287765115129865,
+      "epoch": 0.8464328899637243,
+      "grad_norm": 633.4468460505269,
       "learning_rate": 2.779127764652889e-08,
-      "logits/chosen": -2.5984504222869873,
-      "logits/rejected": -2.536284923553467,
-      "logps/chosen": -1.8758970499038696,
-      "logps/rejected": -2.4569201469421387,
-      "loss": 0.4064,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -2.7078359127044678,
-      "rewards/margins": 1.8303868770599365,
-      "rewards/rejected": -4.538222789764404,
+      "logits/chosen": -2.671653985977173,
+      "logits/rejected": -2.6091012954711914,
+      "logps/chosen": -0.9844328761100769,
+      "logps/rejected": -0.9899514317512512,
+      "loss": 0.8681,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.1147739887237549,
+      "rewards/margins": 3.46244478225708,
+      "rewards/rejected": -4.577218532562256,
       "step": 350
     },
     {
-      "epoch": 0.87,
-      "grad_norm": 35.97823366978786,
+      "epoch": 0.8706166868198307,
+      "grad_norm": 790.3321831454036,
       "learning_rate": 1.9806226419516193e-08,
-      "logits/chosen": -2.608383893966675,
-      "logits/rejected": -2.5496582984924316,
-      "logps/chosen": -1.9528089761734009,
-      "logps/rejected": -2.4915452003479004,
-      "loss": 0.402,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -2.9561896324157715,
-      "rewards/margins": 1.5498435497283936,
-      "rewards/rejected": -4.506033420562744,
+      "logits/chosen": -2.682281494140625,
+      "logits/rejected": -2.6280040740966797,
+      "logps/chosen": -0.9713428616523743,
+      "logps/rejected": -1.028427004814148,
+      "loss": 0.8939,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.3930521309375763,
+      "rewards/margins": 3.496274948120117,
+      "rewards/rejected": -3.889326810836792,
       "step": 360
     },
     {
-      "epoch": 0.89,
-      "grad_norm": 39.263164350081915,
+      "epoch": 0.8948004836759371,
+      "grad_norm": 758.0022509460326,
       "learning_rate": 1.3112490146559552e-08,
-      "logits/chosen": -2.6401114463806152,
-      "logits/rejected": -2.5702717304229736,
-      "logps/chosen": -1.842989206314087,
-      "logps/rejected": -2.4201440811157227,
-      "loss": 0.3876,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -2.8394322395324707,
-      "rewards/margins": 1.7951984405517578,
-      "rewards/rejected": -4.634631156921387,
+      "logits/chosen": -2.7291171550750732,
+      "logits/rejected": -2.6654868125915527,
+      "logps/chosen": -0.8997148275375366,
+      "logps/rejected": -0.9173600077629089,
+      "loss": 0.6798,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -0.3203060030937195,
+      "rewards/margins": 3.888981580734253,
+      "rewards/rejected": -4.209287166595459,
       "step": 370
     },
     {
-      "epoch": 0.92,
-      "grad_norm": 42.24637972019108,
+      "epoch": 0.9189842805320435,
+      "grad_norm": 580.6775247669598,
       "learning_rate": 7.758037864413247e-09,
-      "logits/chosen": -2.6185803413391113,
-      "logits/rejected": -2.5831003189086914,
-      "logps/chosen": -1.9194902181625366,
-      "logps/rejected": -2.553614854812622,
-      "loss": 0.4006,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -3.042757034301758,
-      "rewards/margins": 1.7644752264022827,
-      "rewards/rejected": -4.807231903076172,
+      "logits/chosen": -2.6995849609375,
+      "logits/rejected": -2.6686558723449707,
+      "logps/chosen": -0.906518280506134,
+      "logps/rejected": -0.9883183240890503,
+      "loss": 0.7423,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -0.12804336845874786,
+      "rewards/margins": 3.583407163619995,
+      "rewards/rejected": -3.7114500999450684,
       "step": 380
     },
     {
-      "epoch": 0.94,
-      "grad_norm": 42.39006737727194,
+      "epoch": 0.9431680773881499,
+      "grad_norm": 740.0766659575787,
       "learning_rate": 3.78124095609087e-09,
-      "logits/chosen": -2.5883195400238037,
-      "logits/rejected": -2.53429913520813,
-      "logps/chosen": -1.866962194442749,
-      "logps/rejected": -2.517702579498291,
-      "loss": 0.3967,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -2.8133203983306885,
-      "rewards/margins": 1.8152778148651123,
-      "rewards/rejected": -4.628598213195801,
+      "logits/chosen": -2.677032947540283,
+      "logits/rejected": -2.63029408454895,
+      "logps/chosen": -0.9278643727302551,
+      "logps/rejected": -1.0108085870742798,
+      "loss": 0.7435,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.13243435323238373,
+      "rewards/margins": 3.7296395301818848,
+      "rewards/rejected": -3.597205400466919,
       "step": 390
     },
     {
-      "epoch": 0.97,
-      "grad_norm": 37.06032817801063,
+      "epoch": 0.9673518742442564,
+      "grad_norm": 664.330995388725,
       "learning_rate": 1.2105981716597603e-09,
-      "logits/chosen": -2.6171584129333496,
-      "logits/rejected": -2.5108449459075928,
-      "logps/chosen": -1.9496452808380127,
-      "logps/rejected": -2.580923080444336,
-      "loss": 0.4194,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -2.9390242099761963,
-      "rewards/margins": 2.0721981525421143,
-      "rewards/rejected": -5.011221885681152,
+      "logits/chosen": -2.711874485015869,
+      "logits/rejected": -2.614854574203491,
+      "logps/chosen": -0.9732038378715515,
+      "logps/rejected": -0.9536064863204956,
+      "loss": 0.955,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.32332465052604675,
+      "rewards/margins": 3.985724925994873,
+      "rewards/rejected": -4.309049129486084,
       "step": 400
     },
     {
-      "epoch": 0.99,
-      "grad_norm": 36.54571174339,
+      "epoch": 0.9915356711003628,
+      "grad_norm": 1022.9589946634649,
       "learning_rate": 6.453139886395398e-11,
-      "logits/chosen": -2.637455463409424,
-      "logits/rejected": -2.5771093368530273,
-      "logps/chosen": -1.9152857065200806,
-      "logps/rejected": -2.5553231239318848,
-      "loss": 0.4027,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -2.9583487510681152,
-      "rewards/margins": 1.9200756549835205,
-      "rewards/rejected": -4.878424167633057,
+      "logits/chosen": -2.710538864135742,
+      "logits/rejected": -2.6645543575286865,
+      "logps/chosen": -0.9343970417976379,
+      "logps/rejected": -0.9787089228630066,
+      "loss": 0.8541,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.5227594375610352,
+      "rewards/margins": 4.429939270019531,
+      "rewards/rejected": -4.952698707580566,
       "step": 410
     },
     {
-      "epoch": 1.0,
+      "epoch": 0.9987908101571947,
       "step": 413,
       "total_flos": 0.0,
-      "train_loss": 0.47487434232494735,
-      "train_runtime": 6395.0149,
-      "train_samples_per_second": 8.276,
-      "train_steps_per_second": 0.065
+      "train_loss": 0.9438655665076674,
+      "train_runtime": 7020.7263,
+      "train_samples_per_second": 7.538,
+      "train_steps_per_second": 0.059
     }
   ],
   "logging_steps": 10,
@@ -653,6 +653,18 @@
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
   "total_flos": 0.0,
   "train_batch_size": 4,
   "trial_name": null,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:93c9cdc5d47cf48041862ca767083718bee40c0c9d421521bddf246b76bcabfa
-size 6264
+oid sha256:c3a645c7a17ab4c7ed2624bb6f63a7530266096f7cd34d78eb56492568bf376c
+size 6456