Model save

README.md

@@ -52,7 +52,7 @@ The following hyperparameters were used during training:
 
 ### Framework versions
 
-- Transformers 4.39.3
+- Transformers 4.41.1
 - Pytorch 2.1.2+cu118
 - Datasets 2.16.1
-- Tokenizers 0.15.2
+- Tokenizers 0.19.1

all_results.json

@@ -1,8 +1,9 @@
 {
-    "epoch": 1.0,
-    "train_loss": 1.2283447361731714,
-    "train_runtime": 7984.1405,
+    "epoch": 0.9990319457889641,
+    "total_flos": 0.0,
+    "train_loss": 1.2622329578843228,
+    "train_runtime": 8815.3772,
     "train_samples": 66084,
-    "train_samples_per_second": 8.277,
-    "train_steps_per_second": 0.065
+    "train_samples_per_second": 7.496,
+    "train_steps_per_second": 0.059
 }

config.json

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.39.3",
+  "transformers_version": "4.41.1",
   "use_cache": false,
   "vocab_size": 32000
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "transformers_version": "4.39.3"
+  "transformers_version": "4.41.1"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b6137f8b0700bad4f2654974773825d428b43e1035f3f4e0d37e8828850a29b
+oid sha256:ea42776f0193c66b4e372e22097d2fc65400f7bdb2f3ca46f982c79196cf49f7
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d1ed4892d990ee5ca205366b7df90cb53ab573a0b504df190226b7d026710941
+oid sha256:6eb3a99a80f98514c49768d558a81159ceee2e5cf406379207b21f1de70e3733
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d1213c9ea174e5541cc65d907a55897621ce9630ffd180bef0bfb80e5753ab0b
+oid sha256:4c08bb1fbdcc4816d468af7fec45a822a9da800bb4f19eff2e0946970a0e8a2a
 size 4540516344

runs/Jul08_05-29-38_n136-082-130/events.out.tfevents.1720387900.n136-082-130.2711574.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:03326fe07ddd4b507a3ec9ac088acef22cba1ea772f58653e02f8ea1833f74a1
-size 40126
+oid sha256:8233040679669edc5de74dbf2c369f093b9a8fb17949255be5d238d6a8a3326b
+size 41168

tokenizer.json

@@ -134,6 +134,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
+    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

train_results.json

@@ -1,8 +1,9 @@
 {
-    "epoch": 1.0,
-    "train_loss": 1.2283447361731714,
-    "train_runtime": 7984.1405,
+    "epoch": 0.9990319457889641,
+    "total_flos": 0.0,
+    "train_loss": 1.2622329578843228,
+    "train_runtime": 8815.3772,
     "train_samples": 66084,
-    "train_samples_per_second": 8.277,
-    "train_steps_per_second": 0.065
+    "train_samples_per_second": 7.496,
+    "train_steps_per_second": 0.059
 }

trainer_state.json

@@ -9,8 +9,8 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0,
-      "grad_norm": 3433.252872729438,
+      "epoch": 0.001936108422071636,
+      "grad_norm": 4289.771574316242,
       "learning_rate": 1.9230769230769234e-11,
       "logits/chosen": -1.8683955669403076,
       "logits/rejected": -1.7658718824386597,
@@ -24,778 +24,778 @@
       "step": 1
     },
     {
-      "epoch": 0.02,
-      "grad_norm": 3412.8742259369806,
+      "epoch": 0.01936108422071636,
+      "grad_norm": 4190.035270648744,
       "learning_rate": 1.9230769230769234e-10,
-      "logits/chosen": -1.6615941524505615,
-      "logits/rejected": -1.6194777488708496,
-      "logps/chosen": -0.9486016035079956,
-      "logps/rejected": -0.9299763441085815,
-      "loss": 1.1995,
-      "rewards/accuracies": 0.3541666567325592,
-      "rewards/chosen": -0.02906438708305359,
-      "rewards/margins": -0.07331237941980362,
-      "rewards/rejected": 0.044248003512620926,
+      "logits/chosen": -1.6616647243499756,
+      "logits/rejected": -1.6193790435791016,
+      "logps/chosen": -0.9486603140830994,
+      "logps/rejected": -0.9298955202102661,
+      "loss": 1.2029,
+      "rewards/accuracies": 0.3333333432674408,
+      "rewards/chosen": -0.0510055311024189,
+      "rewards/margins": -0.1265178769826889,
+      "rewards/rejected": 0.0755123570561409,
       "step": 10
     },
     {
-      "epoch": 0.04,
-      "grad_norm": 2858.7837493194793,
+      "epoch": 0.03872216844143272,
+      "grad_norm": 3083.054373722184,
       "learning_rate": 3.8461538461538467e-10,
-      "logits/chosen": -1.5834972858428955,
-      "logits/rejected": -1.535605549812317,
-      "logps/chosen": -1.024587631225586,
-      "logps/rejected": -0.970314621925354,
-      "loss": 1.2258,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.014270206913352013,
-      "rewards/margins": 0.010384765453636646,
-      "rewards/rejected": -0.02465497888624668,
+      "logits/chosen": -1.5834848880767822,
+      "logits/rejected": -1.5355803966522217,
+      "logps/chosen": -1.0245015621185303,
+      "logps/rejected": -0.9704240560531616,
+      "loss": 1.2623,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": 0.0036691308487206697,
+      "rewards/margins": 0.06183544546365738,
+      "rewards/rejected": -0.05816630274057388,
       "step": 20
     },
     {
-      "epoch": 0.06,
-      "grad_norm": 3802.89735188374,
+      "epoch": 0.05808325266214908,
+      "grad_norm": 4908.628774557542,
       "learning_rate": 5.769230769230769e-10,
-      "logits/chosen": -1.5293736457824707,
-      "logits/rejected": -1.4700099229812622,
-      "logps/chosen": -1.003350019454956,
-      "logps/rejected": -0.9810702204704285,
-      "loss": 1.2405,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": -0.04970189183950424,
-      "rewards/margins": 0.030842384323477745,
-      "rewards/rejected": -0.08054427057504654,
+      "logits/chosen": -1.5294702053070068,
+      "logits/rejected": -1.4707310199737549,
+      "logps/chosen": -1.0035126209259033,
+      "logps/rejected": -0.9810468554496765,
+      "loss": 1.2911,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.10282345861196518,
+      "rewards/margins": -0.00801654439419508,
+      "rewards/rejected": -0.09480690211057663,
       "step": 30
     },
     {
-      "epoch": 0.08,
-      "grad_norm": 3128.321622751937,
+      "epoch": 0.07744433688286544,
+      "grad_norm": 3634.399853197314,
       "learning_rate": 7.692307692307693e-10,
-      "logits/chosen": -1.554677963256836,
-      "logits/rejected": -1.496991515159607,
-      "logps/chosen": -1.0003422498703003,
-      "logps/rejected": -0.9374068379402161,
-      "loss": 1.216,
-      "rewards/accuracies": 0.45625001192092896,
-      "rewards/chosen": -0.05540431663393974,
-      "rewards/margins": -0.12218408286571503,
-      "rewards/rejected": 0.06677977740764618,
+      "logits/chosen": -1.5545880794525146,
+      "logits/rejected": -1.4971392154693604,
+      "logps/chosen": -1.0002901554107666,
+      "logps/rejected": -0.9374505281448364,
+      "loss": 1.2544,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.056221622973680496,
+      "rewards/margins": -0.12876693904399872,
+      "rewards/rejected": 0.07254532724618912,
       "step": 40
     },
     {
-      "epoch": 0.1,
-      "grad_norm": 3052.7964454970747,
+      "epoch": 0.0968054211035818,
+      "grad_norm": 3743.017714256134,
       "learning_rate": 9.615384615384616e-10,
-      "logits/chosen": -1.649513840675354,
-      "logits/rejected": -1.6041488647460938,
-      "logps/chosen": -0.9904917478561401,
-      "logps/rejected": -0.9689529538154602,
-      "loss": 1.2316,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -0.022712722420692444,
-      "rewards/margins": 0.059487856924533844,
-      "rewards/rejected": -0.08220058679580688,
+      "logits/chosen": -1.6496717929840088,
+      "logits/rejected": -1.6047435998916626,
+      "logps/chosen": -0.990554928779602,
+      "logps/rejected": -0.968321681022644,
+      "loss": 1.2784,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.044189296662807465,
+      "rewards/margins": -0.09925868362188339,
+      "rewards/rejected": 0.05506938695907593,
       "step": 50
     },
     {
-      "epoch": 0.12,
-      "grad_norm": 2902.252196246418,
+      "epoch": 0.11616650532429816,
+      "grad_norm": 3989.4789916346926,
       "learning_rate": 9.99266706925562e-10,
-      "logits/chosen": -1.6003596782684326,
-      "logits/rejected": -1.547429084777832,
-      "logps/chosen": -0.9953937530517578,
-      "logps/rejected": -0.9311510324478149,
-      "loss": 1.2409,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.011818816885352135,
-      "rewards/margins": -0.030796002596616745,
-      "rewards/rejected": 0.04261482506990433,
+      "logits/chosen": -1.6014108657836914,
+      "logits/rejected": -1.5485652685165405,
+      "logps/chosen": -0.9952412843704224,
+      "logps/rejected": -0.9312202334403992,
+      "loss": 1.2834,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.05288747698068619,
+      "rewards/margins": 0.01693376898765564,
+      "rewards/rejected": 0.03595370799303055,
       "step": 60
     },
     {
-      "epoch": 0.14,
-      "grad_norm": 3820.124830724004,
+      "epoch": 0.1355275895450145,
+      "grad_norm": 4801.740697750488,
       "learning_rate": 9.96291389741603e-10,
-      "logits/chosen": -1.5929685831069946,
-      "logits/rejected": -1.5064890384674072,
-      "logps/chosen": -0.9960752725601196,
-      "logps/rejected": -0.9465099573135376,
-      "loss": 1.2095,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.005996152758598328,
-      "rewards/margins": 0.0470072403550148,
-      "rewards/rejected": -0.05300339311361313,
+      "logits/chosen": -1.5917268991470337,
+      "logits/rejected": -1.505897045135498,
+      "logps/chosen": -0.9961442947387695,
+      "logps/rejected": -0.946877658367157,
+      "loss": 1.2574,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.024737417697906494,
+      "rewards/margins": 0.13343551754951477,
+      "rewards/rejected": -0.15817293524742126,
       "step": 70
     },
     {
-      "epoch": 0.15,
-      "grad_norm": 3594.9924550893647,
+      "epoch": 0.15488867376573087,
+      "grad_norm": 4317.0356676312595,
       "learning_rate": 9.91041841371078e-10,
-      "logits/chosen": -1.5460221767425537,
-      "logits/rejected": -1.5286107063293457,
-      "logps/chosen": -1.0364538431167603,
-      "logps/rejected": -0.9859156608581543,
-      "loss": 1.2414,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.10518498718738556,
-      "rewards/margins": -0.0029987855814397335,
-      "rewards/rejected": -0.10218620300292969,
+      "logits/chosen": -1.5460880994796753,
+      "logits/rejected": -1.5285828113555908,
+      "logps/chosen": -1.0357868671417236,
+      "logps/rejected": -0.985648512840271,
+      "loss": 1.2938,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.03525885194540024,
+      "rewards/margins": 0.09618574380874634,
+      "rewards/rejected": -0.0609268844127655,
       "step": 80
     },
     {
-      "epoch": 0.17,
-      "grad_norm": 3142.656864448476,
+      "epoch": 0.17424975798644723,
+      "grad_norm": 4079.0129090665787,
       "learning_rate": 9.835421176144035e-10,
-      "logits/chosen": -1.6783430576324463,
-      "logits/rejected": -1.6198842525482178,
-      "logps/chosen": -1.0106076002120972,
-      "logps/rejected": -0.9245842099189758,
-      "loss": 1.2421,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": 0.11870207637548447,
-      "rewards/margins": 0.038075316697359085,
-      "rewards/rejected": 0.08062675595283508,
+      "logits/chosen": -1.678056001663208,
+      "logits/rejected": -1.6193243265151978,
+      "logps/chosen": -1.0110713243484497,
+      "logps/rejected": -0.9249661564826965,
+      "loss": 1.2586,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.03245333582162857,
+      "rewards/margins": 0.027137070894241333,
+      "rewards/rejected": 0.005316261202096939,
       "step": 90
     },
     {
-      "epoch": 0.19,
-      "grad_norm": 3296.038760800627,
+      "epoch": 0.1936108422071636,
+      "grad_norm": 4056.2821430675544,
       "learning_rate": 9.738265855914014e-10,
-      "logits/chosen": -1.6321983337402344,
-      "logits/rejected": -1.5683090686798096,
-      "logps/chosen": -0.9782571792602539,
-      "logps/rejected": -0.9384816288948059,
-      "loss": 1.2618,
-      "rewards/accuracies": 0.46875,
-      "rewards/chosen": -0.02403583563864231,
-      "rewards/margins": -0.07467617094516754,
-      "rewards/rejected": 0.05064033344388008,
+      "logits/chosen": -1.6319992542266846,
+      "logits/rejected": -1.5686506032943726,
+      "logps/chosen": -0.978125274181366,
+      "logps/rejected": -0.9383082389831543,
+      "loss": 1.2723,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.002950614783912897,
+      "rewards/margins": -0.10372404754161835,
+      "rewards/rejected": 0.10667465627193451,
       "step": 100
     },
     {
-      "epoch": 0.21,
-      "grad_norm": 4108.091542633096,
+      "epoch": 0.21297192642787996,
+      "grad_norm": 5228.540901226267,
       "learning_rate": 9.619397662556434e-10,
-      "logits/chosen": -1.6637523174285889,
-      "logits/rejected": -1.5871461629867554,
-      "logps/chosen": -0.8984575271606445,
-      "logps/rejected": -0.8728235363960266,
-      "loss": 1.2356,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": 0.010543391108512878,
-      "rewards/margins": -0.026898717507719994,
-      "rewards/rejected": 0.03744211047887802,
+      "logits/chosen": -1.6645421981811523,
+      "logits/rejected": -1.588181495666504,
+      "logps/chosen": -0.8982473611831665,
+      "logps/rejected": -0.8726890683174133,
+      "loss": 1.2749,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": 0.06572394073009491,
+      "rewards/margins": -0.014699941501021385,
+      "rewards/rejected": 0.08042389899492264,
       "step": 110
     },
     {
-      "epoch": 0.23,
-      "grad_norm": 3179.4902720939494,
+      "epoch": 0.23233301064859632,
+      "grad_norm": 3837.5381027526573,
       "learning_rate": 9.47936130379344e-10,
-      "logits/chosen": -1.542689323425293,
-      "logits/rejected": -1.5236142873764038,
-      "logps/chosen": -0.9706069827079773,
-      "logps/rejected": -0.9399257898330688,
-      "loss": 1.2417,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": 0.06870836764574051,
-      "rewards/margins": 0.023814253509044647,
-      "rewards/rejected": 0.04489411413669586,
+      "logits/chosen": -1.5425523519515991,
+      "logits/rejected": -1.5234339237213135,
+      "logps/chosen": -0.9705616235733032,
+      "logps/rejected": -0.9401592016220093,
+      "loss": 1.2288,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.0972176343202591,
+      "rewards/margins": 0.09945462644100189,
+      "rewards/rejected": -0.0022369951475411654,
       "step": 120
     },
     {
-      "epoch": 0.25,
-      "grad_norm": 3098.1125119687267,
+      "epoch": 0.25169409486931266,
+      "grad_norm": 3814.5542415994746,
       "learning_rate": 9.318798489436919e-10,
-      "logits/chosen": -1.5796587467193604,
-      "logits/rejected": -1.489713430404663,
-      "logps/chosen": -0.9658042788505554,
-      "logps/rejected": -0.9291502237319946,
-      "loss": 1.2341,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.04142928868532181,
-      "rewards/margins": 0.09168253093957901,
-      "rewards/rejected": -0.050253234803676605,
+      "logits/chosen": -1.579369306564331,
+      "logits/rejected": -1.4889588356018066,
+      "logps/chosen": -0.9654415845870972,
+      "logps/rejected": -0.9286600351333618,
+      "loss": 1.2977,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.14246916770935059,
+      "rewards/margins": 0.08274303376674652,
+      "rewards/rejected": 0.05972614139318466,
       "step": 130
     },
     {
-      "epoch": 0.27,
-      "grad_norm": 3014.945040340012,
+      "epoch": 0.271055179090029,
+      "grad_norm": 3751.478373618925,
       "learning_rate": 9.138444990784454e-10,
-      "logits/chosen": -1.575457215309143,
-      "logits/rejected": -1.5276520252227783,
-      "logps/chosen": -0.993288516998291,
-      "logps/rejected": -0.9939154386520386,
-      "loss": 1.2323,
-      "rewards/accuracies": 0.46875,
-      "rewards/chosen": -0.0799776241183281,
-      "rewards/margins": -0.10535556077957153,
-      "rewards/rejected": 0.025377947837114334,
+      "logits/chosen": -1.5760021209716797,
+      "logits/rejected": -1.5278505086898804,
+      "logps/chosen": -0.9930068850517273,
+      "logps/rejected": -0.9939811825752258,
+      "loss": 1.2583,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.029570287093520164,
+      "rewards/margins": -0.044873449951410294,
+      "rewards/rejected": 0.015303166583180428,
       "step": 140
     },
     {
-      "epoch": 0.29,
-      "grad_norm": 3223.119338557624,
+      "epoch": 0.2904162633107454,
+      "grad_norm": 3907.5968331867316,
       "learning_rate": 8.939127268983109e-10,
-      "logits/chosen": -1.5617806911468506,
-      "logits/rejected": -1.5375396013259888,
-      "logps/chosen": -1.0731405019760132,
-      "logps/rejected": -0.9959096908569336,
-      "loss": 1.2408,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.13223543763160706,
-      "rewards/margins": 0.09327533096075058,
-      "rewards/rejected": 0.038960110396146774,
+      "logits/chosen": -1.5609729290008545,
+      "logits/rejected": -1.5375111103057861,
+      "logps/chosen": -1.0732953548431396,
+      "logps/rejected": -0.9959889650344849,
+      "loss": 1.2631,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.12664008140563965,
+      "rewards/margins": 0.0977608785033226,
+      "rewards/rejected": 0.02887919172644615,
       "step": 150
     },
     {
-      "epoch": 0.31,
-      "grad_norm": 3104.4317123654555,
+      "epoch": 0.30977734753146174,
+      "grad_norm": 4306.071797965924,
       "learning_rate": 8.721758687811352e-10,
-      "logits/chosen": -1.6764981746673584,
-      "logits/rejected": -1.6017143726348877,
-      "logps/chosen": -0.9720491170883179,
-      "logps/rejected": -0.9432562589645386,
-      "loss": 1.2573,
-      "rewards/accuracies": 0.41874998807907104,
-      "rewards/chosen": -0.04389803856611252,
-      "rewards/margins": -0.03539767116308212,
-      "rewards/rejected": -0.008500367403030396,
+      "logits/chosen": -1.6766704320907593,
+      "logits/rejected": -1.6015819311141968,
+      "logps/chosen": -0.9719392657279968,
+      "logps/rejected": -0.9425565600395203,
+      "loss": 1.3024,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.02740299701690674,
+      "rewards/margins": -0.19174733757972717,
+      "rewards/rejected": 0.16434435546398163,
       "step": 160
     },
     {
-      "epoch": 0.33,
-      "grad_norm": 3100.0404111969096,
+      "epoch": 0.3291384317521781,
+      "grad_norm": 4085.902013165705,
       "learning_rate": 8.487335328233912e-10,
-      "logits/chosen": -1.5355565547943115,
-      "logits/rejected": -1.4408550262451172,
-      "logps/chosen": -0.9955608248710632,
-      "logps/rejected": -0.9735702276229858,
-      "loss": 1.2433,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.03805801272392273,
-      "rewards/margins": 0.007754004094749689,
-      "rewards/rejected": 0.030304009094834328,
+      "logits/chosen": -1.5358489751815796,
+      "logits/rejected": -1.4411264657974243,
+      "logps/chosen": -0.9960495829582214,
+      "logps/rejected": -0.9736671447753906,
+      "loss": 1.2834,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.07462203502655029,
+      "rewards/margins": -0.08827298879623413,
+      "rewards/rejected": 0.013650953769683838,
       "step": 170
     },
     {
-      "epoch": 0.35,
-      "grad_norm": 3504.3078457085194,
+      "epoch": 0.34849951597289447,
+      "grad_norm": 4830.939949744352,
       "learning_rate": 8.236931423909139e-10,
-      "logits/chosen": -1.6720529794692993,
-      "logits/rejected": -1.574114203453064,
-      "logps/chosen": -0.9757201075553894,
-      "logps/rejected": -0.9518011808395386,
-      "loss": 1.2196,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.01711990498006344,
-      "rewards/margins": 0.07902441173791885,
-      "rewards/rejected": -0.06190451234579086,
+      "logits/chosen": -1.6721004247665405,
+      "logits/rejected": -1.5736545324325562,
+      "logps/chosen": -0.9753511548042297,
+      "logps/rejected": -0.9517275094985962,
+      "loss": 1.2611,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.11362428963184357,
+      "rewards/margins": 0.17257475852966309,
+      "rewards/rejected": -0.05895046144723892,
       "step": 180
     },
     {
-      "epoch": 0.37,
-      "grad_norm": 3090.0873072741374,
+      "epoch": 0.36786060019361083,
+      "grad_norm": 4117.362535725707,
       "learning_rate": 7.971694438565449e-10,
-      "logits/chosen": -1.6263911724090576,
-      "logits/rejected": -1.563900113105774,
-      "logps/chosen": -0.973171055316925,
-      "logps/rejected": -0.9747421145439148,
-      "loss": 1.226,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.01540590263903141,
-      "rewards/margins": 0.06777185946702957,
-      "rewards/rejected": -0.08317776024341583,
+      "logits/chosen": -1.6258817911148071,
+      "logits/rejected": -1.563320517539978,
+      "logps/chosen": -0.9729933738708496,
+      "logps/rejected": -0.9748104810714722,
+      "loss": 1.2509,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.025180306285619736,
+      "rewards/margins": 0.146215558052063,
+      "rewards/rejected": -0.12103524059057236,
       "step": 190
     },
     {
-      "epoch": 0.39,
-      "grad_norm": 3280.208428429287,
+      "epoch": 0.3872216844143272,
+      "grad_norm": 4321.444634590991,
       "learning_rate": 7.692839807804521e-10,
-      "logits/chosen": -1.630990982055664,
-      "logits/rejected": -1.5965864658355713,
-      "logps/chosen": -0.973315417766571,
-      "logps/rejected": -0.9266045689582825,
-      "loss": 1.2517,
-      "rewards/accuracies": 0.40625,
-      "rewards/chosen": -0.06684601306915283,
-      "rewards/margins": -0.1311662197113037,
-      "rewards/rejected": 0.06432018429040909,
+      "logits/chosen": -1.6311115026474,
+      "logits/rejected": -1.5964945554733276,
+      "logps/chosen": -0.9733870625495911,
+      "logps/rejected": -0.9268864393234253,
+      "loss": 1.2991,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -0.10145823657512665,
+      "rewards/margins": -0.11137993633747101,
+      "rewards/rejected": 0.009921704418957233,
       "step": 200
     },
     {
-      "epoch": 0.41,
-      "grad_norm": 2910.1442572596316,
+      "epoch": 0.40658276863504356,
+      "grad_norm": 4028.0205254352395,
       "learning_rate": 7.401645369426697e-10,
-      "logits/chosen": -1.606448769569397,
-      "logits/rejected": -1.540171504020691,
-      "logps/chosen": -0.9637352824211121,
-      "logps/rejected": -0.9145343899726868,
-      "loss": 1.215,
-      "rewards/accuracies": 0.45625001192092896,
-      "rewards/chosen": 0.009862110018730164,
-      "rewards/margins": -0.023509711027145386,
-      "rewards/rejected": 0.03337181732058525,
+      "logits/chosen": -1.6063648462295532,
+      "logits/rejected": -1.5398415327072144,
+      "logps/chosen": -0.9638331532478333,
+      "logps/rejected": -0.9147375822067261,
+      "loss": 1.2499,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.012107854709029198,
+      "rewards/margins": -0.00303336838260293,
+      "rewards/rejected": -0.009074489586055279,
       "step": 210
     },
     {
-      "epoch": 0.43,
-      "grad_norm": 2974.409845511936,
+      "epoch": 0.4259438528557599,
+      "grad_norm": 4531.775441431939,
       "learning_rate": 7.099445507801324e-10,
-      "logits/chosen": -1.6375468969345093,
-      "logits/rejected": -1.5900648832321167,
-      "logps/chosen": -0.9990792274475098,
-      "logps/rejected": -0.9409869909286499,
-      "loss": 1.2144,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.09886778891086578,
-      "rewards/margins": 0.0811978206038475,
-      "rewards/rejected": 0.01766996458172798,
+      "logits/chosen": -1.6380093097686768,
+      "logits/rejected": -1.589648962020874,
+      "logps/chosen": -0.999239444732666,
+      "logps/rejected": -0.9409993886947632,
+      "loss": 1.2661,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.08351825177669525,
+      "rewards/margins": 0.06452666223049164,
+      "rewards/rejected": 0.018991602584719658,
       "step": 220
     },
     {
-      "epoch": 0.45,
-      "grad_norm": 3525.800329043237,
+      "epoch": 0.4453049370764763,
+      "grad_norm": 4190.546129288723,
       "learning_rate": 6.7876250391152e-10,
-      "logits/chosen": -1.5894132852554321,
-      "logits/rejected": -1.5480855703353882,
-      "logps/chosen": -0.951377272605896,
-      "logps/rejected": -0.9713876843452454,
-      "loss": 1.2267,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": -0.002721360418945551,
-      "rewards/margins": 0.05332719162106514,
-      "rewards/rejected": -0.05604856088757515,
+      "logits/chosen": -1.5896263122558594,
+      "logits/rejected": -1.5483357906341553,
+      "logps/chosen": -0.9513187408447266,
+      "logps/rejected": -0.9711192846298218,
+      "loss": 1.2465,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.011254754848778248,
+      "rewards/margins": 0.014234659262001514,
+      "rewards/rejected": -0.002979907440021634,
       "step": 230
     },
     {
-      "epoch": 0.46,
-      "grad_norm": 3442.0681577443684,
+      "epoch": 0.46466602129719264,
+      "grad_norm": 4241.882144515242,
       "learning_rate": 6.467612865519674e-10,
-      "logits/chosen": -1.6259033679962158,
-      "logits/rejected": -1.600040078163147,
-      "logps/chosen": -0.9880603551864624,
-      "logps/rejected": -0.9100335836410522,
-      "loss": 1.2605,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.07632036507129669,
-      "rewards/margins": -0.12639155983924866,
-      "rewards/rejected": 0.050071217119693756,
+      "logits/chosen": -1.6262489557266235,
+      "logits/rejected": -1.6008937358856201,
+      "logps/chosen": -0.9875959157943726,
+      "logps/rejected": -0.9101887941360474,
+      "loss": 1.2863,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": 0.02070781961083412,
+      "rewards/margins": -0.0031043351627886295,
+      "rewards/rejected": 0.023812144994735718,
       "step": 240
     },
     {
-      "epoch": 0.48,
-      "grad_norm": 3065.812622616001,
+      "epoch": 0.484027105517909,
+      "grad_norm": 3918.096198448725,
       "learning_rate": 6.14087542725593e-10,
-      "logits/chosen": -1.6481239795684814,
-      "logits/rejected": -1.6146821975708008,
-      "logps/chosen": -1.0201656818389893,
-      "logps/rejected": -0.9347532391548157,
-      "loss": 1.2724,
-      "rewards/accuracies": 0.4312500059604645,
-      "rewards/chosen": -0.04044584929943085,
-      "rewards/margins": -0.13707883656024933,
-      "rewards/rejected": 0.09663298726081848,
+      "logits/chosen": -1.6483112573623657,
+      "logits/rejected": -1.6147336959838867,
+      "logps/chosen": -1.020185112953186,
+      "logps/rejected": -0.9349339604377747,
+      "loss": 1.2945,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.05541396886110306,
+      "rewards/margins": -0.13103903830051422,
+      "rewards/rejected": 0.07562507688999176,
       "step": 250
     },
     {
-      "epoch": 0.5,
-      "grad_norm": 2828.71184697319,
+      "epoch": 0.5033881897386253,
+      "grad_norm": 3626.964808344265,
       "learning_rate": 5.808909982763825e-10,
-      "logits/chosen": -1.6454213857650757,
-      "logits/rejected": -1.5548063516616821,
-      "logps/chosen": -0.9938017129898071,
-      "logps/rejected": -0.9485760927200317,
-      "loss": 1.2281,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": 0.030869582667946815,
-      "rewards/margins": 0.039691779762506485,
-      "rewards/rejected": -0.00882219523191452,
+      "logits/chosen": -1.6457252502441406,
+      "logits/rejected": -1.5552377700805664,
+      "logps/chosen": -0.9940276145935059,
+      "logps/rejected": -0.9484678506851196,
+      "loss": 1.274,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -0.017910266295075417,
+      "rewards/margins": -0.03398443013429642,
+      "rewards/rejected": 0.016074160113930702,
       "step": 260
     },
     {
-      "epoch": 0.52,
-      "grad_norm": 3024.1321839759044,
+      "epoch": 0.5227492739593417,
+      "grad_norm": 3993.5474645065597,
       "learning_rate": 5.473237747567806e-10,
-      "logits/chosen": -1.6354761123657227,
-      "logits/rejected": -1.5714191198349,
-      "logps/chosen": -0.9696062803268433,
-      "logps/rejected": -0.9574581980705261,
-      "loss": 1.2206,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.027052313089370728,
-      "rewards/margins": -0.07983547449111938,
-      "rewards/rejected": 0.05278315395116806,
+      "logits/chosen": -1.6354029178619385,
+      "logits/rejected": -1.5715049505233765,
+      "logps/chosen": -0.9696807861328125,
+      "logps/rejected": -0.9575467109680176,
+      "loss": 1.2403,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.052422285079956055,
+      "rewards/margins": -0.0962902158498764,
+      "rewards/rejected": 0.04386794939637184,
       "step": 270
     },
     {
-      "epoch": 0.54,
-      "grad_norm": 3087.829887959573,
+      "epoch": 0.542110358180058,
+      "grad_norm": 4125.713570850562,
       "learning_rate": 5.135396923380673e-10,
-      "logits/chosen": -1.568947434425354,
-      "logits/rejected": -1.495872974395752,
-      "logps/chosen": -0.9841231107711792,
-      "logps/rejected": -0.9465670585632324,
-      "loss": 1.2426,
+      "logits/chosen": -1.5687922239303589,
+      "logits/rejected": -1.495790719985962,
+      "logps/chosen": -0.9840999841690063,
+      "logps/rejected": -0.9467118978500366,
+      "loss": 1.274,
       "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.05226878449320793,
-      "rewards/margins": -0.027065182104706764,
-      "rewards/rejected": 0.07933396100997925,
+      "rewards/chosen": 0.07109468430280685,
+      "rewards/margins": 0.008125528693199158,
+      "rewards/rejected": 0.0629691407084465,
       "step": 280
     },
     {
-      "epoch": 0.56,
-      "grad_norm": 3324.8251041085505,
+      "epoch": 0.5614714424007744,
+      "grad_norm": 4172.164367513521,
       "learning_rate": 4.796935649368935e-10,
-      "logits/chosen": -1.574679970741272,
-      "logits/rejected": -1.4959328174591064,
-      "logps/chosen": -1.0472538471221924,
-      "logps/rejected": -0.993445098400116,
-      "loss": 1.2037,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.01891358196735382,
-      "rewards/margins": 0.11695466190576553,
-      "rewards/rejected": -0.09804105758666992,
+      "logits/chosen": -1.5752254724502563,
+      "logits/rejected": -1.4961917400360107,
+      "logps/chosen": -1.047147512435913,
+      "logps/rejected": -0.9933904409408569,
+      "loss": 1.2562,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0501960925757885,
+      "rewards/margins": 0.15902431309223175,
+      "rewards/rejected": -0.10882820934057236,
       "step": 290
     },
     {
-      "epoch": 0.58,
-      "grad_norm": 3258.5962797542716,
+      "epoch": 0.5808325266214908,
+      "grad_norm": 4075.4312369744443,
       "learning_rate": 4.4594049078802925e-10,
-      "logits/chosen": -1.599393606185913,
-      "logits/rejected": -1.5013492107391357,
-      "logps/chosen": -0.9566828012466431,
-      "logps/rejected": -0.9199537038803101,
-      "loss": 1.216,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.06600774824619293,
-      "rewards/margins": 0.06115701049566269,
-      "rewards/rejected": 0.004850737750530243,
+      "logits/chosen": -1.5983613729476929,
+      "logits/rejected": -1.501022458076477,
+      "logps/chosen": -0.9565431475639343,
+      "logps/rejected": -0.9199585914611816,
+      "loss": 1.251,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.11743637174367905,
+      "rewards/margins": 0.11259187757968903,
+      "rewards/rejected": 0.004844509996473789,
       "step": 300
     },
     {
-      "epoch": 0.6,
-      "grad_norm": 4156.011936794382,
+      "epoch": 0.6001936108422071,
+      "grad_norm": 4415.771404967329,
       "learning_rate": 4.1243514171423466e-10,
-      "logits/chosen": -1.5815703868865967,
-      "logits/rejected": -1.5408540964126587,
-      "logps/chosen": -0.9799707531929016,
-      "logps/rejected": -0.9496101140975952,
-      "loss": 1.2371,
-      "rewards/accuracies": 0.45625001192092896,
-      "rewards/chosen": -0.013506487011909485,
-      "rewards/margins": -0.11432155221700668,
-      "rewards/rejected": 0.10081508010625839,
+      "logits/chosen": -1.5818376541137695,
+      "logits/rejected": -1.5412604808807373,
+      "logps/chosen": -0.9797090291976929,
+      "logps/rejected": -0.9499006271362305,
+      "loss": 1.2733,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 0.04855315759778023,
+      "rewards/margins": -0.004827280528843403,
+      "rewards/rejected": 0.053380437195301056,
       "step": 310
     },
     {
-      "epoch": 0.62,
-      "grad_norm": 3302.7976660213035,
+      "epoch": 0.6195546950629235,
+      "grad_norm": 4020.4622713375675,
       "learning_rate": 3.793310543501473e-10,
-      "logits/chosen": -1.6618106365203857,
-      "logits/rejected": -1.5896971225738525,
-      "logps/chosen": -0.9786826968193054,
-      "logps/rejected": -0.9558340907096863,
-      "loss": 1.2255,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": -0.017104968428611755,
-      "rewards/margins": 0.0024414777290076017,
-      "rewards/rejected": -0.019546449184417725,
+      "logits/chosen": -1.661425232887268,
+      "logits/rejected": -1.589224100112915,
+      "logps/chosen": -0.9785275459289551,
+      "logps/rejected": -0.9554710388183594,
+      "loss": 1.281,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": 0.017457595095038414,
+      "rewards/margins": -0.04888144135475159,
+      "rewards/rejected": 0.06633903831243515,
       "step": 320
     },
     {
-      "epoch": 0.64,
-      "grad_norm": 3509.209967592506,
+      "epoch": 0.6389157792836399,
+      "grad_norm": 4011.0955669502655,
       "learning_rate": 3.4677992656811053e-10,
-      "logits/chosen": -1.6321818828582764,
-      "logits/rejected": -1.6005859375,
-      "logps/chosen": -1.0184519290924072,
-      "logps/rejected": -0.9701558947563171,
-      "loss": 1.2375,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": -0.007846951484680176,
-      "rewards/margins": 0.018898997455835342,
-      "rewards/rejected": -0.026745948940515518,
+      "logits/chosen": -1.6328061819076538,
+      "logits/rejected": -1.6010538339614868,
+      "logps/chosen": -1.0184727907180786,
+      "logps/rejected": -0.9700371623039246,
+      "loss": 1.2624,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.015060502104461193,
+      "rewards/margins": -0.0113009512424469,
+      "rewards/rejected": -0.0037595562171190977,
       "step": 330
     },
     {
-      "epoch": 0.66,
-      "grad_norm": 3710.6450277616004,
+      "epoch": 0.6582768635043562,
+      "grad_norm": 4234.390545263189,
       "learning_rate": 3.149309223300428e-10,
-      "logits/chosen": -1.5329395532608032,
-      "logits/rejected": -1.5036195516586304,
-      "logps/chosen": -1.0616767406463623,
-      "logps/rejected": -0.9750264883041382,
-      "loss": 1.2303,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": 0.15331539511680603,
-      "rewards/margins": 0.07396470010280609,
-      "rewards/rejected": 0.07935069501399994,
+      "logits/chosen": -1.532965064048767,
+      "logits/rejected": -1.5039539337158203,
+      "logps/chosen": -1.0615012645721436,
+      "logps/rejected": -0.975587010383606,
+      "loss": 1.2536,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": 0.23547323048114777,
+      "rewards/margins": 0.2764241397380829,
+      "rewards/rejected": -0.04095090553164482,
       "step": 340
     },
     {
-      "epoch": 0.68,
-      "grad_norm": 3547.547904460223,
+      "epoch": 0.6776379477250726,
+      "grad_norm": 4310.225886680266,
       "learning_rate": 2.8392998815082717e-10,
-      "logits/chosen": -1.65792715549469,
-      "logits/rejected": -1.564651608467102,
-      "logps/chosen": -1.0541561841964722,
-      "logps/rejected": -1.0552833080291748,
-      "loss": 1.2058,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.09947662055492401,
-      "rewards/margins": 0.0944891944527626,
-      "rewards/rejected": 0.0049874186515808105,
+      "logits/chosen": -1.6574161052703857,
+      "logits/rejected": -1.5642915964126587,
+      "logps/chosen": -1.0545518398284912,
+      "logps/rejected": -1.055479645729065,
+      "loss": 1.2634,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.025425389409065247,
+      "rewards/margins": 0.06826835870742798,
+      "rewards/rejected": -0.04284298047423363,
       "step": 350
     },
     {
-      "epoch": 0.7,
-      "grad_norm": 3220.5618942165656,
+      "epoch": 0.6969990319457889,
+      "grad_norm": 3627.3654619519734,
       "learning_rate": 2.5391918430549634e-10,
-      "logits/chosen": -1.691556692123413,
-      "logits/rejected": -1.629815697669983,
-      "logps/chosen": -1.0243303775787354,
-      "logps/rejected": -0.9425641894340515,
-      "loss": 1.2104,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.0219818577170372,
-      "rewards/margins": 0.0395941361784935,
-      "rewards/rejected": -0.017612282186746597,
+      "logits/chosen": -1.6917625665664673,
+      "logits/rejected": -1.6304069757461548,
+      "logps/chosen": -1.0242336988449097,
+      "logps/rejected": -0.9425589442253113,
+      "loss": 1.2358,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0516384020447731,
+      "rewards/margins": 0.07235859334468842,
+      "rewards/rejected": -0.020720209926366806,
       "step": 360
     },
     {
-      "epoch": 0.72,
-      "grad_norm": 3066.9468052938237,
+      "epoch": 0.7163601161665053,
+      "grad_norm": 3729.350909018597,
       "learning_rate": 2.250360338449226e-10,
-      "logits/chosen": -1.7176358699798584,
-      "logits/rejected": -1.705020546913147,
-      "logps/chosen": -0.9747244119644165,
-      "logps/rejected": -0.9268182516098022,
-      "loss": 1.2136,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": 0.00225704163312912,
-      "rewards/margins": 0.037154071033000946,
-      "rewards/rejected": -0.034897033125162125,
+      "logits/chosen": -1.7181346416473389,
+      "logits/rejected": -1.7049201726913452,
+      "logps/chosen": -0.9749780893325806,
+      "logps/rejected": -0.9269036054611206,
+      "loss": 1.2295,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.060578178614377975,
+      "rewards/margins": 0.004380314145237207,
+      "rewards/rejected": -0.06495849788188934,
       "step": 370
     },
     {
-      "epoch": 0.74,
-      "grad_norm": 3016.00166770142,
+      "epoch": 0.7357212003872217,
+      "grad_norm": 3635.364845851746,
       "learning_rate": 1.9741289240311756e-10,
-      "logits/chosen": -1.6244051456451416,
-      "logits/rejected": -1.5759707689285278,
-      "logps/chosen": -0.9922860860824585,
-      "logps/rejected": -0.9526718854904175,
-      "loss": 1.2207,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": 0.030346695333719254,
-      "rewards/margins": 0.0755487009882927,
-      "rewards/rejected": -0.04520200192928314,
+      "logits/chosen": -1.6250957250595093,
+      "logits/rejected": -1.5766202211380005,
+      "logps/chosen": -0.9923038482666016,
+      "logps/rejected": -0.95228511095047,
+      "loss": 1.279,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": 0.03353533893823624,
+      "rewards/margins": -0.006679633166640997,
+      "rewards/rejected": 0.04021497443318367,
       "step": 380
     },
     {
-      "epoch": 0.76,
-      "grad_norm": 2468.5679131308602,
+      "epoch": 0.755082284607938,
+      "grad_norm": 3489.267034208794,
       "learning_rate": 1.7117634168396773e-10,
-      "logits/chosen": -1.6217002868652344,
-      "logits/rejected": -1.5572141408920288,
-      "logps/chosen": -1.0028107166290283,
-      "logps/rejected": -0.9800904989242554,
-      "loss": 1.224,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": 0.1051943302154541,
-      "rewards/margins": 0.10344062000513077,
-      "rewards/rejected": 0.0017537251114845276,
+      "logits/chosen": -1.6205106973648071,
+      "logits/rejected": -1.5565264225006104,
+      "logps/chosen": -1.0025845766067505,
+      "logps/rejected": -0.9802389144897461,
+      "loss": 1.2588,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.1880512535572052,
+      "rewards/margins": 0.2229468822479248,
+      "rewards/rejected": -0.034895628690719604,
       "step": 390
     },
     {
-      "epoch": 0.77,
-      "grad_norm": 2795.143859667373,
+      "epoch": 0.7744433688286544,
+      "grad_norm": 3578.1741607281892,
       "learning_rate": 1.4644660940672628e-10,
-      "logits/chosen": -1.6635463237762451,
-      "logits/rejected": -1.5782734155654907,
-      "logps/chosen": -0.9968119859695435,
-      "logps/rejected": -0.95280522108078,
-      "loss": 1.2085,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": 0.030616816133260727,
-      "rewards/margins": -0.05183488130569458,
-      "rewards/rejected": 0.08245169371366501,
+      "logits/chosen": -1.6632875204086304,
+      "logits/rejected": -1.5778895616531372,
+      "logps/chosen": -0.9968886375427246,
+      "logps/rejected": -0.9527746438980103,
+      "loss": 1.2605,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.01907891035079956,
+      "rewards/margins": -0.09163277596235275,
+      "rewards/rejected": 0.11071167141199112,
       "step": 400
     },
     {
-      "epoch": 0.79,
-      "grad_norm": 4061.2855167991283,
+      "epoch": 0.7938044530493708,
+      "grad_norm": 4697.104208557154,
       "learning_rate": 1.2333701836832813e-10,
-      "logits/chosen": -1.6263242959976196,
-      "logits/rejected": -1.561570405960083,
-      "logps/chosen": -0.9721932411193848,
-      "logps/rejected": -0.9435796737670898,
-      "loss": 1.207,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.027876421809196472,
-      "rewards/margins": 0.05892045423388481,
-      "rewards/rejected": -0.031044036149978638,
+      "logits/chosen": -1.6258773803710938,
+      "logits/rejected": -1.5613044500350952,
+      "logps/chosen": -0.9721547365188599,
+      "logps/rejected": -0.9434949159622192,
+      "loss": 1.2668,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.04442809149622917,
+      "rewards/margins": 0.06200051307678223,
+      "rewards/rejected": -0.01757242903113365,
       "step": 410
     },
     {
-      "epoch": 0.81,
-      "grad_norm": 3787.0389105243557,
+      "epoch": 0.8131655372700871,
+      "grad_norm": 4633.7276813164935,
       "learning_rate": 1.0195346714717813e-10,
-      "logits/chosen": -1.5375678539276123,
-      "logits/rejected": -1.5268394947052002,
-      "logps/chosen": -0.9648769497871399,
-      "logps/rejected": -0.9464458227157593,
-      "loss": 1.2424,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": -0.017999447882175446,
-      "rewards/margins": -0.051319561898708344,
-      "rewards/rejected": 0.0333201140165329,
+      "logits/chosen": -1.537536859512329,
+      "logits/rejected": -1.5266730785369873,
+      "logps/chosen": -0.965029239654541,
+      "logps/rejected": -0.9463868141174316,
+      "loss": 1.2845,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": -0.0605890154838562,
+      "rewards/margins": -0.1169745922088623,
+      "rewards/rejected": 0.05638556554913521,
       "step": 420
     },
     {
-      "epoch": 0.83,
-      "grad_norm": 3059.7951831954097,
+      "epoch": 0.8325266214908035,
+      "grad_norm": 4035.95185171689,
       "learning_rate": 8.239394482805996e-11,
-      "logits/chosen": -1.593246340751648,
-      "logits/rejected": -1.5492476224899292,
-      "logps/chosen": -1.0189164876937866,
-      "logps/rejected": -0.9625579118728638,
-      "loss": 1.2289,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.06651192903518677,
-      "rewards/margins": 0.12220799922943115,
-      "rewards/rejected": -0.05569607764482498,
+      "logits/chosen": -1.5938284397125244,
+      "logits/rejected": -1.5492713451385498,
+      "logps/chosen": -1.0192432403564453,
+      "logps/rejected": -0.9626362919807434,
+      "loss": 1.2589,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.001451274729333818,
+      "rewards/margins": 0.09064897149801254,
+      "rewards/rejected": -0.08919770270586014,
       "step": 430
     },
     {
-      "epoch": 0.85,
-      "grad_norm": 3247.488164594565,
+      "epoch": 0.8518877057115198,
+      "grad_norm": 4317.7906947065,
       "learning_rate": 6.474808197191401e-11,
-      "logits/chosen": -1.6315078735351562,
-      "logits/rejected": -1.5818345546722412,
-      "logps/chosen": -1.046481966972351,
-      "logps/rejected": -0.9752964973449707,
-      "loss": 1.2091,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.041832178831100464,
-      "rewards/margins": 0.09823881089687347,
-      "rewards/rejected": -0.05640662834048271,
+      "logits/chosen": -1.631870985031128,
+      "logits/rejected": -1.5819487571716309,
+      "logps/chosen": -1.0460567474365234,
+      "logps/rejected": -0.9752202033996582,
+      "loss": 1.2476,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.15858839452266693,
+      "rewards/margins": 0.21004195511341095,
+      "rewards/rejected": -0.051453519612550735,
       "step": 440
     },
     {
-      "epoch": 0.87,
-      "grad_norm": 3377.303480206014,
+      "epoch": 0.8712487899322362,
+      "grad_norm": 4211.087790819232,
       "learning_rate": 4.9096739888146e-11,
-      "logits/chosen": -1.6218818426132202,
-      "logits/rejected": -1.5289857387542725,
-      "logps/chosen": -1.0135427713394165,
-      "logps/rejected": -0.9735710024833679,
-      "loss": 1.2589,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.15450438857078552,
-      "rewards/margins": 0.1440540999174118,
-      "rewards/rejected": 0.01045025885105133,
+      "logits/chosen": -1.6219565868377686,
+      "logits/rejected": -1.5289008617401123,
+      "logps/chosen": -1.013887643814087,
+      "logps/rejected": -0.9734300374984741,
+      "loss": 1.2654,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.10689739137887955,
+      "rewards/margins": 0.05857623741030693,
+      "rewards/rejected": 0.04832116514444351,
       "step": 450
     },
     {
-      "epoch": 0.89,
-      "grad_norm": 3083.4878845586436,
+      "epoch": 0.8906098741529526,
+      "grad_norm": 3769.26467222043,
       "learning_rate": 3.5511640091604293e-11,
-      "logits/chosen": -1.5636273622512817,
-      "logits/rejected": -1.5262852907180786,
-      "logps/chosen": -1.056396722793579,
-      "logps/rejected": -0.9384132623672485,
-      "loss": 1.2066,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.023233562707901,
-      "rewards/margins": 0.048523545265197754,
-      "rewards/rejected": -0.07175710052251816,
+      "logits/chosen": -1.5646103620529175,
+      "logits/rejected": -1.5271342992782593,
+      "logps/chosen": -1.0559688806533813,
+      "logps/rejected": -0.9385896921157837,
+      "loss": 1.2187,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.07790811359882355,
+      "rewards/margins": 0.21173524856567383,
+      "rewards/rejected": -0.13382713496685028,
       "step": 460
     },
     {
-      "epoch": 0.91,
-      "grad_norm": 3283.918271980478,
+      "epoch": 0.9099709583736689,
+      "grad_norm": 4757.142899979977,
       "learning_rate": 2.4055035642222225e-11,
-      "logits/chosen": -1.637670874595642,
-      "logits/rejected": -1.5689923763275146,
-      "logps/chosen": -0.9840338826179504,
-      "logps/rejected": -0.9136601686477661,
-      "loss": 1.2087,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.046017058193683624,
-      "rewards/margins": -0.03135695680975914,
-      "rewards/rejected": 0.07737401872873306,
+      "logits/chosen": -1.6382324695587158,
+      "logits/rejected": -1.5700418949127197,
+      "logps/chosen": -0.9839082956314087,
+      "logps/rejected": -0.9138771295547485,
+      "loss": 1.2532,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.08895201981067657,
+      "rewards/margins": 0.04646927863359451,
+      "rewards/rejected": 0.04248274117708206,
       "step": 470
     },
     {
-      "epoch": 0.93,
-      "grad_norm": 2984.402833421987,
+      "epoch": 0.9293320425943853,
+      "grad_norm": 3836.5764604419264,
       "learning_rate": 1.477942587339426e-11,
-      "logits/chosen": -1.6090877056121826,
-      "logits/rejected": -1.5660176277160645,
-      "logps/chosen": -0.9489718675613403,
-      "logps/rejected": -0.9073503613471985,
-      "loss": 1.2087,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.04631403833627701,
-      "rewards/margins": -0.06630536168813705,
-      "rewards/rejected": 0.019991323351860046,
+      "logits/chosen": -1.6090694665908813,
+      "logits/rejected": -1.5662223100662231,
+      "logps/chosen": -0.9489457011222839,
+      "logps/rejected": -0.90757817029953,
+      "loss": 1.2642,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.05137147754430771,
+      "rewards/margins": -0.01941109262406826,
+      "rewards/rejected": -0.0319603867828846,
       "step": 480
     },
     {
-      "epoch": 0.95,
-      "grad_norm": 3305.4541171804303,
+      "epoch": 0.9486931268151017,
+      "grad_norm": 4250.965954777766,
       "learning_rate": 7.727315816331515e-12,
-      "logits/chosen": -1.657076120376587,
-      "logits/rejected": -1.607134461402893,
-      "logps/chosen": -1.0778063535690308,
-      "logps/rejected": -1.0003712177276611,
-      "loss": 1.2118,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": 0.044107623398303986,
-      "rewards/margins": 0.06955888122320175,
-      "rewards/rejected": -0.025451254099607468,
+      "logits/chosen": -1.6554877758026123,
+      "logits/rejected": -1.6055545806884766,
+      "logps/chosen": -1.0777404308319092,
+      "logps/rejected": -1.0004560947418213,
+      "loss": 1.2376,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.07162754982709885,
+      "rewards/margins": 0.12462921440601349,
+      "rewards/rejected": -0.05300166457891464,
       "step": 490
     },
     {
-      "epoch": 0.97,
-      "grad_norm": 3282.0354146265427,
+      "epoch": 0.968054211035818,
+      "grad_norm": 3914.5437511702235,
       "learning_rate": 2.9310214228202016e-12,
-      "logits/chosen": -1.6225248575210571,
-      "logits/rejected": -1.5484001636505127,
-      "logps/chosen": -0.9589713215827942,
-      "logps/rejected": -0.9175511598587036,
-      "loss": 1.2114,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.06731443107128143,
-      "rewards/margins": -0.1021970883011818,
-      "rewards/rejected": 0.034882646054029465,
+      "logits/chosen": -1.6229422092437744,
+      "logits/rejected": -1.5482664108276367,
+      "logps/chosen": -0.9587628245353699,
+      "logps/rejected": -0.9172463417053223,
+      "loss": 1.2265,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.032034747302532196,
+      "rewards/margins": -0.1518467366695404,
+      "rewards/rejected": 0.1198119968175888,
       "step": 500
     },
     {
-      "epoch": 0.99,
-      "grad_norm": 3258.514052979456,
+      "epoch": 0.9874152952565344,
+      "grad_norm": 4236.531801010516,
       "learning_rate": 4.125214789427734e-13,
-      "logits/chosen": -1.564023733139038,
-      "logits/rejected": -1.50169837474823,
-      "logps/chosen": -0.9931696653366089,
-      "logps/rejected": -0.9385188221931458,
-      "loss": 1.2461,
-      "rewards/accuracies": 0.45625001192092896,
-      "rewards/chosen": 0.05308717489242554,
-      "rewards/margins": 0.01174162793904543,
-      "rewards/rejected": 0.041345540434122086,
+      "logits/chosen": -1.5640289783477783,
+      "logits/rejected": -1.5025882720947266,
+      "logps/chosen": -0.9930515289306641,
+      "logps/rejected": -0.9385469555854797,
+      "loss": 1.254,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.09590280055999756,
+      "rewards/margins": 0.0512530580163002,
+      "rewards/rejected": 0.04464975371956825,
       "step": 510
     },
     {
-      "epoch": 1.0,
+      "epoch": 0.9990319457889641,
       "step": 516,
       "total_flos": 0.0,
-      "train_loss": 1.2283447361731714,
-      "train_runtime": 7984.1405,
-      "train_samples_per_second": 8.277,
-      "train_steps_per_second": 0.065
+      "train_loss": 1.2622329578843228,
+      "train_runtime": 8815.3772,
+      "train_samples_per_second": 7.496,
+      "train_steps_per_second": 0.059
     }
   ],
   "logging_steps": 10,
@@ -803,6 +803,18 @@
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
   "total_flos": 0.0,
   "train_batch_size": 4,
   "trial_name": null,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b98ffc796b97b2fbb0115b6523bcf5cf2f91af7851d4af544ee956eaa778a5f3
-size 6328
+oid sha256:cd72d8e0cd286217c9a7fddf58a057686f97c0f94432feaca0ca148ca8dbc12e
+size 6520