Model save

README.md

@@ -2,7 +2,6 @@
 tags:
 - trl
 - dpo
-- alignment-handbook
 - generated_from_trainer
 model-index:
 - name: zephyr-7b-dpo-full
@@ -14,17 +13,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model was trained from scratch on an unknown dataset.
+This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Logits/chosen: -5.3452
-- Logits/rejected: -5.6032
-- Logps/chosen: -585.4705
-- Logps/rejected: -737.1887
-- Loss: 0.7740
-- Rewards/accuracies: 0.5664
-- Rewards/chosen: -1.9500
-- Rewards/margins: 0.2381
-- Rewards/rejected: -2.1880
+- Loss: 1.1302
+- Rewards/chosen: -1.0433
+- Rewards/rejected: -0.6633
+- Rewards/accuracies: 0.4102
+- Rewards/margins: -0.3800
+- Logps/rejected: -531.6535
+- Logps/chosen: -411.3397
+- Logits/rejected: -4.8567
+- Logits/chosen: -4.6242
 
 ## Model description
 
@@ -43,7 +42,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 1e-08
+- learning_rate: 1e-07
 - train_batch_size: 8
 - eval_batch_size: 8
 - seed: 42
@@ -59,16 +58,16 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch  | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
-|:-------------:|:------:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
-| 0.4622        | 0.2558 | 100  | -4.7883       | -5.0069         | -439.2997    | -570.1285      | 0.7195          | 0.4883             | -0.4883        | 0.0292          | -0.5174          |
-| 0.3618        | 0.5115 | 200  | -5.1462       | -5.4044         | -551.2876    | -696.0472      | 0.7717          | 0.5703             | -1.6081        | 0.1685          | -1.7766          |
-| 0.3249        | 0.7673 | 300  | -5.3452       | -5.6032         | -585.4705    | -737.1887      | 0.7740          | 0.5664             | -1.9500        | 0.2381          | -2.1880          |
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.2472        | 0.26  | 100  | 0.9611          | -0.2112        | 0.1236           | 0.3906             | -0.3347         | -515.9166      | -394.6981    | -4.8019         | -4.5851       |
+| 0.2112        | 0.51  | 200  | 1.1025          | -0.7299        | -0.3200          | 0.375              | -0.4098         | -524.7885      | -405.0722    | -4.8388         | -4.6065       |
+| 0.195         | 0.77  | 300  | 1.1302          | -1.0433        | -0.6633          | 0.4102             | -0.3800         | -531.6535      | -411.3397    | -4.8567         | -4.6242       |
 
 
 ### Framework versions
 
-- Transformers 4.40.2
+- Transformers 4.38.2
 - Pytorch 2.1.2+cu118
-- Datasets 2.19.1
-- Tokenizers 0.19.1
+- Datasets 2.16.1
+- Tokenizers 0.15.2

all_results.json

@@ -1,22 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -5.382973670959473,
-    "eval_logits/rejected": -5.635280132293701,
-    "eval_logps/chosen": -592.3199462890625,
-    "eval_logps/rejected": -744.3624877929688,
-    "eval_loss": 0.7785406708717346,
-    "eval_rewards/accuracies": 0.57421875,
-    "eval_rewards/chosen": -2.0184545516967773,
-    "eval_rewards/margins": 0.24129416048526764,
-    "eval_rewards/rejected": -2.259748935699463,
-    "eval_runtime": 97.5883,
-    "eval_samples": 2000,
-    "eval_samples_per_second": 20.494,
-    "eval_steps_per_second": 0.328,
-    "total_flos": 0.0,
-    "train_loss": 0.0756377827785814,
-    "train_runtime": 1327.4397,
+    "train_loss": 0.256967593336959,
+    "train_runtime": 6146.1986,
     "train_samples": 50000,
-    "train_samples_per_second": 37.666,
-    "train_steps_per_second": 0.295
+    "train_samples_per_second": 8.135,
+    "train_steps_per_second": 0.064
 }

config.json

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.40.2",
+  "transformers_version": "4.38.2",
   "use_cache": false,
   "vocab_size": 32002
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 32000,
-  "transformers_version": "4.40.2"
+  "transformers_version": "4.38.2"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:37bb171e534385dad7304abb74d8ce7b17254c662782c0bba2f79c82a353549b
+oid sha256:b773863c1c34f71a12f146b8f74ffc662775755518fb796340f28308788aed9b
 size 4943178720

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1ef099be007fed6d276729f594e479f44624be82506119b73084fbf8fa37df2e
+oid sha256:03e30dfe84f62b6452cd39b1564f484dca8c20abaae2c9fb750f42eb765dad5b
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac0cbe20b18932e70e062265222254736aae966444d5c53190bd1f029c5b41dd
+oid sha256:146df42653d4b3f0e5289c416cd3c0ea883bd1b3e641c90176feec5bc2b98424
 size 4540532728

runs/May28_08-05-26_n136-100-194/events.out.tfevents.1716854751.n136-100-194.686462.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eeb102a6a961fc7e7539d1a3ae091dabd9b7ddc9ba247f87e9997f9aceff3dbd
-size 28266
+oid sha256:9cc9a190ba3803a02df56b7c56954abe9fb71c889378030299424e8d0d0e9f01
+size 34812

tokenizer.json

@@ -152,7 +152,6 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
-    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

train_results.json

@@ -1,9 +1,8 @@
 {
     "epoch": 1.0,
-    "total_flos": 0.0,
-    "train_loss": 0.0756377827785814,
-    "train_runtime": 1327.4397,
+    "train_loss": 0.256967593336959,
+    "train_runtime": 6146.1986,
     "train_samples": 50000,
-    "train_samples_per_second": 37.666,
-    "train_steps_per_second": 0.295
+    "train_samples_per_second": 8.135,
+    "train_steps_per_second": 0.064
 }

trainer_state.json

@@ -9,14 +9,14 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0025575447570332483,
-      "grad_norm": 39.26094609355056,
+      "epoch": 0.0,
+      "grad_norm": 210.34713052784278,
       "learning_rate": 2.5e-09,
-      "logits/chosen": -4.633832931518555,
-      "logits/rejected": -4.853580951690674,
-      "logps/chosen": -233.06472778320312,
-      "logps/rejected": -207.33529663085938,
-      "loss": 0.6931,
+      "logits/chosen": -4.623842239379883,
+      "logits/rejected": -4.85917854309082,
+      "logps/chosen": -239.31422424316406,
+      "logps/rejected": -207.56365966796875,
+      "loss": 0.6927,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -24,646 +24,646 @@
       "step": 1
     },
     {
-      "epoch": 0.02557544757033248,
-      "grad_norm": 36.68935602174721,
+      "epoch": 0.03,
+      "grad_norm": 198.95172630432864,
       "learning_rate": 2.5e-08,
-      "logits/chosen": -4.338743686676025,
-      "logits/rejected": -4.649880886077881,
-      "logps/chosen": -262.4483642578125,
-      "logps/rejected": -221.99293518066406,
-      "loss": 0.693,
-      "rewards/accuracies": 0.4791666567325592,
-      "rewards/chosen": 0.00011811345757450908,
-      "rewards/margins": 0.0013624767307192087,
-      "rewards/rejected": -0.0012443631421774626,
+      "logits/chosen": -4.3338446617126465,
+      "logits/rejected": -4.64424991607666,
+      "logps/chosen": -265.20184326171875,
+      "logps/rejected": -215.72174072265625,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.4236111044883728,
+      "rewards/chosen": -0.004745930898934603,
+      "rewards/margins": -0.004067909903824329,
+      "rewards/rejected": -0.0006780209369026124,
       "step": 10
     },
     {
-      "epoch": 0.05115089514066496,
-      "grad_norm": 39.59560101486009,
+      "epoch": 0.05,
+      "grad_norm": 204.7891876677461,
       "learning_rate": 5e-08,
-      "logits/chosen": -4.518132209777832,
-      "logits/rejected": -4.7573723793029785,
-      "logps/chosen": -265.6820373535156,
-      "logps/rejected": -224.56332397460938,
-      "loss": 0.6925,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": 0.0007334979018196464,
-      "rewards/margins": 0.0019747079350054264,
-      "rewards/rejected": -0.00124121003318578,
+      "logits/chosen": -4.509727478027344,
+      "logits/rejected": -4.74410343170166,
+      "logps/chosen": -267.73052978515625,
+      "logps/rejected": -216.7478485107422,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.010470375418663025,
+      "rewards/margins": 0.01739482954144478,
+      "rewards/rejected": -0.006924452725797892,
       "step": 20
     },
     {
-      "epoch": 0.07672634271099744,
-      "grad_norm": 42.07292039672945,
+      "epoch": 0.08,
+      "grad_norm": 204.94575488992174,
       "learning_rate": 7.5e-08,
-      "logits/chosen": -4.586892127990723,
-      "logits/rejected": -4.772247314453125,
-      "logps/chosen": -254.6095428466797,
-      "logps/rejected": -221.1687469482422,
-      "loss": 0.6881,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 0.004800780676305294,
-      "rewards/margins": 0.01116220187395811,
-      "rewards/rejected": -0.006361422594636679,
+      "logits/chosen": -4.5970940589904785,
+      "logits/rejected": -4.777865409851074,
+      "logps/chosen": -257.5598449707031,
+      "logps/rejected": -215.4015350341797,
+      "loss": 0.6544,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": 0.04864828661084175,
+      "rewards/margins": 0.09208732843399048,
+      "rewards/rejected": -0.04343904182314873,
       "step": 30
     },
     {
-      "epoch": 0.10230179028132992,
-      "grad_norm": 39.99985818605015,
+      "epoch": 0.1,
+      "grad_norm": 163.67699084811588,
       "learning_rate": 1e-07,
-      "logits/chosen": -4.643338680267334,
-      "logits/rejected": -4.734442234039307,
-      "logps/chosen": -250.8112030029297,
-      "logps/rejected": -227.5035858154297,
-      "loss": 0.674,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": 0.015374640934169292,
-      "rewards/margins": 0.04068901389837265,
-      "rewards/rejected": -0.025314366444945335,
+      "logits/chosen": -4.643096923828125,
+      "logits/rejected": -4.7387237548828125,
+      "logps/chosen": -249.96743774414062,
+      "logps/rejected": -223.3234405517578,
+      "loss": 0.5584,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 0.18158790469169617,
+      "rewards/margins": 0.36420467495918274,
+      "rewards/rejected": -0.18261677026748657,
       "step": 40
     },
     {
-      "epoch": 0.1278772378516624,
-      "grad_norm": 39.42008970309411,
+      "epoch": 0.13,
+      "grad_norm": 125.1152304775479,
       "learning_rate": 9.979985922607475e-08,
-      "logits/chosen": -4.586745262145996,
-      "logits/rejected": -4.8019514083862305,
-      "logps/chosen": -266.86578369140625,
-      "logps/rejected": -236.32333374023438,
-      "loss": 0.6472,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.025944575667381287,
-      "rewards/margins": 0.09520022571086884,
-      "rewards/rejected": -0.06925566494464874,
+      "logits/chosen": -4.558148384094238,
+      "logits/rejected": -4.785082817077637,
+      "logps/chosen": -265.6357727050781,
+      "logps/rejected": -234.0360107421875,
+      "loss": 0.45,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.3166799247264862,
+      "rewards/margins": 0.7249041795730591,
+      "rewards/rejected": -0.4082241952419281,
       "step": 50
     },
     {
-      "epoch": 0.1534526854219949,
-      "grad_norm": 38.00390022566866,
+      "epoch": 0.15,
+      "grad_norm": 110.5697848266263,
       "learning_rate": 9.92010391574745e-08,
-      "logits/chosen": -4.757911205291748,
-      "logits/rejected": -4.887364387512207,
-      "logps/chosen": -239.2329559326172,
-      "logps/rejected": -240.6675262451172,
-      "loss": 0.5963,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.0015811750199645758,
-      "rewards/margins": 0.20014917850494385,
-      "rewards/rejected": -0.20173034071922302,
+      "logits/chosen": -4.701218605041504,
+      "logits/rejected": -4.855440139770508,
+      "logps/chosen": -232.1560821533203,
+      "logps/rejected": -235.8180389404297,
+      "loss": 0.3379,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.44831886887550354,
+      "rewards/margins": 1.4881489276885986,
+      "rewards/rejected": -1.039829969406128,
       "step": 60
     },
     {
-      "epoch": 0.17902813299232737,
-      "grad_norm": 40.739477874841256,
+      "epoch": 0.18,
+      "grad_norm": 103.09926490168155,
       "learning_rate": 9.820833372667812e-08,
-      "logits/chosen": -4.700837135314941,
-      "logits/rejected": -4.890389442443848,
-      "logps/chosen": -262.3451843261719,
-      "logps/rejected": -265.8101501464844,
-      "loss": 0.552,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.08908917009830475,
-      "rewards/margins": 0.345054566860199,
-      "rewards/rejected": -0.4341437816619873,
+      "logits/chosen": -4.597586631774902,
+      "logits/rejected": -4.846543312072754,
+      "logps/chosen": -243.5035858154297,
+      "logps/rejected": -245.3424072265625,
+      "loss": 0.3085,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 0.4819186329841614,
+      "rewards/margins": 1.8609161376953125,
+      "rewards/rejected": -1.378997564315796,
       "step": 70
     },
     {
-      "epoch": 0.20460358056265984,
-      "grad_norm": 39.05798828996261,
+      "epoch": 0.2,
+      "grad_norm": 89.87848352821936,
       "learning_rate": 9.682969016701356e-08,
-      "logits/chosen": -4.702755451202393,
-      "logits/rejected": -4.89903450012207,
-      "logps/chosen": -271.28289794921875,
-      "logps/rejected": -292.86474609375,
-      "loss": 0.4978,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.11036349833011627,
-      "rewards/margins": 0.46904292702674866,
-      "rewards/rejected": -0.5794063806533813,
+      "logits/chosen": -4.592278957366943,
+      "logits/rejected": -4.840281963348389,
+      "logps/chosen": -249.3519744873047,
+      "logps/rejected": -261.445068359375,
+      "loss": 0.2624,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 0.593399703502655,
+      "rewards/margins": 2.1497161388397217,
+      "rewards/rejected": -1.5563163757324219,
       "step": 80
     },
     {
-      "epoch": 0.23017902813299232,
-      "grad_norm": 35.899018808141314,
+      "epoch": 0.23,
+      "grad_norm": 98.45898295424381,
       "learning_rate": 9.507614539004081e-08,
-      "logits/chosen": -4.802427768707275,
-      "logits/rejected": -5.013784408569336,
-      "logps/chosen": -254.51138305664062,
-      "logps/rejected": -277.52117919921875,
-      "loss": 0.4653,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -0.007465996779501438,
-      "rewards/margins": 0.6835082769393921,
-      "rewards/rejected": -0.6909742951393127,
+      "logits/chosen": -4.667254447937012,
+      "logits/rejected": -4.913816928863525,
+      "logps/chosen": -235.763427734375,
+      "logps/rejected": -244.2578582763672,
+      "loss": 0.2462,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 0.7724655866622925,
+      "rewards/margins": 2.8438591957092285,
+      "rewards/rejected": -2.0713934898376465,
       "step": 90
     },
     {
-      "epoch": 0.2557544757033248,
-      "grad_norm": 36.72256884048717,
+      "epoch": 0.26,
+      "grad_norm": 87.96881533227138,
       "learning_rate": 9.296173762811083e-08,
-      "logits/chosen": -4.685734748840332,
-      "logits/rejected": -4.969820976257324,
-      "logps/chosen": -261.9112243652344,
-      "logps/rejected": -306.5802001953125,
-      "loss": 0.4622,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.03183584660291672,
-      "rewards/margins": 0.7450669407844543,
-      "rewards/rejected": -0.7769027948379517,
+      "logits/chosen": -4.5116472244262695,
+      "logits/rejected": -4.829812049865723,
+      "logps/chosen": -238.08468627929688,
+      "logps/rejected": -269.5484619140625,
+      "loss": 0.2472,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.931675910949707,
+      "rewards/margins": 3.0536458492279053,
+      "rewards/rejected": -2.1219699382781982,
       "step": 100
     },
     {
-      "epoch": 0.2557544757033248,
-      "eval_logits/chosen": -4.788325309753418,
-      "eval_logits/rejected": -5.006863594055176,
-      "eval_logps/chosen": -439.2997131347656,
-      "eval_logps/rejected": -570.1284790039062,
-      "eval_loss": 0.7195349335670471,
-      "eval_rewards/accuracies": 0.48828125,
-      "eval_rewards/chosen": -0.4882526695728302,
-      "eval_rewards/margins": 0.029155578464269638,
-      "eval_rewards/rejected": -0.5174082517623901,
-      "eval_runtime": 98.6046,
-      "eval_samples_per_second": 20.283,
-      "eval_steps_per_second": 0.325,
+      "epoch": 0.26,
+      "eval_logits/chosen": -4.58513879776001,
+      "eval_logits/rejected": -4.80186128616333,
+      "eval_logps/chosen": -394.6981201171875,
+      "eval_logps/rejected": -515.9166259765625,
+      "eval_loss": 0.9610964059829712,
+      "eval_rewards/accuracies": 0.390625,
+      "eval_rewards/chosen": -0.21118265390396118,
+      "eval_rewards/margins": -0.3347358703613281,
+      "eval_rewards/rejected": 0.12355318665504456,
+      "eval_runtime": 97.8315,
+      "eval_samples_per_second": 20.443,
+      "eval_steps_per_second": 0.327,
       "step": 100
     },
     {
-      "epoch": 0.2813299232736573,
-      "grad_norm": 37.92952007987077,
+      "epoch": 0.28,
+      "grad_norm": 84.98735748868098,
       "learning_rate": 9.050339404945832e-08,
-      "logits/chosen": -4.800631999969482,
-      "logits/rejected": -5.014934539794922,
-      "logps/chosen": -261.18560791015625,
-      "logps/rejected": -321.3470458984375,
-      "loss": 0.4274,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -0.1468946933746338,
-      "rewards/margins": 0.8339446783065796,
-      "rewards/rejected": -0.9808392524719238,
+      "logits/chosen": -4.55401611328125,
+      "logits/rejected": -4.845933437347412,
+      "logps/chosen": -229.4434356689453,
+      "logps/rejected": -257.52984619140625,
+      "loss": 0.2226,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.9871166348457336,
+      "rewards/margins": 3.0293149948120117,
+      "rewards/rejected": -2.042198419570923,
       "step": 110
     },
     {
-      "epoch": 0.3069053708439898,
-      "grad_norm": 34.34594445177828,
+      "epoch": 0.31,
+      "grad_norm": 75.78122724506682,
       "learning_rate": 8.77207952455395e-08,
-      "logits/chosen": -4.794532775878906,
-      "logits/rejected": -5.045430660247803,
-      "logps/chosen": -288.1357727050781,
-      "logps/rejected": -340.7710876464844,
-      "loss": 0.4231,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -0.13442620635032654,
-      "rewards/margins": 0.9264079928398132,
-      "rewards/rejected": -1.0608341693878174,
+      "logits/chosen": -4.49249792098999,
+      "logits/rejected": -4.787415981292725,
+      "logps/chosen": -252.7578125,
+      "logps/rejected": -273.38555908203125,
+      "loss": 0.2215,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 1.1615877151489258,
+      "rewards/margins": 3.400435209274292,
+      "rewards/rejected": -2.238847255706787,
       "step": 120
     },
     {
-      "epoch": 0.33248081841432225,
-      "grad_norm": 38.8387240251847,
+      "epoch": 0.33,
+      "grad_norm": 109.1136183108071,
       "learning_rate": 8.463621767547997e-08,
-      "logits/chosen": -4.894143104553223,
-      "logits/rejected": -5.190768718719482,
-      "logps/chosen": -281.547607421875,
-      "logps/rejected": -352.4320983886719,
-      "loss": 0.4055,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -0.25050926208496094,
-      "rewards/margins": 1.063452959060669,
-      "rewards/rejected": -1.3139623403549194,
+      "logits/chosen": -4.589264869689941,
+      "logits/rejected": -4.87318229675293,
+      "logps/chosen": -239.29531860351562,
+      "logps/rejected": -265.04693603515625,
+      "loss": 0.2169,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 0.9574294090270996,
+      "rewards/margins": 3.4433422088623047,
+      "rewards/rejected": -2.485912799835205,
       "step": 130
     },
     {
-      "epoch": 0.35805626598465473,
-      "grad_norm": 36.960561396839225,
+      "epoch": 0.36,
+      "grad_norm": 79.72525878658313,
       "learning_rate": 8.127435532896387e-08,
-      "logits/chosen": -4.948976993560791,
-      "logits/rejected": -5.229872703552246,
-      "logps/chosen": -320.14996337890625,
-      "logps/rejected": -386.9942932128906,
-      "loss": 0.392,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -0.3797687590122223,
-      "rewards/margins": 1.1361901760101318,
-      "rewards/rejected": -1.5159590244293213,
+      "logits/chosen": -4.636221885681152,
+      "logits/rejected": -4.9098310470581055,
+      "logps/chosen": -267.59625244140625,
+      "logps/rejected": -288.02349853515625,
+      "loss": 0.2063,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 0.8299416303634644,
+      "rewards/margins": 3.622443675994873,
+      "rewards/rejected": -2.792501926422119,
       "step": 140
     },
     {
-      "epoch": 0.3836317135549872,
-      "grad_norm": 43.84353984691604,
+      "epoch": 0.38,
+      "grad_norm": 94.45112212404622,
       "learning_rate": 7.766212203526569e-08,
-      "logits/chosen": -5.017346382141113,
-      "logits/rejected": -5.299422740936279,
-      "logps/chosen": -287.28350830078125,
-      "logps/rejected": -373.42730712890625,
-      "loss": 0.3891,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -0.374920517206192,
-      "rewards/margins": 1.0954115390777588,
-      "rewards/rejected": -1.4703319072723389,
+      "logits/chosen": -4.643942832946777,
+      "logits/rejected": -4.911728382110596,
+      "logps/chosen": -233.4263153076172,
+      "logps/rejected": -277.07818603515625,
+      "loss": 0.2098,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 0.9495984315872192,
+      "rewards/margins": 3.8475449085235596,
+      "rewards/rejected": -2.89794659614563,
       "step": 150
     },
     {
-      "epoch": 0.4092071611253197,
-      "grad_norm": 45.6486792527098,
+      "epoch": 0.41,
+      "grad_norm": 85.35291313866578,
       "learning_rate": 7.382843600106538e-08,
-      "logits/chosen": -5.09747314453125,
-      "logits/rejected": -5.334885120391846,
-      "logps/chosen": -294.9914855957031,
-      "logps/rejected": -389.2686462402344,
-      "loss": 0.3586,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -0.47372356057167053,
-      "rewards/margins": 1.2067290544509888,
-      "rewards/rejected": -1.680452585220337,
+      "logits/chosen": -4.690377235412598,
+      "logits/rejected": -4.9024457931518555,
+      "logps/chosen": -233.21981811523438,
+      "logps/rejected": -271.2682189941406,
+      "loss": 0.1861,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 0.9118326306343079,
+      "rewards/margins": 3.6947906017303467,
+      "rewards/rejected": -2.7829582691192627,
       "step": 160
     },
     {
-      "epoch": 0.43478260869565216,
-      "grad_norm": 41.55934089834119,
+      "epoch": 0.43,
+      "grad_norm": 75.89103255157417,
       "learning_rate": 6.980398830195784e-08,
-      "logits/chosen": -5.022572040557861,
-      "logits/rejected": -5.321537494659424,
-      "logps/chosen": -307.09521484375,
-      "logps/rejected": -427.2223205566406,
-      "loss": 0.3397,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -0.5031875967979431,
-      "rewards/margins": 1.4829540252685547,
-      "rewards/rejected": -1.9861418008804321,
+      "logits/chosen": -4.554282188415527,
+      "logits/rejected": -4.874223232269287,
+      "logps/chosen": -236.4412078857422,
+      "logps/rejected": -279.4911804199219,
+      "loss": 0.1833,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 1.2316501140594482,
+      "rewards/margins": 4.21605920791626,
+      "rewards/rejected": -2.9844090938568115,
       "step": 170
     },
     {
-      "epoch": 0.46035805626598464,
-      "grad_norm": 37.76415174175888,
+      "epoch": 0.46,
+      "grad_norm": 77.81291773020575,
       "learning_rate": 6.562099718102787e-08,
-      "logits/chosen": -5.190648078918457,
-      "logits/rejected": -5.442112922668457,
-      "logps/chosen": -293.28350830078125,
-      "logps/rejected": -403.9653625488281,
-      "loss": 0.3551,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -0.5512182116508484,
-      "rewards/margins": 1.491098165512085,
-      "rewards/rejected": -2.042316198348999,
+      "logits/chosen": -4.651320934295654,
+      "logits/rejected": -4.9173784255981445,
+      "logps/chosen": -215.70126342773438,
+      "logps/rejected": -251.5159149169922,
+      "loss": 0.2065,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 1.0581190586090088,
+      "rewards/margins": 3.832904815673828,
+      "rewards/rejected": -2.7747855186462402,
       "step": 180
     },
     {
-      "epoch": 0.4859335038363171,
-      "grad_norm": 52.26242886674056,
+      "epoch": 0.49,
+      "grad_norm": 86.461876717381,
       "learning_rate": 6.131295012148612e-08,
-      "logits/chosen": -5.1211113929748535,
-      "logits/rejected": -5.269497871398926,
-      "logps/chosen": -328.1192321777344,
-      "logps/rejected": -461.3627014160156,
-      "loss": 0.3605,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -0.7219178080558777,
-      "rewards/margins": 1.4698314666748047,
-      "rewards/rejected": -2.1917495727539062,
+      "logits/chosen": -4.617634296417236,
+      "logits/rejected": -4.793360233306885,
+      "logps/chosen": -239.32681274414062,
+      "logps/rejected": -286.96124267578125,
+      "loss": 0.2013,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 1.0783030986785889,
+      "rewards/margins": 3.6680614948272705,
+      "rewards/rejected": -2.5897579193115234,
       "step": 190
     },
     {
-      "epoch": 0.5115089514066496,
-      "grad_norm": 41.210687347051376,
+      "epoch": 0.51,
+      "grad_norm": 84.24320751887706,
       "learning_rate": 5.691433575823665e-08,
-      "logits/chosen": -5.160891056060791,
-      "logits/rejected": -5.3507490158081055,
-      "logps/chosen": -323.77532958984375,
-      "logps/rejected": -448.09271240234375,
-      "loss": 0.3618,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.7529298067092896,
-      "rewards/margins": 1.5445276498794556,
-      "rewards/rejected": -2.297457456588745,
+      "logits/chosen": -4.624228477478027,
+      "logits/rejected": -4.830000877380371,
+      "logps/chosen": -233.09713745117188,
+      "logps/rejected": -271.84051513671875,
+      "loss": 0.2112,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.9521042108535767,
+      "rewards/margins": 3.7768027782440186,
+      "rewards/rejected": -2.8246986865997314,
       "step": 200
     },
     {
-      "epoch": 0.5115089514066496,
-      "eval_logits/chosen": -5.146201133728027,
-      "eval_logits/rejected": -5.404381275177002,
-      "eval_logps/chosen": -551.28759765625,
-      "eval_logps/rejected": -696.0471801757812,
-      "eval_loss": 0.7717023491859436,
-      "eval_rewards/accuracies": 0.5703125,
-      "eval_rewards/chosen": -1.6081316471099854,
-      "eval_rewards/margins": 0.16846325993537903,
-      "eval_rewards/rejected": -1.7765947580337524,
-      "eval_runtime": 98.5146,
-      "eval_samples_per_second": 20.302,
-      "eval_steps_per_second": 0.325,
+      "epoch": 0.51,
+      "eval_logits/chosen": -4.606511116027832,
+      "eval_logits/rejected": -4.8388166427612305,
+      "eval_logps/chosen": -405.0722351074219,
+      "eval_logps/rejected": -524.7885131835938,
+      "eval_loss": 1.102483868598938,
+      "eval_rewards/accuracies": 0.375,
+      "eval_rewards/chosen": -0.729888916015625,
+      "eval_rewards/margins": -0.40984660387039185,
+      "eval_rewards/rejected": -0.32004231214523315,
+      "eval_runtime": 97.8012,
+      "eval_samples_per_second": 20.45,
+      "eval_steps_per_second": 0.327,
       "step": 200
     },
     {
-      "epoch": 0.5370843989769821,
-      "grad_norm": 33.990804012478485,
+      "epoch": 0.54,
+      "grad_norm": 70.97621814359026,
       "learning_rate": 5.2460367774593905e-08,
-      "logits/chosen": -5.241884708404541,
-      "logits/rejected": -5.514366626739502,
-      "logps/chosen": -325.63385009765625,
-      "logps/rejected": -494.1783142089844,
-      "loss": 0.3361,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": -0.631729245185852,
-      "rewards/margins": 1.9169912338256836,
-      "rewards/rejected": -2.548720359802246,
+      "logits/chosen": -4.6944451332092285,
+      "logits/rejected": -4.962179183959961,
+      "logps/chosen": -243.93307495117188,
+      "logps/rejected": -297.62066650390625,
+      "loss": 0.1723,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 1.0976004600524902,
+      "rewards/margins": 4.546332836151123,
+      "rewards/rejected": -3.448732376098633,
       "step": 210
     },
     {
-      "epoch": 0.5626598465473146,
-      "grad_norm": 43.799668582141386,
+      "epoch": 0.56,
+      "grad_norm": 71.88477654183092,
       "learning_rate": 4.798670299452925e-08,
-      "logits/chosen": -5.0720014572143555,
-      "logits/rejected": -5.483579635620117,
-      "logps/chosen": -326.8290100097656,
-      "logps/rejected": -476.4143981933594,
-      "loss": 0.3361,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -0.6208838224411011,
-      "rewards/margins": 1.8470569849014282,
-      "rewards/rejected": -2.4679408073425293,
+      "logits/chosen": -4.529160499572754,
+      "logits/rejected": -4.8643479347229,
+      "logps/chosen": -241.5579833984375,
+      "logps/rejected": -293.224365234375,
+      "loss": 0.1923,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 1.0626676082611084,
+      "rewards/margins": 4.490227699279785,
+      "rewards/rejected": -3.4275600910186768,
       "step": 220
     },
     {
-      "epoch": 0.5882352941176471,
-      "grad_norm": 49.054323066624825,
+      "epoch": 0.59,
+      "grad_norm": 83.09100453064212,
       "learning_rate": 4.3529155927297226e-08,
-      "logits/chosen": -5.175291538238525,
-      "logits/rejected": -5.545914649963379,
-      "logps/chosen": -346.18603515625,
-      "logps/rejected": -467.39093017578125,
-      "loss": 0.3482,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -0.8635517954826355,
-      "rewards/margins": 1.513214111328125,
-      "rewards/rejected": -2.376765727996826,
+      "logits/chosen": -4.6047258377075195,
+      "logits/rejected": -4.93651008605957,
+      "logps/chosen": -241.11477661132812,
+      "logps/rejected": -293.9808044433594,
+      "loss": 0.2012,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 0.9927155375480652,
+      "rewards/margins": 4.5062031745910645,
+      "rewards/rejected": -3.5134873390197754,
       "step": 230
     },
     {
-      "epoch": 0.6138107416879796,
-      "grad_norm": 39.76260402422977,
+      "epoch": 0.61,
+      "grad_norm": 70.11336436391163,
       "learning_rate": 3.9123412049691636e-08,
-      "logits/chosen": -5.179227828979492,
-      "logits/rejected": -5.498720645904541,
-      "logps/chosen": -356.7921142578125,
-      "logps/rejected": -494.8655700683594,
-      "loss": 0.3289,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -0.8680986166000366,
-      "rewards/margins": 1.7818403244018555,
-      "rewards/rejected": -2.6499390602111816,
+      "logits/chosen": -4.588685035705566,
+      "logits/rejected": -4.866146087646484,
+      "logps/chosen": -252.31533813476562,
+      "logps/rejected": -294.6343688964844,
+      "loss": 0.1875,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 1.0253931283950806,
+      "rewards/margins": 4.710432529449463,
+      "rewards/rejected": -3.6850390434265137,
       "step": 240
     },
     {
-      "epoch": 0.639386189258312,
-      "grad_norm": 39.31547537418584,
+      "epoch": 0.64,
+      "grad_norm": 77.75874575792918,
       "learning_rate": 3.480474212128766e-08,
-      "logits/chosen": -5.349845886230469,
-      "logits/rejected": -5.674462795257568,
-      "logps/chosen": -343.0057067871094,
-      "logps/rejected": -444.6160583496094,
-      "loss": 0.3376,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -0.9290262460708618,
-      "rewards/margins": 1.3633143901824951,
-      "rewards/rejected": -2.2923407554626465,
+      "logits/chosen": -4.716187000274658,
+      "logits/rejected": -4.966707229614258,
+      "logps/chosen": -231.89279174804688,
+      "logps/rejected": -266.51666259765625,
+      "loss": 0.1825,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.840434193611145,
+      "rewards/margins": 3.7858078479766846,
+      "rewards/rejected": -2.94537353515625,
       "step": 250
     },
     {
-      "epoch": 0.6649616368286445,
-      "grad_norm": 47.933967621919784,
+      "epoch": 0.66,
+      "grad_norm": 97.12524424809816,
       "learning_rate": 3.060771981975726e-08,
-      "logits/chosen": -5.2227373123168945,
-      "logits/rejected": -5.566103935241699,
-      "logps/chosen": -334.39306640625,
-      "logps/rejected": -502.48211669921875,
-      "loss": 0.3271,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -0.8672464489936829,
-      "rewards/margins": 1.90872323513031,
-      "rewards/rejected": -2.7759692668914795,
+      "logits/chosen": -4.585513114929199,
+      "logits/rejected": -4.878482341766357,
+      "logps/chosen": -234.92617797851562,
+      "logps/rejected": -297.1214904785156,
+      "loss": 0.1837,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": 0.9561206102371216,
+      "rewards/margins": 4.824769973754883,
+      "rewards/rejected": -3.86864972114563,
       "step": 260
     },
     {
-      "epoch": 0.690537084398977,
-      "grad_norm": 37.918454908956036,
+      "epoch": 0.69,
+      "grad_norm": 92.49874438996748,
       "learning_rate": 2.6565944956764818e-08,
-      "logits/chosen": -5.3946356773376465,
-      "logits/rejected": -5.655900478363037,
-      "logps/chosen": -349.11785888671875,
-      "logps/rejected": -490.94720458984375,
-      "loss": 0.3334,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -0.8713266253471375,
-      "rewards/margins": 1.8250868320465088,
-      "rewards/rejected": -2.696413516998291,
+      "logits/chosen": -4.684746742248535,
+      "logits/rejected": -4.911890983581543,
+      "logps/chosen": -243.29568481445312,
+      "logps/rejected": -288.39111328125,
+      "loss": 0.1961,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.8868792653083801,
+      "rewards/margins": 4.555182456970215,
+      "rewards/rejected": -3.6683037281036377,
       "step": 270
     },
     {
-      "epoch": 0.7161125319693095,
-      "grad_norm": 43.57355633058329,
+      "epoch": 0.72,
+      "grad_norm": 73.7028241699641,
       "learning_rate": 2.2711774490274766e-08,
-      "logits/chosen": -5.313577651977539,
-      "logits/rejected": -5.556331157684326,
-      "logps/chosen": -347.73529052734375,
-      "logps/rejected": -519.9879150390625,
-      "loss": 0.3132,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -0.8589982986450195,
-      "rewards/margins": 1.8729617595672607,
-      "rewards/rejected": -2.731959819793701,
+      "logits/chosen": -4.634344577789307,
+      "logits/rejected": -4.873081207275391,
+      "logps/chosen": -245.1703338623047,
+      "logps/rejected": -317.2539978027344,
+      "loss": 0.1644,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.9668266177177429,
+      "rewards/margins": 4.682557582855225,
+      "rewards/rejected": -3.715731143951416,
       "step": 280
     },
     {
-      "epoch": 0.7416879795396419,
-      "grad_norm": 48.474190706284276,
+      "epoch": 0.74,
+      "grad_norm": 142.25337407808868,
       "learning_rate": 1.9076063486687256e-08,
-      "logits/chosen": -5.153687477111816,
-      "logits/rejected": -5.569068908691406,
-      "logps/chosen": -342.547607421875,
-      "logps/rejected": -473.8719787597656,
-      "loss": 0.3082,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": -0.7416984438896179,
-      "rewards/margins": 1.8063793182373047,
-      "rewards/rejected": -2.5480778217315674,
+      "logits/chosen": -4.503401756286621,
+      "logits/rejected": -4.866554260253906,
+      "logps/chosen": -250.9346160888672,
+      "logps/rejected": -283.41046142578125,
+      "loss": 0.1799,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 1.083687424659729,
+      "rewards/margins": 4.472739219665527,
+      "rewards/rejected": -3.389052152633667,
       "step": 290
     },
     {
-      "epoch": 0.7672634271099744,
-      "grad_norm": 46.47610707831878,
+      "epoch": 0.77,
+      "grad_norm": 88.66793876665662,
       "learning_rate": 1.5687918106563324e-08,
-      "logits/chosen": -5.316671371459961,
-      "logits/rejected": -5.562603950500488,
-      "logps/chosen": -339.9733581542969,
-      "logps/rejected": -511.3289489746094,
-      "loss": 0.3249,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -0.8634950518608093,
-      "rewards/margins": 2.052011251449585,
-      "rewards/rejected": -2.915506601333618,
+      "logits/chosen": -4.625166416168213,
+      "logits/rejected": -4.831929683685303,
+      "logps/chosen": -232.6981658935547,
+      "logps/rejected": -288.00457763671875,
+      "loss": 0.195,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 1.0354994535446167,
+      "rewards/margins": 4.637454509735107,
+      "rewards/rejected": -3.6019554138183594,
       "step": 300
     },
     {
-      "epoch": 0.7672634271099744,
-      "eval_logits/chosen": -5.345163345336914,
-      "eval_logits/rejected": -5.603174209594727,
-      "eval_logps/chosen": -585.470458984375,
-      "eval_logps/rejected": -737.188720703125,
-      "eval_loss": 0.7740334272384644,
-      "eval_rewards/accuracies": 0.56640625,
-      "eval_rewards/chosen": -1.9499597549438477,
-      "eval_rewards/margins": 0.23805105686187744,
-      "eval_rewards/rejected": -2.1880106925964355,
-      "eval_runtime": 98.564,
-      "eval_samples_per_second": 20.291,
-      "eval_steps_per_second": 0.325,
+      "epoch": 0.77,
+      "eval_logits/chosen": -4.624210357666016,
+      "eval_logits/rejected": -4.856749057769775,
+      "eval_logps/chosen": -411.3396911621094,
+      "eval_logps/rejected": -531.6535034179688,
+      "eval_loss": 1.1301820278167725,
+      "eval_rewards/accuracies": 0.41015625,
+      "eval_rewards/chosen": -1.0432608127593994,
+      "eval_rewards/margins": -0.3799673318862915,
+      "eval_rewards/rejected": -0.6632934212684631,
+      "eval_runtime": 97.9609,
+      "eval_samples_per_second": 20.416,
+      "eval_steps_per_second": 0.327,
       "step": 300
     },
     {
-      "epoch": 0.7928388746803069,
-      "grad_norm": 43.4969717520313,
+      "epoch": 0.79,
+      "grad_norm": 90.90394303193246,
       "learning_rate": 1.257446259144494e-08,
-      "logits/chosen": -5.234282493591309,
-      "logits/rejected": -5.613511085510254,
-      "logps/chosen": -346.8320007324219,
-      "logps/rejected": -529.3019409179688,
-      "loss": 0.3197,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -0.8631784319877625,
-      "rewards/margins": 2.172049045562744,
-      "rewards/rejected": -3.0352275371551514,
+      "logits/chosen": -4.541079044342041,
+      "logits/rejected": -4.873132228851318,
+      "logps/chosen": -239.60592651367188,
+      "logps/rejected": -298.159423828125,
+      "loss": 0.1847,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 1.076683759689331,
+      "rewards/margins": 4.874758243560791,
+      "rewards/rejected": -3.7980740070343018,
       "step": 310
     },
     {
-      "epoch": 0.8184143222506394,
-      "grad_norm": 49.480305879245726,
+      "epoch": 0.82,
+      "grad_norm": 87.85310576006609,
       "learning_rate": 9.760622117187234e-09,
-      "logits/chosen": -5.375542640686035,
-      "logits/rejected": -5.707432746887207,
-      "logps/chosen": -341.70611572265625,
-      "logps/rejected": -499.69854736328125,
-      "loss": 0.3333,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": -0.9386850595474243,
-      "rewards/margins": 1.9452975988388062,
-      "rewards/rejected": -2.8839828968048096,
+      "logits/chosen": -4.597599029541016,
+      "logits/rejected": -4.9500837326049805,
+      "logps/chosen": -227.94247436523438,
+      "logps/rejected": -279.3809814453125,
+      "loss": 0.1934,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 0.8610901832580566,
+      "rewards/margins": 4.507565498352051,
+      "rewards/rejected": -3.6464743614196777,
       "step": 320
     },
     {
-      "epoch": 0.8439897698209718,
-      "grad_norm": 47.987548127540066,
+      "epoch": 0.84,
+      "grad_norm": 85.81889719468313,
       "learning_rate": 7.2689232521989885e-09,
-      "logits/chosen": -5.299598693847656,
-      "logits/rejected": -5.636387348175049,
-      "logps/chosen": -363.62799072265625,
-      "logps/rejected": -517.6014404296875,
-      "loss": 0.314,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -0.9615789651870728,
-      "rewards/margins": 1.8379528522491455,
-      "rewards/rejected": -2.799531936645508,
+      "logits/chosen": -4.554391860961914,
+      "logits/rejected": -4.864416599273682,
+      "logps/chosen": -249.89169311523438,
+      "logps/rejected": -304.54913330078125,
+      "loss": 0.1773,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.9128581881523132,
+      "rewards/margins": 4.5053324699401855,
+      "rewards/rejected": -3.5924744606018066,
       "step": 330
     },
     {
-      "epoch": 0.8695652173913043,
-      "grad_norm": 39.276792588944474,
+      "epoch": 0.87,
+      "grad_norm": 77.2990699180903,
       "learning_rate": 5.119313618049309e-09,
-      "logits/chosen": -5.3245110511779785,
-      "logits/rejected": -5.723358154296875,
-      "logps/chosen": -366.1358947753906,
-      "logps/rejected": -471.16693115234375,
-      "loss": 0.3133,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -0.9128230214118958,
-      "rewards/margins": 1.6382230520248413,
-      "rewards/rejected": -2.5510458946228027,
+      "logits/chosen": -4.570425987243652,
+      "logits/rejected": -4.913475513458252,
+      "logps/chosen": -250.6792449951172,
+      "logps/rejected": -277.26556396484375,
+      "loss": 0.1723,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 1.1733391284942627,
+      "rewards/margins": 4.697513580322266,
+      "rewards/rejected": -3.524174451828003,
       "step": 340
     },
     {
-      "epoch": 0.8951406649616368,
-      "grad_norm": 44.505022117748766,
+      "epoch": 0.9,
+      "grad_norm": 74.61892537865367,
       "learning_rate": 3.3290021961708158e-09,
-      "logits/chosen": -5.358895778656006,
-      "logits/rejected": -5.502166271209717,
-      "logps/chosen": -351.8700256347656,
-      "logps/rejected": -492.72833251953125,
-      "loss": 0.3421,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.0121654272079468,
-      "rewards/margins": 1.6023184061050415,
-      "rewards/rejected": -2.6144840717315674,
+      "logits/chosen": -4.588479995727539,
+      "logits/rejected": -4.761317253112793,
+      "logps/chosen": -238.91921997070312,
+      "logps/rejected": -291.2458190917969,
+      "loss": 0.1937,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.8275976181030273,
+      "rewards/margins": 4.001389026641846,
+      "rewards/rejected": -3.1737911701202393,
       "step": 350
     },
     {
-      "epoch": 0.9207161125319693,
-      "grad_norm": 43.52722682791457,
+      "epoch": 0.92,
+      "grad_norm": 81.22407668854541,
       "learning_rate": 1.9123215591052013e-09,
-      "logits/chosen": -5.307016372680664,
-      "logits/rejected": -5.524683952331543,
-      "logps/chosen": -356.2999267578125,
-      "logps/rejected": -509.2779235839844,
-      "loss": 0.3231,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -0.950890839099884,
-      "rewards/margins": 1.7997655868530273,
-      "rewards/rejected": -2.7506566047668457,
+      "logits/chosen": -4.583038806915283,
+      "logits/rejected": -4.805889129638672,
+      "logps/chosen": -244.8368682861328,
+      "logps/rejected": -294.9869079589844,
+      "loss": 0.1907,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 0.8399018049240112,
+      "rewards/margins": 4.216121673583984,
+      "rewards/rejected": -3.3762192726135254,
       "step": 360
     },
     {
-      "epoch": 0.9462915601023018,
-      "grad_norm": 45.19067766851852,
+      "epoch": 0.95,
+      "grad_norm": 75.17805842008224,
       "learning_rate": 8.806131292167618e-10,
-      "logits/chosen": -5.336704730987549,
-      "logits/rejected": -5.556437969207764,
-      "logps/chosen": -349.86932373046875,
-      "logps/rejected": -495.8702087402344,
-      "loss": 0.341,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.957863450050354,
-      "rewards/margins": 1.5969595909118652,
-      "rewards/rejected": -2.5548229217529297,
+      "logits/chosen": -4.595518112182617,
+      "logits/rejected": -4.752079010009766,
+      "logps/chosen": -239.1554412841797,
+      "logps/rejected": -302.4869079589844,
+      "loss": 0.1904,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 0.8832891583442688,
+      "rewards/margins": 4.165283679962158,
+      "rewards/rejected": -3.281994581222534,
       "step": 370
     },
     {
-      "epoch": 0.9718670076726342,
-      "grad_norm": 44.75028680489971,
+      "epoch": 0.97,
+      "grad_norm": 91.00267878372446,
       "learning_rate": 2.4213638345040867e-10,
-      "logits/chosen": -5.498834133148193,
-      "logits/rejected": -5.772934913635254,
-      "logps/chosen": -356.33770751953125,
-      "logps/rejected": -501.0440979003906,
-      "loss": 0.3268,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -0.9707822799682617,
-      "rewards/margins": 1.7587143182754517,
-      "rewards/rejected": -2.729496717453003,
+      "logits/chosen": -4.70483922958374,
+      "logits/rejected": -4.97845983505249,
+      "logps/chosen": -242.5469207763672,
+      "logps/rejected": -292.7474670410156,
+      "loss": 0.1788,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 0.9767888188362122,
+      "rewards/margins": 4.587931156158447,
+      "rewards/rejected": -3.61114239692688,
       "step": 380
     },
     {
-      "epoch": 0.9974424552429667,
-      "grad_norm": 54.761084032021465,
+      "epoch": 1.0,
+      "grad_norm": 100.5241948062632,
       "learning_rate": 2.0027310073833516e-12,
-      "logits/chosen": -5.484150409698486,
-      "logits/rejected": -5.723382949829102,
-      "logps/chosen": -359.4778747558594,
-      "logps/rejected": -514.458251953125,
-      "loss": 0.3144,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -0.9768111109733582,
-      "rewards/margins": 1.897681474685669,
-      "rewards/rejected": -2.8744921684265137,
+      "logits/chosen": -4.696263313293457,
+      "logits/rejected": -4.96966028213501,
+      "logps/chosen": -238.3385772705078,
+      "logps/rejected": -292.5868835449219,
+      "loss": 0.1773,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 1.066699743270874,
+      "rewards/margins": 4.670289516448975,
+      "rewards/rejected": -3.6035892963409424,
       "step": 390
     },
     {
       "epoch": 1.0,
       "step": 391,
       "total_flos": 0.0,
-      "train_loss": 0.0756377827785814,
-      "train_runtime": 1327.4397,
-      "train_samples_per_second": 37.666,
-      "train_steps_per_second": 0.295
+      "train_loss": 0.256967593336959,
+      "train_runtime": 6146.1986,
+      "train_samples_per_second": 8.135,
+      "train_steps_per_second": 0.064
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e599c16ce070b390131b8a105cd9f3d8874b8b69ec427608b7bb0892f924d35f
-size 6328
+oid sha256:62fd36d8ce66e8becc572379f18c762113afc3bd8c05085b884b7f3d305aff55
+size 6264