Model save

Browse files

Files changed (12) hide show

README.md +19 -19
all_results.json +5 -4
config.json +1 -1
generation_config.json +1 -1
model-00001-of-00003.safetensors +1 -1
model-00002-of-00003.safetensors +1 -1
model-00003-of-00003.safetensors +1 -1
runs/May28_06-41-22_n136-112-146/events.out.tfevents.1716850671.n136-112-146.393581.0 +2 -2
tokenizer.json +1 -0
train_results.json +5 -4
trainer_state.json +518 -518
training_args.bin +2 -2

README.md CHANGED Viewed

@@ -13,17 +13,17 @@ should probably proofread and complete it, then remove this comment. -->
 # zephyr-7b-dpo-full
-This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.5791
-- Rewards/chosen: 0.3072
-- Rewards/rejected: 1.0868
-- Rewards/accuracies: 0.3984
-- Rewards/margins: -0.7796
-- Logps/rejected: -516.2141
-- Logps/chosen: -389.8601
-- Logits/rejected: -4.7141
-- Logits/chosen: -4.4983
 ## Model description
@@ -42,7 +42,7 @@ More information needed
 ### Training hyperparameters
 The following hyperparameters were used during training:
-- learning_rate: 5e-08
 - train_batch_size: 8
 - eval_batch_size: 8
 - seed: 42
@@ -58,16 +58,16 @@ The following hyperparameters were used during training:
 ### Training results
-| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
-|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.2235        | 0.26  | 100  | 1.4095          | 0.3999         | 1.0916           | 0.4023             | -0.6917         | -516.2045      | -389.6747    | -4.7259         | -4.5122       |
-| 0.1957        | 0.51  | 200  | 1.5616          | 0.3683         | 1.1571           | 0.3945             | -0.7887         | -516.0735      | -389.7379    | -4.7163         | -4.5006       |
-| 0.1905        | 0.77  | 300  | 1.5791          | 0.3072         | 1.0868           | 0.3984             | -0.7796         | -516.2141      | -389.8601    | -4.7141         | -4.4983       |
 ### Framework versions
-- Transformers 4.38.2
 - Pytorch 2.1.2+cu118
-- Datasets 2.16.1
-- Tokenizers 0.15.2

 # zephyr-7b-dpo-full
+This model was trained from scratch on an unknown dataset.
 It achieves the following results on the evaluation set:
+- Loss: 0.7740
+- Rewards/chosen: -1.9500
+- Rewards/rejected: -2.1880
+- Rewards/accuracies: 0.5664
+- Rewards/margins: 0.2381
+- Logps/rejected: -737.1887
+- Logps/chosen: -585.4705
+- Logits/rejected: -5.6032
+- Logits/chosen: -5.3452
 ## Model description
 ### Training hyperparameters
 The following hyperparameters were used during training:
+- learning_rate: 1e-07
 - train_batch_size: 8
 - eval_batch_size: 8
 - seed: 42
 ### Training results
+| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.4622        | 0.2558 | 100  | 0.7195          | -0.4883        | -0.5174          | 0.4883             | 0.0292          | -570.1285      | -439.2997    | -5.0069         | -4.7883       |
+| 0.3618        | 0.5115 | 200  | 0.7717          | -1.6081        | -1.7766          | 0.5703             | 0.1685          | -696.0472      | -551.2876    | -5.4044         | -5.1462       |
+| 0.3249        | 0.7673 | 300  | 0.7740          | -1.9500        | -2.1880          | 0.5664             | 0.2381          | -737.1887      | -585.4705    | -5.6032         | -5.3452       |
 ### Framework versions
+- Transformers 4.40.2
 - Pytorch 2.1.2+cu118
+- Datasets 2.19.1
+- Tokenizers 0.19.1

all_results.json CHANGED Viewed

@@ -1,8 +1,9 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.240954409779795,
-    "train_runtime": 6147.1036,
     "train_samples": 50000,
-    "train_samples_per_second": 8.134,
-    "train_steps_per_second": 0.064
 }

 {
     "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.4100414538932273,
+    "train_runtime": 6205.0725,
     "train_samples": 50000,
+    "train_samples_per_second": 8.058,
+    "train_steps_per_second": 0.063
 }

config.json CHANGED Viewed

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.38.2",
   "use_cache": false,
   "vocab_size": 32002
 }

   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
+  "transformers_version": "4.40.2",
   "use_cache": false,
   "vocab_size": 32002
 }

generation_config.json CHANGED Viewed

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 32000,
-  "transformers_version": "4.38.2"
 }

   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 32000,
+  "transformers_version": "4.40.2"
 }

model-00001-of-00003.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1bc3fd278cbe14a39e172890c4aabe2c20a2257df6b0c76ecb41da3a95c4e415
 size 4943178720

 version https://git-lfs.github.com/spec/v1
+oid sha256:20c9414c680bd55f51253e0e4dfdbacc85dff3cfb76ed2d4bd550f1830a171b4
 size 4943178720

model-00002-of-00003.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5760e6042dc762e1e811acc0d083f514e84b0f14a3b257fb2feb22883320d90a
 size 4999819336

 version https://git-lfs.github.com/spec/v1
+oid sha256:474dd254cc45ede807d9f0a8b68dfb008d4ddc55d3a0ff14db78753609a61d0c
 size 4999819336

model-00003-of-00003.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:91024b816086db126cfb61e13397356a73e0aac26213a216ea62483e24d52156
 size 4540532728

 version https://git-lfs.github.com/spec/v1
+oid sha256:1ad09cd99e0738430a25a2fb1677e46fb1ea87cbd34429e65175e901c8d92668
 size 4540532728

runs/May28_06-41-22_n136-112-146/events.out.tfevents.1716850671.n136-112-146.393581.0 CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e05ea396a3a7c9245aef69b3de6c17481466a691810fe5099b55a27bcd455000
-size 28347

 version https://git-lfs.github.com/spec/v1
+oid sha256:2a1e7691a040571f5ad20207152ba3f51f3e12251eb1473b2611db1643b0afb1
+size 34893

tokenizer.json CHANGED Viewed

@@ -152,6 +152,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
+    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

train_results.json CHANGED Viewed

@@ -1,8 +1,9 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.240954409779795,
-    "train_runtime": 6147.1036,
     "train_samples": 50000,
-    "train_samples_per_second": 8.134,
-    "train_steps_per_second": 0.064
 }

 {
     "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.4100414538932273,
+    "train_runtime": 6205.0725,
     "train_samples": 50000,
+    "train_samples_per_second": 8.058,
+    "train_steps_per_second": 0.063
 }

trainer_state.json CHANGED Viewed

@@ -9,14 +9,14 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0,
-      "grad_norm": 2119.773933625158,
-      "learning_rate": 1.25e-09,
-      "logits/chosen": -4.623842239379883,
-      "logits/rejected": -4.85917854309082,
-      "logps/chosen": -239.31422424316406,
-      "logps/rejected": -207.56365966796875,
-      "loss": 0.7016,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -24,646 +24,646 @@
       "step": 1
     },
     {
-      "epoch": 0.03,
-      "grad_norm": 2039.8145241987106,
-      "learning_rate": 1.25e-08,
-      "logits/chosen": -4.334660530090332,
-      "logits/rejected": -4.644404888153076,
-      "logps/chosen": -265.1518859863281,
-      "logps/rejected": -215.78053283691406,
-      "loss": 0.7158,
       "rewards/accuracies": 0.4791666567325592,
-      "rewards/chosen": -0.022490110248327255,
-      "rewards/margins": 0.013700157403945923,
-      "rewards/rejected": -0.03619026765227318,
       "step": 10
     },
     {
-      "epoch": 0.05,
-      "grad_norm": 2123.0791902752153,
-      "learning_rate": 2.5e-08,
-      "logits/chosen": -4.507326126098633,
-      "logits/rejected": -4.741320610046387,
-      "logps/chosen": -267.782958984375,
-      "logps/rejected": -216.6239013671875,
-      "loss": 0.7083,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.0784914419054985,
-      "rewards/margins": 0.08576976507902145,
-      "rewards/rejected": -0.007278320845216513,
       "step": 20
     },
     {
-      "epoch": 0.08,
-      "grad_norm": 1853.4496230505663,
-      "learning_rate": 3.75e-08,
-      "logits/chosen": -4.582863807678223,
-      "logits/rejected": -4.761076927185059,
-      "logps/chosen": -258.3573913574219,
-      "logps/rejected": -214.7029266357422,
-      "loss": 0.6439,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": 0.08770741522312164,
-      "rewards/margins": 0.17278313636779785,
-      "rewards/rejected": -0.08507572114467621,
       "step": 30
     },
     {
-      "epoch": 0.1,
-      "grad_norm": 1327.9844892722813,
-      "learning_rate": 5e-08,
-      "logits/chosen": -4.618801593780518,
-      "logits/rejected": -4.703221321105957,
-      "logps/chosen": -252.527587890625,
-      "logps/rejected": -220.14675903320312,
-      "loss": 0.4963,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.5357844233512878,
-      "rewards/margins": 0.7736073732376099,
-      "rewards/rejected": -0.2378229796886444,
       "step": 40
     },
     {
-      "epoch": 0.13,
-      "grad_norm": 1069.4622580863575,
-      "learning_rate": 4.989992961303737e-08,
-      "logits/chosen": -4.50632381439209,
-      "logits/rejected": -4.698899269104004,
-      "logps/chosen": -269.89434814453125,
-      "logps/rejected": -226.6046142578125,
-      "loss": 0.3766,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 1.0375183820724487,
-      "rewards/margins": 1.404066562652588,
-      "rewards/rejected": -0.36654800176620483,
       "step": 50
     },
     {
-      "epoch": 0.15,
-      "grad_norm": 918.9889625128512,
-      "learning_rate": 4.960051957873725e-08,
-      "logits/chosen": -4.588326930999756,
-      "logits/rejected": -4.701268196105957,
-      "logps/chosen": -237.8685302734375,
-      "logps/rejected": -215.9205780029297,
-      "loss": 0.2737,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 1.6269744634628296,
-      "rewards/margins": 2.0765488147735596,
-      "rewards/rejected": -0.44957447052001953,
       "step": 60
     },
     {
-      "epoch": 0.18,
-      "grad_norm": 946.4459060471729,
-      "learning_rate": 4.910416686333906e-08,
-      "logits/chosen": -4.468060493469238,
-      "logits/rejected": -4.6865234375,
-      "logps/chosen": -249.1713104248047,
-      "logps/rejected": -219.0127410888672,
-      "loss": 0.2613,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 1.9853293895721436,
-      "rewards/margins": 2.6104633808135986,
-      "rewards/rejected": -0.6251339912414551,
       "step": 70
     },
     {
-      "epoch": 0.2,
-      "grad_norm": 679.1661186320622,
-      "learning_rate": 4.841484508350678e-08,
-      "logits/chosen": -4.460832118988037,
-      "logits/rejected": -4.670706748962402,
-      "logps/chosen": -256.6949768066406,
-      "logps/rejected": -231.5639190673828,
-      "loss": 0.2268,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 2.2625014781951904,
-      "rewards/margins": 2.885091781616211,
-      "rewards/rejected": -0.6225907206535339,
       "step": 80
     },
     {
-      "epoch": 0.23,
-      "grad_norm": 1183.979446523898,
-      "learning_rate": 4.7538072695020406e-08,
-      "logits/chosen": -4.5435404777526855,
-      "logits/rejected": -4.737183570861816,
-      "logps/chosen": -246.42739868164062,
-      "logps/rejected": -204.73475646972656,
-      "loss": 0.218,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 2.3926777839660645,
-      "rewards/margins": 3.3450546264648438,
-      "rewards/rejected": -0.9523773193359375,
       "step": 90
     },
     {
-      "epoch": 0.26,
-      "grad_norm": 908.0652921589165,
-      "learning_rate": 4.6480868814055416e-08,
-      "logits/chosen": -4.413487911224365,
-      "logits/rejected": -4.6725263595581055,
-      "logps/chosen": -251.57327270507812,
-      "logps/rejected": -229.37167358398438,
-      "loss": 0.2235,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 2.572472095489502,
-      "rewards/margins": 3.7037949562072754,
-      "rewards/rejected": -1.131322979927063,
       "step": 100
     },
     {
-      "epoch": 0.26,
-      "eval_logits/chosen": -4.512205123901367,
-      "eval_logits/rejected": -4.725853443145752,
-      "eval_logps/chosen": -389.6747131347656,
-      "eval_logps/rejected": -516.2044677734375,
-      "eval_loss": 1.4095032215118408,
-      "eval_rewards/accuracies": 0.40234375,
-      "eval_rewards/chosen": 0.3998846113681793,
-      "eval_rewards/margins": -0.6917239427566528,
-      "eval_rewards/rejected": 1.0916085243225098,
-      "eval_runtime": 97.9662,
-      "eval_samples_per_second": 20.415,
-      "eval_steps_per_second": 0.327,
       "step": 100
     },
     {
-      "epoch": 0.28,
-      "grad_norm": 712.5121868885494,
-      "learning_rate": 4.525169702472916e-08,
-      "logits/chosen": -4.465897560119629,
-      "logits/rejected": -4.702467441558838,
-      "logps/chosen": -243.669677734375,
-      "logps/rejected": -218.7489776611328,
-      "loss": 0.2035,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 2.758031129837036,
-      "rewards/margins": 3.7895760536193848,
-      "rewards/rejected": -1.0315452814102173,
       "step": 110
     },
     {
-      "epoch": 0.31,
-      "grad_norm": 667.5412222468834,
-      "learning_rate": 4.386039762276975e-08,
-      "logits/chosen": -4.414495944976807,
-      "logits/rejected": -4.629802703857422,
-      "logps/chosen": -269.6809997558594,
-      "logps/rejected": -230.2985076904297,
-      "loss": 0.2011,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 3.1542792320251465,
-      "rewards/margins": 3.9992222785949707,
-      "rewards/rejected": -0.8449428677558899,
       "step": 120
     },
     {
-      "epoch": 0.33,
-      "grad_norm": 923.4850160841356,
-      "learning_rate": 4.231810883773999e-08,
-      "logits/chosen": -4.481413841247559,
-      "logits/rejected": -4.726315498352051,
-      "logps/chosen": -252.7733917236328,
-      "logps/rejected": -217.5948486328125,
-      "loss": 0.2074,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 2.835254669189453,
-      "rewards/margins": 3.9683444499969482,
-      "rewards/rejected": -1.1330897808074951,
       "step": 130
     },
     {
-      "epoch": 0.36,
-      "grad_norm": 616.0428688374469,
-      "learning_rate": 4.063717766448194e-08,
-      "logits/chosen": -4.507636070251465,
-      "logits/rejected": -4.760751247406006,
-      "logps/chosen": -278.6434631347656,
-      "logps/rejected": -235.1444854736328,
-      "loss": 0.1987,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 2.7758140563964844,
-      "rewards/margins": 4.261317253112793,
-      "rewards/rejected": -1.4855029582977295,
       "step": 140
     },
     {
-      "epoch": 0.38,
-      "grad_norm": 741.9538552087575,
-      "learning_rate": 3.8831061017632845e-08,
-      "logits/chosen": -4.500254154205322,
-      "logits/rejected": -4.711678504943848,
-      "logps/chosen": -246.7163543701172,
-      "logps/rejected": -221.5094451904297,
-      "loss": 0.2003,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 2.8509652614593506,
-      "rewards/margins": 4.0460524559021,
-      "rewards/rejected": -1.1950870752334595,
       "step": 150
     },
     {
-      "epoch": 0.41,
-      "grad_norm": 815.598001293288,
-      "learning_rate": 3.691421800053269e-08,
-      "logits/chosen": -4.561967849731445,
-      "logits/rejected": -4.702834606170654,
-      "logps/chosen": -245.543701171875,
-      "logps/rejected": -217.7865447998047,
-      "loss": 0.1766,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 2.9564106464385986,
-      "rewards/margins": 4.045149803161621,
-      "rewards/rejected": -1.088739275932312,
       "step": 160
     },
     {
-      "epoch": 0.43,
-      "grad_norm": 844.389651127701,
-      "learning_rate": 3.490199415097892e-08,
-      "logits/chosen": -4.448655128479004,
-      "logits/rejected": -4.690133094787598,
-      "logps/chosen": -253.94808959960938,
-      "logps/rejected": -222.57205200195312,
-      "loss": 0.1679,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 3.5630671977996826,
-      "rewards/margins": 4.9475860595703125,
-      "rewards/rejected": -1.3845189809799194,
       "step": 170
     },
     {
-      "epoch": 0.46,
-      "grad_norm": 691.9187241462795,
-      "learning_rate": 3.2810498590513937e-08,
-      "logits/chosen": -4.5445475578308105,
-      "logits/rejected": -4.739317893981934,
-      "logps/chosen": -230.8211212158203,
-      "logps/rejected": -198.8353271484375,
-      "loss": 0.1892,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 3.0212557315826416,
-      "rewards/margins": 4.4287896156311035,
-      "rewards/rejected": -1.4075340032577515,
       "step": 180
     },
     {
-      "epoch": 0.49,
-      "grad_norm": 827.5041864981521,
-      "learning_rate": 3.065647506074306e-08,
-      "logits/chosen": -4.514561176300049,
-      "logits/rejected": -4.631649017333984,
-      "logps/chosen": -254.4134521484375,
-      "logps/rejected": -237.3285675048828,
-      "loss": 0.1891,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 3.2396960258483887,
-      "rewards/margins": 4.320940971374512,
-      "rewards/rejected": -1.081244707107544,
       "step": 190
     },
     {
-      "epoch": 0.51,
-      "grad_norm": 642.9556336647308,
-      "learning_rate": 2.8457167879118325e-08,
-      "logits/chosen": -4.507633209228516,
-      "logits/rejected": -4.638915061950684,
-      "logps/chosen": -245.82296752929688,
-      "logps/rejected": -217.69921875,
-      "loss": 0.1957,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 3.158137798309326,
-      "rewards/margins": 4.334475517272949,
-      "rewards/rejected": -1.176337480545044,
       "step": 200
     },
     {
-      "epoch": 0.51,
-      "eval_logits/chosen": -4.50057315826416,
-      "eval_logits/rejected": -4.716313362121582,
-      "eval_logps/chosen": -389.73785400390625,
-      "eval_logps/rejected": -516.0735473632812,
-      "eval_loss": 1.561594009399414,
-      "eval_rewards/accuracies": 0.39453125,
-      "eval_rewards/chosen": 0.36831969022750854,
-      "eval_rewards/margins": -0.7887431383132935,
-      "eval_rewards/rejected": 1.1570627689361572,
-      "eval_runtime": 97.7279,
-      "eval_samples_per_second": 20.465,
-      "eval_steps_per_second": 0.327,
       "step": 200
     },
     {
-      "epoch": 0.54,
-      "grad_norm": 568.3174347805576,
-      "learning_rate": 2.6230183887296952e-08,
-      "logits/chosen": -4.570614337921143,
-      "logits/rejected": -4.765097141265869,
-      "logps/chosen": -258.71307373046875,
-      "logps/rejected": -232.0406951904297,
-      "loss": 0.1515,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 3.5860073566436768,
-      "rewards/margins": 5.283370018005371,
-      "rewards/rejected": -1.6973628997802734,
       "step": 210
     },
     {
-      "epoch": 0.56,
-      "grad_norm": 649.5951829889209,
-      "learning_rate": 2.3993351497264626e-08,
-      "logits/chosen": -4.4200639724731445,
-      "logits/rejected": -4.70514440536499,
-      "logps/chosen": -256.3812255859375,
-      "logps/rejected": -228.02224731445312,
-      "loss": 0.1838,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 3.2150566577911377,
-      "rewards/margins": 4.889595985412598,
-      "rewards/rejected": -1.674539566040039,
       "step": 220
     },
     {
-      "epoch": 0.59,
-      "grad_norm": 867.6806752902361,
-      "learning_rate": 2.1764577963648613e-08,
-      "logits/chosen": -4.4975972175598145,
-      "logits/rejected": -4.75902795791626,
-      "logps/chosen": -254.38192749023438,
-      "logps/rejected": -227.0271759033203,
-      "loss": 0.1975,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 3.2935631275177,
-      "rewards/margins": 4.951591968536377,
-      "rewards/rejected": -1.6580288410186768,
       "step": 230
     },
     {
-      "epoch": 0.61,
-      "grad_norm": 538.1398453260755,
-      "learning_rate": 1.9561706024845818e-08,
-      "logits/chosen": -4.4658050537109375,
-      "logits/rejected": -4.660877227783203,
-      "logps/chosen": -266.124267578125,
-      "logps/rejected": -224.49252319335938,
-      "loss": 0.1765,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 3.349458694458008,
-      "rewards/margins": 5.12891149520874,
-      "rewards/rejected": -1.7794525623321533,
       "step": 240
     },
     {
-      "epoch": 0.64,
-      "grad_norm": 679.3262102929132,
-      "learning_rate": 1.740237106064383e-08,
-      "logits/chosen": -4.591132164001465,
-      "logits/rejected": -4.800902366638184,
-      "logps/chosen": -242.6898193359375,
-      "logps/rejected": -210.1246795654297,
-      "loss": 0.1648,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 3.0058376789093018,
-      "rewards/margins": 4.263583183288574,
-      "rewards/rejected": -1.2577459812164307,
       "step": 250
     },
     {
-      "epoch": 0.66,
-      "grad_norm": 954.5022870918694,
-      "learning_rate": 1.530385990987863e-08,
-      "logits/chosen": -4.4684882164001465,
-      "logits/rejected": -4.688414573669434,
-      "logps/chosen": -247.53207397460938,
-      "logps/rejected": -224.006103515625,
-      "loss": 0.1732,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 3.2582428455352783,
-      "rewards/margins": 5.38706111907959,
-      "rewards/rejected": -2.1288187503814697,
       "step": 260
     },
     {
-      "epoch": 0.69,
-      "grad_norm": 1275.8554204119202,
-      "learning_rate": 1.3282972478382409e-08,
-      "logits/chosen": -4.555933952331543,
-      "logits/rejected": -4.729363918304443,
-      "logps/chosen": -254.4961395263672,
-      "logps/rejected": -218.8379669189453,
-      "loss": 0.1762,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 3.268549680709839,
-      "rewards/margins": 5.175016403198242,
-      "rewards/rejected": -1.9064662456512451,
       "step": 270
     },
     {
-      "epoch": 0.72,
-      "grad_norm": 714.3522909489556,
-      "learning_rate": 1.1355887245137383e-08,
-      "logits/chosen": -4.508360385894775,
-      "logits/rejected": -4.665677070617676,
-      "logps/chosen": -257.4315490722656,
-      "logps/rejected": -246.13681030273438,
-      "loss": 0.1551,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 3.5376486778259277,
-      "rewards/margins": 5.136354446411133,
-      "rewards/rejected": -1.598705530166626,
       "step": 280
     },
     {
-      "epoch": 0.74,
-      "grad_norm": 1801.7427609134184,
-      "learning_rate": 9.538031743343628e-09,
-      "logits/chosen": -4.377167701721191,
-      "logits/rejected": -4.655056476593018,
-      "logps/chosen": -265.31396484375,
-      "logps/rejected": -218.6363525390625,
-      "loss": 0.1633,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 3.6472067832946777,
-      "rewards/margins": 5.1506853103637695,
-      "rewards/rejected": -1.5034778118133545,
       "step": 290
     },
     {
-      "epoch": 0.77,
-      "grad_norm": 920.076671733992,
-      "learning_rate": 7.843959053281662e-09,
-      "logits/chosen": -4.4860124588012695,
-      "logits/rejected": -4.640582084655762,
-      "logps/chosen": -246.03854370117188,
-      "logps/rejected": -219.10879516601562,
-      "loss": 0.1905,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 3.684807538986206,
-      "rewards/margins": 5.256471633911133,
-      "rewards/rejected": -1.571663498878479,
       "step": 300
     },
     {
-      "epoch": 0.77,
-      "eval_logits/chosen": -4.498342514038086,
-      "eval_logits/rejected": -4.714066028594971,
-      "eval_logps/chosen": -389.8600769042969,
-      "eval_logps/rejected": -516.214111328125,
-      "eval_loss": 1.5790551900863647,
-      "eval_rewards/accuracies": 0.3984375,
-      "eval_rewards/chosen": 0.3071921765804291,
-      "eval_rewards/margins": -0.7796162366867065,
-      "eval_rewards/rejected": 1.0868083238601685,
-      "eval_runtime": 98.0146,
-      "eval_samples_per_second": 20.405,
-      "eval_steps_per_second": 0.326,
       "step": 300
     },
     {
-      "epoch": 0.79,
-      "grad_norm": 698.2991090149884,
-      "learning_rate": 6.28723129572247e-09,
-      "logits/chosen": -4.4101176261901855,
-      "logits/rejected": -4.666938781738281,
-      "logps/chosen": -253.66043090820312,
-      "logps/rejected": -226.02029418945312,
-      "loss": 0.1773,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 3.739584445953369,
-      "rewards/margins": 5.650748252868652,
-      "rewards/rejected": -1.9111639261245728,
       "step": 310
     },
     {
-      "epoch": 0.82,
-      "grad_norm": 989.4087037650046,
-      "learning_rate": 4.880311058593617e-09,
-      "logits/chosen": -4.468711853027344,
-      "logits/rejected": -4.748659610748291,
-      "logps/chosen": -238.40478515625,
-      "logps/rejected": -209.8579864501953,
-      "loss": 0.1722,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 3.379748582839966,
-      "rewards/margins": 5.082991600036621,
-      "rewards/rejected": -1.7032434940338135,
       "step": 320
     },
     {
-      "epoch": 0.84,
-      "grad_norm": 763.7163191438951,
-      "learning_rate": 3.6344616260994942e-09,
-      "logits/chosen": -4.420454978942871,
-      "logits/rejected": -4.672707557678223,
-      "logps/chosen": -260.8804626464844,
-      "logps/rejected": -236.47921752929688,
-      "loss": 0.1613,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 3.634185791015625,
-      "rewards/margins": 5.5239667892456055,
-      "rewards/rejected": -1.8897815942764282,
       "step": 330
     },
     {
-      "epoch": 0.87,
-      "grad_norm": 694.9124867513493,
-      "learning_rate": 2.5596568090246547e-09,
-      "logits/chosen": -4.442999839782715,
-      "logits/rejected": -4.721438407897949,
-      "logps/chosen": -266.19195556640625,
-      "logps/rejected": -210.20596313476562,
-      "loss": 0.1687,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 3.9770240783691406,
-      "rewards/margins": 5.688981056213379,
-      "rewards/rejected": -1.7119563817977905,
       "step": 340
     },
     {
-      "epoch": 0.9,
-      "grad_norm": 787.4371760833645,
-      "learning_rate": 1.6645010980854079e-09,
-      "logits/chosen": -4.456801414489746,
-      "logits/rejected": -4.582957744598389,
-      "logps/chosen": -249.213623046875,
-      "logps/rejected": -230.64990234375,
-      "loss": 0.1886,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 3.1287765502929688,
-      "rewards/margins": 4.568727493286133,
-      "rewards/rejected": -1.4399511814117432,
       "step": 350
     },
     {
-      "epoch": 0.92,
-      "grad_norm": 796.3842389013369,
-      "learning_rate": 9.561607795526006e-10,
-      "logits/chosen": -4.446815490722656,
-      "logits/rejected": -4.6302266120910645,
-      "logps/chosen": -255.41098022460938,
-      "logps/rejected": -230.77975463867188,
-      "loss": 0.1792,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 3.111967086791992,
-      "rewards/margins": 4.770585060119629,
-      "rewards/rejected": -1.6586182117462158,
       "step": 360
     },
     {
-      "epoch": 0.95,
-      "grad_norm": 746.9931263686806,
-      "learning_rate": 4.403065646083809e-10,
-      "logits/chosen": -4.472794532775879,
-      "logits/rejected": -4.599184513092041,
-      "logps/chosen": -249.98641967773438,
-      "logps/rejected": -239.8127899169922,
-      "loss": 0.1796,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 3.417390823364258,
-      "rewards/margins": 4.900269031524658,
-      "rewards/rejected": -1.4828782081604004,
       "step": 370
     },
     {
-      "epoch": 0.97,
-      "grad_norm": 693.0631323897318,
-      "learning_rate": 1.2106819172520434e-10,
-      "logits/chosen": -4.5681304931640625,
-      "logits/rejected": -4.779508590698242,
-      "logps/chosen": -255.0944366455078,
-      "logps/rejected": -223.9898223876953,
-      "loss": 0.1785,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 3.494133710861206,
-      "rewards/margins": 5.226744174957275,
-      "rewards/rejected": -1.7326103448867798,
       "step": 380
     },
     {
-      "epoch": 1.0,
-      "grad_norm": 1239.893679079081,
-      "learning_rate": 1.0013655036916758e-12,
-      "logits/chosen": -4.557173252105713,
-      "logits/rejected": -4.7594380378723145,
-      "logps/chosen": -252.3553466796875,
-      "logps/rejected": -223.79635620117188,
-      "loss": 0.1719,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 3.6586031913757324,
-      "rewards/margins": 5.299229145050049,
-      "rewards/rejected": -1.6406255960464478,
       "step": 390
     },
     {
       "epoch": 1.0,
       "step": 391,
       "total_flos": 0.0,
-      "train_loss": 0.240954409779795,
-      "train_runtime": 6147.1036,
-      "train_samples_per_second": 8.134,
-      "train_steps_per_second": 0.064
     }
   ],
   "logging_steps": 10,

   "is_world_process_zero": true,
   "log_history": [
     {
+      "epoch": 0.0025575447570332483,
+      "grad_norm": 39.26094609355056,
+      "learning_rate": 2.5e-09,
+      "logits/chosen": -4.633832931518555,
+      "logits/rejected": -4.853580951690674,
+      "logps/chosen": -233.06472778320312,
+      "logps/rejected": -207.33529663085938,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
       "step": 1
     },
     {
+      "epoch": 0.02557544757033248,
+      "grad_norm": 36.68935602174721,
+      "learning_rate": 2.5e-08,
+      "logits/chosen": -4.338743686676025,
+      "logits/rejected": -4.649880886077881,
+      "logps/chosen": -262.4483642578125,
+      "logps/rejected": -221.99293518066406,
+      "loss": 0.693,
       "rewards/accuracies": 0.4791666567325592,
+      "rewards/chosen": 0.00011811345757450908,
+      "rewards/margins": 0.0013624767307192087,
+      "rewards/rejected": -0.0012443631421774626,
       "step": 10
     },
     {
+      "epoch": 0.05115089514066496,
+      "grad_norm": 39.59560101486009,
+      "learning_rate": 5e-08,
+      "logits/chosen": -4.518132209777832,
+      "logits/rejected": -4.7573723793029785,
+      "logps/chosen": -265.6820373535156,
+      "logps/rejected": -224.56332397460938,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": 0.0007334979018196464,
+      "rewards/margins": 0.0019747079350054264,
+      "rewards/rejected": -0.00124121003318578,
       "step": 20
     },
     {
+      "epoch": 0.07672634271099744,
+      "grad_norm": 42.07292039672945,
+      "learning_rate": 7.5e-08,
+      "logits/chosen": -4.586892127990723,
+      "logits/rejected": -4.772247314453125,
+      "logps/chosen": -254.6095428466797,
+      "logps/rejected": -221.1687469482422,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.004800780676305294,
+      "rewards/margins": 0.01116220187395811,
+      "rewards/rejected": -0.006361422594636679,
       "step": 30
     },
     {
+      "epoch": 0.10230179028132992,
+      "grad_norm": 39.99985818605015,
+      "learning_rate": 1e-07,
+      "logits/chosen": -4.643338680267334,
+      "logits/rejected": -4.734442234039307,
+      "logps/chosen": -250.8112030029297,
+      "logps/rejected": -227.5035858154297,
+      "loss": 0.674,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": 0.015374640934169292,
+      "rewards/margins": 0.04068901389837265,
+      "rewards/rejected": -0.025314366444945335,
       "step": 40
     },
     {
+      "epoch": 0.1278772378516624,
+      "grad_norm": 39.42008970309411,
+      "learning_rate": 9.979985922607475e-08,
+      "logits/chosen": -4.586745262145996,
+      "logits/rejected": -4.8019514083862305,
+      "logps/chosen": -266.86578369140625,
+      "logps/rejected": -236.32333374023438,
+      "loss": 0.6472,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 0.025944575667381287,
+      "rewards/margins": 0.09520022571086884,
+      "rewards/rejected": -0.06925566494464874,
       "step": 50
     },
     {
+      "epoch": 0.1534526854219949,
+      "grad_norm": 38.00390022566866,
+      "learning_rate": 9.92010391574745e-08,
+      "logits/chosen": -4.757911205291748,
+      "logits/rejected": -4.887364387512207,
+      "logps/chosen": -239.2329559326172,
+      "logps/rejected": -240.6675262451172,
+      "loss": 0.5963,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.0015811750199645758,
+      "rewards/margins": 0.20014917850494385,
+      "rewards/rejected": -0.20173034071922302,
       "step": 60
     },
     {
+      "epoch": 0.17902813299232737,
+      "grad_norm": 40.739477874841256,
+      "learning_rate": 9.820833372667812e-08,
+      "logits/chosen": -4.700837135314941,
+      "logits/rejected": -4.890389442443848,
+      "logps/chosen": -262.3451843261719,
+      "logps/rejected": -265.8101501464844,
+      "loss": 0.552,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.08908917009830475,
+      "rewards/margins": 0.345054566860199,
+      "rewards/rejected": -0.4341437816619873,
       "step": 70
     },
     {
+      "epoch": 0.20460358056265984,
+      "grad_norm": 39.05798828996261,
+      "learning_rate": 9.682969016701356e-08,
+      "logits/chosen": -4.702755451202393,
+      "logits/rejected": -4.89903450012207,
+      "logps/chosen": -271.28289794921875,
+      "logps/rejected": -292.86474609375,
+      "loss": 0.4978,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.11036349833011627,
+      "rewards/margins": 0.46904292702674866,
+      "rewards/rejected": -0.5794063806533813,
       "step": 80
     },
     {
+      "epoch": 0.23017902813299232,
+      "grad_norm": 35.899018808141314,
+      "learning_rate": 9.507614539004081e-08,
+      "logits/chosen": -4.802427768707275,
+      "logits/rejected": -5.013784408569336,
+      "logps/chosen": -254.51138305664062,
+      "logps/rejected": -277.52117919921875,
+      "loss": 0.4653,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.007465996779501438,
+      "rewards/margins": 0.6835082769393921,
+      "rewards/rejected": -0.6909742951393127,
       "step": 90
     },
     {
+      "epoch": 0.2557544757033248,
+      "grad_norm": 36.72256884048717,
+      "learning_rate": 9.296173762811083e-08,
+      "logits/chosen": -4.685734748840332,
+      "logits/rejected": -4.969820976257324,
+      "logps/chosen": -261.9112243652344,
+      "logps/rejected": -306.5802001953125,
+      "loss": 0.4622,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.03183584660291672,
+      "rewards/margins": 0.7450669407844543,
+      "rewards/rejected": -0.7769027948379517,
       "step": 100
     },
     {
+      "epoch": 0.2557544757033248,
+      "eval_logits/chosen": -4.788325309753418,
+      "eval_logits/rejected": -5.006863594055176,
+      "eval_logps/chosen": -439.2997131347656,
+      "eval_logps/rejected": -570.1284790039062,
+      "eval_loss": 0.7195349335670471,
+      "eval_rewards/accuracies": 0.48828125,
+      "eval_rewards/chosen": -0.4882526695728302,
+      "eval_rewards/margins": 0.029155578464269638,
+      "eval_rewards/rejected": -0.5174082517623901,
+      "eval_runtime": 98.6046,
+      "eval_samples_per_second": 20.283,
+      "eval_steps_per_second": 0.325,
       "step": 100
     },
     {
+      "epoch": 0.2813299232736573,
+      "grad_norm": 37.92952007987077,
+      "learning_rate": 9.050339404945832e-08,
+      "logits/chosen": -4.800631999969482,
+      "logits/rejected": -5.014934539794922,
+      "logps/chosen": -261.18560791015625,
+      "logps/rejected": -321.3470458984375,
+      "loss": 0.4274,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.1468946933746338,
+      "rewards/margins": 0.8339446783065796,
+      "rewards/rejected": -0.9808392524719238,
       "step": 110
     },
     {
+      "epoch": 0.3069053708439898,
+      "grad_norm": 34.34594445177828,
+      "learning_rate": 8.77207952455395e-08,
+      "logits/chosen": -4.794532775878906,
+      "logits/rejected": -5.045430660247803,
+      "logps/chosen": -288.1357727050781,
+      "logps/rejected": -340.7710876464844,
+      "loss": 0.4231,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -0.13442620635032654,
+      "rewards/margins": 0.9264079928398132,
+      "rewards/rejected": -1.0608341693878174,
       "step": 120
     },
     {
+      "epoch": 0.33248081841432225,
+      "grad_norm": 38.8387240251847,
+      "learning_rate": 8.463621767547997e-08,
+      "logits/chosen": -4.894143104553223,
+      "logits/rejected": -5.190768718719482,
+      "logps/chosen": -281.547607421875,
+      "logps/rejected": -352.4320983886719,
+      "loss": 0.4055,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -0.25050926208496094,
+      "rewards/margins": 1.063452959060669,
+      "rewards/rejected": -1.3139623403549194,
       "step": 130
     },
     {
+      "epoch": 0.35805626598465473,
+      "grad_norm": 36.960561396839225,
+      "learning_rate": 8.127435532896387e-08,
+      "logits/chosen": -4.948976993560791,
+      "logits/rejected": -5.229872703552246,
+      "logps/chosen": -320.14996337890625,
+      "logps/rejected": -386.9942932128906,
+      "loss": 0.392,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.3797687590122223,
+      "rewards/margins": 1.1361901760101318,
+      "rewards/rejected": -1.5159590244293213,
       "step": 140
     },
     {
+      "epoch": 0.3836317135549872,
+      "grad_norm": 43.84353984691604,
+      "learning_rate": 7.766212203526569e-08,
+      "logits/chosen": -5.017346382141113,
+      "logits/rejected": -5.299422740936279,
+      "logps/chosen": -287.28350830078125,
+      "logps/rejected": -373.42730712890625,
+      "loss": 0.3891,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.374920517206192,
+      "rewards/margins": 1.0954115390777588,
+      "rewards/rejected": -1.4703319072723389,
       "step": 150
     },
     {
+      "epoch": 0.4092071611253197,
+      "grad_norm": 45.6486792527098,
+      "learning_rate": 7.382843600106538e-08,
+      "logits/chosen": -5.09747314453125,
+      "logits/rejected": -5.334885120391846,
+      "logps/chosen": -294.9914855957031,
+      "logps/rejected": -389.2686462402344,
+      "loss": 0.3586,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -0.47372356057167053,
+      "rewards/margins": 1.2067290544509888,
+      "rewards/rejected": -1.680452585220337,
       "step": 160
     },
     {
+      "epoch": 0.43478260869565216,
+      "grad_norm": 41.55934089834119,
+      "learning_rate": 6.980398830195784e-08,
+      "logits/chosen": -5.022572040557861,
+      "logits/rejected": -5.321537494659424,
+      "logps/chosen": -307.09521484375,
+      "logps/rejected": -427.2223205566406,
+      "loss": 0.3397,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.5031875967979431,
+      "rewards/margins": 1.4829540252685547,
+      "rewards/rejected": -1.9861418008804321,
       "step": 170
     },
     {
+      "epoch": 0.46035805626598464,
+      "grad_norm": 37.76415174175888,
+      "learning_rate": 6.562099718102787e-08,
+      "logits/chosen": -5.190648078918457,
+      "logits/rejected": -5.442112922668457,
+      "logps/chosen": -293.28350830078125,
+      "logps/rejected": -403.9653625488281,
+      "loss": 0.3551,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -0.5512182116508484,
+      "rewards/margins": 1.491098165512085,
+      "rewards/rejected": -2.042316198348999,
       "step": 180
     },
     {
+      "epoch": 0.4859335038363171,
+      "grad_norm": 52.26242886674056,
+      "learning_rate": 6.131295012148612e-08,
+      "logits/chosen": -5.1211113929748535,
+      "logits/rejected": -5.269497871398926,
+      "logps/chosen": -328.1192321777344,
+      "logps/rejected": -461.3627014160156,
+      "loss": 0.3605,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.7219178080558777,
+      "rewards/margins": 1.4698314666748047,
+      "rewards/rejected": -2.1917495727539062,
       "step": 190
     },
     {
+      "epoch": 0.5115089514066496,
+      "grad_norm": 41.210687347051376,
+      "learning_rate": 5.691433575823665e-08,
+      "logits/chosen": -5.160891056060791,
+      "logits/rejected": -5.3507490158081055,
+      "logps/chosen": -323.77532958984375,
+      "logps/rejected": -448.09271240234375,
+      "loss": 0.3618,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.7529298067092896,
+      "rewards/margins": 1.5445276498794556,
+      "rewards/rejected": -2.297457456588745,
       "step": 200
     },
     {
+      "epoch": 0.5115089514066496,
+      "eval_logits/chosen": -5.146201133728027,
+      "eval_logits/rejected": -5.404381275177002,
+      "eval_logps/chosen": -551.28759765625,
+      "eval_logps/rejected": -696.0471801757812,
+      "eval_loss": 0.7717023491859436,
+      "eval_rewards/accuracies": 0.5703125,
+      "eval_rewards/chosen": -1.6081316471099854,
+      "eval_rewards/margins": 0.16846325993537903,
+      "eval_rewards/rejected": -1.7765947580337524,
+      "eval_runtime": 98.5146,
+      "eval_samples_per_second": 20.302,
+      "eval_steps_per_second": 0.325,
       "step": 200
     },
     {
+      "epoch": 0.5370843989769821,
+      "grad_norm": 33.990804012478485,
+      "learning_rate": 5.2460367774593905e-08,
+      "logits/chosen": -5.241884708404541,
+      "logits/rejected": -5.514366626739502,
+      "logps/chosen": -325.63385009765625,
+      "logps/rejected": -494.1783142089844,
+      "loss": 0.3361,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -0.631729245185852,
+      "rewards/margins": 1.9169912338256836,
+      "rewards/rejected": -2.548720359802246,
       "step": 210
     },
     {
+      "epoch": 0.5626598465473146,
+      "grad_norm": 43.799668582141386,
+      "learning_rate": 4.798670299452925e-08,
+      "logits/chosen": -5.0720014572143555,
+      "logits/rejected": -5.483579635620117,
+      "logps/chosen": -326.8290100097656,
+      "logps/rejected": -476.4143981933594,
+      "loss": 0.3361,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.6208838224411011,
+      "rewards/margins": 1.8470569849014282,
+      "rewards/rejected": -2.4679408073425293,
       "step": 220
     },
     {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 49.054323066624825,
+      "learning_rate": 4.3529155927297226e-08,
+      "logits/chosen": -5.175291538238525,
+      "logits/rejected": -5.545914649963379,
+      "logps/chosen": -346.18603515625,
+      "logps/rejected": -467.39093017578125,
+      "loss": 0.3482,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.8635517954826355,
+      "rewards/margins": 1.513214111328125,
+      "rewards/rejected": -2.376765727996826,
       "step": 230
     },
     {
+      "epoch": 0.6138107416879796,
+      "grad_norm": 39.76260402422977,
+      "learning_rate": 3.9123412049691636e-08,
+      "logits/chosen": -5.179227828979492,
+      "logits/rejected": -5.498720645904541,
+      "logps/chosen": -356.7921142578125,
+      "logps/rejected": -494.8655700683594,
+      "loss": 0.3289,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -0.8680986166000366,
+      "rewards/margins": 1.7818403244018555,
+      "rewards/rejected": -2.6499390602111816,
       "step": 240
     },
     {
+      "epoch": 0.639386189258312,
+      "grad_norm": 39.31547537418584,
+      "learning_rate": 3.480474212128766e-08,
+      "logits/chosen": -5.349845886230469,
+      "logits/rejected": -5.674462795257568,
+      "logps/chosen": -343.0057067871094,
+      "logps/rejected": -444.6160583496094,
+      "loss": 0.3376,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.9290262460708618,
+      "rewards/margins": 1.3633143901824951,
+      "rewards/rejected": -2.2923407554626465,
       "step": 250
     },
     {
+      "epoch": 0.6649616368286445,
+      "grad_norm": 47.933967621919784,
+      "learning_rate": 3.060771981975726e-08,
+      "logits/chosen": -5.2227373123168945,
+      "logits/rejected": -5.566103935241699,
+      "logps/chosen": -334.39306640625,
+      "logps/rejected": -502.48211669921875,
+      "loss": 0.3271,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -0.8672464489936829,
+      "rewards/margins": 1.90872323513031,
+      "rewards/rejected": -2.7759692668914795,
       "step": 260
     },
     {
+      "epoch": 0.690537084398977,
+      "grad_norm": 37.918454908956036,
+      "learning_rate": 2.6565944956764818e-08,
+      "logits/chosen": -5.3946356773376465,
+      "logits/rejected": -5.655900478363037,
+      "logps/chosen": -349.11785888671875,
+      "logps/rejected": -490.94720458984375,
+      "loss": 0.3334,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.8713266253471375,
+      "rewards/margins": 1.8250868320465088,
+      "rewards/rejected": -2.696413516998291,
       "step": 270
     },
     {
+      "epoch": 0.7161125319693095,
+      "grad_norm": 43.57355633058329,
+      "learning_rate": 2.2711774490274766e-08,
+      "logits/chosen": -5.313577651977539,
+      "logits/rejected": -5.556331157684326,
+      "logps/chosen": -347.73529052734375,
+      "logps/rejected": -519.9879150390625,
+      "loss": 0.3132,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -0.8589982986450195,
+      "rewards/margins": 1.8729617595672607,
+      "rewards/rejected": -2.731959819793701,
       "step": 280
     },
     {
+      "epoch": 0.7416879795396419,
+      "grad_norm": 48.474190706284276,
+      "learning_rate": 1.9076063486687256e-08,
+      "logits/chosen": -5.153687477111816,
+      "logits/rejected": -5.569068908691406,
+      "logps/chosen": -342.547607421875,
+      "logps/rejected": -473.8719787597656,
+      "loss": 0.3082,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -0.7416984438896179,
+      "rewards/margins": 1.8063793182373047,
+      "rewards/rejected": -2.5480778217315674,
       "step": 290
     },
     {
+      "epoch": 0.7672634271099744,
+      "grad_norm": 46.47610707831878,
+      "learning_rate": 1.5687918106563324e-08,
+      "logits/chosen": -5.316671371459961,
+      "logits/rejected": -5.562603950500488,
+      "logps/chosen": -339.9733581542969,
+      "logps/rejected": -511.3289489746094,
+      "loss": 0.3249,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.8634950518608093,
+      "rewards/margins": 2.052011251449585,
+      "rewards/rejected": -2.915506601333618,
       "step": 300
     },
     {
+      "epoch": 0.7672634271099744,
+      "eval_logits/chosen": -5.345163345336914,
+      "eval_logits/rejected": -5.603174209594727,
+      "eval_logps/chosen": -585.470458984375,
+      "eval_logps/rejected": -737.188720703125,
+      "eval_loss": 0.7740334272384644,
+      "eval_rewards/accuracies": 0.56640625,
+      "eval_rewards/chosen": -1.9499597549438477,
+      "eval_rewards/margins": 0.23805105686187744,
+      "eval_rewards/rejected": -2.1880106925964355,
+      "eval_runtime": 98.564,
+      "eval_samples_per_second": 20.291,
+      "eval_steps_per_second": 0.325,
       "step": 300
     },
     {
+      "epoch": 0.7928388746803069,
+      "grad_norm": 43.72352052570602,
+      "learning_rate": 1.257446259144494e-08,
+      "logits/chosen": -5.234416961669922,
+      "logits/rejected": -5.61405611038208,
+      "logps/chosen": -347.01263427734375,
+      "logps/rejected": -529.2141723632812,
+      "loss": 0.3198,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.8649846911430359,
+      "rewards/margins": 2.1693644523620605,
+      "rewards/rejected": -3.034349203109741,
       "step": 310
     },
     {
+      "epoch": 0.8184143222506394,
+      "grad_norm": 49.6074266455155,
+      "learning_rate": 9.760622117187234e-09,
+      "logits/chosen": -5.37489128112793,
+      "logits/rejected": -5.706605434417725,
+      "logps/chosen": -341.7244567871094,
+      "logps/rejected": -499.4188537597656,
+      "loss": 0.3337,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.9388681650161743,
+      "rewards/margins": 1.9423179626464844,
+      "rewards/rejected": -2.881186008453369,
       "step": 320
     },
     {
+      "epoch": 0.8439897698209718,
+      "grad_norm": 47.67492578137724,
+      "learning_rate": 7.2689232521989885e-09,
+      "logits/chosen": -5.29935884475708,
+      "logits/rejected": -5.635709762573242,
+      "logps/chosen": -363.49591064453125,
+      "logps/rejected": -517.4185180664062,
+      "loss": 0.314,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.9602580070495605,
+      "rewards/margins": 1.8374437093734741,
+      "rewards/rejected": -2.797701597213745,
       "step": 330
     },
     {
+      "epoch": 0.8695652173913043,
+      "grad_norm": 39.135188403156114,
+      "learning_rate": 5.119313618049309e-09,
+      "logits/chosen": -5.324515342712402,
+      "logits/rejected": -5.7237324714660645,
+      "logps/chosen": -366.15264892578125,
+      "logps/rejected": -471.0101623535156,
+      "loss": 0.3135,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.9129904508590698,
+      "rewards/margins": 1.6364881992340088,
+      "rewards/rejected": -2.549478530883789,
       "step": 340
     },
     {
+      "epoch": 0.8951406649616368,
+      "grad_norm": 43.15700039763216,
+      "learning_rate": 3.3290021961708158e-09,
+      "logits/chosen": -5.358675956726074,
+      "logits/rejected": -5.501515865325928,
+      "logps/chosen": -351.61016845703125,
+      "logps/rejected": -492.70294189453125,
+      "loss": 0.3421,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -1.0095672607421875,
+      "rewards/margins": 1.6046628952026367,
+      "rewards/rejected": -2.614229917526245,
       "step": 350
     },
     {
+      "epoch": 0.9207161125319693,
+      "grad_norm": 43.21141833066524,
+      "learning_rate": 1.9123215591052013e-09,
+      "logits/chosen": -5.308594703674316,
+      "logits/rejected": -5.525307655334473,
+      "logps/chosen": -355.9210510253906,
+      "logps/rejected": -509.302001953125,
+      "loss": 0.3229,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.9471023678779602,
+      "rewards/margins": 1.8037946224212646,
+      "rewards/rejected": -2.75089693069458,
       "step": 360
     },
     {
+      "epoch": 0.9462915601023018,
+      "grad_norm": 45.460982961010096,
+      "learning_rate": 8.806131292167618e-10,
+      "logits/chosen": -5.335926055908203,
+      "logits/rejected": -5.556703567504883,
+      "logps/chosen": -349.836181640625,
+      "logps/rejected": -495.7314453125,
+      "loss": 0.3405,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.957531750202179,
+      "rewards/margins": 1.595903754234314,
+      "rewards/rejected": -2.5534353256225586,
       "step": 370
     },
     {
+      "epoch": 0.9718670076726342,
+      "grad_norm": 44.49515613968937,
+      "learning_rate": 2.4213638345040867e-10,
+      "logits/chosen": -5.496818542480469,
+      "logits/rejected": -5.771291255950928,
+      "logps/chosen": -356.3369445800781,
+      "logps/rejected": -501.03448486328125,
+      "loss": 0.3269,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.9707748293876648,
+      "rewards/margins": 1.7586256265640259,
+      "rewards/rejected": -2.729400634765625,
       "step": 380
     },
     {
+      "epoch": 0.9974424552429667,
+      "grad_norm": 54.69003383274636,
+      "learning_rate": 2.0027310073833516e-12,
+      "logits/chosen": -5.4842376708984375,
+      "logits/rejected": -5.72286319732666,
+      "logps/chosen": -359.2212829589844,
+      "logps/rejected": -514.3115234375,
+      "loss": 0.3144,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.9742454290390015,
+      "rewards/margins": 1.8987795114517212,
+      "rewards/rejected": -2.8730247020721436,
       "step": 390
     },
     {
       "epoch": 1.0,
       "step": 391,
       "total_flos": 0.0,
+      "train_loss": 0.4100414538932273,
+      "train_runtime": 6205.0725,
+      "train_samples_per_second": 8.058,
+      "train_steps_per_second": 0.063
     }
   ],
   "logging_steps": 10,

training_args.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:62fd36d8ce66e8becc572379f18c762113afc3bd8c05085b884b7f3d305aff55
-size 6264

 version https://git-lfs.github.com/spec/v1
+oid sha256:08b44cbfb47e0b0ab717461874f5af1f5bae2515baf1a1af924d72d425712001
+size 6328