Model save

Browse files

Files changed (12) hide show

README.md +19 -19
all_results.json +4 -5
config.json +1 -1
generation_config.json +1 -1
model-00001-of-00003.safetensors +1 -1
model-00002-of-00003.safetensors +1 -1
model-00003-of-00003.safetensors +1 -1
runs/May28_00-20-53_n136-100-194/events.out.tfevents.1716826873.n136-100-194.565990.0 +2 -2
tokenizer.json +0 -1
train_results.json +4 -5
trainer_state.json +515 -515
training_args.bin +2 -2

README.md CHANGED Viewed

@@ -13,17 +13,17 @@ should probably proofread and complete it, then remove this comment. -->
 # zephyr-7b-dpo-full
-This model was trained from scratch on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.7111
-- Rewards/chosen: -0.8296
-- Rewards/rejected: -0.9542
-- Rewards/accuracies: 0.5625
-- Rewards/margins: 0.1246
-- Logps/rejected: -613.8080
-- Logps/chosen: -473.4393
-- Logits/rejected: -5.2824
-- Logits/chosen: -5.0285
 ## Model description
@@ -42,7 +42,7 @@ More information needed
 ### Training hyperparameters
 The following hyperparameters were used during training:
-- learning_rate: 5e-08
 - train_batch_size: 8
 - eval_batch_size: 8
 - seed: 42
@@ -58,16 +58,16 @@ The following hyperparameters were used during training:
 ### Training results
-| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
-|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.5044        | 0.2558 | 100  | 0.7105          | -0.1129        | -0.0877          | 0.4727             | -0.0252         | -527.1570      | -401.7669    | -4.8307         | -4.6554       |
-| 0.3343        | 0.5115 | 200  | 0.6982          | -0.5200        | -0.6117          | 0.5586             | 0.0918          | -579.5609      | -442.4707    | -5.1101         | -4.8657       |
-| 0.2972        | 0.7673 | 300  | 0.7111          | -0.8296        | -0.9542          | 0.5625             | 0.1246          | -613.8080      | -473.4393    | -5.2824         | -5.0285       |
 ### Framework versions
-- Transformers 4.40.2
 - Pytorch 2.1.2+cu118
-- Datasets 2.19.1
-- Tokenizers 0.19.1

 # zephyr-7b-dpo-full
+This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
+- Loss: 1.3650
+- Rewards/chosen: -0.6695
+- Rewards/rejected: -0.0468
+- Rewards/accuracies: 0.3828
+- Rewards/margins: -0.6227
+- Logps/rejected: -518.8555
+- Logps/chosen: -397.1692
+- Logits/rejected: -4.7484
+- Logits/chosen: -4.5219
 ## Model description
 ### Training hyperparameters
 The following hyperparameters were used during training:
+- learning_rate: 1e-07
 - train_batch_size: 8
 - eval_batch_size: 8
 - seed: 42
 ### Training results
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.216         | 0.26  | 100  | 1.1906          | -0.1088        | 0.4672           | 0.3984             | -0.5759         | -513.7159      | -391.5621    | -4.7718         | -4.5558       |
+| 0.1925        | 0.51  | 200  | 1.3563          | -0.3996        | 0.2819           | 0.375              | -0.6815         | -515.5683      | -394.4702    | -4.7427         | -4.5166       |
+| 0.179         | 0.77  | 300  | 1.3650          | -0.6695        | -0.0468          | 0.3828             | -0.6227         | -518.8555      | -397.1692    | -4.7484         | -4.5219       |
 ### Framework versions
+- Transformers 4.38.2
 - Pytorch 2.1.2+cu118
+- Datasets 2.16.1
+- Tokenizers 0.15.2

all_results.json CHANGED Viewed

@@ -1,9 +1,8 @@
 {
     "epoch": 1.0,
-    "total_flos": 0.0,
-    "train_loss": 0.4007269041922391,
-    "train_runtime": 6210.4356,
     "train_samples": 50000,
-    "train_samples_per_second": 8.051,
-    "train_steps_per_second": 0.063
 }

 {
     "epoch": 1.0,
+    "train_loss": 0.23296360492401416,
+    "train_runtime": 6150.4195,
     "train_samples": 50000,
+    "train_samples_per_second": 8.13,
+    "train_steps_per_second": 0.064
 }

config.json CHANGED Viewed

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.40.2",
   "use_cache": false,
   "vocab_size": 32002
 }

   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
+  "transformers_version": "4.38.2",
   "use_cache": false,
   "vocab_size": 32002
 }

generation_config.json CHANGED Viewed

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 32000,
-  "transformers_version": "4.40.2"
 }

   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 32000,
+  "transformers_version": "4.38.2"
 }

model-00001-of-00003.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a9f251e96294c9492756c37d895155f82d849066c2471fd0aca8729a7aede122
 size 4943178720

 version https://git-lfs.github.com/spec/v1
+oid sha256:e8b81887b85183d4a4fce153e9d2d2fb4c1c12ff26edfc47a967614085e4ba54
 size 4943178720

model-00002-of-00003.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e9fd416637a101fc45a516e2d108838af2259d6ce2b49a13aba516cf189b7f0
 size 4999819336

 version https://git-lfs.github.com/spec/v1
+oid sha256:12f930756c5e712e7f34db59925b5831febcdc775f5258f7984274f15dffbba9
 size 4999819336

model-00003-of-00003.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:551b3715b02a97fd2af4d39745e82e183fc10809e578801af9fb5fdd39551e6f
 size 4540532728

 version https://git-lfs.github.com/spec/v1
+oid sha256:ab661929946b1e6f5490a343690d65f428f51fd0146f96739d6af9238c402383
 size 4540532728

runs/May28_00-20-53_n136-100-194/events.out.tfevents.1716826873.n136-100-194.565990.0 CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2149d80eff0b4e9c3632536caa9097575c852acbe5bdb07b4f2eb54ac92a49ba
-size 28262

 version https://git-lfs.github.com/spec/v1
+oid sha256:830a8dc2fb23b4bd12d7b631d68a03e221c221a0629bb5c2bbc516375052ce5e
+size 34808

tokenizer.json CHANGED Viewed

@@ -152,7 +152,6 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
-    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

train_results.json CHANGED Viewed

@@ -1,9 +1,8 @@
 {
     "epoch": 1.0,
-    "total_flos": 0.0,
-    "train_loss": 0.4007269041922391,
-    "train_runtime": 6210.4356,
     "train_samples": 50000,
-    "train_samples_per_second": 8.051,
-    "train_steps_per_second": 0.063
 }

 {
     "epoch": 1.0,
+    "train_loss": 0.23296360492401416,
+    "train_runtime": 6150.4195,
     "train_samples": 50000,
+    "train_samples_per_second": 8.13,
+    "train_steps_per_second": 0.064
 }

trainer_state.json CHANGED Viewed

@@ -9,14 +9,14 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0025575447570332483,
-      "grad_norm": 42.05566856949971,
-      "learning_rate": 1.25e-09,
       "logits/chosen": -4.623842239379883,
       "logits/rejected": -4.85917854309082,
       "logps/chosen": -239.31422424316406,
       "logps/rejected": -207.56365966796875,
-      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -24,646 +24,646 @@
       "step": 1
     },
     {
-      "epoch": 0.02557544757033248,
-      "grad_norm": 39.368501883614215,
-      "learning_rate": 1.25e-08,
-      "logits/chosen": -4.334437370300293,
-      "logits/rejected": -4.64446496963501,
-      "logps/chosen": -265.1294250488281,
-      "logps/rejected": -215.75079345703125,
-      "loss": 0.6929,
-      "rewards/accuracies": 0.4513888955116272,
-      "rewards/chosen": -0.00022488113609142601,
-      "rewards/margins": 0.0002015345817198977,
-      "rewards/rejected": -0.0004264157032594085,
       "step": 10
     },
     {
-      "epoch": 0.05115089514066496,
-      "grad_norm": 41.32160817525484,
-      "learning_rate": 2.5e-08,
-      "logits/chosen": -4.507131576538086,
-      "logits/rejected": -4.741620063781738,
-      "logps/chosen": -267.7641906738281,
-      "logps/rejected": -216.6431427001953,
-      "loss": 0.6928,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.001757637015543878,
-      "rewards/margins": 0.0020956869702786207,
-      "rewards/rejected": -0.000338050042046234,
       "step": 20
     },
     {
-      "epoch": 0.07672634271099744,
-      "grad_norm": 43.9248767727728,
-      "learning_rate": 3.75e-08,
-      "logits/chosen": -4.58504581451416,
-      "logits/rejected": -4.763764381408691,
-      "logps/chosen": -258.29669189453125,
-      "logps/rejected": -214.74630737304688,
-      "loss": 0.6912,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.00236110738478601,
-      "rewards/margins": 0.0044965180568397045,
-      "rewards/rejected": -0.002135410439223051,
       "step": 30
     },
     {
-      "epoch": 0.10230179028132992,
-      "grad_norm": 43.260460439031476,
-      "learning_rate": 5e-08,
-      "logits/chosen": -4.622879981994629,
-      "logits/rejected": -4.708461284637451,
-      "logps/chosen": -252.55868530273438,
-      "logps/rejected": -220.5706329345703,
-      "loss": 0.6843,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 0.010404979810118675,
-      "rewards/margins": 0.019400382414460182,
-      "rewards/rejected": -0.008995403535664082,
       "step": 40
     },
     {
-      "epoch": 0.1278772378516624,
-      "grad_norm": 42.12222343545266,
-      "learning_rate": 4.989992961303737e-08,
-      "logits/chosen": -4.523015022277832,
-      "logits/rejected": -4.722769737243652,
-      "logps/chosen": -269.7854919433594,
-      "logps/rejected": -228.0283660888672,
-      "loss": 0.6709,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": 0.021838409826159477,
-      "rewards/margins": 0.04340698570013046,
-      "rewards/rejected": -0.021568577736616135,
       "step": 50
     },
     {
-      "epoch": 0.1534526854219949,
-      "grad_norm": 41.2743182340779,
-      "learning_rate": 4.960051957873725e-08,
-      "logits/chosen": -4.632551670074463,
-      "logits/rejected": -4.7638654708862305,
-      "logps/chosen": -237.92984008789062,
-      "logps/rejected": -220.9913787841797,
-      "loss": 0.647,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.0319262370467186,
-      "rewards/margins": 0.09162561595439911,
-      "rewards/rejected": -0.05969937518239021,
       "step": 60
     },
     {
-      "epoch": 0.17902813299232737,
-      "grad_norm": 39.1856071564541,
-      "learning_rate": 4.910416686333906e-08,
-      "logits/chosen": -4.551320552825928,
-      "logits/rejected": -4.791552543640137,
-      "logps/chosen": -249.8735809326172,
-      "logps/rejected": -230.2266082763672,
-      "loss": 0.6219,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.032683782279491425,
-      "rewards/margins": 0.1573253571987152,
-      "rewards/rejected": -0.12464158236980438,
       "step": 70
     },
     {
-      "epoch": 0.20460358056265984,
-      "grad_norm": 40.933579332132446,
-      "learning_rate": 4.841484508350678e-08,
-      "logits/chosen": -4.6027655601501465,
-      "logits/rejected": -4.830323696136475,
-      "logps/chosen": -258.7153625488281,
-      "logps/rejected": -251.62368774414062,
-      "loss": 0.5731,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.025046274065971375,
-      "rewards/margins": 0.2380959540605545,
-      "rewards/rejected": -0.21304969489574432,
       "step": 80
     },
     {
-      "epoch": 0.23017902813299232,
-      "grad_norm": 40.245773651041624,
-      "learning_rate": 4.7538072695020406e-08,
-      "logits/chosen": -4.726979732513428,
-      "logits/rejected": -4.926435470581055,
-      "logps/chosen": -248.79373168945312,
-      "logps/rejected": -241.45919799804688,
-      "loss": 0.5258,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.024190178140997887,
-      "rewards/margins": 0.41048234701156616,
-      "rewards/rejected": -0.38629215955734253,
       "step": 90
     },
     {
-      "epoch": 0.2557544757033248,
-      "grad_norm": 36.66878437086382,
-      "learning_rate": 4.6480868814055416e-08,
-      "logits/chosen": -4.612161636352539,
-      "logits/rejected": -4.8301801681518555,
-      "logps/chosen": -252.9623260498047,
-      "logps/rejected": -275.35980224609375,
-      "loss": 0.5044,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": 0.03755917400121689,
-      "rewards/margins": 0.5200667977333069,
-      "rewards/rejected": -0.4825075566768646,
       "step": 100
     },
     {
-      "epoch": 0.2557544757033248,
-      "eval_logits/chosen": -4.655425071716309,
-      "eval_logits/rejected": -4.8306565284729,
-      "eval_logps/chosen": -401.76690673828125,
-      "eval_logps/rejected": -527.1570434570312,
-      "eval_loss": 0.7104954123497009,
-      "eval_rewards/accuracies": 0.47265625,
-      "eval_rewards/chosen": -0.11292455345392227,
-      "eval_rewards/margins": -0.02523117884993553,
-      "eval_rewards/rejected": -0.08769337832927704,
-      "eval_runtime": 98.6531,
-      "eval_samples_per_second": 20.273,
-      "eval_steps_per_second": 0.324,
       "step": 100
     },
     {
-      "epoch": 0.2813299232736573,
-      "grad_norm": 33.07145934836895,
-      "learning_rate": 4.525169702472916e-08,
-      "logits/chosen": -4.64766788482666,
-      "logits/rejected": -4.833477020263672,
-      "logps/chosen": -244.29296875,
-      "logps/rejected": -270.46356201171875,
-      "loss": 0.4684,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 0.04892749339342117,
-      "rewards/margins": 0.586704432964325,
-      "rewards/rejected": -0.5377769470214844,
       "step": 110
     },
     {
-      "epoch": 0.3069053708439898,
-      "grad_norm": 33.5848438881061,
-      "learning_rate": 4.386039762276975e-08,
-      "logits/chosen": -4.567343235015869,
-      "logits/rejected": -4.763147830963135,
-      "logps/chosen": -260.3200988769531,
-      "logps/rejected": -289.777587890625,
-      "loss": 0.4413,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.15669476985931396,
-      "rewards/margins": 0.7683843374252319,
-      "rewards/rejected": -0.611689567565918,
       "step": 120
     },
     {
-      "epoch": 0.33248081841432225,
-      "grad_norm": 33.705176699923506,
-      "learning_rate": 4.231810883773999e-08,
-      "logits/chosen": -4.627255916595459,
-      "logits/rejected": -4.849207878112793,
-      "logps/chosen": -243.1725616455078,
-      "logps/rejected": -288.8663635253906,
-      "loss": 0.4074,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 0.15271326899528503,
-      "rewards/margins": 0.8880899548530579,
-      "rewards/rejected": -0.7353767156600952,
       "step": 130
     },
     {
-      "epoch": 0.35805626598465473,
-      "grad_norm": 28.886999424269227,
-      "learning_rate": 4.063717766448194e-08,
-      "logits/chosen": -4.684683322906494,
-      "logits/rejected": -4.887022972106934,
-      "logps/chosen": -271.5019226074219,
-      "logps/rejected": -315.28717041015625,
-      "loss": 0.3858,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.1269315481185913,
-      "rewards/margins": 0.9580682516098022,
-      "rewards/rejected": -0.8311365842819214,
       "step": 140
     },
     {
-      "epoch": 0.3836317135549872,
-      "grad_norm": 34.24616985646881,
-      "learning_rate": 3.8831061017632845e-08,
-      "logits/chosen": -4.733763694763184,
-      "logits/rejected": -4.927274703979492,
-      "logps/chosen": -237.88955688476562,
-      "logps/rejected": -312.8567199707031,
-      "loss": 0.3767,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 0.14528730511665344,
-      "rewards/margins": 1.0826618671417236,
-      "rewards/rejected": -0.937374472618103,
       "step": 150
     },
     {
-      "epoch": 0.4092071611253197,
-      "grad_norm": 32.75808894341811,
-      "learning_rate": 3.691421800053269e-08,
-      "logits/chosen": -4.803037166595459,
-      "logits/rejected": -4.954171657562256,
-      "logps/chosen": -236.3014373779297,
-      "logps/rejected": -314.69329833984375,
-      "loss": 0.3428,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 0.1515505015850067,
-      "rewards/margins": 1.1423927545547485,
-      "rewards/rejected": -0.9908422231674194,
       "step": 160
     },
     {
-      "epoch": 0.43478260869565216,
-      "grad_norm": 27.946215188997492,
-      "learning_rate": 3.490199415097892e-08,
-      "logits/chosen": -4.722456932067871,
-      "logits/rejected": -4.946799278259277,
-      "logps/chosen": -246.8963623046875,
-      "logps/rejected": -339.89215087890625,
-      "loss": 0.3253,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 0.14177896082401276,
-      "rewards/margins": 1.3426704406738281,
-      "rewards/rejected": -1.2008916139602661,
       "step": 170
     },
     {
-      "epoch": 0.46035805626598464,
-      "grad_norm": 30.919214925357334,
-      "learning_rate": 3.2810498590513937e-08,
-      "logits/chosen": -4.84631872177124,
-      "logits/rejected": -5.061424255371094,
-      "logps/chosen": -226.3748016357422,
-      "logps/rejected": -312.2584533691406,
-      "loss": 0.3453,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 0.10488839447498322,
-      "rewards/margins": 1.2672706842422485,
-      "rewards/rejected": -1.1623823642730713,
       "step": 180
     },
     {
-      "epoch": 0.4859335038363171,
-      "grad_norm": 35.218898553113945,
-      "learning_rate": 3.065647506074306e-08,
-      "logits/chosen": -4.800053596496582,
-      "logits/rejected": -4.945647716522217,
-      "logps/chosen": -250.15170288085938,
-      "logps/rejected": -353.6054382324219,
-      "loss": 0.3307,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 0.10741142183542252,
-      "rewards/margins": 1.2918050289154053,
-      "rewards/rejected": -1.1843936443328857,
       "step": 190
     },
     {
-      "epoch": 0.5115089514066496,
-      "grad_norm": 34.27582139337608,
-      "learning_rate": 2.8457167879118325e-08,
-      "logits/chosen": -4.8446269035339355,
-      "logits/rejected": -5.029626846313477,
-      "logps/chosen": -245.35513305664062,
-      "logps/rejected": -340.13250732421875,
-      "loss": 0.3343,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.06784123182296753,
-      "rewards/margins": 1.3157011270523071,
-      "rewards/rejected": -1.2478597164154053,
       "step": 200
     },
     {
-      "epoch": 0.5115089514066496,
-      "eval_logits/chosen": -4.865734100341797,
-      "eval_logits/rejected": -5.110110282897949,
-      "eval_logps/chosen": -442.470703125,
-      "eval_logps/rejected": -579.5608520507812,
-      "eval_loss": 0.6981855630874634,
-      "eval_rewards/accuracies": 0.55859375,
-      "eval_rewards/chosen": -0.51996248960495,
-      "eval_rewards/margins": 0.09176936745643616,
-      "eval_rewards/rejected": -0.6117318868637085,
-      "eval_runtime": 98.533,
-      "eval_samples_per_second": 20.298,
-      "eval_steps_per_second": 0.325,
       "step": 200
     },
     {
-      "epoch": 0.5370843989769821,
-      "grad_norm": 27.49657850496402,
-      "learning_rate": 2.6230183887296952e-08,
-      "logits/chosen": -4.9495344161987305,
-      "logits/rejected": -5.170707702636719,
-      "logps/chosen": -253.7008514404297,
-      "logps/rejected": -388.9451904296875,
-      "loss": 0.3045,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 0.12184244394302368,
-      "rewards/margins": 1.7248340845108032,
-      "rewards/rejected": -1.6029917001724243,
       "step": 210
     },
     {
-      "epoch": 0.5626598465473146,
-      "grad_norm": 31.60560013082535,
-      "learning_rate": 2.3993351497264626e-08,
-      "logits/chosen": -4.796377658843994,
-      "logits/rejected": -5.133594989776611,
-      "logps/chosen": -251.61669921875,
-      "logps/rejected": -385.34405517578125,
-      "loss": 0.3118,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.11194615066051483,
-      "rewards/margins": 1.718654990196228,
-      "rewards/rejected": -1.6067088842391968,
       "step": 220
     },
     {
-      "epoch": 0.5882352941176471,
-      "grad_norm": 31.96350965289743,
-      "learning_rate": 2.1764577963648613e-08,
-      "logits/chosen": -4.8748345375061035,
-      "logits/rejected": -5.184638977050781,
-      "logps/chosen": -258.0860900878906,
-      "logps/rejected": -386.3064270019531,
-      "loss": 0.3152,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 0.028830066323280334,
-      "rewards/margins": 1.6547828912734985,
-      "rewards/rejected": -1.6259527206420898,
       "step": 230
     },
     {
-      "epoch": 0.6138107416879796,
-      "grad_norm": 34.58110281626622,
-      "learning_rate": 1.9561706024845818e-08,
-      "logits/chosen": -4.866055488586426,
-      "logits/rejected": -5.124758243560791,
-      "logps/chosen": -271.0610046386719,
-      "logps/rejected": -398.76727294921875,
-      "loss": 0.3015,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.017621681094169617,
-      "rewards/margins": 1.7959582805633545,
-      "rewards/rejected": -1.778336763381958,
       "step": 240
     },
     {
-      "epoch": 0.639386189258312,
-      "grad_norm": 34.427123510289576,
-      "learning_rate": 1.740237106064383e-08,
-      "logits/chosen": -5.0298542976379395,
-      "logits/rejected": -5.25708532333374,
-      "logps/chosen": -248.6929168701172,
-      "logps/rejected": -353.6767272949219,
-      "loss": 0.2999,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 8.549987978767604e-05,
-      "rewards/margins": 1.4607607126235962,
-      "rewards/rejected": -1.4606752395629883,
       "step": 250
     },
     {
-      "epoch": 0.6649616368286445,
-      "grad_norm": 36.61434250379136,
-      "learning_rate": 1.530385990987863e-08,
-      "logits/chosen": -4.881124019622803,
-      "logits/rejected": -5.179129600524902,
-      "logps/chosen": -254.1592254638672,
-      "logps/rejected": -414.9335021972656,
-      "loss": 0.2863,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": -0.001106788171455264,
-      "rewards/margins": 1.950743317604065,
-      "rewards/rejected": -1.9518499374389648,
       "step": 260
     },
     {
-      "epoch": 0.690537084398977,
-      "grad_norm": 30.68695587447776,
-      "learning_rate": 1.3282972478382409e-08,
-      "logits/chosen": -5.040741920471191,
-      "logits/rejected": -5.266045570373535,
-      "logps/chosen": -260.43017578125,
-      "logps/rejected": -401.3609924316406,
-      "loss": 0.2997,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 0.006030815653502941,
-      "rewards/margins": 1.8693902492523193,
-      "rewards/rejected": -1.8633596897125244,
       "step": 270
     },
     {
-      "epoch": 0.7161125319693095,
-      "grad_norm": 31.65072997235767,
-      "learning_rate": 1.1355887245137383e-08,
-      "logits/chosen": -4.975480556488037,
-      "logits/rejected": -5.191180229187012,
-      "logps/chosen": -262.6047668457031,
-      "logps/rejected": -436.56243896484375,
-      "loss": 0.2763,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.019021058455109596,
-      "rewards/margins": 1.9552510976791382,
-      "rewards/rejected": -1.9362300634384155,
       "step": 280
     },
     {
-      "epoch": 0.7416879795396419,
-      "grad_norm": 44.14600223942157,
-      "learning_rate": 9.538031743343628e-09,
-      "logits/chosen": -4.857875823974609,
-      "logits/rejected": -5.190948009490967,
-      "logps/chosen": -269.9219055175781,
-      "logps/rejected": -394.4927978515625,
-      "loss": 0.281,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 0.02686493471264839,
-      "rewards/margins": 1.815498948097229,
-      "rewards/rejected": -1.7886340618133545,
       "step": 290
     },
     {
-      "epoch": 0.7672634271099744,
-      "grad_norm": 39.91454735081478,
-      "learning_rate": 7.843959053281662e-09,
-      "logits/chosen": -4.9824910163879395,
-      "logits/rejected": -5.185948371887207,
-      "logps/chosen": -254.4317626953125,
-      "logps/rejected": -409.8275451660156,
-      "loss": 0.2972,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -0.01023593544960022,
-      "rewards/margins": 1.928384780883789,
-      "rewards/rejected": -1.9386205673217773,
       "step": 300
     },
     {
-      "epoch": 0.7672634271099744,
-      "eval_logits/chosen": -5.028459072113037,
-      "eval_logits/rejected": -5.28243350982666,
-      "eval_logps/chosen": -473.4393005371094,
-      "eval_logps/rejected": -613.8079833984375,
-      "eval_loss": 0.7111232280731201,
-      "eval_rewards/accuracies": 0.5625,
-      "eval_rewards/chosen": -0.8296481966972351,
-      "eval_rewards/margins": 0.12455525994300842,
-      "eval_rewards/rejected": -0.9542034864425659,
-      "eval_runtime": 98.5847,
-      "eval_samples_per_second": 20.287,
-      "eval_steps_per_second": 0.325,
       "step": 300
     },
     {
-      "epoch": 0.7928388746803069,
-      "grad_norm": 28.907759477900694,
-      "learning_rate": 6.28723129572247e-09,
-      "logits/chosen": -4.889031887054443,
-      "logits/rejected": -5.217998504638672,
-      "logps/chosen": -259.29278564453125,
-      "logps/rejected": -429.45001220703125,
-      "loss": 0.2863,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.01846831850707531,
-      "rewards/margins": 2.0909886360168457,
-      "rewards/rejected": -2.0725202560424805,
       "step": 310
     },
     {
-      "epoch": 0.8184143222506394,
-      "grad_norm": 35.34285789208096,
-      "learning_rate": 4.880311058593617e-09,
-      "logits/chosen": -5.006392002105713,
-      "logits/rejected": -5.3091888427734375,
-      "logps/chosen": -254.5725555419922,
-      "logps/rejected": -404.92706298828125,
-      "loss": 0.2987,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -0.094082310795784,
-      "rewards/margins": 1.8906733989715576,
-      "rewards/rejected": -1.9847558736801147,
       "step": 320
     },
     {
-      "epoch": 0.8439897698209718,
-      "grad_norm": 31.108205637173466,
-      "learning_rate": 3.6344616260994942e-09,
-      "logits/chosen": -4.935029983520508,
-      "logits/rejected": -5.206645965576172,
-      "logps/chosen": -279.4919128417969,
-      "logps/rejected": -436.8589782714844,
-      "loss": 0.2806,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -0.11343076080083847,
-      "rewards/margins": 1.9281622171401978,
-      "rewards/rejected": -2.041593074798584,
       "step": 330
     },
     {
-      "epoch": 0.8695652173913043,
-      "grad_norm": 37.630889083981245,
-      "learning_rate": 2.5596568090246547e-09,
-      "logits/chosen": -4.956147193908691,
-      "logits/rejected": -5.322269439697266,
-      "logps/chosen": -274.47479248046875,
-      "logps/rejected": -390.50115966796875,
-      "loss": 0.2837,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": -0.0032872497104108334,
-      "rewards/margins": 1.8339039087295532,
-      "rewards/rejected": -1.8371912240982056,
       "step": 340
     },
     {
-      "epoch": 0.8951406649616368,
-      "grad_norm": 33.50567544254766,
-      "learning_rate": 1.6645010980854079e-09,
-      "logits/chosen": -4.992954254150391,
-      "logits/rejected": -5.131080150604248,
-      "logps/chosen": -267.6521911621094,
-      "logps/rejected": -406.9331359863281,
-      "loss": 0.3087,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -0.12181039899587631,
-      "rewards/margins": 1.66982102394104,
-      "rewards/rejected": -1.7916314601898193,
       "step": 350
     },
     {
-      "epoch": 0.9207161125319693,
-      "grad_norm": 36.41873473091959,
-      "learning_rate": 9.561607795526006e-10,
-      "logits/chosen": -4.958062171936035,
-      "logits/rejected": -5.163455963134766,
-      "logps/chosen": -269.3998107910156,
-      "logps/rejected": -414.72039794921875,
-      "loss": 0.2917,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": -0.07764892280101776,
-      "rewards/margins": 1.7949295043945312,
-      "rewards/rejected": -1.8725783824920654,
       "step": 360
     },
     {
-      "epoch": 0.9462915601023018,
-      "grad_norm": 29.734783261980905,
-      "learning_rate": 4.403065646083809e-10,
-      "logits/chosen": -4.9909467697143555,
-      "logits/rejected": -5.146109104156494,
-      "logps/chosen": -261.77947998046875,
-      "logps/rejected": -422.8021545410156,
-      "loss": 0.292,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -0.04958271235227585,
-      "rewards/margins": 1.8099679946899414,
-      "rewards/rejected": -1.8595508337020874,
       "step": 370
     },
     {
-      "epoch": 0.9718670076726342,
-      "grad_norm": 37.248731518669295,
-      "learning_rate": 1.2106819172520434e-10,
-      "logits/chosen": -5.114525318145752,
-      "logits/rejected": -5.3700361251831055,
-      "logps/chosen": -265.19122314453125,
-      "logps/rejected": -416.8501892089844,
-      "loss": 0.2878,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": -0.0310853011906147,
-      "rewards/margins": 1.9321701526641846,
-      "rewards/rejected": -1.9632552862167358,
       "step": 380
     },
     {
-      "epoch": 0.9974424552429667,
-      "grad_norm": 52.48837519556826,
-      "learning_rate": 1.0013655036916758e-12,
-      "logits/chosen": -5.090936660766602,
-      "logits/rejected": -5.337624549865723,
-      "logps/chosen": -265.92279052734375,
-      "logps/rejected": -421.610595703125,
-      "loss": 0.2755,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -0.06250263750553131,
-      "rewards/margins": 1.948452353477478,
-      "rewards/rejected": -2.0109550952911377,
       "step": 390
     },
     {
       "epoch": 1.0,
       "step": 391,
       "total_flos": 0.0,
-      "train_loss": 0.4007269041922391,
-      "train_runtime": 6210.4356,
-      "train_samples_per_second": 8.051,
-      "train_steps_per_second": 0.063
     }
   ],
   "logging_steps": 10,

   "is_world_process_zero": true,
   "log_history": [
     {
+      "epoch": 0.0,
+      "grad_norm": 421.7748930851862,
+      "learning_rate": 2.5e-09,
       "logits/chosen": -4.623842239379883,
       "logits/rejected": -4.85917854309082,
       "logps/chosen": -239.31422424316406,
       "logps/rejected": -207.56365966796875,
+      "loss": 0.693,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
       "step": 1
     },
     {
+      "epoch": 0.03,
+      "grad_norm": 395.00460150231794,
+      "learning_rate": 2.5e-08,
+      "logits/chosen": -4.33427619934082,
+      "logits/rejected": -4.64423942565918,
+      "logps/chosen": -265.1573791503906,
+      "logps/rejected": -215.7504425048828,
+      "loss": 0.693,
+      "rewards/accuracies": 0.4444444477558136,
+      "rewards/chosen": -0.0050499276258051395,
+      "rewards/margins": -0.0008213710389100015,
+      "rewards/rejected": -0.004228556528687477,
       "step": 10
     },
     {
+      "epoch": 0.05,
+      "grad_norm": 398.86324560087934,
+      "learning_rate": 5e-08,
+      "logits/chosen": -4.5085954666137695,
+      "logits/rejected": -4.7428812980651855,
+      "logps/chosen": -267.77325439453125,
+      "logps/rejected": -216.7412567138672,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.016667194664478302,
+      "rewards/margins": 0.029861312359571457,
+      "rewards/rejected": -0.01319411676377058,
       "step": 20
     },
     {
+      "epoch": 0.08,
+      "grad_norm": 388.1898348935921,
+      "learning_rate": 7.5e-08,
+      "logits/chosen": -4.594970703125,
+      "logits/rejected": -4.776912689208984,
+      "logps/chosen": -257.5873107910156,
+      "logps/rejected": -215.4453582763672,
+      "loss": 0.6195,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.09455005824565887,
+      "rewards/margins": 0.18580853939056396,
+      "rewards/rejected": -0.09125850349664688,
       "step": 30
     },
     {
+      "epoch": 0.1,
+      "grad_norm": 267.0875315068977,
+      "learning_rate": 1e-07,
+      "logits/chosen": -4.637397766113281,
+      "logits/rejected": -4.731802940368652,
+      "logps/chosen": -249.911865234375,
+      "logps/rejected": -222.6023712158203,
+      "loss": 0.4813,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 0.36873525381088257,
+      "rewards/margins": 0.661862313747406,
+      "rewards/rejected": -0.29312705993652344,
       "step": 40
     },
     {
+      "epoch": 0.13,
+      "grad_norm": 193.9339508971275,
+      "learning_rate": 9.979985922607475e-08,
+      "logits/chosen": -4.531577110290527,
+      "logits/rejected": -4.7472710609436035,
+      "logps/chosen": -265.0107116699219,
+      "logps/rejected": -231.0427703857422,
+      "loss": 0.3721,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 0.6958638429641724,
+      "rewards/margins": 1.2129887342453003,
+      "rewards/rejected": -0.5171248316764832,
       "step": 50
     },
     {
+      "epoch": 0.15,
+      "grad_norm": 197.66681943180416,
+      "learning_rate": 9.92010391574745e-08,
+      "logits/chosen": -4.6342668533325195,
+      "logits/rejected": -4.783431053161621,
+      "logps/chosen": -230.5199432373047,
+      "logps/rejected": -224.8899383544922,
+      "loss": 0.281,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 1.0602517127990723,
+      "rewards/margins": 2.0471043586730957,
+      "rewards/rejected": -0.9868523478507996,
       "step": 60
     },
     {
+      "epoch": 0.18,
+      "grad_norm": 162.90653777538404,
+      "learning_rate": 9.820833372667812e-08,
+      "logits/chosen": -4.519268989562988,
+      "logits/rejected": -4.774823188781738,
+      "logps/chosen": -240.6663818359375,
+      "logps/rejected": -230.677490234375,
+      "loss": 0.2583,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 1.247557520866394,
+      "rewards/margins": 2.539060592651367,
+      "rewards/rejected": -1.2915031909942627,
       "step": 70
     },
     {
+      "epoch": 0.2,
+      "grad_norm": 139.86229845766394,
+      "learning_rate": 9.682969016701356e-08,
+      "logits/chosen": -4.514620780944824,
+      "logits/rejected": -4.760144233703613,
+      "logps/chosen": -247.26364135742188,
+      "logps/rejected": -244.28866577148438,
+      "loss": 0.2248,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 1.3956347703933716,
+      "rewards/margins": 2.7926278114318848,
+      "rewards/rejected": -1.3969930410385132,
       "step": 80
     },
     {
+      "epoch": 0.23,
+      "grad_norm": 208.01416582057018,
+      "learning_rate": 9.507614539004081e-08,
+      "logits/chosen": -4.604180812835693,
+      "logits/rejected": -4.849738597869873,
+      "logps/chosen": -236.37899780273438,
+      "logps/rejected": -221.75564575195312,
+      "loss": 0.2139,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 1.4833731651306152,
+      "rewards/margins": 3.3759377002716064,
+      "rewards/rejected": -1.8925644159317017,
       "step": 90
     },
     {
+      "epoch": 0.26,
+      "grad_norm": 168.83388514254116,
+      "learning_rate": 9.296173762811083e-08,
+      "logits/chosen": -4.470773220062256,
+      "logits/rejected": -4.786902904510498,
+      "logps/chosen": -240.86477661132812,
+      "logps/rejected": -247.6592559814453,
+      "loss": 0.216,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 1.5853464603424072,
+      "rewards/margins": 3.6403656005859375,
+      "rewards/rejected": -2.0550191402435303,
       "step": 100
     },
     {
+      "epoch": 0.26,
+      "eval_logits/chosen": -4.555809497833252,
+      "eval_logits/rejected": -4.771756172180176,
+      "eval_logps/chosen": -391.56207275390625,
+      "eval_logps/rejected": -513.7159423828125,
+      "eval_loss": 1.1906155347824097,
+      "eval_rewards/accuracies": 0.3984375,
+      "eval_rewards/chosen": -0.10876215249300003,
+      "eval_rewards/margins": -0.575933575630188,
+      "eval_rewards/rejected": 0.46717149019241333,
+      "eval_runtime": 97.8557,
+      "eval_samples_per_second": 20.438,
+      "eval_steps_per_second": 0.327,
       "step": 100
     },
     {
+      "epoch": 0.28,
+      "grad_norm": 158.8626687509829,
+      "learning_rate": 9.050339404945832e-08,
+      "logits/chosen": -4.515485763549805,
+      "logits/rejected": -4.795956134796143,
+      "logps/chosen": -232.4200439453125,
+      "logps/rejected": -236.90969848632812,
+      "loss": 0.1985,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 1.6765697002410889,
+      "rewards/margins": 3.698953628540039,
+      "rewards/rejected": -2.02238392829895,
       "step": 110
     },
     {
+      "epoch": 0.31,
+      "grad_norm": 135.50801896928647,
+      "learning_rate": 8.77207952455395e-08,
+      "logits/chosen": -4.454637050628662,
+      "logits/rejected": -4.722542762756348,
+      "logps/chosen": -256.7890930175781,
+      "logps/rejected": -248.82601928710938,
+      "loss": 0.1975,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 1.9200481176376343,
+      "rewards/margins": 3.9417896270751953,
+      "rewards/rejected": -2.0217413902282715,
       "step": 120
     },
     {
+      "epoch": 0.33,
+      "grad_norm": 154.49491996909583,
+      "learning_rate": 8.463621767547997e-08,
+      "logits/chosen": -4.527877330780029,
+      "logits/rejected": -4.81399393081665,
+      "logps/chosen": -241.6123809814453,
+      "logps/rejected": -239.6664276123047,
+      "loss": 0.1993,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 1.683153748512268,
+      "rewards/margins": 4.116929054260254,
+      "rewards/rejected": -2.4337756633758545,
       "step": 130
     },
     {
+      "epoch": 0.36,
+      "grad_norm": 161.00485250742125,
+      "learning_rate": 8.127435532896387e-08,
+      "logits/chosen": -4.5556840896606445,
+      "logits/rejected": -4.849120140075684,
+      "logps/chosen": -268.4659118652344,
+      "logps/rejected": -258.8057556152344,
+      "loss": 0.1874,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 1.5729143619537354,
+      "rewards/margins": 4.236142158508301,
+      "rewards/rejected": -2.6632275581359863,
       "step": 140
     },
     {
+      "epoch": 0.38,
+      "grad_norm": 157.59104792116975,
+      "learning_rate": 7.766212203526569e-08,
+      "logits/chosen": -4.545467853546143,
+      "logits/rejected": -4.822617053985596,
+      "logps/chosen": -235.0808563232422,
+      "logps/rejected": -246.62606811523438,
+      "loss": 0.1878,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 1.7337404489517212,
+      "rewards/margins": 4.484419822692871,
+      "rewards/rejected": -2.7506790161132812,
       "step": 150
     },
     {
+      "epoch": 0.41,
+      "grad_norm": 153.9528654470227,
+      "learning_rate": 7.382843600106538e-08,
+      "logits/chosen": -4.591027736663818,
+      "logits/rejected": -4.801583290100098,
+      "logps/chosen": -234.0863494873047,
+      "logps/rejected": -241.58761596679688,
+      "loss": 0.1687,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 1.7370154857635498,
+      "rewards/margins": 4.334869861602783,
+      "rewards/rejected": -2.5978541374206543,
       "step": 160
     },
     {
+      "epoch": 0.43,
+      "grad_norm": 134.86505617626943,
+      "learning_rate": 6.980398830195784e-08,
+      "logits/chosen": -4.457674980163574,
+      "logits/rejected": -4.7707014083862305,
+      "logps/chosen": -239.03787231445312,
+      "logps/rejected": -248.50393676757812,
+      "loss": 0.1647,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 2.203636407852173,
+      "rewards/margins": 5.073732852935791,
+      "rewards/rejected": -2.870095729827881,
       "step": 170
     },
     {
+      "epoch": 0.46,
+      "grad_norm": 134.91302963185322,
+      "learning_rate": 6.562099718102787e-08,
+      "logits/chosen": -4.556507110595703,
+      "logits/rejected": -4.814910411834717,
+      "logps/chosen": -217.7527618408203,
+      "logps/rejected": -222.974365234375,
+      "loss": 0.1885,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 1.9110908508300781,
+      "rewards/margins": 4.606502532958984,
+      "rewards/rejected": -2.695411443710327,
       "step": 180
     },
     {
+      "epoch": 0.49,
+      "grad_norm": 164.38739923975325,
+      "learning_rate": 6.131295012148612e-08,
+      "logits/chosen": -4.534274101257324,
+      "logits/rejected": -4.6962666511535645,
+      "logps/chosen": -241.34304809570312,
+      "logps/rejected": -259.50860595703125,
+      "loss": 0.1854,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 1.9549798965454102,
+      "rewards/margins": 4.389232158660889,
+      "rewards/rejected": -2.434252977371216,
       "step": 190
     },
     {
+      "epoch": 0.51,
+      "grad_norm": 145.61398569600198,
+      "learning_rate": 5.691433575823665e-08,
+      "logits/chosen": -4.528763771057129,
+      "logits/rejected": -4.724316596984863,
+      "logps/chosen": -234.06503295898438,
+      "logps/rejected": -241.2750701904297,
+      "loss": 0.1925,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 1.8074201345443726,
+      "rewards/margins": 4.400271892547607,
+      "rewards/rejected": -2.5928521156311035,
       "step": 200
     },
     {
+      "epoch": 0.51,
+      "eval_logits/chosen": -4.5165863037109375,
+      "eval_logits/rejected": -4.742655277252197,
+      "eval_logps/chosen": -394.47015380859375,
+      "eval_logps/rejected": -515.5682983398438,
+      "eval_loss": 1.3562896251678467,
+      "eval_rewards/accuracies": 0.375,
+      "eval_rewards/chosen": -0.39956870675086975,
+      "eval_rewards/margins": -0.6815085411071777,
+      "eval_rewards/rejected": 0.28193986415863037,
+      "eval_runtime": 97.991,
+      "eval_samples_per_second": 20.41,
+      "eval_steps_per_second": 0.327,
       "step": 200
     },
     {
+      "epoch": 0.54,
+      "grad_norm": 126.33823591196861,
+      "learning_rate": 5.2460367774593905e-08,
+      "logits/chosen": -4.592769622802734,
+      "logits/rejected": -4.851358413696289,
+      "logps/chosen": -244.64755249023438,
+      "logps/rejected": -259.65142822265625,
+      "loss": 0.1489,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 2.1237542629241943,
+      "rewards/margins": 5.224299907684326,
+      "rewards/rejected": -3.10054612159729,
       "step": 210
     },
     {
+      "epoch": 0.56,
+      "grad_norm": 139.63020748735394,
+      "learning_rate": 4.798670299452925e-08,
+      "logits/chosen": -4.435277462005615,
+      "logits/rejected": -4.771773338317871,
+      "logps/chosen": -242.7268524169922,
+      "logps/rejected": -258.1542053222656,
+      "loss": 0.1754,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 2.0084481239318848,
+      "rewards/margins": 5.356553077697754,
+      "rewards/rejected": -3.348104953765869,
       "step": 220
     },
     {
+      "epoch": 0.59,
+      "grad_norm": 146.4580429105822,
+      "learning_rate": 4.3529155927297226e-08,
+      "logits/chosen": -4.511857986450195,
+      "logits/rejected": -4.845773696899414,
+      "logps/chosen": -241.4152374267578,
+      "logps/rejected": -257.61944580078125,
+      "loss": 0.1815,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 1.9553873538970947,
+      "rewards/margins": 5.346222877502441,
+      "rewards/rejected": -3.390836000442505,
       "step": 230
     },
     {
+      "epoch": 0.61,
+      "grad_norm": 120.36905618473185,
+      "learning_rate": 3.9123412049691636e-08,
+      "logits/chosen": -4.488260269165039,
+      "logits/rejected": -4.767331123352051,
+      "logps/chosen": -252.67538452148438,
+      "logps/rejected": -255.2183837890625,
+      "loss": 0.1687,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 2.014777421951294,
+      "rewards/margins": 5.443254470825195,
+      "rewards/rejected": -3.4284775257110596,
       "step": 240
     },
     {
+      "epoch": 0.64,
+      "grad_norm": 143.11323504040695,
+      "learning_rate": 3.480474212128766e-08,
+      "logits/chosen": -4.618422031402588,
+      "logits/rejected": -4.878515243530273,
+      "logps/chosen": -231.21737670898438,
+      "logps/rejected": -236.189208984375,
+      "loss": 0.1614,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 1.7484108209609985,
+      "rewards/margins": 4.606410503387451,
+      "rewards/rejected": -2.857999324798584,
       "step": 250
     },
     {
+      "epoch": 0.66,
+      "grad_norm": 193.1124896787253,
+      "learning_rate": 3.060771981975726e-08,
+      "logits/chosen": -4.48922872543335,
+      "logits/rejected": -4.790060997009277,
+      "logps/chosen": -234.9071502685547,
+      "logps/rejected": -258.86639404296875,
+      "loss": 0.1653,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 1.9141381978988647,
+      "rewards/margins": 5.825932025909424,
+      "rewards/rejected": -3.9117939472198486,
       "step": 260
     },
     {
+      "epoch": 0.69,
+      "grad_norm": 194.5027032928207,
+      "learning_rate": 2.6565944956764818e-08,
+      "logits/chosen": -4.580440044403076,
+      "logits/rejected": -4.814294815063477,
+      "logps/chosen": -242.62802124023438,
+      "logps/rejected": -251.90444946289062,
+      "loss": 0.1739,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 1.84052312374115,
+      "rewards/margins": 5.5284624099731445,
+      "rewards/rejected": -3.687939405441284,
       "step": 270
     },
     {
+      "epoch": 0.72,
+      "grad_norm": 134.789590992645,
+      "learning_rate": 2.2711774490274766e-08,
+      "logits/chosen": -4.531357765197754,
+      "logits/rejected": -4.778365135192871,
+      "logps/chosen": -244.4646453857422,
+      "logps/rejected": -277.8075866699219,
+      "loss": 0.148,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 2.004220485687256,
+      "rewards/margins": 5.491039276123047,
+      "rewards/rejected": -3.48681902885437,
       "step": 280
     },
     {
+      "epoch": 0.74,
+      "grad_norm": 281.0472172318649,
+      "learning_rate": 1.9076063486687256e-08,
+      "logits/chosen": -4.403564453125,
+      "logits/rejected": -4.759608268737793,
+      "logps/chosen": -251.33419799804688,
+      "logps/rejected": -248.35458374023438,
+      "loss": 0.1633,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 2.1274170875549316,
+      "rewards/margins": 5.399934768676758,
+      "rewards/rejected": -3.272517681121826,
       "step": 290
     },
     {
+      "epoch": 0.77,
+      "grad_norm": 153.20971602377742,
+      "learning_rate": 1.5687918106563324e-08,
+      "logits/chosen": -4.5167622566223145,
+      "logits/rejected": -4.735228061676025,
+      "logps/chosen": -232.205322265625,
+      "logps/rejected": -250.81884765625,
+      "loss": 0.179,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 2.1202826499938965,
+      "rewards/margins": 5.605620861053467,
+      "rewards/rejected": -3.4853386878967285,
       "step": 300
     },
     {
+      "epoch": 0.77,
+      "eval_logits/chosen": -4.521942615509033,
+      "eval_logits/rejected": -4.748414039611816,
+      "eval_logps/chosen": -397.169189453125,
+      "eval_logps/rejected": -518.8555297851562,
+      "eval_loss": 1.3649907112121582,
+      "eval_rewards/accuracies": 0.3828125,
+      "eval_rewards/chosen": -0.6694751977920532,
+      "eval_rewards/margins": -0.6226927638053894,
+      "eval_rewards/rejected": -0.04678243398666382,
+      "eval_runtime": 97.9822,
+      "eval_samples_per_second": 20.412,
+      "eval_steps_per_second": 0.327,
       "step": 300
     },
     {
+      "epoch": 0.79,
+      "grad_norm": 149.9412754602278,
+      "learning_rate": 1.257446259144494e-08,
+      "logits/chosen": -4.436882972717285,
+      "logits/rejected": -4.769654750823975,
+      "logps/chosen": -239.6269073486328,
+      "logps/rejected": -259.68951416015625,
+      "loss": 0.1671,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 2.1512722969055176,
+      "rewards/margins": 5.90042781829834,
+      "rewards/rejected": -3.7491557598114014,
       "step": 310
     },
     {
+      "epoch": 0.82,
+      "grad_norm": 162.84992879153398,
+      "learning_rate": 9.760622117187234e-09,
+      "logits/chosen": -4.495532512664795,
+      "logits/rejected": -4.839716911315918,
+      "logps/chosen": -226.40927124023438,
+      "logps/rejected": -241.6173553466797,
+      "loss": 0.1706,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 1.8755052089691162,
+      "rewards/margins": 5.392090797424316,
+      "rewards/rejected": -3.5165855884552,
       "step": 320
     },
     {
+      "epoch": 0.84,
+      "grad_norm": 140.15174845446813,
+      "learning_rate": 7.2689232521989885e-09,
+      "logits/chosen": -4.450541019439697,
+      "logits/rejected": -4.755129814147949,
+      "logps/chosen": -248.2396240234375,
+      "logps/rejected": -268.4003601074219,
+      "loss": 0.159,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 1.9909223318099976,
+      "rewards/margins": 5.5609917640686035,
+      "rewards/rejected": -3.5700690746307373,
       "step": 330
     },
     {
+      "epoch": 0.87,
+      "grad_norm": 126.08085826498804,
+      "learning_rate": 5.119313618049309e-09,
+      "logits/chosen": -4.463766574859619,
+      "logits/rejected": -4.806758403778076,
+      "logps/chosen": -251.0575714111328,
+      "logps/rejected": -241.99609375,
+      "loss": 0.1558,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 2.308844804763794,
+      "rewards/margins": 5.830248832702637,
+      "rewards/rejected": -3.5214037895202637,
       "step": 340
     },
     {
+      "epoch": 0.9,
+      "grad_norm": 122.26451804586765,
+      "learning_rate": 3.3290021961708158e-09,
+      "logits/chosen": -4.487333297729492,
+      "logits/rejected": -4.672482967376709,
+      "logps/chosen": -237.46456909179688,
+      "logps/rejected": -258.9417419433594,
+      "loss": 0.1724,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 1.8006614446640015,
+      "rewards/margins": 4.9178361892700195,
+      "rewards/rejected": -3.117175340652466,
       "step": 350
     },
     {
+      "epoch": 0.92,
+      "grad_norm": 144.944818423744,
+      "learning_rate": 1.9123215591052013e-09,
+      "logits/chosen": -4.482793807983398,
+      "logits/rejected": -4.7083001136779785,
+      "logps/chosen": -243.4811248779297,
+      "logps/rejected": -260.84857177734375,
+      "loss": 0.1742,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 1.8153793811798096,
+      "rewards/margins": 5.153982162475586,
+      "rewards/rejected": -3.3386025428771973,
       "step": 360
     },
     {
+      "epoch": 0.95,
+      "grad_norm": 132.15693545315082,
+      "learning_rate": 8.806131292167618e-10,
+      "logits/chosen": -4.498129367828369,
+      "logits/rejected": -4.671444892883301,
+      "logps/chosen": -237.79995727539062,
+      "logps/rejected": -269.40106201171875,
+      "loss": 0.1751,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 1.9021259546279907,
+      "rewards/margins": 5.157528877258301,
+      "rewards/rejected": -3.2554030418395996,
       "step": 370
     },
     {
+      "epoch": 0.97,
+      "grad_norm": 150.94805422584076,
+      "learning_rate": 2.4213638345040867e-10,
+      "logits/chosen": -4.600647926330566,
+      "logits/rejected": -4.883923053741455,
+      "logps/chosen": -242.57833862304688,
+      "logps/rejected": -258.09075927734375,
+      "loss": 0.1659,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 1.9504365921020508,
+      "rewards/margins": 5.70705509185791,
+      "rewards/rejected": -3.7566189765930176,
       "step": 380
     },
     {
+      "epoch": 1.0,
+      "grad_norm": 216.65022888736644,
+      "learning_rate": 2.0027310073833516e-12,
+      "logits/chosen": -4.585878372192383,
+      "logits/rejected": -4.851449966430664,
+      "logps/chosen": -237.5348663330078,
+      "logps/rejected": -255.65542602539062,
+      "loss": 0.1632,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 2.213772773742676,
+      "rewards/margins": 5.727804183959961,
+      "rewards/rejected": -3.514031171798706,
       "step": 390
     },
     {
       "epoch": 1.0,
       "step": 391,
       "total_flos": 0.0,
+      "train_loss": 0.23296360492401416,
+      "train_runtime": 6150.4195,
+      "train_samples_per_second": 8.13,
+      "train_steps_per_second": 0.064
     }
   ],
   "logging_steps": 10,

training_args.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0fa7e3e53a21d58800798c272a65a4f1f4bdea1a718fe29f85dc9ce2a41691db
-size 6328

 version https://git-lfs.github.com/spec/v1
+oid sha256:415d261f85430b964f160785ea3246fdc1bd7cc63556f01f25275fb3ec7835a0
+size 6264