Model save

Browse files

Files changed (12) hide show

README.md +19 -19
all_results.json +5 -4
config.json +1 -1
generation_config.json +1 -1
model-00001-of-00003.safetensors +1 -1
model-00002-of-00003.safetensors +1 -1
model-00003-of-00003.safetensors +1 -1
runs/May28_02-13-45_n136-112-146/events.out.tfevents.1716834544.n136-112-146.326388.0 +2 -2
tokenizer.json +1 -0
train_results.json +5 -4
trainer_state.json +511 -511
training_args.bin +2 -2

README.md CHANGED Viewed

@@ -13,17 +13,17 @@ should probably proofread and complete it, then remove this comment. -->
 # zephyr-7b-dpo-full
-This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.1360
-- Rewards/chosen: -0.1815
-- Rewards/rejected: 0.2981
-- Rewards/accuracies: 0.4102
-- Rewards/margins: -0.4796
-- Logps/rejected: -515.4065
-- Logps/chosen: -392.2896
-- Logits/rejected: -4.7596
-- Logits/chosen: -4.5397
 ## Model description
@@ -42,7 +42,7 @@ More information needed
 ### Training hyperparameters
 The following hyperparameters were used during training:
-- learning_rate: 5e-08
 - train_batch_size: 8
 - eval_batch_size: 8
 - seed: 42
@@ -58,16 +58,16 @@ The following hyperparameters were used during training:
 ### Training results
-| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
-|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.2984        | 0.26  | 100  | 0.9403          | -0.0277        | 0.3050           | 0.4062             | -0.3328         | -515.3372      | -390.7516    | -4.7620         | -4.5507       |
-| 0.2338        | 0.51  | 200  | 1.0997          | -0.0605        | 0.4142           | 0.3867             | -0.4747         | -514.2458      | -391.0792    | -4.7584         | -4.5386       |
-| 0.2158        | 0.77  | 300  | 1.1360          | -0.1815        | 0.2981           | 0.4102             | -0.4796         | -515.4065      | -392.2896    | -4.7596         | -4.5397       |
 ### Framework versions
-- Transformers 4.38.2
 - Pytorch 2.1.2+cu118
-- Datasets 2.16.1
-- Tokenizers 0.15.2

 # zephyr-7b-dpo-full
+This model was trained from scratch on an unknown dataset.
 It achieves the following results on the evaluation set:
+- Loss: 0.8003
+- Rewards/chosen: -1.8897
+- Rewards/rejected: -2.0004
+- Rewards/accuracies: 0.5273
+- Rewards/margins: 0.1107
+- Logps/rejected: -718.4238
+- Logps/chosen: -579.4417
+- Logits/rejected: -5.6556
+- Logits/chosen: -5.3947
 ## Model description
 ### Training hyperparameters
 The following hyperparameters were used during training:
+- learning_rate: 1e-07
 - train_batch_size: 8
 - eval_batch_size: 8
 - seed: 42
 ### Training results
+| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.3413        | 0.2558 | 100  | 0.7230          | -0.5409        | -0.5757          | 0.5156             | 0.0348          | -575.9554      | -444.5646    | -5.0451         | -4.8217       |
+| 0.2653        | 0.5115 | 200  | 0.7765          | -1.4996        | -1.6149          | 0.5430             | 0.1153          | -679.8810      | -540.4390    | -5.5042         | -5.2262       |
+| 0.2424        | 0.7673 | 300  | 0.8003          | -1.8897        | -2.0004          | 0.5273             | 0.1107          | -718.4238      | -579.4417    | -5.6556         | -5.3947       |
 ### Framework versions
+- Transformers 4.40.2
 - Pytorch 2.1.2+cu118
+- Datasets 2.19.1
+- Tokenizers 0.19.1

all_results.json CHANGED Viewed

@@ -1,8 +1,9 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.29024992444935965,
-    "train_runtime": 6148.7126,
     "train_samples": 50000,
-    "train_samples_per_second": 8.132,
-    "train_steps_per_second": 0.064
 }

 {
     "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.3220548828315857,
+    "train_runtime": 6253.066,
     "train_samples": 50000,
+    "train_samples_per_second": 7.996,
+    "train_steps_per_second": 0.063
 }

config.json CHANGED Viewed

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.38.2",
   "use_cache": false,
   "vocab_size": 32002
 }

   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
+  "transformers_version": "4.40.2",
   "use_cache": false,
   "vocab_size": 32002
 }

generation_config.json CHANGED Viewed

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 32000,
-  "transformers_version": "4.38.2"
 }

   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 32000,
+  "transformers_version": "4.40.2"
 }

model-00001-of-00003.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7b1567f82232ae4543a6b8c947587ca02d81b5b1a6d612f6c48505a456bcdaef
 size 4943178720

 version https://git-lfs.github.com/spec/v1
+oid sha256:704bf613eb27caf571ef226e414e9760489580da4614aaafb40003e65ec17441
 size 4943178720

model-00002-of-00003.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b63c90bc00eb3c20661922864d3e4196b77a831cb51f070d94bfb2c31046bb6
 size 4999819336

 version https://git-lfs.github.com/spec/v1
+oid sha256:ed52e2d62c0960fa29471d46fbfd02182f6f32018d50ca876ce32d35000541e1
 size 4999819336

model-00003-of-00003.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dccab9d7e3cbbc2fe51be697f1fb53d79146c3c708bad129f55934efe50fd566
 size 4540532728

 version https://git-lfs.github.com/spec/v1
+oid sha256:e8183375d0ae670479a1b45a3914794934dd63d5f69e72e6f0fdde82f4a37170
 size 4540532728

runs/May28_02-13-45_n136-112-146/events.out.tfevents.1716834544.n136-112-146.326388.0 CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a712b7a97539c383168f6e5e3530ecfe86d5b039a3a87ba7fd6b692854b1ad7d
-size 28347

 version https://git-lfs.github.com/spec/v1
+oid sha256:894cad00dcd7b13143689d809d7c35a6c332b87eeae40025e75ae3c6de784805
+size 34893

tokenizer.json CHANGED Viewed

@@ -152,6 +152,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
+    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

train_results.json CHANGED Viewed

@@ -1,8 +1,9 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.29024992444935965,
-    "train_runtime": 6148.7126,
     "train_samples": 50000,
-    "train_samples_per_second": 8.132,
-    "train_steps_per_second": 0.064
 }

 {
     "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.3220548828315857,
+    "train_runtime": 6253.066,
     "train_samples": 50000,
+    "train_samples_per_second": 7.996,
+    "train_steps_per_second": 0.063
 }

trainer_state.json CHANGED Viewed

@@ -9,14 +9,14 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0,
-      "grad_norm": 422.289956912934,
-      "learning_rate": 1.25e-09,
       "logits/chosen": -4.623842239379883,
       "logits/rejected": -4.85917854309082,
       "logps/chosen": -239.31422424316406,
       "logps/rejected": -207.56365966796875,
-      "loss": 0.6959,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -24,646 +24,646 @@
       "step": 1
     },
     {
-      "epoch": 0.03,
-      "grad_norm": 397.335507450448,
-      "learning_rate": 1.25e-08,
-      "logits/chosen": -4.334493160247803,
-      "logits/rejected": -4.643917560577393,
-      "logps/chosen": -265.1265563964844,
-      "logps/rejected": -215.76223754882812,
-      "loss": 0.6932,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.0019649432506412268,
-      "rewards/margins": 0.0034448718652129173,
-      "rewards/rejected": -0.005409814417362213,
       "step": 10
     },
     {
-      "epoch": 0.05,
-      "grad_norm": 411.8624893441942,
-      "learning_rate": 2.5e-08,
-      "logits/chosen": -4.506826400756836,
-      "logits/rejected": -4.740732192993164,
-      "logps/chosen": -267.86932373046875,
-      "logps/rejected": -216.64578247070312,
-      "loss": 0.6921,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": 0.007061410695314407,
-      "rewards/margins": 0.010701502673327923,
-      "rewards/rejected": -0.003640091512352228,
       "step": 20
     },
     {
-      "epoch": 0.08,
-      "grad_norm": 426.4548766919431,
-      "learning_rate": 3.75e-08,
-      "logits/chosen": -4.585576057434082,
-      "logits/rejected": -4.762608528137207,
-      "logps/chosen": -258.25,
-      "logps/rejected": -214.71231079101562,
-      "loss": 0.6756,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.028281379491090775,
-      "rewards/margins": 0.04623327776789665,
-      "rewards/rejected": -0.017951902002096176,
       "step": 30
     },
     {
-      "epoch": 0.1,
-      "grad_norm": 368.66232685986097,
-      "learning_rate": 5e-08,
-      "logits/chosen": -4.62213659286499,
-      "logits/rejected": -4.706842422485352,
-      "logps/chosen": -252.6122283935547,
-      "logps/rejected": -220.41427612304688,
-      "loss": 0.6177,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": 0.0986957773566246,
-      "rewards/margins": 0.1730131059885025,
-      "rewards/rejected": -0.07431730628013611,
       "step": 40
     },
     {
-      "epoch": 0.13,
-      "grad_norm": 294.81806277707193,
-      "learning_rate": 4.989992961303737e-08,
-      "logits/chosen": -4.516692161560059,
-      "logits/rejected": -4.714283466339111,
-      "logps/chosen": -269.63470458984375,
-      "logps/rejected": -227.5962371826172,
-      "loss": 0.5368,
       "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.2334650307893753,
-      "rewards/margins": 0.4059367775917053,
-      "rewards/rejected": -0.17247170209884644,
       "step": 50
     },
     {
-      "epoch": 0.15,
-      "grad_norm": 263.2732793061953,
-      "learning_rate": 4.960051957873725e-08,
-      "logits/chosen": -4.610293388366699,
-      "logits/rejected": -4.734058856964111,
-      "logps/chosen": -237.3783721923828,
-      "logps/rejected": -218.6456298828125,
-      "loss": 0.4381,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.3744103014469147,
-      "rewards/margins": 0.7368327975273132,
-      "rewards/rejected": -0.3624224364757538,
       "step": 60
     },
     {
-      "epoch": 0.18,
-      "grad_norm": 243.25478495437125,
-      "learning_rate": 4.910416686333906e-08,
-      "logits/chosen": -4.496267795562744,
-      "logits/rejected": -4.724743843078613,
-      "logps/chosen": -248.19540405273438,
-      "logps/rejected": -222.69140625,
-      "loss": 0.4014,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 0.49465441703796387,
-      "rewards/margins": 0.9875491261482239,
-      "rewards/rejected": -0.49289458990097046,
       "step": 70
     },
     {
-      "epoch": 0.2,
-      "grad_norm": 203.40516001004536,
-      "learning_rate": 4.841484508350678e-08,
-      "logits/chosen": -4.488529682159424,
-      "logits/rejected": -4.711686611175537,
-      "logps/chosen": -254.6974639892578,
-      "logps/rejected": -235.7099151611328,
-      "loss": 0.3429,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.652252197265625,
-      "rewards/margins": 1.1913691759109497,
-      "rewards/rejected": -0.5391170978546143,
       "step": 80
     },
     {
-      "epoch": 0.23,
-      "grad_norm": 237.79736326938396,
-      "learning_rate": 4.7538072695020406e-08,
-      "logits/chosen": -4.58192777633667,
-      "logits/rejected": -4.797459602355957,
-      "logps/chosen": -243.06143188476562,
-      "logps/rejected": -210.63308715820312,
-      "loss": 0.3085,
       "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 0.8151354789733887,
-      "rewards/margins": 1.595442533493042,
-      "rewards/rejected": -0.7803069353103638,
       "step": 90
     },
     {
-      "epoch": 0.26,
-      "grad_norm": 222.07031689896021,
-      "learning_rate": 4.6480868814055416e-08,
-      "logits/chosen": -4.459914207458496,
-      "logits/rejected": -4.747165679931641,
-      "logps/chosen": -247.2704620361328,
-      "logps/rejected": -236.6487274169922,
-      "loss": 0.2984,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.9447771906852722,
-      "rewards/margins": 1.8987438678741455,
-      "rewards/rejected": -0.953966498374939,
       "step": 100
     },
     {
-      "epoch": 0.26,
-      "eval_logits/chosen": -4.55070686340332,
-      "eval_logits/rejected": -4.762002944946289,
-      "eval_logps/chosen": -390.7516174316406,
-      "eval_logps/rejected": -515.337158203125,
-      "eval_loss": 0.9402573704719543,
-      "eval_rewards/accuracies": 0.40625,
-      "eval_rewards/chosen": -0.027714576572179794,
-      "eval_rewards/margins": -0.3327641487121582,
-      "eval_rewards/rejected": 0.3050495684146881,
-      "eval_runtime": 97.8238,
-      "eval_samples_per_second": 20.445,
-      "eval_steps_per_second": 0.327,
       "step": 100
     },
     {
-      "epoch": 0.28,
-      "grad_norm": 179.38114597248955,
-      "learning_rate": 4.525169702472916e-08,
-      "logits/chosen": -4.51773738861084,
-      "logits/rejected": -4.777291297912598,
-      "logps/chosen": -238.6410369873047,
-      "logps/rejected": -227.3874053955078,
-      "loss": 0.2711,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 1.0544707775115967,
-      "rewards/margins": 2.1246237754821777,
-      "rewards/rejected": -1.070152997970581,
       "step": 110
     },
     {
-      "epoch": 0.31,
-      "grad_norm": 163.17108435846185,
-      "learning_rate": 4.386039762276975e-08,
-      "logits/chosen": -4.462746620178223,
-      "logits/rejected": -4.7056145668029785,
-      "logps/chosen": -262.86920166015625,
-      "logps/rejected": -238.99801635742188,
-      "loss": 0.267,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": 1.3120397329330444,
-      "rewards/margins": 2.3509771823883057,
-      "rewards/rejected": -1.0389372110366821,
       "step": 120
     },
     {
-      "epoch": 0.33,
-      "grad_norm": 229.70994586547334,
-      "learning_rate": 4.231810883773999e-08,
-      "logits/chosen": -4.532160758972168,
-      "logits/rejected": -4.803128719329834,
-      "logps/chosen": -246.3385009765625,
-      "logps/rejected": -227.83792114257812,
-      "loss": 0.2577,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 1.2105365991592407,
-      "rewards/margins": 2.461458921432495,
-      "rewards/rejected": -1.2509223222732544,
       "step": 130
     },
     {
-      "epoch": 0.36,
-      "grad_norm": 152.87001146329087,
-      "learning_rate": 4.063717766448194e-08,
-      "logits/chosen": -4.559675216674805,
-      "logits/rejected": -4.842075824737549,
-      "logps/chosen": -272.3346252441406,
-      "logps/rejected": -246.54464721679688,
-      "loss": 0.2453,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 1.1860424280166626,
-      "rewards/margins": 2.6231608390808105,
-      "rewards/rejected": -1.4371181726455688,
       "step": 140
     },
     {
-      "epoch": 0.38,
-      "grad_norm": 176.7311824941399,
-      "learning_rate": 3.8831061017632845e-08,
-      "logits/chosen": -4.557957172393799,
-      "logits/rejected": -4.812293529510498,
-      "logps/chosen": -239.32144165039062,
-      "logps/rejected": -232.82479858398438,
-      "loss": 0.2425,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 1.3096827268600464,
-      "rewards/margins": 2.680234432220459,
-      "rewards/rejected": -1.3705517053604126,
       "step": 150
     },
     {
-      "epoch": 0.41,
-      "grad_norm": 179.5862213559593,
-      "learning_rate": 3.691421800053269e-08,
-      "logits/chosen": -4.614952564239502,
-      "logits/rejected": -4.799678802490234,
-      "logps/chosen": -238.4506378173828,
-      "logps/rejected": -229.4785614013672,
-      "loss": 0.2216,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 1.300586462020874,
-      "rewards/margins": 2.687537908554077,
-      "rewards/rejected": -1.3869514465332031,
       "step": 160
     },
     {
-      "epoch": 0.43,
-      "grad_norm": 160.70849517962517,
-      "learning_rate": 3.490199415097892e-08,
-      "logits/chosen": -4.499081611633301,
-      "logits/rejected": -4.779529571533203,
-      "logps/chosen": -244.9915771484375,
-      "logps/rejected": -235.46743774414062,
-      "loss": 0.2108,
       "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.6082652807235718,
-      "rewards/margins": 3.174710988998413,
-      "rewards/rejected": -1.5664453506469727,
       "step": 170
     },
     {
-      "epoch": 0.46,
-      "grad_norm": 149.11333079529007,
-      "learning_rate": 3.2810498590513937e-08,
-      "logits/chosen": -4.59390926361084,
-      "logits/rejected": -4.832152366638184,
-      "logps/chosen": -222.95986938476562,
-      "logps/rejected": -211.53585815429688,
-      "loss": 0.2274,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.3903782367706299,
-      "rewards/margins": 2.9419426918029785,
-      "rewards/rejected": -1.5515644550323486,
       "step": 180
     },
     {
-      "epoch": 0.49,
-      "grad_norm": 171.77003361632143,
-      "learning_rate": 3.065647506074306e-08,
-      "logits/chosen": -4.56182861328125,
-      "logits/rejected": -4.7075724601745605,
-      "logps/chosen": -245.95556640625,
-      "logps/rejected": -247.3394012451172,
-      "loss": 0.2299,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 1.4937294721603394,
-      "rewards/margins": 2.711061477661133,
-      "rewards/rejected": -1.2173320055007935,
       "step": 190
     },
     {
-      "epoch": 0.51,
-      "grad_norm": 152.3542212939215,
-      "learning_rate": 2.8457167879118325e-08,
-      "logits/chosen": -4.556639194488525,
-      "logits/rejected": -4.735670566558838,
-      "logps/chosen": -237.8849639892578,
-      "logps/rejected": -229.5240020751953,
-      "loss": 0.2338,
       "rewards/accuracies": 0.875,
-      "rewards/chosen": 1.4254279136657715,
-      "rewards/margins": 2.843172073364258,
-      "rewards/rejected": -1.4177442789077759,
       "step": 200
     },
     {
-      "epoch": 0.51,
-      "eval_logits/chosen": -4.538640975952148,
-      "eval_logits/rejected": -4.758352279663086,
-      "eval_logps/chosen": -391.07916259765625,
-      "eval_logps/rejected": -514.2457885742188,
-      "eval_loss": 1.0996507406234741,
-      "eval_rewards/accuracies": 0.38671875,
-      "eval_rewards/chosen": -0.06046929210424423,
-      "eval_rewards/margins": -0.4746614694595337,
-      "eval_rewards/rejected": 0.41419219970703125,
-      "eval_runtime": 98.0841,
-      "eval_samples_per_second": 20.391,
-      "eval_steps_per_second": 0.326,
       "step": 200
     },
     {
-      "epoch": 0.54,
-      "grad_norm": 139.65082338502864,
-      "learning_rate": 2.6230183887296952e-08,
-      "logits/chosen": -4.619709014892578,
-      "logits/rejected": -4.859663963317871,
-      "logps/chosen": -249.32113647460938,
-      "logps/rejected": -246.33468627929688,
-      "loss": 0.1967,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 1.6563961505889893,
-      "rewards/margins": 3.425267457962036,
-      "rewards/rejected": -1.768871545791626,
       "step": 210
     },
     {
-      "epoch": 0.56,
-      "grad_norm": 147.42119588032455,
-      "learning_rate": 2.3993351497264626e-08,
-      "logits/chosen": -4.466108798980713,
-      "logits/rejected": -4.793113708496094,
-      "logps/chosen": -247.28756713867188,
-      "logps/rejected": -244.1797637939453,
-      "loss": 0.2159,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 1.5523773431777954,
-      "rewards/margins": 3.5030410289764404,
-      "rewards/rejected": -1.9506635665893555,
       "step": 220
     },
     {
-      "epoch": 0.59,
-      "grad_norm": 158.88183877851495,
-      "learning_rate": 2.1764577963648613e-08,
-      "logits/chosen": -4.541924476623535,
-      "logits/rejected": -4.858447074890137,
-      "logps/chosen": -245.6726531982422,
-      "logps/rejected": -243.28677368164062,
-      "loss": 0.2197,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 1.5296446084976196,
-      "rewards/margins": 3.487210750579834,
-      "rewards/rejected": -1.957566499710083,
       "step": 230
     },
     {
-      "epoch": 0.61,
-      "grad_norm": 157.203527489415,
-      "learning_rate": 1.9561706024845818e-08,
-      "logits/chosen": -4.5143561363220215,
-      "logits/rejected": -4.772491455078125,
-      "logps/chosen": -256.7393798828125,
-      "logps/rejected": -240.91226196289062,
-      "loss": 0.2123,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 1.6083800792694092,
-      "rewards/margins": 3.60624623298645,
-      "rewards/rejected": -1.9978656768798828,
       "step": 240
     },
     {
-      "epoch": 0.64,
-      "grad_norm": 159.9556615524972,
-      "learning_rate": 1.740237106064383e-08,
-      "logits/chosen": -4.641883850097656,
-      "logits/rejected": -4.893360614776611,
-      "logps/chosen": -234.55264282226562,
-      "logps/rejected": -224.25631713867188,
-      "loss": 0.2024,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.414884090423584,
-      "rewards/margins": 3.079598903656006,
-      "rewards/rejected": -1.6647151708602905,
       "step": 250
     },
     {
-      "epoch": 0.66,
-      "grad_norm": 198.60636587673295,
-      "learning_rate": 1.530385990987863e-08,
-      "logits/chosen": -4.517378330230713,
-      "logits/rejected": -4.799233913421631,
-      "logps/chosen": -238.4044189453125,
-      "logps/rejected": -242.4611358642578,
-      "loss": 0.2025,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 1.5644124746322632,
-      "rewards/margins": 3.8356785774230957,
-      "rewards/rejected": -2.271266460418701,
       "step": 260
     },
     {
-      "epoch": 0.69,
-      "grad_norm": 186.10967020286805,
-      "learning_rate": 1.3282972478382409e-08,
-      "logits/chosen": -4.603947639465332,
-      "logits/rejected": -4.826247215270996,
-      "logps/chosen": -245.811279296875,
-      "logps/rejected": -236.3955841064453,
-      "loss": 0.2127,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 1.5221991539001465,
-      "rewards/margins": 3.6592516899108887,
-      "rewards/rejected": -2.137052536010742,
       "step": 270
     },
     {
-      "epoch": 0.72,
-      "grad_norm": 144.59147008918274,
-      "learning_rate": 1.1355887245137383e-08,
-      "logits/chosen": -4.557550430297852,
-      "logits/rejected": -4.780216217041016,
-      "logps/chosen": -248.1402587890625,
-      "logps/rejected": -262.3576354980469,
-      "loss": 0.1884,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": 1.636661171913147,
-      "rewards/margins": 3.578484296798706,
-      "rewards/rejected": -1.9418232440948486,
       "step": 280
     },
     {
-      "epoch": 0.74,
-      "grad_norm": 331.8489693457681,
-      "learning_rate": 9.538031743343628e-09,
-      "logits/chosen": -4.426544666290283,
-      "logits/rejected": -4.761611461639404,
-      "logps/chosen": -255.4153289794922,
-      "logps/rejected": -234.84487915039062,
-      "loss": 0.1966,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.7193052768707275,
-      "rewards/margins": 3.6408512592315674,
-      "rewards/rejected": -1.9215457439422607,
       "step": 290
     },
     {
-      "epoch": 0.77,
-      "grad_norm": 177.15049850318,
-      "learning_rate": 7.843959053281662e-09,
-      "logits/chosen": -4.541173934936523,
-      "logits/rejected": -4.741909503936768,
-      "logps/chosen": -236.41854858398438,
-      "logps/rejected": -236.483642578125,
-      "loss": 0.2158,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 1.6989631652832031,
-      "rewards/margins": 3.7507786750793457,
-      "rewards/rejected": -2.0518155097961426,
       "step": 300
     },
     {
-      "epoch": 0.77,
-      "eval_logits/chosen": -4.539734840393066,
-      "eval_logits/rejected": -4.759631633758545,
-      "eval_logps/chosen": -392.2895812988281,
-      "eval_logps/rejected": -515.406494140625,
-      "eval_loss": 1.1359957456588745,
-      "eval_rewards/accuracies": 0.41015625,
-      "eval_rewards/chosen": -0.18151262402534485,
-      "eval_rewards/margins": -0.47963014245033264,
-      "eval_rewards/rejected": 0.2981175184249878,
-      "eval_runtime": 97.905,
-      "eval_samples_per_second": 20.428,
-      "eval_steps_per_second": 0.327,
       "step": 300
     },
     {
-      "epoch": 0.79,
-      "grad_norm": 159.8961908197972,
-      "learning_rate": 6.28723129572247e-09,
-      "logits/chosen": -4.461672782897949,
-      "logits/rejected": -4.776505470275879,
-      "logps/chosen": -244.0063934326172,
-      "logps/rejected": -244.4510955810547,
-      "loss": 0.2028,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 1.7133222818374634,
-      "rewards/margins": 3.9386374950408936,
-      "rewards/rejected": -2.2253153324127197,
       "step": 310
     },
     {
-      "epoch": 0.82,
-      "grad_norm": 185.55678167306448,
-      "learning_rate": 4.880311058593617e-09,
-      "logits/chosen": -4.521292209625244,
-      "logits/rejected": -4.848372936248779,
-      "logps/chosen": -230.04397583007812,
-      "logps/rejected": -226.9331817626953,
-      "loss": 0.2095,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 1.5120347738265991,
-      "rewards/margins": 3.560204267501831,
-      "rewards/rejected": -2.0481698513031006,
       "step": 320
     },
     {
-      "epoch": 0.84,
-      "grad_norm": 155.26912676521727,
-      "learning_rate": 3.6344616260994942e-09,
-      "logits/chosen": -4.473151683807373,
-      "logits/rejected": -4.766911029815674,
-      "logps/chosen": -251.7646484375,
-      "logps/rejected": -254.7379913330078,
-      "loss": 0.1928,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 1.638421654701233,
-      "rewards/margins": 3.8422539234161377,
-      "rewards/rejected": -2.2038321495056152,
       "step": 330
     },
     {
-      "epoch": 0.87,
-      "grad_norm": 151.70242269299357,
-      "learning_rate": 2.5596568090246547e-09,
-      "logits/chosen": -4.4894232749938965,
-      "logits/rejected": -4.816695213317871,
-      "logps/chosen": -255.334716796875,
-      "logps/rejected": -228.4678192138672,
-      "loss": 0.1963,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 1.8811309337615967,
-      "rewards/margins": 4.049709320068359,
-      "rewards/rejected": -2.1685783863067627,
       "step": 340
     },
     {
-      "epoch": 0.9,
-      "grad_norm": 150.47860807724385,
-      "learning_rate": 1.6645010980854079e-09,
-      "logits/chosen": -4.505433559417725,
-      "logits/rejected": -4.675290584564209,
-      "logps/chosen": -240.54714965820312,
-      "logps/rejected": -246.03665161132812,
-      "loss": 0.2168,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 1.4924026727676392,
-      "rewards/margins": 3.3190674781799316,
-      "rewards/rejected": -1.8266645669937134,
       "step": 350
     },
     {
-      "epoch": 0.92,
-      "grad_norm": 165.46679700251414,
-      "learning_rate": 9.561607795526006e-10,
-      "logits/chosen": -4.49678373336792,
-      "logits/rejected": -4.711674690246582,
-      "logps/chosen": -246.95388793945312,
-      "logps/rejected": -247.2928009033203,
-      "loss": 0.2121,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 1.468100905418396,
-      "rewards/margins": 3.451129913330078,
-      "rewards/rejected": -1.9830286502838135,
       "step": 360
     },
     {
-      "epoch": 0.95,
-      "grad_norm": 149.25359071163066,
-      "learning_rate": 4.403065646083809e-10,
-      "logits/chosen": -4.518364906311035,
-      "logits/rejected": -4.680220603942871,
-      "logps/chosen": -241.2300262451172,
-      "logps/rejected": -255.6038818359375,
-      "loss": 0.2103,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 1.5591168403625488,
-      "rewards/margins": 3.434800386428833,
-      "rewards/rejected": -1.8756835460662842,
       "step": 370
     },
     {
-      "epoch": 0.97,
-      "grad_norm": 160.57572024314433,
-      "learning_rate": 1.2106819172520434e-10,
-      "logits/chosen": -4.618407726287842,
-      "logits/rejected": -4.8883843421936035,
-      "logps/chosen": -246.15853881835938,
-      "logps/rejected": -243.1090087890625,
-      "loss": 0.2034,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 1.5924150943756104,
-      "rewards/margins": 3.8508572578430176,
-      "rewards/rejected": -2.2584421634674072,
       "step": 380
     },
     {
-      "epoch": 1.0,
-      "grad_norm": 235.40189038757265,
-      "learning_rate": 1.0013655036916758e-12,
-      "logits/chosen": -4.611303329467773,
-      "logits/rejected": -4.857443809509277,
-      "logps/chosen": -242.1800994873047,
-      "logps/rejected": -241.1270294189453,
-      "loss": 0.199,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 1.7492481470108032,
-      "rewards/margins": 3.8104407787323,
-      "rewards/rejected": -2.0611929893493652,
       "step": 390
     },
     {
       "epoch": 1.0,
       "step": 391,
       "total_flos": 0.0,
-      "train_loss": 0.29024992444935965,
-      "train_runtime": 6148.7126,
-      "train_samples_per_second": 8.132,
-      "train_steps_per_second": 0.064
     }
   ],
   "logging_steps": 10,

   "is_world_process_zero": true,
   "log_history": [
     {
+      "epoch": 0.0025575447570332483,
+      "grad_norm": 42.05885932037307,
+      "learning_rate": 2.5e-09,
       "logits/chosen": -4.623842239379883,
       "logits/rejected": -4.85917854309082,
       "logps/chosen": -239.31422424316406,
       "logps/rejected": -207.56365966796875,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
       "step": 1
     },
     {
+      "epoch": 0.02557544757033248,
+      "grad_norm": 39.560773735648084,
+      "learning_rate": 2.5e-08,
+      "logits/chosen": -4.334544658660889,
+      "logits/rejected": -4.644796848297119,
+      "logps/chosen": -265.15618896484375,
+      "logps/rejected": -215.6714630126953,
+      "loss": 0.693,
+      "rewards/accuracies": 0.4166666567325592,
+      "rewards/chosen": -0.0004928099224343896,
+      "rewards/margins": -0.0008595392573624849,
+      "rewards/rejected": 0.00036672933492809534,
       "step": 10
     },
     {
+      "epoch": 0.05115089514066496,
+      "grad_norm": 40.83271143256618,
+      "learning_rate": 5e-08,
+      "logits/chosen": -4.509532928466797,
+      "logits/rejected": -4.744012832641602,
+      "logps/chosen": -267.80267333984375,
+      "logps/rejected": -216.80471801757812,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.0013727399054914713,
+      "rewards/margins": 0.0033264080993831158,
+      "rewards/rejected": -0.0019536681938916445,
       "step": 20
     },
     {
+      "epoch": 0.07672634271099744,
+      "grad_norm": 43.48154475134036,
+      "learning_rate": 7.5e-08,
+      "logits/chosen": -4.5965423583984375,
+      "logits/rejected": -4.777901649475098,
+      "logps/chosen": -257.59088134765625,
+      "logps/rejected": -215.49658203125,
+      "loss": 0.6845,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.00941941887140274,
+      "rewards/margins": 0.019057607278227806,
+      "rewards/rejected": -0.00963818933814764,
       "step": 30
     },
     {
+      "epoch": 0.10230179028132992,
+      "grad_norm": 43.11247032025707,
+      "learning_rate": 1e-07,
+      "logits/chosen": -4.648722171783447,
+      "logits/rejected": -4.745718002319336,
+      "logps/chosen": -250.10897827148438,
+      "logps/rejected": -223.86532592773438,
+      "loss": 0.6588,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 0.03490210697054863,
+      "rewards/margins": 0.07684428989887238,
+      "rewards/rejected": -0.041942186653614044,
       "step": 40
     },
     {
+      "epoch": 0.1278772378516624,
+      "grad_norm": 47.11742069616159,
+      "learning_rate": 9.979985922607475e-08,
+      "logits/chosen": -4.593738555908203,
+      "logits/rejected": -4.8337082862854,
+      "logps/chosen": -267.30694580078125,
+      "logps/rejected": -239.9588623046875,
+      "loss": 0.6057,
       "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.04662395641207695,
+      "rewards/margins": 0.1874973475933075,
+      "rewards/rejected": -0.14087337255477905,
       "step": 50
     },
     {
+      "epoch": 0.1534526854219949,
+      "grad_norm": 39.18274034042972,
+      "learning_rate": 9.92010391574745e-08,
+      "logits/chosen": -4.788964748382568,
+      "logits/rejected": -4.883444786071777,
+      "logps/chosen": -237.8981475830078,
+      "logps/rejected": -257.84942626953125,
+      "loss": 0.5174,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.03224308043718338,
+      "rewards/margins": 0.46052321791648865,
+      "rewards/rejected": -0.4282800555229187,
       "step": 60
     },
     {
+      "epoch": 0.17902813299232737,
+      "grad_norm": 34.832880831116846,
+      "learning_rate": 9.820833372667812e-08,
+      "logits/chosen": -4.657534599304199,
+      "logits/rejected": -4.817151069641113,
+      "logps/chosen": -249.996337890625,
+      "logps/rejected": -280.097412109375,
+      "loss": 0.4614,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 0.031456105411052704,
+      "rewards/margins": 0.6548057198524475,
+      "rewards/rejected": -0.6233495473861694,
       "step": 70
     },
     {
+      "epoch": 0.20460358056265984,
+      "grad_norm": 36.47722570862778,
+      "learning_rate": 9.682969016701356e-08,
+      "logits/chosen": -4.626967430114746,
+      "logits/rejected": -4.778214454650879,
+      "logps/chosen": -250.9975128173828,
+      "logps/rejected": -311.1219177246094,
+      "loss": 0.3904,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.10222460329532623,
+      "rewards/margins": 0.9102567434310913,
+      "rewards/rejected": -0.8080320358276367,
       "step": 80
     },
     {
+      "epoch": 0.23017902813299232,
+      "grad_norm": 30.998854450156045,
+      "learning_rate": 9.507614539004081e-08,
+      "logits/chosen": -4.739785194396973,
+      "logits/rejected": -4.909841060638428,
+      "logps/chosen": -237.671875,
+      "logps/rejected": -307.8204040527344,
+      "loss": 0.3509,
       "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 0.1354086697101593,
+      "rewards/margins": 1.1853126287460327,
+      "rewards/rejected": -1.0499038696289062,
       "step": 90
     },
     {
+      "epoch": 0.2557544757033248,
+      "grad_norm": 42.52785579314538,
+      "learning_rate": 9.296173762811083e-08,
+      "logits/chosen": -4.647661209106445,
+      "logits/rejected": -4.924945831298828,
+      "logps/chosen": -244.45303344726562,
+      "logps/rejected": -355.6828918457031,
+      "loss": 0.3413,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 0.12265179306268692,
+      "rewards/margins": 1.4083904027938843,
+      "rewards/rejected": -1.2857385873794556,
       "step": 100
     },
     {
+      "epoch": 0.2557544757033248,
+      "eval_logits/chosen": -4.821703910827637,
+      "eval_logits/rejected": -5.045117378234863,
+      "eval_logps/chosen": -444.5645751953125,
+      "eval_logps/rejected": -575.9554443359375,
+      "eval_loss": 0.7230384349822998,
+      "eval_rewards/accuracies": 0.515625,
+      "eval_rewards/chosen": -0.5409007668495178,
+      "eval_rewards/margins": 0.03477693349123001,
+      "eval_rewards/rejected": -0.575677752494812,
+      "eval_runtime": 98.6304,
+      "eval_samples_per_second": 20.278,
+      "eval_steps_per_second": 0.324,
       "step": 100
     },
     {
+      "epoch": 0.2813299232736573,
+      "grad_norm": 42.40417010662429,
+      "learning_rate": 9.050339404945832e-08,
+      "logits/chosen": -4.8084492683410645,
+      "logits/rejected": -5.027788162231445,
+      "logps/chosen": -247.86376953125,
+      "logps/rejected": -369.55267333984375,
+      "loss": 0.3143,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.01321962010115385,
+      "rewards/margins": 1.5418872833251953,
+      "rewards/rejected": -1.528667688369751,
       "step": 110
     },
     {
+      "epoch": 0.3069053708439898,
+      "grad_norm": 31.11429497548564,
+      "learning_rate": 8.77207952455395e-08,
+      "logits/chosen": -4.781357765197754,
+      "logits/rejected": -5.055319786071777,
+      "logps/chosen": -271.8451843261719,
+      "logps/rejected": -396.73046875,
+      "loss": 0.3042,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 0.041443757712841034,
+      "rewards/margins": 1.7226619720458984,
+      "rewards/rejected": -1.6812183856964111,
       "step": 120
     },
     {
+      "epoch": 0.33248081841432225,
+      "grad_norm": 32.05773581279916,
+      "learning_rate": 8.463621767547997e-08,
+      "logits/chosen": -4.876931190490723,
+      "logits/rejected": -5.202266693115234,
+      "logps/chosen": -264.2982482910156,
+      "logps/rejected": -409.0570983886719,
+      "loss": 0.2914,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.058543670922517776,
+      "rewards/margins": 1.8787403106689453,
+      "rewards/rejected": -1.937284231185913,
       "step": 130
     },
     {
+      "epoch": 0.35805626598465473,
+      "grad_norm": 31.96087329942538,
+      "learning_rate": 8.127435532896387e-08,
+      "logits/chosen": -4.971903324127197,
+      "logits/rejected": -5.277985095977783,
+      "logps/chosen": -305.4132385253906,
+      "logps/rejected": -457.46343994140625,
+      "loss": 0.274,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -0.2121816873550415,
+      "rewards/margins": 2.040717601776123,
+      "rewards/rejected": -2.252899408340454,
       "step": 140
     },
     {
+      "epoch": 0.3836317135549872,
+      "grad_norm": 40.46461234858551,
+      "learning_rate": 7.766212203526569e-08,
+      "logits/chosen": -5.087113857269287,
+      "logits/rejected": -5.368134498596191,
+      "logps/chosen": -274.01080322265625,
+      "logps/rejected": -457.38330078125,
+      "loss": 0.2819,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.2159254252910614,
+      "rewards/margins": 2.166714906692505,
+      "rewards/rejected": -2.3826401233673096,
       "step": 150
     },
     {
+      "epoch": 0.4092071611253197,
+      "grad_norm": 32.057320142788335,
+      "learning_rate": 7.382843600106538e-08,
+      "logits/chosen": -5.177260398864746,
+      "logits/rejected": -5.416450023651123,
+      "logps/chosen": -284.1901550292969,
+      "logps/rejected": -474.3257751464844,
+      "loss": 0.2436,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.3273366093635559,
+      "rewards/margins": 2.2598299980163574,
+      "rewards/rejected": -2.5871663093566895,
       "step": 160
     },
     {
+      "epoch": 0.43478260869565216,
+      "grad_norm": 33.151157821087715,
+      "learning_rate": 6.980398830195784e-08,
+      "logits/chosen": -5.109088897705078,
+      "logits/rejected": -5.438628196716309,
+      "logps/chosen": -296.1925964355469,
+      "logps/rejected": -516.4288940429688,
+      "loss": 0.2364,
       "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.3511837124824524,
+      "rewards/margins": 2.6150753498077393,
+      "rewards/rejected": -2.966259002685547,
       "step": 170
     },
     {
+      "epoch": 0.46035805626598464,
+      "grad_norm": 34.18806970089564,
+      "learning_rate": 6.562099718102787e-08,
+      "logits/chosen": -5.2773332595825195,
+      "logits/rejected": -5.568037509918213,
+      "logps/chosen": -284.951904296875,
+      "logps/rejected": -486.5365295410156,
+      "loss": 0.2628,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.480882465839386,
+      "rewards/margins": 2.4242804050445557,
+      "rewards/rejected": -2.9051625728607178,
       "step": 180
     },
     {
+      "epoch": 0.4859335038363171,
+      "grad_norm": 33.03269272782741,
+      "learning_rate": 6.131295012148612e-08,
+      "logits/chosen": -5.19248104095459,
+      "logits/rejected": -5.355208396911621,
+      "logps/chosen": -311.060791015625,
+      "logps/rejected": -542.6156005859375,
+      "loss": 0.2517,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -0.5016793012619019,
+      "rewards/margins": 2.5728163719177246,
+      "rewards/rejected": -3.074495792388916,
       "step": 190
     },
     {
+      "epoch": 0.5115089514066496,
+      "grad_norm": 40.925552268276135,
+      "learning_rate": 5.691433575823665e-08,
+      "logits/chosen": -5.236765384674072,
+      "logits/rejected": -5.465119361877441,
+      "logps/chosen": -302.1981201171875,
+      "logps/rejected": -515.2794799804688,
+      "loss": 0.2653,
       "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.5005888342857361,
+      "rewards/margins": 2.4987406730651855,
+      "rewards/rejected": -2.9993293285369873,
       "step": 200
     },
     {
+      "epoch": 0.5115089514066496,
+      "eval_logits/chosen": -5.226232528686523,
+      "eval_logits/rejected": -5.50424337387085,
+      "eval_logps/chosen": -540.43896484375,
+      "eval_logps/rejected": -679.8809814453125,
+      "eval_loss": 0.7765124440193176,
+      "eval_rewards/accuracies": 0.54296875,
+      "eval_rewards/chosen": -1.4996453523635864,
+      "eval_rewards/margins": 0.11528739333152771,
+      "eval_rewards/rejected": -1.6149327754974365,
+      "eval_runtime": 98.5941,
+      "eval_samples_per_second": 20.285,
+      "eval_steps_per_second": 0.325,
       "step": 200
     },
     {
+      "epoch": 0.5370843989769821,
+      "grad_norm": 26.659672604447973,
+      "learning_rate": 5.2460367774593905e-08,
+      "logits/chosen": -5.310137748718262,
+      "logits/rejected": -5.583542823791504,
+      "logps/chosen": -303.67047119140625,
+      "logps/rejected": -573.1016845703125,
+      "loss": 0.2296,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.37785404920578003,
+      "rewards/margins": 3.0667028427124023,
+      "rewards/rejected": -3.444556713104248,
       "step": 210
     },
     {
+      "epoch": 0.5626598465473146,
+      "grad_norm": 35.355172011912686,
+      "learning_rate": 4.798670299452925e-08,
+      "logits/chosen": -5.1389665603637695,
+      "logits/rejected": -5.567061424255371,
+      "logps/chosen": -304.0540466308594,
+      "logps/rejected": -569.4851684570312,
+      "loss": 0.245,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.41242700815200806,
+      "rewards/margins": 3.0356929302215576,
+      "rewards/rejected": -3.4481201171875,
       "step": 220
     },
     {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 37.35765448344736,
+      "learning_rate": 4.3529155927297226e-08,
+      "logits/chosen": -5.210625648498535,
+      "logits/rejected": -5.601117134094238,
+      "logps/chosen": -323.33135986328125,
+      "logps/rejected": -591.130126953125,
+      "loss": 0.2477,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.623622715473175,
+      "rewards/margins": 3.050567150115967,
+      "rewards/rejected": -3.674190044403076,
       "step": 230
     },
     {
+      "epoch": 0.6138107416879796,
+      "grad_norm": 27.168387739658527,
+      "learning_rate": 3.9123412049691636e-08,
+      "logits/chosen": -5.26107120513916,
+      "logits/rejected": -5.582613945007324,
+      "logps/chosen": -341.65289306640625,
+      "logps/rejected": -593.1688232421875,
+      "loss": 0.2349,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.68829745054245,
+      "rewards/margins": 3.034055233001709,
+      "rewards/rejected": -3.7223525047302246,
       "step": 240
     },
     {
+      "epoch": 0.639386189258312,
+      "grad_norm": 34.59601076495169,
+      "learning_rate": 3.480474212128766e-08,
+      "logits/chosen": -5.441601753234863,
+      "logits/rejected": -5.72822380065918,
+      "logps/chosen": -329.5417175292969,
+      "logps/rejected": -537.7394409179688,
+      "loss": 0.2339,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.8084025382995605,
+      "rewards/margins": 2.4929001331329346,
+      "rewards/rejected": -3.301302433013916,
       "step": 250
     },
     {
+      "epoch": 0.6649616368286445,
+      "grad_norm": 44.5395657806438,
+      "learning_rate": 3.060771981975726e-08,
+      "logits/chosen": -5.302738666534424,
+      "logits/rejected": -5.622676372528076,
+      "logps/chosen": -326.24041748046875,
+      "logps/rejected": -637.6575927734375,
+      "loss": 0.2325,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.7219182848930359,
+      "rewards/margins": 3.4571731090545654,
+      "rewards/rejected": -4.179091453552246,
       "step": 260
     },
     {
+      "epoch": 0.690537084398977,
+      "grad_norm": 33.64914034772639,
+      "learning_rate": 2.6565944956764818e-08,
+      "logits/chosen": -5.4421281814575195,
+      "logits/rejected": -5.695931911468506,
+      "logps/chosen": -332.70892333984375,
+      "logps/rejected": -598.5055541992188,
+      "loss": 0.2433,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -0.7167563438415527,
+      "rewards/margins": 3.118049144744873,
+      "rewards/rejected": -3.834805727005005,
       "step": 270
     },
     {
+      "epoch": 0.7161125319693095,
+      "grad_norm": 38.28164920230575,
+      "learning_rate": 2.2711774490274766e-08,
+      "logits/chosen": -5.344332695007324,
+      "logits/rejected": -5.591184616088867,
+      "logps/chosen": -331.06939697265625,
+      "logps/rejected": -640.959228515625,
+      "loss": 0.2174,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.665625810623169,
+      "rewards/margins": 3.3145720958709717,
+      "rewards/rejected": -3.9801979064941406,
       "step": 280
     },
     {
+      "epoch": 0.7416879795396419,
+      "grad_norm": 42.555865291815444,
+      "learning_rate": 1.9076063486687256e-08,
+      "logits/chosen": -5.223475933074951,
+      "logits/rejected": -5.618660926818848,
+      "logps/chosen": -328.63055419921875,
+      "logps/rejected": -579.0905151367188,
+      "loss": 0.2228,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -0.5602216124534607,
+      "rewards/margins": 3.074389696121216,
+      "rewards/rejected": -3.634611129760742,
       "step": 290
     },
     {
+      "epoch": 0.7672634271099744,
+      "grad_norm": 40.820437800178965,
+      "learning_rate": 1.5687918106563324e-08,
+      "logits/chosen": -5.369271755218506,
+      "logits/rejected": -5.632781028747559,
+      "logps/chosen": -320.268798828125,
+      "logps/rejected": -608.9943237304688,
+      "loss": 0.2424,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.6686061024665833,
+      "rewards/margins": 3.2616829872131348,
+      "rewards/rejected": -3.9302895069122314,
       "step": 300
     },
     {
+      "epoch": 0.7672634271099744,
+      "eval_logits/chosen": -5.394677639007568,
+      "eval_logits/rejected": -5.655616283416748,
+      "eval_logps/chosen": -579.441650390625,
+      "eval_logps/rejected": -718.423828125,
+      "eval_loss": 0.8002758622169495,
+      "eval_rewards/accuracies": 0.52734375,
+      "eval_rewards/chosen": -1.889671802520752,
+      "eval_rewards/margins": 0.11068924516439438,
+      "eval_rewards/rejected": -2.000361442565918,
+      "eval_runtime": 98.5861,
+      "eval_samples_per_second": 20.287,
+      "eval_steps_per_second": 0.325,
       "step": 300
     },
     {
+      "epoch": 0.7928388746803069,
+      "grad_norm": 33.64379879568246,
+      "learning_rate": 1.257446259144494e-08,
+      "logits/chosen": -5.246872425079346,
+      "logits/rejected": -5.653367042541504,
+      "logps/chosen": -315.7105407714844,
+      "logps/rejected": -625.9619140625,
+      "loss": 0.2323,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -0.5457090735435486,
+      "rewards/margins": 3.491931200027466,
+      "rewards/rejected": -4.03764009475708,
       "step": 310
     },
     {
+      "epoch": 0.8184143222506394,
+      "grad_norm": 35.35694379401523,
+      "learning_rate": 9.760622117187234e-09,
+      "logits/chosen": -5.381436824798584,
+      "logits/rejected": -5.7473673820495605,
+      "logps/chosen": -314.6531677246094,
+      "logps/rejected": -594.841552734375,
+      "loss": 0.2466,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.6948888897895813,
+      "rewards/margins": 3.1890125274658203,
+      "rewards/rejected": -3.8839008808135986,
       "step": 320
     },
     {
+      "epoch": 0.8439897698209718,
+      "grad_norm": 34.515465680243125,
+      "learning_rate": 7.2689232521989885e-09,
+      "logits/chosen": -5.308783531188965,
+      "logits/rejected": -5.656357765197754,
+      "logps/chosen": -347.4857482910156,
+      "logps/rejected": -629.4615478515625,
+      "loss": 0.2233,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.7933691143989563,
+      "rewards/margins": 3.1742498874664307,
+      "rewards/rejected": -3.967618942260742,
       "step": 330
     },
     {
+      "epoch": 0.8695652173913043,
+      "grad_norm": 28.542655038843865,
+      "learning_rate": 5.119313618049309e-09,
+      "logits/chosen": -5.346091270446777,
+      "logits/rejected": -5.756931781768799,
+      "logps/chosen": -338.65509033203125,
+      "logps/rejected": -575.4979858398438,
+      "loss": 0.2174,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.6450907588005066,
+      "rewards/margins": 3.0420687198638916,
+      "rewards/rejected": -3.687159299850464,
       "step": 340
     },
     {
+      "epoch": 0.8951406649616368,
+      "grad_norm": 33.42105425863571,
+      "learning_rate": 3.3290021961708158e-09,
+      "logits/chosen": -5.374421119689941,
+      "logits/rejected": -5.536851406097412,
+      "logps/chosen": -333.8661193847656,
+      "logps/rejected": -595.2741088867188,
+      "loss": 0.2467,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.7839492559432983,
+      "rewards/margins": 2.891091823577881,
+      "rewards/rejected": -3.6750411987304688,
       "step": 350
     },
     {
+      "epoch": 0.9207161125319693,
+      "grad_norm": 33.175441995042306,
+      "learning_rate": 1.9123215591052013e-09,
+      "logits/chosen": -5.3232526779174805,
+      "logits/rejected": -5.559803485870361,
+      "logps/chosen": -337.17694091796875,
+      "logps/rejected": -596.7660522460938,
+      "loss": 0.2397,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.7554206252098083,
+      "rewards/margins": 2.937615156173706,
+      "rewards/rejected": -3.693035840988159,
       "step": 360
     },
     {
+      "epoch": 0.9462915601023018,
+      "grad_norm": 33.50889046296721,
+      "learning_rate": 8.806131292167618e-10,
+      "logits/chosen": -5.363125801086426,
+      "logits/rejected": -5.561426162719727,
+      "logps/chosen": -327.09295654296875,
+      "logps/rejected": -608.7786865234375,
+      "loss": 0.238,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.7027177214622498,
+      "rewards/margins": 3.01659893989563,
+      "rewards/rejected": -3.719316005706787,
       "step": 370
     },
     {
+      "epoch": 0.9718670076726342,
+      "grad_norm": 42.30140132740828,
+      "learning_rate": 2.4213638345040867e-10,
+      "logits/chosen": -5.489308834075928,
+      "logits/rejected": -5.787456512451172,
+      "logps/chosen": -332.35858154296875,
+      "logps/rejected": -607.3480224609375,
+      "loss": 0.2341,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -0.7027586698532104,
+      "rewards/margins": 3.165475368499756,
+      "rewards/rejected": -3.8682339191436768,
       "step": 380
     },
     {
+      "epoch": 0.9974424552429667,
+      "grad_norm": 37.88179259111206,
+      "learning_rate": 2.0027310073833516e-12,
+      "logits/chosen": -5.485334873199463,
+      "logits/rejected": -5.764852046966553,
+      "logps/chosen": -331.56610107421875,
+      "logps/rejected": -614.2138061523438,
+      "loss": 0.2223,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.7189357876777649,
+      "rewards/margins": 3.2180511951446533,
+      "rewards/rejected": -3.9369864463806152,
       "step": 390
     },
     {
       "epoch": 1.0,
       "step": 391,
       "total_flos": 0.0,
+      "train_loss": 0.3220548828315857,
+      "train_runtime": 6253.066,
+      "train_samples_per_second": 7.996,
+      "train_steps_per_second": 0.063
     }
   ],
   "logging_steps": 10,

training_args.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2ba26fafd03dc99e7d3db95db6c6e4c6d32ca5324c807136077205b8734c2180
-size 6264

 version https://git-lfs.github.com/spec/v1
+oid sha256:233762ce2f7a1cfa30251c51e84fc095bdb7579faa7bbc23d8ce39a8e51f366d
+size 6328