Model save

README.md

@@ -17,15 +17,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.0194
-- Rewards/chosen: -1.8199
-- Rewards/rejected: -2.5238
+- Loss: 0.0247
+- Rewards/chosen: -1.3723
+- Rewards/rejected: -2.0629
 - Rewards/accuracies: 0.7031
-- Rewards/margins: 0.7039
-- Logps/rejected: -509.7351
-- Logps/chosen: -439.0294
-- Logits/rejected: -2.2378
-- Logits/chosen: -2.2625
+- Rewards/margins: 0.6905
+- Logps/rejected: -463.6402
+- Logps/chosen: -394.2719
+- Logits/rejected: -2.4588
+- Logits/chosen: -2.4710
 
 ## Model description
 
@@ -47,7 +47,7 @@ The following hyperparameters were used during training:
 - learning_rate: 5e-07
 - train_batch_size: 8
 - eval_batch_size: 8
-- seed: 2
+- seed: 3
 - distributed_type: multi-GPU
 - num_devices: 8
 - gradient_accumulation_steps: 2
@@ -62,10 +62,10 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.0502        | 0.21  | 100  | 0.0391          | -0.7692        | -1.1939          | 0.6914             | 0.4246          | -376.7401      | -333.9633    | -2.7065         | -2.7235       |
-| 0.0237        | 0.42  | 200  | 0.0268          | -1.2426        | -1.8386          | 0.6680             | 0.5960          | -441.2099      | -381.2964    | -2.4965         | -2.5105       |
-| 0.0248        | 0.63  | 300  | 0.0254          | -1.3833        | -2.0353          | 0.6914             | 0.6521          | -460.8870      | -395.3680    | -2.3196         | -2.3397       |
-| 0.0194        | 0.84  | 400  | 0.0194          | -1.8199        | -2.5238          | 0.7031             | 0.7039          | -509.7351      | -439.0294    | -2.2378         | -2.2625       |
+| 0.0464        | 0.21  | 100  | 0.0506          | -0.5333        | -0.9231          | 0.6953             | 0.3898          | -349.6644      | -310.3744    | -2.8682         | -2.8854       |
+| 0.0311        | 0.42  | 200  | 0.0300          | -1.0702        | -1.6770          | 0.7070             | 0.6068          | -425.0558      | -364.0610    | -2.6481         | -2.6609       |
+| 0.0213        | 0.63  | 300  | 0.0222          | -1.5349        | -2.2068          | 0.6992             | 0.6719          | -478.0340      | -410.5276    | -2.5034         | -2.5167       |
+| 0.0241        | 0.84  | 400  | 0.0247          | -1.3723        | -2.0629          | 0.7031             | 0.6905          | -463.6402      | -394.2719    | -2.4588         | -2.4710       |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.03868693582495386,
-    "train_runtime": 4353.4885,
+    "train_loss": 0.03922173811052634,
+    "train_runtime": 4355.9498,
     "train_samples": 61134,
-    "train_samples_per_second": 14.043,
+    "train_samples_per_second": 14.035,
     "train_steps_per_second": 0.11
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:472b04e0cc50bda621fe5cbd642762d1767be557c1f585354c28a09172156977
+oid sha256:e102f6e41eeedda46ea8fa8b45653a0c40d55a5b79dd8aa2a7cfb1c15d9bf062
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aea925956bfb36d8ce6bf0529378c0a70c0af339b58425c70ea48f9146d63695
+oid sha256:90f5cb81b2cc945396ef323471b1a17835c064e85a1024b470b61fc36a1a6730
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:70949f8d5a33dbbc2818f352d2585dd202e52822e017939aa5d1bc67dd71cac9
+oid sha256:5cb938d285713cc1c1a09480fb89fd5b2eb7d3847d7525197bf7129d3bbc1911
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.03868693582495386,
-    "train_runtime": 4353.4885,
+    "train_loss": 0.03922173811052634,
+    "train_runtime": 4355.9498,
     "train_samples": 61134,
-    "train_samples_per_second": 14.043,
+    "train_samples_per_second": 14.035,
     "train_steps_per_second": 0.11
 }

trainer_state.json

@@ -11,11 +11,11 @@
     {
       "epoch": 0.0,
       "learning_rate": 1.0416666666666666e-08,
-      "logits/chosen": -2.7050857543945312,
-      "logits/rejected": -2.7461352348327637,
-      "logps/chosen": -137.25845336914062,
-      "logps/rejected": -163.38693237304688,
-      "loss": 0.0968,
+      "logits/chosen": -2.7515339851379395,
+      "logits/rejected": -2.7739691734313965,
+      "logps/chosen": -226.12843322753906,
+      "logps/rejected": -195.60536193847656,
+      "loss": 0.1023,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,732 +25,732 @@
     {
       "epoch": 0.02,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -2.731086015701294,
-      "logits/rejected": -2.69549298286438,
-      "logps/chosen": -255.12014770507812,
-      "logps/rejected": -238.85885620117188,
-      "loss": 0.1059,
-      "rewards/accuracies": 0.4722222089767456,
-      "rewards/chosen": 0.00021500665752682835,
-      "rewards/margins": 0.00010767025378299877,
-      "rewards/rejected": 0.00010733638919191435,
+      "logits/chosen": -2.7851314544677734,
+      "logits/rejected": -2.7735650539398193,
+      "logps/chosen": -296.5018005371094,
+      "logps/rejected": -257.1644287109375,
+      "loss": 0.1034,
+      "rewards/accuracies": 0.4583333432674408,
+      "rewards/chosen": 1.3278131518745795e-05,
+      "rewards/margins": 0.0003422351146582514,
+      "rewards/rejected": -0.00032895695767365396,
       "step": 10
     },
     {
       "epoch": 0.04,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -2.7516608238220215,
-      "logits/rejected": -2.7480316162109375,
-      "logps/chosen": -265.412109375,
-      "logps/rejected": -255.61865234375,
-      "loss": 0.1048,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -4.060972059960477e-05,
-      "rewards/margins": 0.0010573023464530706,
-      "rewards/rejected": -0.0010979120852425694,
+      "logits/chosen": -2.780992031097412,
+      "logits/rejected": -2.7528281211853027,
+      "logps/chosen": -267.0114440917969,
+      "logps/rejected": -264.90545654296875,
+      "loss": 0.1026,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.00017318663594778627,
+      "rewards/margins": 0.0009094638517126441,
+      "rewards/rejected": -0.0007362772594206035,
       "step": 20
     },
     {
       "epoch": 0.06,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -2.741386651992798,
-      "logits/rejected": -2.698482036590576,
-      "logps/chosen": -253.40029907226562,
-      "logps/rejected": -252.72500610351562,
-      "loss": 0.1012,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.0009587205713614821,
-      "rewards/margins": 0.009668193757534027,
-      "rewards/rejected": -0.010626914910972118,
+      "logits/chosen": -2.7856814861297607,
+      "logits/rejected": -2.759723663330078,
+      "logps/chosen": -246.8136444091797,
+      "logps/rejected": -251.91244506835938,
+      "loss": 0.1036,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0007418844616040587,
+      "rewards/margins": 0.006723286118358374,
+      "rewards/rejected": -0.00598140200600028,
       "step": 30
     },
     {
       "epoch": 0.08,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.792792797088623,
-      "logits/rejected": -2.778289556503296,
-      "logps/chosen": -274.712158203125,
-      "logps/rejected": -279.0121154785156,
-      "loss": 0.1022,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.0003971463011112064,
-      "rewards/margins": 0.02575758472084999,
-      "rewards/rejected": -0.02615473046898842,
+      "logits/chosen": -2.781876564025879,
+      "logits/rejected": -2.738236427307129,
+      "logps/chosen": -276.0296936035156,
+      "logps/rejected": -258.5984802246094,
+      "loss": 0.1004,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.007513026241213083,
+      "rewards/margins": 0.010224683210253716,
+      "rewards/rejected": -0.01773770898580551,
       "step": 40
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999733114418725e-07,
-      "logits/chosen": -2.760355234146118,
-      "logits/rejected": -2.7391133308410645,
-      "logps/chosen": -286.1962890625,
-      "logps/rejected": -295.90069580078125,
-      "loss": 0.0957,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.041570521891117096,
-      "rewards/margins": 0.03570377081632614,
-      "rewards/rejected": -0.07727430015802383,
+      "logits/chosen": -2.779130458831787,
+      "logits/rejected": -2.769984006881714,
+      "logps/chosen": -309.8009338378906,
+      "logps/rejected": -307.19244384765625,
+      "loss": 0.0954,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.01474276464432478,
+      "rewards/margins": 0.05062633752822876,
+      "rewards/rejected": -0.06536909192800522,
       "step": 50
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.990398100856366e-07,
-      "logits/chosen": -2.776111125946045,
-      "logits/rejected": -2.7512311935424805,
-      "logps/chosen": -276.09942626953125,
-      "logps/rejected": -258.3904724121094,
-      "loss": 0.0889,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.06735710799694061,
-      "rewards/margins": 0.1039455384016037,
-      "rewards/rejected": -0.1713026612997055,
+      "logits/chosen": -2.7530159950256348,
+      "logits/rejected": -2.7202906608581543,
+      "logps/chosen": -291.90826416015625,
+      "logps/rejected": -281.79998779296875,
+      "loss": 0.0857,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.08949585258960724,
+      "rewards/margins": 0.05581193417310715,
+      "rewards/rejected": -0.14530780911445618,
       "step": 60
     },
     {
       "epoch": 0.15,
       "learning_rate": 4.967775735898179e-07,
-      "logits/chosen": -2.768890857696533,
-      "logits/rejected": -2.7490429878234863,
-      "logps/chosen": -270.66156005859375,
-      "logps/rejected": -301.940185546875,
-      "loss": 0.0783,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.15578190982341766,
-      "rewards/margins": 0.11446709930896759,
-      "rewards/rejected": -0.27024897933006287,
+      "logits/chosen": -2.755706310272217,
+      "logits/rejected": -2.7257895469665527,
+      "logps/chosen": -273.61529541015625,
+      "logps/rejected": -275.41033935546875,
+      "loss": 0.0748,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.20192591845989227,
+      "rewards/margins": 0.10217342525720596,
+      "rewards/rejected": -0.30409935116767883,
       "step": 70
     },
     {
       "epoch": 0.17,
       "learning_rate": 4.931986719649298e-07,
-      "logits/chosen": -2.7411890029907227,
-      "logits/rejected": -2.71295428276062,
-      "logps/chosen": -270.99969482421875,
-      "logps/rejected": -290.6012878417969,
-      "loss": 0.0734,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.17659564316272736,
-      "rewards/margins": 0.2667813003063202,
-      "rewards/rejected": -0.44337692856788635,
+      "logits/chosen": -2.784632921218872,
+      "logits/rejected": -2.7770540714263916,
+      "logps/chosen": -293.8019104003906,
+      "logps/rejected": -307.91998291015625,
+      "loss": 0.0608,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.3351340591907501,
+      "rewards/margins": 0.2123802900314331,
+      "rewards/rejected": -0.5475143194198608,
       "step": 80
     },
     {
       "epoch": 0.19,
       "learning_rate": 4.883222001996351e-07,
-      "logits/chosen": -2.7742576599121094,
-      "logits/rejected": -2.7353005409240723,
-      "logps/chosen": -315.1212463378906,
-      "logps/rejected": -318.26153564453125,
-      "loss": 0.0628,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.37408870458602905,
-      "rewards/margins": 0.32524389028549194,
-      "rewards/rejected": -0.699332594871521,
+      "logits/chosen": -2.818525791168213,
+      "logits/rejected": -2.8273603916168213,
+      "logps/chosen": -288.95440673828125,
+      "logps/rejected": -299.98297119140625,
+      "loss": 0.0557,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.4987950325012207,
+      "rewards/margins": 0.29060596227645874,
+      "rewards/rejected": -0.7894009947776794,
       "step": 90
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.821741763807186e-07,
-      "logits/chosen": -2.7804133892059326,
-      "logits/rejected": -2.755944013595581,
-      "logps/chosen": -322.177978515625,
-      "logps/rejected": -350.9091796875,
-      "loss": 0.0502,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.5584506988525391,
-      "rewards/margins": 0.32031145691871643,
-      "rewards/rejected": -0.8787621259689331,
+      "logits/chosen": -2.8322081565856934,
+      "logits/rejected": -2.8068549633026123,
+      "logps/chosen": -305.04168701171875,
+      "logps/rejected": -348.1548156738281,
+      "loss": 0.0464,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.6313897967338562,
+      "rewards/margins": 0.2933662533760071,
+      "rewards/rejected": -0.9247561693191528,
       "step": 100
     },
     {
       "epoch": 0.21,
-      "eval_logits/chosen": -2.7234888076782227,
-      "eval_logits/rejected": -2.7065277099609375,
-      "eval_logps/chosen": -333.9632873535156,
-      "eval_logps/rejected": -376.7401428222656,
-      "eval_loss": 0.03910641372203827,
-      "eval_rewards/accuracies": 0.69140625,
-      "eval_rewards/chosen": -0.76923668384552,
-      "eval_rewards/margins": 0.4246319532394409,
-      "eval_rewards/rejected": -1.193868637084961,
-      "eval_runtime": 53.2662,
-      "eval_samples_per_second": 37.547,
+      "eval_logits/chosen": -2.8853633403778076,
+      "eval_logits/rejected": -2.868199348449707,
+      "eval_logps/chosen": -310.3743896484375,
+      "eval_logps/rejected": -349.66436767578125,
+      "eval_loss": 0.05058940127491951,
+      "eval_rewards/accuracies": 0.6953125,
+      "eval_rewards/chosen": -0.5333478450775146,
+      "eval_rewards/margins": 0.389763206243515,
+      "eval_rewards/rejected": -0.923111081123352,
+      "eval_runtime": 53.2701,
+      "eval_samples_per_second": 37.545,
       "eval_steps_per_second": 0.601,
       "step": 100
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.747874028753375e-07,
-      "logits/chosen": -2.6888976097106934,
-      "logits/rejected": -2.650428056716919,
-      "logps/chosen": -366.8619079589844,
-      "logps/rejected": -376.2767028808594,
-      "loss": 0.0371,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.7290430665016174,
-      "rewards/margins": 0.3944670259952545,
-      "rewards/rejected": -1.1235100030899048,
+      "logits/chosen": -2.8740365505218506,
+      "logits/rejected": -2.8250865936279297,
+      "logps/chosen": -330.26556396484375,
+      "logps/rejected": -312.4544372558594,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.6077455282211304,
+      "rewards/margins": 0.28130462765693665,
+      "rewards/rejected": -0.889050304889679,
       "step": 110
     },
     {
       "epoch": 0.25,
       "learning_rate": 4.662012913161997e-07,
-      "logits/chosen": -2.733076810836792,
-      "logits/rejected": -2.6883604526519775,
-      "logps/chosen": -342.3517150878906,
-      "logps/rejected": -385.7752990722656,
-      "loss": 0.0398,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.7808613181114197,
-      "rewards/margins": 0.501408040523529,
-      "rewards/rejected": -1.2822694778442383,
+      "logits/chosen": -2.8701424598693848,
+      "logits/rejected": -2.8420445919036865,
+      "logps/chosen": -354.23553466796875,
+      "logps/rejected": -347.61614990234375,
+      "loss": 0.0375,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.7496905326843262,
+      "rewards/margins": 0.3611409664154053,
+      "rewards/rejected": -1.1108314990997314,
       "step": 120
     },
     {
       "epoch": 0.27,
       "learning_rate": 4.5646165232345103e-07,
-      "logits/chosen": -2.709172248840332,
-      "logits/rejected": -2.6797726154327393,
-      "logps/chosen": -348.2884216308594,
-      "logps/rejected": -364.2542724609375,
-      "loss": 0.0333,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.8973625898361206,
-      "rewards/margins": 0.41800689697265625,
-      "rewards/rejected": -1.3153694868087769,
+      "logits/chosen": -2.843061923980713,
+      "logits/rejected": -2.8619065284729004,
+      "logps/chosen": -348.7552795410156,
+      "logps/rejected": -376.20318603515625,
+      "loss": 0.0402,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.7844843864440918,
+      "rewards/margins": 0.3621848225593567,
+      "rewards/rejected": -1.1466691493988037,
       "step": 130
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.456204510851956e-07,
-      "logits/chosen": -2.6971547603607178,
-      "logits/rejected": -2.6667492389678955,
-      "logps/chosen": -369.86016845703125,
-      "logps/rejected": -398.45721435546875,
-      "loss": 0.0363,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.8229220509529114,
-      "rewards/margins": 0.38307899236679077,
-      "rewards/rejected": -1.2060010433197021,
+      "logits/chosen": -2.8098416328430176,
+      "logits/rejected": -2.7964279651641846,
+      "logps/chosen": -330.7640075683594,
+      "logps/rejected": -352.5660705566406,
+      "loss": 0.0395,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.6989365816116333,
+      "rewards/margins": 0.326722115278244,
+      "rewards/rejected": -1.0256588459014893,
       "step": 140
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.337355301007335e-07,
-      "logits/chosen": -2.5674502849578857,
-      "logits/rejected": -2.5130200386047363,
-      "logps/chosen": -363.6653747558594,
-      "logps/rejected": -388.075439453125,
-      "loss": 0.0322,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.874055027961731,
-      "rewards/margins": 0.542679488658905,
-      "rewards/rejected": -1.4167345762252808,
+      "logits/chosen": -2.8092267513275146,
+      "logits/rejected": -2.7995684146881104,
+      "logps/chosen": -340.68218994140625,
+      "logps/rejected": -377.41815185546875,
+      "loss": 0.0348,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.8084400296211243,
+      "rewards/margins": 0.5104471445083618,
+      "rewards/rejected": -1.3188872337341309,
       "step": 150
     },
     {
       "epoch": 0.33,
       "learning_rate": 4.2087030056579986e-07,
-      "logits/chosen": -2.5751426219940186,
-      "logits/rejected": -2.5513062477111816,
-      "logps/chosen": -367.0979919433594,
-      "logps/rejected": -382.1893615722656,
-      "loss": 0.0365,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.9746230840682983,
-      "rewards/margins": 0.31770768761634827,
-      "rewards/rejected": -1.2923307418823242,
+      "logits/chosen": -2.7137389183044434,
+      "logits/rejected": -2.656611919403076,
+      "logps/chosen": -353.7635803222656,
+      "logps/rejected": -371.21478271484375,
+      "loss": 0.0413,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.7515512108802795,
+      "rewards/margins": 0.42916202545166016,
+      "rewards/rejected": -1.1807130575180054,
       "step": 160
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.070934040463998e-07,
-      "logits/chosen": -2.5913219451904297,
-      "logits/rejected": -2.5757110118865967,
-      "logps/chosen": -371.7869567871094,
-      "logps/rejected": -361.24609375,
-      "loss": 0.0393,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.6607332825660706,
-      "rewards/margins": 0.31646376848220825,
-      "rewards/rejected": -0.977196991443634,
+      "logits/chosen": -2.711103916168213,
+      "logits/rejected": -2.6679797172546387,
+      "logps/chosen": -371.1010437011719,
+      "logps/rejected": -426.967041015625,
+      "loss": 0.0336,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.0021672248840332,
+      "rewards/margins": 0.5754579901695251,
+      "rewards/rejected": -1.5776251554489136,
       "step": 170
     },
     {
       "epoch": 0.38,
       "learning_rate": 3.9247834624635404e-07,
-      "logits/chosen": -2.561448812484741,
-      "logits/rejected": -2.5715978145599365,
-      "logps/chosen": -352.5904846191406,
-      "logps/rejected": -386.1707763671875,
-      "loss": 0.0355,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.8926581144332886,
-      "rewards/margins": 0.39244139194488525,
-      "rewards/rejected": -1.2850996255874634,
+      "logits/chosen": -2.6752336025238037,
+      "logits/rejected": -2.659677743911743,
+      "logps/chosen": -407.52606201171875,
+      "logps/rejected": -394.50433349609375,
+      "loss": 0.0335,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.046008825302124,
+      "rewards/margins": 0.40840545296669006,
+      "rewards/rejected": -1.4544141292572021,
       "step": 180
     },
     {
       "epoch": 0.4,
       "learning_rate": 3.7710310482256523e-07,
-      "logits/chosen": -2.510988473892212,
-      "logits/rejected": -2.493633985519409,
-      "logps/chosen": -343.94232177734375,
-      "logps/rejected": -387.73297119140625,
-      "loss": 0.0284,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.0100765228271484,
-      "rewards/margins": 0.5000672340393066,
-      "rewards/rejected": -1.5101438760757446,
+      "logits/chosen": -2.6721465587615967,
+      "logits/rejected": -2.642923593521118,
+      "logps/chosen": -363.59368896484375,
+      "logps/rejected": -352.25311279296875,
+      "loss": 0.0312,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.9571406245231628,
+      "rewards/margins": 0.4024307131767273,
+      "rewards/rejected": -1.3595713376998901,
       "step": 190
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.610497133404795e-07,
-      "logits/chosen": -2.456496238708496,
-      "logits/rejected": -2.442700147628784,
-      "logps/chosen": -379.01141357421875,
-      "logps/rejected": -432.1864318847656,
-      "loss": 0.0237,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -1.3147755861282349,
-      "rewards/margins": 0.4870442748069763,
-      "rewards/rejected": -1.8018200397491455,
+      "logits/chosen": -2.640389919281006,
+      "logits/rejected": -2.5935587882995605,
+      "logps/chosen": -369.52587890625,
+      "logps/rejected": -404.99945068359375,
+      "loss": 0.0311,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.0596716403961182,
+      "rewards/margins": 0.49687033891677856,
+      "rewards/rejected": -1.5565420389175415,
       "step": 200
     },
     {
       "epoch": 0.42,
-      "eval_logits/chosen": -2.5105109214782715,
-      "eval_logits/rejected": -2.496497392654419,
-      "eval_logps/chosen": -381.2964172363281,
-      "eval_logps/rejected": -441.2099304199219,
-      "eval_loss": 0.026758365333080292,
-      "eval_rewards/accuracies": 0.66796875,
-      "eval_rewards/chosen": -1.242567539215088,
-      "eval_rewards/margins": 0.5959988832473755,
-      "eval_rewards/rejected": -1.8385663032531738,
-      "eval_runtime": 53.1945,
-      "eval_samples_per_second": 37.598,
-      "eval_steps_per_second": 0.602,
+      "eval_logits/chosen": -2.660949230194092,
+      "eval_logits/rejected": -2.648130416870117,
+      "eval_logps/chosen": -364.06097412109375,
+      "eval_logps/rejected": -425.0558166503906,
+      "eval_loss": 0.029988564550876617,
+      "eval_rewards/accuracies": 0.70703125,
+      "eval_rewards/chosen": -1.0702130794525146,
+      "eval_rewards/margins": 0.6068122982978821,
+      "eval_rewards/rejected": -1.6770254373550415,
+      "eval_runtime": 53.293,
+      "eval_samples_per_second": 37.528,
+      "eval_steps_per_second": 0.6,
       "step": 200
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4440382358952115e-07,
-      "logits/chosen": -2.464259624481201,
-      "logits/rejected": -2.4774632453918457,
-      "logps/chosen": -408.1732482910156,
-      "logps/rejected": -461.95947265625,
-      "loss": 0.0259,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.0447111129760742,
-      "rewards/margins": 0.6814562678337097,
-      "rewards/rejected": -1.7261674404144287,
+      "logits/chosen": -2.5870018005371094,
+      "logits/rejected": -2.529308557510376,
+      "logps/chosen": -375.44476318359375,
+      "logps/rejected": -419.3675842285156,
+      "loss": 0.0237,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.068831205368042,
+      "rewards/margins": 0.7056142091751099,
+      "rewards/rejected": -1.7744452953338623,
       "step": 210
     },
     {
       "epoch": 0.46,
       "learning_rate": 3.272542485937368e-07,
-      "logits/chosen": -2.454061985015869,
-      "logits/rejected": -2.4306459426879883,
-      "logps/chosen": -375.48541259765625,
-      "logps/rejected": -397.7698059082031,
-      "loss": 0.0304,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.1511690616607666,
-      "rewards/margins": 0.46213898062705994,
-      "rewards/rejected": -1.6133081912994385,
+      "logits/chosen": -2.566018581390381,
+      "logits/rejected": -2.5462303161621094,
+      "logps/chosen": -344.32403564453125,
+      "logps/rejected": -405.7695007324219,
+      "loss": 0.0282,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.0890238285064697,
+      "rewards/margins": 0.6549443006515503,
+      "rewards/rejected": -1.7439680099487305,
       "step": 220
     },
     {
       "epoch": 0.48,
       "learning_rate": 3.096924887558854e-07,
-      "logits/chosen": -2.4601683616638184,
-      "logits/rejected": -2.445401430130005,
-      "logps/chosen": -377.16510009765625,
-      "logps/rejected": -408.50640869140625,
-      "loss": 0.0273,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.183936595916748,
-      "rewards/margins": 0.452188640832901,
-      "rewards/rejected": -1.6361253261566162,
+      "logits/chosen": -2.584099531173706,
+      "logits/rejected": -2.57403826713562,
+      "logps/chosen": -390.76983642578125,
+      "logps/rejected": -428.17291259765625,
+      "loss": 0.0283,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.1449739933013916,
+      "rewards/margins": 0.5948079824447632,
+      "rewards/rejected": -1.7397819757461548,
       "step": 230
     },
     {
       "epoch": 0.5,
       "learning_rate": 2.9181224366319943e-07,
-      "logits/chosen": -2.386798620223999,
-      "logits/rejected": -2.3921284675598145,
-      "logps/chosen": -433.074951171875,
-      "logps/rejected": -453.2076110839844,
-      "loss": 0.0283,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -1.21742844581604,
-      "rewards/margins": 0.5803453326225281,
-      "rewards/rejected": -1.7977739572525024,
+      "logits/chosen": -2.61647629737854,
+      "logits/rejected": -2.589991807937622,
+      "logps/chosen": -407.5807189941406,
+      "logps/rejected": -457.05426025390625,
+      "loss": 0.0218,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.4289559125900269,
+      "rewards/margins": 0.5248817801475525,
+      "rewards/rejected": -1.9538379907608032,
       "step": 240
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7370891215954565e-07,
-      "logits/chosen": -2.4028992652893066,
-      "logits/rejected": -2.392509937286377,
-      "logps/chosen": -390.3121032714844,
-      "logps/rejected": -434.3692321777344,
-      "loss": 0.0275,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.207838535308838,
-      "rewards/margins": 0.5656585097312927,
-      "rewards/rejected": -1.773497223854065,
+      "logits/chosen": -2.571653127670288,
+      "logits/rejected": -2.5496749877929688,
+      "logps/chosen": -422.95263671875,
+      "logps/rejected": -438.4296875,
+      "loss": 0.0259,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.4188787937164307,
+      "rewards/margins": 0.41885384917259216,
+      "rewards/rejected": -1.8377326726913452,
       "step": 250
     },
     {
       "epoch": 0.54,
       "learning_rate": 2.55479083351317e-07,
-      "logits/chosen": -2.3397555351257324,
-      "logits/rejected": -2.3070874214172363,
-      "logps/chosen": -451.30047607421875,
-      "logps/rejected": -458.3089294433594,
-      "loss": 0.0239,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.6053054332733154,
-      "rewards/margins": 0.562503457069397,
-      "rewards/rejected": -2.167808771133423,
+      "logits/chosen": -2.6116561889648438,
+      "logits/rejected": -2.5952515602111816,
+      "logps/chosen": -384.0877685546875,
+      "logps/rejected": -404.0052795410156,
+      "loss": 0.0303,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.1095821857452393,
+      "rewards/margins": 0.602874755859375,
+      "rewards/rejected": -1.7124569416046143,
       "step": 260
     },
     {
       "epoch": 0.56,
       "learning_rate": 2.3722002126275822e-07,
-      "logits/chosen": -2.328000068664551,
-      "logits/rejected": -2.298135280609131,
-      "logps/chosen": -414.98504638671875,
-      "logps/rejected": -454.4877014160156,
-      "loss": 0.0259,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.4466311931610107,
-      "rewards/margins": 0.555857241153717,
-      "rewards/rejected": -2.002488374710083,
+      "logits/chosen": -2.5404951572418213,
+      "logits/rejected": -2.5122456550598145,
+      "logps/chosen": -408.5818786621094,
+      "logps/rejected": -428.281982421875,
+      "loss": 0.0255,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.1992963552474976,
+      "rewards/margins": 0.4924868643283844,
+      "rewards/rejected": -1.6917833089828491,
       "step": 270
     },
     {
       "epoch": 0.59,
       "learning_rate": 2.19029145890313e-07,
-      "logits/chosen": -2.3714160919189453,
-      "logits/rejected": -2.3525238037109375,
-      "logps/chosen": -457.384033203125,
-      "logps/rejected": -497.83660888671875,
-      "loss": 0.0262,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.3860464096069336,
-      "rewards/margins": 0.6043224930763245,
-      "rewards/rejected": -1.9903688430786133,
+      "logits/chosen": -2.5082573890686035,
+      "logits/rejected": -2.484790563583374,
+      "logps/chosen": -413.8412170410156,
+      "logps/rejected": -446.98504638671875,
+      "loss": 0.025,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.3821589946746826,
+      "rewards/margins": 0.5924533009529114,
+      "rewards/rejected": -1.9746122360229492,
       "step": 280
     },
     {
       "epoch": 0.61,
       "learning_rate": 2.0100351342479216e-07,
-      "logits/chosen": -2.3231723308563232,
-      "logits/rejected": -2.3119874000549316,
-      "logps/chosen": -416.6573791503906,
-      "logps/rejected": -427.49945068359375,
-      "loss": 0.0251,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.4526689052581787,
-      "rewards/margins": 0.5181004405021667,
-      "rewards/rejected": -1.9707695245742798,
+      "logits/chosen": -2.4998109340667725,
+      "logits/rejected": -2.4744560718536377,
+      "logps/chosen": -446.154052734375,
+      "logps/rejected": -489.60235595703125,
+      "loss": 0.021,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.4215105772018433,
+      "rewards/margins": 0.7660014033317566,
+      "rewards/rejected": -2.187511920928955,
       "step": 290
     },
     {
       "epoch": 0.63,
       "learning_rate": 1.8323929841460178e-07,
-      "logits/chosen": -2.3192086219787598,
-      "logits/rejected": -2.2759287357330322,
-      "logps/chosen": -440.58935546875,
-      "logps/rejected": -448.8992614746094,
-      "loss": 0.0248,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.4383071660995483,
-      "rewards/margins": 0.5324681997299194,
-      "rewards/rejected": -1.9707752466201782,
+      "logits/chosen": -2.4917101860046387,
+      "logits/rejected": -2.46440052986145,
+      "logps/chosen": -439.48468017578125,
+      "logps/rejected": -490.96673583984375,
+      "loss": 0.0213,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.6839507818222046,
+      "rewards/margins": 0.6170190572738647,
+      "rewards/rejected": -2.3009700775146484,
       "step": 300
     },
     {
       "epoch": 0.63,
-      "eval_logits/chosen": -2.339747190475464,
-      "eval_logits/rejected": -2.319640874862671,
-      "eval_logps/chosen": -395.36798095703125,
-      "eval_logps/rejected": -460.886962890625,
-      "eval_loss": 0.025441216304898262,
-      "eval_rewards/accuracies": 0.69140625,
-      "eval_rewards/chosen": -1.383283257484436,
-      "eval_rewards/margins": 0.6520535945892334,
-      "eval_rewards/rejected": -2.03533673286438,
-      "eval_runtime": 53.2655,
-      "eval_samples_per_second": 37.548,
+      "eval_logits/chosen": -2.516728162765503,
+      "eval_logits/rejected": -2.5033957958221436,
+      "eval_logps/chosen": -410.5275573730469,
+      "eval_logps/rejected": -478.0339660644531,
+      "eval_loss": 0.02223433367908001,
+      "eval_rewards/accuracies": 0.69921875,
+      "eval_rewards/chosen": -1.5348790884017944,
+      "eval_rewards/margins": 0.6719276905059814,
+      "eval_rewards/rejected": -2.2068068981170654,
+      "eval_runtime": 53.2736,
+      "eval_samples_per_second": 37.542,
       "eval_steps_per_second": 0.601,
       "step": 300
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6583128063291573e-07,
-      "logits/chosen": -2.256671667098999,
-      "logits/rejected": -2.2072629928588867,
-      "logps/chosen": -441.985107421875,
-      "logps/rejected": -454.2076721191406,
-      "loss": 0.0255,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.5002824068069458,
-      "rewards/margins": 0.5682298541069031,
-      "rewards/rejected": -2.068512439727783,
+      "logits/chosen": -2.4991133213043213,
+      "logits/rejected": -2.481350898742676,
+      "logps/chosen": -425.85052490234375,
+      "logps/rejected": -514.1547241210938,
+      "loss": 0.0209,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.5156629085540771,
+      "rewards/margins": 0.8620169758796692,
+      "rewards/rejected": -2.3776798248291016,
       "step": 310
     },
     {
       "epoch": 0.67,
       "learning_rate": 1.488723393865766e-07,
-      "logits/chosen": -2.318877696990967,
-      "logits/rejected": -2.311671495437622,
-      "logps/chosen": -368.1661071777344,
-      "logps/rejected": -419.93499755859375,
-      "loss": 0.0257,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.1376068592071533,
-      "rewards/margins": 0.6408455967903137,
-      "rewards/rejected": -1.7784525156021118,
+      "logits/chosen": -2.5260231494903564,
+      "logits/rejected": -2.471526861190796,
+      "logps/chosen": -440.5318908691406,
+      "logps/rejected": -486.42193603515625,
+      "loss": 0.0224,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.47434401512146,
+      "rewards/margins": 0.6616085767745972,
+      "rewards/rejected": -2.1359527111053467,
       "step": 320
     },
     {
       "epoch": 0.69,
       "learning_rate": 1.3245295796480788e-07,
-      "logits/chosen": -2.307875156402588,
-      "logits/rejected": -2.287741184234619,
-      "logps/chosen": -388.35369873046875,
-      "logps/rejected": -418.2744140625,
-      "loss": 0.0264,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -1.2828859090805054,
-      "rewards/margins": 0.3729013502597809,
-      "rewards/rejected": -1.6557872295379639,
+      "logits/chosen": -2.437394142150879,
+      "logits/rejected": -2.434614896774292,
+      "logps/chosen": -429.20745849609375,
+      "logps/rejected": -491.5396423339844,
+      "loss": 0.0214,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.5614187717437744,
+      "rewards/margins": 0.6547269821166992,
+      "rewards/rejected": -2.2161457538604736,
       "step": 330
     },
     {
       "epoch": 0.71,
       "learning_rate": 1.1666074087171627e-07,
-      "logits/chosen": -2.3257226943969727,
-      "logits/rejected": -2.3047900199890137,
-      "logps/chosen": -400.9947204589844,
-      "logps/rejected": -466.18438720703125,
-      "loss": 0.0244,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -1.4033421277999878,
-      "rewards/margins": 0.6501402854919434,
-      "rewards/rejected": -2.0534825325012207,
+      "logits/chosen": -2.444455862045288,
+      "logits/rejected": -2.4428887367248535,
+      "logps/chosen": -467.06573486328125,
+      "logps/rejected": -497.69110107421875,
+      "loss": 0.0245,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.6300328969955444,
+      "rewards/margins": 0.43098846077919006,
+      "rewards/rejected": -2.061021327972412,
       "step": 340
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0157994641835734e-07,
-      "logits/chosen": -2.2791500091552734,
-      "logits/rejected": -2.228987216949463,
-      "logps/chosen": -438.2213439941406,
-      "logps/rejected": -482.71185302734375,
-      "loss": 0.0231,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.380692720413208,
-      "rewards/margins": 0.6693974733352661,
-      "rewards/rejected": -2.0500900745391846,
+      "logits/chosen": -2.4948933124542236,
+      "logits/rejected": -2.440502405166626,
+      "logps/chosen": -382.00372314453125,
+      "logps/rejected": -429.8780822753906,
+      "loss": 0.0225,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.3501756191253662,
+      "rewards/margins": 0.703359842300415,
+      "rewards/rejected": -2.0535354614257812,
       "step": 350
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.729103716819111e-08,
-      "logits/chosen": -2.3203213214874268,
-      "logits/rejected": -2.2689433097839355,
-      "logps/chosen": -447.54644775390625,
-      "logps/rejected": -481.5086364746094,
-      "loss": 0.0244,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.4591137170791626,
-      "rewards/margins": 0.7254992723464966,
-      "rewards/rejected": -2.184612989425659,
+      "logits/chosen": -2.4288363456726074,
+      "logits/rejected": -2.426206588745117,
+      "logps/chosen": -418.43243408203125,
+      "logps/rejected": -456.20526123046875,
+      "loss": 0.0232,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.3951587677001953,
+      "rewards/margins": 0.7315441370010376,
+      "rewards/rejected": -2.1267027854919434,
       "step": 360
     },
     {
       "epoch": 0.77,
       "learning_rate": 7.387025063449081e-08,
-      "logits/chosen": -2.2618608474731445,
-      "logits/rejected": -2.2370240688323975,
-      "logps/chosen": -463.3082580566406,
-      "logps/rejected": -520.4660034179688,
-      "loss": 0.0211,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.6034772396087646,
-      "rewards/margins": 0.8156415820121765,
-      "rewards/rejected": -2.419118881225586,
+      "logits/chosen": -2.449120044708252,
+      "logits/rejected": -2.4348466396331787,
+      "logps/chosen": -399.43878173828125,
+      "logps/rejected": -476.8487854003906,
+      "loss": 0.0227,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.4563090801239014,
+      "rewards/margins": 0.7731784582138062,
+      "rewards/rejected": -2.229487657546997,
       "step": 370
     },
     {
       "epoch": 0.79,
       "learning_rate": 6.138919252022435e-08,
-      "logits/chosen": -2.198676347732544,
-      "logits/rejected": -2.1428768634796143,
-      "logps/chosen": -469.5831604003906,
-      "logps/rejected": -499.9231872558594,
-      "loss": 0.0227,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.7702577114105225,
-      "rewards/margins": 0.7163289189338684,
-      "rewards/rejected": -2.4865870475769043,
+      "logits/chosen": -2.4416422843933105,
+      "logits/rejected": -2.397697925567627,
+      "logps/chosen": -472.70196533203125,
+      "logps/rejected": -477.952392578125,
+      "loss": 0.0247,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.4779317378997803,
+      "rewards/margins": 0.6161085367202759,
+      "rewards/rejected": -2.0940403938293457,
       "step": 380
     },
     {
       "epoch": 0.82,
       "learning_rate": 4.991445467064689e-08,
-      "logits/chosen": -2.2201147079467773,
-      "logits/rejected": -2.207123041152954,
-      "logps/chosen": -446.3898010253906,
-      "logps/rejected": -482.3876953125,
-      "loss": 0.0202,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -1.8155654668807983,
-      "rewards/margins": 0.5150431394577026,
-      "rewards/rejected": -2.330608367919922,
+      "logits/chosen": -2.4285051822662354,
+      "logits/rejected": -2.4042510986328125,
+      "logps/chosen": -400.11761474609375,
+      "logps/rejected": -447.89788818359375,
+      "loss": 0.0236,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.4530318975448608,
+      "rewards/margins": 0.5990349054336548,
+      "rewards/rejected": -2.0520665645599365,
       "step": 390
     },
     {
       "epoch": 0.84,
       "learning_rate": 3.9507259776993954e-08,
-      "logits/chosen": -2.2568085193634033,
-      "logits/rejected": -2.2447478771209717,
-      "logps/chosen": -449.7113342285156,
-      "logps/rejected": -495.636474609375,
-      "loss": 0.0194,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.930235505104065,
-      "rewards/margins": 0.5124340653419495,
-      "rewards/rejected": -2.442669630050659,
+      "logits/chosen": -2.447537899017334,
+      "logits/rejected": -2.4286274909973145,
+      "logps/chosen": -399.5050048828125,
+      "logps/rejected": -446.0399475097656,
+      "loss": 0.0241,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.3466806411743164,
+      "rewards/margins": 0.6658064126968384,
+      "rewards/rejected": -2.012486696243286,
       "step": 400
     },
     {
       "epoch": 0.84,
-      "eval_logits/chosen": -2.262547254562378,
-      "eval_logits/rejected": -2.237802267074585,
-      "eval_logps/chosen": -439.0294189453125,
-      "eval_logps/rejected": -509.7351379394531,
-      "eval_loss": 0.019433461129665375,
+      "eval_logits/chosen": -2.4709630012512207,
+      "eval_logits/rejected": -2.4588139057159424,
+      "eval_logps/chosen": -394.27191162109375,
+      "eval_logps/rejected": -463.64019775390625,
+      "eval_loss": 0.024743441492319107,
       "eval_rewards/accuracies": 0.703125,
-      "eval_rewards/chosen": -1.8198977708816528,
-      "eval_rewards/margins": 0.7039206027984619,
-      "eval_rewards/rejected": -2.523818254470825,
-      "eval_runtime": 53.2518,
-      "eval_samples_per_second": 37.557,
-      "eval_steps_per_second": 0.601,
+      "eval_rewards/chosen": -1.372322916984558,
+      "eval_rewards/margins": 0.6905463337898254,
+      "eval_rewards/rejected": -2.062869071960449,
+      "eval_runtime": 53.299,
+      "eval_samples_per_second": 37.524,
+      "eval_steps_per_second": 0.6,
       "step": 400
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.022313472693447e-08,
-      "logits/chosen": -2.219531536102295,
-      "logits/rejected": -2.177171468734741,
-      "logps/chosen": -435.68316650390625,
-      "logps/rejected": -480.1519470214844,
-      "loss": 0.0193,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.8031377792358398,
-      "rewards/margins": 0.5790697336196899,
-      "rewards/rejected": -2.3822076320648193,
+      "logits/chosen": -2.3729207515716553,
+      "logits/rejected": -2.3796212673187256,
+      "logps/chosen": -403.90350341796875,
+      "logps/rejected": -431.29766845703125,
+      "loss": 0.0239,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.3825476169586182,
+      "rewards/margins": 0.5267915725708008,
+      "rewards/rejected": -1.9093395471572876,
       "step": 410
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.2111614344599684e-08,
-      "logits/chosen": -2.261929988861084,
-      "logits/rejected": -2.20975661277771,
-      "logps/chosen": -446.5043029785156,
-      "logps/rejected": -468.5169982910156,
-      "loss": 0.0187,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.7549480199813843,
-      "rewards/margins": 0.5446823239326477,
-      "rewards/rejected": -2.2996301651000977,
+      "logits/chosen": -2.4164390563964844,
+      "logits/rejected": -2.399444103240967,
+      "logps/chosen": -389.99127197265625,
+      "logps/rejected": -439.9002990722656,
+      "loss": 0.0245,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.3119112253189087,
+      "rewards/margins": 0.6596888303756714,
+      "rewards/rejected": -1.9715999364852905,
       "step": 420
     },
     {
       "epoch": 0.9,
       "learning_rate": 1.521597710086439e-08,
-      "logits/chosen": -2.2168891429901123,
-      "logits/rejected": -2.1971139907836914,
-      "logps/chosen": -465.6109313964844,
-      "logps/rejected": -488.16912841796875,
-      "loss": 0.0192,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.8082774877548218,
-      "rewards/margins": 0.5722709894180298,
-      "rewards/rejected": -2.3805484771728516,
+      "logits/chosen": -2.4601497650146484,
+      "logits/rejected": -2.424562454223633,
+      "logps/chosen": -424.79266357421875,
+      "logps/rejected": -446.62359619140625,
+      "loss": 0.0255,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.4399926662445068,
+      "rewards/margins": 0.5878643989562988,
+      "rewards/rejected": -2.0278568267822266,
       "step": 430
     },
     {
       "epoch": 0.92,
       "learning_rate": 9.57301420397924e-09,
-      "logits/chosen": -2.231231212615967,
-      "logits/rejected": -2.209394931793213,
-      "logps/chosen": -448.60986328125,
-      "logps/rejected": -509.45770263671875,
-      "loss": 0.0187,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.8875834941864014,
-      "rewards/margins": 0.6209200620651245,
-      "rewards/rejected": -2.5085036754608154,
+      "logits/chosen": -2.5213966369628906,
+      "logits/rejected": -2.4663472175598145,
+      "logps/chosen": -409.0346984863281,
+      "logps/rejected": -405.43157958984375,
+      "loss": 0.0243,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.3315532207489014,
+      "rewards/margins": 0.5227077603340149,
+      "rewards/rejected": -1.8542606830596924,
       "step": 440
     },
     {
       "epoch": 0.94,
       "learning_rate": 5.212833302556258e-09,
-      "logits/chosen": -2.2720608711242676,
-      "logits/rejected": -2.2489547729492188,
-      "logps/chosen": -436.8863830566406,
-      "logps/rejected": -479.250244140625,
-      "loss": 0.0196,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.7134939432144165,
-      "rewards/margins": 0.5517303347587585,
-      "rewards/rejected": -2.2652244567871094,
+      "logits/chosen": -2.4338746070861816,
+      "logits/rejected": -2.422412395477295,
+      "logps/chosen": -418.519775390625,
+      "logps/rejected": -436.580810546875,
+      "loss": 0.0236,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.315696358680725,
+      "rewards/margins": 0.5186929702758789,
+      "rewards/rejected": -1.834389328956604,
       "step": 450
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.158697848236607e-09,
-      "logits/chosen": -2.2524914741516113,
-      "logits/rejected": -2.1975879669189453,
-      "logps/chosen": -454.9847717285156,
-      "logps/rejected": -471.0829162597656,
-      "loss": 0.0196,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.871111273765564,
-      "rewards/margins": 0.5668479204177856,
-      "rewards/rejected": -2.4379591941833496,
+      "logits/chosen": -2.4857373237609863,
+      "logits/rejected": -2.4624526500701904,
+      "logps/chosen": -393.2152404785156,
+      "logps/rejected": -430.68231201171875,
+      "loss": 0.0254,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.3136705160140991,
+      "rewards/margins": 0.5981560945510864,
+      "rewards/rejected": -1.911826491355896,
       "step": 460
     },
     {
       "epoch": 0.98,
       "learning_rate": 4.269029751107489e-10,
-      "logits/chosen": -2.2418127059936523,
-      "logits/rejected": -2.2091026306152344,
-      "logps/chosen": -438.8106384277344,
-      "logps/rejected": -487.1082458496094,
-      "loss": 0.0201,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.6759856939315796,
-      "rewards/margins": 0.6907237768173218,
-      "rewards/rejected": -2.3667094707489014,
+      "logits/chosen": -2.44291353225708,
+      "logits/rejected": -2.417865514755249,
+      "logps/chosen": -370.2296142578125,
+      "logps/rejected": -439.4507751464844,
+      "loss": 0.0244,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.3117094039916992,
+      "rewards/margins": 0.6579588651657104,
+      "rewards/rejected": -1.9696681499481201,
       "step": 470
     },
     {
       "epoch": 1.0,
       "step": 478,
       "total_flos": 0.0,
-      "train_loss": 0.03868693582495386,
-      "train_runtime": 4353.4885,
-      "train_samples_per_second": 14.043,
+      "train_loss": 0.03922173811052634,
+      "train_runtime": 4355.9498,
+      "train_samples_per_second": 14.035,
       "train_steps_per_second": 0.11
     }
   ],