Model save

README.md

@@ -17,15 +17,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.0581
-- Rewards/chosen: -1.0898
-- Rewards/rejected: -1.5220
-- Rewards/accuracies: 0.7031
-- Rewards/margins: 0.4322
-- Logps/rejected: -409.5538
-- Logps/chosen: -366.0210
-- Logits/rejected: -2.4659
-- Logits/chosen: -2.4909
+- Loss: 0.5327
+- Rewards/chosen: -1.3058
+- Rewards/rejected: -2.0790
+- Rewards/accuracies: 0.7695
+- Rewards/margins: 0.7731
+- Logps/rejected: -465.2492
+- Logps/chosen: -387.6228
+- Logits/rejected: -2.0418
+- Logits/chosen: -2.0914
 
 ## Model description
 
@@ -62,10 +62,10 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.2635        | 0.21  | 100  | 0.1337          | -0.2670        | -0.4352          | 0.6328             | 0.1682          | -300.8777      | -283.7432    | -2.5846         | -2.6032       |
-| 0.1263        | 0.42  | 200  | 0.0696          | -0.9052        | -1.2607          | 0.6719             | 0.3556          | -383.4268      | -347.5578    | -2.5095         | -2.5332       |
-| 0.0853        | 0.63  | 300  | 0.0518          | -1.2778        | -1.7148          | 0.6914             | 0.4369          | -428.8290      | -384.8238    | -2.4603         | -2.4861       |
-| 0.1028        | 0.84  | 400  | 0.0581          | -1.0898        | -1.5220          | 0.7031             | 0.4322          | -409.5538      | -366.0210    | -2.4659         | -2.4909       |
+| 0.645         | 0.21  | 100  | 0.6138          | -0.3445        | -0.6635          | 0.7109             | 0.3190          | -323.7003      | -291.4871    | -2.5877         | -2.6051       |
+| 0.6088        | 0.42  | 200  | 0.5601          | -1.2478        | -1.8506          | 0.7383             | 0.6028          | -442.4111      | -381.8215    | -2.2506         | -2.2970       |
+| 0.5713        | 0.63  | 300  | 0.5404          | -1.1777        | -1.9907          | 0.7656             | 0.8130          | -456.4196      | -374.8101    | -2.1131         | -2.1610       |
+| 0.585         | 0.84  | 400  | 0.5327          | -1.3058        | -2.0790          | 0.7695             | 0.7731          | -465.2492      | -387.6228    | -2.0418         | -2.0914       |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.1551352257384416,
-    "train_runtime": 4354.6676,
+    "train_loss": 0.6134292899814111,
+    "train_runtime": 4347.6461,
     "train_samples": 61133,
-    "train_samples_per_second": 14.038,
+    "train_samples_per_second": 14.061,
     "train_steps_per_second": 0.11
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b5dc7472e82af20819c0ee4f93694f6c4d9dcc1e6c1ee25a146244bf25f25f4
+oid sha256:b13fa5b299368c2259ecf6b9c726c87b393aad8fdb4a6a12109793d8ab38623a
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0f973d409e09bf00754e9f50e78e67e2ccbc40e09c192f3e900dff31bfc716c0
+oid sha256:51dcea949b3cd74c461e76864fae30cd7745945766344c2326a7f912be62de9c
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f1dd4da84f1ab66646a25fdeb975d56a2f09d385ad4fff83b7606597ba95eaaf
+oid sha256:4d306517620b0dd0829ce6b274c3e2da445360756a701278d964f5c4d7153017
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.1551352257384416,
-    "train_runtime": 4354.6676,
+    "train_loss": 0.6134292899814111,
+    "train_runtime": 4347.6461,
     "train_samples": 61133,
-    "train_samples_per_second": 14.038,
+    "train_samples_per_second": 14.061,
     "train_steps_per_second": 0.11
 }

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -2.6760501861572266,
       "logps/chosen": -153.28274536132812,
       "logps/rejected": -127.2953872680664,
-      "loss": 0.3484,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,732 +25,732 @@
     {
       "epoch": 0.02,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -2.7934648990631104,
-      "logits/rejected": -2.792778968811035,
-      "logps/chosen": -169.8660125732422,
-      "logps/rejected": -174.6484375,
-      "loss": 0.3419,
-      "rewards/accuracies": 0.4791666567325592,
-      "rewards/chosen": 0.000645098858512938,
-      "rewards/margins": 0.00047150938189588487,
-      "rewards/rejected": 0.00017358947661705315,
+      "logits/chosen": -2.793574094772339,
+      "logits/rejected": -2.7928600311279297,
+      "logps/chosen": -169.88270568847656,
+      "logps/rejected": -174.67208862304688,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.0004781727911904454,
+      "rewards/margins": 0.0005411193706095219,
+      "rewards/rejected": -6.294667400652543e-05,
       "step": 10
     },
     {
       "epoch": 0.04,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -2.755502700805664,
-      "logits/rejected": -2.7592272758483887,
-      "logps/chosen": -173.93276977539062,
-      "logps/rejected": -172.51002502441406,
-      "loss": 0.3374,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": 0.00027031515492126346,
-      "rewards/margins": 0.0004853919963352382,
-      "rewards/rejected": -0.00021507684141397476,
+      "logits/chosen": -2.75508975982666,
+      "logits/rejected": -2.7587742805480957,
+      "logps/chosen": -173.97691345214844,
+      "logps/rejected": -172.56187438964844,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.00017117620154749602,
+      "rewards/margins": 0.00056259281700477,
+      "rewards/rejected": -0.0007337690331041813,
       "step": 20
     },
     {
       "epoch": 0.06,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -2.7985167503356934,
-      "logits/rejected": -2.7822909355163574,
-      "logps/chosen": -149.1220245361328,
-      "logps/rejected": -148.69937133789062,
-      "loss": 0.3436,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": -0.0010819969465956092,
-      "rewards/margins": 0.0008986530010588467,
-      "rewards/rejected": -0.001980650005862117,
+      "logits/chosen": -2.7977731227874756,
+      "logits/rejected": -2.7815966606140137,
+      "logps/chosen": -149.19393920898438,
+      "logps/rejected": -148.75160217285156,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.0018012176733464003,
+      "rewards/margins": 0.0007020062766969204,
+      "rewards/rejected": -0.0025032239500433207,
       "step": 30
     },
     {
       "epoch": 0.08,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.7784383296966553,
-      "logits/rejected": -2.7649941444396973,
-      "logps/chosen": -145.487060546875,
-      "logps/rejected": -153.00625610351562,
-      "loss": 0.3369,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": 0.0003283992991782725,
-      "rewards/margins": 0.0034873399417847395,
-      "rewards/rejected": -0.0031589411664754152,
+      "logits/chosen": -2.7791695594787598,
+      "logits/rejected": -2.765839099884033,
+      "logps/chosen": -145.63992309570312,
+      "logps/rejected": -153.24330139160156,
+      "loss": 0.691,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0012001590803265572,
+      "rewards/margins": 0.004328931216150522,
+      "rewards/rejected": -0.005529090296477079,
       "step": 40
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999733114418725e-07,
-      "logits/chosen": -2.7593047618865967,
-      "logits/rejected": -2.7510294914245605,
-      "logps/chosen": -168.7542266845703,
-      "logps/rejected": -149.18936157226562,
-      "loss": 0.3507,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.0038483496755361557,
-      "rewards/margins": 0.008144570514559746,
-      "rewards/rejected": -0.0042962199077010155,
+      "logits/chosen": -2.765580415725708,
+      "logits/rejected": -2.7574915885925293,
+      "logps/chosen": -168.78195190429688,
+      "logps/rejected": -149.6485595703125,
+      "loss": 0.6867,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": 0.0035712323151528835,
+      "rewards/margins": 0.012459376826882362,
+      "rewards/rejected": -0.008888144977390766,
       "step": 50
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.990398100856366e-07,
-      "logits/chosen": -2.718301296234131,
-      "logits/rejected": -2.7141382694244385,
-      "logps/chosen": -149.05760192871094,
-      "logps/rejected": -140.6719512939453,
-      "loss": 0.3549,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": 0.04143524542450905,
-      "rewards/margins": 0.029302945360541344,
-      "rewards/rejected": 0.012132294476032257,
+      "logits/chosen": -2.7321128845214844,
+      "logits/rejected": -2.7279860973358154,
+      "logps/chosen": -148.0186004638672,
+      "logps/rejected": -141.0856475830078,
+      "loss": 0.6803,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.05182523652911186,
+      "rewards/margins": 0.043829865753650665,
+      "rewards/rejected": 0.00799537356942892,
       "step": 60
     },
     {
       "epoch": 0.15,
       "learning_rate": 4.967775735898179e-07,
-      "logits/chosen": -2.7313451766967773,
-      "logits/rejected": -2.725338935852051,
-      "logps/chosen": -171.82577514648438,
-      "logps/rejected": -164.0020751953125,
-      "loss": 0.3442,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.012257634662091732,
-      "rewards/margins": 0.044421594589948654,
-      "rewards/rejected": -0.032163962721824646,
+      "logits/chosen": -2.74849271774292,
+      "logits/rejected": -2.7429099082946777,
+      "logps/chosen": -175.6835174560547,
+      "logps/rejected": -169.5352783203125,
+      "loss": 0.6726,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.026319989934563637,
+      "rewards/margins": 0.061176080256700516,
+      "rewards/rejected": -0.0874960646033287,
       "step": 70
     },
     {
       "epoch": 0.17,
       "learning_rate": 4.931986719649298e-07,
-      "logits/chosen": -2.6190171241760254,
-      "logits/rejected": -2.615912914276123,
-      "logps/chosen": -172.01473999023438,
-      "logps/rejected": -163.3651885986328,
-      "loss": 0.2755,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.14659154415130615,
-      "rewards/margins": 0.04203266650438309,
-      "rewards/rejected": -0.18862418830394745,
+      "logits/chosen": -2.633291244506836,
+      "logits/rejected": -2.629636764526367,
+      "logps/chosen": -169.12416076660156,
+      "logps/rejected": -163.40679931640625,
+      "loss": 0.6604,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.1176857128739357,
+      "rewards/margins": 0.0713546946644783,
+      "rewards/rejected": -0.1890403926372528,
       "step": 80
     },
     {
       "epoch": 0.19,
       "learning_rate": 4.883222001996351e-07,
-      "logits/chosen": -2.691314220428467,
-      "logits/rejected": -2.6700475215911865,
-      "logps/chosen": -187.34765625,
-      "logps/rejected": -169.58822631835938,
-      "loss": 0.2654,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.16760021448135376,
-      "rewards/margins": 0.06520744413137436,
-      "rewards/rejected": -0.23280766606330872,
+      "logits/chosen": -2.708993911743164,
+      "logits/rejected": -2.6869051456451416,
+      "logps/chosen": -195.36166381835938,
+      "logps/rejected": -183.77658081054688,
+      "loss": 0.6512,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.2477399855852127,
+      "rewards/margins": 0.12695127725601196,
+      "rewards/rejected": -0.37469127774238586,
       "step": 90
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.821741763807186e-07,
-      "logits/chosen": -2.566351890563965,
-      "logits/rejected": -2.5563173294067383,
-      "logps/chosen": -182.60443115234375,
-      "logps/rejected": -181.93276977539062,
-      "loss": 0.2635,
+      "logits/chosen": -2.5655856132507324,
+      "logits/rejected": -2.5531303882598877,
+      "logps/chosen": -200.37026977539062,
+      "logps/rejected": -205.42471313476562,
+      "loss": 0.645,
       "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.23503470420837402,
-      "rewards/margins": 0.052919644862413406,
-      "rewards/rejected": -0.28795433044433594,
+      "rewards/chosen": -0.4126931130886078,
+      "rewards/margins": 0.11018091440200806,
+      "rewards/rejected": -0.5228740572929382,
       "step": 100
     },
     {
       "epoch": 0.21,
-      "eval_logits/chosen": -2.603199005126953,
-      "eval_logits/rejected": -2.5846238136291504,
-      "eval_logps/chosen": -283.7431945800781,
-      "eval_logps/rejected": -300.87774658203125,
-      "eval_loss": 0.13367339968681335,
-      "eval_rewards/accuracies": 0.6328125,
-      "eval_rewards/chosen": -0.26703548431396484,
-      "eval_rewards/margins": 0.16820889711380005,
-      "eval_rewards/rejected": -0.4352443814277649,
-      "eval_runtime": 53.4179,
-      "eval_samples_per_second": 37.441,
-      "eval_steps_per_second": 0.599,
+      "eval_logits/chosen": -2.6051313877105713,
+      "eval_logits/rejected": -2.5876898765563965,
+      "eval_logps/chosen": -291.487060546875,
+      "eval_logps/rejected": -323.70025634765625,
+      "eval_loss": 0.6137697696685791,
+      "eval_rewards/accuracies": 0.7109375,
+      "eval_rewards/chosen": -0.34447404742240906,
+      "eval_rewards/margins": 0.31899595260620117,
+      "eval_rewards/rejected": -0.6634700298309326,
+      "eval_runtime": 53.8393,
+      "eval_samples_per_second": 37.148,
+      "eval_steps_per_second": 0.594,
       "step": 100
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.747874028753375e-07,
-      "logits/chosen": -2.552278518676758,
-      "logits/rejected": -2.5532853603363037,
-      "logps/chosen": -217.91488647460938,
-      "logps/rejected": -202.06771850585938,
-      "loss": 0.2208,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.3952718675136566,
-      "rewards/margins": 0.07539477944374084,
-      "rewards/rejected": -0.47066664695739746,
+      "logits/chosen": -2.5329718589782715,
+      "logits/rejected": -2.5321857929229736,
+      "logps/chosen": -226.0692596435547,
+      "logps/rejected": -221.8550262451172,
+      "loss": 0.6378,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.476815402507782,
+      "rewards/margins": 0.19172437489032745,
+      "rewards/rejected": -0.6685398817062378,
       "step": 110
     },
     {
       "epoch": 0.25,
       "learning_rate": 4.662012913161997e-07,
-      "logits/chosen": -2.555584669113159,
-      "logits/rejected": -2.5512847900390625,
-      "logps/chosen": -196.34860229492188,
-      "logps/rejected": -197.1841583251953,
-      "loss": 0.2068,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.4823318421840668,
-      "rewards/margins": 0.09269986301660538,
-      "rewards/rejected": -0.5750316977500916,
+      "logits/chosen": -2.5088794231414795,
+      "logits/rejected": -2.5054843425750732,
+      "logps/chosen": -209.64541625976562,
+      "logps/rejected": -223.4107208251953,
+      "loss": 0.6382,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.6152997612953186,
+      "rewards/margins": 0.22199754416942596,
+      "rewards/rejected": -0.837297260761261,
       "step": 120
     },
     {
       "epoch": 0.27,
       "learning_rate": 4.5646165232345103e-07,
-      "logits/chosen": -2.5616307258605957,
-      "logits/rejected": -2.5439343452453613,
-      "logps/chosen": -219.60317993164062,
-      "logps/rejected": -223.775390625,
-      "loss": 0.1801,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.5193461179733276,
-      "rewards/margins": 0.1517525166273117,
-      "rewards/rejected": -0.6710985898971558,
+      "logits/chosen": -2.5064823627471924,
+      "logits/rejected": -2.4858460426330566,
+      "logps/chosen": -223.5966339111328,
+      "logps/rejected": -242.69461059570312,
+      "loss": 0.6146,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.559280514717102,
+      "rewards/margins": 0.3010101616382599,
+      "rewards/rejected": -0.8602906465530396,
       "step": 130
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.456204510851956e-07,
-      "logits/chosen": -2.4612019062042236,
-      "logits/rejected": -2.4220564365386963,
-      "logps/chosen": -235.089599609375,
-      "logps/rejected": -286.7164306640625,
-      "loss": 0.1601,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.5644978284835815,
-      "rewards/margins": 0.198373943567276,
-      "rewards/rejected": -0.7628717422485352,
+      "logits/chosen": -2.4447319507598877,
+      "logits/rejected": -2.4147696495056152,
+      "logps/chosen": -247.92050170898438,
+      "logps/rejected": -308.4876708984375,
+      "loss": 0.6237,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.6928067207336426,
+      "rewards/margins": 0.2877771854400635,
+      "rewards/rejected": -0.9805838465690613,
       "step": 140
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.337355301007335e-07,
-      "logits/chosen": -2.549870252609253,
-      "logits/rejected": -2.506094455718994,
-      "logps/chosen": -225.95242309570312,
-      "logps/rejected": -232.50741577148438,
-      "loss": 0.1409,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.7746833562850952,
-      "rewards/margins": 0.17926892638206482,
-      "rewards/rejected": -0.9539523124694824,
+      "logits/chosen": -2.5137321949005127,
+      "logits/rejected": -2.46433687210083,
+      "logps/chosen": -230.3649139404297,
+      "logps/rejected": -242.9630126953125,
+      "loss": 0.6261,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.818808376789093,
+      "rewards/margins": 0.23969991505146027,
+      "rewards/rejected": -1.0585081577301025,
       "step": 150
     },
     {
       "epoch": 0.33,
       "learning_rate": 4.2087030056579986e-07,
-      "logits/chosen": -2.4901137351989746,
-      "logits/rejected": -2.494145631790161,
-      "logps/chosen": -260.5132751464844,
-      "logps/rejected": -295.6642761230469,
-      "loss": 0.1158,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.9862159490585327,
-      "rewards/margins": 0.26035162806510925,
-      "rewards/rejected": -1.2465674877166748,
+      "logits/chosen": -2.4309020042419434,
+      "logits/rejected": -2.4267687797546387,
+      "logps/chosen": -253.67276000976562,
+      "logps/rejected": -301.7275390625,
+      "loss": 0.62,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.9178106188774109,
+      "rewards/margins": 0.3893894851207733,
+      "rewards/rejected": -1.3072000741958618,
       "step": 160
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.070934040463998e-07,
-      "logits/chosen": -2.4662399291992188,
-      "logits/rejected": -2.4258627891540527,
-      "logps/chosen": -289.4188232421875,
-      "logps/rejected": -315.436279296875,
-      "loss": 0.1128,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -1.126964807510376,
-      "rewards/margins": 0.24998700618743896,
-      "rewards/rejected": -1.3769519329071045,
+      "logits/chosen": -2.3415331840515137,
+      "logits/rejected": -2.289360284805298,
+      "logps/chosen": -309.43841552734375,
+      "logps/rejected": -346.57745361328125,
+      "loss": 0.6056,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.3271605968475342,
+      "rewards/margins": 0.3612029254436493,
+      "rewards/rejected": -1.6883633136749268,
       "step": 170
     },
     {
       "epoch": 0.38,
       "learning_rate": 3.9247834624635404e-07,
-      "logits/chosen": -2.472064733505249,
-      "logits/rejected": -2.461550235748291,
-      "logps/chosen": -270.08148193359375,
-      "logps/rejected": -308.64593505859375,
-      "loss": 0.1069,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -1.2109320163726807,
-      "rewards/margins": 0.21587257087230682,
-      "rewards/rejected": -1.426804780960083,
+      "logits/chosen": -2.3409111499786377,
+      "logits/rejected": -2.3219058513641357,
+      "logps/chosen": -260.40386962890625,
+      "logps/rejected": -308.46649169921875,
+      "loss": 0.6178,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.1141560077667236,
+      "rewards/margins": 0.31085366010665894,
+      "rewards/rejected": -1.425009846687317,
       "step": 180
     },
     {
       "epoch": 0.4,
       "learning_rate": 3.7710310482256523e-07,
-      "logits/chosen": -2.4616963863372803,
-      "logits/rejected": -2.4687294960021973,
-      "logps/chosen": -271.68658447265625,
-      "logps/rejected": -283.86602783203125,
-      "loss": 0.1142,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -1.0631568431854248,
-      "rewards/margins": 0.16678860783576965,
-      "rewards/rejected": -1.229945421218872,
+      "logits/chosen": -2.3359875679016113,
+      "logits/rejected": -2.3345441818237305,
+      "logps/chosen": -248.61245727539062,
+      "logps/rejected": -272.8714599609375,
+      "loss": 0.6087,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.832414984703064,
+      "rewards/margins": 0.28758493065834045,
+      "rewards/rejected": -1.119999885559082,
       "step": 190
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.610497133404795e-07,
-      "logits/chosen": -2.521026134490967,
-      "logits/rejected": -2.4863734245300293,
-      "logps/chosen": -263.0998229980469,
-      "logps/rejected": -271.73101806640625,
-      "loss": 0.1263,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -1.022953987121582,
-      "rewards/margins": 0.1595546305179596,
-      "rewards/rejected": -1.1825085878372192,
+      "logits/chosen": -2.3152999877929688,
+      "logits/rejected": -2.2719802856445312,
+      "logps/chosen": -283.94561767578125,
+      "logps/rejected": -298.6034240722656,
+      "loss": 0.6088,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2314116954803467,
+      "rewards/margins": 0.2198210060596466,
+      "rewards/rejected": -1.451232671737671,
       "step": 200
     },
     {
       "epoch": 0.42,
-      "eval_logits/chosen": -2.5332112312316895,
-      "eval_logits/rejected": -2.5095102787017822,
-      "eval_logps/chosen": -347.55780029296875,
-      "eval_logps/rejected": -383.42681884765625,
-      "eval_loss": 0.06963210552930832,
-      "eval_rewards/accuracies": 0.671875,
-      "eval_rewards/chosen": -0.9051821231842041,
-      "eval_rewards/margins": 0.3555533289909363,
-      "eval_rewards/rejected": -1.260735273361206,
-      "eval_runtime": 53.0896,
-      "eval_samples_per_second": 37.672,
-      "eval_steps_per_second": 0.603,
+      "eval_logits/chosen": -2.296954393386841,
+      "eval_logits/rejected": -2.2505648136138916,
+      "eval_logps/chosen": -381.8215026855469,
+      "eval_logps/rejected": -442.4111328125,
+      "eval_loss": 0.5601309537887573,
+      "eval_rewards/accuracies": 0.73828125,
+      "eval_rewards/chosen": -1.2478187084197998,
+      "eval_rewards/margins": 0.6027593016624451,
+      "eval_rewards/rejected": -1.8505780696868896,
+      "eval_runtime": 53.1318,
+      "eval_samples_per_second": 37.642,
+      "eval_steps_per_second": 0.602,
       "step": 200
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4440382358952115e-07,
-      "logits/chosen": -2.4805960655212402,
-      "logits/rejected": -2.454671621322632,
-      "logps/chosen": -276.58099365234375,
-      "logps/rejected": -316.8192443847656,
-      "loss": 0.1238,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.9177879095077515,
-      "rewards/margins": 0.2953142523765564,
-      "rewards/rejected": -1.213102102279663,
+      "logits/chosen": -2.245332717895508,
+      "logits/rejected": -2.208613634109497,
+      "logps/chosen": -322.2597351074219,
+      "logps/rejected": -380.9275817871094,
+      "loss": 0.5753,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.3745754957199097,
+      "rewards/margins": 0.4796102046966553,
+      "rewards/rejected": -1.854185700416565,
       "step": 210
     },
     {
       "epoch": 0.46,
       "learning_rate": 3.272542485937368e-07,
-      "logits/chosen": -2.4566211700439453,
-      "logits/rejected": -2.4362823963165283,
-      "logps/chosen": -264.9664611816406,
-      "logps/rejected": -277.8113708496094,
-      "loss": 0.1132,
+      "logits/chosen": -2.236409902572632,
+      "logits/rejected": -2.2042953968048096,
+      "logps/chosen": -291.12939453125,
+      "logps/rejected": -318.00677490234375,
+      "loss": 0.6031,
       "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.9819846153259277,
-      "rewards/margins": 0.19036361575126648,
-      "rewards/rejected": -1.1723482608795166,
+      "rewards/chosen": -1.2436137199401855,
+      "rewards/margins": 0.3306885063648224,
+      "rewards/rejected": -1.5743024349212646,
       "step": 220
     },
     {
       "epoch": 0.48,
       "learning_rate": 3.096924887558854e-07,
-      "logits/chosen": -2.441197633743286,
-      "logits/rejected": -2.4596199989318848,
-      "logps/chosen": -276.02557373046875,
-      "logps/rejected": -300.5140380859375,
-      "loss": 0.1025,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -1.212254285812378,
-      "rewards/margins": 0.1722598373889923,
-      "rewards/rejected": -1.3845142126083374,
+      "logits/chosen": -2.22724986076355,
+      "logits/rejected": -2.243987798690796,
+      "logps/chosen": -267.0990295410156,
+      "logps/rejected": -302.5461120605469,
+      "loss": 0.6021,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.1229889392852783,
+      "rewards/margins": 0.2818460762500763,
+      "rewards/rejected": -1.4048349857330322,
       "step": 230
     },
     {
       "epoch": 0.5,
       "learning_rate": 2.9181224366319943e-07,
-      "logits/chosen": -2.333803653717041,
-      "logits/rejected": -2.3507378101348877,
-      "logps/chosen": -310.8798522949219,
-      "logps/rejected": -304.9378356933594,
-      "loss": 0.0863,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -1.4252393245697021,
-      "rewards/margins": 0.1391422301530838,
-      "rewards/rejected": -1.5643815994262695,
+      "logits/chosen": -2.167080879211426,
+      "logits/rejected": -2.164618492126465,
+      "logps/chosen": -282.51898193359375,
+      "logps/rejected": -297.18804931640625,
+      "loss": 0.5853,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.141631007194519,
+      "rewards/margins": 0.3452518582344055,
+      "rewards/rejected": -1.4868830442428589,
       "step": 240
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7370891215954565e-07,
-      "logits/chosen": -2.401367664337158,
-      "logits/rejected": -2.3665804862976074,
-      "logps/chosen": -291.9630432128906,
-      "logps/rejected": -310.1883239746094,
-      "loss": 0.0876,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -1.2684155702590942,
-      "rewards/margins": 0.14334025979042053,
-      "rewards/rejected": -1.4117558002471924,
+      "logits/chosen": -2.233933687210083,
+      "logits/rejected": -2.1930792331695557,
+      "logps/chosen": -265.99151611328125,
+      "logps/rejected": -305.2124328613281,
+      "loss": 0.6125,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.0087003707885742,
+      "rewards/margins": 0.35329657793045044,
+      "rewards/rejected": -1.3619968891143799,
       "step": 250
     },
     {
       "epoch": 0.54,
       "learning_rate": 2.55479083351317e-07,
-      "logits/chosen": -2.4047045707702637,
-      "logits/rejected": -2.3763346672058105,
-      "logps/chosen": -265.778564453125,
-      "logps/rejected": -270.27850341796875,
-      "loss": 0.1013,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -1.1575849056243896,
-      "rewards/margins": 0.19892552495002747,
-      "rewards/rejected": -1.3565104007720947,
+      "logits/chosen": -2.2159643173217773,
+      "logits/rejected": -2.176429033279419,
+      "logps/chosen": -258.05267333984375,
+      "logps/rejected": -276.36248779296875,
+      "loss": 0.6118,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.0803258419036865,
+      "rewards/margins": 0.3370239734649658,
+      "rewards/rejected": -1.4173498153686523,
       "step": 260
     },
     {
       "epoch": 0.56,
       "learning_rate": 2.3722002126275822e-07,
-      "logits/chosen": -2.436124324798584,
-      "logits/rejected": -2.4261937141418457,
-      "logps/chosen": -277.248779296875,
-      "logps/rejected": -299.70343017578125,
-      "loss": 0.1014,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -1.1114938259124756,
-      "rewards/margins": 0.2822580635547638,
-      "rewards/rejected": -1.393751859664917,
+      "logits/chosen": -2.1824676990509033,
+      "logits/rejected": -2.1633520126342773,
+      "logps/chosen": -275.42816162109375,
+      "logps/rejected": -318.5284423828125,
+      "loss": 0.5836,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.093287706375122,
+      "rewards/margins": 0.48871397972106934,
+      "rewards/rejected": -1.582001805305481,
       "step": 270
     },
     {
       "epoch": 0.59,
       "learning_rate": 2.19029145890313e-07,
-      "logits/chosen": -2.4304685592651367,
-      "logits/rejected": -2.419506549835205,
-      "logps/chosen": -281.5838928222656,
-      "logps/rejected": -291.45440673828125,
-      "loss": 0.0969,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -1.1346423625946045,
-      "rewards/margins": 0.18906255066394806,
-      "rewards/rejected": -1.3237049579620361,
+      "logits/chosen": -2.166625738143921,
+      "logits/rejected": -2.1442408561706543,
+      "logps/chosen": -299.334228515625,
+      "logps/rejected": -323.8576354980469,
+      "loss": 0.5885,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.312145709991455,
+      "rewards/margins": 0.33559149503707886,
+      "rewards/rejected": -1.6477371454238892,
       "step": 280
     },
     {
       "epoch": 0.61,
       "learning_rate": 2.0100351342479216e-07,
-      "logits/chosen": -2.432145118713379,
-      "logits/rejected": -2.432448148727417,
-      "logps/chosen": -270.7576904296875,
-      "logps/rejected": -286.2849426269531,
-      "loss": 0.0976,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.1866267919540405,
-      "rewards/margins": 0.2962873876094818,
-      "rewards/rejected": -1.4829142093658447,
+      "logits/chosen": -2.1124095916748047,
+      "logits/rejected": -2.091318368911743,
+      "logps/chosen": -273.45526123046875,
+      "logps/rejected": -320.23291015625,
+      "loss": 0.5729,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.2136024236679077,
+      "rewards/margins": 0.6087918281555176,
+      "rewards/rejected": -1.8223940134048462,
       "step": 290
     },
     {
       "epoch": 0.63,
       "learning_rate": 1.8323929841460178e-07,
-      "logits/chosen": -2.416377544403076,
-      "logits/rejected": -2.384713649749756,
-      "logps/chosen": -311.925537109375,
-      "logps/rejected": -328.46063232421875,
-      "loss": 0.0853,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -1.347588300704956,
-      "rewards/margins": 0.2518860101699829,
-      "rewards/rejected": -1.599474310874939,
+      "logits/chosen": -2.0944509506225586,
+      "logits/rejected": -2.042977809906006,
+      "logps/chosen": -318.8859558105469,
+      "logps/rejected": -362.031005859375,
+      "loss": 0.5713,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.417192816734314,
+      "rewards/margins": 0.5179849863052368,
+      "rewards/rejected": -1.9351780414581299,
       "step": 300
     },
     {
       "epoch": 0.63,
-      "eval_logits/chosen": -2.486100673675537,
-      "eval_logits/rejected": -2.4602901935577393,
-      "eval_logps/chosen": -384.8237609863281,
-      "eval_logps/rejected": -428.8289794921875,
-      "eval_loss": 0.05176469311118126,
-      "eval_rewards/accuracies": 0.69140625,
-      "eval_rewards/chosen": -1.2778410911560059,
-      "eval_rewards/margins": 0.4369162321090698,
-      "eval_rewards/rejected": -1.7147574424743652,
-      "eval_runtime": 53.0827,
-      "eval_samples_per_second": 37.677,
-      "eval_steps_per_second": 0.603,
+      "eval_logits/chosen": -2.1609904766082764,
+      "eval_logits/rejected": -2.1131389141082764,
+      "eval_logps/chosen": -374.8100891113281,
+      "eval_logps/rejected": -456.4195556640625,
+      "eval_loss": 0.5404335856437683,
+      "eval_rewards/accuracies": 0.765625,
+      "eval_rewards/chosen": -1.1777048110961914,
+      "eval_rewards/margins": 0.8129580616950989,
+      "eval_rewards/rejected": -1.9906628131866455,
+      "eval_runtime": 53.1243,
+      "eval_samples_per_second": 37.648,
+      "eval_steps_per_second": 0.602,
       "step": 300
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6583128063291573e-07,
-      "logits/chosen": -2.4378726482391357,
-      "logits/rejected": -2.4306111335754395,
-      "logps/chosen": -290.81195068359375,
-      "logps/rejected": -343.064453125,
-      "loss": 0.0895,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.2710740566253662,
-      "rewards/margins": 0.37092748284339905,
-      "rewards/rejected": -1.642001748085022,
+      "logits/chosen": -2.0995776653289795,
+      "logits/rejected": -2.084737539291382,
+      "logps/chosen": -293.4931640625,
+      "logps/rejected": -358.046142578125,
+      "loss": 0.5972,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.297886610031128,
+      "rewards/margins": 0.4939315915107727,
+      "rewards/rejected": -1.7918182611465454,
       "step": 310
     },
     {
       "epoch": 0.67,
       "learning_rate": 1.488723393865766e-07,
-      "logits/chosen": -2.445270538330078,
-      "logits/rejected": -2.425755023956299,
-      "logps/chosen": -313.85882568359375,
-      "logps/rejected": -326.77911376953125,
-      "loss": 0.0948,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -1.347819447517395,
-      "rewards/margins": 0.2622388005256653,
-      "rewards/rejected": -1.6100581884384155,
+      "logits/chosen": -2.1189067363739014,
+      "logits/rejected": -2.0870201587677,
+      "logps/chosen": -323.58111572265625,
+      "logps/rejected": -349.4317932128906,
+      "loss": 0.5988,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.445042371749878,
+      "rewards/margins": 0.39154213666915894,
+      "rewards/rejected": -1.8365846872329712,
       "step": 320
     },
     {
       "epoch": 0.69,
       "learning_rate": 1.3245295796480788e-07,
-      "logits/chosen": -2.386258602142334,
-      "logits/rejected": -2.374406099319458,
-      "logps/chosen": -287.63970947265625,
-      "logps/rejected": -297.44610595703125,
-      "loss": 0.099,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -1.1716187000274658,
-      "rewards/margins": 0.23826315999031067,
-      "rewards/rejected": -1.409881830215454,
+      "logits/chosen": -2.028104543685913,
+      "logits/rejected": -2.0024235248565674,
+      "logps/chosen": -300.0089111328125,
+      "logps/rejected": -327.14935302734375,
+      "loss": 0.5806,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.295310378074646,
+      "rewards/margins": 0.41160374879837036,
+      "rewards/rejected": -1.7069141864776611,
       "step": 330
     },
     {
       "epoch": 0.71,
       "learning_rate": 1.1666074087171627e-07,
-      "logits/chosen": -2.4657585620880127,
-      "logits/rejected": -2.4581031799316406,
-      "logps/chosen": -309.5735778808594,
-      "logps/rejected": -323.62957763671875,
-      "loss": 0.1019,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -1.299774408340454,
-      "rewards/margins": 0.27329882979393005,
-      "rewards/rejected": -1.5730730295181274,
+      "logits/chosen": -2.106621265411377,
+      "logits/rejected": -2.084312677383423,
+      "logps/chosen": -303.54571533203125,
+      "logps/rejected": -323.85528564453125,
+      "loss": 0.5842,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.239495873451233,
+      "rewards/margins": 0.33583495020866394,
+      "rewards/rejected": -1.5753308534622192,
       "step": 340
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0157994641835734e-07,
-      "logits/chosen": -2.4129486083984375,
-      "logits/rejected": -2.410670518875122,
-      "logps/chosen": -305.326416015625,
-      "logps/rejected": -335.5422058105469,
-      "loss": 0.093,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -1.2445926666259766,
-      "rewards/margins": 0.33277326822280884,
-      "rewards/rejected": -1.5773658752441406,
+      "logits/chosen": -2.0272693634033203,
+      "logits/rejected": -2.014112949371338,
+      "logps/chosen": -298.81707763671875,
+      "logps/rejected": -342.98248291015625,
+      "loss": 0.602,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.1794991493225098,
+      "rewards/margins": 0.47226953506469727,
+      "rewards/rejected": -1.651768684387207,
       "step": 350
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.729103716819111e-08,
-      "logits/chosen": -2.453228712081909,
-      "logits/rejected": -2.4483864307403564,
-      "logps/chosen": -293.7769775390625,
-      "logps/rejected": -314.35516357421875,
-      "loss": 0.0913,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -1.3550714254379272,
-      "rewards/margins": 0.18412788212299347,
-      "rewards/rejected": -1.5391991138458252,
+      "logits/chosen": -2.0633723735809326,
+      "logits/rejected": -2.032658100128174,
+      "logps/chosen": -288.3153991699219,
+      "logps/rejected": -329.348876953125,
+      "loss": 0.5949,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.3004556894302368,
+      "rewards/margins": 0.3886806070804596,
+      "rewards/rejected": -1.6891365051269531,
       "step": 360
     },
     {
       "epoch": 0.77,
       "learning_rate": 7.387025063449081e-08,
-      "logits/chosen": -2.45651912689209,
-      "logits/rejected": -2.427839517593384,
-      "logps/chosen": -282.86907958984375,
-      "logps/rejected": -292.1116027832031,
-      "loss": 0.0865,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -1.2447216510772705,
-      "rewards/margins": 0.21571063995361328,
-      "rewards/rejected": -1.4604320526123047,
+      "logits/chosen": -2.0877535343170166,
+      "logits/rejected": -2.0346691608428955,
+      "logps/chosen": -288.3091125488281,
+      "logps/rejected": -319.13067626953125,
+      "loss": 0.5867,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.2991220951080322,
+      "rewards/margins": 0.4315001368522644,
+      "rewards/rejected": -1.7306222915649414,
       "step": 370
     },
     {
       "epoch": 0.79,
       "learning_rate": 6.138919252022435e-08,
-      "logits/chosen": -2.4189741611480713,
-      "logits/rejected": -2.4400978088378906,
-      "logps/chosen": -265.35223388671875,
-      "logps/rejected": -311.8428649902344,
-      "loss": 0.0957,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -1.2127383947372437,
-      "rewards/margins": 0.3842606544494629,
-      "rewards/rejected": -1.5969990491867065,
+      "logits/chosen": -2.0052437782287598,
+      "logits/rejected": -2.026928663253784,
+      "logps/chosen": -279.9800720214844,
+      "logps/rejected": -343.59759521484375,
+      "loss": 0.5808,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.3590168952941895,
+      "rewards/margins": 0.5555295348167419,
+      "rewards/rejected": -1.9145463705062866,
       "step": 380
     },
     {
       "epoch": 0.82,
       "learning_rate": 4.991445467064689e-08,
-      "logits/chosen": -2.4274165630340576,
-      "logits/rejected": -2.423893451690674,
-      "logps/chosen": -304.935302734375,
-      "logps/rejected": -313.81036376953125,
-      "loss": 0.1009,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -1.1629902124404907,
-      "rewards/margins": 0.18845006823539734,
-      "rewards/rejected": -1.351440191268921,
+      "logits/chosen": -2.0364718437194824,
+      "logits/rejected": -2.0138964653015137,
+      "logps/chosen": -342.26214599609375,
+      "logps/rejected": -371.9187316894531,
+      "loss": 0.5832,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.5362588167190552,
+      "rewards/margins": 0.39626526832580566,
+      "rewards/rejected": -1.9325239658355713,
       "step": 390
     },
     {
       "epoch": 0.84,
       "learning_rate": 3.9507259776993954e-08,
-      "logits/chosen": -2.4709787368774414,
-      "logits/rejected": -2.4519667625427246,
-      "logps/chosen": -296.1802673339844,
-      "logps/rejected": -301.5351257324219,
-      "loss": 0.1028,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -1.2224457263946533,
-      "rewards/margins": 0.24025912582874298,
-      "rewards/rejected": -1.4627048969268799,
+      "logits/chosen": -2.0687859058380127,
+      "logits/rejected": -2.024091958999634,
+      "logps/chosen": -327.12469482421875,
+      "logps/rejected": -350.4504699707031,
+      "loss": 0.585,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.5318901538848877,
+      "rewards/margins": 0.41996774077415466,
+      "rewards/rejected": -1.9518581628799438,
       "step": 400
     },
     {
       "epoch": 0.84,
-      "eval_logits/chosen": -2.490896224975586,
-      "eval_logits/rejected": -2.4658799171447754,
-      "eval_logps/chosen": -366.0210266113281,
-      "eval_logps/rejected": -409.55377197265625,
-      "eval_loss": 0.05811962112784386,
-      "eval_rewards/accuracies": 0.703125,
-      "eval_rewards/chosen": -1.0898137092590332,
-      "eval_rewards/margins": 0.4321911036968231,
-      "eval_rewards/rejected": -1.5220048427581787,
-      "eval_runtime": 53.0562,
-      "eval_samples_per_second": 37.696,
-      "eval_steps_per_second": 0.603,
+      "eval_logits/chosen": -2.0913608074188232,
+      "eval_logits/rejected": -2.0418004989624023,
+      "eval_logps/chosen": -387.622802734375,
+      "eval_logps/rejected": -465.2492370605469,
+      "eval_loss": 0.532730221748352,
+      "eval_rewards/accuracies": 0.76953125,
+      "eval_rewards/chosen": -1.3058316707611084,
+      "eval_rewards/margins": 0.7731280326843262,
+      "eval_rewards/rejected": -2.0789597034454346,
+      "eval_runtime": 53.1693,
+      "eval_samples_per_second": 37.616,
+      "eval_steps_per_second": 0.602,
       "step": 400
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.022313472693447e-08,
-      "logits/chosen": -2.4692976474761963,
-      "logits/rejected": -2.4374513626098633,
-      "logps/chosen": -270.84930419921875,
-      "logps/rejected": -300.2176208496094,
-      "loss": 0.0979,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -1.1441447734832764,
-      "rewards/margins": 0.3019401431083679,
-      "rewards/rejected": -1.4460850954055786,
+      "logits/chosen": -2.051201820373535,
+      "logits/rejected": -2.0001602172851562,
+      "logps/chosen": -301.98968505859375,
+      "logps/rejected": -340.74334716796875,
+      "loss": 0.6013,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.455548882484436,
+      "rewards/margins": 0.3957931697368622,
+      "rewards/rejected": -1.8513422012329102,
       "step": 410
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.2111614344599684e-08,
-      "logits/chosen": -2.4384732246398926,
-      "logits/rejected": -2.4019885063171387,
-      "logps/chosen": -277.5793151855469,
-      "logps/rejected": -309.7054138183594,
-      "loss": 0.1012,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -1.2275290489196777,
-      "rewards/margins": 0.209476500749588,
-      "rewards/rejected": -1.437005639076233,
+      "logits/chosen": -2.003211736679077,
+      "logits/rejected": -1.9390989542007446,
+      "logps/chosen": -301.6129455566406,
+      "logps/rejected": -358.48712158203125,
+      "loss": 0.588,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.4678657054901123,
+      "rewards/margins": 0.4569568634033203,
+      "rewards/rejected": -1.9248225688934326,
       "step": 420
     },
     {
       "epoch": 0.9,
       "learning_rate": 1.521597710086439e-08,
-      "logits/chosen": -2.446877956390381,
-      "logits/rejected": -2.4304375648498535,
-      "logps/chosen": -249.771240234375,
-      "logps/rejected": -290.00628662109375,
-      "loss": 0.0996,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.139901041984558,
-      "rewards/margins": 0.34011808037757874,
-      "rewards/rejected": -1.480019211769104,
+      "logits/chosen": -2.049384593963623,
+      "logits/rejected": -2.022660732269287,
+      "logps/chosen": -276.34271240234375,
+      "logps/rejected": -341.69525146484375,
+      "loss": 0.577,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.4056159257888794,
+      "rewards/margins": 0.5912927985191345,
+      "rewards/rejected": -1.9969087839126587,
       "step": 430
     },
     {
       "epoch": 0.92,
       "learning_rate": 9.57301420397924e-09,
-      "logits/chosen": -2.4680263996124268,
-      "logits/rejected": -2.4480295181274414,
-      "logps/chosen": -292.5712585449219,
-      "logps/rejected": -327.3638610839844,
-      "loss": 0.0962,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -1.197205901145935,
-      "rewards/margins": 0.3009336292743683,
-      "rewards/rejected": -1.4981396198272705,
+      "logits/chosen": -2.0629231929779053,
+      "logits/rejected": -2.0302374362945557,
+      "logps/chosen": -324.0228576660156,
+      "logps/rejected": -372.5835266113281,
+      "loss": 0.5727,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.5117217302322388,
+      "rewards/margins": 0.4386143088340759,
+      "rewards/rejected": -1.9503358602523804,
       "step": 440
     },
     {
       "epoch": 0.94,
       "learning_rate": 5.212833302556258e-09,
-      "logits/chosen": -2.389580249786377,
-      "logits/rejected": -2.383848190307617,
-      "logps/chosen": -311.14459228515625,
-      "logps/rejected": -335.6319885253906,
-      "loss": 0.0973,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.1656043529510498,
-      "rewards/margins": 0.3214489221572876,
-      "rewards/rejected": -1.4870531558990479,
+      "logits/chosen": -2.0090086460113525,
+      "logits/rejected": -1.9827800989151,
+      "logps/chosen": -339.3094482421875,
+      "logps/rejected": -378.0882873535156,
+      "loss": 0.5813,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.4472534656524658,
+      "rewards/margins": 0.464362770318985,
+      "rewards/rejected": -1.9116159677505493,
       "step": 450
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.158697848236607e-09,
-      "logits/chosen": -2.4592392444610596,
-      "logits/rejected": -2.4107604026794434,
-      "logps/chosen": -272.6553039550781,
-      "logps/rejected": -301.021728515625,
-      "loss": 0.0994,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -1.1737346649169922,
-      "rewards/margins": 0.2486286610364914,
-      "rewards/rejected": -1.4223631620407104,
+      "logits/chosen": -2.037766218185425,
+      "logits/rejected": -1.9724490642547607,
+      "logps/chosen": -298.0849609375,
+      "logps/rejected": -351.5104675292969,
+      "loss": 0.5765,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.4280312061309814,
+      "rewards/margins": 0.4992195665836334,
+      "rewards/rejected": -1.927250623703003,
       "step": 460
     },
     {
       "epoch": 0.98,
       "learning_rate": 4.269029751107489e-10,
-      "logits/chosen": -2.4406816959381104,
-      "logits/rejected": -2.423938274383545,
-      "logps/chosen": -276.0345153808594,
-      "logps/rejected": -281.52838134765625,
-      "loss": 0.094,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -1.1824878454208374,
-      "rewards/margins": 0.23259492218494415,
-      "rewards/rejected": -1.4150828123092651,
+      "logits/chosen": -2.028768301010132,
+      "logits/rejected": -1.9958856105804443,
+      "logps/chosen": -302.0503845214844,
+      "logps/rejected": -326.9120178222656,
+      "loss": 0.5964,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.4426469802856445,
+      "rewards/margins": 0.42627257108688354,
+      "rewards/rejected": -1.8689197301864624,
       "step": 470
     },
     {
       "epoch": 1.0,
       "step": 478,
       "total_flos": 0.0,
-      "train_loss": 0.1551352257384416,
-      "train_runtime": 4354.6676,
-      "train_samples_per_second": 14.038,
+      "train_loss": 0.6134292899814111,
+      "train_runtime": 4347.6461,
+      "train_samples_per_second": 14.061,
       "train_steps_per_second": 0.11
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:32e413cb9b194c99d709f023ce90ecb160397c928f7dc78084b24ba0db113938
-size 5944
+oid sha256:68de4b692d7e681073120f26d1d80e9c8ef5217f3be7d9dc2f4046cf139290cd
+size 6008