Model save

README.md

@@ -32,7 +32,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 5e-09
+- learning_rate: 1e-09
 - train_batch_size: 4
 - eval_batch_size: 4
 - seed: 42
@@ -52,7 +52,7 @@ The following hyperparameters were used during training:
 
 ### Framework versions
 
-- Transformers 4.41.1
+- Transformers 4.39.3
 - Pytorch 2.1.2+cu118
 - Datasets 2.16.1
-- Tokenizers 0.19.1
+- Tokenizers 0.15.2

all_results.json

@@ -1,9 +1,8 @@
 {
-    "epoch": 0.9990783410138249,
-    "total_flos": 0.0,
-    "train_loss": 0.7834695007968213,
-    "train_runtime": 9239.3637,
-    "train_samples": 69410,
-    "train_samples_per_second": 7.512,
-    "train_steps_per_second": 0.059
+    "epoch": 1.0,
+    "train_loss": 0.7340851113084674,
+    "train_runtime": 8012.0764,
+    "train_samples": 66084,
+    "train_samples_per_second": 8.248,
+    "train_steps_per_second": 0.064
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "transformers_version": "4.41.1"
+  "transformers_version": "4.39.3"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6c9c59d81358b922cc7dc4a0f1212ed3989092cc463daa04e06aade722a12a55
+oid sha256:98b359f2ed439c25068388b47040cdb10b77f5f5fa53649105ea9d05368882f0
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ad6c951a013cca13cc4d3888f5fd9cebd9dbd9d0db94933753fda8a549ad3f8
+oid sha256:c3fff5c90a9f6a4b054b84b812f4f3aef8e56cc79b45cf52084d4bd2d0be53fc
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:432431dead4b9b1064f55e868128f357e49949a9557abbb160b8a3320007e2bd
+oid sha256:e93c1614085b19e11689c215dea89f11cb83c8457a6e6cc90cb2d26115dc084f
 size 4540516344

runs/Jul07_23-56-32_n136-100-194/events.out.tfevents.1720368281.n136-100-194.1414729.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:441817b12b3830eb99b45ae5ebc5129b9e7a0311347eafc843945004a5dffb69
-size 39904
+oid sha256:d166b3928c1bc2acc55a3c2e2a9d6496526373bf273f5d85b4f36b96b6565889
+size 40946

train_results.json

@@ -1,9 +1,8 @@
 {
-    "epoch": 0.9990783410138249,
-    "total_flos": 0.0,
-    "train_loss": 0.7834695007968213,
-    "train_runtime": 9239.3637,
-    "train_samples": 69410,
-    "train_samples_per_second": 7.512,
-    "train_steps_per_second": 0.059
+    "epoch": 1.0,
+    "train_loss": 0.7340851113084674,
+    "train_runtime": 8012.0764,
+    "train_samples": 66084,
+    "train_samples_per_second": 8.248,
+    "train_steps_per_second": 0.064
 }

trainer_state.json

@@ -1,22 +1,22 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9990783410138249,
+  "epoch": 0.9990319457889641,
   "eval_steps": 10000000,
-  "global_step": 542,
+  "global_step": 516,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0018433179723502304,
-      "grad_norm": 3898.482747931532,
-      "learning_rate": 9.090909090909091e-11,
-      "logits/chosen": -1.6609081029891968,
-      "logits/rejected": -1.6088519096374512,
-      "logps/chosen": -0.9401239156723022,
-      "logps/rejected": -0.9000049829483032,
-      "loss": 0.913,
+      "epoch": 0.0,
+      "grad_norm": 1841.589330811575,
+      "learning_rate": 1.9230769230769234e-11,
+      "logits/chosen": -1.8683955669403076,
+      "logits/rejected": -1.7658718824386597,
+      "logps/chosen": -1.0707917213439941,
+      "logps/rejected": -1.2424218654632568,
+      "loss": 0.6843,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -24,842 +24,785 @@
       "step": 1
     },
     {
-      "epoch": 0.018433179723502304,
-      "grad_norm": 4090.8547186973697,
-      "learning_rate": 9.090909090909091e-10,
-      "logits/chosen": -1.655083417892456,
-      "logits/rejected": -1.5546139478683472,
-      "logps/chosen": -0.9712722301483154,
-      "logps/rejected": -0.9472112655639648,
-      "loss": 0.9926,
-      "rewards/accuracies": 0.3472222089767456,
-      "rewards/chosen": 0.004639791324734688,
-      "rewards/margins": -0.15443584322929382,
-      "rewards/rejected": 0.15907563269138336,
+      "epoch": 0.02,
+      "grad_norm": 1996.1274022232458,
+      "learning_rate": 1.9230769230769234e-10,
+      "logits/chosen": -1.661571979522705,
+      "logits/rejected": -1.6195077896118164,
+      "logps/chosen": -0.9485270380973816,
+      "logps/rejected": -0.9299606680870056,
+      "loss": 0.7139,
+      "rewards/accuracies": 0.3402777910232544,
+      "rewards/chosen": -0.010606925934553146,
+      "rewards/margins": -0.04614371806383133,
+      "rewards/rejected": 0.035536784678697586,
       "step": 10
     },
     {
-      "epoch": 0.03686635944700461,
-      "grad_norm": 5249.87937646366,
-      "learning_rate": 1.8181818181818182e-09,
-      "logits/chosen": -1.6007907390594482,
-      "logits/rejected": -1.5696344375610352,
-      "logps/chosen": -1.0197725296020508,
-      "logps/rejected": -0.950878918170929,
-      "loss": 0.9901,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": 0.06652222573757172,
-      "rewards/margins": 0.1879386305809021,
-      "rewards/rejected": -0.12141638994216919,
+      "epoch": 0.04,
+      "grad_norm": 1476.5922039435188,
+      "learning_rate": 3.8461538461538467e-10,
+      "logits/chosen": -1.5835134983062744,
+      "logits/rejected": -1.5358213186264038,
+      "logps/chosen": -1.0245001316070557,
+      "logps/rejected": -0.9702553749084473,
+      "loss": 0.7328,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.002407646970823407,
+      "rewards/margins": 0.012017359957098961,
+      "rewards/rejected": -0.009609714150428772,
       "step": 20
     },
     {
-      "epoch": 0.055299539170506916,
-      "grad_norm": 3473.4229634351905,
-      "learning_rate": 2.727272727272727e-09,
-      "logits/chosen": -1.6087005138397217,
-      "logits/rejected": -1.5368092060089111,
-      "logps/chosen": -0.9995329976081848,
-      "logps/rejected": -0.999841570854187,
-      "loss": 1.0333,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.08179456740617752,
-      "rewards/margins": -0.014446260407567024,
-      "rewards/rejected": 0.09624083340167999,
+      "epoch": 0.06,
+      "grad_norm": 2327.1082042354296,
+      "learning_rate": 5.769230769230769e-10,
+      "logits/chosen": -1.5297813415527344,
+      "logits/rejected": -1.470786690711975,
+      "logps/chosen": -1.003636360168457,
+      "logps/rejected": -0.9809616208076477,
+      "loss": 0.7501,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.0802309662103653,
+      "rewards/margins": -0.03612281754612923,
+      "rewards/rejected": -0.04410814121365547,
       "step": 30
     },
     {
-      "epoch": 0.07373271889400922,
-      "grad_norm": 4366.816381959647,
-      "learning_rate": 3.6363636363636364e-09,
-      "logits/chosen": -1.6297199726104736,
-      "logits/rejected": -1.527199625968933,
-      "logps/chosen": -0.9844824075698853,
-      "logps/rejected": -0.9811803102493286,
-      "loss": 1.0512,
-      "rewards/accuracies": 0.45625001192092896,
-      "rewards/chosen": 0.014942830428481102,
-      "rewards/margins": -0.04575050622224808,
-      "rewards/rejected": 0.06069333478808403,
+      "epoch": 0.08,
+      "grad_norm": 1754.5725922762788,
+      "learning_rate": 7.692307692307693e-10,
+      "logits/chosen": -1.5550693273544312,
+      "logits/rejected": -1.4969508647918701,
+      "logps/chosen": -0.9998669624328613,
+      "logps/rejected": -0.9377104640007019,
+      "loss": 0.7259,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.02974640764296055,
+      "rewards/margins": 0.025205513462424278,
+      "rewards/rejected": 0.004540898837149143,
       "step": 40
     },
     {
-      "epoch": 0.09216589861751152,
-      "grad_norm": 4310.555577886656,
-      "learning_rate": 4.545454545454545e-09,
-      "logits/chosen": -1.5640833377838135,
-      "logits/rejected": -1.491424322128296,
-      "logps/chosen": -0.979416012763977,
-      "logps/rejected": -0.8847635984420776,
-      "loss": 1.0104,
-      "rewards/accuracies": 0.46875,
-      "rewards/chosen": -0.07140497118234634,
-      "rewards/margins": -0.12020325660705566,
-      "rewards/rejected": 0.04879828169941902,
+      "epoch": 0.1,
+      "grad_norm": 1834.9151562164598,
+      "learning_rate": 9.615384615384616e-10,
+      "logits/chosen": -1.6489346027374268,
+      "logits/rejected": -1.603512167930603,
+      "logps/chosen": -0.9903966784477234,
+      "logps/rejected": -0.968778133392334,
+      "loss": 0.7275,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.0027869821060448885,
+      "rewards/margins": 0.03262440487742424,
+      "rewards/rejected": -0.03541138768196106,
       "step": 50
     },
     {
-      "epoch": 0.11059907834101383,
-      "grad_norm": 3816.3756252973885,
-      "learning_rate": 4.99869966817273e-09,
-      "logits/chosen": -1.5969488620758057,
-      "logits/rejected": -1.5034626722335815,
-      "logps/chosen": -1.027340054512024,
-      "logps/rejected": -1.0131927728652954,
-      "loss": 1.0351,
-      "rewards/accuracies": 0.4437499940395355,
-      "rewards/chosen": -0.028199095278978348,
-      "rewards/margins": -0.04337681084871292,
-      "rewards/rejected": 0.015177717432379723,
+      "epoch": 0.12,
+      "grad_norm": 1858.8148428923591,
+      "learning_rate": 9.99266706925562e-10,
+      "logits/chosen": -1.6008754968643188,
+      "logits/rejected": -1.5484288930892944,
+      "logps/chosen": -0.9955031275749207,
+      "logps/rejected": -0.931098461151123,
+      "loss": 0.7562,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.00754826795309782,
+      "rewards/margins": -0.04739421606063843,
+      "rewards/rejected": 0.039845943450927734,
       "step": 60
     },
     {
-      "epoch": 0.12903225806451613,
-      "grad_norm": 4072.2596400359776,
-      "learning_rate": 4.98830512828915e-09,
-      "logits/chosen": -1.7021442651748657,
-      "logits/rejected": -1.6100256443023682,
-      "logps/chosen": -0.9533787965774536,
-      "logps/rejected": -0.9375246167182922,
-      "loss": 1.0037,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": 0.08136853575706482,
-      "rewards/margins": -0.00438536424189806,
-      "rewards/rejected": 0.08575389534235,
+      "epoch": 0.14,
+      "grad_norm": 2425.8586612790064,
+      "learning_rate": 9.96291389741603e-10,
+      "logits/chosen": -1.5926530361175537,
+      "logits/rejected": -1.5061492919921875,
+      "logps/chosen": -0.9957473874092102,
+      "logps/rejected": -0.9462421536445618,
+      "loss": 0.7475,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.044678620994091034,
+      "rewards/margins": 0.04425480216741562,
+      "rewards/rejected": 0.0004238195833750069,
       "step": 70
     },
     {
-      "epoch": 0.14746543778801843,
-      "grad_norm": 4118.420754219571,
-      "learning_rate": 4.967559289596846e-09,
-      "logits/chosen": -1.5828940868377686,
-      "logits/rejected": -1.5408188104629517,
-      "logps/chosen": -0.991844654083252,
-      "logps/rejected": -0.9241277575492859,
-      "loss": 1.006,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.0019341229926794767,
-      "rewards/margins": -0.02749333344399929,
-      "rewards/rejected": 0.029427463188767433,
+      "epoch": 0.15,
+      "grad_norm": 1974.3624260956421,
+      "learning_rate": 9.91041841371078e-10,
+      "logits/chosen": -1.5462015867233276,
+      "logits/rejected": -1.52878737449646,
+      "logps/chosen": -1.0357959270477295,
+      "logps/rejected": -0.9859043955802917,
+      "loss": 0.737,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.019811829552054405,
+      "rewards/margins": 0.09474115073680878,
+      "rewards/rejected": -0.07492931932210922,
       "step": 80
     },
     {
-      "epoch": 0.16589861751152074,
-      "grad_norm": 3629.5409761234123,
-      "learning_rate": 4.936548454363532e-09,
-      "logits/chosen": -1.6866261959075928,
-      "logits/rejected": -1.618090271949768,
-      "logps/chosen": -1.0860463380813599,
-      "logps/rejected": -0.9448333978652954,
-      "loss": 0.9926,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.08469289541244507,
-      "rewards/margins": 0.13724537193775177,
-      "rewards/rejected": -0.0525524728000164,
+      "epoch": 0.17,
+      "grad_norm": 1805.8384539876447,
+      "learning_rate": 9.835421176144035e-10,
+      "logits/chosen": -1.6784532070159912,
+      "logits/rejected": -1.6200278997421265,
+      "logps/chosen": -1.0107990503311157,
+      "logps/rejected": -0.924695611000061,
+      "loss": 0.7294,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.06031709909439087,
+      "rewards/margins": 0.016556955873966217,
+      "rewards/rejected": 0.04376014322042465,
       "step": 90
     },
     {
-      "epoch": 0.18433179723502305,
-      "grad_norm": 3898.733244088321,
-      "learning_rate": 4.895401627034106e-09,
-      "logits/chosen": -1.5633373260498047,
-      "logits/rejected": -1.4719525575637817,
-      "logps/chosen": -1.0307183265686035,
-      "logps/rejected": -0.9637019038200378,
-      "loss": 0.9422,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": 0.1219116598367691,
-      "rewards/margins": 0.014806958846747875,
-      "rewards/rejected": 0.10710470378398895,
+      "epoch": 0.19,
+      "grad_norm": 1972.7826610383331,
+      "learning_rate": 9.738265855914014e-10,
+      "logits/chosen": -1.632147192955017,
+      "logits/rejected": -1.56899094581604,
+      "logps/chosen": -0.9780662655830383,
+      "logps/rejected": -0.9384719133377075,
+      "loss": 0.7363,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 0.010622555390000343,
+      "rewards/margins": -0.028813939541578293,
+      "rewards/rejected": 0.03943649306893349,
       "step": 100
     },
     {
-      "epoch": 0.20276497695852536,
-      "grad_norm": 3696.4743498583503,
-      "learning_rate": 4.844289977574774e-09,
-      "logits/chosen": -1.5632708072662354,
-      "logits/rejected": -1.496795892715454,
-      "logps/chosen": -1.0272352695465088,
-      "logps/rejected": -0.9775659441947937,
-      "loss": 0.9388,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.17640800774097443,
-      "rewards/margins": 0.18908366560935974,
-      "rewards/rejected": -0.012675672769546509,
+      "epoch": 0.21,
+      "grad_norm": 2332.6350905446534,
+      "learning_rate": 9.619397662556434e-10,
+      "logits/chosen": -1.6633880138397217,
+      "logits/rejected": -1.5872291326522827,
+      "logps/chosen": -0.8982691764831543,
+      "logps/rejected": -0.8728898167610168,
+      "loss": 0.7458,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": 0.03617207705974579,
+      "rewards/margins": 0.01802017167210579,
+      "rewards/rejected": 0.01815190538764,
       "step": 110
     },
     {
-      "epoch": 0.22119815668202766,
-      "grad_norm": 3181.2117454771997,
-      "learning_rate": 4.783426129409464e-09,
-      "logits/chosen": -1.6399650573730469,
-      "logits/rejected": -1.528159737586975,
-      "logps/chosen": -0.9425110816955566,
-      "logps/rejected": -0.9258049130439758,
-      "loss": 0.9311,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.13392865657806396,
-      "rewards/margins": 0.16803190112113953,
-      "rewards/rejected": -0.03410324081778526,
+      "epoch": 0.23,
+      "grad_norm": 1941.4125890247126,
+      "learning_rate": 9.47936130379344e-10,
+      "logits/chosen": -1.54219651222229,
+      "logits/rejected": -1.522878885269165,
+      "logps/chosen": -0.9705562591552734,
+      "logps/rejected": -0.9400444030761719,
+      "loss": 0.7336,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.059115856885910034,
+      "rewards/margins": 0.04324622079730034,
+      "rewards/rejected": 0.01586962677538395,
       "step": 120
     },
     {
-      "epoch": 0.23963133640552994,
-      "grad_norm": 2982.5336500421463,
-      "learning_rate": 4.713063274910708e-09,
-      "logits/chosen": -1.5521310567855835,
-      "logits/rejected": -1.4859836101531982,
-      "logps/chosen": -1.0397251844406128,
-      "logps/rejected": -0.9695302844047546,
-      "loss": 0.9569,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": 0.1583443433046341,
-      "rewards/margins": 0.1197737455368042,
-      "rewards/rejected": 0.03857060521841049,
+      "epoch": 0.25,
+      "grad_norm": 1715.1411693424232,
+      "learning_rate": 9.318798489436919e-10,
+      "logits/chosen": -1.57927405834198,
+      "logits/rejected": -1.489496111869812,
+      "logps/chosen": -0.9656535983085632,
+      "logps/rejected": -0.9289010167121887,
+      "loss": 0.7313,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.053683798760175705,
+      "rewards/margins": 0.05398359149694443,
+      "rewards/rejected": -0.00029979198006913066,
       "step": 130
     },
     {
-      "epoch": 0.25806451612903225,
-      "grad_norm": 3620.919118773496,
-      "learning_rate": 4.633494122124504e-09,
-      "logits/chosen": -1.5602772235870361,
-      "logits/rejected": -1.5094670057296753,
-      "logps/chosen": -1.0015778541564941,
-      "logps/rejected": -0.9640843272209167,
-      "loss": 0.8817,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.1787305474281311,
-      "rewards/margins": 0.17238874733448029,
-      "rewards/rejected": 0.006341800093650818,
+      "epoch": 0.27,
+      "grad_norm": 1655.2138808718737,
+      "learning_rate": 9.138444990784454e-10,
+      "logits/chosen": -1.575226068496704,
+      "logits/rejected": -1.5274800062179565,
+      "logps/chosen": -0.9927815198898315,
+      "logps/rejected": -0.9941291809082031,
+      "loss": 0.7376,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.016064394265413284,
+      "rewards/margins": 0.029078301042318344,
+      "rewards/rejected": -0.013013908639550209,
       "step": 140
     },
     {
-      "epoch": 0.2764976958525346,
-      "grad_norm": 4013.4092182868308,
-      "learning_rate": 4.545049677110793e-09,
-      "logits/chosen": -1.5850383043289185,
-      "logits/rejected": -1.5053658485412598,
-      "logps/chosen": -0.9671676754951477,
-      "logps/rejected": -0.9508814811706543,
-      "loss": 0.8971,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.3438531458377838,
-      "rewards/margins": 0.29903319478034973,
-      "rewards/rejected": 0.044819992035627365,
+      "epoch": 0.29,
+      "grad_norm": 1948.0751706535148,
+      "learning_rate": 8.939127268983109e-10,
+      "logits/chosen": -1.560948133468628,
+      "logits/rejected": -1.5372627973556519,
+      "logps/chosen": -1.0732929706573486,
+      "logps/rejected": -0.9957958459854126,
+      "loss": 0.7216,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.07632608711719513,
+      "rewards/margins": 0.030022624880075455,
+      "rewards/rejected": 0.046303462237119675,
       "step": 150
     },
     {
-      "epoch": 0.29493087557603687,
-      "grad_norm": 3947.6099295976574,
-      "learning_rate": 4.4480978669649716e-09,
-      "logits/chosen": -1.5827395915985107,
-      "logits/rejected": -1.4827316999435425,
-      "logps/chosen": -0.9799680709838867,
-      "logps/rejected": -0.9488567113876343,
-      "loss": 0.8639,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.31666576862335205,
-      "rewards/margins": 0.38411837816238403,
-      "rewards/rejected": -0.06745252758264542,
+      "epoch": 0.31,
+      "grad_norm": 1823.5612035138797,
+      "learning_rate": 8.721758687811352e-10,
+      "logits/chosen": -1.6760711669921875,
+      "logits/rejected": -1.6009712219238281,
+      "logps/chosen": -0.9717607498168945,
+      "logps/rejected": -0.9428688883781433,
+      "loss": 0.7558,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": 0.010309430770576,
+      "rewards/margins": -0.04144478961825371,
+      "rewards/rejected": 0.05175423622131348,
       "step": 160
     },
     {
-      "epoch": 0.31336405529953915,
-      "grad_norm": 3208.2709652021513,
-      "learning_rate": 4.343042009248641e-09,
-      "logits/chosen": -1.6824086904525757,
-      "logits/rejected": -1.6013917922973633,
-      "logps/chosen": -1.0053058862686157,
-      "logps/rejected": -0.9279009699821472,
-      "loss": 0.8277,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.48275047540664673,
-      "rewards/margins": 0.4515896737575531,
-      "rewards/rejected": 0.03116079606115818,
+      "epoch": 0.33,
+      "grad_norm": 1740.6546374262923,
+      "learning_rate": 8.487335328233912e-10,
+      "logits/chosen": -1.5352542400360107,
+      "logits/rejected": -1.4410475492477417,
+      "logps/chosen": -0.9954330325126648,
+      "logps/rejected": -0.9735302925109863,
+      "loss": 0.7379,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.04769414663314819,
+      "rewards/margins": 0.01896754838526249,
+      "rewards/rejected": 0.028726596385240555,
       "step": 170
     },
     {
-      "epoch": 0.3317972350230415,
-      "grad_norm": 3131.9144039177236,
-      "learning_rate": 4.230319134196747e-09,
-      "logits/chosen": -1.5756456851959229,
-      "logits/rejected": -1.4971188306808472,
-      "logps/chosen": -1.0033220052719116,
-      "logps/rejected": -0.9541338086128235,
-      "loss": 0.8293,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.5112671852111816,
-      "rewards/margins": 0.4981989860534668,
-      "rewards/rejected": 0.013068236410617828,
+      "epoch": 0.35,
+      "grad_norm": 1979.6537790499867,
+      "learning_rate": 8.236931423909139e-10,
+      "logits/chosen": -1.672133207321167,
+      "logits/rejected": -1.5740010738372803,
+      "logps/chosen": -0.9754056930541992,
+      "logps/rejected": -0.951374351978302,
+      "loss": 0.724,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.05998270958662033,
+      "rewards/margins": 0.04238981008529663,
+      "rewards/rejected": 0.017592918127775192,
       "step": 180
     },
     {
-      "epoch": 0.35023041474654376,
-      "grad_norm": 2715.064519767043,
-      "learning_rate": 4.110398166680671e-09,
-      "logits/chosen": -1.5660308599472046,
-      "logits/rejected": -1.4755548238754272,
-      "logps/chosen": -0.9532132148742676,
-      "logps/rejected": -0.8907047510147095,
-      "loss": 0.7906,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.6197713017463684,
-      "rewards/margins": 0.528595507144928,
-      "rewards/rejected": 0.09117577970027924,
+      "epoch": 0.37,
+      "grad_norm": 1975.0077002715911,
+      "learning_rate": 7.971694438565449e-10,
+      "logits/chosen": -1.6265771389007568,
+      "logits/rejected": -1.5644137859344482,
+      "logps/chosen": -0.9732062220573425,
+      "logps/rejected": -0.9743107557296753,
+      "loss": 0.7296,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.016829270869493484,
+      "rewards/margins": -0.01913767121732235,
+      "rewards/rejected": 0.0023084029089659452,
       "step": 190
     },
     {
-      "epoch": 0.3686635944700461,
-      "grad_norm": 3468.2349150799664,
-      "learning_rate": 3.9837779754902876e-09,
-      "logits/chosen": -1.7327858209609985,
-      "logits/rejected": -1.6148407459259033,
-      "logps/chosen": -1.0091302394866943,
-      "logps/rejected": -0.9430079460144043,
-      "loss": 0.7922,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.5713268518447876,
-      "rewards/margins": 0.5340843200683594,
-      "rewards/rejected": 0.03724261373281479,
+      "epoch": 0.39,
+      "grad_norm": 1823.8748366422865,
+      "learning_rate": 7.692839807804521e-10,
+      "logits/chosen": -1.6312087774276733,
+      "logits/rejected": -1.5969831943511963,
+      "logps/chosen": -0.9732693433761597,
+      "logps/rejected": -0.9268602132797241,
+      "loss": 0.7454,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.04321768134832382,
+      "rewards/margins": -0.05310916155576706,
+      "rewards/rejected": 0.009891483001410961,
       "step": 200
     },
     {
-      "epoch": 0.3870967741935484,
-      "grad_norm": 5541.889609678496,
-      "learning_rate": 3.8509852980499295e-09,
-      "logits/chosen": -1.5810222625732422,
-      "logits/rejected": -1.5188120603561401,
-      "logps/chosen": -0.9732600450515747,
-      "logps/rejected": -0.8989869356155396,
-      "loss": 0.8017,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.5253499746322632,
-      "rewards/margins": 0.5527364015579224,
-      "rewards/rejected": -0.027386415749788284,
+      "epoch": 0.41,
+      "grad_norm": 1665.7874215210654,
+      "learning_rate": 7.401645369426697e-10,
+      "logits/chosen": -1.606499433517456,
+      "logits/rejected": -1.5401887893676758,
+      "logps/chosen": -0.9636400938034058,
+      "logps/rejected": -0.9143346548080444,
+      "loss": 0.7287,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": 0.021665044128894806,
+      "rewards/margins": -0.03331884369254112,
+      "rewards/rejected": 0.05498389154672623,
       "step": 210
     },
     {
-      "epoch": 0.4055299539170507,
-      "grad_norm": 3226.71468419498,
-      "learning_rate": 3.7125725492013986e-09,
-      "logits/chosen": -1.6166194677352905,
-      "logits/rejected": -1.5297718048095703,
-      "logps/chosen": -0.9580337405204773,
-      "logps/rejected": -0.9028044939041138,
-      "loss": 0.7911,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": 0.6303132772445679,
-      "rewards/margins": 0.5151628255844116,
-      "rewards/rejected": 0.11515048891305923,
+      "epoch": 0.43,
+      "grad_norm": 1814.5837493015931,
+      "learning_rate": 7.099445507801324e-10,
+      "logits/chosen": -1.6374841928482056,
+      "logits/rejected": -1.590040922164917,
+      "logps/chosen": -0.9993332624435425,
+      "logps/rejected": -0.9410373568534851,
+      "loss": 0.7117,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.03604020178318024,
+      "rewards/margins": 0.030328240245580673,
+      "rewards/rejected": 0.005711960140615702,
       "step": 220
     },
     {
-      "epoch": 0.423963133640553,
-      "grad_norm": 3730.157392610105,
-      "learning_rate": 3.5691155231694373e-09,
-      "logits/chosen": -1.6063117980957031,
-      "logits/rejected": -1.5548478364944458,
-      "logps/chosen": -0.9622556567192078,
-      "logps/rejected": -0.9150797128677368,
-      "loss": 0.7521,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.6007626056671143,
-      "rewards/margins": 0.6586055755615234,
-      "rewards/rejected": -0.057842958718538284,
+      "epoch": 0.45,
+      "grad_norm": 2021.643245419743,
+      "learning_rate": 6.7876250391152e-10,
+      "logits/chosen": -1.5899055004119873,
+      "logits/rejected": -1.5487146377563477,
+      "logps/chosen": -0.9511580467224121,
+      "logps/rejected": -0.9713302850723267,
+      "loss": 0.7297,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.03084571287035942,
+      "rewards/margins": 0.06426262110471725,
+      "rewards/rejected": -0.03341691941022873,
       "step": 230
     },
     {
-      "epoch": 0.4423963133640553,
-      "grad_norm": 3478.733032746658,
-      "learning_rate": 3.421210998269447e-09,
-      "logits/chosen": -1.6624641418457031,
-      "logits/rejected": -1.6196787357330322,
-      "logps/chosen": -1.015082597732544,
-      "logps/rejected": -0.9574974775314331,
-      "loss": 0.7876,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.6061643362045288,
-      "rewards/margins": 0.4766843914985657,
-      "rewards/rejected": 0.1294800341129303,
+      "epoch": 0.46,
+      "grad_norm": 2420.5535415129702,
+      "learning_rate": 6.467612865519674e-10,
+      "logits/chosen": -1.6261504888534546,
+      "logits/rejected": -1.6002610921859741,
+      "logps/chosen": -0.9878660440444946,
+      "logps/rejected": -0.9101985692977905,
+      "loss": 0.7436,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -0.028089094907045364,
+      "rewards/margins": -0.04090605676174164,
+      "rewards/rejected": 0.012816962786018848,
       "step": 240
     },
     {
-      "epoch": 0.4608294930875576,
-      "grad_norm": 4199.026342735118,
-      "learning_rate": 3.269474254321818e-09,
-      "logits/chosen": -1.6640870571136475,
-      "logits/rejected": -1.611088514328003,
-      "logps/chosen": -0.9812959432601929,
-      "logps/rejected": -0.9682718515396118,
-      "loss": 0.7719,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.734277606010437,
-      "rewards/margins": 0.5840839147567749,
-      "rewards/rejected": 0.15019364655017853,
+      "epoch": 0.48,
+      "grad_norm": 1826.8127431178382,
+      "learning_rate": 6.14087542725593e-10,
+      "logits/chosen": -1.6483690738677979,
+      "logits/rejected": -1.6149110794067383,
+      "logps/chosen": -1.0198554992675781,
+      "logps/rejected": -0.9351280331611633,
+      "loss": 0.7446,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.016190512105822563,
+      "rewards/margins": -7.110387377906591e-05,
+      "rewards/rejected": 0.016261618584394455,
       "step": 250
     },
     {
-      "epoch": 0.4792626728110599,
-      "grad_norm": 2843.441875870463,
-      "learning_rate": 3.1145365131003605e-09,
-      "logits/chosen": -1.6078466176986694,
-      "logits/rejected": -1.5514041185379028,
-      "logps/chosen": -0.9493719935417175,
-      "logps/rejected": -0.8929805755615234,
-      "loss": 0.7262,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 0.7151438593864441,
-      "rewards/margins": 0.7053098082542419,
-      "rewards/rejected": 0.00983402505517006,
+      "epoch": 0.5,
+      "grad_norm": 1733.386808938765,
+      "learning_rate": 5.808909982763825e-10,
+      "logits/chosen": -1.6453545093536377,
+      "logits/rejected": -1.554999828338623,
+      "logps/chosen": -0.9937711954116821,
+      "logps/rejected": -0.9480551481246948,
+      "loss": 0.737,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": 0.027714818716049194,
+      "rewards/margins": -0.043810728937387466,
+      "rewards/rejected": 0.07152555137872696,
       "step": 260
     },
     {
-      "epoch": 0.4976958525345622,
-      "grad_norm": 3021.910017633022,
-      "learning_rate": 2.95704231246255e-09,
-      "logits/chosen": -1.641345739364624,
-      "logits/rejected": -1.5682355165481567,
-      "logps/chosen": -0.9710962176322937,
-      "logps/rejected": -0.9229364395141602,
-      "loss": 0.7197,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.8893934488296509,
-      "rewards/margins": 0.7715497016906738,
-      "rewards/rejected": 0.11784378439188004,
+      "epoch": 0.52,
+      "grad_norm": 1823.0310181293855,
+      "learning_rate": 5.473237747567806e-10,
+      "logits/chosen": -1.6353635787963867,
+      "logits/rejected": -1.5713129043579102,
+      "logps/chosen": -0.9697279930114746,
+      "logps/rejected": -0.957770824432373,
+      "loss": 0.7381,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.038542490452528,
+      "rewards/margins": -0.031249618157744408,
+      "rewards/rejected": -0.007292867638170719,
       "step": 270
     },
     {
-      "epoch": 0.5161290322580645,
-      "grad_norm": 2769.0118344627704,
-      "learning_rate": 2.797646825085125e-09,
-      "logits/chosen": -1.6461395025253296,
-      "logits/rejected": -1.614833116531372,
-      "logps/chosen": -0.9965534210205078,
-      "logps/rejected": -0.9103838801383972,
-      "loss": 0.7102,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 0.900854229927063,
-      "rewards/margins": 0.7569249868392944,
-      "rewards/rejected": 0.14392916858196259,
+      "epoch": 0.54,
+      "grad_norm": 1896.714047804176,
+      "learning_rate": 5.135396923380673e-10,
+      "logits/chosen": -1.5689613819122314,
+      "logits/rejected": -1.4959182739257812,
+      "logps/chosen": -0.9839603304862976,
+      "logps/rejected": -0.9465781450271606,
+      "loss": 0.7359,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.06361061334609985,
+      "rewards/margins": 0.005773247219622135,
+      "rewards/rejected": 0.05783736705780029,
       "step": 280
     },
     {
-      "epoch": 0.5345622119815668,
-      "grad_norm": 3201.218696167003,
-      "learning_rate": 2.6370131329590557e-09,
-      "logits/chosen": -1.6856491565704346,
-      "logits/rejected": -1.5777591466903687,
-      "logps/chosen": -0.9708482623100281,
-      "logps/rejected": -0.9296213984489441,
-      "loss": 0.6843,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": 0.9540289044380188,
-      "rewards/margins": 0.750830352306366,
-      "rewards/rejected": 0.2031985968351364,
+      "epoch": 0.56,
+      "grad_norm": 2320.8508484306008,
+      "learning_rate": 4.796935649368935e-10,
+      "logits/chosen": -1.574792504310608,
+      "logits/rejected": -1.4957481622695923,
+      "logps/chosen": -1.0472533702850342,
+      "logps/rejected": -0.993044376373291,
+      "loss": 0.7367,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.014282060787081718,
+      "rewards/margins": 0.02767084166407585,
+      "rewards/rejected": -0.013388775289058685,
       "step": 290
     },
     {
-      "epoch": 0.5529953917050692,
-      "grad_norm": 2497.168902383888,
-      "learning_rate": 2.4758094689819246e-09,
-      "logits/chosen": -1.6694984436035156,
-      "logits/rejected": -1.5708268880844116,
-      "logps/chosen": -0.9928863644599915,
-      "logps/rejected": -0.8953903913497925,
-      "loss": 0.6994,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": 0.7281567454338074,
-      "rewards/margins": 0.802379310131073,
-      "rewards/rejected": -0.07422257214784622,
+      "epoch": 0.58,
+      "grad_norm": 1825.0797751905636,
+      "learning_rate": 4.4594049078802925e-10,
+      "logits/chosen": -1.5986864566802979,
+      "logits/rejected": -1.5016899108886719,
+      "logps/chosen": -0.9563978314399719,
+      "logps/rejected": -0.9198349118232727,
+      "loss": 0.7259,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.09224653244018555,
+      "rewards/margins": 0.07078908383846283,
+      "rewards/rejected": 0.021457448601722717,
       "step": 300
     },
     {
-      "epoch": 0.5714285714285714,
-      "grad_norm": 4042.9572154642865,
-      "learning_rate": 2.3147064371226394e-09,
-      "logits/chosen": -1.6897979974746704,
-      "logits/rejected": -1.6063530445098877,
-      "logps/chosen": -0.9562484622001648,
-      "logps/rejected": -0.8966572880744934,
-      "loss": 0.7118,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.9196362495422363,
-      "rewards/margins": 0.9230009317398071,
-      "rewards/rejected": -0.0033645853400230408,
+      "epoch": 0.6,
+      "grad_norm": 2580.7259376510724,
+      "learning_rate": 4.1243514171423466e-10,
+      "logits/chosen": -1.5819472074508667,
+      "logits/rejected": -1.5410352945327759,
+      "logps/chosen": -0.9796406030654907,
+      "logps/rejected": -0.9497900009155273,
+      "loss": 0.7348,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.03940460830926895,
+      "rewards/margins": -0.009207022376358509,
+      "rewards/rejected": 0.048611629754304886,
       "step": 310
     },
     {
-      "epoch": 0.5898617511520737,
-      "grad_norm": 2587.202113246667,
-      "learning_rate": 2.154374222722545e-09,
-      "logits/chosen": -1.5904594659805298,
-      "logits/rejected": -1.5287564992904663,
-      "logps/chosen": -1.0147100687026978,
-      "logps/rejected": -0.9374237060546875,
-      "loss": 0.7142,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 0.939732551574707,
-      "rewards/margins": 0.9331587553024292,
-      "rewards/rejected": 0.006573830731213093,
+      "epoch": 0.62,
+      "grad_norm": 1729.5664068861677,
+      "learning_rate": 3.793310543501473e-10,
+      "logits/chosen": -1.6621357202529907,
+      "logits/rejected": -1.5896662473678589,
+      "logps/chosen": -0.9787474870681763,
+      "logps/rejected": -0.9559493064880371,
+      "loss": 0.7398,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.022534608840942383,
+      "rewards/margins": 0.009389793500304222,
+      "rewards/rejected": -0.031924404203891754,
       "step": 320
     },
     {
-      "epoch": 0.6082949308755761,
-      "grad_norm": 3045.5495188898726,
-      "learning_rate": 1.995479804538004e-09,
-      "logits/chosen": -1.6013281345367432,
-      "logits/rejected": -1.518786072731018,
-      "logps/chosen": -1.043778419494629,
-      "logps/rejected": -0.9321629405021667,
-      "loss": 0.693,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": 0.9917459487915039,
-      "rewards/margins": 0.8832038640975952,
-      "rewards/rejected": 0.10854210704565048,
+      "epoch": 0.64,
+      "grad_norm": 2107.6254901282878,
+      "learning_rate": 3.4677992656811053e-10,
+      "logits/chosen": -1.6322323083877563,
+      "logits/rejected": -1.6013027429580688,
+      "logps/chosen": -1.018520712852478,
+      "logps/rejected": -0.9697484970092773,
+      "loss": 0.7397,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.016207193955779076,
+      "rewards/margins": -0.05724817514419556,
+      "rewards/rejected": 0.04104097932577133,
       "step": 330
     },
     {
-      "epoch": 0.6267281105990783,
-      "grad_norm": 3007.069780597671,
-      "learning_rate": 1.8386841801223184e-09,
-      "logits/chosen": -1.6415650844573975,
-      "logits/rejected": -1.5566316843032837,
-      "logps/chosen": -0.9854591488838196,
-      "logps/rejected": -0.9954888224601746,
-      "loss": 0.6912,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.7283134460449219,
-      "rewards/margins": 0.7907330989837646,
-      "rewards/rejected": -0.06241961196064949,
+      "epoch": 0.66,
+      "grad_norm": 2346.5024165631285,
+      "learning_rate": 3.149309223300428e-10,
+      "logits/chosen": -1.5328480005264282,
+      "logits/rejected": -1.5037202835083008,
+      "logps/chosen": -1.0620964765548706,
+      "logps/rejected": -0.9756827354431152,
+      "loss": 0.7423,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.051995206624269485,
+      "rewards/margins": 0.0909401923418045,
+      "rewards/rejected": -0.03894497826695442,
       "step": 340
     },
     {
-      "epoch": 0.6451612903225806,
-      "grad_norm": 4406.908866729794,
-      "learning_rate": 1.6846396160893263e-09,
-      "logits/chosen": -1.6521275043487549,
-      "logits/rejected": -1.563493013381958,
-      "logps/chosen": -1.016737699508667,
-      "logps/rejected": -0.9376832246780396,
-      "loss": 0.7053,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": 1.0002192258834839,
-      "rewards/margins": 0.828022837638855,
-      "rewards/rejected": 0.17219629883766174,
+      "epoch": 0.68,
+      "grad_norm": 1860.3254470648908,
+      "learning_rate": 2.8392998815082717e-10,
+      "logits/chosen": -1.6585187911987305,
+      "logits/rejected": -1.5651119947433472,
+      "logps/chosen": -1.0541408061981201,
+      "logps/rejected": -1.0554534196853638,
+      "loss": 0.7345,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.07690231502056122,
+      "rewards/margins": 0.0986892357468605,
+      "rewards/rejected": -0.02178690955042839,
       "step": 350
     },
     {
-      "epoch": 0.663594470046083,
-      "grad_norm": 2676.172542864631,
-      "learning_rate": 1.5339869346975361e-09,
-      "logits/chosen": -1.6332248449325562,
-      "logits/rejected": -1.5798089504241943,
-      "logps/chosen": -1.0168434381484985,
-      "logps/rejected": -0.923861026763916,
-      "loss": 0.6317,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 1.0311250686645508,
-      "rewards/margins": 1.0256620645523071,
-      "rewards/rejected": 0.005463090725243092,
+      "epoch": 0.7,
+      "grad_norm": 1848.6713797847476,
+      "learning_rate": 2.5391918430549634e-10,
+      "logits/chosen": -1.6919893026351929,
+      "logits/rejected": -1.630860686302185,
+      "logps/chosen": -1.0240895748138428,
+      "logps/rejected": -0.9422369003295898,
+      "loss": 0.7225,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": 0.052596330642700195,
+      "rewards/margins": 0.016732681542634964,
+      "rewards/rejected": 0.03586364910006523,
       "step": 360
     },
     {
-      "epoch": 0.6820276497695853,
-      "grad_norm": 3538.899443074702,
-      "learning_rate": 1.3873528480425386e-09,
-      "logits/chosen": -1.6682748794555664,
-      "logits/rejected": -1.5604602098464966,
-      "logps/chosen": -0.9834707379341125,
-      "logps/rejected": -0.918461799621582,
-      "loss": 0.6548,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": 1.1218345165252686,
-      "rewards/margins": 0.9969808459281921,
-      "rewards/rejected": 0.1248536929488182,
+      "epoch": 0.72,
+      "grad_norm": 1745.482314493582,
+      "learning_rate": 2.250360338449226e-10,
+      "logits/chosen": -1.7182960510253906,
+      "logits/rejected": -1.705248236656189,
+      "logps/chosen": -0.9747020602226257,
+      "logps/rejected": -0.927233874797821,
+      "loss": 0.716,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.005029407795518637,
+      "rewards/margins": 0.09355296194553375,
+      "rewards/rejected": -0.08852354437112808,
       "step": 370
     },
     {
-      "epoch": 0.7004608294930875,
-      "grad_norm": 2976.475346671877,
-      "learning_rate": 1.2453473509474186e-09,
-      "logits/chosen": -1.660499930381775,
-      "logits/rejected": -1.5915935039520264,
-      "logps/chosen": -0.9880379438400269,
-      "logps/rejected": -0.9460450410842896,
-      "loss": 0.7138,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 1.1316003799438477,
-      "rewards/margins": 0.9571952819824219,
-      "rewards/rejected": 0.17440509796142578,
+      "epoch": 0.74,
+      "grad_norm": 1796.573665803334,
+      "learning_rate": 1.9741289240311756e-10,
+      "logits/chosen": -1.6247894763946533,
+      "logits/rejected": -1.5761014223098755,
+      "logps/chosen": -0.9921888113021851,
+      "logps/rejected": -0.9521619081497192,
+      "loss": 0.7394,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.03736250475049019,
+      "rewards/margins": -0.005258217919617891,
+      "rewards/rejected": 0.04262072592973709,
       "step": 380
     },
     {
-      "epoch": 0.7188940092165899,
-      "grad_norm": 2873.8297776136696,
-      "learning_rate": 1.1085611833966747e-09,
-      "logits/chosen": -1.594157338142395,
-      "logits/rejected": -1.5532026290893555,
-      "logps/chosen": -1.0556713342666626,
-      "logps/rejected": -0.9804666638374329,
-      "loss": 0.7064,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 1.1315768957138062,
-      "rewards/margins": 1.1887867450714111,
-      "rewards/rejected": -0.057209838181734085,
+      "epoch": 0.76,
+      "grad_norm": 1721.8345915070997,
+      "learning_rate": 1.7117634168396773e-10,
+      "logits/chosen": -1.620558500289917,
+      "logits/rejected": -1.5566461086273193,
+      "logps/chosen": -1.003068208694458,
+      "logps/rejected": -0.9802311658859253,
+      "loss": 0.7272,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.0402844175696373,
+      "rewards/margins": 0.06007431820034981,
+      "rewards/rejected": -0.01978989504277706,
       "step": 390
     },
     {
-      "epoch": 0.7373271889400922,
-      "grad_norm": 3810.435964314268,
-      "learning_rate": 9.77563373069879e-10,
-      "logits/chosen": -1.6148662567138672,
-      "logits/rejected": -1.4739339351654053,
-      "logps/chosen": -0.9978437423706055,
-      "logps/rejected": -0.9537578821182251,
-      "loss": 0.655,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 1.0335350036621094,
-      "rewards/margins": 0.9007118940353394,
-      "rewards/rejected": 0.13282322883605957,
+      "epoch": 0.77,
+      "grad_norm": 1711.1281038444054,
+      "learning_rate": 1.4644660940672628e-10,
+      "logits/chosen": -1.6632341146469116,
+      "logits/rejected": -1.5769034624099731,
+      "logps/chosen": -0.996769905090332,
+      "logps/rejected": -0.9531647562980652,
+      "loss": 0.7362,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.02927534654736519,
+      "rewards/margins": 0.02135869301855564,
+      "rewards/rejected": 0.007916651666164398,
       "step": 400
     },
     {
-      "epoch": 0.7557603686635944,
-      "grad_norm": 3353.3656429864536,
-      "learning_rate": 8.528988681980848e-10,
-      "logits/chosen": -1.525498390197754,
-      "logits/rejected": -1.5023047924041748,
-      "logps/chosen": -1.0074002742767334,
-      "logps/rejected": -0.9325093030929565,
-      "loss": 0.6624,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 0.9953389167785645,
-      "rewards/margins": 1.0517241954803467,
-      "rewards/rejected": -0.05638519674539566,
+      "epoch": 0.79,
+      "grad_norm": 2141.9848630404285,
+      "learning_rate": 1.2333701836832813e-10,
+      "logits/chosen": -1.6260700225830078,
+      "logits/rejected": -1.5618007183074951,
+      "logps/chosen": -0.9725567698478699,
+      "logps/rejected": -0.9434686899185181,
+      "loss": 0.7235,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.033636633306741714,
+      "rewards/margins": -0.02703043445944786,
+      "rewards/rejected": -0.006606197915971279,
       "step": 410
     },
     {
-      "epoch": 0.7741935483870968,
-      "grad_norm": 3108.3553021985713,
-      "learning_rate": 7.350862705902492e-10,
-      "logits/chosen": -1.612502098083496,
-      "logits/rejected": -1.5601556301116943,
-      "logps/chosen": -0.9830726385116577,
-      "logps/rejected": -0.9087923169136047,
-      "loss": 0.668,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": 1.2218666076660156,
-      "rewards/margins": 1.0628268718719482,
-      "rewards/rejected": 0.15903989970684052,
+      "epoch": 0.81,
+      "grad_norm": 2481.4760711884373,
+      "learning_rate": 1.0195346714717813e-10,
+      "logits/chosen": -1.5375875234603882,
+      "logits/rejected": -1.5268070697784424,
+      "logps/chosen": -0.9649657011032104,
+      "logps/rejected": -0.9462583661079407,
+      "loss": 0.7543,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.02682650461792946,
+      "rewards/margins": -0.07992960512638092,
+      "rewards/rejected": 0.05310310050845146,
       "step": 420
     },
     {
-      "epoch": 0.7926267281105991,
-      "grad_norm": 2848.4192218675507,
-      "learning_rate": 6.246156782602395e-10,
-      "logits/chosen": -1.6182113885879517,
-      "logits/rejected": -1.5692577362060547,
-      "logps/chosen": -0.9712175130844116,
-      "logps/rejected": -0.9052525758743286,
-      "loss": 0.6509,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.8137975931167603,
-      "rewards/margins": 0.871743381023407,
-      "rewards/rejected": -0.057945869863033295,
+      "epoch": 0.83,
+      "grad_norm": 1967.5434793497254,
+      "learning_rate": 8.239394482805996e-11,
+      "logits/chosen": -1.5937187671661377,
+      "logits/rejected": -1.5494930744171143,
+      "logps/chosen": -1.0188677310943604,
+      "logps/rejected": -0.9621112942695618,
+      "loss": 0.7302,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": 0.057189978659152985,
+      "rewards/margins": 0.0319669134914875,
+      "rewards/rejected": 0.02522306516766548,
       "step": 430
     },
     {
-      "epoch": 0.8110599078341014,
-      "grad_norm": 3760.6065599338026,
-      "learning_rate": 5.219466466290479e-10,
-      "logits/chosen": -1.582889199256897,
-      "logits/rejected": -1.4811861515045166,
-      "logps/chosen": -0.9767929911613464,
-      "logps/rejected": -0.9605540037155151,
-      "loss": 0.6505,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 0.7346361875534058,
-      "rewards/margins": 1.012524962425232,
-      "rewards/rejected": -0.27788880467414856,
+      "epoch": 0.85,
+      "grad_norm": 1970.7510190413827,
+      "learning_rate": 6.474808197191401e-11,
+      "logits/chosen": -1.631608247756958,
+      "logits/rejected": -1.5823745727539062,
+      "logps/chosen": -1.0462229251861572,
+      "logps/rejected": -0.9751529693603516,
+      "loss": 0.7293,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.0702277421951294,
+      "rewards/margins": 0.0910101979970932,
+      "rewards/rejected": -0.020782459527254105,
       "step": 440
     },
     {
-      "epoch": 0.8294930875576036,
-      "grad_norm": 4053.0728361684164,
-      "learning_rate": 4.2750627678356803e-10,
-      "logits/chosen": -1.5421695709228516,
-      "logits/rejected": -1.4773906469345093,
-      "logps/chosen": -0.9987564086914062,
-      "logps/rejected": -0.9237726330757141,
-      "loss": 0.6655,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 1.0704389810562134,
-      "rewards/margins": 1.020028829574585,
-      "rewards/rejected": 0.05040997266769409,
+      "epoch": 0.87,
+      "grad_norm": 2032.2295716484484,
+      "learning_rate": 4.9096739888146e-11,
+      "logits/chosen": -1.622164011001587,
+      "logits/rejected": -1.529624342918396,
+      "logps/chosen": -1.013816475868225,
+      "logps/rejected": -0.9728986620903015,
+      "loss": 0.7463,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": 0.07482706010341644,
+      "rewards/margins": -0.03386348485946655,
+      "rewards/rejected": 0.10869055986404419,
       "step": 450
     },
     {
-      "epoch": 0.847926267281106,
-      "grad_norm": 3444.670240085631,
-      "learning_rate": 3.4168743874474826e-10,
-      "logits/chosen": -1.6144073009490967,
-      "logits/rejected": -1.5566074848175049,
-      "logps/chosen": -1.0152393579483032,
-      "logps/rejected": -0.929918646812439,
-      "loss": 0.6458,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 0.7092639207839966,
-      "rewards/margins": 0.7927274107933044,
-      "rewards/rejected": -0.08346347510814667,
+      "epoch": 0.89,
+      "grad_norm": 1776.741650731322,
+      "learning_rate": 3.5511640091604293e-11,
+      "logits/chosen": -1.5639044046401978,
+      "logits/rejected": -1.5262724161148071,
+      "logps/chosen": -1.0560028553009033,
+      "logps/rejected": -0.9387346506118774,
+      "loss": 0.7216,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.04164951294660568,
+      "rewards/margins": 0.14367111027240753,
+      "rewards/rejected": -0.10202159732580185,
       "step": 460
     },
     {
-      "epoch": 0.8663594470046083,
-      "grad_norm": 3106.2819052615973,
-      "learning_rate": 2.6484713713628097e-10,
-      "logits/chosen": -1.6743053197860718,
-      "logits/rejected": -1.5681962966918945,
-      "logps/chosen": -0.9611037969589233,
-      "logps/rejected": -0.9157723188400269,
-      "loss": 0.6663,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 0.9063161611557007,
-      "rewards/margins": 0.7436623573303223,
-      "rewards/rejected": 0.16265380382537842,
+      "epoch": 0.91,
+      "grad_norm": 1907.6345668857712,
+      "learning_rate": 2.4055035642222225e-11,
+      "logits/chosen": -1.6381734609603882,
+      "logits/rejected": -1.5699491500854492,
+      "logps/chosen": -0.9839171171188354,
+      "logps/rejected": -0.9137029647827148,
+      "loss": 0.7333,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": 0.05203929543495178,
+      "rewards/margins": 0.0004328913928475231,
+      "rewards/rejected": 0.05160640552639961,
       "step": 470
     },
     {
-      "epoch": 0.8847926267281107,
-      "grad_norm": 3028.8363800911566,
-      "learning_rate": 1.9730502605261797e-10,
-      "logits/chosen": -1.5557960271835327,
-      "logits/rejected": -1.5075175762176514,
-      "logps/chosen": -0.9503531455993652,
-      "logps/rejected": -0.919326663017273,
-      "loss": 0.6366,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": 0.9357951283454895,
-      "rewards/margins": 0.953781247138977,
-      "rewards/rejected": -0.01798621378839016,
+      "epoch": 0.93,
+      "grad_norm": 1909.400973276575,
+      "learning_rate": 1.477942587339426e-11,
+      "logits/chosen": -1.609994649887085,
+      "logits/rejected": -1.5665223598480225,
+      "logps/chosen": -0.9485662579536438,
+      "logps/rejected": -0.9070123434066772,
+      "loss": 0.7389,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": 0.026085853576660156,
+      "rewards/margins": -0.039593569934368134,
+      "rewards/rejected": 0.06567941606044769,
       "step": 480
     },
     {
-      "epoch": 0.9032258064516129,
-      "grad_norm": 3953.758778776682,
-      "learning_rate": 1.39342079304427e-10,
-      "logits/chosen": -1.6463558673858643,
-      "logits/rejected": -1.5752487182617188,
-      "logps/chosen": -1.0051090717315674,
-      "logps/rejected": -0.980335533618927,
-      "loss": 0.6936,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 1.2566089630126953,
-      "rewards/margins": 1.2254388332366943,
-      "rewards/rejected": 0.031170058995485306,
+      "epoch": 0.95,
+      "grad_norm": 1986.3370377223296,
+      "learning_rate": 7.727315816331515e-12,
+      "logits/chosen": -1.656057357788086,
+      "logits/rejected": -1.605974793434143,
+      "logps/chosen": -1.0781848430633545,
+      "logps/rejected": -0.9999237060546875,
+      "loss": 0.733,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.023666301742196083,
+      "rewards/margins": -0.07170303910970688,
+      "rewards/rejected": 0.04803674668073654,
       "step": 490
     },
     {
-      "epoch": 0.9216589861751152,
-      "grad_norm": 2794.294715742837,
-      "learning_rate": 9.119942157324367e-11,
-      "logits/chosen": -1.6388028860092163,
-      "logits/rejected": -1.5481789112091064,
-      "logps/chosen": -1.0447478294372559,
-      "logps/rejected": -1.0064018964767456,
-      "loss": 0.6587,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": 1.1869680881500244,
-      "rewards/margins": 1.1323801279067993,
-      "rewards/rejected": 0.054588038474321365,
+      "epoch": 0.97,
+      "grad_norm": 1606.2598352691068,
+      "learning_rate": 2.9310214228202016e-12,
+      "logits/chosen": -1.6220519542694092,
+      "logits/rejected": -1.5478252172470093,
+      "logps/chosen": -0.9590246081352234,
+      "logps/rejected": -0.9175594449043274,
+      "loss": 0.7359,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -0.058479368686676025,
+      "rewards/margins": -0.08338940888643265,
+      "rewards/rejected": 0.024910034611821175,
       "step": 500
     },
     {
-      "epoch": 0.9400921658986175,
-      "grad_norm": 2259.2971695939314,
-      "learning_rate": 5.3077325337695935e-11,
-      "logits/chosen": -1.6551536321640015,
-      "logits/rejected": -1.5781865119934082,
-      "logps/chosen": -0.9668482542037964,
-      "logps/rejected": -0.9251588582992554,
-      "loss": 0.6534,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 1.0340436697006226,
-      "rewards/margins": 0.9162915349006653,
-      "rewards/rejected": 0.1177520900964737,
+      "epoch": 0.99,
+      "grad_norm": 2004.3062556110438,
+      "learning_rate": 4.125214789427734e-13,
+      "logits/chosen": -1.5623807907104492,
+      "logits/rejected": -1.499871015548706,
+      "logps/chosen": -0.993099570274353,
+      "logps/rejected": -0.9391816854476929,
+      "loss": 0.7238,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.05033453181385994,
+      "rewards/margins": 0.11875885725021362,
+      "rewards/rejected": -0.06842432916164398,
       "step": 510
     },
     {
-      "epoch": 0.9585253456221198,
-      "grad_norm": 2756.1908243931707,
-      "learning_rate": 2.5134377744054636e-11,
-      "logits/chosen": -1.6491291522979736,
-      "logits/rejected": -1.5655778646469116,
-      "logps/chosen": -1.0213550329208374,
-      "logps/rejected": -0.9846656918525696,
-      "loss": 0.664,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 1.088184118270874,
-      "rewards/margins": 0.7975610494613647,
-      "rewards/rejected": 0.29062318801879883,
-      "step": 520
-    },
-    {
-      "epoch": 0.9769585253456221,
-      "grad_norm": 3373.335584334466,
-      "learning_rate": 7.486820886929147e-12,
-      "logits/chosen": -1.6995586156845093,
-      "logits/rejected": -1.606285810470581,
-      "logps/chosen": -0.9370242953300476,
-      "logps/rejected": -0.935525119304657,
-      "loss": 0.6841,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 0.8417491912841797,
-      "rewards/margins": 0.7081137895584106,
-      "rewards/rejected": 0.13363537192344666,
-      "step": 530
-    },
-    {
-      "epoch": 0.9953917050691244,
-      "grad_norm": 3339.7952039913303,
-      "learning_rate": 2.080682445118609e-13,
-      "logits/chosen": -1.6441770792007446,
-      "logits/rejected": -1.5704596042633057,
-      "logps/chosen": -0.9470025897026062,
-      "logps/rejected": -0.9030858278274536,
-      "loss": 0.659,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.9892306327819824,
-      "rewards/margins": 0.7913962602615356,
-      "rewards/rejected": 0.19783440232276917,
-      "step": 540
-    },
-    {
-      "epoch": 0.9990783410138249,
-      "step": 542,
+      "epoch": 1.0,
+      "step": 516,
       "total_flos": 0.0,
-      "train_loss": 0.7834695007968213,
-      "train_runtime": 9239.3637,
-      "train_samples_per_second": 7.512,
-      "train_steps_per_second": 0.059
+      "train_loss": 0.7340851113084674,
+      "train_runtime": 8012.0764,
+      "train_samples_per_second": 8.248,
+      "train_steps_per_second": 0.064
     }
   ],
   "logging_steps": 10,
-  "max_steps": 542,
+  "max_steps": 516,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
-  "stateful_callbacks": {
-    "TrainerControl": {
-      "args": {
-        "should_epoch_stop": false,
-        "should_evaluate": false,
-        "should_log": false,
-        "should_save": true,
-        "should_training_stop": false
-      },
-      "attributes": {}
-    }
-  },
   "total_flos": 0.0,
   "train_batch_size": 4,
   "trial_name": null,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5ea88b5abf47b03971988b97f0d835c6b3e16e6b3ba4dd4ea50e5d4a14e5831b
+oid sha256:3b0f084c2495920c1f1c1e0d0165b9a5be48649d9a063da77553719561d5c998
 size 6328