Model save

README.md

@@ -32,7 +32,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 5e-07
+- learning_rate: 2e-07
 - train_batch_size: 4
 - eval_batch_size: 4
 - seed: 42
@@ -52,7 +52,7 @@ The following hyperparameters were used during training:
 
 ### Framework versions
 
-- Transformers 4.39.3
+- Transformers 4.41.1
 - Pytorch 2.1.2+cu118
 - Datasets 2.16.1
-- Tokenizers 0.15.2
+- Tokenizers 0.19.1

all_results.json

@@ -1,8 +1,9 @@
 {
-    "epoch": 1.0,
-    "train_loss": 0.39275855483152927,
-    "train_runtime": 5850.2913,
+    "epoch": 0.9980806142034548,
+    "total_flos": 0.0,
+    "train_loss": 0.2947830175742125,
+    "train_runtime": 6391.2384,
     "train_samples": 50000,
-    "train_samples_per_second": 8.547,
-    "train_steps_per_second": 0.067
+    "train_samples_per_second": 7.823,
+    "train_steps_per_second": 0.061
 }

config.json

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.39.3",
+  "transformers_version": "4.41.1",
   "use_cache": false,
   "vocab_size": 32000
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "transformers_version": "4.39.3"
+  "transformers_version": "4.41.1"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c0dc0c292d489e5548e3b75999fe6cd030919ab96386b961a7432dab2fccb25
+oid sha256:c879c438925d1255e6bb62e95443c7b9be4c54b554c5661246d16917a1112924
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:78055b666c3b09cd419d126d46142b7e4cbc261fd5db79d249138904dfc1a57d
+oid sha256:0c2dd72f78dcf556e6528d803c589c6105e3a837532325d354a805e21db0d419
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:04307748a9391cbe41e028618c312d319041f81f61a38ff22b6d44f0630d8343
+oid sha256:0a55211455915c4c6cc8aa0486c54d0d029dd088874b044c923db3b11eb37773
 size 4540516344

runs/Jun22_03-45-12_n136-082-130/events.out.tfevents.1718999150.n136-082-130.2429577.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:28126c3869c0fe579ef32582aed3225c779b7e3bdb966a58d323f9c2b91f62be
-size 26299
+oid sha256:76c76e9e221e4f8cc6ab660e0755399876a9ef2ac84345577d63bbe730e2236f
+size 32845

tokenizer.json

@@ -134,6 +134,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
+    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

train_results.json

@@ -1,8 +1,9 @@
 {
-    "epoch": 1.0,
-    "train_loss": 0.39275855483152927,
-    "train_runtime": 5850.2913,
+    "epoch": 0.9980806142034548,
+    "total_flos": 0.0,
+    "train_loss": 0.2947830175742125,
+    "train_runtime": 6391.2384,
     "train_samples": 50000,
-    "train_samples_per_second": 8.547,
-    "train_steps_per_second": 0.067
+    "train_samples_per_second": 7.823,
+    "train_steps_per_second": 0.061
 }

trainer_state.json

@@ -9,14 +9,14 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0,
-      "grad_norm": 992.8142742850837,
-      "learning_rate": 1.282051282051282e-08,
+      "epoch": 0.0025591810620601407,
+      "grad_norm": 709.6319834867974,
+      "learning_rate": 5.128205128205128e-09,
       "logits/chosen": -2.5583817958831787,
       "logits/rejected": -2.4487552642822266,
       "logps/chosen": -258.1644592285156,
       "logps/rejected": -216.25729370117188,
-      "loss": 0.6965,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -24,598 +24,598 @@
       "step": 1
     },
     {
-      "epoch": 0.03,
-      "grad_norm": 972.5242632724868,
-      "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -2.606093168258667,
-      "logits/rejected": -2.5532331466674805,
-      "logps/chosen": -267.5519714355469,
-      "logps/rejected": -217.63278198242188,
-      "loss": 0.7101,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.008765195496380329,
-      "rewards/margins": 0.004809141159057617,
-      "rewards/rejected": 0.0039560552686452866,
+      "epoch": 0.025591810620601407,
+      "grad_norm": 670.9725237886468,
+      "learning_rate": 5.128205128205127e-08,
+      "logits/chosen": -2.606004238128662,
+      "logits/rejected": -2.553126573562622,
+      "logps/chosen": -267.5910339355469,
+      "logps/rejected": -217.67140197753906,
+      "loss": 0.6976,
+      "rewards/accuracies": 0.4513888955116272,
+      "rewards/chosen": -0.013270225375890732,
+      "rewards/margins": 0.0032119054812937975,
+      "rewards/rejected": -0.01648213155567646,
       "step": 10
     },
     {
-      "epoch": 0.05,
-      "grad_norm": 659.3539113664132,
-      "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -2.6304450035095215,
-      "logits/rejected": -2.5677618980407715,
-      "logps/chosen": -260.57421875,
-      "logps/rejected": -207.02157592773438,
-      "loss": 0.5261,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.5109978318214417,
-      "rewards/margins": 0.48820367455482483,
-      "rewards/rejected": 0.022794198244810104,
+      "epoch": 0.05118362124120281,
+      "grad_norm": 590.5735450897452,
+      "learning_rate": 1.0256410256410255e-07,
+      "logits/chosen": -2.6245453357696533,
+      "logits/rejected": -2.562206745147705,
+      "logps/chosen": -261.142333984375,
+      "logps/rejected": -207.0738067626953,
+      "loss": 0.6676,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.08092932403087616,
+      "rewards/margins": 0.09077399969100952,
+      "rewards/rejected": -0.009844672866165638,
       "step": 20
     },
     {
-      "epoch": 0.08,
-      "grad_norm": 779.0248633950893,
-      "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -2.648696184158325,
-      "logits/rejected": -2.573786973953247,
-      "logps/chosen": -250.76904296875,
-      "logps/rejected": -198.43540954589844,
-      "loss": 0.3369,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 3.0647289752960205,
-      "rewards/margins": 2.7615580558776855,
-      "rewards/rejected": 0.3031708300113678,
+      "epoch": 0.07677543186180422,
+      "grad_norm": 644.2505273786248,
+      "learning_rate": 1.5384615384615385e-07,
+      "logits/chosen": -2.6289420127868652,
+      "logits/rejected": -2.5566790103912354,
+      "logps/chosen": -253.9628143310547,
+      "logps/rejected": -198.890380859375,
+      "loss": 0.5156,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.5922147035598755,
+      "rewards/margins": 0.6031575798988342,
+      "rewards/rejected": -0.010942881926894188,
       "step": 30
     },
     {
-      "epoch": 0.1,
-      "grad_norm": 384.3988310437092,
-      "learning_rate": 4.99989986344963e-07,
-      "logits/chosen": -2.6418910026550293,
-      "logits/rejected": -2.5625271797180176,
-      "logps/chosen": -243.13668823242188,
-      "logps/rejected": -192.8317108154297,
-      "loss": 0.3186,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 5.048842430114746,
-      "rewards/margins": 4.450641632080078,
-      "rewards/rejected": 0.5982006192207336,
+      "epoch": 0.10236724248240563,
+      "grad_norm": 428.1589759637874,
+      "learning_rate": 1.999959945379852e-07,
+      "logits/chosen": -2.637456178665161,
+      "logits/rejected": -2.5609130859375,
+      "logps/chosen": -247.48666381835938,
+      "logps/rejected": -193.59280395507812,
+      "loss": 0.3866,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 1.4313427209854126,
+      "rewards/margins": 1.3846019506454468,
+      "rewards/rejected": 0.04674070328474045,
       "step": 40
     },
     {
-      "epoch": 0.13,
-      "grad_norm": 557.6508821014772,
-      "learning_rate": 4.987893180827479e-07,
-      "logits/chosen": -2.651369333267212,
-      "logits/rejected": -2.5784316062927246,
-      "logps/chosen": -257.6221923828125,
-      "logps/rejected": -203.21920776367188,
-      "loss": 0.3491,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 7.43048620223999,
-      "rewards/margins": 6.079649448394775,
-      "rewards/rejected": 1.3508365154266357,
+      "epoch": 0.12795905310300704,
+      "grad_norm": 366.0325785868742,
+      "learning_rate": 1.9951572723309917e-07,
+      "logits/chosen": -2.684983730316162,
+      "logits/rejected": -2.618076801300049,
+      "logps/chosen": -262.3794250488281,
+      "logps/rejected": -204.0382843017578,
+      "loss": 0.3488,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 2.9288697242736816,
+      "rewards/margins": 2.3735203742980957,
+      "rewards/rejected": 0.5553494095802307,
       "step": 50
     },
     {
-      "epoch": 0.15,
-      "grad_norm": 595.0405769986861,
-      "learning_rate": 4.955969343539162e-07,
-      "logits/chosen": -2.606095552444458,
-      "logits/rejected": -2.5313313007354736,
-      "logps/chosen": -261.7010192871094,
-      "logps/rejected": -209.05029296875,
-      "loss": 0.3678,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 4.988667011260986,
-      "rewards/margins": 5.682937145233154,
-      "rewards/rejected": -0.6942700147628784,
+      "epoch": 0.15355086372360843,
+      "grad_norm": 375.0458504104559,
+      "learning_rate": 1.9823877374156647e-07,
+      "logits/chosen": -2.669403553009033,
+      "logits/rejected": -2.6018548011779785,
+      "logps/chosen": -262.41534423828125,
+      "logps/rejected": -206.5387420654297,
+      "loss": 0.311,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 3.2061855792999268,
+      "rewards/margins": 2.446310520172119,
+      "rewards/rejected": 0.7598745822906494,
       "step": 60
     },
     {
-      "epoch": 0.18,
-      "grad_norm": 406.24830016448993,
-      "learning_rate": 4.90438392204474e-07,
-      "logits/chosen": -2.57716703414917,
-      "logits/rejected": -2.5019829273223877,
-      "logps/chosen": -291.1893615722656,
-      "logps/rejected": -227.8859100341797,
-      "loss": 0.3366,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 4.704905033111572,
-      "rewards/margins": 6.591601371765137,
-      "rewards/rejected": -1.8866965770721436,
+      "epoch": 0.17914267434420986,
+      "grad_norm": 371.9136724850062,
+      "learning_rate": 1.9617535688178958e-07,
+      "logits/chosen": -2.6535415649414062,
+      "logits/rejected": -2.590367078781128,
+      "logps/chosen": -290.74871826171875,
+      "logps/rejected": -224.1929168701172,
+      "loss": 0.3051,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 3.5809733867645264,
+      "rewards/margins": 3.0821049213409424,
+      "rewards/rejected": 0.4988683760166168,
       "step": 70
     },
     {
-      "epoch": 0.2,
-      "grad_norm": 771.1936928093826,
-      "learning_rate": 4.83354989019146e-07,
-      "logits/chosen": -2.534698247909546,
-      "logits/rejected": -2.4588425159454346,
-      "logps/chosen": -259.4831848144531,
-      "logps/rejected": -204.6412353515625,
-      "loss": 0.3162,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 5.652478218078613,
-      "rewards/margins": 6.747942924499512,
-      "rewards/rejected": -1.0954645872116089,
+      "epoch": 0.20473448496481125,
+      "grad_norm": 381.95411975681657,
+      "learning_rate": 1.9334199560765839e-07,
+      "logits/chosen": -2.6085634231567383,
+      "logits/rejected": -2.5423429012298584,
+      "logps/chosen": -260.9337463378906,
+      "logps/rejected": -203.12725830078125,
+      "loss": 0.2569,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 3.3122222423553467,
+      "rewards/margins": 3.3377063274383545,
+      "rewards/rejected": -0.025484371930360794,
       "step": 80
     },
     {
-      "epoch": 0.23,
-      "grad_norm": 746.9475535161084,
-      "learning_rate": 4.7440343190975353e-07,
-      "logits/chosen": -2.5645816326141357,
-      "logits/rejected": -2.5068752765655518,
-      "logps/chosen": -257.1772155761719,
-      "logps/rejected": -217.83139038085938,
-      "loss": 0.31,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": 3.197655200958252,
-      "rewards/margins": 5.532810211181641,
-      "rewards/rejected": -2.3351550102233887,
+      "epoch": 0.23032629558541268,
+      "grad_norm": 331.9831865016541,
+      "learning_rate": 1.897613727639014e-07,
+      "logits/chosen": -2.610903739929199,
+      "logits/rejected": -2.556959867477417,
+      "logps/chosen": -256.61138916015625,
+      "logps/rejected": -215.09701538085938,
+      "loss": 0.2658,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 2.5669326782226562,
+      "rewards/margins": 2.867710590362549,
+      "rewards/rejected": -0.3007778227329254,
       "step": 90
     },
     {
-      "epoch": 0.26,
-      "grad_norm": 823.7911810864297,
-      "learning_rate": 4.6365538373900506e-07,
-      "logits/chosen": -2.6224541664123535,
-      "logits/rejected": -2.5483615398406982,
-      "logps/chosen": -236.20339965820312,
-      "logps/rejected": -201.47940063476562,
-      "loss": 0.5348,
+      "epoch": 0.2559181062060141,
+      "grad_norm": 634.1345493995642,
+      "learning_rate": 1.8546215349560202e-07,
+      "logits/chosen": -2.6384997367858887,
+      "logits/rejected": -2.563520908355713,
+      "logps/chosen": -236.5784912109375,
+      "logps/rejected": -199.6844940185547,
+      "loss": 0.3117,
       "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 4.17586088180542,
-      "rewards/margins": 6.1418962478637695,
-      "rewards/rejected": -1.9660362005233765,
+      "rewards/chosen": 2.7952113151550293,
+      "rewards/margins": 3.302074909210205,
+      "rewards/rejected": -0.5068637132644653,
       "step": 100
     },
     {
-      "epoch": 0.28,
-      "grad_norm": 579.991739437595,
-      "learning_rate": 4.5119688941406386e-07,
-      "logits/chosen": -2.6264939308166504,
-      "logits/rejected": -2.5470776557922363,
-      "logps/chosen": -256.7718200683594,
-      "logps/rejected": -209.46231079101562,
-      "loss": 0.4091,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 6.23184061050415,
-      "rewards/margins": 6.958665370941162,
-      "rewards/rejected": -0.7268255352973938,
+      "epoch": 0.28150991682661547,
+      "grad_norm": 319.85079714910466,
+      "learning_rate": 1.8047875576562553e-07,
+      "logits/chosen": -2.630192518234253,
+      "logits/rejected": -2.549891710281372,
+      "logps/chosen": -258.26971435546875,
+      "logps/rejected": -208.4285430908203,
+      "loss": 0.282,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 3.702361583709717,
+      "rewards/margins": 3.704643726348877,
+      "rewards/rejected": -0.002281466033309698,
       "step": 110
     },
     {
-      "epoch": 0.31,
-      "grad_norm": 690.1539082640518,
-      "learning_rate": 4.3712768704277524e-07,
-      "logits/chosen": -2.6043858528137207,
-      "logits/rejected": -2.5360493659973145,
-      "logps/chosen": -261.69561767578125,
-      "logps/rejected": -208.3325653076172,
-      "loss": 0.4048,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 5.436749458312988,
-      "rewards/margins": 6.8130621910095215,
-      "rewards/rejected": -1.3763134479522705,
+      "epoch": 0.30710172744721687,
+      "grad_norm": 376.18943633320504,
+      "learning_rate": 1.748510748171101e-07,
+      "logits/chosen": -2.6130287647247314,
+      "logits/rejected": -2.543454170227051,
+      "logps/chosen": -262.194580078125,
+      "logps/rejected": -206.72488403320312,
+      "loss": 0.2802,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 3.633894443511963,
+      "rewards/margins": 3.8131375312805176,
+      "rewards/rejected": -0.17924347519874573,
       "step": 120
     },
     {
-      "epoch": 0.33,
-      "grad_norm": 451.1743521072461,
-      "learning_rate": 4.2156040946718343e-07,
-      "logits/chosen": -2.578007698059082,
-      "logits/rejected": -2.5135886669158936,
-      "logps/chosen": -250.7688446044922,
-      "logps/rejected": -196.73779296875,
-      "loss": 0.3698,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 4.574068069458008,
-      "rewards/margins": 6.677270412445068,
-      "rewards/rejected": -2.1032023429870605,
+      "epoch": 0.3326935380678183,
+      "grad_norm": 258.08224292776333,
+      "learning_rate": 1.6862416378687336e-07,
+      "logits/chosen": -2.587902069091797,
+      "logits/rejected": -2.5218377113342285,
+      "logps/chosen": -250.2627410888672,
+      "logps/rejected": -193.98731994628906,
+      "loss": 0.2644,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 3.5202510356903076,
+      "rewards/margins": 3.647292375564575,
+      "rewards/rejected": -0.12704159319400787,
       "step": 130
     },
     {
-      "epoch": 0.36,
-      "grad_norm": 465.41383196085104,
-      "learning_rate": 4.046196825665637e-07,
-      "logits/chosen": -2.600147247314453,
-      "logits/rejected": -2.534235715866089,
-      "logps/chosen": -269.448974609375,
-      "logps/rejected": -216.6135711669922,
-      "loss": 0.4171,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 4.009121417999268,
-      "rewards/margins": 6.148682594299316,
-      "rewards/rejected": -2.139561176300049,
+      "epoch": 0.3582853486884197,
+      "grad_norm": 312.9816016514414,
+      "learning_rate": 1.6184787302662547e-07,
+      "logits/chosen": -2.6076598167419434,
+      "logits/rejected": -2.539602279663086,
+      "logps/chosen": -267.7660217285156,
+      "logps/rejected": -213.330078125,
+      "loss": 0.3055,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 3.705143690109253,
+      "rewards/margins": 3.5916450023651123,
+      "rewards/rejected": 0.11349865049123764,
       "step": 140
     },
     {
-      "epoch": 0.38,
-      "grad_norm": 485.4989695160128,
-      "learning_rate": 3.864411275486261e-07,
-      "logits/chosen": -2.583752155303955,
-      "logits/rejected": -2.51660418510437,
-      "logps/chosen": -262.35479736328125,
-      "logps/rejected": -212.38021850585938,
-      "loss": 0.429,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 5.765866756439209,
-      "rewards/margins": 7.155278205871582,
-      "rewards/rejected": -1.3894122838974,
+      "epoch": 0.3838771593090211,
+      "grad_norm": 298.63492430036257,
+      "learning_rate": 1.5457645101945046e-07,
+      "logits/chosen": -2.588223457336426,
+      "logits/rejected": -2.521327495574951,
+      "logps/chosen": -262.39776611328125,
+      "logps/rejected": -210.2900848388672,
+      "loss": 0.2675,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 4.096975803375244,
+      "rewards/margins": 4.044339656829834,
+      "rewards/rejected": 0.05263558775186539,
       "step": 150
     },
     {
-      "epoch": 0.41,
-      "grad_norm": 546.3479101912284,
-      "learning_rate": 3.671702752161759e-07,
-      "logits/chosen": -2.5821540355682373,
-      "logits/rejected": -2.514200210571289,
-      "logps/chosen": -242.9073944091797,
-      "logps/rejected": -197.12026977539062,
-      "loss": 0.4562,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 4.221369743347168,
-      "rewards/margins": 7.090517520904541,
-      "rewards/rejected": -2.869147777557373,
+      "epoch": 0.4094689699296225,
+      "grad_norm": 277.3876418755778,
+      "learning_rate": 1.4686811008647035e-07,
+      "logits/chosen": -2.5860092639923096,
+      "logits/rejected": -2.5171661376953125,
+      "logps/chosen": -241.29501342773438,
+      "logps/rejected": -192.94955444335938,
+      "loss": 0.2544,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 3.8214523792266846,
+      "rewards/margins": 3.785486936569214,
+      "rewards/rejected": 0.0359656922519207,
       "step": 160
     },
     {
-      "epoch": 0.44,
-      "grad_norm": 655.017465367729,
-      "learning_rate": 3.4696140090121375e-07,
-      "logits/chosen": -2.582387685775757,
-      "logits/rejected": -2.517115831375122,
-      "logps/chosen": -264.36541748046875,
-      "logps/rejected": -210.4468231201172,
-      "loss": 0.3517,
+      "epoch": 0.4350607805502239,
+      "grad_norm": 339.61682629601955,
+      "learning_rate": 1.387845603604855e-07,
+      "logits/chosen": -2.5826542377471924,
+      "logits/rejected": -2.51751446723938,
+      "logps/chosen": -262.99298095703125,
+      "logps/rejected": -206.5457000732422,
+      "loss": 0.2334,
       "rewards/accuracies": 0.875,
-      "rewards/chosen": 4.428440570831299,
-      "rewards/margins": 7.276960849761963,
-      "rewards/rejected": -2.8485207557678223,
+      "rewards/chosen": 3.8493824005126953,
+      "rewards/margins": 3.933476209640503,
+      "rewards/rejected": -0.08409398794174194,
       "step": 170
     },
     {
-      "epoch": 0.46,
-      "grad_norm": 492.2183630043871,
-      "learning_rate": 3.259762893935617e-07,
-      "logits/chosen": -2.6398727893829346,
-      "logits/rejected": -2.5528438091278076,
-      "logps/chosen": -235.0208282470703,
-      "logps/rejected": -185.29156494140625,
-      "loss": 0.4065,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 4.908794403076172,
-      "rewards/margins": 6.33758544921875,
-      "rewards/rejected": -1.4287910461425781,
+      "epoch": 0.46065259117082535,
+      "grad_norm": 293.5783730044579,
+      "learning_rate": 1.3039051575742468e-07,
+      "logits/chosen": -2.6330299377441406,
+      "logits/rejected": -2.547426700592041,
+      "logps/chosen": -235.4442596435547,
+      "logps/rejected": -183.63687133789062,
+      "loss": 0.2871,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 3.294564723968506,
+      "rewards/margins": 3.4877796173095703,
+      "rewards/rejected": -0.193215012550354,
       "step": 180
     },
     {
-      "epoch": 0.49,
-      "grad_norm": 502.0957454157663,
-      "learning_rate": 3.0438293975154184e-07,
-      "logits/chosen": -2.5951266288757324,
-      "logits/rejected": -2.519169330596924,
-      "logps/chosen": -258.04022216796875,
-      "logps/rejected": -202.89727783203125,
-      "loss": 0.3426,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 4.746270179748535,
-      "rewards/margins": 7.24340295791626,
-      "rewards/rejected": -2.4971330165863037,
+      "epoch": 0.48624440179142675,
+      "grad_norm": 298.0737945063753,
+      "learning_rate": 1.2175317590061675e-07,
+      "logits/chosen": -2.5894675254821777,
+      "logits/rejected": -2.51374888420105,
+      "logps/chosen": -257.42132568359375,
+      "logps/rejected": -199.62106323242188,
+      "loss": 0.239,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 3.699636936187744,
+      "rewards/margins": 3.845207929611206,
+      "rewards/rejected": -0.1455712914466858,
       "step": 190
     },
     {
-      "epoch": 0.51,
-      "grad_norm": 536.1992163678008,
-      "learning_rate": 2.823542203635138e-07,
-      "logits/chosen": -2.6266205310821533,
-      "logits/rejected": -2.5365912914276123,
-      "logps/chosen": -273.0335388183594,
-      "logps/rejected": -217.910888671875,
-      "loss": 0.3772,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 5.242685794830322,
-      "rewards/margins": 8.247489929199219,
-      "rewards/rejected": -3.004803419113159,
+      "epoch": 0.5118362124120281,
+      "grad_norm": 326.3771289852159,
+      "learning_rate": 1.1294168814540553e-07,
+      "logits/chosen": -2.625087261199951,
+      "logits/rejected": -2.5350747108459473,
+      "logps/chosen": -271.9561462402344,
+      "logps/rejected": -214.1556854248047,
+      "loss": 0.2317,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 4.283473014831543,
+      "rewards/margins": 4.552158832550049,
+      "rewards/rejected": -0.2686860263347626,
       "step": 200
     },
     {
-      "epoch": 0.54,
-      "grad_norm": 550.4099704660905,
-      "learning_rate": 2.600664850273538e-07,
-      "logits/chosen": -2.6228280067443848,
-      "logits/rejected": -2.550497531890869,
-      "logps/chosen": -265.55755615234375,
-      "logps/rejected": -210.61660766601562,
-      "loss": 0.5254,
+      "epoch": 0.5374280230326296,
+      "grad_norm": 510.1343443372893,
+      "learning_rate": 1.0402659401094151e-07,
+      "logits/chosen": -2.602785587310791,
+      "logits/rejected": -2.5288333892822266,
+      "logps/chosen": -264.21099853515625,
+      "logps/rejected": -207.3443145751953,
+      "loss": 0.3153,
       "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 3.847407102584839,
-      "rewards/margins": 6.826489448547363,
-      "rewards/rejected": -2.979081869125366,
+      "rewards/chosen": 3.421428680419922,
+      "rewards/margins": 3.9131996631622314,
+      "rewards/rejected": -0.4917708933353424,
       "step": 210
     },
     {
-      "epoch": 0.56,
-      "grad_norm": 581.5523533407168,
-      "learning_rate": 2.3769816112703045e-07,
-      "logits/chosen": -2.6491611003875732,
-      "logits/rejected": -2.5865907669067383,
-      "logps/chosen": -255.453857421875,
-      "logps/rejected": -212.5009765625,
-      "loss": 0.4229,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 4.356105327606201,
-      "rewards/margins": 6.26999568939209,
-      "rewards/rejected": -1.9138901233673096,
+      "epoch": 0.5630198336532309,
+      "grad_norm": 362.61099626331173,
+      "learning_rate": 9.507926445081218e-08,
+      "logits/chosen": -2.607333183288574,
+      "logits/rejected": -2.5410573482513428,
+      "logps/chosen": -255.5386199951172,
+      "logps/rejected": -210.58261108398438,
+      "loss": 0.2657,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 3.069124698638916,
+      "rewards/margins": 3.4770140647888184,
+      "rewards/rejected": -0.4078896641731262,
       "step": 220
     },
     {
-      "epoch": 0.59,
-      "grad_norm": 486.64295333011063,
-      "learning_rate": 2.1542832120881677e-07,
-      "logits/chosen": -2.6853079795837402,
-      "logits/rejected": -2.600752830505371,
-      "logps/chosen": -264.80615234375,
-      "logps/rejected": -214.556396484375,
-      "loss": 0.3669,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 6.017881393432617,
-      "rewards/margins": 7.167391300201416,
-      "rewards/rejected": -1.1495091915130615,
+      "epoch": 0.5886116442738324,
+      "grad_norm": 270.86067735708986,
+      "learning_rate": 8.61713284835267e-08,
+      "logits/chosen": -2.6318671703338623,
+      "logits/rejected": -2.5431408882141113,
+      "logps/chosen": -265.3260803222656,
+      "logps/rejected": -212.6390380859375,
+      "loss": 0.249,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 4.038520812988281,
+      "rewards/margins": 3.9009087085723877,
+      "rewards/rejected": 0.1376120150089264,
       "step": 230
     },
     {
-      "epoch": 0.61,
-      "grad_norm": 831.6224733555287,
-      "learning_rate": 1.934352493925695e-07,
-      "logits/chosen": -2.667158603668213,
-      "logits/rejected": -2.620408058166504,
-      "logps/chosen": -261.4100036621094,
-      "logps/rejected": -219.61032104492188,
-      "loss": 0.3738,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 4.424870014190674,
-      "rewards/margins": 8.652185440063477,
-      "rewards/rejected": -4.227315425872803,
+      "epoch": 0.6142034548944337,
+      "grad_norm": 496.26361322510223,
+      "learning_rate": 7.73740997570278e-08,
+      "logits/chosen": -2.6172757148742676,
+      "logits/rejected": -2.566117763519287,
+      "logps/chosen": -259.39569091796875,
+      "logps/rejected": -214.13717651367188,
+      "loss": 0.2362,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 4.167778968811035,
+      "rewards/margins": 4.450715065002441,
+      "rewards/rejected": -0.28293582797050476,
       "step": 240
     },
     {
-      "epoch": 0.64,
-      "grad_norm": 563.6526543418182,
-      "learning_rate": 1.7189501409486059e-07,
-      "logits/chosen": -2.674989938735962,
-      "logits/rejected": -2.606071949005127,
-      "logps/chosen": -266.0426940917969,
-      "logps/rejected": -221.0687713623047,
-      "loss": 0.367,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": 4.015201091766357,
-      "rewards/margins": 7.296302795410156,
-      "rewards/rejected": -3.281102418899536,
+      "epoch": 0.6397952655150352,
+      "grad_norm": 385.39306248287744,
+      "learning_rate": 6.875800563794424e-08,
+      "logits/chosen": -2.6301958560943604,
+      "logits/rejected": -2.5562362670898438,
+      "logps/chosen": -264.5022277832031,
+      "logps/rejected": -217.06472778320312,
+      "loss": 0.2541,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 3.638233184814453,
+      "rewards/margins": 3.9798378944396973,
+      "rewards/rejected": -0.34160494804382324,
       "step": 250
     },
     {
-      "epoch": 0.67,
-      "grad_norm": 457.0558678943042,
-      "learning_rate": 1.5098005849021078e-07,
-      "logits/chosen": -2.6647772789001465,
-      "logits/rejected": -2.6081786155700684,
-      "logps/chosen": -260.12384033203125,
-      "logps/rejected": -207.2174835205078,
-      "loss": 0.3387,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 4.0153021812438965,
-      "rewards/margins": 6.823422908782959,
-      "rewards/rejected": -2.8081207275390625,
+      "epoch": 0.6653870761356366,
+      "grad_norm": 246.0647695237683,
+      "learning_rate": 6.039202339608431e-08,
+      "logits/chosen": -2.6222901344299316,
+      "logits/rejected": -2.5577120780944824,
+      "logps/chosen": -258.84637451171875,
+      "logps/rejected": -203.7962188720703,
+      "loss": 0.2317,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 3.5067996978759766,
+      "rewards/margins": 3.801957607269287,
+      "rewards/rejected": -0.295158326625824,
       "step": 260
     },
     {
-      "epoch": 0.69,
-      "grad_norm": 478.6275517216598,
-      "learning_rate": 1.30857819994673e-07,
-      "logits/chosen": -2.641366481781006,
-      "logits/rejected": -2.558147668838501,
-      "logps/chosen": -272.80377197265625,
-      "logps/rejected": -228.5403594970703,
-      "loss": 0.4885,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 2.646123170852661,
-      "rewards/margins": 8.172276496887207,
-      "rewards/rejected": -5.526152610778809,
+      "epoch": 0.690978886756238,
+      "grad_norm": 382.6551210350461,
+      "learning_rate": 5.2343127997869205e-08,
+      "logits/chosen": -2.602179765701294,
+      "logits/rejected": -2.5085558891296387,
+      "logps/chosen": -270.6271057128906,
+      "logps/rejected": -222.81655883789062,
+      "loss": 0.2926,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 2.9784107208251953,
+      "rewards/margins": 4.063758850097656,
+      "rewards/rejected": -1.085348129272461,
       "step": 270
     },
     {
-      "epoch": 0.72,
-      "grad_norm": 369.20521183996,
-      "learning_rate": 1.116893898236716e-07,
-      "logits/chosen": -2.6526267528533936,
-      "logits/rejected": -2.597562789916992,
-      "logps/chosen": -268.1092224121094,
-      "logps/rejected": -217.9169464111328,
-      "loss": 0.3441,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 3.9179558753967285,
-      "rewards/margins": 7.3985185623168945,
-      "rewards/rejected": -3.480562686920166,
+      "epoch": 0.7165706973768394,
+      "grad_norm": 291.15530026271324,
+      "learning_rate": 4.4675755929468636e-08,
+      "logits/chosen": -2.627225160598755,
+      "logits/rejected": -2.565744638442993,
+      "logps/chosen": -266.7916259765625,
+      "logps/rejected": -214.30709838867188,
+      "loss": 0.2375,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 3.4573512077331543,
+      "rewards/margins": 4.138546943664551,
+      "rewards/rejected": -0.6811951398849487,
       "step": 280
     },
     {
-      "epoch": 0.74,
-      "grad_norm": 498.11211487860106,
-      "learning_rate": 9.362822335518062e-08,
-      "logits/chosen": -2.6078855991363525,
-      "logits/rejected": -2.5603675842285156,
-      "logps/chosen": -265.8270263671875,
-      "logps/rejected": -215.421142578125,
-      "loss": 0.3216,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 4.283304691314697,
-      "rewards/margins": 7.295420169830322,
-      "rewards/rejected": -3.012115478515625,
+      "epoch": 0.7421625079974408,
+      "grad_norm": 302.9770657235692,
+      "learning_rate": 3.745128934207224e-08,
+      "logits/chosen": -2.590620279312134,
+      "logits/rejected": -2.5388922691345215,
+      "logps/chosen": -265.51971435546875,
+      "logps/rejected": -212.6427764892578,
+      "loss": 0.2286,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 3.2131683826446533,
+      "rewards/margins": 3.975499391555786,
+      "rewards/rejected": -0.7623313665390015,
       "step": 290
     },
     {
-      "epoch": 0.77,
-      "grad_norm": 433.49223304725587,
-      "learning_rate": 7.681891162260015e-08,
-      "logits/chosen": -2.6258552074432373,
-      "logits/rejected": -2.5720603466033936,
-      "logps/chosen": -272.7791442871094,
-      "logps/rejected": -219.44140625,
-      "loss": 0.3717,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 4.056910514831543,
-      "rewards/margins": 7.150809288024902,
-      "rewards/rejected": -3.093899726867676,
+      "epoch": 0.7677543186180422,
+      "grad_norm": 339.1306333767976,
+      "learning_rate": 3.0727564649040063e-08,
+      "logits/chosen": -2.6111063957214355,
+      "logits/rejected": -2.5518836975097656,
+      "logps/chosen": -271.9001159667969,
+      "logps/rejected": -216.2710723876953,
+      "loss": 0.2446,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 3.3372998237609863,
+      "rewards/margins": 3.9620718955993652,
+      "rewards/rejected": -0.6247718930244446,
       "step": 300
     },
     {
-      "epoch": 0.79,
-      "grad_norm": 369.6165987845038,
-      "learning_rate": 6.139602377230247e-08,
-      "logits/chosen": -2.5929980278015137,
-      "logits/rejected": -2.524878978729248,
-      "logps/chosen": -276.72418212890625,
-      "logps/rejected": -214.73757934570312,
-      "loss": 0.3974,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 3.884862184524536,
-      "rewards/margins": 7.488539695739746,
-      "rewards/rejected": -3.6036782264709473,
+      "epoch": 0.7933461292386437,
+      "grad_norm": 235.34557136252968,
+      "learning_rate": 2.4558409508920985e-08,
+      "logits/chosen": -2.5742857456207275,
+      "logits/rejected": -2.499851703643799,
+      "logps/chosen": -275.28167724609375,
+      "logps/rejected": -211.0914764404297,
+      "loss": 0.2598,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 3.4961440563201904,
+      "rewards/margins": 4.247152805328369,
+      "rewards/rejected": -0.7510083913803101,
       "step": 310
     },
     {
-      "epoch": 0.82,
-      "grad_norm": 542.2853213894015,
-      "learning_rate": 4.748302975270837e-08,
-      "logits/chosen": -2.618070125579834,
-      "logits/rejected": -2.5723397731781006,
-      "logps/chosen": -259.38433837890625,
-      "logps/rejected": -202.79501342773438,
-      "loss": 0.3593,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 3.869190216064453,
-      "rewards/margins": 6.644041538238525,
-      "rewards/rejected": -2.7748515605926514,
+      "epoch": 0.818937939859245,
+      "grad_norm": 368.2414347720992,
+      "learning_rate": 1.899321190108335e-08,
+      "logits/chosen": -2.59656023979187,
+      "logits/rejected": -2.545151472091675,
+      "logps/chosen": -258.7973327636719,
+      "logps/rejected": -200.56219482421875,
+      "loss": 0.2329,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 3.0572078227996826,
+      "rewards/margins": 3.9228408336639404,
+      "rewards/rejected": -0.8656327128410339,
       "step": 320
     },
     {
-      "epoch": 0.84,
-      "grad_norm": 1179.920803140471,
-      "learning_rate": 3.5191311859445795e-08,
-      "logits/chosen": -2.639923572540283,
-      "logits/rejected": -2.582639455795288,
-      "logps/chosen": -262.33123779296875,
-      "logps/rejected": -215.505615234375,
-      "loss": 0.3667,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 5.157271385192871,
-      "rewards/margins": 7.100647926330566,
-      "rewards/rejected": -1.9433765411376953,
+      "epoch": 0.8445297504798465,
+      "grad_norm": 412.2800478150093,
+      "learning_rate": 1.4076524743778317e-08,
+      "logits/chosen": -2.6195101737976074,
+      "logits/rejected": -2.55584716796875,
+      "logps/chosen": -262.2277526855469,
+      "logps/rejected": -213.35403442382812,
+      "loss": 0.231,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 3.7354748249053955,
+      "rewards/margins": 4.0478105545043945,
+      "rewards/rejected": -0.31233564019203186,
       "step": 330
     },
     {
-      "epoch": 0.87,
-      "grad_norm": 467.7658799716397,
-      "learning_rate": 2.4619273049795996e-08,
-      "logits/chosen": -2.6265780925750732,
-      "logits/rejected": -2.5702362060546875,
-      "logps/chosen": -257.75689697265625,
-      "logps/rejected": -209.0670623779297,
-      "loss": 0.324,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 4.93801212310791,
-      "rewards/margins": 7.8753981590271,
-      "rewards/rejected": -2.9373855590820312,
+      "epoch": 0.8701215611004478,
+      "grad_norm": 291.1936087568274,
+      "learning_rate": 9.847709219918398e-09,
+      "logits/chosen": -2.6071531772613525,
+      "logits/rejected": -2.5460381507873535,
+      "logps/chosen": -257.75164794921875,
+      "logps/rejected": -206.60986328125,
+      "loss": 0.221,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 3.5297904014587402,
+      "rewards/margins": 4.399306297302246,
+      "rewards/rejected": -0.8695155382156372,
       "step": 340
     },
     {
-      "epoch": 0.9,
-      "grad_norm": 345.87107109384476,
-      "learning_rate": 1.5851549164932115e-08,
-      "logits/chosen": -2.637718677520752,
-      "logits/rejected": -2.589808940887451,
-      "logps/chosen": -267.7754821777344,
-      "logps/rejected": -225.0698699951172,
-      "loss": 0.3399,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 4.532751560211182,
-      "rewards/margins": 7.0415191650390625,
-      "rewards/rejected": -2.508768320083618,
+      "epoch": 0.8957133717210493,
+      "grad_norm": 301.99507828681374,
+      "learning_rate": 6.340619665972846e-09,
+      "logits/chosen": -2.617502212524414,
+      "logits/rejected": -2.5671114921569824,
+      "logps/chosen": -267.4635009765625,
+      "logps/rejected": -222.5786895751953,
+      "loss": 0.2579,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 3.393681287765503,
+      "rewards/margins": 3.9400668144226074,
+      "rewards/rejected": -0.546385645866394,
       "step": 350
     },
     {
-      "epoch": 0.92,
-      "grad_norm": 491.63656420568554,
-      "learning_rate": 8.958331366609423e-09,
-      "logits/chosen": -2.641171932220459,
-      "logits/rejected": -2.574852705001831,
-      "logps/chosen": -272.898193359375,
-      "logps/rejected": -217.81997680664062,
-      "loss": 0.3962,
+      "epoch": 0.9213051823416507,
+      "grad_norm": 348.99526280716225,
+      "learning_rate": 3.583332546643769e-09,
+      "logits/chosen": -2.6180646419525146,
+      "logits/rejected": -2.5445971488952637,
+      "logps/chosen": -272.35406494140625,
+      "logps/rejected": -214.8975067138672,
+      "loss": 0.2825,
       "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 4.423092842102051,
-      "rewards/margins": 7.286849021911621,
-      "rewards/rejected": -2.863755464553833,
+      "rewards/chosen": 3.431410551071167,
+      "rewards/margins": 4.0157270431518555,
+      "rewards/rejected": -0.584316074848175,
       "step": 360
     },
     {
-      "epoch": 0.95,
-      "grad_norm": 863.5117187901297,
-      "learning_rate": 3.994804212627461e-09,
-      "logits/chosen": -2.600586175918579,
-      "logits/rejected": -2.565150737762451,
-      "logps/chosen": -271.3792419433594,
-      "logps/rejected": -227.0088348388672,
-      "loss": 0.4043,
+      "epoch": 0.946896992962252,
+      "grad_norm": 251.0460885716357,
+      "learning_rate": 1.5979216850509847e-09,
+      "logits/chosen": -2.5794007778167725,
+      "logits/rejected": -2.5391972064971924,
+      "logps/chosen": -271.82513427734375,
+      "logps/rejected": -225.1968231201172,
+      "loss": 0.2529,
       "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 5.524130821228027,
-      "rewards/margins": 7.229277610778809,
-      "rewards/rejected": -1.7051467895507812,
+      "rewards/chosen": 3.722885847091675,
+      "rewards/margins": 4.0348358154296875,
+      "rewards/rejected": -0.3119499981403351,
       "step": 370
     },
     {
-      "epoch": 0.97,
-      "grad_norm": 386.7539067890418,
-      "learning_rate": 1.0007038696262516e-09,
-      "logits/chosen": -2.649324893951416,
-      "logits/rejected": -2.608961820602417,
-      "logps/chosen": -261.44415283203125,
-      "logps/rejected": -229.10617065429688,
-      "loss": 0.3514,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 4.858848571777344,
-      "rewards/margins": 7.17071008682251,
-      "rewards/rejected": -2.311861276626587,
+      "epoch": 0.9724888035828535,
+      "grad_norm": 201.79043052952954,
+      "learning_rate": 4.002815478505006e-10,
+      "logits/chosen": -2.6285715103149414,
+      "logits/rejected": -2.585494041442871,
+      "logps/chosen": -261.1199951171875,
+      "logps/rejected": -226.61105346679688,
+      "loss": 0.255,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 3.6326937675476074,
+      "rewards/margins": 4.036477088928223,
+      "rewards/rejected": -0.40378379821777344,
       "step": 380
     },
     {
-      "epoch": 1.0,
-      "grad_norm": 658.7609084129675,
+      "epoch": 0.9980806142034548,
+      "grad_norm": 458.24518941114127,
       "learning_rate": 0.0,
-      "logits/chosen": -2.657970428466797,
-      "logits/rejected": -2.5974597930908203,
-      "logps/chosen": -249.08193969726562,
-      "logps/rejected": -208.86935424804688,
-      "loss": 0.389,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 4.138771057128906,
-      "rewards/margins": 6.9808807373046875,
-      "rewards/rejected": -2.8421101570129395,
+      "logits/chosen": -2.641213893890381,
+      "logits/rejected": -2.575150966644287,
+      "logps/chosen": -248.17904663085938,
+      "logps/rejected": -206.1395721435547,
+      "loss": 0.2365,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 3.4077067375183105,
+      "rewards/margins": 4.0728960037231445,
+      "rewards/rejected": -0.6651893854141235,
       "step": 390
     },
     {
-      "epoch": 1.0,
+      "epoch": 0.9980806142034548,
       "step": 390,
       "total_flos": 0.0,
-      "train_loss": 0.39275855483152927,
-      "train_runtime": 5850.2913,
-      "train_samples_per_second": 8.547,
-      "train_steps_per_second": 0.067
+      "train_loss": 0.2947830175742125,
+      "train_runtime": 6391.2384,
+      "train_samples_per_second": 7.823,
+      "train_steps_per_second": 0.061
     }
   ],
   "logging_steps": 10,
@@ -623,6 +623,18 @@
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
   "total_flos": 0.0,
   "train_batch_size": 4,
   "trial_name": null,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69755ef6c720cdfd7d14b231fbf0ad5a8fc29c369a6e770d1d3031286dc30b37
-size 6264
+oid sha256:4794be63c2fe75e163cfc9de1b0a4ccd9a5f02340e7eb2b99ae28feb9b6d4609
+size 6520