Model save

README.md

@@ -53,7 +53,7 @@ The following hyperparameters were used during training:
 
 ### Framework versions
 
-- Transformers 4.39.3
+- Transformers 4.41.1
 - Pytorch 2.1.2+cu118
 - Datasets 2.16.1
-- Tokenizers 0.15.2
+- Tokenizers 0.19.1

all_results.json

@@ -1,8 +1,9 @@
 {
-    "epoch": 1.0,
-    "train_loss": 0.048529281703106095,
-    "train_runtime": 584.5423,
+    "epoch": 0.9988623435722411,
+    "total_flos": 0.0,
+    "train_loss": 0.6069056124241857,
+    "train_runtime": 7568.165,
     "train_samples": 56236,
-    "train_samples_per_second": 96.205,
-    "train_steps_per_second": 0.751
+    "train_samples_per_second": 7.431,
+    "train_steps_per_second": 0.058
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "transformers_version": "4.39.3"
+  "transformers_version": "4.41.1"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0f0917430ce074f3dea083f0ccf860b56e464900291bf59b18d535777662e5e2
+oid sha256:2835b01b1f54d9984513be70fe4bafe1b10401e57498769928bbbf3c2d6b4dde
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:767ab10c5d7cbffb20427d5ee7b697153038525334a0025d3e17b6ef308adea3
+oid sha256:9c039a5b3b2ed233596c44fca798b2e85594ca522af8b24fe4ca55392c759054
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:232f07d779218e892df908602977cfff9f29af01717e280a568b132d80e0d8ca
+oid sha256:71be880c3a748306bfa87fb501066232d5e06f6a5cc376b7e465bd0722e93c57
 size 4540516344

runs/Jul04_01-05-39_n136-082-130/events.out.tfevents.1720026360.n136-082-130.847024.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ceda3f271a3ff2ca04c61767fe156c63f71f8d089d9e811b514d0b254b21afc3
-size 33139
+oid sha256:f8fde53d5c3fe3c01a4416c94e8daf422b4e0ec68210f7a63e0f168b9236dfed
+size 35557

train_results.json

@@ -1,8 +1,9 @@
 {
-    "epoch": 1.0,
-    "train_loss": 0.048529281703106095,
-    "train_runtime": 584.5423,
+    "epoch": 0.9988623435722411,
+    "total_flos": 0.0,
+    "train_loss": 0.6069056124241857,
+    "train_runtime": 7568.165,
     "train_samples": 56236,
-    "train_samples_per_second": 96.205,
-    "train_steps_per_second": 0.751
+    "train_samples_per_second": 7.431,
+    "train_steps_per_second": 0.058
 }

trainer_state.json

@@ -9,14 +9,14 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0,
-      "grad_norm": 23.98402260612519,
+      "epoch": 0.0022753128555176336,
+      "grad_norm": 2401.7651304741935,
       "learning_rate": 2.2727272727272727e-09,
       "logits/chosen": -1.6768856048583984,
       "logits/rejected": -1.7259055376052856,
       "logps/chosen": -394.9654541015625,
       "logps/rejected": -320.0859069824219,
-      "loss": 0.693,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -24,658 +24,658 @@
       "step": 1
     },
     {
-      "epoch": 0.02,
-      "grad_norm": 31.95891283237601,
+      "epoch": 0.022753128555176336,
+      "grad_norm": 3674.27103893147,
       "learning_rate": 2.2727272727272725e-08,
-      "logits/chosen": -1.7029528617858887,
-      "logits/rejected": -1.6683764457702637,
-      "logps/chosen": -429.5246887207031,
-      "logps/rejected": -403.747314453125,
-      "loss": 0.6933,
-      "rewards/accuracies": 0.4305555522441864,
-      "rewards/chosen": 2.8328322514425963e-05,
-      "rewards/margins": -0.0006085141212679446,
-      "rewards/rejected": 0.0006368425092659891,
+      "logits/chosen": -1.7033692598342896,
+      "logits/rejected": -1.668839931488037,
+      "logps/chosen": -429.4827575683594,
+      "logps/rejected": -403.7341613769531,
+      "loss": 0.8236,
+      "rewards/accuracies": 0.4930555522441864,
+      "rewards/chosen": 0.04477702081203461,
+      "rewards/margins": -0.03203389421105385,
+      "rewards/rejected": 0.07681091874837875,
       "step": 10
     },
     {
-      "epoch": 0.05,
-      "grad_norm": 49.89152060792663,
+      "epoch": 0.04550625711035267,
+      "grad_norm": 7689.946378851128,
       "learning_rate": 4.545454545454545e-08,
-      "logits/chosen": -1.7806730270385742,
-      "logits/rejected": -1.7358741760253906,
-      "logps/chosen": -442.21636962890625,
-      "logps/rejected": -401.44000244140625,
-      "loss": 0.6922,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.0007078830385580659,
-      "rewards/margins": 0.0019328873604536057,
-      "rewards/rejected": -0.0026407705154269934,
+      "logits/chosen": -1.7804689407348633,
+      "logits/rejected": -1.7352758646011353,
+      "logps/chosen": -442.10357666015625,
+      "logps/rejected": -401.1966857910156,
+      "loss": 0.7984,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": 0.04200217127799988,
+      "rewards/margins": 0.06276695430278778,
+      "rewards/rejected": -0.0207647867500782,
       "step": 20
     },
     {
-      "epoch": 0.07,
-      "grad_norm": 35.37044354910807,
+      "epoch": 0.06825938566552901,
+      "grad_norm": 3320.742356538349,
       "learning_rate": 6.818181818181817e-08,
-      "logits/chosen": -1.751228928565979,
-      "logits/rejected": -1.6827186346054077,
-      "logps/chosen": -440.0089416503906,
-      "logps/rejected": -401.7633361816406,
-      "loss": 0.6874,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.002162130083888769,
-      "rewards/margins": 0.012089937925338745,
-      "rewards/rejected": -0.014252068474888802,
+      "logits/chosen": -1.748478889465332,
+      "logits/rejected": -1.6795032024383545,
+      "logps/chosen": -439.0035095214844,
+      "logps/rejected": -399.9698181152344,
+      "loss": 0.7209,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.7892183661460876,
+      "rewards/margins": 0.4209066331386566,
+      "rewards/rejected": 0.36831170320510864,
       "step": 30
     },
     {
-      "epoch": 0.09,
-      "grad_norm": 24.433299552566275,
+      "epoch": 0.09101251422070535,
+      "grad_norm": 2645.399792158171,
       "learning_rate": 9.09090909090909e-08,
-      "logits/chosen": -1.7572921514511108,
-      "logits/rejected": -1.6904194355010986,
-      "logps/chosen": -431.66802978515625,
-      "logps/rejected": -390.01898193359375,
-      "loss": 0.6773,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.00594430323690176,
-      "rewards/margins": 0.07497542351484299,
-      "rewards/rejected": -0.06903111934661865,
+      "logits/chosen": -1.7387292385101318,
+      "logits/rejected": -1.6709461212158203,
+      "logps/chosen": -429.72637939453125,
+      "logps/rejected": -381.5823059082031,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 2.5361411571502686,
+      "rewards/margins": 1.0025578737258911,
+      "rewards/rejected": 1.533583641052246,
       "step": 40
     },
     {
-      "epoch": 0.11,
-      "grad_norm": 31.92529179185726,
+      "epoch": 0.11376564277588168,
+      "grad_norm": 2727.7867200257297,
       "learning_rate": 9.994307990108962e-08,
-      "logits/chosen": -1.7621084451675415,
-      "logits/rejected": -1.6976591348648071,
-      "logps/chosen": -444.54913330078125,
-      "logps/rejected": -393.5503234863281,
-      "loss": 0.6599,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": 0.04167298600077629,
-      "rewards/margins": 0.09635747224092484,
-      "rewards/rejected": -0.05468447878956795,
+      "logits/chosen": -1.7028394937515259,
+      "logits/rejected": -1.6375631093978882,
+      "logps/chosen": -447.03662109375,
+      "logps/rejected": -387.6848449707031,
+      "loss": 0.6375,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 1.6797555685043335,
+      "rewards/margins": 1.2827322483062744,
+      "rewards/rejected": 0.39702337980270386,
       "step": 50
     },
     {
-      "epoch": 0.14,
-      "grad_norm": 28.704756119671675,
+      "epoch": 0.13651877133105803,
+      "grad_norm": 3144.6788910608248,
       "learning_rate": 9.959570405988094e-08,
-      "logits/chosen": -1.8457626104354858,
-      "logits/rejected": -1.7673368453979492,
-      "logps/chosen": -385.0513916015625,
-      "logps/rejected": -358.7567443847656,
-      "loss": 0.6566,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": 0.10071470588445663,
-      "rewards/margins": 0.06823419034481049,
-      "rewards/rejected": 0.032480526715517044,
+      "logits/chosen": -1.7310268878936768,
+      "logits/rejected": -1.6494038105010986,
+      "logps/chosen": -393.791748046875,
+      "logps/rejected": -361.52777099609375,
+      "loss": 0.6436,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 1.3311208486557007,
+      "rewards/margins": 0.8541172742843628,
+      "rewards/rejected": 0.47700363397598267,
       "step": 60
     },
     {
-      "epoch": 0.16,
-      "grad_norm": 24.531377945654775,
+      "epoch": 0.15927189988623436,
+      "grad_norm": 2122.8952693977944,
       "learning_rate": 9.893476820924666e-08,
-      "logits/chosen": -1.9651501178741455,
-      "logits/rejected": -1.8813607692718506,
-      "logps/chosen": -403.64105224609375,
-      "logps/rejected": -371.46234130859375,
-      "loss": 0.6395,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.2845568060874939,
-      "rewards/margins": 0.13415710628032684,
-      "rewards/rejected": 0.15039967000484467,
+      "logits/chosen": -1.8110435009002686,
+      "logits/rejected": -1.7250308990478516,
+      "logps/chosen": -430.95562744140625,
+      "logps/rejected": -386.60845947265625,
+      "loss": 0.5572,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 1.141100525856018,
+      "rewards/margins": 1.2472059726715088,
+      "rewards/rejected": -0.10610540211200714,
       "step": 70
     },
     {
-      "epoch": 0.18,
-      "grad_norm": 27.08910191775396,
+      "epoch": 0.1820250284414107,
+      "grad_norm": 3330.6701664231987,
       "learning_rate": 9.796445099843647e-08,
-      "logits/chosen": -2.0368916988372803,
-      "logits/rejected": -1.9531713724136353,
-      "logps/chosen": -407.11553955078125,
-      "logps/rejected": -381.14794921875,
-      "loss": 0.6473,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.3643694519996643,
-      "rewards/margins": 0.1756153404712677,
-      "rewards/rejected": 0.18875406682491302,
+      "logits/chosen": -1.7953722476959229,
+      "logits/rejected": -1.7064487934112549,
+      "logps/chosen": -441.93267822265625,
+      "logps/rejected": -399.75653076171875,
+      "loss": 0.6418,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 1.6198523044586182,
+      "rewards/margins": 1.3530175685882568,
+      "rewards/rejected": 0.2668345868587494,
       "step": 80
     },
     {
-      "epoch": 0.2,
-      "grad_norm": 38.20992212526602,
+      "epoch": 0.20477815699658702,
+      "grad_norm": 2946.6918997559205,
       "learning_rate": 9.669088708527066e-08,
-      "logits/chosen": -2.025817394256592,
-      "logits/rejected": -1.9624574184417725,
-      "logps/chosen": -415.40020751953125,
-      "logps/rejected": -390.00140380859375,
-      "loss": 0.6291,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": 0.36643102765083313,
-      "rewards/margins": 0.1800619214773178,
-      "rewards/rejected": 0.1863691359758377,
+      "logits/chosen": -1.7440869808197021,
+      "logits/rejected": -1.6746982336044312,
+      "logps/chosen": -450.5009765625,
+      "logps/rejected": -408.64202880859375,
+      "loss": 0.615,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 1.5422661304473877,
+      "rewards/margins": 1.545967698097229,
+      "rewards/rejected": -0.003701579524204135,
       "step": 90
     },
     {
-      "epoch": 0.23,
-      "grad_norm": 27.439269349288335,
+      "epoch": 0.22753128555176336,
+      "grad_norm": 3604.7041576262422,
       "learning_rate": 9.512212835085849e-08,
-      "logits/chosen": -2.1013572216033936,
-      "logits/rejected": -2.0152974128723145,
-      "logps/chosen": -388.7185974121094,
-      "logps/rejected": -383.03546142578125,
-      "loss": 0.6197,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.37033963203430176,
-      "rewards/margins": 0.19511529803276062,
-      "rewards/rejected": 0.17522430419921875,
+      "logits/chosen": -1.7728557586669922,
+      "logits/rejected": -1.6836904287338257,
+      "logps/chosen": -424.36199951171875,
+      "logps/rejected": -400.8880310058594,
+      "loss": 0.6302,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 1.3905924558639526,
+      "rewards/margins": 1.7207343578338623,
+      "rewards/rejected": -0.33014196157455444,
       "step": 100
     },
     {
-      "epoch": 0.25,
-      "grad_norm": 26.198395783666243,
+      "epoch": 0.2502844141069397,
+      "grad_norm": 2323.5410392013223,
       "learning_rate": 9.326809299301306e-08,
-      "logits/chosen": -2.1095404624938965,
-      "logits/rejected": -2.0083327293395996,
-      "logps/chosen": -427.6131896972656,
-      "logps/rejected": -393.1694030761719,
-      "loss": 0.6112,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 0.37072473764419556,
-      "rewards/margins": 0.26910915970802307,
-      "rewards/rejected": 0.10161559283733368,
+      "logits/chosen": -1.7771885395050049,
+      "logits/rejected": -1.6686515808105469,
+      "logps/chosen": -463.02960205078125,
+      "logps/rejected": -403.10284423828125,
+      "loss": 0.5955,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": 1.656102180480957,
+      "rewards/margins": 1.4279956817626953,
+      "rewards/rejected": 0.22810666263103485,
       "step": 110
     },
     {
-      "epoch": 0.27,
-      "grad_norm": 26.3105476660504,
+      "epoch": 0.27303754266211605,
+      "grad_norm": 3204.4228883901383,
       "learning_rate": 9.114050282021158e-08,
-      "logits/chosen": -2.1161999702453613,
-      "logits/rejected": -2.05537748336792,
-      "logps/chosen": -435.20086669921875,
-      "logps/rejected": -412.62078857421875,
-      "loss": 0.607,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.268494188785553,
-      "rewards/margins": 0.23757004737854004,
-      "rewards/rejected": 0.030924171209335327,
+      "logits/chosen": -1.765499472618103,
+      "logits/rejected": -1.7019010782241821,
+      "logps/chosen": -460.32476806640625,
+      "logps/rejected": -415.9939880371094,
+      "loss": 0.5703,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 1.7254955768585205,
+      "rewards/margins": 2.0062484741210938,
+      "rewards/rejected": -0.280752956867218,
       "step": 120
     },
     {
-      "epoch": 0.3,
-      "grad_norm": 27.752798140691322,
+      "epoch": 0.29579067121729236,
+      "grad_norm": 2041.539950600557,
       "learning_rate": 8.875280914254802e-08,
-      "logits/chosen": -2.1366939544677734,
-      "logits/rejected": -2.0489516258239746,
-      "logps/chosen": -392.9619140625,
-      "logps/rejected": -363.98553466796875,
-      "loss": 0.602,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 0.2517772316932678,
-      "rewards/margins": 0.2715442478656769,
-      "rewards/rejected": -0.01976701058447361,
+      "logits/chosen": -1.7614352703094482,
+      "logits/rejected": -1.666844129562378,
+      "logps/chosen": -416.5196228027344,
+      "logps/rejected": -362.18475341796875,
+      "loss": 0.6694,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 1.6200507879257202,
+      "rewards/margins": 1.7960132360458374,
+      "rewards/rejected": -0.17596255242824554,
       "step": 130
     },
     {
-      "epoch": 0.32,
-      "grad_norm": 24.888665971199188,
+      "epoch": 0.3185437997724687,
+      "grad_norm": 2554.0547403903993,
       "learning_rate": 8.612010772821971e-08,
-      "logits/chosen": -2.180723190307617,
-      "logits/rejected": -2.1342811584472656,
-      "logps/chosen": -445.15472412109375,
-      "logps/rejected": -407.91998291015625,
-      "loss": 0.6048,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 0.32857105135917664,
-      "rewards/margins": 0.3196411728858948,
-      "rewards/rejected": 0.008929857984185219,
+      "logits/chosen": -1.791283369064331,
+      "logits/rejected": -1.7399561405181885,
+      "logps/chosen": -475.14630126953125,
+      "logps/rejected": -407.97772216796875,
+      "loss": 0.552,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.865530490875244,
+      "rewards/margins": 2.030369281768799,
+      "rewards/rejected": 0.8351613879203796,
       "step": 140
     },
     {
-      "epoch": 0.34,
-      "grad_norm": 27.895144442719968,
+      "epoch": 0.3412969283276451,
+      "grad_norm": 2569.885762906874,
       "learning_rate": 8.325904336322055e-08,
-      "logits/chosen": -2.172715663909912,
-      "logits/rejected": -2.112837314605713,
-      "logps/chosen": -402.29217529296875,
-      "logps/rejected": -380.25335693359375,
-      "loss": 0.6071,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.1980087012052536,
-      "rewards/margins": 0.2475043088197708,
-      "rewards/rejected": -0.049495600163936615,
+      "logits/chosen": -1.7574840784072876,
+      "logits/rejected": -1.6957889795303345,
+      "logps/chosen": -419.8956604003906,
+      "logps/rejected": -374.81927490234375,
+      "loss": 0.6146,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 2.1974282264709473,
+      "rewards/margins": 1.7128820419311523,
+      "rewards/rejected": 0.48454588651657104,
       "step": 150
     },
     {
-      "epoch": 0.36,
-      "grad_norm": 24.760628369907216,
+      "epoch": 0.3640500568828214,
+      "grad_norm": 1870.9610749832216,
       "learning_rate": 8.01877046176447e-08,
-      "logits/chosen": -2.131298303604126,
-      "logits/rejected": -2.059814453125,
-      "logps/chosen": -396.01788330078125,
-      "logps/rejected": -379.63665771484375,
-      "loss": 0.5888,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.12456746399402618,
-      "rewards/margins": 0.23458366096019745,
-      "rewards/rejected": -0.11001620441675186,
+      "logits/chosen": -1.704718828201294,
+      "logits/rejected": -1.627812147140503,
+      "logps/chosen": -406.99969482421875,
+      "logps/rejected": -368.83868408203125,
+      "loss": 0.6199,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 1.4749443531036377,
+      "rewards/margins": 1.6785873174667358,
+      "rewards/rejected": -0.20364296436309814,
       "step": 160
     },
     {
-      "epoch": 0.39,
-      "grad_norm": 22.13569267518615,
+      "epoch": 0.38680318543799774,
+      "grad_norm": 1849.2938090361138,
       "learning_rate": 7.692550948392249e-08,
-      "logits/chosen": -2.1969549655914307,
-      "logits/rejected": -2.1351304054260254,
-      "logps/chosen": -421.291015625,
-      "logps/rejected": -387.77484130859375,
-      "loss": 0.589,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.19595439732074738,
-      "rewards/margins": 0.26243916153907776,
-      "rewards/rejected": -0.06648479402065277,
+      "logits/chosen": -1.767721176147461,
+      "logits/rejected": -1.6935087442398071,
+      "logps/chosen": -438.52667236328125,
+      "logps/rejected": -380.55413818359375,
+      "loss": 0.6083,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 2.3597300052642822,
+      "rewards/margins": 1.7874879837036133,
+      "rewards/rejected": 0.5722419023513794,
       "step": 170
     },
     {
-      "epoch": 0.41,
-      "grad_norm": 29.37383088378205,
+      "epoch": 0.40955631399317405,
+      "grad_norm": 2648.5827284102115,
       "learning_rate": 7.349308261002021e-08,
-      "logits/chosen": -2.1651830673217773,
-      "logits/rejected": -2.1070454120635986,
-      "logps/chosen": -426.6806640625,
-      "logps/rejected": -408.07781982421875,
-      "loss": 0.5855,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.2470887452363968,
-      "rewards/margins": 0.2750667631626129,
-      "rewards/rejected": -0.02797803282737732,
+      "logits/chosen": -1.719022512435913,
+      "logits/rejected": -1.6547473669052124,
+      "logps/chosen": -448.7548828125,
+      "logps/rejected": -404.3783264160156,
+      "loss": 0.6855,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 2.634617328643799,
+      "rewards/margins": 1.7329174280166626,
+      "rewards/rejected": 0.9016996622085571,
       "step": 180
     },
     {
-      "epoch": 0.43,
-      "grad_norm": 29.642029984354075,
+      "epoch": 0.4323094425483504,
+      "grad_norm": 2073.609691895701,
       "learning_rate": 6.991212490377531e-08,
-      "logits/chosen": -2.239077091217041,
-      "logits/rejected": -2.188472270965576,
-      "logps/chosen": -460.720947265625,
-      "logps/rejected": -433.29608154296875,
-      "loss": 0.58,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": 0.17696049809455872,
-      "rewards/margins": 0.3639344274997711,
-      "rewards/rejected": -0.1869739592075348,
+      "logits/chosen": -1.7709083557128906,
+      "logits/rejected": -1.7132765054702759,
+      "logps/chosen": -477.26641845703125,
+      "logps/rejected": -415.4395446777344,
+      "loss": 0.7128,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 1.1506327390670776,
+      "rewards/margins": 1.991530179977417,
+      "rewards/rejected": -0.8408973813056946,
       "step": 190
     },
     {
-      "epoch": 0.46,
-      "grad_norm": 40.17804362794063,
+      "epoch": 0.4550625711035267,
+      "grad_norm": 3176.161253146297,
       "learning_rate": 6.620527633276978e-08,
-      "logits/chosen": -2.1767477989196777,
-      "logits/rejected": -2.1026599407196045,
-      "logps/chosen": -425.94677734375,
-      "logps/rejected": -428.23846435546875,
-      "loss": 0.5857,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": 0.10997031629085541,
-      "rewards/margins": 0.38212892413139343,
-      "rewards/rejected": -0.2721586227416992,
+      "logits/chosen": -1.7090890407562256,
+      "logits/rejected": -1.6279356479644775,
+      "logps/chosen": -435.232666015625,
+      "logps/rejected": -401.6995544433594,
+      "loss": 0.6138,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 1.711168646812439,
+      "rewards/margins": 2.3881287574768066,
+      "rewards/rejected": -0.6769603490829468,
       "step": 200
     },
     {
-      "epoch": 0.48,
-      "grad_norm": 23.44190889972074,
+      "epoch": 0.4778156996587031,
+      "grad_norm": 2116.394638213222,
       "learning_rate": 6.239597278716581e-08,
-      "logits/chosen": -2.2649803161621094,
-      "logits/rejected": -2.202864646911621,
-      "logps/chosen": -410.3318786621094,
-      "logps/rejected": -393.8056640625,
-      "loss": 0.5721,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 0.14795458316802979,
-      "rewards/margins": 0.41721048951148987,
-      "rewards/rejected": -0.26925593614578247,
+      "logits/chosen": -1.7814315557479858,
+      "logits/rejected": -1.7139074802398682,
+      "logps/chosen": -423.193115234375,
+      "logps/rejected": -367.2490539550781,
+      "loss": 0.545,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 1.9342952966690063,
+      "rewards/margins": 2.303255796432495,
+      "rewards/rejected": -0.3689606785774231,
       "step": 210
     },
     {
-      "epoch": 0.5,
-      "grad_norm": 25.725130617472978,
+      "epoch": 0.5005688282138794,
+      "grad_norm": 2816.6422041241376,
       "learning_rate": 5.8508297910462456e-08,
-      "logits/chosen": -2.2174572944641113,
-      "logits/rejected": -2.1320137977600098,
-      "logps/chosen": -415.51153564453125,
-      "logps/rejected": -422.1241149902344,
-      "loss": 0.5729,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": 0.06063612177968025,
-      "rewards/margins": 0.38312196731567383,
-      "rewards/rejected": -0.32248586416244507,
+      "logits/chosen": -1.7332464456558228,
+      "logits/rejected": -1.6482226848602295,
+      "logps/chosen": -419.59423828125,
+      "logps/rejected": -390.67681884765625,
+      "loss": 0.598,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 1.980918288230896,
+      "rewards/margins": 2.782224655151367,
+      "rewards/rejected": -0.8013065457344055,
       "step": 220
     },
     {
-      "epoch": 0.52,
-      "grad_norm": 33.748772620696556,
+      "epoch": 0.5233219567690558,
+      "grad_norm": 4902.899944638479,
       "learning_rate": 5.456683083494731e-08,
-      "logits/chosen": -2.204589366912842,
-      "logits/rejected": -2.164806842803955,
-      "logps/chosen": -464.0941467285156,
-      "logps/rejected": -462.9925231933594,
-      "loss": 0.5856,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.1456037312746048,
-      "rewards/margins": 0.29207009077072144,
-      "rewards/rejected": -0.14646635949611664,
+      "logits/chosen": -1.7302262783050537,
+      "logits/rejected": -1.6885974407196045,
+      "logps/chosen": -477.3504943847656,
+      "logps/rejected": -448.8739318847656,
+      "loss": 0.6141,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 1.3040168285369873,
+      "rewards/margins": 1.832013726234436,
+      "rewards/rejected": -0.527996838092804,
       "step": 230
     },
     {
-      "epoch": 0.55,
-      "grad_norm": 35.95467752105537,
+      "epoch": 0.5460750853242321,
+      "grad_norm": 5130.479717563698,
       "learning_rate": 5.059649078450834e-08,
-      "logits/chosen": -2.2099993228912354,
-      "logits/rejected": -2.1632981300354004,
-      "logps/chosen": -434.889404296875,
-      "logps/rejected": -441.2001037597656,
-      "loss": 0.5675,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 0.03340945765376091,
-      "rewards/margins": 0.3251820206642151,
-      "rewards/rejected": -0.2917725443840027,
+      "logits/chosen": -1.7282073497772217,
+      "logits/rejected": -1.6770431995391846,
+      "logps/chosen": -436.76788330078125,
+      "logps/rejected": -412.71893310546875,
+      "loss": 0.554,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 1.4624868631362915,
+      "rewards/margins": 2.158628225326538,
+      "rewards/rejected": -0.696141242980957,
       "step": 240
     },
     {
-      "epoch": 0.57,
-      "grad_norm": 21.692896426665428,
+      "epoch": 0.5688282138794084,
+      "grad_norm": 1909.2473147410115,
       "learning_rate": 4.6622379527277186e-08,
-      "logits/chosen": -2.2246696949005127,
-      "logits/rejected": -2.1740729808807373,
-      "logps/chosen": -406.245361328125,
-      "logps/rejected": -402.9896545410156,
-      "loss": 0.5644,
+      "logits/chosen": -1.7340736389160156,
+      "logits/rejected": -1.6779124736785889,
+      "logps/chosen": -397.98602294921875,
+      "logps/rejected": -363.1806335449219,
+      "loss": 0.5418,
       "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.07031740248203278,
-      "rewards/margins": 0.33437561988830566,
-      "rewards/rejected": -0.40469303727149963,
+      "rewards/chosen": 1.2276204824447632,
+      "rewards/margins": 1.8878930807113647,
+      "rewards/rejected": -0.6602726578712463,
       "step": 250
     },
     {
-      "epoch": 0.59,
-      "grad_norm": 39.74076278883575,
+      "epoch": 0.5915813424345847,
+      "grad_norm": 3341.9896511224047,
       "learning_rate": 4.26696226741691e-08,
-      "logits/chosen": -2.2478280067443848,
-      "logits/rejected": -2.1786162853240967,
-      "logps/chosen": -446.00897216796875,
-      "logps/rejected": -439.8905334472656,
-      "loss": 0.5747,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.13826611638069153,
-      "rewards/margins": 0.3671457767486572,
-      "rewards/rejected": -0.5054119229316711,
+      "logits/chosen": -1.7472769021987915,
+      "logits/rejected": -1.6690517663955688,
+      "logps/chosen": -431.0550231933594,
+      "logps/rejected": -389.91729736328125,
+      "loss": 0.5596,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 1.1272859573364258,
+      "rewards/margins": 1.6952396631240845,
+      "rewards/rejected": -0.5679537653923035,
       "step": 260
     },
     {
-      "epoch": 0.61,
-      "grad_norm": 24.611550372970484,
+      "epoch": 0.6143344709897611,
+      "grad_norm": 2341.4958744103656,
       "learning_rate": 3.876321082668098e-08,
-      "logits/chosen": -2.306821346282959,
-      "logits/rejected": -2.2316880226135254,
-      "logps/chosen": -465.56439208984375,
-      "logps/rejected": -459.61700439453125,
-      "loss": 0.564,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.019088072702288628,
-      "rewards/margins": 0.4447278082370758,
-      "rewards/rejected": -0.463815838098526,
+      "logits/chosen": -1.808306097984314,
+      "logits/rejected": -1.7300828695297241,
+      "logps/chosen": -461.818359375,
+      "logps/rejected": -413.60040283203125,
+      "loss": 0.5324,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 1.8372154235839844,
+      "rewards/margins": 2.2021522521972656,
+      "rewards/rejected": -0.36493679881095886,
       "step": 270
     },
     {
-      "epoch": 0.64,
-      "grad_norm": 39.92086883833645,
+      "epoch": 0.6370875995449374,
+      "grad_norm": 3913.185491878204,
       "learning_rate": 3.492784157826244e-08,
-      "logits/chosen": -2.236232280731201,
-      "logits/rejected": -2.1331005096435547,
-      "logps/chosen": -452.8758850097656,
-      "logps/rejected": -426.164306640625,
-      "loss": 0.5642,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.054852552711963654,
-      "rewards/margins": 0.43687576055526733,
-      "rewards/rejected": -0.49172839522361755,
+      "logits/chosen": -1.7311302423477173,
+      "logits/rejected": -1.6296806335449219,
+      "logps/chosen": -445.4042053222656,
+      "logps/rejected": -377.4632568359375,
+      "loss": 0.714,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 1.9863967895507812,
+      "rewards/margins": 2.4581451416015625,
+      "rewards/rejected": -0.47174835205078125,
       "step": 280
     },
     {
-      "epoch": 0.66,
-      "grad_norm": 30.441413413514407,
+      "epoch": 0.6598407281001137,
+      "grad_norm": 2536.8509305796933,
       "learning_rate": 3.118776336817812e-08,
-      "logits/chosen": -2.286245346069336,
-      "logits/rejected": -2.217912435531616,
-      "logps/chosen": -441.1709899902344,
-      "logps/rejected": -427.1979064941406,
-      "loss": 0.5531,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.020920906215906143,
-      "rewards/margins": 0.44389209151268005,
-      "rewards/rejected": -0.4648129940032959,
+      "logits/chosen": -1.7828449010849,
+      "logits/rejected": -1.7099231481552124,
+      "logps/chosen": -436.8643493652344,
+      "logps/rejected": -380.64483642578125,
+      "loss": 0.6404,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 2.214547634124756,
+      "rewards/margins": 2.1427664756774902,
+      "rewards/rejected": 0.0717814713716507,
       "step": 290
     },
     {
-      "epoch": 0.68,
-      "grad_norm": 27.497209882056495,
+      "epoch": 0.6825938566552902,
+      "grad_norm": 4257.034541972718,
       "learning_rate": 2.7566622175067443e-08,
-      "logits/chosen": -2.2853000164031982,
-      "logits/rejected": -2.2172305583953857,
-      "logps/chosen": -439.1316833496094,
-      "logps/rejected": -443.3128967285156,
-      "loss": 0.5682,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.03673394396901131,
-      "rewards/margins": 0.42858797311782837,
-      "rewards/rejected": -0.46532192826271057,
+      "logits/chosen": -1.7593892812728882,
+      "logits/rejected": -1.688314437866211,
+      "logps/chosen": -432.97064208984375,
+      "logps/rejected": -396.4273376464844,
+      "loss": 0.6727,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 2.487614870071411,
+      "rewards/margins": 2.1342551708221436,
+      "rewards/rejected": 0.3533596992492676,
       "step": 300
     },
     {
-      "epoch": 0.71,
-      "grad_norm": 35.65048747036057,
+      "epoch": 0.7053469852104665,
+      "grad_norm": 2159.6702091791217,
       "learning_rate": 2.408731201945432e-08,
-      "logits/chosen": -2.270371437072754,
-      "logits/rejected": -2.2171905040740967,
-      "logps/chosen": -437.3782653808594,
-      "logps/rejected": -453.547607421875,
-      "loss": 0.5546,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.06338675320148468,
-      "rewards/margins": 0.3352622985839844,
-      "rewards/rejected": -0.39864906668663025,
+      "logits/chosen": -1.75957453250885,
+      "logits/rejected": -1.709562063217163,
+      "logps/chosen": -429.00360107421875,
+      "logps/rejected": -413.32177734375,
+      "loss": 0.5967,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 2.035928249359131,
+      "rewards/margins": 1.6749908924102783,
+      "rewards/rejected": 0.36093729734420776,
       "step": 310
     },
     {
-      "epoch": 0.73,
-      "grad_norm": 34.348920131629896,
+      "epoch": 0.7281001137656428,
+      "grad_norm": 4478.383858957151,
       "learning_rate": 2.0771830220378112e-08,
-      "logits/chosen": -2.223020076751709,
-      "logits/rejected": -2.158247709274292,
-      "logps/chosen": -462.366455078125,
-      "logps/rejected": -461.9310607910156,
-      "loss": 0.5476,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.025463122874498367,
-      "rewards/margins": 0.39216503500938416,
-      "rewards/rejected": -0.41762813925743103,
+      "logits/chosen": -1.7119019031524658,
+      "logits/rejected": -1.6519428491592407,
+      "logps/chosen": -457.98175048828125,
+      "logps/rejected": -420.1382751464844,
+      "loss": 0.5886,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 1.8383922576904297,
+      "rewards/margins": 1.8084253072738647,
+      "rewards/rejected": 0.02996676042675972,
       "step": 320
     },
     {
-      "epoch": 0.75,
-      "grad_norm": 27.509152301102358,
+      "epoch": 0.7508532423208191,
+      "grad_norm": 2972.164463500601,
       "learning_rate": 1.7641138321260257e-08,
-      "logits/chosen": -2.2643046379089355,
-      "logits/rejected": -2.188499689102173,
-      "logps/chosen": -440.750732421875,
-      "logps/rejected": -428.7913513183594,
-      "loss": 0.5541,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.06256476789712906,
-      "rewards/margins": 0.46622103452682495,
-      "rewards/rejected": -0.5287858247756958,
+      "logits/chosen": -1.7484897375106812,
+      "logits/rejected": -1.670183539390564,
+      "logps/chosen": -432.33966064453125,
+      "logps/rejected": -376.2579650878906,
+      "loss": 0.5475,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 2.154627799987793,
+      "rewards/margins": 2.4997963905334473,
+      "rewards/rejected": -0.34516826272010803,
       "step": 330
     },
     {
-      "epoch": 0.77,
-      "grad_norm": 27.197721402060907,
+      "epoch": 0.7736063708759955,
+      "grad_norm": 2288.763315736693,
       "learning_rate": 1.4715029564277793e-08,
-      "logits/chosen": -2.340247631072998,
-      "logits/rejected": -2.2862460613250732,
-      "logps/chosen": -450.05523681640625,
-      "logps/rejected": -448.6039123535156,
-      "loss": 0.5553,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.009611198678612709,
-      "rewards/margins": 0.4653921127319336,
-      "rewards/rejected": -0.45578089356422424,
+      "logits/chosen": -1.8215715885162354,
+      "logits/rejected": -1.7631429433822632,
+      "logps/chosen": -448.9734802246094,
+      "logps/rejected": -402.9405822753906,
+      "loss": 0.5402,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 2.042846202850342,
+      "rewards/margins": 1.9576542377471924,
+      "rewards/rejected": 0.08519164472818375,
       "step": 340
     },
     {
-      "epoch": 0.8,
-      "grad_norm": 23.708931464533432,
+      "epoch": 0.7963594994311718,
+      "grad_norm": 2817.482587758628,
       "learning_rate": 1.2012003751113343e-08,
-      "logits/chosen": -2.325669288635254,
-      "logits/rejected": -2.259887933731079,
-      "logps/chosen": -452.64862060546875,
-      "logps/rejected": -454.2327575683594,
-      "loss": 0.538,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.17242571711540222,
-      "rewards/margins": 0.4136194586753845,
-      "rewards/rejected": -0.5860452055931091,
+      "logits/chosen": -1.8010448217391968,
+      "logits/rejected": -1.7336769104003906,
+      "logps/chosen": -433.62738037109375,
+      "logps/rejected": -396.10821533203125,
+      "loss": 0.4871,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 1.7786489725112915,
+      "rewards/margins": 2.2586090564727783,
+      "rewards/rejected": -0.47995972633361816,
       "step": 350
     },
     {
-      "epoch": 0.82,
-      "grad_norm": 32.57218388588953,
+      "epoch": 0.8191126279863481,
+      "grad_norm": 2893.893774716542,
       "learning_rate": 9.549150281252633e-09,
-      "logits/chosen": -2.284872531890869,
-      "logits/rejected": -2.2292895317077637,
-      "logps/chosen": -473.66009521484375,
-      "logps/rejected": -472.35919189453125,
-      "loss": 0.547,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.11653508991003036,
-      "rewards/margins": 0.4531070590019226,
-      "rewards/rejected": -0.5696421265602112,
+      "logits/chosen": -1.7686980962753296,
+      "logits/rejected": -1.7077505588531494,
+      "logps/chosen": -459.77239990234375,
+      "logps/rejected": -415.1337890625,
+      "loss": 0.5747,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 2.2342071533203125,
+      "rewards/margins": 1.9730135202407837,
+      "rewards/rejected": 0.2611936926841736,
       "step": 360
     },
     {
-      "epoch": 0.84,
-      "grad_norm": 36.6888447736333,
+      "epoch": 0.8418657565415245,
+      "grad_norm": 2035.2633008736568,
       "learning_rate": 7.3420401072985306e-09,
-      "logits/chosen": -2.322558641433716,
-      "logits/rejected": -2.267219066619873,
-      "logps/chosen": -451.59295654296875,
-      "logps/rejected": -459.50665283203125,
-      "loss": 0.5582,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.14963454008102417,
-      "rewards/margins": 0.4179585874080658,
-      "rewards/rejected": -0.5675932168960571,
+      "logits/chosen": -1.7992585897445679,
+      "logits/rejected": -1.7426245212554932,
+      "logps/chosen": -434.25457763671875,
+      "logps/rejected": -402.511962890625,
+      "loss": 0.5722,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 2.3749587535858154,
+      "rewards/margins": 2.13959002494812,
+      "rewards/rejected": 0.23536887764930725,
       "step": 370
     },
     {
-      "epoch": 0.86,
-      "grad_norm": 25.485415219281943,
+      "epoch": 0.8646188850967008,
+      "grad_norm": 2185.073098497136,
       "learning_rate": 5.404627290395369e-09,
-      "logits/chosen": -2.2841172218322754,
-      "logits/rejected": -2.213308334350586,
-      "logps/chosen": -445.1299743652344,
-      "logps/rejected": -450.1717224121094,
-      "loss": 0.5491,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.027432629838585854,
-      "rewards/margins": 0.4644508957862854,
-      "rewards/rejected": -0.4918835163116455,
+      "logits/chosen": -1.7635869979858398,
+      "logits/rejected": -1.69329035282135,
+      "logps/chosen": -439.9053649902344,
+      "logps/rejected": -400.70074462890625,
+      "loss": 0.6333,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.4813942909240723,
+      "rewards/margins": 2.198753833770752,
+      "rewards/rejected": 0.2826406955718994,
       "step": 380
     },
     {
-      "epoch": 0.89,
-      "grad_norm": 37.42685396157706,
+      "epoch": 0.8873720136518771,
+      "grad_norm": 3553.679444582505,
       "learning_rate": 3.74916077816162e-09,
-      "logits/chosen": -2.292269706726074,
-      "logits/rejected": -2.226890802383423,
-      "logps/chosen": -436.8121643066406,
-      "logps/rejected": -433.2718200683594,
-      "loss": 0.5552,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.1562749743461609,
-      "rewards/margins": 0.388277530670166,
-      "rewards/rejected": -0.5445524454116821,
+      "logits/chosen": -1.774793267250061,
+      "logits/rejected": -1.7063766717910767,
+      "logps/chosen": -419.1651306152344,
+      "logps/rejected": -378.5888671875,
+      "loss": 0.5591,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 2.019554615020752,
+      "rewards/margins": 1.7918205261230469,
+      "rewards/rejected": 0.22773392498493195,
       "step": 390
     },
     {
-      "epoch": 0.91,
-      "grad_norm": 24.281602089300215,
+      "epoch": 0.9101251422070534,
+      "grad_norm": 1943.3403398028058,
       "learning_rate": 2.386106962899165e-09,
-      "logits/chosen": -2.218071460723877,
-      "logits/rejected": -2.1384034156799316,
-      "logps/chosen": -447.8563537597656,
-      "logps/rejected": -437.1399841308594,
-      "loss": 0.5478,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.219069242477417,
-      "rewards/margins": 0.37984699010849,
-      "rewards/rejected": -0.598916232585907,
+      "logits/chosen": -1.6973623037338257,
+      "logits/rejected": -1.6196067333221436,
+      "logps/chosen": -424.16998291015625,
+      "logps/rejected": -377.53851318359375,
+      "loss": 0.5123,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 1.7794551849365234,
+      "rewards/margins": 2.069586992263794,
+      "rewards/rejected": -0.2901318371295929,
       "step": 400
     },
     {
-      "epoch": 0.93,
-      "grad_norm": 28.633671701703214,
+      "epoch": 0.9328782707622298,
+      "grad_norm": 3047.206047019034,
       "learning_rate": 1.3240835096913706e-09,
-      "logits/chosen": -2.2583167552948,
-      "logits/rejected": -2.1519083976745605,
-      "logps/chosen": -440.5953063964844,
-      "logps/rejected": -431.39501953125,
-      "loss": 0.5514,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.04258907586336136,
-      "rewards/margins": 0.5055002570152283,
-      "rewards/rejected": -0.5480893850326538,
+      "logits/chosen": -1.7308810949325562,
+      "logits/rejected": -1.619262456893921,
+      "logps/chosen": -433.6739196777344,
+      "logps/rejected": -376.218505859375,
+      "loss": 0.6337,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 2.662471294403076,
+      "rewards/margins": 2.294875144958496,
+      "rewards/rejected": 0.36759620904922485,
       "step": 410
     },
     {
-      "epoch": 0.96,
-      "grad_norm": 25.241002875120977,
+      "epoch": 0.9556313993174061,
+      "grad_norm": 2161.402141420211,
       "learning_rate": 5.698048727497462e-10,
-      "logits/chosen": -2.2798821926116943,
-      "logits/rejected": -2.2044219970703125,
-      "logps/chosen": -424.9578552246094,
-      "logps/rejected": -435.0152893066406,
-      "loss": 0.5514,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.11878044903278351,
-      "rewards/margins": 0.5139662623405457,
-      "rewards/rejected": -0.632746696472168,
+      "logits/chosen": -1.7543413639068604,
+      "logits/rejected": -1.6781110763549805,
+      "logps/chosen": -410.76763916015625,
+      "logps/rejected": -371.71697998046875,
+      "loss": 0.5539,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 2.3121988773345947,
+      "rewards/margins": 2.288635015487671,
+      "rewards/rejected": 0.02356356382369995,
       "step": 420
     },
     {
-      "epoch": 0.98,
-      "grad_norm": 23.361501927411613,
+      "epoch": 0.9783845278725825,
+      "grad_norm": 2190.386640999091,
       "learning_rate": 1.2803984447259387e-10,
-      "logits/chosen": -2.295804977416992,
-      "logits/rejected": -2.235670328140259,
-      "logps/chosen": -475.6675720214844,
-      "logps/rejected": -457.9853515625,
-      "loss": 0.5428,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.12296229600906372,
-      "rewards/margins": 0.4703094959259033,
-      "rewards/rejected": -0.593271791934967,
+      "logits/chosen": -1.7616386413574219,
+      "logits/rejected": -1.6976215839385986,
+      "logps/chosen": -461.20989990234375,
+      "logps/rejected": -398.564453125,
+      "loss": 0.5426,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 2.1613998413085938,
+      "rewards/margins": 2.0676867961883545,
+      "rewards/rejected": 0.09371300041675568,
       "step": 430
     },
     {
-      "epoch": 1.0,
+      "epoch": 0.9988623435722411,
       "step": 439,
       "total_flos": 0.0,
-      "train_loss": 0.048529281703106095,
-      "train_runtime": 584.5423,
-      "train_samples_per_second": 96.205,
-      "train_steps_per_second": 0.751
+      "train_loss": 0.6069056124241857,
+      "train_runtime": 7568.165,
+      "train_samples_per_second": 7.431,
+      "train_steps_per_second": 0.058
     }
   ],
   "logging_steps": 10,
@@ -683,6 +683,18 @@
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
   "total_flos": 0.0,
   "train_batch_size": 4,
   "trial_name": null,