Model save

README.md

@@ -14,16 +14,6 @@ should probably proofread and complete it, then remove this comment. -->
 # zephyr-7b-dpo-full
 
 This model was trained from scratch on the None dataset.
-It achieves the following results on the evaluation set:
-- Loss: 0.5054
-- Rewards/chosen: -1.3174
-- Rewards/rejected: -2.4481
-- Rewards/accuracies: 0.7773
-- Rewards/margins: 1.1307
-- Logps/rejected: -556.4355
-- Logps/chosen: -435.3261
-- Logits/rejected: -0.1274
-- Logits/chosen: -0.0846
 
 ## Model description
 
@@ -42,13 +32,13 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 3e-06
-- train_batch_size: 4
+- learning_rate: 1e-06
+- train_batch_size: 2
 - eval_batch_size: 8
-- seed: 3
+- seed: 1
 - distributed_type: multi-GPU
 - num_devices: 8
-- gradient_accumulation_steps: 4
+- gradient_accumulation_steps: 8
 - total_train_batch_size: 128
 - total_eval_batch_size: 64
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
@@ -58,12 +48,6 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
-|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.5502        | 0.21  | 100  | 0.5705          | -1.1110        | -1.9243          | 0.7578             | 0.8133          | -504.0533      | -414.6870    | -0.1991         | -0.2234       |
-| 0.539         | 0.42  | 200  | 0.5325          | -1.2893        | -2.1763          | 0.7578             | 0.8870          | -529.2560      | -432.5124    | 0.1917          | 0.1517        |
-| 0.528         | 0.63  | 300  | 0.5161          | -1.4780        | -2.6138          | 0.7578             | 1.1358          | -573.0014      | -451.3821    | 0.1045          | 0.1296        |
-| 0.5049        | 0.84  | 400  | 0.5054          | -1.3174        | -2.4481          | 0.7773             | 1.1307          | -556.4355      | -435.3261    | -0.1274         | -0.0846       |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5426262839535247,
-    "train_runtime": 4328.2164,
-    "train_samples": 61134,
-    "train_samples_per_second": 14.125,
-    "train_steps_per_second": 0.11
+    "train_loss": 0.464236611379704,
+    "train_runtime": 5271.2295,
+    "train_samples": 45548,
+    "train_samples_per_second": 8.641,
+    "train_steps_per_second": 0.067
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c1ad713d321f72b4f6dee955bf279416bc9a375ca6b8ee6bd1648ec8f26dc08d
+oid sha256:366bb9560cc47efbca54034565f6513238203e5c0f566fddc150a9d3e6085bfa
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3ab3eb1ce23074d25dd5354b0630fc90dcb29f8a44bd558e73222da30d71e1b7
+oid sha256:e2120eb8af6751d1817786193d0d4ef3daf4da774c7049a6883347a4245178ff
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:951657e3c35c06e4d339cd40ffbd2fb90d4fa240970e6a9a05e6beede4607a5a
+oid sha256:c6e8c563ecda5484c9528a79ef74141bcc4fc0c565785c2d6adf764a77545114
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:52adaaebdf92d341736955fe50fe4178cf74714cb5d47cf7ec73b6ef772497db
+oid sha256:0a7239e159560eaf7980449da5998b76d7130edd7090b3808a7b04eeed33600e
 size 1168138808

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5426262839535247,
-    "train_runtime": 4328.2164,
-    "train_samples": 61134,
-    "train_samples_per_second": 14.125,
-    "train_steps_per_second": 0.11
+    "train_loss": 0.464236611379704,
+    "train_runtime": 5271.2295,
+    "train_samples": 45548,
+    "train_samples_per_second": 8.641,
+    "train_steps_per_second": 0.067
 }

trainer_state.json

@@ -1,749 +1,517 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9984301412872841,
-  "eval_steps": 100,
-  "global_step": 477,
+  "epoch": 0.9975412715138743,
+  "eval_steps": 10000,
+  "global_step": 355,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.02,
-      "learning_rate": 6.25e-07,
-      "logits/chosen": 0.2209470570087433,
-      "logits/rejected": 0.3076450824737549,
-      "logps/chosen": -324.3524169921875,
-      "logps/rejected": -285.35931396484375,
-      "loss": 0.6932,
-      "rewards/accuracies": 0.45625001192092896,
-      "rewards/chosen": 0.0010343596804887056,
-      "rewards/margins": 0.0006591519340872765,
-      "rewards/rejected": 0.0003752077464014292,
+      "epoch": 0.03,
+      "learning_rate": 2.7777777777777776e-07,
+      "logits/chosen": -0.08267850428819656,
+      "logits/rejected": -0.0387466736137867,
+      "logps/chosen": -327.2626037597656,
+      "logps/rejected": -244.530517578125,
+      "loss": 0.5075,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.0007698397384956479,
+      "rewards/margins": 0.0003367254394106567,
+      "rewards/rejected": -0.0011065651196986437,
       "step": 10
     },
     {
-      "epoch": 0.04,
-      "learning_rate": 1.25e-06,
-      "logits/chosen": 0.3226737976074219,
-      "logits/rejected": 0.4124608039855957,
-      "logps/chosen": -323.83349609375,
-      "logps/rejected": -290.1531677246094,
-      "loss": 0.6877,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.0006101070903241634,
-      "rewards/margins": 0.011370119638741016,
-      "rewards/rejected": -0.011980227194726467,
+      "epoch": 0.06,
+      "learning_rate": 5.555555555555555e-07,
+      "logits/chosen": -0.05577896907925606,
+      "logits/rejected": -0.02608281373977661,
+      "logps/chosen": -291.40679931640625,
+      "logps/rejected": -195.19332885742188,
+      "loss": 0.5124,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.0003207808767911047,
+      "rewards/margins": 0.007293092552572489,
+      "rewards/rejected": -0.007613874040544033,
       "step": 20
     },
     {
-      "epoch": 0.06,
-      "learning_rate": 1.875e-06,
-      "logits/chosen": 0.37067800760269165,
-      "logits/rejected": 0.4101516604423523,
-      "logps/chosen": -312.599609375,
-      "logps/rejected": -317.5506896972656,
-      "loss": 0.6643,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.022324323654174805,
-      "rewards/margins": 0.08467327058315277,
-      "rewards/rejected": -0.10699759423732758,
+      "epoch": 0.08,
+      "learning_rate": 8.333333333333333e-07,
+      "logits/chosen": -0.019027356058359146,
+      "logits/rejected": 0.024959497153759003,
+      "logps/chosen": -348.4835205078125,
+      "logps/rejected": -209.26522827148438,
+      "loss": 0.5167,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.006022198125720024,
+      "rewards/margins": 0.05245554447174072,
+      "rewards/rejected": -0.046433351933956146,
       "step": 30
     },
     {
-      "epoch": 0.08,
-      "learning_rate": 2.5e-06,
-      "logits/chosen": 0.3572315275669098,
-      "logits/rejected": 0.4867202639579773,
-      "logps/chosen": -369.9970703125,
-      "logps/rejected": -357.2043762207031,
-      "loss": 0.6422,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.2718835771083832,
-      "rewards/margins": 0.19485555589199066,
-      "rewards/rejected": -0.46673911809921265,
+      "epoch": 0.11,
+      "learning_rate": 9.99612097830993e-07,
+      "logits/chosen": -0.012003236450254917,
+      "logits/rejected": 0.02850813791155815,
+      "logps/chosen": -310.19439697265625,
+      "logps/rejected": -234.8997039794922,
+      "loss": 0.5424,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.06786860525608063,
+      "rewards/margins": 0.05475381761789322,
+      "rewards/rejected": -0.12262241542339325,
       "step": 40
     },
     {
-      "epoch": 0.1,
-      "learning_rate": 2.999839121261416e-06,
-      "logits/chosen": 0.02461487613618374,
-      "logits/rejected": 0.1311604082584381,
-      "logps/chosen": -397.81427001953125,
-      "logps/rejected": -374.7628173828125,
-      "loss": 0.6205,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.30967187881469727,
-      "rewards/margins": 0.26407095789909363,
-      "rewards/rejected": -0.5737428069114685,
+      "epoch": 0.14,
+      "learning_rate": 9.952551076085863e-07,
+      "logits/chosen": 0.0373210608959198,
+      "logits/rejected": 0.07148866355419159,
+      "logps/chosen": -325.3377990722656,
+      "logps/rejected": -265.7305908203125,
+      "loss": 0.5454,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.09839601814746857,
+      "rewards/margins": 0.1340223103761673,
+      "rewards/rejected": -0.23241834342479706,
       "step": 50
     },
     {
-      "epoch": 0.13,
-      "learning_rate": 2.994211988057582e-06,
-      "logits/chosen": -0.08134393393993378,
-      "logits/rejected": -0.0017544396687299013,
-      "logps/chosen": -334.15093994140625,
-      "logps/rejected": -356.8257141113281,
-      "loss": 0.5972,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.4183749258518219,
-      "rewards/margins": 0.3044845461845398,
-      "rewards/rejected": -0.7228595018386841,
+      "epoch": 0.17,
+      "learning_rate": 9.860986139994238e-07,
+      "logits/chosen": -0.04613853245973587,
+      "logits/rejected": 0.015464186668395996,
+      "logps/chosen": -391.8591003417969,
+      "logps/rejected": -237.02279663085938,
+      "loss": 0.5384,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.04945594444870949,
+      "rewards/margins": 0.2955462634563446,
+      "rewards/rejected": -0.3450022041797638,
       "step": 60
     },
     {
-      "epoch": 0.15,
-      "learning_rate": 2.9805753939568693e-06,
-      "logits/chosen": -0.12287361919879913,
-      "logits/rejected": -0.03368464112281799,
-      "logps/chosen": -365.60296630859375,
-      "logps/rejected": -404.632568359375,
-      "loss": 0.5902,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.6079816818237305,
-      "rewards/margins": 0.4893825948238373,
-      "rewards/rejected": -1.0973644256591797,
+      "epoch": 0.2,
+      "learning_rate": 9.722313523268027e-07,
+      "logits/chosen": -0.05867184326052666,
+      "logits/rejected": 0.02377297915518284,
+      "logps/chosen": -370.95343017578125,
+      "logps/rejected": -244.0323028564453,
+      "loss": 0.5278,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.024297554045915604,
+      "rewards/margins": 0.19173592329025269,
+      "rewards/rejected": -0.21603348851203918,
       "step": 70
     },
     {
-      "epoch": 0.17,
-      "learning_rate": 2.959002435526626e-06,
-      "logits/chosen": -0.17652544379234314,
-      "logits/rejected": -0.11289135366678238,
-      "logps/chosen": -397.5993957519531,
-      "logps/rejected": -454.41571044921875,
-      "loss": 0.5711,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.8409433364868164,
-      "rewards/margins": 0.5162743926048279,
-      "rewards/rejected": -1.357217788696289,
+      "epoch": 0.22,
+      "learning_rate": 9.537877098354784e-07,
+      "logits/chosen": -0.010928474366664886,
+      "logits/rejected": -0.0014367073308676481,
+      "logps/chosen": -264.022216796875,
+      "logps/rejected": -202.35671997070312,
+      "loss": 0.5297,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.04622369259595871,
+      "rewards/margins": 0.17698831856250763,
+      "rewards/rejected": -0.22321197390556335,
       "step": 80
     },
     {
-      "epoch": 0.19,
-      "learning_rate": 2.929608750821129e-06,
-      "logits/chosen": -0.3760049641132355,
-      "logits/rejected": -0.28880029916763306,
-      "logps/chosen": -430.817138671875,
-      "logps/rejected": -447.759033203125,
-      "loss": 0.5512,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.9714916944503784,
-      "rewards/margins": 0.6265453696250916,
-      "rewards/rejected": -1.5980370044708252,
+      "epoch": 0.25,
+      "learning_rate": 9.309464233486386e-07,
+      "logits/chosen": -0.10472371429204941,
+      "logits/rejected": -0.04422920569777489,
+      "logps/chosen": -364.49853515625,
+      "logps/rejected": -217.35791015625,
+      "loss": 0.516,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.05392575263977051,
+      "rewards/margins": 0.32242026925086975,
+      "rewards/rejected": -0.26849451661109924,
       "step": 90
     },
     {
-      "epoch": 0.21,
-      "learning_rate": 2.892551899524109e-06,
-      "logits/chosen": -0.30831730365753174,
-      "logits/rejected": -0.2624167501926422,
-      "logps/chosen": -448.5101623535156,
-      "logps/rejected": -532.5578002929688,
-      "loss": 0.5502,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.1964621543884277,
-      "rewards/margins": 0.740709662437439,
-      "rewards/rejected": -1.9371719360351562,
+      "epoch": 0.28,
+      "learning_rate": 9.039288471343504e-07,
+      "logits/chosen": -0.06652472913265228,
+      "logits/rejected": -0.046227507293224335,
+      "logps/chosen": -343.7643127441406,
+      "logps/rejected": -256.42938232421875,
+      "loss": 0.5032,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.0055793882347643375,
+      "rewards/margins": 0.1420070379972458,
+      "rewards/rejected": -0.1475864201784134,
       "step": 100
     },
     {
-      "epoch": 0.21,
-      "eval_logits/chosen": -0.22339792549610138,
-      "eval_logits/rejected": -0.1990877389907837,
-      "eval_logps/chosen": -414.68701171875,
-      "eval_logps/rejected": -504.0533142089844,
-      "eval_loss": 0.5704939365386963,
-      "eval_rewards/accuracies": 0.7578125,
-      "eval_rewards/chosen": -1.1110235452651978,
-      "eval_rewards/margins": 0.8132585287094116,
-      "eval_rewards/rejected": -1.9242819547653198,
-      "eval_runtime": 64.9663,
-      "eval_samples_per_second": 30.785,
-      "eval_steps_per_second": 0.493,
-      "step": 100
-    },
-    {
-      "epoch": 0.23,
-      "learning_rate": 2.848030518377739e-06,
-      "logits/chosen": -0.23117919266223907,
-      "logits/rejected": -0.1292828470468521,
-      "logps/chosen": -420.55633544921875,
-      "logps/rejected": -438.8388671875,
-      "loss": 0.5707,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.9971814155578613,
-      "rewards/margins": 0.49195510149002075,
-      "rewards/rejected": -1.4891364574432373,
+      "epoch": 0.31,
+      "learning_rate": 8.729968077675454e-07,
+      "logits/chosen": -0.14867696166038513,
+      "logits/rejected": -0.09129262715578079,
+      "logps/chosen": -295.33929443359375,
+      "logps/rejected": -247.62142944335938,
+      "loss": 0.4879,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.0303972028195858,
+      "rewards/margins": 0.09183444827795029,
+      "rewards/rejected": -0.12223164737224579,
       "step": 110
     },
     {
-      "epoch": 0.25,
-      "learning_rate": 2.7962832564252724e-06,
-      "logits/chosen": -0.19192993640899658,
-      "logits/rejected": -0.07996977120637894,
-      "logps/chosen": -423.1610412597656,
-      "logps/rejected": -425.53839111328125,
-      "loss": 0.5379,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.8748111724853516,
-      "rewards/margins": 0.6682790517807007,
-      "rewards/rejected": -1.5430901050567627,
+      "epoch": 0.34,
+      "learning_rate": 8.384500667760089e-07,
+      "logits/chosen": -0.13717997074127197,
+      "logits/rejected": -0.10880477726459503,
+      "logps/chosen": -314.3236999511719,
+      "logps/rejected": -198.9403533935547,
+      "loss": 0.4789,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.03523362800478935,
+      "rewards/margins": 0.17541493475437164,
+      "rewards/rejected": -0.140181303024292,
       "step": 120
     },
     {
-      "epoch": 0.27,
-      "learning_rate": 2.7375874957747644e-06,
-      "logits/chosen": -0.1726234257221222,
-      "logits/rejected": -0.08887670934200287,
-      "logps/chosen": -431.18365478515625,
-      "logps/rejected": -499.1405334472656,
-      "loss": 0.5636,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.0607411861419678,
-      "rewards/margins": 0.934187114238739,
-      "rewards/rejected": -1.9949283599853516,
+      "epoch": 0.37,
+      "learning_rate": 8.006234156598042e-07,
+      "logits/chosen": -0.14056822657585144,
+      "logits/rejected": -0.09251274913549423,
+      "logps/chosen": -349.345458984375,
+      "logps/rejected": -211.16281127929688,
+      "loss": 0.478,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.007185462862253189,
+      "rewards/margins": 0.2594950795173645,
+      "rewards/rejected": -0.2523096203804016,
       "step": 130
     },
     {
-      "epoch": 0.29,
-      "learning_rate": 2.672257864741005e-06,
-      "logits/chosen": -0.15248467028141022,
-      "logits/rejected": -0.09780655056238174,
-      "logps/chosen": -401.5557556152344,
-      "logps/rejected": -453.06341552734375,
-      "loss": 0.5422,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.9503288269042969,
-      "rewards/margins": 0.6491050124168396,
-      "rewards/rejected": -1.5994337797164917,
+      "epoch": 0.39,
+      "learning_rate": 7.59883431436215e-07,
+      "logits/chosen": -0.11560620367527008,
+      "logits/rejected": -0.09353138506412506,
+      "logps/chosen": -307.52716064453125,
+      "logps/rejected": -232.7269287109375,
+      "loss": 0.4694,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.086158387362957,
+      "rewards/margins": 0.15163089334964752,
+      "rewards/rejected": -0.23778927326202393,
       "step": 140
     },
     {
-      "epoch": 0.31,
-      "learning_rate": 2.600644551335706e-06,
-      "logits/chosen": -0.06402697414159775,
-      "logits/rejected": -0.022289589047431946,
-      "logps/chosen": -436.34893798828125,
-      "logps/rejected": -519.2093505859375,
-      "loss": 0.5418,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.1614066362380981,
-      "rewards/margins": 1.014080286026001,
-      "rewards/rejected": -2.1754870414733887,
+      "epoch": 0.42,
+      "learning_rate": 7.166249241521318e-07,
+      "logits/chosen": -0.09010031074285507,
+      "logits/rejected": -0.050861239433288574,
+      "logps/chosen": -286.13616943359375,
+      "logps/rejected": -242.67953491210938,
+      "loss": 0.4676,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.16256649792194366,
+      "rewards/margins": 0.15163187682628632,
+      "rewards/rejected": -0.3141983449459076,
       "step": 150
     },
     {
-      "epoch": 0.33,
-      "learning_rate": 2.5231314261461732e-06,
-      "logits/chosen": -0.13501138985157013,
-      "logits/rejected": -0.03512698411941528,
-      "logps/chosen": -405.4765319824219,
-      "logps/rejected": -474.7965393066406,
-      "loss": 0.5315,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.0841046571731567,
-      "rewards/margins": 0.7699181437492371,
-      "rewards/rejected": -1.854022741317749,
+      "epoch": 0.45,
+      "learning_rate": 6.712671107909358e-07,
+      "logits/chosen": -0.18100903928279877,
+      "logits/rejected": -0.10084307193756104,
+      "logps/chosen": -358.4205322265625,
+      "logps/rejected": -247.5520782470703,
+      "loss": 0.4636,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.09556841105222702,
+      "rewards/margins": 0.2233666479587555,
+      "rewards/rejected": -0.31893500685691833,
       "step": 160
     },
     {
-      "epoch": 0.36,
-      "learning_rate": 2.440133984664454e-06,
-      "logits/chosen": -0.11433794349431992,
-      "logits/rejected": -0.04908312112092972,
-      "logps/chosen": -425.5057678222656,
-      "logps/rejected": -513.4010009765625,
-      "loss": 0.5526,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -1.2522752285003662,
-      "rewards/margins": 0.9037529230117798,
-      "rewards/rejected": -2.1560280323028564,
+      "epoch": 0.48,
+      "learning_rate": 6.24249552652447e-07,
+      "logits/chosen": -0.1316126585006714,
+      "logits/rejected": -0.10074617713689804,
+      "logps/chosen": -309.5055236816406,
+      "logps/rejected": -261.95977783203125,
+      "loss": 0.4396,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.14085456728935242,
+      "rewards/margins": 0.18683212995529175,
+      "rewards/rejected": -0.32768669724464417,
       "step": 170
     },
     {
-      "epoch": 0.38,
-      "learning_rate": 2.3520971200967337e-06,
-      "logits/chosen": -0.2898910641670227,
-      "logits/rejected": -0.11892978847026825,
-      "logps/chosen": -461.5921936035156,
-      "logps/rejected": -437.5884704589844,
-      "loss": 0.5491,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.0226460695266724,
-      "rewards/margins": 0.5135943293571472,
-      "rewards/rejected": -1.5362403392791748,
+      "epoch": 0.51,
+      "learning_rate": 5.760278955766694e-07,
+      "logits/chosen": -0.23117272555828094,
+      "logits/rejected": -0.16750793159008026,
+      "logps/chosen": -324.4317932128906,
+      "logps/rejected": -253.4479217529297,
+      "loss": 0.4428,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.22457607090473175,
+      "rewards/margins": 0.1670912355184555,
+      "rewards/rejected": -0.39166730642318726,
       "step": 180
     },
     {
-      "epoch": 0.4,
-      "learning_rate": 2.2594927385914546e-06,
-      "logits/chosen": -0.11167088896036148,
-      "logits/rejected": -0.024417612701654434,
-      "logps/chosen": -459.20880126953125,
-      "logps/rejected": -484.350341796875,
-      "loss": 0.537,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.387242078781128,
-      "rewards/margins": 0.6068987250328064,
-      "rewards/rejected": -1.994140863418579,
+      "epoch": 0.53,
+      "learning_rate": 5.270694542927088e-07,
+      "logits/chosen": -0.17674800753593445,
+      "logits/rejected": -0.13648080825805664,
+      "logps/chosen": -336.358642578125,
+      "logps/rejected": -226.1509246826172,
+      "loss": 0.4263,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.160172238945961,
+      "rewards/margins": 0.2827780246734619,
+      "rewards/rejected": -0.4429502487182617,
       "step": 190
     },
     {
-      "epoch": 0.42,
-      "learning_rate": 2.1628172296692954e-06,
-      "logits/chosen": 0.08049922436475754,
-      "logits/rejected": 0.168174147605896,
-      "logps/chosen": -408.86431884765625,
-      "logps/rejected": -484.8013610839844,
-      "loss": 0.539,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.2882415056228638,
-      "rewards/margins": 0.807357668876648,
-      "rewards/rejected": -2.095599412918091,
+      "epoch": 0.56,
+      "learning_rate": 4.778486836848107e-07,
+      "logits/chosen": -0.21899878978729248,
+      "logits/rejected": -0.13763020932674408,
+      "logps/chosen": -316.164794921875,
+      "logps/rejected": -248.52218627929688,
+      "loss": 0.4349,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.2667180895805359,
+      "rewards/margins": 0.17106209695339203,
+      "rewards/rejected": -0.43778014183044434,
       "step": 200
     },
     {
-      "epoch": 0.42,
-      "eval_logits/chosen": 0.15173441171646118,
-      "eval_logits/rejected": 0.19174005091190338,
-      "eval_logps/chosen": -432.51239013671875,
-      "eval_logps/rejected": -529.2559814453125,
-      "eval_loss": 0.5325208306312561,
-      "eval_rewards/accuracies": 0.7578125,
-      "eval_rewards/chosen": -1.289277195930481,
-      "eval_rewards/margins": 0.8870314955711365,
-      "eval_rewards/rejected": -2.1763086318969727,
-      "eval_runtime": 64.8606,
-      "eval_samples_per_second": 30.835,
-      "eval_steps_per_second": 0.493,
-      "step": 200
-    },
-    {
-      "epoch": 0.44,
-      "learning_rate": 2.062588805414343e-06,
-      "logits/chosen": 0.2150273621082306,
-      "logits/rejected": 0.3071494400501251,
-      "logps/chosen": -429.50146484375,
-      "logps/rejected": -501.4217834472656,
-      "loss": 0.5217,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.4701905250549316,
-      "rewards/margins": 0.7591425776481628,
-      "rewards/rejected": -2.2293331623077393,
+      "epoch": 0.59,
+      "learning_rate": 4.2884258086335745e-07,
+      "logits/chosen": -0.10616914182901382,
+      "logits/rejected": -0.06204689294099808,
+      "logps/chosen": -380.58349609375,
+      "logps/rejected": -267.28692626953125,
+      "loss": 0.427,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.19200220704078674,
+      "rewards/margins": 0.2422538697719574,
+      "rewards/rejected": -0.43425607681274414,
       "step": 210
     },
     {
-      "epoch": 0.46,
-      "learning_rate": 1.9593447226892386e-06,
-      "logits/chosen": 0.1677493005990982,
-      "logits/rejected": 0.25236162543296814,
-      "logps/chosen": -453.639892578125,
-      "logps/rejected": -529.4569091796875,
-      "loss": 0.5335,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.4858437776565552,
-      "rewards/margins": 0.8870512247085571,
-      "rewards/rejected": -2.372894763946533,
+      "epoch": 0.62,
+      "learning_rate": 3.8052606259922095e-07,
+      "logits/chosen": -0.2504015564918518,
+      "logits/rejected": -0.1911773979663849,
+      "logps/chosen": -355.21234130859375,
+      "logps/rejected": -239.8585968017578,
+      "loss": 0.432,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.2169329822063446,
+      "rewards/margins": 0.1906813234090805,
+      "rewards/rejected": -0.4076143205165863,
       "step": 220
     },
     {
-      "epoch": 0.48,
-      "learning_rate": 1.853638403264141e-06,
-      "logits/chosen": 0.18635836243629456,
-      "logits/rejected": 0.2622009217739105,
-      "logps/chosen": -424.3346252441406,
-      "logps/rejected": -484.9342346191406,
-      "loss": 0.5094,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.2930189371109009,
-      "rewards/margins": 0.7800495028495789,
-      "rewards/rejected": -2.073068380355835,
+      "epoch": 0.65,
+      "learning_rate": 3.333673629186279e-07,
+      "logits/chosen": -0.1560947299003601,
+      "logits/rejected": -0.10800876468420029,
+      "logps/chosen": -324.886962890625,
+      "logps/rejected": -239.11428833007812,
+      "loss": 0.4215,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.20993120968341827,
+      "rewards/margins": 0.2062130719423294,
+      "rewards/rejected": -0.4161442816257477,
       "step": 230
     },
     {
-      "epoch": 0.5,
-      "learning_rate": 1.7460364672965328e-06,
-      "logits/chosen": 0.06422718614339828,
-      "logits/rejected": 0.0699523538351059,
-      "logps/chosen": -472.2220153808594,
-      "logps/rejected": -553.8200073242188,
-      "loss": 0.5098,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.5787314176559448,
-      "rewards/margins": 0.7840873003005981,
-      "rewards/rejected": -2.362818717956543,
+      "epoch": 0.67,
+      "learning_rate": 2.878234954603167e-07,
+      "logits/chosen": -0.18873067200183868,
+      "logits/rejected": -0.10890357196331024,
+      "logps/chosen": -367.36383056640625,
+      "logps/rejected": -254.75003051757812,
+      "loss": 0.4096,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.1997281014919281,
+      "rewards/margins": 0.2125912606716156,
+      "rewards/rejected": -0.4123193621635437,
       "step": 240
     },
     {
-      "epoch": 0.52,
-      "learning_rate": 1.637115696063402e-06,
-      "logits/chosen": 0.09498562663793564,
-      "logits/rejected": 0.23446598649024963,
-      "logps/chosen": -436.4678649902344,
-      "logps/rejected": -487.33929443359375,
-      "loss": 0.5396,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -1.1849793195724487,
-      "rewards/margins": 0.8154090642929077,
-      "rewards/rejected": -2.0003883838653564,
+      "epoch": 0.7,
+      "learning_rate": 2.443358245691555e-07,
+      "logits/chosen": -0.20429889857769012,
+      "logits/rejected": -0.15454210340976715,
+      "logps/chosen": -369.3761291503906,
+      "logps/rejected": -248.98831176757812,
+      "loss": 0.4184,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.16036532819271088,
+      "rewards/margins": 0.31407758593559265,
+      "rewards/rejected": -0.4744429588317871,
       "step": 250
     },
     {
-      "epoch": 0.54,
-      "learning_rate": 1.5274599402265162e-06,
-      "logits/chosen": 0.08814150094985962,
-      "logits/rejected": 0.16783395409584045,
-      "logps/chosen": -431.7601623535156,
-      "logps/rejected": -454.37799072265625,
-      "loss": 0.5208,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.176255464553833,
-      "rewards/margins": 0.7015119791030884,
-      "rewards/rejected": -1.877767562866211,
+      "epoch": 0.73,
+      "learning_rate": 2.0332578804662782e-07,
+      "logits/chosen": -0.21500757336616516,
+      "logits/rejected": -0.16170722246170044,
+      "logps/chosen": -355.4232482910156,
+      "logps/rejected": -257.4869689941406,
+      "loss": 0.4354,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.22508692741394043,
+      "rewards/margins": 0.26084834337234497,
+      "rewards/rejected": -0.4859352707862854,
       "step": 260
     },
     {
-      "epoch": 0.57,
-      "learning_rate": 1.4176569902035088e-06,
-      "logits/chosen": 0.013920878991484642,
-      "logits/rejected": -0.04539678990840912,
-      "logps/chosen": -459.3199157714844,
-      "logps/rejected": -532.9890747070312,
-      "loss": 0.5266,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.3778207302093506,
-      "rewards/margins": 0.8610237240791321,
-      "rewards/rejected": -2.238844633102417,
+      "epoch": 0.76,
+      "learning_rate": 1.651908130088947e-07,
+      "logits/chosen": -0.1587514579296112,
+      "logits/rejected": -0.14712968468666077,
+      "logps/chosen": -329.80462646484375,
+      "logps/rejected": -258.6466369628906,
+      "loss": 0.4171,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.31652820110321045,
+      "rewards/margins": 0.23292319476604462,
+      "rewards/rejected": -0.5494514107704163,
       "step": 270
     },
     {
-      "epoch": 0.59,
-      "learning_rate": 1.308295425420593e-06,
-      "logits/chosen": 0.09688195586204529,
-      "logits/rejected": 0.10716281831264496,
-      "logps/chosen": -416.7201232910156,
-      "logps/rejected": -503.9598083496094,
-      "loss": 0.5308,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.1698511838912964,
-      "rewards/margins": 0.8221219778060913,
-      "rewards/rejected": -1.9919731616973877,
+      "epoch": 0.79,
+      "learning_rate": 1.3030046443173442e-07,
+      "logits/chosen": -0.14641807973384857,
+      "logits/rejected": -0.10507597029209137,
+      "logps/chosen": -379.27020263671875,
+      "logps/rejected": -258.7084655761719,
+      "loss": 0.4366,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.2606154978275299,
+      "rewards/margins": 0.2980353832244873,
+      "rewards/rejected": -0.5586508512496948,
       "step": 280
     },
     {
-      "epoch": 0.61,
-      "learning_rate": 1.1999614593359337e-06,
-      "logits/chosen": 0.04211825877428055,
-      "logits/rejected": 0.12247484922409058,
-      "logps/chosen": -450.50439453125,
-      "logps/rejected": -509.80059814453125,
-      "loss": 0.4921,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.9760421514511108,
-      "rewards/margins": 0.8807096481323242,
-      "rewards/rejected": -1.856751799583435,
+      "epoch": 0.81,
+      "learning_rate": 9.899286370670574e-08,
+      "logits/chosen": -0.1658031940460205,
+      "logits/rejected": -0.0985722690820694,
+      "logps/chosen": -343.7666320800781,
+      "logps/rejected": -271.5211181640625,
+      "loss": 0.4418,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.3174339830875397,
+      "rewards/margins": 0.17954358458518982,
+      "rewards/rejected": -0.49697762727737427,
       "step": 290
     },
     {
-      "epoch": 0.63,
-      "learning_rate": 1.0932357971453745e-06,
-      "logits/chosen": 0.1820950210094452,
-      "logits/rejected": 0.16093513369560242,
-      "logps/chosen": -430.36566162109375,
-      "logps/rejected": -520.7562866210938,
-      "loss": 0.528,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.4484156370162964,
-      "rewards/margins": 0.7810145020484924,
-      "rewards/rejected": -2.2294304370880127,
-      "step": 300
-    },
-    {
-      "epoch": 0.63,
-      "eval_logits/chosen": 0.12960398197174072,
-      "eval_logits/rejected": 0.10445590317249298,
-      "eval_logps/chosen": -451.38214111328125,
-      "eval_logps/rejected": -573.0014038085938,
-      "eval_loss": 0.5161482691764832,
-      "eval_rewards/accuracies": 0.7578125,
-      "eval_rewards/chosen": -1.4779750108718872,
-      "eval_rewards/margins": 1.135788083076477,
-      "eval_rewards/rejected": -2.6137630939483643,
-      "eval_runtime": 65.1979,
-      "eval_samples_per_second": 30.676,
-      "eval_steps_per_second": 0.491,
+      "epoch": 0.84,
+      "learning_rate": 7.157141191620548e-08,
+      "logits/chosen": -0.2034151554107666,
+      "logits/rejected": -0.10025110095739365,
+      "logps/chosen": -365.5801086425781,
+      "logps/rejected": -251.5866241455078,
+      "loss": 0.4302,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.19396531581878662,
+      "rewards/margins": 0.27736470103263855,
+      "rewards/rejected": -0.47133007645606995,
       "step": 300
     },
     {
-      "epoch": 0.65,
-      "learning_rate": 9.886905230142433e-07,
-      "logits/chosen": -0.021408915519714355,
-      "logits/rejected": -0.032668907195329666,
-      "logps/chosen": -442.490478515625,
-      "logps/rejected": -543.8944091796875,
-      "loss": 0.4964,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.465606927871704,
-      "rewards/margins": 1.0980336666107178,
-      "rewards/rejected": -2.563640594482422,
+      "epoch": 0.87,
+      "learning_rate": 4.830184958207006e-08,
+      "logits/chosen": -0.20084922015666962,
+      "logits/rejected": -0.11563346534967422,
+      "logps/chosen": -336.1097106933594,
+      "logps/rejected": -272.353515625,
+      "loss": 0.4291,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.24448652565479279,
+      "rewards/margins": 0.242269366979599,
+      "rewards/rejected": -0.48675593733787537,
       "step": 310
     },
     {
-      "epoch": 0.67,
-      "learning_rate": 8.868860335206678e-07,
-      "logits/chosen": -0.20789632201194763,
-      "logits/rejected": -0.17585983872413635,
-      "logps/chosen": -477.2587890625,
-      "logps/rejected": -549.9071044921875,
-      "loss": 0.5025,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.500857949256897,
-      "rewards/margins": 0.9439705014228821,
-      "rewards/rejected": -2.4448282718658447,
+      "epoch": 0.9,
+      "learning_rate": 2.940968138161731e-08,
+      "logits/chosen": -0.13686858117580414,
+      "logits/rejected": -0.11668189615011215,
+      "logps/chosen": -317.7008361816406,
+      "logps/rejected": -261.1730041503906,
+      "loss": 0.4301,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.2534436285495758,
+      "rewards/margins": 0.16237813234329224,
+      "rewards/rejected": -0.41582173109054565,
       "step": 320
     },
     {
-      "epoch": 0.69,
-      "learning_rate": 7.883680337481599e-07,
-      "logits/chosen": -0.021555980667471886,
-      "logits/rejected": -0.07717858999967575,
-      "logps/chosen": -488.15985107421875,
-      "logps/rejected": -586.7195434570312,
-      "loss": 0.5155,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.5513451099395752,
-      "rewards/margins": 0.9236253499984741,
-      "rewards/rejected": -2.4749703407287598,
+      "epoch": 0.93,
+      "learning_rate": 1.507799078812799e-08,
+      "logits/chosen": -0.23092389106750488,
+      "logits/rejected": -0.16825535893440247,
+      "logps/chosen": -412.626708984375,
+      "logps/rejected": -301.6609802246094,
+      "loss": 0.4204,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.27390944957733154,
+      "rewards/margins": 0.2617945373058319,
+      "rewards/rejected": -0.5357040166854858,
       "step": 330
     },
-    {
-      "epoch": 0.71,
-      "learning_rate": 6.936646121293654e-07,
-      "logits/chosen": -0.0836385041475296,
-      "logits/rejected": -0.039361923933029175,
-      "logps/chosen": -532.11181640625,
-      "logps/rejected": -552.9940185546875,
-      "loss": 0.5255,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.5922324657440186,
-      "rewards/margins": 0.7127285003662109,
-      "rewards/rejected": -2.3049609661102295,
-      "step": 340
-    },
-    {
-      "epoch": 0.73,
-      "learning_rate": 6.032834097207889e-07,
-      "logits/chosen": 0.012007070705294609,
-      "logits/rejected": 0.023168018087744713,
-      "logps/chosen": -392.9766845703125,
-      "logps/rejected": -494.611328125,
-      "loss": 0.5176,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.1996792554855347,
-      "rewards/margins": 0.8586348295211792,
-      "rewards/rejected": -2.058314085006714,
-      "step": 350
-    },
-    {
-      "epoch": 0.75,
-      "learning_rate": 5.177088990820725e-07,
-      "logits/chosen": -0.050304025411605835,
-      "logits/rejected": -0.019231608137488365,
-      "logps/chosen": -416.4642639160156,
-      "logps/rejected": -483.6036071777344,
-      "loss": 0.5229,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.2121403217315674,
-      "rewards/margins": 0.8643258810043335,
-      "rewards/rejected": -2.0764665603637695,
-      "step": 360
-    },
-    {
-      "epoch": 0.77,
-      "learning_rate": 4.3739978734594494e-07,
-      "logits/chosen": -0.08456006646156311,
-      "logits/rejected": -0.12612244486808777,
-      "logps/chosen": -400.55206298828125,
-      "logps/rejected": -503.98419189453125,
-      "loss": 0.491,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.2748231887817383,
-      "rewards/margins": 0.9540689587593079,
-      "rewards/rejected": -2.2288920879364014,
-      "step": 370
-    },
-    {
-      "epoch": 0.8,
-      "learning_rate": 3.627865573992087e-07,
-      "logits/chosen": -0.15401089191436768,
-      "logits/rejected": -0.20363488793373108,
-      "logps/chosen": -478.338623046875,
-      "logps/rejected": -561.3379516601562,
-      "loss": 0.5324,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.4220519065856934,
-      "rewards/margins": 1.054446816444397,
-      "rewards/rejected": -2.4764983654022217,
-      "step": 380
-    },
-    {
-      "epoch": 0.82,
-      "learning_rate": 2.9426916035484166e-07,
-      "logits/chosen": -0.21970228850841522,
-      "logits/rejected": -0.12131069600582123,
-      "logps/chosen": -452.88702392578125,
-      "logps/rejected": -512.7882690429688,
-      "loss": 0.4985,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.302075743675232,
-      "rewards/margins": 0.9814373254776001,
-      "rewards/rejected": -2.2835135459899902,
-      "step": 390
-    },
-    {
-      "epoch": 0.84,
-      "learning_rate": 2.322148716843081e-07,
-      "logits/chosen": -0.10807213932275772,
-      "logits/rejected": -0.1505809724330902,
-      "logps/chosen": -412.93658447265625,
-      "logps/rejected": -500.96142578125,
-      "loss": 0.5049,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.183068037033081,
-      "rewards/margins": 0.9983505010604858,
-      "rewards/rejected": -2.1814186573028564,
-      "step": 400
-    },
-    {
-      "epoch": 0.84,
-      "eval_logits/chosen": -0.08459039032459259,
-      "eval_logits/rejected": -0.12737514078617096,
-      "eval_logps/chosen": -435.32611083984375,
-      "eval_logps/rejected": -556.4354858398438,
-      "eval_loss": 0.5054404735565186,
-      "eval_rewards/accuracies": 0.77734375,
-      "eval_rewards/chosen": -1.3174140453338623,
-      "eval_rewards/margins": 1.1306906938552856,
-      "eval_rewards/rejected": -2.4481046199798584,
-      "eval_runtime": 63.6745,
-      "eval_samples_per_second": 31.41,
-      "eval_steps_per_second": 0.503,
-      "step": 400
-    },
-    {
-      "epoch": 0.86,
-      "learning_rate": 1.7695632250191002e-07,
-      "logits/chosen": -0.10287277400493622,
-      "logits/rejected": -0.09486471116542816,
-      "logps/chosen": -460.506103515625,
-      "logps/rejected": -528.74365234375,
-      "loss": 0.5085,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.3777108192443848,
-      "rewards/margins": 0.9177919626235962,
-      "rewards/rejected": -2.2955029010772705,
-      "step": 410
-    },
-    {
-      "epoch": 0.88,
-      "learning_rate": 1.2878971655412515e-07,
-      "logits/chosen": -0.18269723653793335,
-      "logits/rejected": -0.24112820625305176,
-      "logps/chosen": -422.0091857910156,
-      "logps/rejected": -485.4613342285156,
-      "loss": 0.5023,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.350635290145874,
-      "rewards/margins": 0.7604650259017944,
-      "rewards/rejected": -2.1110999584198,
-      "step": 420
-    },
-    {
-      "epoch": 0.9,
-      "learning_rate": 8.797324247145411e-08,
-      "logits/chosen": -0.24929451942443848,
-      "logits/rejected": -0.2101927250623703,
-      "logps/chosen": -461.18609619140625,
-      "logps/rejected": -513.12841796875,
-      "loss": 0.498,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.363139271736145,
-      "rewards/margins": 0.9195320010185242,
-      "rewards/rejected": -2.2826714515686035,
-      "step": 430
-    },
-    {
-      "epoch": 0.92,
-      "learning_rate": 5.472568979361853e-08,
-      "logits/chosen": -0.2508998513221741,
-      "logits/rejected": -0.23314960300922394,
-      "logps/chosen": -452.8556213378906,
-      "logps/rejected": -510.5809020996094,
-      "loss": 0.5278,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.4022619724273682,
-      "rewards/margins": 0.9772977828979492,
-      "rewards/rejected": -2.3795602321624756,
-      "step": 440
-    },
-    {
-      "epoch": 0.94,
-      "learning_rate": 2.922527618666465e-08,
-      "logits/chosen": -0.3062313199043274,
-      "logits/rejected": -0.20936842262744904,
-      "logps/chosen": -482.806884765625,
-      "logps/rejected": -512.525390625,
-      "loss": 0.5118,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.379194736480713,
-      "rewards/margins": 0.6993511915206909,
-      "rewards/rejected": -2.0785460472106934,
-      "step": 450
-    },
     {
       "epoch": 0.96,
-      "learning_rate": 1.1608692138469379e-08,
-      "logits/chosen": -0.1790022850036621,
-      "logits/rejected": -0.2542126774787903,
-      "logps/chosen": -446.405517578125,
-      "logps/rejected": -562.390869140625,
-      "loss": 0.5164,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.2966983318328857,
-      "rewards/margins": 1.0804041624069214,
-      "rewards/rejected": -2.3771026134490967,
-      "step": 460
+      "learning_rate": 5.445665814031941e-09,
+      "logits/chosen": -0.1695241779088974,
+      "logits/rejected": -0.12149496376514435,
+      "logps/chosen": -358.09063720703125,
+      "logps/rejected": -265.50189208984375,
+      "loss": 0.4321,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.2663304805755615,
+      "rewards/margins": 0.24073641002178192,
+      "rewards/rejected": -0.5070669054985046,
+      "step": 340
     },
     {
       "epoch": 0.98,
-      "learning_rate": 1.970368253390198e-09,
-      "logits/chosen": -0.26300907135009766,
-      "logits/rejected": -0.3493719696998596,
-      "logps/chosen": -444.3451232910156,
-      "logps/rejected": -539.7431030273438,
-      "loss": 0.5011,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.3199527263641357,
-      "rewards/margins": 1.0216041803359985,
-      "rewards/rejected": -2.341557025909424,
-      "step": 470
+      "learning_rate": 6.060530510659245e-10,
+      "logits/chosen": -0.2175191193819046,
+      "logits/rejected": -0.21960613131523132,
+      "logps/chosen": -345.24298095703125,
+      "logps/rejected": -263.9671630859375,
+      "loss": 0.4306,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2784077525138855,
+      "rewards/margins": 0.24674105644226074,
+      "rewards/rejected": -0.5251488089561462,
+      "step": 350
     },
     {
       "epoch": 1.0,
-      "step": 477,
+      "step": 355,
       "total_flos": 0.0,
-      "train_loss": 0.5426262839535247,
-      "train_runtime": 4328.2164,
-      "train_samples_per_second": 14.125,
-      "train_steps_per_second": 0.11
+      "train_loss": 0.464236611379704,
+      "train_runtime": 5271.2295,
+      "train_samples_per_second": 8.641,
+      "train_steps_per_second": 0.067
     }
   ],
   "logging_steps": 10,
-  "max_steps": 477,
+  "max_steps": 355,
   "num_train_epochs": 1,
-  "save_steps": 1000,
+  "save_steps": 10000,
   "total_flos": 0.0,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5a91613e24ef836988f352fb159b97a4baf24844c507e3a389c701dcc985b914
+oid sha256:132930e5c6429d808850916e7587770d7157ebba376ed0e45170c2bd96c2061b
 size 6648