Model save

README.md

@@ -14,16 +14,6 @@ should probably proofread and complete it, then remove this comment. -->
 # zephyr-7b-dpo-full
 
 This model was trained from scratch on the None dataset.
-It achieves the following results on the evaluation set:
-- Loss: 0.0415
-- Rewards/chosen: -1.1176
-- Rewards/rejected: -2.0114
-- Rewards/accuracies: 0.7070
-- Rewards/margins: 0.8938
-- Logps/rejected: -531.2747
-- Logps/chosen: -435.5875
-- Logits/rejected: 0.8196
-- Logits/chosen: 0.7291
 
 ## Model description
 
@@ -43,12 +33,12 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 3e-06
-- train_batch_size: 4
+- train_batch_size: 2
 - eval_batch_size: 8
-- seed: 5
+- seed: 1
 - distributed_type: multi-GPU
 - num_devices: 8
-- gradient_accumulation_steps: 4
+- gradient_accumulation_steps: 8
 - total_train_batch_size: 128
 - total_eval_batch_size: 64
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
@@ -58,12 +48,6 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
-|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.066         | 0.21  | 100  | 0.0702          | -0.4714        | -1.0800          | 0.7266             | 0.6086          | -438.1371      | -370.9747    | 0.7687          | 0.6183        |
-| 0.0477        | 0.42  | 200  | 0.0505          | -1.0382        | -1.8566          | 0.7461             | 0.8184          | -515.7967      | -427.6501    | 0.5198          | 0.4181        |
-| 0.0313        | 0.63  | 300  | 0.0344          | -1.3029        | -2.2224          | 0.7227             | 0.9195          | -552.3698      | -454.1193    | 1.0434          | 0.9401        |
-| 0.0359        | 0.84  | 400  | 0.0415          | -1.1176        | -2.0114          | 0.7070             | 0.8938          | -531.2747      | -435.5875    | 0.8196          | 0.7291        |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.055112330793584664,
-    "train_runtime": 4571.3444,
-    "train_samples": 61134,
-    "train_samples_per_second": 13.373,
-    "train_steps_per_second": 0.104
+    "train_loss": 0.3547657697973117,
+    "train_runtime": 5270.9361,
+    "train_samples": 45548,
+    "train_samples_per_second": 8.641,
+    "train_steps_per_second": 0.067
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:77c78f44ae927b8c5f876cba766716862c391ff327d777f630df2273dc608ad2
+oid sha256:85e0dec32f242e5185356c9aabdb63b6361f0e76923db502922b96ae33954e21
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:52890ec8e3b01c2a425c75a5fe8026fad3760550ffe4ecc542adabcb6547e556
+oid sha256:4c71f08ad1d050aa51128e168230fbf7b08a12a7469048f453be868dc4b011c1
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c4e28b526b64115f67f1a7d9ceb1156546b14ddfbf6c799c751ac2c949af93b
+oid sha256:57e7b9d8250962948a9f4a596ab9c8e0fba7ec3e5cbb4089122bb5bb2d64378b
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:331daeef21c9b60a293872df524529661446efaf2f056cc336b124cce438e3cb
+oid sha256:213a31aa46d216f4a6147988c22f9d730650373e142b430f375ebcf8f54ab823
 size 1168138808

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.055112330793584664,
-    "train_runtime": 4571.3444,
-    "train_samples": 61134,
-    "train_samples_per_second": 13.373,
-    "train_steps_per_second": 0.104
+    "train_loss": 0.3547657697973117,
+    "train_runtime": 5270.9361,
+    "train_samples": 45548,
+    "train_samples_per_second": 8.641,
+    "train_steps_per_second": 0.067
 }

trainer_state.json

@@ -1,763 +1,517 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9984301412872841,
-  "eval_steps": 100,
-  "global_step": 477,
+  "epoch": 0.9975412715138743,
+  "eval_steps": 10000,
+  "global_step": 355,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0,
-      "learning_rate": 6.25e-08,
-      "logits/chosen": 0.10802720487117767,
-      "logits/rejected": 0.30745893716812134,
-      "logps/chosen": -475.5745544433594,
-      "logps/rejected": -317.21234130859375,
-      "loss": 0.1378,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": 0.0,
-      "rewards/margins": 0.0,
-      "rewards/rejected": 0.0,
-      "step": 1
-    },
-    {
-      "epoch": 0.02,
-      "learning_rate": 6.25e-07,
-      "logits/chosen": 0.21480141580104828,
-      "logits/rejected": 0.3137889802455902,
-      "logps/chosen": -308.09619140625,
-      "logps/rejected": -308.85736083984375,
-      "loss": 0.1432,
-      "rewards/accuracies": 0.4791666567325592,
-      "rewards/chosen": -0.0008134886738844216,
-      "rewards/margins": 0.0006454013055190444,
-      "rewards/rejected": -0.001458889921195805,
+      "epoch": 0.03,
+      "learning_rate": 8.333333333333334e-07,
+      "logits/chosen": -0.08247309923171997,
+      "logits/rejected": -0.0386468842625618,
+      "logps/chosen": -327.3994140625,
+      "logps/rejected": -244.6085968017578,
+      "loss": 0.5077,
+      "rewards/accuracies": 0.3812499940395355,
+      "rewards/chosen": -0.002138084964826703,
+      "rewards/margins": -0.0002509051118977368,
+      "rewards/rejected": -0.0018871795618906617,
       "step": 10
     },
     {
-      "epoch": 0.04,
-      "learning_rate": 1.25e-06,
-      "logits/chosen": 0.249754399061203,
-      "logits/rejected": 0.2825905978679657,
-      "logps/chosen": -304.75286865234375,
-      "logps/rejected": -317.61688232421875,
-      "loss": 0.1418,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.0010095896432176232,
-      "rewards/margins": 0.010475357994437218,
-      "rewards/rejected": -0.009465768001973629,
+      "epoch": 0.06,
+      "learning_rate": 1.6666666666666669e-06,
+      "logits/chosen": -0.045555226504802704,
+      "logits/rejected": -0.013953140005469322,
+      "logps/chosen": -293.5519104003906,
+      "logps/rejected": -201.40576171875,
+      "loss": 0.5253,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.021771755069494247,
+      "rewards/margins": 0.0479663722217083,
+      "rewards/rejected": -0.06973812729120255,
       "step": 20
     },
     {
-      "epoch": 0.06,
-      "learning_rate": 1.875e-06,
-      "logits/chosen": 0.24968624114990234,
-      "logits/rejected": 0.2685222029685974,
-      "logps/chosen": -366.27813720703125,
-      "logps/rejected": -365.3521728515625,
-      "loss": 0.1431,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.014242827892303467,
-      "rewards/margins": 0.06069143861532211,
-      "rewards/rejected": -0.046448610723018646,
+      "epoch": 0.08,
+      "learning_rate": 2.5e-06,
+      "logits/chosen": -0.0032004565000534058,
+      "logits/rejected": 0.0453818179666996,
+      "logps/chosen": -361.3478088378906,
+      "logps/rejected": -237.36703491210938,
+      "loss": 0.5488,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.12262026220560074,
+      "rewards/margins": 0.2048310935497284,
+      "rewards/rejected": -0.32745134830474854,
       "step": 30
     },
     {
-      "epoch": 0.08,
-      "learning_rate": 2.5e-06,
-      "logits/chosen": 0.5138859748840332,
-      "logits/rejected": 0.6031057238578796,
-      "logps/chosen": -333.85650634765625,
-      "logps/rejected": -331.0009765625,
-      "loss": 0.1181,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.17486190795898438,
-      "rewards/margins": 0.1082921177148819,
-      "rewards/rejected": -0.2831540107727051,
+      "epoch": 0.11,
+      "learning_rate": 2.9988362934929793e-06,
+      "logits/chosen": -0.09940309822559357,
+      "logits/rejected": -0.05378924682736397,
+      "logps/chosen": -314.38812255859375,
+      "logps/rejected": -247.3548583984375,
+      "loss": 0.5321,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.10980594158172607,
+      "rewards/margins": 0.13736829161643982,
+      "rewards/rejected": -0.24717426300048828,
       "step": 40
     },
     {
-      "epoch": 0.1,
-      "learning_rate": 2.999839121261416e-06,
-      "logits/chosen": 0.7348484992980957,
-      "logits/rejected": 0.8855365514755249,
-      "logps/chosen": -370.4933776855469,
-      "logps/rejected": -411.83404541015625,
-      "loss": 0.0741,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.4408305287361145,
-      "rewards/margins": 0.4697234034538269,
-      "rewards/rejected": -0.9105539321899414,
+      "epoch": 0.14,
+      "learning_rate": 2.985765322825759e-06,
+      "logits/chosen": -0.1258987933397293,
+      "logits/rejected": -0.09743531048297882,
+      "logps/chosen": -320.26220703125,
+      "logps/rejected": -259.95989990234375,
+      "loss": 0.484,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.047640036791563034,
+      "rewards/margins": 0.12707160413265228,
+      "rewards/rejected": -0.17471164464950562,
       "step": 50
     },
     {
-      "epoch": 0.13,
-      "learning_rate": 2.994211988057582e-06,
-      "logits/chosen": 0.7168207764625549,
-      "logits/rejected": 0.8200086355209351,
-      "logps/chosen": -341.53277587890625,
-      "logps/rejected": -380.68243408203125,
-      "loss": 0.0819,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.4272558093070984,
-      "rewards/margins": 0.4549214839935303,
-      "rewards/rejected": -0.8821773529052734,
+      "epoch": 0.17,
+      "learning_rate": 2.9582958419982717e-06,
+      "logits/chosen": -0.20755784213542938,
+      "logits/rejected": -0.14810998737812042,
+      "logps/chosen": -397.5509948730469,
+      "logps/rejected": -250.44027709960938,
+      "loss": 0.463,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.10637468099594116,
+      "rewards/margins": 0.37280240654945374,
+      "rewards/rejected": -0.4791770875453949,
       "step": 60
     },
     {
-      "epoch": 0.15,
-      "learning_rate": 2.9805753939568693e-06,
-      "logits/chosen": 0.5615164041519165,
-      "logits/rejected": 0.7741672396659851,
-      "logps/chosen": -347.7218017578125,
-      "logps/rejected": -330.172607421875,
-      "loss": 0.0929,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.3559855818748474,
-      "rewards/margins": 0.2854944169521332,
-      "rewards/rejected": -0.6414799690246582,
+      "epoch": 0.2,
+      "learning_rate": 2.916694056980408e-06,
+      "logits/chosen": -0.19774258136749268,
+      "logits/rejected": -0.12501199543476105,
+      "logps/chosen": -388.07427978515625,
+      "logps/rejected": -266.5827331542969,
+      "loss": 0.4471,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.19550617039203644,
+      "rewards/margins": 0.24603180587291718,
+      "rewards/rejected": -0.4415379464626312,
       "step": 70
     },
     {
-      "epoch": 0.17,
-      "learning_rate": 2.959002435526626e-06,
-      "logits/chosen": 0.5198915004730225,
-      "logits/rejected": 0.725387454032898,
-      "logps/chosen": -389.0698547363281,
-      "logps/rejected": -371.3795471191406,
-      "loss": 0.0736,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.4846402108669281,
-      "rewards/margins": 0.395100474357605,
-      "rewards/rejected": -0.8797407150268555,
+      "epoch": 0.22,
+      "learning_rate": 2.8613631295064357e-06,
+      "logits/chosen": -0.168908953666687,
+      "logits/rejected": -0.16765542328357697,
+      "logps/chosen": -288.5673828125,
+      "logps/rejected": -233.2335968017578,
+      "loss": 0.4418,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.2916755676269531,
+      "rewards/margins": 0.2403053343296051,
+      "rewards/rejected": -0.5319808721542358,
       "step": 80
     },
     {
-      "epoch": 0.19,
-      "learning_rate": 2.929608750821129e-06,
-      "logits/chosen": 0.3736918568611145,
-      "logits/rejected": 0.5658319592475891,
-      "logps/chosen": -444.59234619140625,
-      "logps/rejected": -464.6935119628906,
-      "loss": 0.0491,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.8689848184585571,
-      "rewards/margins": 0.6035453081130981,
-      "rewards/rejected": -1.4725301265716553,
+      "epoch": 0.25,
+      "learning_rate": 2.792839270045916e-06,
+      "logits/chosen": -0.3168974816799164,
+      "logits/rejected": -0.27101725339889526,
+      "logps/chosen": -395.9394226074219,
+      "logps/rejected": -256.7802734375,
+      "loss": 0.4071,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.26048293709754944,
+      "rewards/margins": 0.4022350311279297,
+      "rewards/rejected": -0.6627179980278015,
       "step": 90
     },
     {
-      "epoch": 0.21,
-      "learning_rate": 2.892551899524109e-06,
-      "logits/chosen": 0.3380030393600464,
-      "logits/rejected": 0.443446546792984,
-      "logps/chosen": -408.71551513671875,
-      "logps/rejected": -431.513671875,
-      "loss": 0.066,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.8214343786239624,
-      "rewards/margins": 0.3987075388431549,
-      "rewards/rejected": -1.2201420068740845,
-      "step": 100
-    },
-    {
-      "epoch": 0.21,
-      "eval_logits/chosen": 0.6183323860168457,
-      "eval_logits/rejected": 0.7686768174171448,
-      "eval_logps/chosen": -370.9747009277344,
-      "eval_logps/rejected": -438.13714599609375,
-      "eval_loss": 0.07016688585281372,
-      "eval_rewards/accuracies": 0.7265625,
-      "eval_rewards/chosen": -0.47144782543182373,
-      "eval_rewards/margins": 0.6085766553878784,
-      "eval_rewards/rejected": -1.0800243616104126,
-      "eval_runtime": 74.3034,
-      "eval_samples_per_second": 26.917,
-      "eval_steps_per_second": 0.431,
+      "epoch": 0.28,
+      "learning_rate": 2.711786541403051e-06,
+      "logits/chosen": -0.33102917671203613,
+      "logits/rejected": -0.338579922914505,
+      "logps/chosen": -387.654296875,
+      "logps/rejected": -305.7378845214844,
+      "loss": 0.379,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.4444798529148102,
+      "rewards/margins": 0.19619156420230865,
+      "rewards/rejected": -0.6406713724136353,
       "step": 100
     },
     {
-      "epoch": 0.23,
-      "learning_rate": 2.848030518377739e-06,
-      "logits/chosen": 0.48754867911338806,
-      "logits/rejected": 0.6056569814682007,
-      "logps/chosen": -394.04449462890625,
-      "logps/rejected": -424.449951171875,
-      "loss": 0.06,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.6658821105957031,
-      "rewards/margins": 0.43674975633621216,
-      "rewards/rejected": -1.1026318073272705,
+      "epoch": 0.31,
+      "learning_rate": 2.6189904233026363e-06,
+      "logits/chosen": -0.4311809539794922,
+      "logits/rejected": -0.3962547183036804,
+      "logps/chosen": -351.5871887207031,
+      "logps/rejected": -305.3436584472656,
+      "loss": 0.3607,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.5928763151168823,
+      "rewards/margins": 0.10657763481140137,
+      "rewards/rejected": -0.6994539499282837,
       "step": 110
     },
     {
-      "epoch": 0.25,
-      "learning_rate": 2.7962832564252724e-06,
-      "logits/chosen": 0.5436107516288757,
-      "logits/rejected": 0.6737319231033325,
-      "logps/chosen": -429.415283203125,
-      "logps/rejected": -469.0088806152344,
-      "loss": 0.0627,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.7700729370117188,
-      "rewards/margins": 0.48356789350509644,
-      "rewards/rejected": -1.2536407709121704,
+      "epoch": 0.34,
+      "learning_rate": 2.515350200328027e-06,
+      "logits/chosen": -0.4205436706542969,
+      "logits/rejected": -0.40109872817993164,
+      "logps/chosen": -366.4017028808594,
+      "logps/rejected": -258.0574951171875,
+      "loss": 0.3644,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.48554643988609314,
+      "rewards/margins": 0.24580618739128113,
+      "rewards/rejected": -0.7313526272773743,
       "step": 120
     },
     {
-      "epoch": 0.27,
-      "learning_rate": 2.7375874957747644e-06,
-      "logits/chosen": 0.5728715062141418,
-      "logits/rejected": 0.7463508248329163,
-      "logps/chosen": -441.0868225097656,
-      "logps/rejected": -454.98748779296875,
-      "loss": 0.0621,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.8495699763298035,
-      "rewards/margins": 0.5289269685745239,
-      "rewards/rejected": -1.3784968852996826,
+      "epoch": 0.37,
+      "learning_rate": 2.401870246979413e-06,
+      "logits/chosen": -0.36771100759506226,
+      "logits/rejected": -0.31752774119377136,
+      "logps/chosen": -408.0353088378906,
+      "logps/rejected": -279.37872314453125,
+      "loss": 0.3791,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.5797132849693298,
+      "rewards/margins": 0.35475510358810425,
+      "rewards/rejected": -0.9344683885574341,
       "step": 130
     },
     {
-      "epoch": 0.29,
-      "learning_rate": 2.672257864741005e-06,
-      "logits/chosen": 0.6253047585487366,
-      "logits/rejected": 0.786455512046814,
-      "logps/chosen": -433.4244079589844,
-      "logps/rejected": -461.5254821777344,
-      "loss": 0.0435,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.083187460899353,
-      "rewards/margins": 0.5303990840911865,
-      "rewards/rejected": -1.61358642578125,
+      "epoch": 0.39,
+      "learning_rate": 2.279650294308645e-06,
+      "logits/chosen": -0.3124179244041443,
+      "logits/rejected": -0.3357269763946533,
+      "logps/chosen": -386.987548828125,
+      "logps/rejected": -315.2652587890625,
+      "loss": 0.3217,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.880761981010437,
+      "rewards/margins": 0.18241076171398163,
+      "rewards/rejected": -1.063172698020935,
       "step": 140
     },
     {
-      "epoch": 0.31,
-      "learning_rate": 2.600644551335706e-06,
-      "logits/chosen": 0.7765518426895142,
-      "logits/rejected": 0.984174907207489,
-      "logps/chosen": -419.31109619140625,
-      "logps/rejected": -431.96795654296875,
-      "loss": 0.0444,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -1.0778591632843018,
-      "rewards/margins": 0.4318017363548279,
-      "rewards/rejected": -1.5096609592437744,
+      "epoch": 0.42,
+      "learning_rate": 2.1498747724563957e-06,
+      "logits/chosen": -0.237782284617424,
+      "logits/rejected": -0.2272220402956009,
+      "logps/chosen": -347.40618896484375,
+      "logps/rejected": -309.7053527832031,
+      "loss": 0.32,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.775266706943512,
+      "rewards/margins": 0.20919008553028107,
+      "rewards/rejected": -0.9844567179679871,
       "step": 150
     },
     {
-      "epoch": 0.33,
-      "learning_rate": 2.5231314261461732e-06,
-      "logits/chosen": 0.513221025466919,
-      "logits/rejected": 0.7459092140197754,
-      "logps/chosen": -418.07421875,
-      "logps/rejected": -463.25408935546875,
-      "loss": 0.0586,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.6770002245903015,
-      "rewards/margins": 0.5035561323165894,
-      "rewards/rejected": -1.1805565357208252,
+      "epoch": 0.45,
+      "learning_rate": 2.0138013323728074e-06,
+      "logits/chosen": -0.3776048719882965,
+      "logits/rejected": -0.3391488790512085,
+      "logps/chosen": -430.7017517089844,
+      "logps/rejected": -327.8933410644531,
+      "loss": 0.3115,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.8183802366256714,
+      "rewards/margins": 0.30396735668182373,
+      "rewards/rejected": -1.1223475933074951,
       "step": 160
     },
     {
-      "epoch": 0.36,
-      "learning_rate": 2.440133984664454e-06,
-      "logits/chosen": 0.5670315027236938,
-      "logits/rejected": 0.8073333501815796,
-      "logps/chosen": -390.5821228027344,
-      "logps/rejected": -419.92626953125,
-      "loss": 0.0562,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.8130921125411987,
-      "rewards/margins": 0.4765067994594574,
-      "rewards/rejected": -1.289598822593689,
+      "epoch": 0.48,
+      "learning_rate": 1.8727486579573409e-06,
+      "logits/chosen": -0.3819672465324402,
+      "logits/rejected": -0.38335323333740234,
+      "logps/chosen": -376.9703369140625,
+      "logps/rejected": -336.7782287597656,
+      "loss": 0.3038,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.815502941608429,
+      "rewards/margins": 0.2603687345981598,
+      "rewards/rejected": -1.0758715867996216,
       "step": 170
     },
     {
-      "epoch": 0.38,
-      "learning_rate": 2.3520971200967337e-06,
-      "logits/chosen": 0.39020082354545593,
-      "logits/rejected": 0.4927116334438324,
-      "logps/chosen": -379.1041259765625,
-      "logps/rejected": -440.0082092285156,
-      "loss": 0.0533,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.8111687898635864,
-      "rewards/margins": 0.5016359090805054,
-      "rewards/rejected": -1.3128045797348022,
+      "epoch": 0.51,
+      "learning_rate": 1.7280836867300083e-06,
+      "logits/chosen": -0.5107148885726929,
+      "logits/rejected": -0.5026928186416626,
+      "logps/chosen": -388.6782531738281,
+      "logps/rejected": -321.05328369140625,
+      "loss": 0.3147,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.8670403361320496,
+      "rewards/margins": 0.20068030059337616,
+      "rewards/rejected": -1.067720651626587,
       "step": 180
     },
     {
-      "epoch": 0.4,
-      "learning_rate": 2.2594927385914546e-06,
-      "logits/chosen": 0.32924190163612366,
-      "logits/rejected": 0.46087831258773804,
-      "logps/chosen": -382.1633605957031,
-      "logps/rejected": -444.0999450683594,
-      "loss": 0.0495,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.7652384042739868,
-      "rewards/margins": 0.6649683117866516,
-      "rewards/rejected": -1.4302066564559937,
+      "epoch": 0.53,
+      "learning_rate": 1.5812083628781265e-06,
+      "logits/chosen": -0.5208634734153748,
+      "logits/rejected": -0.506948709487915,
+      "logps/chosen": -409.93951416015625,
+      "logps/rejected": -310.84381103515625,
+      "loss": 0.2825,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.8959806561470032,
+      "rewards/margins": 0.3938984274864197,
+      "rewards/rejected": -1.2898790836334229,
       "step": 190
     },
     {
-      "epoch": 0.42,
-      "learning_rate": 2.1628172296692954e-06,
-      "logits/chosen": 0.21413707733154297,
-      "logits/rejected": 0.302509069442749,
-      "logps/chosen": -465.3833923339844,
-      "logps/rejected": -511.8447265625,
-      "loss": 0.0477,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.172499179840088,
-      "rewards/margins": 0.573866069316864,
-      "rewards/rejected": -1.7463653087615967,
+      "epoch": 0.56,
+      "learning_rate": 1.433546051054432e-06,
+      "logits/chosen": -0.5745254158973694,
+      "logits/rejected": -0.5329603552818298,
+      "logps/chosen": -414.594970703125,
+      "logps/rejected": -349.5610656738281,
+      "loss": 0.2849,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2510201930999756,
+      "rewards/margins": 0.1971484124660492,
+      "rewards/rejected": -1.4481686353683472,
       "step": 200
     },
     {
-      "epoch": 0.42,
-      "eval_logits/chosen": 0.41806796193122864,
-      "eval_logits/rejected": 0.5197638273239136,
-      "eval_logps/chosen": -427.650146484375,
-      "eval_logps/rejected": -515.7966918945312,
-      "eval_loss": 0.050458863377571106,
-      "eval_rewards/accuracies": 0.74609375,
-      "eval_rewards/chosen": -1.038202166557312,
-      "eval_rewards/margins": 0.8184179663658142,
-      "eval_rewards/rejected": -1.856619954109192,
-      "eval_runtime": 75.1858,
-      "eval_samples_per_second": 26.601,
-      "eval_steps_per_second": 0.426,
-      "step": 200
-    },
-    {
-      "epoch": 0.44,
-      "learning_rate": 2.062588805414343e-06,
-      "logits/chosen": 0.29592061042785645,
-      "logits/rejected": 0.39124542474746704,
-      "logps/chosen": -458.99554443359375,
-      "logps/rejected": -476.7998046875,
-      "loss": 0.0543,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.0953991413116455,
-      "rewards/margins": 0.6356866955757141,
-      "rewards/rejected": -1.731086015701294,
+      "epoch": 0.59,
+      "learning_rate": 1.2865277425900725e-06,
+      "logits/chosen": -0.4251164495944977,
+      "logits/rejected": -0.4111138880252838,
+      "logps/chosen": -446.27685546875,
+      "logps/rejected": -339.98321533203125,
+      "loss": 0.2914,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.8489357829093933,
+      "rewards/margins": 0.3122832179069519,
+      "rewards/rejected": -1.1612190008163452,
       "step": 210
     },
     {
-      "epoch": 0.46,
-      "learning_rate": 1.9593447226892386e-06,
-      "logits/chosen": 0.23310557007789612,
-      "logits/rejected": 0.4742186963558197,
-      "logps/chosen": -441.21649169921875,
-      "logps/rejected": -468.25286865234375,
-      "loss": 0.0599,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.9769255518913269,
-      "rewards/margins": 0.7468104362487793,
-      "rewards/rejected": -1.723736047744751,
+      "epoch": 0.62,
+      "learning_rate": 1.141578187797663e-06,
+      "logits/chosen": -0.5450788736343384,
+      "logits/rejected": -0.5059890151023865,
+      "logps/chosen": -417.80908203125,
+      "logps/rejected": -308.05596923828125,
+      "loss": 0.2944,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.8429006338119507,
+      "rewards/margins": 0.24668729305267334,
+      "rewards/rejected": -1.089587926864624,
       "step": 220
     },
     {
-      "epoch": 0.48,
-      "learning_rate": 1.853638403264141e-06,
-      "logits/chosen": 0.4100280702114105,
-      "logits/rejected": 0.5993035435676575,
-      "logps/chosen": -494.64324951171875,
-      "logps/rejected": -490.0165100097656,
-      "loss": 0.0578,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.2230786085128784,
-      "rewards/margins": 0.6530172824859619,
-      "rewards/rejected": -1.8760957717895508,
+      "epoch": 0.65,
+      "learning_rate": 1.0001020887558839e-06,
+      "logits/chosen": -0.47975045442581177,
+      "logits/rejected": -0.4807058274745941,
+      "logps/chosen": -387.7288818359375,
+      "logps/rejected": -303.9996643066406,
+      "loss": 0.292,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.8383504748344421,
+      "rewards/margins": 0.226647287607193,
+      "rewards/rejected": -1.064997911453247,
       "step": 230
     },
     {
-      "epoch": 0.5,
-      "learning_rate": 1.7460364672965328e-06,
-      "logits/chosen": 0.6504024267196655,
-      "logits/rejected": 0.7802666425704956,
-      "logps/chosen": -466.16973876953125,
-      "logps/rejected": -511.08502197265625,
-      "loss": 0.0549,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -1.191239595413208,
-      "rewards/margins": 0.7851654291152954,
-      "rewards/rejected": -1.976405143737793,
+      "epoch": 0.67,
+      "learning_rate": 8.634704863809502e-07,
+      "logits/chosen": -0.5008819103240967,
+      "logits/rejected": -0.46626418828964233,
+      "logps/chosen": -436.1036071777344,
+      "logps/rejected": -323.905029296875,
+      "loss": 0.2845,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.8871256709098816,
+      "rewards/margins": 0.21674367785453796,
+      "rewards/rejected": -1.1038693189620972,
       "step": 240
     },
     {
-      "epoch": 0.52,
-      "learning_rate": 1.637115696063402e-06,
-      "logits/chosen": 0.7357971668243408,
-      "logits/rejected": 0.8341084718704224,
-      "logps/chosen": -462.93048095703125,
-      "logps/rejected": -550.9013671875,
-      "loss": 0.0342,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.577097773551941,
-      "rewards/margins": 0.7956889271736145,
-      "rewards/rejected": -2.3727867603302,
+      "epoch": 0.7,
+      "learning_rate": 7.330074737074666e-07,
+      "logits/chosen": -0.5267480611801147,
+      "logits/rejected": -0.528629720211029,
+      "logps/chosen": -440.72528076171875,
+      "logps/rejected": -327.6587219238281,
+      "loss": 0.2911,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.8738571405410767,
+      "rewards/margins": 0.38728970289230347,
+      "rewards/rejected": -1.2611467838287354,
       "step": 250
     },
     {
-      "epoch": 0.54,
-      "learning_rate": 1.5274599402265162e-06,
-      "logits/chosen": 0.7676488757133484,
-      "logits/rejected": 0.9279497861862183,
-      "logps/chosen": -490.0227966308594,
-      "logps/rejected": -543.2033081054688,
-      "loss": 0.0336,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.5475876331329346,
-      "rewards/margins": 0.6384353041648865,
-      "rewards/rejected": -2.186022996902466,
+      "epoch": 0.73,
+      "learning_rate": 6.099773641398835e-07,
+      "logits/chosen": -0.5498321056365967,
+      "logits/rejected": -0.5135624408721924,
+      "logps/chosen": -430.474609375,
+      "logps/rejected": -340.8508605957031,
+      "loss": 0.3049,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.9756007194519043,
+      "rewards/margins": 0.3439735174179077,
+      "rewards/rejected": -1.319574236869812,
       "step": 260
     },
     {
-      "epoch": 0.57,
-      "learning_rate": 1.4176569902035088e-06,
-      "logits/chosen": 0.7670334577560425,
-      "logits/rejected": 0.927658200263977,
-      "logps/chosen": -455.6305236816406,
-      "logps/rejected": -507.54913330078125,
-      "loss": 0.0334,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.38298761844635,
-      "rewards/margins": 0.6534308195114136,
-      "rewards/rejected": -2.0364184379577637,
+      "epoch": 0.76,
+      "learning_rate": 4.955724390266841e-07,
+      "logits/chosen": -0.5001212358474731,
+      "logits/rejected": -0.509675920009613,
+      "logps/chosen": -397.41259765625,
+      "logps/rejected": -336.1043395996094,
+      "loss": 0.2934,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.9926079511642456,
+      "rewards/margins": 0.33142027258872986,
+      "rewards/rejected": -1.3240282535552979,
       "step": 270
     },
     {
-      "epoch": 0.59,
-      "learning_rate": 1.308295425420593e-06,
-      "logits/chosen": 0.7235329151153564,
-      "logits/rejected": 0.8158149719238281,
-      "logps/chosen": -491.1328125,
-      "logps/rejected": -560.6801147460938,
-      "loss": 0.0301,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.4047319889068604,
-      "rewards/margins": 0.7390089631080627,
-      "rewards/rejected": -2.1437408924102783,
+      "epoch": 0.79,
+      "learning_rate": 3.9090139329520333e-07,
+      "logits/chosen": -0.4506424367427826,
+      "logits/rejected": -0.4490571916103363,
+      "logps/chosen": -455.0115661621094,
+      "logps/rejected": -344.4391784667969,
+      "loss": 0.3013,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.018028974533081,
+      "rewards/margins": 0.3979285955429077,
+      "rewards/rejected": -1.4159575700759888,
       "step": 280
     },
     {
-      "epoch": 0.61,
-      "learning_rate": 1.1999614593359337e-06,
-      "logits/chosen": 0.7884746789932251,
-      "logits/rejected": 1.0120609998703003,
-      "logps/chosen": -492.41693115234375,
-      "logps/rejected": -518.9060668945312,
-      "loss": 0.03,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.4595239162445068,
-      "rewards/margins": 0.7071082592010498,
-      "rewards/rejected": -2.1666321754455566,
+      "epoch": 0.81,
+      "learning_rate": 2.9697859112011724e-07,
+      "logits/chosen": -0.47781458497047424,
+      "logits/rejected": -0.4574874937534332,
+      "logps/chosen": -417.3575744628906,
+      "logps/rejected": -349.57037353515625,
+      "loss": 0.3021,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.0533437728881836,
+      "rewards/margins": 0.22412605583667755,
+      "rewards/rejected": -1.2774698734283447,
       "step": 290
     },
     {
-      "epoch": 0.63,
-      "learning_rate": 1.0932357971453745e-06,
-      "logits/chosen": 0.8025213479995728,
-      "logits/rejected": 0.9630680084228516,
-      "logps/chosen": -472.7798767089844,
-      "logps/rejected": -523.0516967773438,
-      "loss": 0.0313,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.4041074514389038,
-      "rewards/margins": 0.6285351514816284,
-      "rewards/rejected": -2.0326426029205322,
-      "step": 300
-    },
-    {
-      "epoch": 0.63,
-      "eval_logits/chosen": 0.9400739669799805,
-      "eval_logits/rejected": 1.0433921813964844,
-      "eval_logps/chosen": -454.1192932128906,
-      "eval_logps/rejected": -552.3697509765625,
-      "eval_loss": 0.03436482325196266,
-      "eval_rewards/accuracies": 0.72265625,
-      "eval_rewards/chosen": -1.3028936386108398,
-      "eval_rewards/margins": 0.9194571375846863,
-      "eval_rewards/rejected": -2.222350835800171,
-      "eval_runtime": 75.6069,
-      "eval_samples_per_second": 26.453,
-      "eval_steps_per_second": 0.423,
+      "epoch": 0.84,
+      "learning_rate": 2.1471423574861643e-07,
+      "logits/chosen": -0.49651557207107544,
+      "logits/rejected": -0.46776896715164185,
+      "logps/chosen": -429.81732177734375,
+      "logps/rejected": -326.6114807128906,
+      "loss": 0.2961,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.8363375663757324,
+      "rewards/margins": 0.38524097204208374,
+      "rewards/rejected": -1.2215787172317505,
       "step": 300
     },
     {
-      "epoch": 0.65,
-      "learning_rate": 9.886905230142433e-07,
-      "logits/chosen": 0.7544746398925781,
-      "logits/rejected": 0.9142723083496094,
-      "logps/chosen": -462.0435485839844,
-      "logps/rejected": -525.331298828125,
-      "loss": 0.0346,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.3456170558929443,
-      "rewards/margins": 0.749636709690094,
-      "rewards/rejected": -2.0952537059783936,
+      "epoch": 0.87,
+      "learning_rate": 1.449055487462102e-07,
+      "logits/chosen": -0.47061723470687866,
+      "logits/rejected": -0.4526177942752838,
+      "logps/chosen": -401.2471618652344,
+      "logps/rejected": -344.2327575683594,
+      "loss": 0.2998,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.8958613276481628,
+      "rewards/margins": 0.30968743562698364,
+      "rewards/rejected": -1.205548644065857,
       "step": 310
     },
     {
-      "epoch": 0.67,
-      "learning_rate": 8.868860335206678e-07,
-      "logits/chosen": 0.9283370971679688,
-      "logits/rejected": 1.136993169784546,
-      "logps/chosen": -478.44976806640625,
-      "logps/rejected": -530.1534423828125,
-      "loss": 0.0338,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.244257926940918,
-      "rewards/margins": 0.6402724385261536,
-      "rewards/rejected": -1.8845303058624268,
+      "epoch": 0.9,
+      "learning_rate": 8.822904414485194e-08,
+      "logits/chosen": -0.4069460928440094,
+      "logits/rejected": -0.413198858499527,
+      "logps/chosen": -376.17364501953125,
+      "logps/rejected": -327.5869140625,
+      "loss": 0.2994,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.8381717801094055,
+      "rewards/margins": 0.24178913235664368,
+      "rewards/rejected": -1.079960823059082,
       "step": 320
     },
     {
-      "epoch": 0.69,
-      "learning_rate": 7.883680337481599e-07,
-      "logits/chosen": 0.7307278513908386,
-      "logits/rejected": 0.8725861310958862,
-      "logps/chosen": -448.43280029296875,
-      "logps/rejected": -533.6476440429688,
-      "loss": 0.0375,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -1.2363145351409912,
-      "rewards/margins": 0.7372487187385559,
-      "rewards/rejected": -1.9735629558563232,
+      "epoch": 0.93,
+      "learning_rate": 4.523397236438398e-08,
+      "logits/chosen": -0.5108148455619812,
+      "logits/rejected": -0.49169641733169556,
+      "logps/chosen": -478.1392517089844,
+      "logps/rejected": -382.9190979003906,
+      "loss": 0.2897,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.9290350079536438,
+      "rewards/margins": 0.41925033926963806,
+      "rewards/rejected": -1.3482853174209595,
       "step": 330
     },
-    {
-      "epoch": 0.71,
-      "learning_rate": 6.936646121293654e-07,
-      "logits/chosen": 0.5649510622024536,
-      "logits/rejected": 0.7639907598495483,
-      "logps/chosen": -466.2808532714844,
-      "logps/rejected": -526.1297607421875,
-      "loss": 0.0435,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.0562084913253784,
-      "rewards/margins": 0.7370297312736511,
-      "rewards/rejected": -1.7932384014129639,
-      "step": 340
-    },
-    {
-      "epoch": 0.73,
-      "learning_rate": 6.032834097207889e-07,
-      "logits/chosen": 0.7209309935569763,
-      "logits/rejected": 0.7828409671783447,
-      "logps/chosen": -401.2094421386719,
-      "logps/rejected": -480.31671142578125,
-      "loss": 0.0403,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.098332405090332,
-      "rewards/margins": 0.6962517499923706,
-      "rewards/rejected": -1.7945845127105713,
-      "step": 350
-    },
-    {
-      "epoch": 0.75,
-      "learning_rate": 5.177088990820725e-07,
-      "logits/chosen": 0.6787894368171692,
-      "logits/rejected": 0.8372275233268738,
-      "logps/chosen": -446.56317138671875,
-      "logps/rejected": -465.1809997558594,
-      "loss": 0.0453,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.136115550994873,
-      "rewards/margins": 0.6250497698783875,
-      "rewards/rejected": -1.7611652612686157,
-      "step": 360
-    },
-    {
-      "epoch": 0.77,
-      "learning_rate": 4.3739978734594494e-07,
-      "logits/chosen": 0.6346519589424133,
-      "logits/rejected": 0.867949366569519,
-      "logps/chosen": -439.4676208496094,
-      "logps/rejected": -468.6329040527344,
-      "loss": 0.0364,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.016570806503296,
-      "rewards/margins": 0.8048780560493469,
-      "rewards/rejected": -1.8214489221572876,
-      "step": 370
-    },
-    {
-      "epoch": 0.8,
-      "learning_rate": 3.627865573992087e-07,
-      "logits/chosen": 0.6531890630722046,
-      "logits/rejected": 0.6925245523452759,
-      "logps/chosen": -437.359375,
-      "logps/rejected": -492.814453125,
-      "loss": 0.0425,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -1.09610116481781,
-      "rewards/margins": 0.6472191214561462,
-      "rewards/rejected": -1.7433204650878906,
-      "step": 380
-    },
-    {
-      "epoch": 0.82,
-      "learning_rate": 2.9426916035484166e-07,
-      "logits/chosen": 0.4887206554412842,
-      "logits/rejected": 0.7168077230453491,
-      "logps/chosen": -490.0777893066406,
-      "logps/rejected": -530.9293212890625,
-      "loss": 0.038,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -1.2254283428192139,
-      "rewards/margins": 0.8675802946090698,
-      "rewards/rejected": -2.0930087566375732,
-      "step": 390
-    },
-    {
-      "epoch": 0.84,
-      "learning_rate": 2.322148716843081e-07,
-      "logits/chosen": 0.6055541038513184,
-      "logits/rejected": 0.687682032585144,
-      "logps/chosen": -429.68603515625,
-      "logps/rejected": -461.8595275878906,
-      "loss": 0.0359,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.1895955801010132,
-      "rewards/margins": 0.5694113373756409,
-      "rewards/rejected": -1.7590070962905884,
-      "step": 400
-    },
-    {
-      "epoch": 0.84,
-      "eval_logits/chosen": 0.7290832996368408,
-      "eval_logits/rejected": 0.8196390867233276,
-      "eval_logps/chosen": -435.5875244140625,
-      "eval_logps/rejected": -531.2747192382812,
-      "eval_loss": 0.04154704138636589,
-      "eval_rewards/accuracies": 0.70703125,
-      "eval_rewards/chosen": -1.1175758838653564,
-      "eval_rewards/margins": 0.8938245177268982,
-      "eval_rewards/rejected": -2.0114002227783203,
-      "eval_runtime": 75.1852,
-      "eval_samples_per_second": 26.601,
-      "eval_steps_per_second": 0.426,
-      "step": 400
-    },
-    {
-      "epoch": 0.86,
-      "learning_rate": 1.7695632250191002e-07,
-      "logits/chosen": 0.5428584814071655,
-      "logits/rejected": 0.6822582483291626,
-      "logps/chosen": -435.78680419921875,
-      "logps/rejected": -452.6622009277344,
-      "loss": 0.0367,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.1751288175582886,
-      "rewards/margins": 0.5176131129264832,
-      "rewards/rejected": -1.6927419900894165,
-      "step": 410
-    },
-    {
-      "epoch": 0.88,
-      "learning_rate": 1.2878971655412515e-07,
-      "logits/chosen": 0.5744162797927856,
-      "logits/rejected": 0.6994149088859558,
-      "logps/chosen": -474.30908203125,
-      "logps/rejected": -495.92852783203125,
-      "loss": 0.0394,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -1.3165512084960938,
-      "rewards/margins": 0.6040414571762085,
-      "rewards/rejected": -1.9205926656723022,
-      "step": 420
-    },
-    {
-      "epoch": 0.9,
-      "learning_rate": 8.797324247145411e-08,
-      "logits/chosen": 0.6493648290634155,
-      "logits/rejected": 0.6758213043212891,
-      "logps/chosen": -426.60223388671875,
-      "logps/rejected": -521.1129150390625,
-      "loss": 0.0365,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.1816965341567993,
-      "rewards/margins": 0.7749902009963989,
-      "rewards/rejected": -1.9566866159439087,
-      "step": 430
-    },
-    {
-      "epoch": 0.92,
-      "learning_rate": 5.472568979361853e-08,
-      "logits/chosen": 0.7012882232666016,
-      "logits/rejected": 0.7845873832702637,
-      "logps/chosen": -459.6414489746094,
-      "logps/rejected": -518.3292846679688,
-      "loss": 0.0412,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -1.2379354238510132,
-      "rewards/margins": 0.7565950155258179,
-      "rewards/rejected": -1.994530439376831,
-      "step": 440
-    },
-    {
-      "epoch": 0.94,
-      "learning_rate": 2.922527618666465e-08,
-      "logits/chosen": 0.6378465294837952,
-      "logits/rejected": 0.8079195022583008,
-      "logps/chosen": -484.46197509765625,
-      "logps/rejected": -520.6287841796875,
-      "loss": 0.0404,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.2168313264846802,
-      "rewards/margins": 0.676922082901001,
-      "rewards/rejected": -1.8937534093856812,
-      "step": 450
-    },
     {
       "epoch": 0.96,
-      "learning_rate": 1.1608692138469379e-08,
-      "logits/chosen": 0.7224764227867126,
-      "logits/rejected": 0.8670576214790344,
-      "logps/chosen": -398.640869140625,
-      "logps/rejected": -444.4422912597656,
-      "loss": 0.039,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.110528826713562,
-      "rewards/margins": 0.6203423738479614,
-      "rewards/rejected": -1.7308712005615234,
-      "step": 460
+      "learning_rate": 1.6336997442095825e-08,
+      "logits/chosen": -0.4700957238674164,
+      "logits/rejected": -0.43701282143592834,
+      "logps/chosen": -426.40631103515625,
+      "logps/rejected": -343.24334716796875,
+      "loss": 0.3046,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.9494872093200684,
+      "rewards/margins": 0.334994375705719,
+      "rewards/rejected": -1.2844815254211426,
+      "step": 340
     },
     {
       "epoch": 0.98,
-      "learning_rate": 1.970368253390198e-09,
-      "logits/chosen": 0.6133291125297546,
-      "logits/rejected": 0.744029700756073,
-      "logps/chosen": -413.31732177734375,
-      "logps/rejected": -497.829345703125,
-      "loss": 0.0407,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.0763300657272339,
-      "rewards/margins": 0.8329319953918457,
-      "rewards/rejected": -1.9092620611190796,
-      "step": 470
+      "learning_rate": 1.8181591531977737e-09,
+      "logits/chosen": -0.5112472176551819,
+      "logits/rejected": -0.5403722524642944,
+      "logps/chosen": -407.7789611816406,
+      "logps/rejected": -344.8587951660156,
+      "loss": 0.3017,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.9037674069404602,
+      "rewards/margins": 0.4302978515625,
+      "rewards/rejected": -1.334065318107605,
+      "step": 350
     },
     {
       "epoch": 1.0,
-      "step": 477,
+      "step": 355,
       "total_flos": 0.0,
-      "train_loss": 0.055112330793584664,
-      "train_runtime": 4571.3444,
-      "train_samples_per_second": 13.373,
-      "train_steps_per_second": 0.104
+      "train_loss": 0.3547657697973117,
+      "train_runtime": 5270.9361,
+      "train_samples_per_second": 8.641,
+      "train_steps_per_second": 0.067
     }
   ],
   "logging_steps": 10,
-  "max_steps": 477,
+  "max_steps": 355,
   "num_train_epochs": 1,
-  "save_steps": 1000,
+  "save_steps": 10000,
   "total_flos": 0.0,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:200542098b43881df0df6dc0ff3056ca0236db5763f486bb392f305292932d2f
-size 5944
+oid sha256:773f035981526cb91d8da745ac00a062e7cff067bade23d7346497d28717689d
+size 6648