Model save

README.md

@@ -14,6 +14,16 @@ should probably proofread and complete it, then remove this comment. -->
 # zephyr-7b-dpo-full
 
 This model was trained from scratch on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0427
+- Rewards/chosen: -1.1712
+- Rewards/rejected: -2.0556
+- Rewards/accuracies: 0.7266
+- Rewards/margins: 0.8844
+- Logps/rejected: -517.1834
+- Logps/chosen: -420.7032
+- Logits/rejected: -0.0713
+- Logits/chosen: -0.0801
 
 ## Model description
 
@@ -33,12 +43,12 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 3e-06
-- train_batch_size: 2
+- train_batch_size: 4
 - eval_batch_size: 8
-- seed: 1
+- seed: 2
 - distributed_type: multi-GPU
 - num_devices: 8
-- gradient_accumulation_steps: 8
+- gradient_accumulation_steps: 4
 - total_train_batch_size: 128
 - total_eval_batch_size: 64
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
@@ -48,6 +58,12 @@ The following hyperparameters were used during training:
 
 ### Training results
 
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.0642        | 0.21  | 100  | 0.0487          | -1.3984        | -2.4508          | 0.7188             | 1.0524          | -556.7064      | -443.4212    | 0.4579          | 0.4301        |
+| 0.047         | 0.42  | 200  | 0.0461          | -1.1146        | -1.8461          | 0.7422             | 0.7314          | -496.2327      | -415.0494    | 0.1274          | 0.1123        |
+| 0.0401        | 0.63  | 300  | 0.0408          | -1.2816        | -2.1650          | 0.7148             | 0.8834          | -528.1252      | -431.7439    | -0.1765         | -0.1928       |
+| 0.0435        | 0.84  | 400  | 0.0427          | -1.1712        | -2.0556          | 0.7266             | 0.8844          | -517.1834      | -420.7032    | -0.0713         | -0.0801       |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.538863022122493,
-    "train_runtime": 6331.0486,
+    "train_loss": 0.05794826206171288,
+    "train_runtime": 4397.3222,
     "train_samples": 61134,
-    "train_samples_per_second": 9.656,
-    "train_steps_per_second": 0.075
+    "train_samples_per_second": 13.903,
+    "train_steps_per_second": 0.108
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2ca6939233ba1e245fdc46faba9fe5f3921b848a1a3b236d119c079f34232b01
+oid sha256:09b30f17a60c9620148cbd5eb76ec7aa8992e70e6d7a4b2040af7a3e47fd1c42
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc0f5cdd65e6510b7b13813c9be8c354eff275c814bf4fb95f6a95028570f141
+oid sha256:9a2c540e131df728dc7e661af77405e10eb54b3db4281b7eed1e89377b4dc105
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:61203f37794e71715ec3fae92d5dac73a2d855866903bbd863e13bd2efa893fe
+oid sha256:980ee41ac40131d5176d7e7821579b166f4636c9f8b54a41d57709f2dbcd51da
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b9548290db975a59d0450012a02daa4e5d1f873fccf90a8ea347142a091758b1
+oid sha256:23f4c41ec0c959598b2ab2b7e53a1d3f3200d53942ef2e6305032dea475d5633
 size 1168138808

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.538863022122493,
-    "train_runtime": 6331.0486,
+    "train_loss": 0.05794826206171288,
+    "train_runtime": 4397.3222,
     "train_samples": 61134,
-    "train_samples_per_second": 9.656,
-    "train_steps_per_second": 0.075
+    "train_samples_per_second": 13.903,
+    "train_steps_per_second": 0.108
 }

trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.998691442030882,
-  "eval_steps": 10000,
+  "epoch": 0.9984301412872841,
+  "eval_steps": 100,
   "global_step": 477,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
@@ -11,675 +11,739 @@
     {
       "epoch": 0.02,
       "learning_rate": 6.25e-07,
-      "logits/chosen": 0.17749178409576416,
-      "logits/rejected": 0.25463899970054626,
-      "logps/chosen": -354.40045166015625,
-      "logps/rejected": -305.2121276855469,
-      "loss": 0.693,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": -1.9714002519322094e-06,
-      "rewards/margins": -5.996927575324662e-05,
-      "rewards/rejected": 5.799797872896306e-05,
+      "logits/chosen": 0.2953376770019531,
+      "logits/rejected": 0.3199407160282135,
+      "logps/chosen": -295.2257080078125,
+      "logps/rejected": -290.8670959472656,
+      "loss": 0.1859,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": 0.00015557199367322028,
+      "rewards/margins": 0.0003326966252643615,
+      "rewards/rejected": -0.00017712473345454782,
       "step": 10
     },
     {
       "epoch": 0.04,
       "learning_rate": 1.25e-06,
-      "logits/chosen": 0.081016905605793,
-      "logits/rejected": 0.20831938087940216,
-      "logps/chosen": -315.9397277832031,
-      "logps/rejected": -276.8512268066406,
-      "loss": 0.6873,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.007955458015203476,
-      "rewards/margins": 0.01611277461051941,
-      "rewards/rejected": -0.008157314732670784,
+      "logits/chosen": 0.26482802629470825,
+      "logits/rejected": 0.33679550886154175,
+      "logps/chosen": -318.49652099609375,
+      "logps/rejected": -291.0345458984375,
+      "loss": 0.1869,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.002573266625404358,
+      "rewards/margins": 0.011565858498215675,
+      "rewards/rejected": -0.014139125123620033,
       "step": 20
     },
     {
       "epoch": 0.06,
       "learning_rate": 1.875e-06,
-      "logits/chosen": 0.2659609317779541,
-      "logits/rejected": 0.33923760056495667,
-      "logps/chosen": -300.55352783203125,
-      "logps/rejected": -311.36102294921875,
-      "loss": 0.6599,
+      "logits/chosen": 0.3993307650089264,
+      "logits/rejected": 0.45834311842918396,
+      "logps/chosen": -301.6546936035156,
+      "logps/rejected": -295.427734375,
+      "loss": 0.1703,
       "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.05586661025881767,
-      "rewards/margins": 0.0747639387845993,
-      "rewards/rejected": -0.13063055276870728,
+      "rewards/chosen": -0.05365907400846481,
+      "rewards/margins": 0.07345211505889893,
+      "rewards/rejected": -0.12711118161678314,
       "step": 30
     },
     {
       "epoch": 0.08,
       "learning_rate": 2.5e-06,
-      "logits/chosen": 0.2669721245765686,
-      "logits/rejected": 0.390450656414032,
-      "logps/chosen": -370.79107666015625,
-      "logps/rejected": -360.6502685546875,
-      "loss": 0.652,
+      "logits/chosen": 0.35423707962036133,
+      "logits/rejected": 0.4464724063873291,
+      "logps/chosen": -333.11279296875,
+      "logps/rejected": -334.0306091308594,
+      "loss": 0.1389,
       "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.23218843340873718,
-      "rewards/margins": 0.16964344680309296,
-      "rewards/rejected": -0.4018319249153137,
+      "rewards/chosen": -0.2748720049858093,
+      "rewards/margins": 0.14294137060642242,
+      "rewards/rejected": -0.41781339049339294,
       "step": 40
     },
     {
       "epoch": 0.1,
       "learning_rate": 2.999839121261416e-06,
-      "logits/chosen": 0.27385860681533813,
-      "logits/rejected": 0.40135669708251953,
-      "logps/chosen": -344.6741943359375,
-      "logps/rejected": -349.53399658203125,
-      "loss": 0.6226,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.3412158191204071,
-      "rewards/margins": 0.31730175018310547,
-      "rewards/rejected": -0.6585175395011902,
+      "logits/chosen": 0.35561248660087585,
+      "logits/rejected": 0.3796083331108093,
+      "logps/chosen": -374.3013916015625,
+      "logps/rejected": -392.155517578125,
+      "loss": 0.0917,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.6061999201774597,
+      "rewards/margins": 0.2705405652523041,
+      "rewards/rejected": -0.8767404556274414,
       "step": 50
     },
     {
       "epoch": 0.13,
       "learning_rate": 2.994211988057582e-06,
-      "logits/chosen": 0.16504314541816711,
-      "logits/rejected": 0.39565616846084595,
-      "logps/chosen": -338.04547119140625,
-      "logps/rejected": -361.7615966796875,
-      "loss": 0.5965,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.45166701078414917,
-      "rewards/margins": 0.4114951193332672,
-      "rewards/rejected": -0.8631621599197388,
+      "logits/chosen": 0.17622683942317963,
+      "logits/rejected": 0.313021719455719,
+      "logps/chosen": -346.9971618652344,
+      "logps/rejected": -349.66387939453125,
+      "loss": 0.0903,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.5485752820968628,
+      "rewards/margins": 0.28624793887138367,
+      "rewards/rejected": -0.8348232507705688,
       "step": 60
     },
     {
       "epoch": 0.15,
       "learning_rate": 2.9805753939568693e-06,
-      "logits/chosen": 0.2622874975204468,
-      "logits/rejected": 0.40581947565078735,
-      "logps/chosen": -381.7900695800781,
-      "logps/rejected": -426.4183654785156,
-      "loss": 0.5829,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.5206634402275085,
-      "rewards/margins": 0.5540415644645691,
-      "rewards/rejected": -1.0747050046920776,
+      "logits/chosen": 0.1983145773410797,
+      "logits/rejected": 0.23167672753334045,
+      "logps/chosen": -356.7235412597656,
+      "logps/rejected": -386.5905456542969,
+      "loss": 0.0815,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.5771406888961792,
+      "rewards/margins": 0.28289300203323364,
+      "rewards/rejected": -0.8600338101387024,
       "step": 70
     },
     {
       "epoch": 0.17,
       "learning_rate": 2.959002435526626e-06,
-      "logits/chosen": 0.25349122285842896,
-      "logits/rejected": 0.3699270784854889,
-      "logps/chosen": -397.3373718261719,
-      "logps/rejected": -444.7080078125,
-      "loss": 0.5535,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.7867372632026672,
-      "rewards/margins": 0.7896677851676941,
-      "rewards/rejected": -1.5764049291610718,
+      "logits/chosen": 0.35488566756248474,
+      "logits/rejected": 0.4059143662452698,
+      "logps/chosen": -371.07623291015625,
+      "logps/rejected": -389.4164123535156,
+      "loss": 0.0655,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.755893349647522,
+      "rewards/margins": 0.5531964302062988,
+      "rewards/rejected": -1.3090897798538208,
       "step": 80
     },
     {
       "epoch": 0.19,
       "learning_rate": 2.929608750821129e-06,
-      "logits/chosen": 0.05120038986206055,
-      "logits/rejected": 0.2029154747724533,
-      "logps/chosen": -401.5246276855469,
-      "logps/rejected": -380.9539489746094,
-      "loss": 0.5692,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.6458311676979065,
-      "rewards/margins": 0.49296313524246216,
-      "rewards/rejected": -1.138794183731079,
+      "logits/chosen": 0.2614609897136688,
+      "logits/rejected": 0.4048990309238434,
+      "logps/chosen": -387.1451110839844,
+      "logps/rejected": -373.1450500488281,
+      "loss": 0.0699,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.7152177095413208,
+      "rewards/margins": 0.39531436562538147,
+      "rewards/rejected": -1.1105320453643799,
       "step": 90
     },
     {
       "epoch": 0.21,
       "learning_rate": 2.892551899524109e-06,
-      "logits/chosen": 0.048072777688503265,
-      "logits/rejected": 0.24050505459308624,
-      "logps/chosen": -399.18426513671875,
-      "logps/rejected": -435.47479248046875,
-      "loss": 0.5556,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.6462281346321106,
-      "rewards/margins": 0.7846990823745728,
-      "rewards/rejected": -1.4309271574020386,
+      "logits/chosen": 0.35849034786224365,
+      "logits/rejected": 0.3978693187236786,
+      "logps/chosen": -395.839599609375,
+      "logps/rejected": -456.36016845703125,
+      "loss": 0.0642,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.0831067562103271,
+      "rewards/margins": 0.505523681640625,
+      "rewards/rejected": -1.5886304378509521,
+      "step": 100
+    },
+    {
+      "epoch": 0.21,
+      "eval_logits/chosen": 0.4300641715526581,
+      "eval_logits/rejected": 0.45788079500198364,
+      "eval_logps/chosen": -443.42120361328125,
+      "eval_logps/rejected": -556.7063598632812,
+      "eval_loss": 0.04867542162537575,
+      "eval_rewards/accuracies": 0.71875,
+      "eval_rewards/chosen": -1.3983652591705322,
+      "eval_rewards/margins": 1.0524473190307617,
+      "eval_rewards/rejected": -2.450812578201294,
+      "eval_runtime": 73.3058,
+      "eval_samples_per_second": 27.283,
+      "eval_steps_per_second": 0.437,
       "step": 100
     },
     {
       "epoch": 0.23,
       "learning_rate": 2.848030518377739e-06,
-      "logits/chosen": 0.16751229763031006,
-      "logits/rejected": 0.16947798430919647,
-      "logps/chosen": -393.181396484375,
-      "logps/rejected": -448.7366638183594,
-      "loss": 0.5306,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.8056195974349976,
-      "rewards/margins": 0.6872454881668091,
-      "rewards/rejected": -1.492864966392517,
+      "logits/chosen": 0.2927325367927551,
+      "logits/rejected": 0.3830544054508209,
+      "logps/chosen": -459.0276794433594,
+      "logps/rejected": -510.6993103027344,
+      "loss": 0.0405,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.4556727409362793,
+      "rewards/margins": 0.8043819665908813,
+      "rewards/rejected": -2.260054588317871,
       "step": 110
     },
     {
       "epoch": 0.25,
       "learning_rate": 2.7962832564252724e-06,
-      "logits/chosen": 0.15264573693275452,
-      "logits/rejected": 0.25869759917259216,
-      "logps/chosen": -399.55706787109375,
-      "logps/rejected": -474.19659423828125,
-      "loss": 0.5521,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.7900343537330627,
-      "rewards/margins": 0.7399677038192749,
-      "rewards/rejected": -1.5300019979476929,
+      "logits/chosen": 0.15991483628749847,
+      "logits/rejected": 0.2417908012866974,
+      "logps/chosen": -477.8702087402344,
+      "logps/rejected": -537.2095947265625,
+      "loss": 0.0489,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.243250846862793,
+      "rewards/margins": 0.6618646383285522,
+      "rewards/rejected": -1.9051154851913452,
       "step": 120
     },
     {
       "epoch": 0.27,
       "learning_rate": 2.7375874957747644e-06,
-      "logits/chosen": 0.09207181632518768,
-      "logits/rejected": 0.33124327659606934,
-      "logps/chosen": -421.142578125,
-      "logps/rejected": -428.9435119628906,
-      "loss": 0.5221,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.8264646530151367,
-      "rewards/margins": 0.6144661903381348,
-      "rewards/rejected": -1.4409308433532715,
+      "logits/chosen": 0.06199001148343086,
+      "logits/rejected": 0.19189774990081787,
+      "logps/chosen": -411.61602783203125,
+      "logps/rejected": -418.01019287109375,
+      "loss": 0.0665,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.8180139660835266,
+      "rewards/margins": 0.39107373356819153,
+      "rewards/rejected": -1.209087610244751,
       "step": 130
     },
     {
       "epoch": 0.29,
       "learning_rate": 2.672257864741005e-06,
-      "logits/chosen": 0.0794561430811882,
-      "logits/rejected": 0.3778214752674103,
-      "logps/chosen": -430.6768493652344,
-      "logps/rejected": -436.89324951171875,
-      "loss": 0.5419,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.9863343238830566,
-      "rewards/margins": 0.6835289001464844,
-      "rewards/rejected": -1.6698631048202515,
+      "logits/chosen": 0.16753675043582916,
+      "logits/rejected": 0.2094758003950119,
+      "logps/chosen": -402.02484130859375,
+      "logps/rejected": -429.792724609375,
+      "loss": 0.0608,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.8829413652420044,
+      "rewards/margins": 0.4748268127441406,
+      "rewards/rejected": -1.3577684164047241,
       "step": 140
     },
     {
       "epoch": 0.31,
       "learning_rate": 2.600644551335706e-06,
-      "logits/chosen": 0.18784897029399872,
-      "logits/rejected": 0.2580476403236389,
-      "logps/chosen": -402.0419006347656,
-      "logps/rejected": -438.2938537597656,
-      "loss": 0.5609,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.667413055896759,
-      "rewards/margins": 0.5660290718078613,
-      "rewards/rejected": -1.2334421873092651,
+      "logits/chosen": 0.12230317294597626,
+      "logits/rejected": 0.2780611515045166,
+      "logps/chosen": -399.07012939453125,
+      "logps/rejected": -447.2755432128906,
+      "loss": 0.0409,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.1799192428588867,
+      "rewards/margins": 0.7245117425918579,
+      "rewards/rejected": -1.9044307470321655,
       "step": 150
     },
     {
       "epoch": 0.33,
       "learning_rate": 2.5231314261461732e-06,
-      "logits/chosen": 0.01151950377970934,
-      "logits/rejected": 0.06548599898815155,
-      "logps/chosen": -368.8543395996094,
-      "logps/rejected": -422.1419982910156,
-      "loss": 0.5505,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.6679279208183289,
-      "rewards/margins": 0.7465037703514099,
-      "rewards/rejected": -1.4144316911697388,
+      "logits/chosen": 0.15223169326782227,
+      "logits/rejected": 0.2353695183992386,
+      "logps/chosen": -423.8455505371094,
+      "logps/rejected": -493.4649353027344,
+      "loss": 0.0439,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.3035107851028442,
+      "rewards/margins": 0.7260502576828003,
+      "rewards/rejected": -2.0295610427856445,
       "step": 160
     },
     {
       "epoch": 0.36,
       "learning_rate": 2.440133984664454e-06,
-      "logits/chosen": -0.09347765147686005,
-      "logits/rejected": -0.002395985182374716,
-      "logps/chosen": -415.9041442871094,
-      "logps/rejected": -450.20556640625,
-      "loss": 0.5413,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.7953579425811768,
-      "rewards/margins": 0.6282046437263489,
-      "rewards/rejected": -1.4235625267028809,
+      "logits/chosen": 0.07280103117227554,
+      "logits/rejected": 0.15081152319908142,
+      "logps/chosen": -472.6494140625,
+      "logps/rejected": -495.68768310546875,
+      "loss": 0.0448,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.1380369663238525,
+      "rewards/margins": 0.558665931224823,
+      "rewards/rejected": -1.6967029571533203,
       "step": 170
     },
     {
       "epoch": 0.38,
       "learning_rate": 2.3520971200967337e-06,
-      "logits/chosen": -0.25012534856796265,
-      "logits/rejected": -0.054043568670749664,
-      "logps/chosen": -409.1259765625,
-      "logps/rejected": -474.79693603515625,
-      "loss": 0.5326,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.8309396505355835,
-      "rewards/margins": 0.9182770848274231,
-      "rewards/rejected": -1.7492167949676514,
+      "logits/chosen": 0.1431509107351303,
+      "logits/rejected": 0.12718148529529572,
+      "logps/chosen": -421.31591796875,
+      "logps/rejected": -481.0022888183594,
+      "loss": 0.0398,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.277320146560669,
+      "rewards/margins": 0.7172085642814636,
+      "rewards/rejected": -1.9945287704467773,
       "step": 180
     },
     {
       "epoch": 0.4,
       "learning_rate": 2.2594927385914546e-06,
-      "logits/chosen": -0.23637855052947998,
-      "logits/rejected": -0.09897241741418839,
-      "logps/chosen": -418.91204833984375,
-      "logps/rejected": -464.08831787109375,
-      "loss": 0.5339,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.842525839805603,
-      "rewards/margins": 0.6711076498031616,
-      "rewards/rejected": -1.5136334896087646,
+      "logits/chosen": 0.1423339694738388,
+      "logits/rejected": 0.20066264271736145,
+      "logps/chosen": -385.1085205078125,
+      "logps/rejected": -413.58660888671875,
+      "loss": 0.044,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.1722856760025024,
+      "rewards/margins": 0.5436893105506897,
+      "rewards/rejected": -1.715975046157837,
       "step": 190
     },
     {
       "epoch": 0.42,
       "learning_rate": 2.1628172296692954e-06,
-      "logits/chosen": -0.2666473984718323,
-      "logits/rejected": -0.11236460506916046,
-      "logps/chosen": -396.9189453125,
-      "logps/rejected": -428.5235900878906,
-      "loss": 0.5378,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.5642918348312378,
-      "rewards/margins": 0.7796770334243774,
-      "rewards/rejected": -1.3439688682556152,
+      "logits/chosen": 0.08901094645261765,
+      "logits/rejected": 0.13271097838878632,
+      "logps/chosen": -385.1289367675781,
+      "logps/rejected": -454.6835021972656,
+      "loss": 0.047,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.0956504344940186,
+      "rewards/margins": 0.6720448732376099,
+      "rewards/rejected": -1.767695426940918,
+      "step": 200
+    },
+    {
+      "epoch": 0.42,
+      "eval_logits/chosen": 0.11227821558713913,
+      "eval_logits/rejected": 0.12742303311824799,
+      "eval_logps/chosen": -415.04937744140625,
+      "eval_logps/rejected": -496.232666015625,
+      "eval_loss": 0.0460943840444088,
+      "eval_rewards/accuracies": 0.7421875,
+      "eval_rewards/chosen": -1.1146469116210938,
+      "eval_rewards/margins": 0.7314289212226868,
+      "eval_rewards/rejected": -1.8460756540298462,
+      "eval_runtime": 73.4078,
+      "eval_samples_per_second": 27.245,
+      "eval_steps_per_second": 0.436,
       "step": 200
     },
     {
       "epoch": 0.44,
       "learning_rate": 2.062588805414343e-06,
-      "logits/chosen": -0.11552943289279938,
-      "logits/rejected": 0.1089673787355423,
-      "logps/chosen": -382.94049072265625,
-      "logps/rejected": -421.65338134765625,
-      "loss": 0.5452,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.6878319978713989,
-      "rewards/margins": 0.8299639821052551,
-      "rewards/rejected": -1.5177960395812988,
+      "logits/chosen": -0.006222477648407221,
+      "logits/rejected": 0.02312941662967205,
+      "logps/chosen": -435.7500915527344,
+      "logps/rejected": -491.56561279296875,
+      "loss": 0.0379,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.1337292194366455,
+      "rewards/margins": 0.7197057008743286,
+      "rewards/rejected": -1.8534349203109741,
       "step": 210
     },
     {
       "epoch": 0.46,
       "learning_rate": 1.9593447226892386e-06,
-      "logits/chosen": -0.3720184862613678,
-      "logits/rejected": -0.14822836220264435,
-      "logps/chosen": -393.29931640625,
-      "logps/rejected": -419.65283203125,
-      "loss": 0.5192,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.7662023305892944,
-      "rewards/margins": 0.793621838092804,
-      "rewards/rejected": -1.559823989868164,
+      "logits/chosen": -0.17459270358085632,
+      "logits/rejected": -0.05563253164291382,
+      "logps/chosen": -466.9278259277344,
+      "logps/rejected": -490.50567626953125,
+      "loss": 0.033,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.4599158763885498,
+      "rewards/margins": 0.615450382232666,
+      "rewards/rejected": -2.075366497039795,
       "step": 220
     },
     {
       "epoch": 0.48,
       "learning_rate": 1.853638403264141e-06,
-      "logits/chosen": -0.3021862804889679,
-      "logits/rejected": -0.25613459944725037,
-      "logps/chosen": -416.7992248535156,
-      "logps/rejected": -465.28009033203125,
-      "loss": 0.5337,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.9073785543441772,
-      "rewards/margins": 0.9142147898674011,
-      "rewards/rejected": -1.8215935230255127,
+      "logits/chosen": -0.037995658814907074,
+      "logits/rejected": -0.011381956748664379,
+      "logps/chosen": -448.68939208984375,
+      "logps/rejected": -532.2055053710938,
+      "loss": 0.0365,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6667238473892212,
+      "rewards/margins": 0.8385307192802429,
+      "rewards/rejected": -2.5052547454833984,
       "step": 230
     },
     {
       "epoch": 0.5,
       "learning_rate": 1.7460364672965328e-06,
-      "logits/chosen": -0.4147976338863373,
-      "logits/rejected": -0.33779656887054443,
-      "logps/chosen": -408.96173095703125,
-      "logps/rejected": -452.09442138671875,
-      "loss": 0.5465,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.970099925994873,
-      "rewards/margins": 0.5317692756652832,
-      "rewards/rejected": -1.5018692016601562,
+      "logits/chosen": -0.0721597820520401,
+      "logits/rejected": -0.08736585080623627,
+      "logps/chosen": -470.03045654296875,
+      "logps/rejected": -542.5087280273438,
+      "loss": 0.0484,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.167931318283081,
+      "rewards/margins": 0.7865667939186096,
+      "rewards/rejected": -1.954498052597046,
       "step": 240
     },
     {
       "epoch": 0.52,
       "learning_rate": 1.637115696063402e-06,
-      "logits/chosen": -0.4023767113685608,
-      "logits/rejected": -0.32735350728034973,
-      "logps/chosen": -410.4549865722656,
-      "logps/rejected": -460.449462890625,
-      "loss": 0.5349,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.9840686917304993,
-      "rewards/margins": 0.7999861836433411,
-      "rewards/rejected": -1.7840545177459717,
+      "logits/chosen": -0.07914544641971588,
+      "logits/rejected": -0.05659206956624985,
+      "logps/chosen": -430.75726318359375,
+      "logps/rejected": -472.98284912109375,
+      "loss": 0.0475,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.1496978998184204,
+      "rewards/margins": 0.6665709018707275,
+      "rewards/rejected": -1.8162685632705688,
       "step": 250
     },
     {
       "epoch": 0.54,
       "learning_rate": 1.5274599402265162e-06,
-      "logits/chosen": -0.44594064354896545,
-      "logits/rejected": -0.34519004821777344,
-      "logps/chosen": -447.99395751953125,
-      "logps/rejected": -448.3359375,
-      "loss": 0.5503,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.9335018992424011,
-      "rewards/margins": 0.5492356419563293,
-      "rewards/rejected": -1.4827375411987305,
+      "logits/chosen": -0.23472614586353302,
+      "logits/rejected": -0.11506778001785278,
+      "logps/chosen": -493.5909118652344,
+      "logps/rejected": -528.9447021484375,
+      "loss": 0.0421,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5179877281188965,
+      "rewards/margins": 0.8330425024032593,
+      "rewards/rejected": -2.3510303497314453,
       "step": 260
     },
     {
       "epoch": 0.57,
       "learning_rate": 1.4176569902035088e-06,
-      "logits/chosen": -0.35517948865890503,
-      "logits/rejected": -0.41846075654029846,
-      "logps/chosen": -432.1543884277344,
-      "logps/rejected": -498.60821533203125,
-      "loss": 0.5323,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.030205249786377,
-      "rewards/margins": 0.8513900637626648,
-      "rewards/rejected": -1.881595253944397,
+      "logits/chosen": -0.17047128081321716,
+      "logits/rejected": -0.09592470526695251,
+      "logps/chosen": -456.50042724609375,
+      "logps/rejected": -494.05059814453125,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.2792171239852905,
+      "rewards/margins": 0.6969503164291382,
+      "rewards/rejected": -1.9761674404144287,
       "step": 270
     },
     {
       "epoch": 0.59,
       "learning_rate": 1.308295425420593e-06,
-      "logits/chosen": -0.41587767004966736,
-      "logits/rejected": -0.5421104431152344,
-      "logps/chosen": -413.42413330078125,
-      "logps/rejected": -473.20794677734375,
-      "loss": 0.5435,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.1694116592407227,
-      "rewards/margins": 0.6519973278045654,
-      "rewards/rejected": -1.8214088678359985,
+      "logits/chosen": -0.24125584959983826,
+      "logits/rejected": -0.21810472011566162,
+      "logps/chosen": -445.63916015625,
+      "logps/rejected": -531.8533325195312,
+      "loss": 0.0463,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.3838244676589966,
+      "rewards/margins": 0.7514439225196838,
+      "rewards/rejected": -2.135268211364746,
       "step": 280
     },
     {
       "epoch": 0.61,
       "learning_rate": 1.1999614593359337e-06,
-      "logits/chosen": -0.5130027532577515,
-      "logits/rejected": -0.4168172776699066,
-      "logps/chosen": -451.7655334472656,
-      "logps/rejected": -525.0120849609375,
-      "loss": 0.5041,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.0533440113067627,
-      "rewards/margins": 0.9688435792922974,
-      "rewards/rejected": -2.0221877098083496,
+      "logits/chosen": -0.2246803492307663,
+      "logits/rejected": -0.19805817306041718,
+      "logps/chosen": -426.7688903808594,
+      "logps/rejected": -472.63134765625,
+      "loss": 0.0407,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.2982797622680664,
+      "rewards/margins": 0.6589063405990601,
+      "rewards/rejected": -1.9571861028671265,
       "step": 290
     },
     {
       "epoch": 0.63,
       "learning_rate": 1.0932357971453745e-06,
-      "logits/chosen": -0.44245871901512146,
-      "logits/rejected": -0.49341917037963867,
-      "logps/chosen": -408.30517578125,
-      "logps/rejected": -528.8543701171875,
-      "loss": 0.5201,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.1655023097991943,
-      "rewards/margins": 0.8896160125732422,
-      "rewards/rejected": -2.0551185607910156,
+      "logits/chosen": -0.28935036063194275,
+      "logits/rejected": -0.16529107093811035,
+      "logps/chosen": -460.61138916015625,
+      "logps/rejected": -460.7236328125,
+      "loss": 0.0401,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.4895880222320557,
+      "rewards/margins": 0.5250975489616394,
+      "rewards/rejected": -2.0146853923797607,
+      "step": 300
+    },
+    {
+      "epoch": 0.63,
+      "eval_logits/chosen": -0.1928369402885437,
+      "eval_logits/rejected": -0.17653387784957886,
+      "eval_logps/chosen": -431.743896484375,
+      "eval_logps/rejected": -528.1251831054688,
+      "eval_loss": 0.04077731445431709,
+      "eval_rewards/accuracies": 0.71484375,
+      "eval_rewards/chosen": -1.2815920114517212,
+      "eval_rewards/margins": 0.8834086656570435,
+      "eval_rewards/rejected": -2.1650009155273438,
+      "eval_runtime": 75.4051,
+      "eval_samples_per_second": 26.523,
+      "eval_steps_per_second": 0.424,
       "step": 300
     },
     {
       "epoch": 0.65,
       "learning_rate": 9.886905230142433e-07,
-      "logits/chosen": -0.4297795295715332,
-      "logits/rejected": -0.5331634283065796,
-      "logps/chosen": -403.584228515625,
-      "logps/rejected": -518.1731567382812,
-      "loss": 0.4976,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.0785239934921265,
-      "rewards/margins": 0.9861253499984741,
-      "rewards/rejected": -2.0646495819091797,
+      "logits/chosen": -0.19448669254779816,
+      "logits/rejected": -0.2653646469116211,
+      "logps/chosen": -470.20654296875,
+      "logps/rejected": -532.0836181640625,
+      "loss": 0.0436,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.4866724014282227,
+      "rewards/margins": 0.6113816499710083,
+      "rewards/rejected": -2.0980541706085205,
       "step": 310
     },
     {
       "epoch": 0.67,
       "learning_rate": 8.868860335206678e-07,
-      "logits/chosen": -0.4119569659233093,
-      "logits/rejected": -0.3172524571418762,
-      "logps/chosen": -416.0768127441406,
-      "logps/rejected": -491.3486328125,
-      "loss": 0.5269,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.004457712173462,
-      "rewards/margins": 1.0070149898529053,
-      "rewards/rejected": -2.011472702026367,
+      "logits/chosen": -0.274607390165329,
+      "logits/rejected": -0.18271437287330627,
+      "logps/chosen": -476.87591552734375,
+      "logps/rejected": -476.43304443359375,
+      "loss": 0.0457,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.2201156616210938,
+      "rewards/margins": 0.4418094754219055,
+      "rewards/rejected": -1.6619250774383545,
       "step": 320
     },
     {
       "epoch": 0.69,
       "learning_rate": 7.883680337481599e-07,
-      "logits/chosen": -0.32898837327957153,
-      "logits/rejected": -0.31912434101104736,
-      "logps/chosen": -407.608642578125,
-      "logps/rejected": -525.1322021484375,
-      "loss": 0.5093,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.0884016752243042,
-      "rewards/margins": 1.0554091930389404,
-      "rewards/rejected": -2.143810987472534,
+      "logits/chosen": -0.17901699244976044,
+      "logits/rejected": -0.19661930203437805,
+      "logps/chosen": -414.8687438964844,
+      "logps/rejected": -532.6778564453125,
+      "loss": 0.0437,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1104294061660767,
+      "rewards/margins": 0.8232589960098267,
+      "rewards/rejected": -1.9336884021759033,
       "step": 330
     },
     {
       "epoch": 0.71,
       "learning_rate": 6.936646121293654e-07,
-      "logits/chosen": -0.4356309771537781,
-      "logits/rejected": -0.5070624351501465,
-      "logps/chosen": -413.86376953125,
-      "logps/rejected": -487.2752990722656,
-      "loss": 0.5201,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -1.2716727256774902,
-      "rewards/margins": 0.9927502870559692,
-      "rewards/rejected": -2.264423131942749,
+      "logits/chosen": -0.21059110760688782,
+      "logits/rejected": -0.22660131752490997,
+      "logps/chosen": -434.880859375,
+      "logps/rejected": -541.180908203125,
+      "loss": 0.0366,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.4544843435287476,
+      "rewards/margins": 0.8150385618209839,
+      "rewards/rejected": -2.2695229053497314,
       "step": 340
     },
     {
       "epoch": 0.73,
       "learning_rate": 6.032834097207889e-07,
-      "logits/chosen": -0.4740826487541199,
-      "logits/rejected": -0.5902237296104431,
-      "logps/chosen": -432.23675537109375,
-      "logps/rejected": -510.72900390625,
-      "loss": 0.4999,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -1.3073467016220093,
-      "rewards/margins": 0.6941970586776733,
-      "rewards/rejected": -2.0015437602996826,
+      "logits/chosen": -0.17869403958320618,
+      "logits/rejected": -0.1880768984556198,
+      "logps/chosen": -464.3429260253906,
+      "logps/rejected": -565.3632202148438,
+      "loss": 0.0355,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5690315961837769,
+      "rewards/margins": 0.8189191818237305,
+      "rewards/rejected": -2.387950897216797,
       "step": 350
     },
     {
       "epoch": 0.75,
       "learning_rate": 5.177088990820725e-07,
-      "logits/chosen": -0.5474333167076111,
-      "logits/rejected": -0.6527922749519348,
-      "logps/chosen": -466.8818359375,
-      "logps/rejected": -520.4395751953125,
-      "loss": 0.4961,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.3872454166412354,
-      "rewards/margins": 0.9953225255012512,
-      "rewards/rejected": -2.382567882537842,
+      "logits/chosen": -0.2201962023973465,
+      "logits/rejected": -0.17389468848705292,
+      "logps/chosen": -494.4137268066406,
+      "logps/rejected": -554.2338256835938,
+      "loss": 0.0348,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.467558741569519,
+      "rewards/margins": 0.9708374738693237,
+      "rewards/rejected": -2.4383959770202637,
       "step": 360
     },
     {
       "epoch": 0.77,
       "learning_rate": 4.3739978734594494e-07,
-      "logits/chosen": -0.5284720659255981,
-      "logits/rejected": -0.6164287328720093,
-      "logps/chosen": -490.7682189941406,
-      "logps/rejected": -546.9007568359375,
-      "loss": 0.5016,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.4233677387237549,
-      "rewards/margins": 0.9394583702087402,
-      "rewards/rejected": -2.362825870513916,
+      "logits/chosen": -0.2049982100725174,
+      "logits/rejected": -0.13553449511528015,
+      "logps/chosen": -510.56610107421875,
+      "logps/rejected": -544.7944946289062,
+      "loss": 0.0343,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.5760037899017334,
+      "rewards/margins": 0.8068605661392212,
+      "rewards/rejected": -2.382864236831665,
       "step": 370
     },
     {
       "epoch": 0.8,
       "learning_rate": 3.627865573992087e-07,
-      "logits/chosen": -0.33108243346214294,
-      "logits/rejected": -0.3937589228153229,
-      "logps/chosen": -439.3453063964844,
-      "logps/rejected": -516.029052734375,
-      "loss": 0.4983,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.462720513343811,
-      "rewards/margins": 0.9757269024848938,
-      "rewards/rejected": -2.4384474754333496,
+      "logits/chosen": -0.23506097495555878,
+      "logits/rejected": -0.1945139467716217,
+      "logps/chosen": -473.996337890625,
+      "logps/rejected": -546.2703857421875,
+      "loss": 0.0411,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.4194319248199463,
+      "rewards/margins": 0.9198926091194153,
+      "rewards/rejected": -2.339324474334717,
       "step": 380
     },
     {
       "epoch": 0.82,
       "learning_rate": 2.9426916035484166e-07,
-      "logits/chosen": -0.46997910737991333,
-      "logits/rejected": -0.38429608941078186,
-      "logps/chosen": -473.94805908203125,
-      "logps/rejected": -530.4544067382812,
-      "loss": 0.4991,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.322566032409668,
-      "rewards/margins": 0.9721213579177856,
-      "rewards/rejected": -2.294687509536743,
+      "logits/chosen": -0.18075546622276306,
+      "logits/rejected": -0.12812264263629913,
+      "logps/chosen": -451.8818359375,
+      "logps/rejected": -511.1201171875,
+      "loss": 0.0428,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.2848039865493774,
+      "rewards/margins": 0.7686089277267456,
+      "rewards/rejected": -2.053412675857544,
       "step": 390
     },
     {
       "epoch": 0.84,
       "learning_rate": 2.322148716843081e-07,
-      "logits/chosen": -0.40605801343917847,
-      "logits/rejected": -0.407021701335907,
-      "logps/chosen": -454.92510986328125,
-      "logps/rejected": -520.9769287109375,
-      "loss": 0.5107,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.3792002201080322,
-      "rewards/margins": 0.8826921582221985,
-      "rewards/rejected": -2.261892557144165,
+      "logits/chosen": -0.15861022472381592,
+      "logits/rejected": -0.1434582620859146,
+      "logps/chosen": -434.2018127441406,
+      "logps/rejected": -514.0694580078125,
+      "loss": 0.0435,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.3711330890655518,
+      "rewards/margins": 0.6456871032714844,
+      "rewards/rejected": -2.016820192337036,
+      "step": 400
+    },
+    {
+      "epoch": 0.84,
+      "eval_logits/chosen": -0.08009739220142365,
+      "eval_logits/rejected": -0.07131167501211166,
+      "eval_logps/chosen": -420.7032470703125,
+      "eval_logps/rejected": -517.1834106445312,
+      "eval_loss": 0.04270826652646065,
+      "eval_rewards/accuracies": 0.7265625,
+      "eval_rewards/chosen": -1.1711856126785278,
+      "eval_rewards/margins": 0.8843976259231567,
+      "eval_rewards/rejected": -2.0555832386016846,
+      "eval_runtime": 75.7653,
+      "eval_samples_per_second": 26.397,
+      "eval_steps_per_second": 0.422,
       "step": 400
     },
     {
       "epoch": 0.86,
       "learning_rate": 1.7695632250191002e-07,
-      "logits/chosen": -0.5159912705421448,
-      "logits/rejected": -0.6516915559768677,
-      "logps/chosen": -450.1809997558594,
-      "logps/rejected": -543.54443359375,
-      "loss": 0.5087,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.6098343133926392,
-      "rewards/margins": 0.9195775985717773,
-      "rewards/rejected": -2.529412031173706,
+      "logits/chosen": -0.16302308440208435,
+      "logits/rejected": -0.1545868217945099,
+      "logps/chosen": -431.54083251953125,
+      "logps/rejected": -492.18975830078125,
+      "loss": 0.0419,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1856681108474731,
+      "rewards/margins": 0.7893311381340027,
+      "rewards/rejected": -1.9749990701675415,
       "step": 410
     },
     {
       "epoch": 0.88,
       "learning_rate": 1.2878971655412515e-07,
-      "logits/chosen": -0.3541991412639618,
-      "logits/rejected": -0.33743682503700256,
-      "logps/chosen": -428.7999572753906,
-      "logps/rejected": -509.7220764160156,
-      "loss": 0.4935,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.3546617031097412,
-      "rewards/margins": 0.8860130310058594,
-      "rewards/rejected": -2.2406744956970215,
+      "logits/chosen": -0.15071699023246765,
+      "logits/rejected": -0.1457306295633316,
+      "logps/chosen": -424.42547607421875,
+      "logps/rejected": -474.6815490722656,
+      "loss": 0.0388,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.1596481800079346,
+      "rewards/margins": 0.791610836982727,
+      "rewards/rejected": -1.9512590169906616,
       "step": 420
     },
     {
       "epoch": 0.9,
       "learning_rate": 8.797324247145411e-08,
-      "logits/chosen": -0.394558846950531,
-      "logits/rejected": -0.43544620275497437,
-      "logps/chosen": -492.07830810546875,
-      "logps/rejected": -571.8787231445312,
-      "loss": 0.4874,
+      "logits/chosen": -0.16887474060058594,
+      "logits/rejected": -0.11855659633874893,
+      "logps/chosen": -482.5933532714844,
+      "logps/rejected": -524.8614501953125,
+      "loss": 0.0433,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.5800143480300903,
-      "rewards/margins": 0.9009556770324707,
-      "rewards/rejected": -2.4809699058532715,
+      "rewards/chosen": -1.3214952945709229,
+      "rewards/margins": 0.7315466403961182,
+      "rewards/rejected": -2.053041934967041,
       "step": 430
     },
     {
       "epoch": 0.92,
       "learning_rate": 5.472568979361853e-08,
-      "logits/chosen": -0.42609256505966187,
-      "logits/rejected": -0.4700396656990051,
-      "logps/chosen": -444.7997131347656,
-      "logps/rejected": -520.122802734375,
-      "loss": 0.4953,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.6024255752563477,
-      "rewards/margins": 0.8210307359695435,
-      "rewards/rejected": -2.4234564304351807,
+      "logits/chosen": -0.1303589642047882,
+      "logits/rejected": -0.1282893717288971,
+      "logps/chosen": -428.30877685546875,
+      "logps/rejected": -485.59185791015625,
+      "loss": 0.0406,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.3397411108016968,
+      "rewards/margins": 0.8013665080070496,
+      "rewards/rejected": -2.1411080360412598,
       "step": 440
     },
     {
       "epoch": 0.94,
       "learning_rate": 2.922527618666465e-08,
-      "logits/chosen": -0.41642627120018005,
-      "logits/rejected": -0.3031079173088074,
-      "logps/chosen": -515.36328125,
-      "logps/rejected": -556.2701416015625,
-      "loss": 0.4941,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.654118537902832,
-      "rewards/margins": 1.0690773725509644,
-      "rewards/rejected": -2.723196029663086,
+      "logits/chosen": -0.20207130908966064,
+      "logits/rejected": -0.07939404994249344,
+      "logps/chosen": -461.0880432128906,
+      "logps/rejected": -495.314453125,
+      "loss": 0.0414,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.184661626815796,
+      "rewards/margins": 0.7377935647964478,
+      "rewards/rejected": -1.922455072402954,
       "step": 450
     },
     {
       "epoch": 0.96,
       "learning_rate": 1.1608692138469379e-08,
-      "logits/chosen": -0.33327412605285645,
-      "logits/rejected": -0.2631693482398987,
-      "logps/chosen": -445.74749755859375,
-      "logps/rejected": -515.1430053710938,
-      "loss": 0.4794,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.4780994653701782,
-      "rewards/margins": 1.015770673751831,
-      "rewards/rejected": -2.4938700199127197,
+      "logits/chosen": -0.18077249825000763,
+      "logits/rejected": -0.11532274633646011,
+      "logps/chosen": -430.862060546875,
+      "logps/rejected": -464.39385986328125,
+      "loss": 0.0406,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.2867480516433716,
+      "rewards/margins": 0.6147353053092957,
+      "rewards/rejected": -1.9014835357666016,
       "step": 460
     },
     {
       "epoch": 0.98,
       "learning_rate": 1.970368253390198e-09,
-      "logits/chosen": -0.4105471968650818,
-      "logits/rejected": -0.41671761870384216,
-      "logps/chosen": -492.55499267578125,
-      "logps/rejected": -575.6117553710938,
-      "loss": 0.5018,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.5340585708618164,
-      "rewards/margins": 1.2472325563430786,
-      "rewards/rejected": -2.7812907695770264,
+      "logits/chosen": -0.21146085858345032,
+      "logits/rejected": -0.20344269275665283,
+      "logps/chosen": -457.4112243652344,
+      "logps/rejected": -528.634521484375,
+      "loss": 0.039,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.1956093311309814,
+      "rewards/margins": 0.7480987906455994,
+      "rewards/rejected": -1.943708062171936,
       "step": 470
     },
     {
       "epoch": 1.0,
       "step": 477,
       "total_flos": 0.0,
-      "train_loss": 0.538863022122493,
-      "train_runtime": 6331.0486,
-      "train_samples_per_second": 9.656,
-      "train_steps_per_second": 0.075
+      "train_loss": 0.05794826206171288,
+      "train_runtime": 4397.3222,
+      "train_samples_per_second": 13.903,
+      "train_steps_per_second": 0.108
     }
   ],
   "logging_steps": 10,
   "max_steps": 477,
   "num_train_epochs": 1,
-  "save_steps": 10000,
+  "save_steps": 1000,
   "total_flos": 0.0,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d49649ff43e3f71fd7982cbd69b1edd758792760609021ef8b73accbda36f4e0
+oid sha256:e48325f90141d2baab96e119b555cd2507689a536047cb496312be1a6efb1d06
 size 6648