Model save

README.md

@@ -1,16 +1,10 @@
 ---
+license: apache-2.0
 base_model: nnheui/pythia-1.4b-sft-full
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
-- trl
-- dpo
-- alignment-handbook
-- generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: pythia-1.4b-dpo-full
   results: []
@@ -21,19 +15,21 @@ should probably proofread and complete it, then remove this comment. -->
 
 # pythia-1.4b-dpo-full
 
-This model is a fine-tuned version of [nnheui/pythia-1.4b-sft-full](https://huggingface.co/nnheui/pythia-1.4b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [nnheui/pythia-1.4b-sft-full](https://huggingface.co/nnheui/pythia-1.4b-sft-full) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6257
+- Loss: 0.6259
 - Rewards/chosen: -0.5234
-- Rewards/rejected: -0.7812
-- Rewards/accuracies: 0.6597
-- Rewards/margins: 0.2578
+- Rewards/rejected: -0.7852
+- Rewards/accuracies: 0.6567
+- Rewards/margins: 0.2617
 - Logps/rejected: -416.0
 - Logps/chosen: -446.0
 - Logits/rejected: -1.2422
 - Logits/chosen: -1.1953
-- Logps/chosen Bottom Tokens: -0.0007
-- Logps/rejected Bottom Tokens: -0.0007
+- Logps/chosen Top Tokens: -0.0007
+- Logps/rejected Top Tokens: -0.0007
+- Logps/chosen Bottom Tokens: -14.375
+- Logps/rejected Bottom Tokens: -14.3125
 
 ## Model description
 
@@ -68,13 +64,13 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch  | Step | Logits/chosen | Logits/rejected | Logps/bottom Tokens | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
-|:-------------:|:------:|:----:|:-------------:|:---------------:|:-------------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
-| 0.678         | 0.1963 | 100  | -1.0938       | -1.1562         | -0.0009             | -396.0       | -344.0         | 0.6789          | 0.5881             | -0.0275        | 0.0332          | -0.0608          |
-| 0.645         | 0.3925 | 200  | -1.1562       | -1.2031         | -0.0009             | -422.0       | -380.0         | 0.6489          | 0.6448             | -0.2871        | 0.1367          | -0.4238          |
-| 0.6396        | 0.5888 | 300  | -1.1875       | -1.2344         | -0.0008             | -438.0       | -406.0         | 0.6304          | 0.6627             | -0.4512        | 0.2275          | -0.6797          |
-| 0.6102        | 0.7851 | 400  | -1.1875       | -1.2344         | -0.0007             | -444.0       | -414.0         | 0.6268          | 0.6567             | -0.5039        | 0.2578          | -0.7617          |
-| 0.6084        | 0.9814 | 500  | -1.1953       | -1.2422         | -0.0007             | -446.0       | -416.0         | 0.6259          | 0.6567             | -0.5234        | 0.2617          | -0.7852          |
+| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen | Logps/chosen Top Tokens | Logps/rejected Top Tokens | Logps/chosen Bottom Tokens | Logps/rejected Bottom Tokens |
+|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|:-----------------------:|:-------------------------:|:--------------------------:|:----------------------------:|
+| 0.678         | 0.1963 | 100  | 0.6789          | -0.0275        | -0.0608          | 0.5881             | 0.0332          | -344.0         | -396.0       | -1.1562         | -1.0938       | -0.0009                 | -0.0009                   | -14.0625                   | -14.0                        |
+| 0.645         | 0.3925 | 200  | 0.6489          | -0.2871        | -0.4238          | 0.6448             | 0.1367          | -380.0         | -422.0       | -1.2031         | -1.1562       | -0.0009                 | -0.0009                   | -14.375                    | -14.3125                     |
+| 0.6396        | 0.5888 | 300  | 0.6304          | -0.4512        | -0.6797          | 0.6627             | 0.2275          | -406.0         | -438.0       | -1.2344         | -1.1875       | -0.0007                 | -0.0008                   | -14.375                    | -14.3125                     |
+| 0.6102        | 0.7851 | 400  | 0.6268          | -0.5039        | -0.7617          | 0.6567             | 0.2578          | -414.0         | -444.0       | -1.2344         | -1.1875       | -0.0007                 | -0.0007                   | -14.3125                   | -14.25                       |
+| 0.6084        | 0.9814 | 500  | 0.6259          | -0.5234        | -0.7852          | 0.6567             | 0.2617          | -416.0         | -446.0       | -1.2422         | -1.1953       | -0.0007                 | -0.0007                   | -14.375                    | -14.3125                     |
 
 
 ### Framework versions

all_results.json

@@ -1,25 +1,9 @@
 {
     "epoch": 0.9990186457311089,
-    "eval_logits/chosen": -1.1953125,
-    "eval_logits/rejected": -1.2421875,
-    "eval_logps/bottom_tokens": -0.00072479248046875,
-    "eval_logps/chosen": -446.0,
-    "eval_logps/chosen_bottom_tokens": -0.00072479248046875,
-    "eval_logps/rejected": -416.0,
-    "eval_logps/rejected_bottom_tokens": -0.000728607177734375,
-    "eval_loss": 0.6256738305091858,
-    "eval_rewards/accuracies": 0.6597015857696533,
-    "eval_rewards/chosen": -0.5234375,
-    "eval_rewards/margins": 0.2578125,
-    "eval_rewards/rejected": -0.78125,
-    "eval_runtime": 103.3574,
-    "eval_samples": 2000,
-    "eval_samples_per_second": 19.35,
-    "eval_steps_per_second": 0.648,
     "total_flos": 0.0,
-    "train_loss": 0.011100004604616897,
-    "train_runtime": 142.2419,
+    "train_loss": 0.6464882252961105,
+    "train_runtime": 8284.9703,
     "train_samples": 61134,
-    "train_samples_per_second": 429.789,
-    "train_steps_per_second": 3.578
+    "train_samples_per_second": 7.379,
+    "train_steps_per_second": 0.061
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:811b22c69714101dba0a13cf905313113be9cd2fb0da880dd2a6f3f027fb2922
+oid sha256:dee645992f24ee02b486f2e81b344b7a98df284d1f79aa4f2f1679fdd185f99d
 size 2829330208

runs/Jul08_12-10-46_42dbe5cf9ed4/events.out.tfevents.1720441204.42dbe5cf9ed4.850889.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:581c4a7ad45f8f9d52a7cde060b05be03accc097a3afeeaacf2ca6a96d1c0d36
-size 59325
+oid sha256:1b40fca5819f39d3096494f914d8fe42e16c323760be5cc1318d748ca3204d89
+size 59679

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9990186457311089,
     "total_flos": 0.0,
-    "train_loss": 0.011100004604616897,
-    "train_runtime": 142.2419,
+    "train_loss": 0.6464882252961105,
+    "train_runtime": 8284.9703,
     "train_samples": 61134,
-    "train_samples_per_second": 429.789,
-    "train_steps_per_second": 3.578
+    "train_samples_per_second": 7.379,
+    "train_steps_per_second": 0.061
 }

trainer_state.json

@@ -14,9 +14,12 @@
       "learning_rate": 9.803921568627451e-09,
       "logits/chosen": -1.125,
       "logits/rejected": -1.1875,
-      "logps/bottom_tokens": -0.000553131103515625,
       "logps/chosen": -500.0,
+      "logps/chosen_bottom_tokens": -14.5,
+      "logps/chosen_top_tokens": -0.0005645751953125,
       "logps/rejected": -520.0,
+      "logps/rejected_bottom_tokens": -13.9375,
+      "logps/rejected_top_tokens": -0.00054168701171875,
       "loss": 0.6914,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -30,9 +33,12 @@
       "learning_rate": 9.80392156862745e-08,
       "logits/chosen": -1.1640625,
       "logits/rejected": -1.203125,
-      "logps/bottom_tokens": -0.00081634521484375,
       "logps/chosen": -380.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000804901123046875,
       "logps/rejected": -316.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.000827789306640625,
       "loss": 0.6922,
       "rewards/accuracies": 0.41111111640930176,
       "rewards/chosen": -0.000202178955078125,
@@ -46,9 +52,12 @@
       "learning_rate": 1.96078431372549e-07,
       "logits/chosen": -1.0859375,
       "logits/rejected": -1.1484375,
-      "logps/bottom_tokens": -0.0008392333984375,
       "logps/chosen": -374.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000835418701171875,
       "logps/rejected": -324.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.00084686279296875,
       "loss": 0.6913,
       "rewards/accuracies": 0.4399999976158142,
       "rewards/chosen": 0.003662109375,
@@ -62,9 +71,12 @@
       "learning_rate": 2.941176470588235e-07,
       "logits/chosen": -1.125,
       "logits/rejected": -1.1796875,
-      "logps/bottom_tokens": -0.000774383544921875,
       "logps/chosen": -364.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.000762939453125,
       "logps/rejected": -324.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.00078582763671875,
       "loss": 0.6938,
       "rewards/accuracies": 0.3850000202655792,
       "rewards/chosen": -0.0030670166015625,
@@ -78,9 +90,12 @@
       "learning_rate": 3.92156862745098e-07,
       "logits/chosen": -1.125,
       "logits/rejected": -1.15625,
-      "logps/bottom_tokens": -0.000782012939453125,
       "logps/chosen": -378.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.000759124755859375,
       "logps/rejected": -338.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.000804901123046875,
       "loss": 0.6929,
       "rewards/accuracies": 0.445000022649765,
       "rewards/chosen": 0.000881195068359375,
@@ -94,9 +109,12 @@
       "learning_rate": 4.901960784313725e-07,
       "logits/chosen": -1.1171875,
       "logits/rejected": -1.1484375,
-      "logps/bottom_tokens": -0.000789642333984375,
       "logps/chosen": -406.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.000774383544921875,
       "logps/rejected": -352.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.00080108642578125,
       "loss": 0.6915,
       "rewards/accuracies": 0.5400000214576721,
       "rewards/chosen": 0.0026397705078125,
@@ -110,9 +128,12 @@
       "learning_rate": 4.995237599803335e-07,
       "logits/chosen": -1.140625,
       "logits/rejected": -1.203125,
-      "logps/bottom_tokens": -0.000797271728515625,
       "logps/chosen": -406.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.000782012939453125,
       "logps/rejected": -322.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.0008087158203125,
       "loss": 0.6913,
       "rewards/accuracies": 0.42500001192092896,
       "rewards/chosen": 0.00244140625,
@@ -126,9 +147,12 @@
       "learning_rate": 4.978798275112142e-07,
       "logits/chosen": -1.09375,
       "logits/rejected": -1.1328125,
-      "logps/bottom_tokens": -0.00078582763671875,
       "logps/chosen": -372.0,
+      "logps/chosen_bottom_tokens": -14.0625,
+      "logps/chosen_top_tokens": -0.00078582763671875,
       "logps/rejected": -330.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000789642333984375,
       "loss": 0.688,
       "rewards/accuracies": 0.5049999952316284,
       "rewards/chosen": 0.00897216796875,
@@ -142,9 +166,12 @@
       "learning_rate": 4.950700530747689e-07,
       "logits/chosen": -1.078125,
       "logits/rejected": -1.1484375,
-      "logps/bottom_tokens": -0.000904083251953125,
       "logps/chosen": -378.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000934600830078125,
       "logps/rejected": -308.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.00087738037109375,
       "loss": 0.685,
       "rewards/accuracies": 0.5450000166893005,
       "rewards/chosen": -0.00121307373046875,
@@ -158,9 +185,12 @@
       "learning_rate": 4.911076517558622e-07,
       "logits/chosen": -1.125,
       "logits/rejected": -1.15625,
-      "logps/bottom_tokens": -0.000835418701171875,
       "logps/chosen": -382.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000823974609375,
       "logps/rejected": -346.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.00084686279296875,
       "loss": 0.6832,
       "rewards/accuracies": 0.5600000023841858,
       "rewards/chosen": -0.0106201171875,
@@ -174,9 +204,12 @@
       "learning_rate": 4.860112597371772e-07,
       "logits/chosen": -1.125,
       "logits/rejected": -1.171875,
-      "logps/bottom_tokens": -0.0009002685546875,
       "logps/chosen": -372.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000904083251953125,
       "logps/rejected": -328.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.0009002685546875,
       "loss": 0.678,
       "rewards/accuracies": 0.699999988079071,
       "rewards/chosen": -0.017333984375,
@@ -188,17 +221,20 @@
       "epoch": 0.19627085377821393,
       "eval_logits/chosen": -1.09375,
       "eval_logits/rejected": -1.15625,
-      "eval_logps/bottom_tokens": -0.0008697509765625,
       "eval_logps/chosen": -396.0,
+      "eval_logps/chosen_bottom_tokens": -14.0625,
+      "eval_logps/chosen_top_tokens": -0.0008697509765625,
       "eval_logps/rejected": -344.0,
+      "eval_logps/rejected_bottom_tokens": -14.0,
+      "eval_logps/rejected_top_tokens": -0.0008697509765625,
       "eval_loss": 0.6789160370826721,
       "eval_rewards/accuracies": 0.5880597233772278,
       "eval_rewards/chosen": -0.0274658203125,
       "eval_rewards/margins": 0.033203125,
       "eval_rewards/rejected": -0.060791015625,
-      "eval_runtime": 94.4387,
-      "eval_samples_per_second": 21.178,
-      "eval_steps_per_second": 0.709,
+      "eval_runtime": 111.5869,
+      "eval_samples_per_second": 17.923,
+      "eval_steps_per_second": 0.6,
       "step": 100
     },
     {
@@ -207,9 +243,12 @@
       "learning_rate": 4.798048466485017e-07,
       "logits/chosen": -1.1015625,
       "logits/rejected": -1.109375,
-      "logps/bottom_tokens": -0.0008544921875,
       "logps/chosen": -344.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.000835418701171875,
       "logps/rejected": -332.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.000873565673828125,
       "loss": 0.6804,
       "rewards/accuracies": 0.5899999737739563,
       "rewards/chosen": -0.037109375,
@@ -223,9 +262,12 @@
       "learning_rate": 4.725176028314541e-07,
       "logits/chosen": -1.109375,
       "logits/rejected": -1.1171875,
-      "logps/bottom_tokens": -0.0008544921875,
       "logps/chosen": -372.0,
+      "logps/chosen_bottom_tokens": -14.0,
+      "logps/chosen_top_tokens": -0.0008544921875,
       "logps/rejected": -354.0,
+      "logps/rejected_bottom_tokens": -14.0,
+      "logps/rejected_top_tokens": -0.0008544921875,
       "loss": 0.6745,
       "rewards/accuracies": 0.6399999856948853,
       "rewards/chosen": -0.03564453125,
@@ -239,9 +281,12 @@
       "learning_rate": 4.641838020498713e-07,
       "logits/chosen": -1.09375,
       "logits/rejected": -1.1640625,
-      "logps/bottom_tokens": -0.000926971435546875,
       "logps/chosen": -408.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.000934600830078125,
       "logps/rejected": -338.0,
+      "logps/rejected_bottom_tokens": -14.1875,
+      "logps/rejected_top_tokens": -0.00092315673828125,
       "loss": 0.6674,
       "rewards/accuracies": 0.5849999785423279,
       "rewards/chosen": -0.0703125,
@@ -255,9 +300,12 @@
       "learning_rate": 4.5484264029156733e-07,
       "logits/chosen": -1.1015625,
       "logits/rejected": -1.1484375,
-      "logps/bottom_tokens": -0.000823974609375,
       "logps/chosen": -386.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.000812530517578125,
       "logps/rejected": -336.0,
+      "logps/rejected_bottom_tokens": -14.125,
+      "logps/rejected_top_tokens": -0.00083160400390625,
       "loss": 0.6635,
       "rewards/accuracies": 0.5900000333786011,
       "rewards/chosen": -0.1015625,
@@ -271,9 +319,12 @@
       "learning_rate": 4.445380514196192e-07,
       "logits/chosen": -1.09375,
       "logits/rejected": -1.171875,
-      "logps/bottom_tokens": -0.0008697509765625,
       "logps/chosen": -428.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.00087738037109375,
       "logps/rejected": -356.0,
+      "logps/rejected_bottom_tokens": -14.1875,
+      "logps/rejected_top_tokens": -0.000858306884765625,
       "loss": 0.668,
       "rewards/accuracies": 0.6350000500679016,
       "rewards/chosen": -0.12890625,
@@ -287,9 +338,12 @@
       "learning_rate": 4.33318500540218e-07,
       "logits/chosen": -1.0859375,
       "logits/rejected": -1.1328125,
-      "logps/bottom_tokens": -0.000896453857421875,
       "logps/chosen": -408.0,
+      "logps/chosen_bottom_tokens": -14.125,
+      "logps/chosen_top_tokens": -0.00089263916015625,
       "logps/rejected": -368.0,
+      "logps/rejected_bottom_tokens": -14.0625,
+      "logps/rejected_top_tokens": -0.000904083251953125,
       "loss": 0.6655,
       "rewards/accuracies": 0.6050000190734863,
       "rewards/chosen": -0.16796875,
@@ -303,9 +357,12 @@
       "learning_rate": 4.2123675605892985e-07,
       "logits/chosen": -1.078125,
       "logits/rejected": -1.1484375,
-      "logps/bottom_tokens": -0.00099945068359375,
       "logps/chosen": -422.0,
+      "logps/chosen_bottom_tokens": -14.3125,
+      "logps/chosen_top_tokens": -0.00101470947265625,
       "logps/rejected": -364.0,
+      "logps/rejected_bottom_tokens": -14.25,
+      "logps/rejected_top_tokens": -0.00098419189453125,
       "loss": 0.6585,
       "rewards/accuracies": 0.6450000405311584,
       "rewards/chosen": -0.1826171875,
@@ -319,9 +376,12 @@
       "learning_rate": 4.0834964149744333e-07,
       "logits/chosen": -1.1015625,
       "logits/rejected": -1.15625,
-      "logps/bottom_tokens": -0.00096893310546875,
       "logps/chosen": -416.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.00093841552734375,
       "logps/rejected": -380.0,
+      "logps/rejected_bottom_tokens": -14.1875,
+      "logps/rejected_top_tokens": -0.00099945068359375,
       "loss": 0.6643,
       "rewards/accuracies": 0.6100000143051147,
       "rewards/chosen": -0.232421875,
@@ -335,9 +395,12 @@
       "learning_rate": 3.947177682380738e-07,
       "logits/chosen": -1.1796875,
       "logits/rejected": -1.203125,
-      "logps/bottom_tokens": -0.000804901123046875,
       "logps/chosen": -378.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.000789642333984375,
       "logps/rejected": -356.0,
+      "logps/rejected_bottom_tokens": -14.1875,
+      "logps/rejected_top_tokens": -0.000823974609375,
       "loss": 0.6499,
       "rewards/accuracies": 0.6450001001358032,
       "rewards/chosen": -0.2412109375,
@@ -351,9 +414,12 @@
       "learning_rate": 3.804052504529933e-07,
       "logits/chosen": -1.1796875,
       "logits/rejected": -1.2265625,
-      "logps/bottom_tokens": -0.000858306884765625,
       "logps/chosen": -392.0,
+      "logps/chosen_bottom_tokens": -14.375,
+      "logps/chosen_top_tokens": -0.0008392333984375,
       "logps/rejected": -370.0,
+      "logps/rejected_bottom_tokens": -14.375,
+      "logps/rejected_top_tokens": -0.00087738037109375,
       "loss": 0.645,
       "rewards/accuracies": 0.6350000500679016,
       "rewards/chosen": -0.271484375,
@@ -365,17 +431,20 @@
       "epoch": 0.39254170755642787,
       "eval_logits/chosen": -1.15625,
       "eval_logits/rejected": -1.203125,
-      "eval_logps/bottom_tokens": -0.00091552734375,
       "eval_logps/chosen": -422.0,
+      "eval_logps/chosen_bottom_tokens": -14.375,
+      "eval_logps/chosen_top_tokens": -0.000911712646484375,
       "eval_logps/rejected": -380.0,
+      "eval_logps/rejected_bottom_tokens": -14.3125,
+      "eval_logps/rejected_top_tokens": -0.000919342041015625,
       "eval_loss": 0.6488671898841858,
       "eval_rewards/accuracies": 0.6447761058807373,
       "eval_rewards/chosen": -0.287109375,
       "eval_rewards/margins": 0.13671875,
       "eval_rewards/rejected": -0.423828125,
-      "eval_runtime": 94.2281,
-      "eval_samples_per_second": 21.225,
-      "eval_steps_per_second": 0.711,
+      "eval_runtime": 111.5112,
+      "eval_samples_per_second": 17.935,
+      "eval_steps_per_second": 0.601,
       "step": 200
     },
     {
@@ -384,9 +453,12 @@
       "learning_rate": 3.654794035589483e-07,
       "logits/chosen": -1.1328125,
       "logits/rejected": -1.1640625,
-      "logps/bottom_tokens": -0.00092315673828125,
       "logps/chosen": -362.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.00091552734375,
       "logps/rejected": -344.0,
+      "logps/rejected_bottom_tokens": -14.25,
+      "logps/rejected_top_tokens": -0.000926971435546875,
       "loss": 0.6512,
       "rewards/accuracies": 0.6149999499320984,
       "rewards/chosen": -0.298828125,
@@ -400,9 +472,12 @@
       "learning_rate": 3.5001042761570826e-07,
       "logits/chosen": -1.171875,
       "logits/rejected": -1.2109375,
-      "logps/bottom_tokens": -0.000751495361328125,
       "logps/chosen": -414.0,
+      "logps/chosen_bottom_tokens": -14.5,
+      "logps/chosen_top_tokens": -0.000762939453125,
       "logps/rejected": -398.0,
+      "logps/rejected_bottom_tokens": -14.375,
+      "logps/rejected_top_tokens": -0.000743865966796875,
       "loss": 0.6507,
       "rewards/accuracies": 0.5800000429153442,
       "rewards/chosen": -0.333984375,
@@ -416,9 +491,12 @@
       "learning_rate": 3.34071077157304e-07,
       "logits/chosen": -1.171875,
       "logits/rejected": -1.2265625,
-      "logps/bottom_tokens": -0.000789642333984375,
       "logps/chosen": -388.0,
+      "logps/chosen_bottom_tokens": -14.375,
+      "logps/chosen_top_tokens": -0.00075531005859375,
       "logps/rejected": -354.0,
+      "logps/rejected_bottom_tokens": -14.3125,
+      "logps/rejected_top_tokens": -0.000827789306640625,
       "loss": 0.6464,
       "rewards/accuracies": 0.6299999952316284,
       "rewards/chosen": -0.353515625,
@@ -432,9 +510,12 @@
       "learning_rate": 3.1773631900892204e-07,
       "logits/chosen": -1.1484375,
       "logits/rejected": -1.1875,
-      "logps/bottom_tokens": -0.00077056884765625,
       "logps/chosen": -416.0,
+      "logps/chosen_bottom_tokens": -14.375,
+      "logps/chosen_top_tokens": -0.000759124755859375,
       "logps/rejected": -396.0,
+      "logps/rejected_bottom_tokens": -14.375,
+      "logps/rejected_top_tokens": -0.0007781982421875,
       "loss": 0.6442,
       "rewards/accuracies": 0.6200000047683716,
       "rewards/chosen": -0.38671875,
@@ -448,9 +529,12 @@
       "learning_rate": 3.0108297969883103e-07,
       "logits/chosen": -1.1640625,
       "logits/rejected": -1.1953125,
-      "logps/bottom_tokens": -0.000827789306640625,
       "logps/chosen": -426.0,
+      "logps/chosen_bottom_tokens": -14.4375,
+      "logps/chosen_top_tokens": -0.0008392333984375,
       "logps/rejected": -398.0,
+      "logps/rejected_bottom_tokens": -14.375,
+      "logps/rejected_top_tokens": -0.00081634521484375,
       "loss": 0.6347,
       "rewards/accuracies": 0.64000004529953,
       "rewards/chosen": -0.39453125,
@@ -464,9 +548,12 @@
       "learning_rate": 2.8418938412365013e-07,
       "logits/chosen": -1.1640625,
       "logits/rejected": -1.203125,
-      "logps/bottom_tokens": -0.000873565673828125,
       "logps/chosen": -396.0,
+      "logps/chosen_bottom_tokens": -14.3125,
+      "logps/chosen_top_tokens": -0.000865936279296875,
       "logps/rejected": -372.0,
+      "logps/rejected_bottom_tokens": -14.1875,
+      "logps/rejected_top_tokens": -0.000885009765625,
       "loss": 0.6381,
       "rewards/accuracies": 0.6350000500679016,
       "rewards/chosen": -0.373046875,
@@ -480,9 +567,12 @@
       "learning_rate": 2.671349871664101e-07,
       "logits/chosen": -1.1640625,
       "logits/rejected": -1.171875,
-      "logps/bottom_tokens": -0.0007781982421875,
       "logps/chosen": -398.0,
+      "logps/chosen_bottom_tokens": -14.3125,
+      "logps/chosen_top_tokens": -0.000782012939453125,
       "logps/rejected": -386.0,
+      "logps/rejected_bottom_tokens": -14.25,
+      "logps/rejected_top_tokens": -0.00077056884765625,
       "loss": 0.6315,
       "rewards/accuracies": 0.64000004529953,
       "rewards/chosen": -0.40625,
@@ -496,9 +586,12 @@
       "learning_rate": 2.5e-07,
       "logits/chosen": -1.171875,
       "logits/rejected": -1.1953125,
-      "logps/bottom_tokens": -0.000904083251953125,
       "logps/chosen": -438.0,
+      "logps/chosen_bottom_tokens": -14.3125,
+      "logps/chosen_top_tokens": -0.000911712646484375,
       "logps/rejected": -402.0,
+      "logps/rejected_bottom_tokens": -14.25,
+      "logps/rejected_top_tokens": -0.0009002685546875,
       "loss": 0.6384,
       "rewards/accuracies": 0.6600000262260437,
       "rewards/chosen": -0.42578125,
@@ -512,9 +605,12 @@
       "learning_rate": 2.3286501283358982e-07,
       "logits/chosen": -1.1796875,
       "logits/rejected": -1.2421875,
-      "logps/bottom_tokens": -0.00086212158203125,
       "logps/chosen": -412.0,
+      "logps/chosen_bottom_tokens": -14.3125,
+      "logps/chosen_top_tokens": -0.0008392333984375,
       "logps/rejected": -376.0,
+      "logps/rejected_bottom_tokens": -14.25,
+      "logps/rejected_top_tokens": -0.000885009765625,
       "loss": 0.632,
       "rewards/accuracies": 0.5750000476837158,
       "rewards/chosen": -0.4609375,
@@ -528,9 +624,12 @@
       "learning_rate": 2.1581061587634987e-07,
       "logits/chosen": -1.203125,
       "logits/rejected": -1.2421875,
-      "logps/bottom_tokens": -0.000774383544921875,
       "logps/chosen": -428.0,
+      "logps/chosen_bottom_tokens": -14.3125,
+      "logps/chosen_top_tokens": -0.00075531005859375,
       "logps/rejected": -388.0,
+      "logps/rejected_bottom_tokens": -14.3125,
+      "logps/rejected_top_tokens": -0.00079345703125,
       "loss": 0.6396,
       "rewards/accuracies": 0.5999999642372131,
       "rewards/chosen": -0.482421875,
@@ -542,17 +641,20 @@
       "epoch": 0.5888125613346418,
       "eval_logits/chosen": -1.1875,
       "eval_logits/rejected": -1.234375,
-      "eval_logps/bottom_tokens": -0.000751495361328125,
       "eval_logps/chosen": -438.0,
+      "eval_logps/chosen_bottom_tokens": -14.375,
+      "eval_logps/chosen_top_tokens": -0.0007476806640625,
       "eval_logps/rejected": -406.0,
+      "eval_logps/rejected_bottom_tokens": -14.3125,
+      "eval_logps/rejected_top_tokens": -0.000759124755859375,
       "eval_loss": 0.6303857564926147,
       "eval_rewards/accuracies": 0.6626865863800049,
       "eval_rewards/chosen": -0.451171875,
       "eval_rewards/margins": 0.2275390625,
       "eval_rewards/rejected": -0.6796875,
-      "eval_runtime": 94.2601,
-      "eval_samples_per_second": 21.218,
-      "eval_steps_per_second": 0.711,
+      "eval_runtime": 111.5027,
+      "eval_samples_per_second": 17.937,
+      "eval_steps_per_second": 0.601,
       "step": 300
     },
     {
@@ -561,9 +663,12 @@
       "learning_rate": 1.9891702030116897e-07,
       "logits/chosen": -1.140625,
       "logits/rejected": -1.2421875,
-      "logps/bottom_tokens": -0.000728607177734375,
       "logps/chosen": -446.0,
+      "logps/chosen_bottom_tokens": -14.375,
+      "logps/chosen_top_tokens": -0.00074005126953125,
       "logps/rejected": -358.0,
+      "logps/rejected_bottom_tokens": -14.25,
+      "logps/rejected_top_tokens": -0.0007171630859375,
       "loss": 0.6234,
       "rewards/accuracies": 0.6949999928474426,
       "rewards/chosen": -0.408203125,
@@ -577,9 +682,12 @@
       "learning_rate": 1.8226368099107792e-07,
       "logits/chosen": -1.1640625,
       "logits/rejected": -1.2109375,
-      "logps/bottom_tokens": -0.000823974609375,
       "logps/chosen": -424.0,
+      "logps/chosen_bottom_tokens": -14.3125,
+      "logps/chosen_top_tokens": -0.00080108642578125,
       "logps/rejected": -364.0,
+      "logps/rejected_bottom_tokens": -14.25,
+      "logps/rejected_top_tokens": -0.000843048095703125,
       "loss": 0.6241,
       "rewards/accuracies": 0.675000011920929,
       "rewards/chosen": -0.42578125,
@@ -593,9 +701,12 @@
       "learning_rate": 1.6592892284269594e-07,
       "logits/chosen": -1.1796875,
       "logits/rejected": -1.2109375,
-      "logps/bottom_tokens": -0.00072479248046875,
       "logps/chosen": -408.0,
+      "logps/chosen_bottom_tokens": -14.375,
+      "logps/chosen_top_tokens": -0.0007171630859375,
       "logps/rejected": -386.0,
+      "logps/rejected_bottom_tokens": -14.3125,
+      "logps/rejected_top_tokens": -0.000732421875,
       "loss": 0.6224,
       "rewards/accuracies": 0.6799999475479126,
       "rewards/chosen": -0.431640625,
@@ -609,9 +720,12 @@
       "learning_rate": 1.4998957238429172e-07,
       "logits/chosen": -1.21875,
       "logits/rejected": -1.2421875,
-      "logps/bottom_tokens": -0.000789642333984375,
       "logps/chosen": -408.0,
+      "logps/chosen_bottom_tokens": -14.3125,
+      "logps/chosen_top_tokens": -0.000804901123046875,
       "logps/rejected": -380.0,
+      "logps/rejected_bottom_tokens": -14.3125,
+      "logps/rejected_top_tokens": -0.00077056884765625,
       "loss": 0.6204,
       "rewards/accuracies": 0.6300000548362732,
       "rewards/chosen": -0.5078125,
@@ -625,9 +739,12 @@
       "learning_rate": 1.345205964410517e-07,
       "logits/chosen": -1.1953125,
       "logits/rejected": -1.21875,
-      "logps/bottom_tokens": -0.00087738037109375,
       "logps/chosen": -392.0,
+      "logps/chosen_bottom_tokens": -14.375,
+      "logps/chosen_top_tokens": -0.000858306884765625,
       "logps/rejected": -372.0,
+      "logps/rejected_bottom_tokens": -14.3125,
+      "logps/rejected_top_tokens": -0.00089263916015625,
       "loss": 0.627,
       "rewards/accuracies": 0.5850000381469727,
       "rewards/chosen": -0.49609375,
@@ -641,9 +758,12 @@
       "learning_rate": 1.1959474954700665e-07,
       "logits/chosen": -1.1796875,
       "logits/rejected": -1.21875,
-      "logps/bottom_tokens": -0.00067138671875,
       "logps/chosen": -424.0,
+      "logps/chosen_bottom_tokens": -14.375,
+      "logps/chosen_top_tokens": -0.00067138671875,
       "logps/rejected": -416.0,
+      "logps/rejected_bottom_tokens": -14.25,
+      "logps/rejected_top_tokens": -0.000675201416015625,
       "loss": 0.613,
       "rewards/accuracies": 0.7049999833106995,
       "rewards/chosen": -0.44140625,
@@ -657,9 +777,12 @@
       "learning_rate": 1.0528223176192615e-07,
       "logits/chosen": -1.1953125,
       "logits/rejected": -1.234375,
-      "logps/bottom_tokens": -0.0006866455078125,
       "logps/chosen": -442.0,
+      "logps/chosen_bottom_tokens": -14.375,
+      "logps/chosen_top_tokens": -0.00069427490234375,
       "logps/rejected": -398.0,
+      "logps/rejected_bottom_tokens": -14.375,
+      "logps/rejected_top_tokens": -0.00067901611328125,
       "loss": 0.6218,
       "rewards/accuracies": 0.6250000596046448,
       "rewards/chosen": -0.5234375,
@@ -673,9 +796,12 @@
       "learning_rate": 9.16503585025567e-08,
       "logits/chosen": -1.1953125,
       "logits/rejected": -1.21875,
-      "logps/bottom_tokens": -0.0009765625,
       "logps/chosen": -420.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.00090789794921875,
       "logps/rejected": -412.0,
+      "logps/rejected_bottom_tokens": -14.1875,
+      "logps/rejected_top_tokens": -0.00104522705078125,
       "loss": 0.6279,
       "rewards/accuracies": 0.6499999761581421,
       "rewards/chosen": -0.5,
@@ -689,9 +815,12 @@
       "learning_rate": 7.876324394107017e-08,
       "logits/chosen": -1.15625,
       "logits/rejected": -1.203125,
-      "logps/bottom_tokens": -0.0006866455078125,
       "logps/chosen": -442.0,
+      "logps/chosen_bottom_tokens": -14.3125,
+      "logps/chosen_top_tokens": -0.00067901611328125,
       "logps/rejected": -418.0,
+      "logps/rejected_bottom_tokens": -14.25,
+      "logps/rejected_top_tokens": -0.00069427490234375,
       "loss": 0.6289,
       "rewards/accuracies": 0.6350000500679016,
       "rewards/chosen": -0.50390625,
@@ -705,9 +834,12 @@
       "learning_rate": 6.668149945978201e-08,
       "logits/chosen": -1.1953125,
       "logits/rejected": -1.2265625,
-      "logps/bottom_tokens": -0.000720977783203125,
       "logps/chosen": -440.0,
+      "logps/chosen_bottom_tokens": -14.3125,
+      "logps/chosen_top_tokens": -0.00070953369140625,
       "logps/rejected": -420.0,
+      "logps/rejected_bottom_tokens": -14.25,
+      "logps/rejected_top_tokens": -0.000732421875,
       "loss": 0.6102,
       "rewards/accuracies": 0.6700000166893005,
       "rewards/chosen": -0.482421875,
@@ -719,17 +851,20 @@
       "epoch": 0.7850834151128557,
       "eval_logits/chosen": -1.1875,
       "eval_logits/rejected": -1.234375,
-      "eval_logps/bottom_tokens": -0.000667572021484375,
       "eval_logps/chosen": -444.0,
+      "eval_logps/chosen_bottom_tokens": -14.3125,
+      "eval_logps/chosen_top_tokens": -0.00067138671875,
       "eval_logps/rejected": -414.0,
+      "eval_logps/rejected_bottom_tokens": -14.25,
+      "eval_logps/rejected_top_tokens": -0.00066375732421875,
       "eval_loss": 0.6267920136451721,
       "eval_rewards/accuracies": 0.6567164063453674,
       "eval_rewards/chosen": -0.50390625,
       "eval_rewards/margins": 0.2578125,
       "eval_rewards/rejected": -0.76171875,
-      "eval_runtime": 94.2884,
-      "eval_samples_per_second": 21.212,
-      "eval_steps_per_second": 0.711,
+      "eval_runtime": 111.5791,
+      "eval_samples_per_second": 17.925,
+      "eval_steps_per_second": 0.6,
       "step": 400
     },
     {
@@ -738,9 +873,12 @@
       "learning_rate": 5.546194858038072e-08,
       "logits/chosen": -1.171875,
       "logits/rejected": -1.21875,
-      "logps/bottom_tokens": -0.000797271728515625,
       "logps/chosen": -416.0,
+      "logps/chosen_bottom_tokens": -14.3125,
+      "logps/chosen_top_tokens": -0.000766754150390625,
       "logps/rejected": -374.0,
+      "logps/rejected_bottom_tokens": -14.25,
+      "logps/rejected_top_tokens": -0.000827789306640625,
       "loss": 0.6227,
       "rewards/accuracies": 0.6699999570846558,
       "rewards/chosen": -0.51171875,
@@ -754,9 +892,12 @@
       "learning_rate": 4.5157359708432626e-08,
       "logits/chosen": -1.1953125,
       "logits/rejected": -1.2265625,
-      "logps/bottom_tokens": -0.000751495361328125,
       "logps/chosen": -394.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.00075531005859375,
       "logps/rejected": -412.0,
+      "logps/rejected_bottom_tokens": -14.1875,
+      "logps/rejected_top_tokens": -0.0007476806640625,
       "loss": 0.6205,
       "rewards/accuracies": 0.625,
       "rewards/chosen": -0.474609375,
@@ -770,9 +911,12 @@
       "learning_rate": 3.581619795012874e-08,
       "logits/chosen": -1.1796875,
       "logits/rejected": -1.1875,
-      "logps/bottom_tokens": -0.000762939453125,
       "logps/chosen": -400.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.000782012939453125,
       "logps/rejected": -404.0,
+      "logps/rejected_bottom_tokens": -14.1875,
+      "logps/rejected_top_tokens": -0.000743865966796875,
       "loss": 0.6208,
       "rewards/accuracies": 0.675000011920929,
       "rewards/chosen": -0.474609375,
@@ -786,9 +930,12 @@
       "learning_rate": 2.748239716854589e-08,
       "logits/chosen": -1.2109375,
       "logits/rejected": -1.1953125,
-      "logps/bottom_tokens": -0.00074005126953125,
       "logps/chosen": -424.0,
+      "logps/chosen_bottom_tokens": -14.1875,
+      "logps/chosen_top_tokens": -0.000728607177734375,
       "logps/rejected": -420.0,
+      "logps/rejected_bottom_tokens": -14.25,
+      "logps/rejected_top_tokens": -0.000751495361328125,
       "loss": 0.6398,
       "rewards/accuracies": 0.5849999785423279,
       "rewards/chosen": -0.51171875,
@@ -802,9 +949,12 @@
       "learning_rate": 2.0195153351498323e-08,
       "logits/chosen": -1.1796875,
       "logits/rejected": -1.2109375,
-      "logps/bottom_tokens": -0.0007171630859375,
       "logps/chosen": -432.0,
+      "logps/chosen_bottom_tokens": -14.375,
+      "logps/chosen_top_tokens": -0.000732421875,
       "logps/rejected": -420.0,
+      "logps/rejected_bottom_tokens": -14.3125,
+      "logps/rejected_top_tokens": -0.000705718994140625,
       "loss": 0.611,
       "rewards/accuracies": 0.6149999499320984,
       "rewards/chosen": -0.53125,
@@ -818,9 +968,12 @@
       "learning_rate": 1.3988740262822846e-08,
       "logits/chosen": -1.1953125,
       "logits/rejected": -1.203125,
-      "logps/bottom_tokens": -0.00067138671875,
       "logps/chosen": -428.0,
+      "logps/chosen_bottom_tokens": -14.375,
+      "logps/chosen_top_tokens": -0.00067138671875,
       "logps/rejected": -410.0,
+      "logps/rejected_bottom_tokens": -14.3125,
+      "logps/rejected_top_tokens": -0.000667572021484375,
       "loss": 0.6138,
       "rewards/accuracies": 0.6349999904632568,
       "rewards/chosen": -0.490234375,
@@ -834,9 +987,12 @@
       "learning_rate": 8.892348244137788e-09,
       "logits/chosen": -1.1875,
       "logits/rejected": -1.2421875,
-      "logps/bottom_tokens": -0.00066375732421875,
       "logps/chosen": -474.0,
+      "logps/chosen_bottom_tokens": -14.5625,
+      "logps/chosen_top_tokens": -0.000675201416015625,
       "logps/rejected": -444.0,
+      "logps/rejected_bottom_tokens": -14.375,
+      "logps/rejected_top_tokens": -0.000652313232421875,
       "loss": 0.6106,
       "rewards/accuracies": 0.6299999952316284,
       "rewards/chosen": -0.494140625,
@@ -850,9 +1006,12 @@
       "learning_rate": 4.929946925231076e-09,
       "logits/chosen": -1.1328125,
       "logits/rejected": -1.171875,
-      "logps/bottom_tokens": -0.000850677490234375,
       "logps/chosen": -410.0,
+      "logps/chosen_bottom_tokens": -14.375,
+      "logps/chosen_top_tokens": -0.0007476806640625,
       "logps/rejected": -412.0,
+      "logps/rejected_bottom_tokens": -14.25,
+      "logps/rejected_top_tokens": -0.00095367431640625,
       "loss": 0.6203,
       "rewards/accuracies": 0.6049999594688416,
       "rewards/chosen": -0.48046875,
@@ -866,9 +1025,12 @@
       "learning_rate": 2.1201724887858484e-09,
       "logits/chosen": -1.1640625,
       "logits/rejected": -1.171875,
-      "logps/bottom_tokens": -0.000732421875,
       "logps/chosen": -422.0,
+      "logps/chosen_bottom_tokens": -14.25,
+      "logps/chosen_top_tokens": -0.00072479248046875,
       "logps/rejected": -412.0,
+      "logps/rejected_bottom_tokens": -14.1875,
+      "logps/rejected_top_tokens": -0.000743865966796875,
       "loss": 0.6235,
       "rewards/accuracies": 0.5949999690055847,
       "rewards/chosen": -0.5390625,
@@ -882,9 +1044,12 @@
       "learning_rate": 4.762400196664518e-10,
       "logits/chosen": -1.1484375,
       "logits/rejected": -1.1953125,
-      "logps/bottom_tokens": -0.0006256103515625,
       "logps/chosen": -428.0,
+      "logps/chosen_bottom_tokens": -14.375,
+      "logps/chosen_top_tokens": -0.000621795654296875,
       "logps/rejected": -388.0,
+      "logps/rejected_bottom_tokens": -14.3125,
+      "logps/rejected_top_tokens": -0.0006256103515625,
       "loss": 0.6084,
       "rewards/accuracies": 0.625,
       "rewards/chosen": -0.578125,
@@ -896,27 +1061,30 @@
       "epoch": 0.9813542688910697,
       "eval_logits/chosen": -1.1953125,
       "eval_logits/rejected": -1.2421875,
-      "eval_logps/bottom_tokens": -0.000743865966796875,
       "eval_logps/chosen": -446.0,
+      "eval_logps/chosen_bottom_tokens": -14.375,
+      "eval_logps/chosen_top_tokens": -0.000743865966796875,
       "eval_logps/rejected": -416.0,
+      "eval_logps/rejected_bottom_tokens": -14.3125,
+      "eval_logps/rejected_top_tokens": -0.0007476806640625,
       "eval_loss": 0.6259472370147705,
       "eval_rewards/accuracies": 0.6567164659500122,
       "eval_rewards/chosen": -0.5234375,
       "eval_rewards/margins": 0.26171875,
       "eval_rewards/rejected": -0.78515625,
-      "eval_runtime": 94.3436,
-      "eval_samples_per_second": 21.199,
-      "eval_steps_per_second": 0.71,
+      "eval_runtime": 111.4505,
+      "eval_samples_per_second": 17.945,
+      "eval_steps_per_second": 0.601,
       "step": 500
     },
     {
       "epoch": 0.9990186457311089,
       "step": 509,
       "total_flos": 0.0,
-      "train_loss": 0.011100004604616897,
-      "train_runtime": 142.2419,
-      "train_samples_per_second": 429.789,
-      "train_steps_per_second": 3.578
+      "train_loss": 0.6464882252961105,
+      "train_runtime": 8284.9703,
+      "train_samples_per_second": 7.379,
+      "train_steps_per_second": 0.061
     }
   ],
   "logging_steps": 10,