Model save

README.md

@@ -15,15 +15,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.2511
-- Rewards/chosen: 14.1512
-- Rewards/rejected: -27.0299
-- Rewards/accuracies: 0.9297
-- Rewards/margins: 41.1811
-- Logps/rejected: -120.2706
-- Logps/chosen: -123.6211
-- Logits/rejected: -1.8742
-- Logits/chosen: -1.8698
+- Loss: 0.1420
+- Rewards/chosen: 5.3389
+- Rewards/rejected: -11.5415
+- Rewards/accuracies: 0.9258
+- Rewards/margins: 16.8803
+- Logps/rejected: -131.7123
+- Logps/chosen: -119.9761
+- Logits/rejected: -1.8130
+- Logits/chosen: -1.8134
 
 ## Model description
 
@@ -60,10 +60,10 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.1812        | 0.21  | 100  | 0.1474          | 12.7707        | -20.3277         | 0.9180             | 33.0984         | -113.5685      | -125.0015    | -1.7088         | -1.7301       |
-| 0.2958        | 0.42  | 200  | 0.2224          | 15.4746        | -23.1680         | 0.9258             | 38.6426         | -116.4087      | -122.2977    | -1.8350         | -1.8384       |
-| 0.3034        | 0.63  | 300  | 0.2672          | 14.1732        | -27.0300         | 0.9258             | 41.2032         | -120.2707      | -123.5991    | -1.8525         | -1.8496       |
-| 0.3576        | 0.84  | 400  | 0.2511          | 14.1512        | -27.0299         | 0.9297             | 41.1811         | -120.2706      | -123.6211    | -1.8742         | -1.8698       |
+| 0.1496        | 0.21  | 100  | 0.1356          | 4.1320         | -11.2810         | 0.9414             | 15.4129         | -130.8439      | -123.9990    | -1.7910         | -1.8010       |
+| 0.1795        | 0.42  | 200  | 0.1364          | 5.2675         | -11.0420         | 0.9336             | 16.3095         | -130.0476      | -120.2140    | -1.8607         | -1.8614       |
+| 0.1585        | 0.63  | 300  | 0.1425          | 5.1387         | -11.7029         | 0.9258             | 16.8416         | -132.2504      | -120.6432    | -1.7960         | -1.7980       |
+| 0.2005        | 0.84  | 400  | 0.1420          | 5.3389         | -11.5415         | 0.9258             | 16.8803         | -131.7123      | -119.9761    | -1.8130         | -1.8134       |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.2831452648509995,
-    "train_runtime": 7636.09,
+    "train_loss": 0.17749474911510196,
+    "train_runtime": 7645.2484,
     "train_samples": 61135,
-    "train_samples_per_second": 8.006,
+    "train_samples_per_second": 7.996,
     "train_steps_per_second": 0.063
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d9e87c778dd07bc5353b48ab0b1622d62c7f80b637a92d6e104e12a4c096d03
+oid sha256:5505459ad98890a3203705be3e3b4413c1fdb1bed021f564a6052d2b7286ba53
 size 4943178720

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d0e47ee8bdc7394a3c4fff74798dc65703da5a17a7fc91f399e0b692b0c5d987
+oid sha256:f9cd4bbebcca95695bc07db64291f4364758bb91a1c6301d838900b964109691
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a7077d5704ebaba1c41a0334618ab1a9bcc430ac4adbc6d5b16cb3a6fdfddc60
+oid sha256:6603cc6955b683f2147a8107dd2423db37551bb18757806d3089f527c647290d
 size 4540532728

runs/May13_09-51-11_n136-129-074/events.out.tfevents.1715565205.n136-129-074.1804527.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b23f097896970362172141dbd46840a3bc9126f969a3f20bf6441135e008f6da
-size 35913
+oid sha256:e33ac5f6209d31be001170e36d34bbece7451aa9ab83dd828bb1e3deff048718
+size 41083

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.2831452648509995,
-    "train_runtime": 7636.09,
+    "train_loss": 0.17749474911510196,
+    "train_runtime": 7645.2484,
     "train_samples": 61135,
-    "train_samples_per_second": 8.006,
+    "train_samples_per_second": 7.996,
     "train_steps_per_second": 0.063
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 4387.344432836715,
+      "grad_norm": 1316.2617480695828,
       "learning_rate": 1.0416666666666666e-08,
       "logits/chosen": -1.689455509185791,
       "logits/rejected": -1.4794573783874512,
@@ -25,780 +25,780 @@
     },
     {
       "epoch": 0.02,
-      "grad_norm": 2997.186248490448,
+      "grad_norm": 1084.7724692148897,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -1.7082680463790894,
-      "logits/rejected": -1.610369324684143,
-      "logps/chosen": -139.5821990966797,
-      "logps/rejected": -91.33868408203125,
-      "loss": 0.7371,
-      "rewards/accuracies": 0.5138888955116272,
-      "rewards/chosen": 0.14069372415542603,
-      "rewards/margins": 0.1832776665687561,
-      "rewards/rejected": -0.04258394241333008,
+      "logits/chosen": -1.707624912261963,
+      "logits/rejected": -1.6101186275482178,
+      "logps/chosen": -139.66224670410156,
+      "logps/rejected": -91.32621002197266,
+      "loss": 0.686,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.01818913221359253,
+      "rewards/margins": 0.027222516015172005,
+      "rewards/rejected": -0.009033381938934326,
       "step": 10
     },
     {
       "epoch": 0.04,
-      "grad_norm": 978.7690038539965,
+      "grad_norm": 372.5890585979663,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -1.640048623085022,
-      "logits/rejected": -1.6500003337860107,
-      "logps/chosen": -130.82679748535156,
-      "logps/rejected": -93.84379577636719,
-      "loss": 0.3586,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 1.8478235006332397,
-      "rewards/margins": 2.4079792499542236,
-      "rewards/rejected": -0.5601558685302734,
+      "logits/chosen": -1.6384038925170898,
+      "logits/rejected": -1.6487312316894531,
+      "logps/chosen": -130.37515258789062,
+      "logps/rejected": -93.99095153808594,
+      "loss": 0.4495,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 0.6898423433303833,
+      "rewards/margins": 0.9020320177078247,
+      "rewards/rejected": -0.21218962967395782,
       "step": 20
     },
     {
       "epoch": 0.06,
-      "grad_norm": 882.5432957594307,
+      "grad_norm": 363.4016752055454,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -1.7016046047210693,
-      "logits/rejected": -1.6314153671264648,
-      "logps/chosen": -132.51332092285156,
-      "logps/rejected": -104.1015625,
-      "loss": 0.2002,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 5.684301853179932,
-      "rewards/margins": 8.036184310913086,
-      "rewards/rejected": -2.3518824577331543,
+      "logits/chosen": -1.6896642446517944,
+      "logits/rejected": -1.6273959875106812,
+      "logps/chosen": -130.80935668945312,
+      "logps/rejected": -106.37054443359375,
+      "loss": 0.2556,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 2.216484785079956,
+      "rewards/margins": 3.602745771408081,
+      "rewards/rejected": -1.386260986328125,
       "step": 30
     },
     {
       "epoch": 0.08,
-      "grad_norm": 948.7428618668862,
+      "grad_norm": 254.07955635631413,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -1.6547832489013672,
-      "logits/rejected": -1.5681570768356323,
-      "logps/chosen": -143.4683074951172,
-      "logps/rejected": -105.14913177490234,
-      "loss": 0.1727,
+      "logits/chosen": -1.6251739263534546,
+      "logits/rejected": -1.5512189865112305,
+      "logps/chosen": -142.7510528564453,
+      "logps/rejected": -113.31219482421875,
+      "loss": 0.1925,
       "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 8.252656936645508,
-      "rewards/margins": 14.83061695098877,
-      "rewards/rejected": -6.577960968017578,
+      "rewards/chosen": 2.6909842491149902,
+      "rewards/margins": 7.113295078277588,
+      "rewards/rejected": -4.422310829162598,
       "step": 40
     },
     {
       "epoch": 0.1,
-      "grad_norm": 1493.2315961148001,
+      "grad_norm": 450.1083647442556,
       "learning_rate": 4.999733114418725e-07,
-      "logits/chosen": -1.6189439296722412,
-      "logits/rejected": -1.639786958694458,
-      "logps/chosen": -126.5509262084961,
-      "logps/rejected": -110.4835433959961,
-      "loss": 0.1918,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 8.523491859436035,
-      "rewards/margins": 19.890926361083984,
-      "rewards/rejected": -11.367437362670898,
+      "logits/chosen": -1.5827839374542236,
+      "logits/rejected": -1.6095731258392334,
+      "logps/chosen": -127.47169494628906,
+      "logps/rejected": -124.7472152709961,
+      "loss": 0.187,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 2.2808213233947754,
+      "rewards/margins": 9.9701509475708,
+      "rewards/rejected": -7.6893310546875,
       "step": 50
     },
     {
       "epoch": 0.13,
-      "grad_norm": 697.7628654861039,
+      "grad_norm": 256.12553340364644,
       "learning_rate": 4.990398100856366e-07,
-      "logits/chosen": -1.726836919784546,
-      "logits/rejected": -1.664571762084961,
-      "logps/chosen": -140.87637329101562,
-      "logps/rejected": -116.43977355957031,
-      "loss": 0.1811,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 10.12385082244873,
-      "rewards/margins": 24.805103302001953,
-      "rewards/rejected": -14.681253433227539,
+      "logits/chosen": -1.6854372024536133,
+      "logits/rejected": -1.6347172260284424,
+      "logps/chosen": -143.6377410888672,
+      "logps/rejected": -138.22506713867188,
+      "loss": 0.1745,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 2.208740711212158,
+      "rewards/margins": 13.148699760437012,
+      "rewards/rejected": -10.939959526062012,
       "step": 60
     },
     {
       "epoch": 0.15,
-      "grad_norm": 851.3584358950797,
+      "grad_norm": 266.6931182384945,
       "learning_rate": 4.967775735898179e-07,
-      "logits/chosen": -1.670240044593811,
-      "logits/rejected": -1.6913667917251587,
-      "logps/chosen": -134.24986267089844,
-      "logps/rejected": -115.22517395019531,
-      "loss": 0.2071,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 10.848904609680176,
-      "rewards/margins": 28.108707427978516,
-      "rewards/rejected": -17.25979995727539,
+      "logits/chosen": -1.6533622741699219,
+      "logits/rejected": -1.6828343868255615,
+      "logps/chosen": -136.77774047851562,
+      "logps/rejected": -139.76748657226562,
+      "loss": 0.1719,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 2.496314525604248,
+      "rewards/margins": 15.036949157714844,
+      "rewards/rejected": -12.540634155273438,
       "step": 70
     },
     {
       "epoch": 0.17,
-      "grad_norm": 521.7442029758571,
+      "grad_norm": 311.4093156505733,
       "learning_rate": 4.931986719649298e-07,
-      "logits/chosen": -1.6372220516204834,
-      "logits/rejected": -1.618297815322876,
-      "logps/chosen": -128.79197692871094,
-      "logps/rejected": -109.78861999511719,
-      "loss": 0.2008,
+      "logits/chosen": -1.6512008905410767,
+      "logits/rejected": -1.642218828201294,
+      "logps/chosen": -129.7277374267578,
+      "logps/rejected": -130.1470489501953,
+      "loss": 0.1638,
       "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 11.66787052154541,
-      "rewards/margins": 27.729543685913086,
-      "rewards/rejected": -16.061674118041992,
+      "rewards/chosen": 3.2196338176727295,
+      "rewards/margins": 14.145663261413574,
+      "rewards/rejected": -10.926031112670898,
       "step": 80
     },
     {
       "epoch": 0.19,
-      "grad_norm": 1335.0772057896347,
+      "grad_norm": 353.08827705610673,
       "learning_rate": 4.883222001996351e-07,
-      "logits/chosen": -1.6502501964569092,
-      "logits/rejected": -1.6660646200180054,
-      "logps/chosen": -136.35589599609375,
-      "logps/rejected": -121.8341064453125,
-      "loss": 0.1828,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 12.854347229003906,
-      "rewards/margins": 32.79566955566406,
-      "rewards/rejected": -19.941320419311523,
+      "logits/chosen": -1.7057621479034424,
+      "logits/rejected": -1.7253615856170654,
+      "logps/chosen": -136.57843017578125,
+      "logps/rejected": -141.9429931640625,
+      "loss": 0.1501,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 3.7895398139953613,
+      "rewards/margins": 15.804595947265625,
+      "rewards/rejected": -12.015056610107422,
       "step": 90
     },
     {
       "epoch": 0.21,
-      "grad_norm": 597.1667877282997,
+      "grad_norm": 173.53718581228586,
       "learning_rate": 4.821741763807186e-07,
-      "logits/chosen": -1.6900157928466797,
-      "logits/rejected": -1.6581432819366455,
-      "logps/chosen": -116.9823989868164,
-      "logps/rejected": -109.75309753417969,
-      "loss": 0.1812,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 10.995382308959961,
-      "rewards/margins": 30.093975067138672,
-      "rewards/rejected": -19.098596572875977,
+      "logits/chosen": -1.7514231204986572,
+      "logits/rejected": -1.7408148050308228,
+      "logps/chosen": -116.7274169921875,
+      "logps/rejected": -125.6600570678711,
+      "loss": 0.1496,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 3.375108242034912,
+      "rewards/margins": 13.876774787902832,
+      "rewards/rejected": -10.501666069030762,
       "step": 100
     },
     {
       "epoch": 0.21,
-      "eval_logits/chosen": -1.730061650276184,
-      "eval_logits/rejected": -1.7088191509246826,
-      "eval_logps/chosen": -125.00153350830078,
-      "eval_logps/rejected": -113.56846618652344,
-      "eval_loss": 0.14742514491081238,
-      "eval_rewards/accuracies": 0.91796875,
-      "eval_rewards/chosen": 12.770716667175293,
-      "eval_rewards/margins": 33.09844970703125,
-      "eval_rewards/rejected": -20.32773208618164,
-      "eval_runtime": 97.6127,
-      "eval_samples_per_second": 20.489,
+      "eval_logits/chosen": -1.8009788990020752,
+      "eval_logits/rejected": -1.790999174118042,
+      "eval_logps/chosen": -123.99901580810547,
+      "eval_logps/rejected": -130.8439178466797,
+      "eval_loss": 0.13556738197803497,
+      "eval_rewards/accuracies": 0.94140625,
+      "eval_rewards/chosen": 4.131972789764404,
+      "eval_rewards/margins": 15.41292953491211,
+      "eval_rewards/rejected": -11.280956268310547,
+      "eval_runtime": 97.6442,
+      "eval_samples_per_second": 20.483,
       "eval_steps_per_second": 0.328,
       "step": 100
     },
     {
       "epoch": 0.23,
-      "grad_norm": 748.8293985115087,
+      "grad_norm": 220.3790872760113,
       "learning_rate": 4.747874028753375e-07,
-      "logits/chosen": -1.6079628467559814,
-      "logits/rejected": -1.6635444164276123,
-      "logps/chosen": -121.47686767578125,
-      "logps/rejected": -118.15767669677734,
-      "loss": 0.1932,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 10.569832801818848,
-      "rewards/margins": 26.553226470947266,
-      "rewards/rejected": -15.983392715454102,
+      "logits/chosen": -1.6608690023422241,
+      "logits/rejected": -1.7199954986572266,
+      "logps/chosen": -120.6917724609375,
+      "logps/rejected": -133.25762939453125,
+      "loss": 0.1546,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 3.4064812660217285,
+      "rewards/margins": 12.731483459472656,
+      "rewards/rejected": -9.325002670288086,
       "step": 110
     },
     {
       "epoch": 0.25,
-      "grad_norm": 685.6845900004148,
+      "grad_norm": 179.12119430014053,
       "learning_rate": 4.662012913161997e-07,
-      "logits/chosen": -1.67098069190979,
-      "logits/rejected": -1.6562950611114502,
-      "logps/chosen": -120.14444732666016,
-      "logps/rejected": -119.04862976074219,
-      "loss": 0.2,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 12.69383716583252,
-      "rewards/margins": 30.399723052978516,
-      "rewards/rejected": -17.705890655517578,
+      "logits/chosen": -1.7208821773529053,
+      "logits/rejected": -1.7148889303207397,
+      "logps/chosen": -118.548583984375,
+      "logps/rejected": -133.46463012695312,
+      "loss": 0.1456,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 4.286909580230713,
+      "rewards/margins": 13.92347240447998,
+      "rewards/rejected": -9.636563301086426,
       "step": 120
     },
     {
       "epoch": 0.27,
-      "grad_norm": 365.3303217685943,
+      "grad_norm": 119.5811123757762,
       "learning_rate": 4.5646165232345103e-07,
-      "logits/chosen": -1.6567928791046143,
-      "logits/rejected": -1.6678619384765625,
-      "logps/chosen": -126.27610778808594,
-      "logps/rejected": -113.00162506103516,
-      "loss": 0.2132,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 14.438420295715332,
-      "rewards/margins": 34.15327835083008,
-      "rewards/rejected": -19.714855194091797,
+      "logits/chosen": -1.688997507095337,
+      "logits/rejected": -1.7153244018554688,
+      "logps/chosen": -124.4625244140625,
+      "logps/rejected": -129.4587860107422,
+      "loss": 0.1477,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 4.8756022453308105,
+      "rewards/margins": 15.727206230163574,
+      "rewards/rejected": -10.851605415344238,
       "step": 130
     },
     {
       "epoch": 0.29,
-      "grad_norm": 927.4815100211999,
+      "grad_norm": 293.67492501764275,
       "learning_rate": 4.456204510851956e-07,
-      "logits/chosen": -1.5695436000823975,
-      "logits/rejected": -1.5368653535842896,
-      "logps/chosen": -123.9181900024414,
-      "logps/rejected": -111.4970474243164,
-      "loss": 0.2485,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 13.650964736938477,
-      "rewards/margins": 32.08143997192383,
-      "rewards/rejected": -18.43047523498535,
+      "logits/chosen": -1.6008217334747314,
+      "logits/rejected": -1.5764399766921997,
+      "logps/chosen": -121.84040832519531,
+      "logps/rejected": -125.70499420166016,
+      "loss": 0.1534,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 4.718628406524658,
+      "rewards/margins": 14.510149955749512,
+      "rewards/rejected": -9.791521072387695,
       "step": 140
     },
     {
       "epoch": 0.31,
-      "grad_norm": 884.1656325190633,
+      "grad_norm": 250.47040288737202,
       "learning_rate": 4.337355301007335e-07,
-      "logits/chosen": -1.6969578266143799,
-      "logits/rejected": -1.722955346107483,
-      "logps/chosen": -121.66493225097656,
-      "logps/rejected": -109.62093353271484,
-      "loss": 0.1799,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 14.053888320922852,
-      "rewards/margins": 33.25988006591797,
-      "rewards/rejected": -19.20599365234375,
+      "logits/chosen": -1.7394781112670898,
+      "logits/rejected": -1.7636123895645142,
+      "logps/chosen": -119.86863708496094,
+      "logps/rejected": -122.49459075927734,
+      "loss": 0.1403,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 4.755049705505371,
+      "rewards/margins": 14.378946304321289,
+      "rewards/rejected": -9.623896598815918,
       "step": 150
     },
     {
       "epoch": 0.33,
-      "grad_norm": 1383.2317309989403,
+      "grad_norm": 394.54185792168863,
       "learning_rate": 4.2087030056579986e-07,
-      "logits/chosen": -1.5813496112823486,
-      "logits/rejected": -1.5495407581329346,
-      "logps/chosen": -129.09225463867188,
-      "logps/rejected": -117.093994140625,
-      "loss": 0.2686,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 12.873576164245605,
-      "rewards/margins": 31.02213478088379,
-      "rewards/rejected": -18.148557662963867,
+      "logits/chosen": -1.6095482110977173,
+      "logits/rejected": -1.5893223285675049,
+      "logps/chosen": -127.0114517211914,
+      "logps/rejected": -128.70870971679688,
+      "loss": 0.1577,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 4.486311435699463,
+      "rewards/margins": 13.415287971496582,
+      "rewards/rejected": -8.928976058959961,
       "step": 160
     },
     {
       "epoch": 0.36,
-      "grad_norm": 1121.824339354007,
+      "grad_norm": 336.28954028277553,
       "learning_rate": 4.070934040463998e-07,
-      "logits/chosen": -1.7786369323730469,
-      "logits/rejected": -1.7519384622573853,
-      "logps/chosen": -124.59269714355469,
-      "logps/rejected": -116.8897933959961,
-      "loss": 0.2735,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 14.548820495605469,
-      "rewards/margins": 35.767005920410156,
-      "rewards/rejected": -21.218185424804688,
+      "logits/chosen": -1.7846260070800781,
+      "logits/rejected": -1.7679131031036377,
+      "logps/chosen": -121.9030990600586,
+      "logps/rejected": -130.14794921875,
+      "loss": 0.1516,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 5.171528339385986,
+      "rewards/margins": 15.514431953430176,
+      "rewards/rejected": -10.342904090881348,
       "step": 170
     },
     {
       "epoch": 0.38,
-      "grad_norm": 1198.59830465361,
+      "grad_norm": 266.7294156199543,
       "learning_rate": 3.9247834624635404e-07,
-      "logits/chosen": -1.7289392948150635,
-      "logits/rejected": -1.7189258337020874,
-      "logps/chosen": -127.54146575927734,
-      "logps/rejected": -112.2616195678711,
-      "loss": 0.2876,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 15.61170482635498,
-      "rewards/margins": 37.00361251831055,
-      "rewards/rejected": -21.391910552978516,
+      "logits/chosen": -1.7311102151870728,
+      "logits/rejected": -1.725064992904663,
+      "logps/chosen": -125.84059143066406,
+      "logps/rejected": -125.44111633300781,
+      "loss": 0.1616,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 5.19378137588501,
+      "rewards/margins": 15.565200805664062,
+      "rewards/rejected": -10.371419906616211,
       "step": 180
     },
     {
       "epoch": 0.4,
-      "grad_norm": 926.4816546126996,
+      "grad_norm": 243.44642169675112,
       "learning_rate": 3.7710310482256523e-07,
-      "logits/chosen": -1.7749055624008179,
-      "logits/rejected": -1.7469419240951538,
-      "logps/chosen": -116.5710678100586,
-      "logps/rejected": -125.67762756347656,
-      "loss": 0.2836,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 15.228490829467773,
-      "rewards/margins": 38.81087112426758,
-      "rewards/rejected": -23.582382202148438,
+      "logits/chosen": -1.764722466468811,
+      "logits/rejected": -1.7476263046264648,
+      "logps/chosen": -114.77116394042969,
+      "logps/rejected": -139.33917236328125,
+      "loss": 0.1684,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 5.108515739440918,
+      "rewards/margins": 16.28169822692871,
+      "rewards/rejected": -11.173181533813477,
       "step": 190
     },
     {
       "epoch": 0.42,
-      "grad_norm": 963.2845441111758,
+      "grad_norm": 258.83830985573707,
       "learning_rate": 3.610497133404795e-07,
-      "logits/chosen": -1.7404663562774658,
-      "logits/rejected": -1.7393602132797241,
-      "logps/chosen": -122.38069152832031,
-      "logps/rejected": -109.40885925292969,
-      "loss": 0.2958,
+      "logits/chosen": -1.7498699426651,
+      "logits/rejected": -1.7545902729034424,
+      "logps/chosen": -120.42120361328125,
+      "logps/rejected": -123.0103988647461,
+      "loss": 0.1795,
       "rewards/accuracies": 0.9375,
-      "rewards/chosen": 16.75569725036621,
-      "rewards/margins": 38.156455993652344,
-      "rewards/rejected": -21.400760650634766,
+      "rewards/chosen": 5.614555835723877,
+      "rewards/margins": 16.115243911743164,
+      "rewards/rejected": -10.500688552856445,
       "step": 200
     },
     {
       "epoch": 0.42,
-      "eval_logits/chosen": -1.8384383916854858,
-      "eval_logits/rejected": -1.8349756002426147,
-      "eval_logps/chosen": -122.29769134521484,
-      "eval_logps/rejected": -116.40873718261719,
-      "eval_loss": 0.22235894203186035,
-      "eval_rewards/accuracies": 0.92578125,
-      "eval_rewards/chosen": 15.474552154541016,
-      "eval_rewards/margins": 38.64255142211914,
-      "eval_rewards/rejected": -23.167999267578125,
-      "eval_runtime": 97.6373,
-      "eval_samples_per_second": 20.484,
+      "eval_logits/chosen": -1.86138117313385,
+      "eval_logits/rejected": -1.8606913089752197,
+      "eval_logps/chosen": -120.21395111083984,
+      "eval_logps/rejected": -130.0475616455078,
+      "eval_loss": 0.13641399145126343,
+      "eval_rewards/accuracies": 0.93359375,
+      "eval_rewards/chosen": 5.267488479614258,
+      "eval_rewards/margins": 16.30953025817871,
+      "eval_rewards/rejected": -11.042043685913086,
+      "eval_runtime": 97.5652,
+      "eval_samples_per_second": 20.499,
       "eval_steps_per_second": 0.328,
       "step": 200
     },
     {
       "epoch": 0.44,
-      "grad_norm": 707.2538341296229,
+      "grad_norm": 258.76531038458563,
       "learning_rate": 3.4440382358952115e-07,
-      "logits/chosen": -1.6704992055892944,
-      "logits/rejected": -1.605607271194458,
-      "logps/chosen": -120.807373046875,
-      "logps/rejected": -107.77888488769531,
-      "loss": 0.2821,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 11.783378601074219,
-      "rewards/margins": 29.960418701171875,
-      "rewards/rejected": -18.17704200744629,
+      "logits/chosen": -1.689432144165039,
+      "logits/rejected": -1.632364273071289,
+      "logps/chosen": -117.7408447265625,
+      "logps/rejected": -117.53926086425781,
+      "loss": 0.1494,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 4.45497989654541,
+      "rewards/margins": 12.836206436157227,
+      "rewards/rejected": -8.381224632263184,
       "step": 210
     },
     {
       "epoch": 0.46,
-      "grad_norm": 601.6467819271398,
+      "grad_norm": 163.97448768931915,
       "learning_rate": 3.272542485937368e-07,
-      "logits/chosen": -1.8005359172821045,
-      "logits/rejected": -1.845910668373108,
-      "logps/chosen": -122.75736236572266,
-      "logps/rejected": -108.84068298339844,
-      "loss": 0.2678,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 13.717930793762207,
-      "rewards/margins": 35.02998733520508,
-      "rewards/rejected": -21.31205177307129,
+      "logits/chosen": -1.8162240982055664,
+      "logits/rejected": -1.8604834079742432,
+      "logps/chosen": -118.64457702636719,
+      "logps/rejected": -118.34297180175781,
+      "loss": 0.1524,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 5.34921407699585,
+      "rewards/margins": 14.593510627746582,
+      "rewards/rejected": -9.244296073913574,
       "step": 220
     },
     {
       "epoch": 0.48,
-      "grad_norm": 474.6819088542097,
+      "grad_norm": 122.96583033580265,
       "learning_rate": 3.096924887558854e-07,
-      "logits/chosen": -1.7543551921844482,
-      "logits/rejected": -1.703619360923767,
-      "logps/chosen": -134.04598999023438,
-      "logps/rejected": -127.0660400390625,
-      "loss": 0.3505,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 13.662841796875,
-      "rewards/margins": 40.915740966796875,
-      "rewards/rejected": -27.252899169921875,
+      "logits/chosen": -1.7572132349014282,
+      "logits/rejected": -1.708581566810608,
+      "logps/chosen": -131.16842651367188,
+      "logps/rejected": -137.84829711914062,
+      "loss": 0.1815,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 4.962122440338135,
+      "rewards/margins": 16.372663497924805,
+      "rewards/rejected": -11.410540580749512,
       "step": 230
     },
     {
       "epoch": 0.5,
-      "grad_norm": 1030.0648597846362,
+      "grad_norm": 300.9054437339817,
       "learning_rate": 2.9181224366319943e-07,
-      "logits/chosen": -1.7851312160491943,
-      "logits/rejected": -1.8008880615234375,
-      "logps/chosen": -120.84779357910156,
-      "logps/rejected": -118.9937515258789,
-      "loss": 0.343,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 11.07734203338623,
-      "rewards/margins": 32.441627502441406,
-      "rewards/rejected": -21.364286422729492,
+      "logits/chosen": -1.768376111984253,
+      "logits/rejected": -1.7853620052337646,
+      "logps/chosen": -118.08064270019531,
+      "logps/rejected": -129.52337646484375,
+      "loss": 0.1638,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 4.15334415435791,
+      "rewards/margins": 13.721521377563477,
+      "rewards/rejected": -9.568175315856934,
       "step": 240
     },
     {
       "epoch": 0.52,
-      "grad_norm": 984.6682547225865,
+      "grad_norm": 279.7087004166346,
       "learning_rate": 2.7370891215954565e-07,
-      "logits/chosen": -1.7966454029083252,
-      "logits/rejected": -1.8319499492645264,
-      "logps/chosen": -124.015869140625,
-      "logps/rejected": -120.4018783569336,
-      "loss": 0.3481,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 13.399293899536133,
-      "rewards/margins": 36.81779098510742,
-      "rewards/rejected": -23.41849708557129,
+      "logits/chosen": -1.7569881677627563,
+      "logits/rejected": -1.7871322631835938,
+      "logps/chosen": -121.43367767333984,
+      "logps/rejected": -130.93099975585938,
+      "loss": 0.1737,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 4.7944416999816895,
+      "rewards/margins": 14.978727340698242,
+      "rewards/rejected": -10.184286117553711,
       "step": 250
     },
     {
       "epoch": 0.54,
-      "grad_norm": 466.5498060764853,
+      "grad_norm": 163.84527157544522,
       "learning_rate": 2.55479083351317e-07,
-      "logits/chosen": -1.8401811122894287,
-      "logits/rejected": -1.8560435771942139,
-      "logps/chosen": -128.3160858154297,
-      "logps/rejected": -107.73912048339844,
-      "loss": 0.2405,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 15.51091480255127,
-      "rewards/margins": 38.373741149902344,
-      "rewards/rejected": -22.862829208374023,
+      "logits/chosen": -1.7968614101409912,
+      "logits/rejected": -1.8222767114639282,
+      "logps/chosen": -125.540283203125,
+      "logps/rejected": -119.97703552246094,
+      "loss": 0.1484,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 5.486014366149902,
+      "rewards/margins": 16.016239166259766,
+      "rewards/rejected": -10.530224800109863,
       "step": 260
     },
     {
       "epoch": 0.56,
-      "grad_norm": 878.144034078672,
+      "grad_norm": 150.71761898213634,
       "learning_rate": 2.3722002126275822e-07,
-      "logits/chosen": -1.8353513479232788,
-      "logits/rejected": -1.7788803577423096,
-      "logps/chosen": -118.90830993652344,
-      "logps/rejected": -113.4903335571289,
-      "loss": 0.3013,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 13.150169372558594,
-      "rewards/margins": 35.406681060791016,
-      "rewards/rejected": -22.256511688232422,
+      "logits/chosen": -1.787398099899292,
+      "logits/rejected": -1.723350167274475,
+      "logps/chosen": -115.22535705566406,
+      "logps/rejected": -123.31414794921875,
+      "loss": 0.139,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 5.049933433532715,
+      "rewards/margins": 14.674034118652344,
+      "rewards/rejected": -9.624099731445312,
       "step": 270
     },
     {
       "epoch": 0.59,
-      "grad_norm": 1244.2545962296354,
+      "grad_norm": 267.05255152770263,
       "learning_rate": 2.19029145890313e-07,
-      "logits/chosen": -1.6921329498291016,
-      "logits/rejected": -1.778611183166504,
-      "logps/chosen": -124.7625732421875,
-      "logps/rejected": -126.3423080444336,
-      "loss": 0.3747,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 12.294075012207031,
-      "rewards/margins": 37.28684997558594,
-      "rewards/rejected": -24.992773056030273,
+      "logits/chosen": -1.6403396129608154,
+      "logits/rejected": -1.728877067565918,
+      "logps/chosen": -121.6760482788086,
+      "logps/rejected": -137.2602996826172,
+      "loss": 0.1837,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 4.614184379577637,
+      "rewards/margins": 15.387414932250977,
+      "rewards/rejected": -10.773229598999023,
       "step": 280
     },
     {
       "epoch": 0.61,
-      "grad_norm": 970.0623194716495,
+      "grad_norm": 355.7773679008215,
       "learning_rate": 2.0100351342479216e-07,
-      "logits/chosen": -1.788028359413147,
-      "logits/rejected": -1.7638485431671143,
-      "logps/chosen": -115.53651428222656,
-      "logps/rejected": -114.30744934082031,
-      "loss": 0.3137,
+      "logits/chosen": -1.7196556329727173,
+      "logits/rejected": -1.7061948776245117,
+      "logps/chosen": -112.95915222167969,
+      "logps/rejected": -124.29833984375,
+      "loss": 0.1648,
       "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 12.6263427734375,
-      "rewards/margins": 37.81734848022461,
-      "rewards/rejected": -25.19100570678711,
+      "rewards/chosen": 4.5611090660095215,
+      "rewards/margins": 15.115681648254395,
+      "rewards/rejected": -10.554571151733398,
       "step": 290
     },
     {
       "epoch": 0.63,
-      "grad_norm": 817.9238785214287,
+      "grad_norm": 387.9028451533635,
       "learning_rate": 1.8323929841460178e-07,
-      "logits/chosen": -1.7443387508392334,
-      "logits/rejected": -1.6937278509140015,
-      "logps/chosen": -133.70358276367188,
-      "logps/rejected": -131.15541076660156,
-      "loss": 0.3034,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 12.736581802368164,
-      "rewards/margins": 37.65189743041992,
-      "rewards/rejected": -24.91531753540039,
+      "logits/chosen": -1.6907150745391846,
+      "logits/rejected": -1.6335220336914062,
+      "logps/chosen": -130.05313110351562,
+      "logps/rejected": -141.36476135253906,
+      "loss": 0.1585,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 4.916111946105957,
+      "rewards/margins": 15.453518867492676,
+      "rewards/rejected": -10.537405967712402,
       "step": 300
     },
     {
       "epoch": 0.63,
-      "eval_logits/chosen": -1.8495898246765137,
-      "eval_logits/rejected": -1.8524861335754395,
-      "eval_logps/chosen": -123.59905242919922,
-      "eval_logps/rejected": -120.27072143554688,
-      "eval_loss": 0.2672339975833893,
+      "eval_logits/chosen": -1.7980220317840576,
+      "eval_logits/rejected": -1.7959610223770142,
+      "eval_logps/chosen": -120.6431655883789,
+      "eval_logps/rejected": -132.25042724609375,
+      "eval_loss": 0.14247241616249084,
       "eval_rewards/accuracies": 0.92578125,
-      "eval_rewards/chosen": 14.173208236694336,
-      "eval_rewards/margins": 41.203189849853516,
-      "eval_rewards/rejected": -27.02998161315918,
-      "eval_runtime": 97.7326,
-      "eval_samples_per_second": 20.464,
-      "eval_steps_per_second": 0.327,
+      "eval_rewards/chosen": 5.138728141784668,
+      "eval_rewards/margins": 16.841632843017578,
+      "eval_rewards/rejected": -11.702906608581543,
+      "eval_runtime": 97.7011,
+      "eval_samples_per_second": 20.471,
+      "eval_steps_per_second": 0.328,
       "step": 300
     },
     {
       "epoch": 0.65,
-      "grad_norm": 1032.5800107949206,
+      "grad_norm": 308.7683125090126,
       "learning_rate": 1.6583128063291573e-07,
-      "logits/chosen": -1.7456867694854736,
-      "logits/rejected": -1.781978964805603,
-      "logps/chosen": -124.9278335571289,
-      "logps/rejected": -121.29362487792969,
-      "loss": 0.3116,
+      "logits/chosen": -1.6941922903060913,
+      "logits/rejected": -1.7201515436172485,
+      "logps/chosen": -121.898193359375,
+      "logps/rejected": -132.00962829589844,
+      "loss": 0.1619,
       "rewards/accuracies": 0.9375,
-      "rewards/chosen": 14.409858703613281,
-      "rewards/margins": 37.164405822753906,
-      "rewards/rejected": -22.75455093383789,
+      "rewards/chosen": 5.231846809387207,
+      "rewards/margins": 15.2730131149292,
+      "rewards/rejected": -10.041168212890625,
       "step": 310
     },
     {
       "epoch": 0.67,
-      "grad_norm": 852.7418560203273,
+      "grad_norm": 208.88097429038612,
       "learning_rate": 1.488723393865766e-07,
-      "logits/chosen": -1.773827314376831,
-      "logits/rejected": -1.742255449295044,
-      "logps/chosen": -114.78021240234375,
-      "logps/rejected": -120.72232818603516,
-      "loss": 0.307,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 16.016681671142578,
-      "rewards/margins": 40.483619689941406,
-      "rewards/rejected": -24.466938018798828,
+      "logits/chosen": -1.7199640274047852,
+      "logits/rejected": -1.6923316717147827,
+      "logps/chosen": -111.40040588378906,
+      "logps/rejected": -131.6649627685547,
+      "loss": 0.154,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 5.818942070007324,
+      "rewards/margins": 16.441822052001953,
+      "rewards/rejected": -10.62287712097168,
       "step": 320
     },
     {
       "epoch": 0.69,
-      "grad_norm": 530.8053526489199,
+      "grad_norm": 167.39144709959334,
       "learning_rate": 1.3245295796480788e-07,
-      "logits/chosen": -1.7004032135009766,
-      "logits/rejected": -1.7918256521224976,
-      "logps/chosen": -125.2526626586914,
-      "logps/rejected": -116.03642272949219,
-      "loss": 0.3323,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 12.930267333984375,
-      "rewards/margins": 35.218624114990234,
-      "rewards/rejected": -22.28835678100586,
+      "logits/chosen": -1.65103018283844,
+      "logits/rejected": -1.7277615070343018,
+      "logps/chosen": -121.6876449584961,
+      "logps/rejected": -127.0235366821289,
+      "loss": 0.1507,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 4.948590278625488,
+      "rewards/margins": 14.931228637695312,
+      "rewards/rejected": -9.982640266418457,
       "step": 330
     },
     {
       "epoch": 0.71,
-      "grad_norm": 574.055283608638,
+      "grad_norm": 156.90849982717606,
       "learning_rate": 1.1666074087171627e-07,
-      "logits/chosen": -1.6694965362548828,
-      "logits/rejected": -1.7466586828231812,
-      "logps/chosen": -119.57568359375,
-      "logps/rejected": -108.32354736328125,
-      "loss": 0.4687,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 10.914609909057617,
-      "rewards/margins": 31.60161781311035,
-      "rewards/rejected": -20.687007904052734,
+      "logits/chosen": -1.602086067199707,
+      "logits/rejected": -1.6703577041625977,
+      "logps/chosen": -115.71211242675781,
+      "logps/rejected": -116.56599426269531,
+      "loss": 0.1908,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 4.433460235595703,
+      "rewards/margins": 13.112295150756836,
+      "rewards/rejected": -8.678834915161133,
       "step": 340
     },
     {
       "epoch": 0.73,
-      "grad_norm": 1566.6302259536992,
+      "grad_norm": 295.768622683131,
       "learning_rate": 1.0157994641835734e-07,
-      "logits/chosen": -1.7077308893203735,
-      "logits/rejected": -1.7222753763198853,
-      "logps/chosen": -119.43708801269531,
-      "logps/rejected": -127.2352523803711,
-      "loss": 0.2776,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 10.647879600524902,
-      "rewards/margins": 36.324337005615234,
-      "rewards/rejected": -25.67645835876465,
+      "logits/chosen": -1.6467043161392212,
+      "logits/rejected": -1.659854531288147,
+      "logps/chosen": -115.77754974365234,
+      "logps/rejected": -137.26531982421875,
+      "loss": 0.1326,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 4.292226314544678,
+      "rewards/margins": 15.004185676574707,
+      "rewards/rejected": -10.711957931518555,
       "step": 350
     },
     {
       "epoch": 0.75,
-      "grad_norm": 891.7532408538094,
+      "grad_norm": 201.50715466732544,
       "learning_rate": 8.729103716819111e-08,
-      "logits/chosen": -1.6924266815185547,
-      "logits/rejected": -1.7151873111724854,
-      "logps/chosen": -125.8857421875,
-      "logps/rejected": -123.62667083740234,
-      "loss": 0.2483,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 12.26460075378418,
-      "rewards/margins": 36.57375717163086,
-      "rewards/rejected": -24.309158325195312,
+      "logits/chosen": -1.6420570611953735,
+      "logits/rejected": -1.663745641708374,
+      "logps/chosen": -122.47066497802734,
+      "logps/rejected": -131.9073486328125,
+      "loss": 0.1543,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 4.703906059265137,
+      "rewards/margins": 14.480855941772461,
+      "rewards/rejected": -9.776951789855957,
       "step": 360
     },
     {
       "epoch": 0.77,
-      "grad_norm": 1484.9700115288094,
+      "grad_norm": 398.4959635020424,
       "learning_rate": 7.387025063449081e-08,
-      "logits/chosen": -1.7990468740463257,
-      "logits/rejected": -1.7503010034561157,
-      "logps/chosen": -115.22508239746094,
-      "logps/rejected": -112.91387939453125,
-      "loss": 0.3162,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 11.164658546447754,
-      "rewards/margins": 31.5644588470459,
-      "rewards/rejected": -20.399805068969727,
+      "logits/chosen": -1.7505977153778076,
+      "logits/rejected": -1.6930338144302368,
+      "logps/chosen": -112.33251953125,
+      "logps/rejected": -121.57230377197266,
+      "loss": 0.1443,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 4.217165470123291,
+      "rewards/margins": 12.93463134765625,
+      "rewards/rejected": -8.7174654006958,
       "step": 370
     },
     {
       "epoch": 0.79,
-      "grad_norm": 567.3331476500654,
+      "grad_norm": 239.63863129657855,
       "learning_rate": 6.138919252022435e-08,
-      "logits/chosen": -1.8380801677703857,
-      "logits/rejected": -1.860713005065918,
-      "logps/chosen": -123.00642395019531,
-      "logps/rejected": -125.6397933959961,
-      "loss": 0.2616,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 13.647308349609375,
-      "rewards/margins": 40.07811737060547,
-      "rewards/rejected": -26.430805206298828,
+      "logits/chosen": -1.7603282928466797,
+      "logits/rejected": -1.7833961248397827,
+      "logps/chosen": -119.14958190917969,
+      "logps/rejected": -136.85977172851562,
+      "loss": 0.1576,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 5.251246452331543,
+      "rewards/margins": 16.546478271484375,
+      "rewards/rejected": -11.295232772827148,
       "step": 380
     },
     {
       "epoch": 0.82,
-      "grad_norm": 1013.5375269262001,
+      "grad_norm": 197.58750581294214,
       "learning_rate": 4.991445467064689e-08,
-      "logits/chosen": -1.717441201210022,
-      "logits/rejected": -1.7240034341812134,
-      "logps/chosen": -118.6935806274414,
-      "logps/rejected": -114.69548034667969,
-      "loss": 0.2438,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 13.80224609375,
-      "rewards/margins": 38.19357681274414,
-      "rewards/rejected": -24.391324996948242,
+      "logits/chosen": -1.656961441040039,
+      "logits/rejected": -1.6608669757843018,
+      "logps/chosen": -115.59519958496094,
+      "logps/rejected": -124.886474609375,
+      "loss": 0.1242,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 5.070186614990234,
+      "rewards/margins": 15.444877624511719,
+      "rewards/rejected": -10.3746919631958,
       "step": 390
     },
     {
       "epoch": 0.84,
-      "grad_norm": 1086.7904609993896,
+      "grad_norm": 266.2162184394133,
       "learning_rate": 3.9507259776993954e-08,
-      "logits/chosen": -1.6992905139923096,
-      "logits/rejected": -1.7782999277114868,
-      "logps/chosen": -121.951171875,
-      "logps/rejected": -115.44358825683594,
-      "loss": 0.3576,
+      "logits/chosen": -1.636885643005371,
+      "logits/rejected": -1.713200330734253,
+      "logps/chosen": -117.88565826416016,
+      "logps/rejected": -124.86863708496094,
+      "loss": 0.2005,
       "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 10.810310363769531,
-      "rewards/margins": 34.673805236816406,
-      "rewards/rejected": -23.86349868774414,
+      "rewards/chosen": 4.46274471282959,
+      "rewards/margins": 14.449310302734375,
+      "rewards/rejected": -9.986566543579102,
       "step": 400
     },
     {
       "epoch": 0.84,
-      "eval_logits/chosen": -1.869842529296875,
-      "eval_logits/rejected": -1.8741588592529297,
-      "eval_logps/chosen": -123.62108612060547,
-      "eval_logps/rejected": -120.2706298828125,
-      "eval_loss": 0.2510662376880646,
-      "eval_rewards/accuracies": 0.9296875,
-      "eval_rewards/chosen": 14.151167869567871,
-      "eval_rewards/margins": 41.181060791015625,
-      "eval_rewards/rejected": -27.029890060424805,
-      "eval_runtime": 97.6822,
-      "eval_samples_per_second": 20.475,
+      "eval_logits/chosen": -1.8133598566055298,
+      "eval_logits/rejected": -1.8130455017089844,
+      "eval_logps/chosen": -119.97606658935547,
+      "eval_logps/rejected": -131.71229553222656,
+      "eval_loss": 0.14202240109443665,
+      "eval_rewards/accuracies": 0.92578125,
+      "eval_rewards/chosen": 5.338851451873779,
+      "eval_rewards/margins": 16.880319595336914,
+      "eval_rewards/rejected": -11.541468620300293,
+      "eval_runtime": 97.6019,
+      "eval_samples_per_second": 20.491,
       "eval_steps_per_second": 0.328,
       "step": 400
     },
     {
       "epoch": 0.86,
-      "grad_norm": 953.3240690024764,
+      "grad_norm": 300.6588760835356,
       "learning_rate": 3.022313472693447e-08,
-      "logits/chosen": -1.792865514755249,
-      "logits/rejected": -1.8273773193359375,
-      "logps/chosen": -132.37612915039062,
-      "logps/rejected": -118.2215805053711,
-      "loss": 0.3428,
+      "logits/chosen": -1.735870599746704,
+      "logits/rejected": -1.7638452053070068,
+      "logps/chosen": -128.670166015625,
+      "logps/rejected": -129.5255126953125,
+      "loss": 0.1508,
       "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 14.9389009475708,
-      "rewards/margins": 41.05461502075195,
-      "rewards/rejected": -26.115713119506836,
+      "rewards/chosen": 5.593460559844971,
+      "rewards/margins": 16.819358825683594,
+      "rewards/rejected": -11.225897789001465,
       "step": 410
     },
     {
       "epoch": 0.88,
-      "grad_norm": 497.3107054185315,
+      "grad_norm": 184.26802651594184,
       "learning_rate": 2.2111614344599684e-08,
-      "logits/chosen": -1.7695974111557007,
-      "logits/rejected": -1.768599271774292,
-      "logps/chosen": -116.97047424316406,
-      "logps/rejected": -114.58748626708984,
-      "loss": 0.2612,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 11.339197158813477,
-      "rewards/margins": 35.26213455200195,
-      "rewards/rejected": -23.92293357849121,
+      "logits/chosen": -1.7033697366714478,
+      "logits/rejected": -1.704993486404419,
+      "logps/chosen": -113.72819519042969,
+      "logps/rejected": -124.34715270996094,
+      "loss": 0.1419,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 4.3744401931762695,
+      "rewards/margins": 14.479223251342773,
+      "rewards/rejected": -10.104782104492188,
       "step": 420
     },
     {
       "epoch": 0.9,
-      "grad_norm": 978.5564078023309,
+      "grad_norm": 269.33105520831003,
       "learning_rate": 1.521597710086439e-08,
-      "logits/chosen": -1.7172062397003174,
-      "logits/rejected": -1.747591257095337,
-      "logps/chosen": -133.02786254882812,
-      "logps/rejected": -114.73051452636719,
-      "loss": 0.282,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 13.081387519836426,
-      "rewards/margins": 34.326393127441406,
-      "rewards/rejected": -21.245006561279297,
+      "logits/chosen": -1.6633046865463257,
+      "logits/rejected": -1.6779390573501587,
+      "logps/chosen": -129.4523162841797,
+      "logps/rejected": -124.72953796386719,
+      "loss": 0.154,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 4.997079372406006,
+      "rewards/margins": 14.370283126831055,
+      "rewards/rejected": -9.373201370239258,
       "step": 430
     },
     {
       "epoch": 0.92,
-      "grad_norm": 837.4554645359331,
+      "grad_norm": 181.5022378731684,
       "learning_rate": 9.57301420397924e-09,
-      "logits/chosen": -1.8500797748565674,
-      "logits/rejected": -1.8606961965560913,
-      "logps/chosen": -124.95164489746094,
-      "logps/rejected": -119.78662109375,
-      "loss": 0.2435,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 14.02147388458252,
-      "rewards/margins": 40.40496063232422,
-      "rewards/rejected": -26.38348388671875,
+      "logits/chosen": -1.7851531505584717,
+      "logits/rejected": -1.790157675743103,
+      "logps/chosen": -120.8554458618164,
+      "logps/rejected": -131.34410095214844,
+      "loss": 0.144,
+      "rewards/accuracies": 0.96875,
+      "rewards/chosen": 5.435299873352051,
+      "rewards/margins": 16.81759262084961,
+      "rewards/rejected": -11.382290840148926,
       "step": 440
     },
     {
       "epoch": 0.94,
-      "grad_norm": 928.2897315645127,
+      "grad_norm": 227.09412853715097,
       "learning_rate": 5.212833302556258e-09,
-      "logits/chosen": -1.8660519123077393,
-      "logits/rejected": -1.83090078830719,
-      "logps/chosen": -120.15888977050781,
-      "logps/rejected": -115.3041000366211,
-      "loss": 0.2863,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 15.148035049438477,
-      "rewards/margins": 42.080291748046875,
-      "rewards/rejected": -26.9322566986084,
+      "logits/chosen": -1.7998348474502563,
+      "logits/rejected": -1.7618439197540283,
+      "logps/chosen": -116.46590423583984,
+      "logps/rejected": -127.42796325683594,
+      "loss": 0.159,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 5.652300834655762,
+      "rewards/margins": 17.369140625,
+      "rewards/rejected": -11.716839790344238,
       "step": 450
     },
     {
       "epoch": 0.96,
-      "grad_norm": 623.0626054843979,
+      "grad_norm": 225.69247261851913,
       "learning_rate": 2.158697848236607e-09,
-      "logits/chosen": -1.7563292980194092,
-      "logits/rejected": -1.7755203247070312,
-      "logps/chosen": -124.97782897949219,
-      "logps/rejected": -119.69969177246094,
-      "loss": 0.2375,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 14.375999450683594,
-      "rewards/margins": 38.44708251953125,
-      "rewards/rejected": -24.071086883544922,
+      "logits/chosen": -1.6827681064605713,
+      "logits/rejected": -1.7046699523925781,
+      "logps/chosen": -120.79095458984375,
+      "logps/rejected": -130.98043823242188,
+      "loss": 0.1335,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 5.568859100341797,
+      "rewards/margins": 16.174407958984375,
+      "rewards/rejected": -10.605547904968262,
       "step": 460
     },
     {
       "epoch": 0.98,
-      "grad_norm": 652.9309582790959,
+      "grad_norm": 199.48903785359678,
       "learning_rate": 4.269029751107489e-10,
-      "logits/chosen": -1.7278000116348267,
-      "logits/rejected": -1.7499659061431885,
-      "logps/chosen": -118.06231689453125,
-      "logps/rejected": -129.90151977539062,
-      "loss": 0.2885,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 11.649320602416992,
-      "rewards/margins": 37.77583312988281,
-      "rewards/rejected": -26.126514434814453,
+      "logits/chosen": -1.6656444072723389,
+      "logits/rejected": -1.6796722412109375,
+      "logps/chosen": -114.7574462890625,
+      "logps/rejected": -139.92117309570312,
+      "loss": 0.1514,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 4.486257553100586,
+      "rewards/margins": 15.330111503601074,
+      "rewards/rejected": -10.843853950500488,
       "step": 470
     },
     {
       "epoch": 1.0,
       "step": 478,
       "total_flos": 0.0,
-      "train_loss": 0.2831452648509995,
-      "train_runtime": 7636.09,
-      "train_samples_per_second": 8.006,
+      "train_loss": 0.17749474911510196,
+      "train_runtime": 7645.2484,
+      "train_samples_per_second": 7.996,
       "train_steps_per_second": 0.063
     }
   ],