Model save

README.md

@@ -2,7 +2,6 @@
 tags:
 - trl
 - dpo
-- alignment-handbook
 - generated_from_trainer
 model-index:
 - name: zephyr-7b-dpo-full
@@ -16,15 +15,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Logits/chosen: -0.9126
-- Logits/rejected: -0.7601
-- Logps/chosen: -361.4586
-- Logps/rejected: -470.9522
-- Loss: 0.4905
-- Rewards/accuracies: 0.7539
-- Rewards/chosen: -0.7983
-- Rewards/margins: 1.0337
-- Rewards/rejected: -1.8319
+- Loss: 1.8652
+- Rewards/chosen: 125.1359
+- Rewards/rejected: -204.8868
+- Rewards/accuracies: 0.9141
+- Rewards/margins: 330.0227
+- Logps/rejected: -113.7294
+- Logps/chosen: -125.2587
+- Logits/rejected: -1.8708
+- Logits/chosen: -1.8725
 
 ## Model description
 
@@ -59,12 +58,12 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
-|:-------------:|:-----:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
-| 0.5759        | 0.21  | 100  | -1.9151       | -1.8988         | -335.8161    | -395.6910      | 0.5723          | 0.7148             | -0.5418        | 0.5375          | -1.0793          |
-| 0.5391        | 0.42  | 200  | -1.6570       | -1.5774         | -321.1682    | -405.0056      | 0.5138          | 0.7461             | -0.3953        | 0.7771          | -1.1724          |
-| 0.4788        | 0.63  | 300  | -0.9864       | -0.8826         | -367.7277    | -475.5532      | 0.4939          | 0.7578             | -0.8609        | 1.0170          | -1.8779          |
-| 0.4937        | 0.84  | 400  | -0.9126       | -0.7601         | -361.4586    | -470.9522      | 0.4905          | 0.7539             | -0.7983        | 1.0337          | -1.8319          |
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 1.1504        | 0.21  | 100  | 0.9784          | 103.9547       | -116.3378        | 0.9102             | 220.2925        | -104.8745      | -127.3768    | -1.6942         | -1.7242       |
+| 2.8553        | 0.42  | 200  | 1.8849          | 118.2614       | -180.4101        | 0.9102             | 298.6714        | -111.2817      | -125.9461    | -1.8462         | -1.8519       |
+| 2.2897        | 0.63  | 300  | 2.1029          | 127.9046       | -196.9116        | 0.9141             | 324.8163        | -112.9319      | -124.9818    | -1.8642         | -1.8683       |
+| 2.2714        | 0.84  | 400  | 1.8652          | 125.1359       | -204.8868        | 0.9141             | 330.0227        | -113.7294      | -125.2587    | -1.8708         | -1.8725       |
 
 
 ### Framework versions

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -0.8698186874389648,
-    "eval_logits/rejected": -0.7152774930000305,
-    "eval_logps/chosen": -366.0130310058594,
-    "eval_logps/rejected": -477.25433349609375,
-    "eval_loss": 0.48990052938461304,
-    "eval_rewards/accuracies": 0.765625,
-    "eval_rewards/chosen": -0.8437946438789368,
-    "eval_rewards/margins": 1.0511335134506226,
-    "eval_rewards/rejected": -1.894927978515625,
-    "eval_runtime": 97.4253,
-    "eval_samples": 2000,
-    "eval_samples_per_second": 20.529,
-    "eval_steps_per_second": 0.328,
-    "train_loss": 0.07911034258838479,
-    "train_runtime": 1140.8874,
-    "train_samples": 61134,
-    "train_samples_per_second": 53.585,
-    "train_steps_per_second": 0.419
+    "train_loss": 2.0128297995323914,
+    "train_runtime": 7588.5519,
+    "train_samples": 61135,
+    "train_samples_per_second": 8.056,
+    "train_steps_per_second": 0.063
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8993f207919853c6077676c36bf82423d62894d8b5f9f43eb1749a3ca762a15e
+oid sha256:bc91be5bc4e64f8cc3c7d6ab30482d81b7e75dcfe0d795c0d2faa6614fb9916a
 size 4943178720

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cac8f4af54fcab2456b0f93ada8fead3c13b06c0eacdee7245d8ad8692e950bf
+oid sha256:23399768c2f186d099dbf25a818c276f11bd6c402b46b35964c3b41af8d8aed8
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d4c13bcd7d01eec2d330950d0b4cba5bff9d8afbdf18f3ca6d3892559af67e17
+oid sha256:767c51a4eec8ff384baabbf4abb73f66a3893b96d231485e656f7759af483e06
 size 4540532728

runs/May13_00-27-29_n136-129-074/events.out.tfevents.1715531392.n136-129-074.1540539.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e99f027cdbff005e3d44cef3871c8a9622298ff132ffe5e599e17fe45e477850
-size 35909
+oid sha256:1384af147bba702620e92a9f1e4d2ee4900346f5c1d39c78037389b56b149262
+size 41079

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.07911034258838479,
-    "train_runtime": 1140.8874,
-    "train_samples": 61134,
-    "train_samples_per_second": 53.585,
-    "train_steps_per_second": 0.419
+    "train_loss": 2.0128297995323914,
+    "train_runtime": 7588.5519,
+    "train_samples": 61135,
+    "train_samples_per_second": 8.056,
+    "train_steps_per_second": 0.063
 }

trainer_state.json

@@ -10,12 +10,12 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 27.376848082151543,
+      "grad_norm": 43894.48099242753,
       "learning_rate": 1.0416666666666666e-08,
-      "logits/chosen": -1.8783892393112183,
-      "logits/rejected": -1.8756425380706787,
-      "logps/chosen": -298.4870300292969,
-      "logps/rejected": -398.0157165527344,
+      "logits/chosen": -1.689455509185791,
+      "logits/rejected": -1.4794573783874512,
+      "logps/chosen": -126.21005249023438,
+      "logps/rejected": -98.13133239746094,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,781 +25,781 @@
     },
     {
       "epoch": 0.02,
-      "grad_norm": 25.183613892194092,
+      "grad_norm": 32305.118552441847,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -1.7508937120437622,
-      "logits/rejected": -1.7071995735168457,
-      "logps/chosen": -280.5123596191406,
-      "logps/rejected": -271.9100341796875,
-      "loss": 0.6931,
-      "rewards/accuracies": 0.4791666567325592,
-      "rewards/chosen": -0.00033082481240853667,
-      "rewards/margins": 0.0001291988301090896,
-      "rewards/rejected": -0.00046002367162145674,
+      "logits/chosen": -1.7068803310394287,
+      "logits/rejected": -1.6096948385238647,
+      "logps/chosen": -139.68423461914062,
+      "logps/rejected": -91.41385650634766,
+      "loss": 2.648,
+      "rewards/accuracies": 0.5069444179534912,
+      "rewards/chosen": 0.3865443468093872,
+      "rewards/margins": 1.56412935256958,
+      "rewards/rejected": -1.1775851249694824,
       "step": 10
     },
     {
       "epoch": 0.04,
-      "grad_norm": 23.230240384482265,
+      "grad_norm": 12815.76079839475,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -1.832267165184021,
-      "logits/rejected": -1.7261158227920532,
-      "logps/chosen": -298.87811279296875,
-      "logps/rejected": -320.8003845214844,
-      "loss": 0.6919,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": 0.009286092594265938,
-      "rewards/margins": 0.0007404378848150373,
-      "rewards/rejected": 0.008545654825866222,
+      "logits/chosen": -1.6411230564117432,
+      "logits/rejected": -1.6499197483062744,
+      "logps/chosen": -131.1981964111328,
+      "logps/rejected": -93.75257110595703,
+      "loss": 0.8229,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 14.764315605163574,
+      "rewards/margins": 19.453596115112305,
+      "rewards/rejected": -4.68928337097168,
       "step": 20
     },
     {
       "epoch": 0.06,
-      "grad_norm": 22.82438334979188,
+      "grad_norm": 8663.076785137986,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -1.8619463443756104,
-      "logits/rejected": -1.8105783462524414,
-      "logps/chosen": -315.0911865234375,
-      "logps/rejected": -281.7936096191406,
-      "loss": 0.6843,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": 0.06353694945573807,
-      "rewards/margins": 0.020365100353956223,
-      "rewards/rejected": 0.043171852827072144,
+      "logits/chosen": -1.7121353149414062,
+      "logits/rejected": -1.6375898122787476,
+      "logps/chosen": -133.71029663085938,
+      "logps/rejected": -103.07096099853516,
+      "loss": 0.5133,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 44.873260498046875,
+      "rewards/margins": 58.086036682128906,
+      "rewards/rejected": -13.2127685546875,
       "step": 30
     },
     {
       "epoch": 0.08,
-      "grad_norm": 20.30578423378571,
+      "grad_norm": 9873.577427815002,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -1.7271950244903564,
-      "logits/rejected": -1.6917632818222046,
-      "logps/chosen": -269.0830078125,
-      "logps/rejected": -258.13482666015625,
-      "loss": 0.6707,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.15124677121639252,
-      "rewards/margins": 0.058850765228271484,
-      "rewards/rejected": 0.09239600598812103,
+      "logits/chosen": -1.6784517765045166,
+      "logits/rejected": -1.5826914310455322,
+      "logps/chosen": -145.05630493164062,
+      "logps/rejected": -101.44771575927734,
+      "loss": 0.516,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 66.6468734741211,
+      "rewards/margins": 95.41236114501953,
+      "rewards/rejected": -28.765483856201172,
       "step": 40
     },
     {
       "epoch": 0.1,
-      "grad_norm": 18.991767469871117,
+      "grad_norm": 15484.454406367853,
       "learning_rate": 4.999733114418725e-07,
-      "logits/chosen": -1.7609294652938843,
-      "logits/rejected": -1.749925971031189,
-      "logps/chosen": -274.7580261230469,
-      "logps/rejected": -298.2468566894531,
-      "loss": 0.6568,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 0.22399310767650604,
-      "rewards/margins": 0.05384649708867073,
-      "rewards/rejected": 0.1701466143131256,
+      "logits/chosen": -1.6393781900405884,
+      "logits/rejected": -1.6535584926605225,
+      "logps/chosen": -127.37105560302734,
+      "logps/rejected": -104.55952453613281,
+      "loss": 0.6497,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 77.03364562988281,
+      "rewards/margins": 131.46775817871094,
+      "rewards/rejected": -54.434104919433594,
       "step": 50
     },
     {
       "epoch": 0.13,
-      "grad_norm": 22.512008467267577,
+      "grad_norm": 4641.588943610254,
       "learning_rate": 4.990398100856366e-07,
-      "logits/chosen": -1.8478819131851196,
-      "logits/rejected": -1.8104110956192017,
-      "logps/chosen": -268.5568542480469,
-      "logps/rejected": -318.2205505371094,
-      "loss": 0.6419,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.22708478569984436,
-      "rewards/margins": 0.11861655861139297,
-      "rewards/rejected": 0.108468197286129,
+      "logits/chosen": -1.7217296361923218,
+      "logits/rejected": -1.651254653930664,
+      "logps/chosen": -141.35108947753906,
+      "logps/rejected": -108.5528793334961,
+      "loss": 0.7574,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 96.49129486083984,
+      "rewards/margins": 164.4349822998047,
+      "rewards/rejected": -67.94366455078125,
       "step": 60
     },
     {
       "epoch": 0.15,
-      "grad_norm": 24.33517976511266,
+      "grad_norm": 10223.596716214304,
       "learning_rate": 4.967775735898179e-07,
-      "logits/chosen": -1.6735728979110718,
-      "logits/rejected": -1.6890596151351929,
-      "logps/chosen": -274.67156982421875,
-      "logps/rejected": -289.94256591796875,
-      "loss": 0.6121,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": 0.11328981816768646,
-      "rewards/margins": 0.22474534809589386,
-      "rewards/rejected": -0.11145554482936859,
+      "logits/chosen": -1.6282870769500732,
+      "logits/rejected": -1.6370842456817627,
+      "logps/chosen": -134.8829803466797,
+      "logps/rejected": -106.41259765625,
+      "loss": 1.0394,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 102.15782928466797,
+      "rewards/margins": 186.63003540039062,
+      "rewards/rejected": -84.47221374511719,
       "step": 70
     },
     {
       "epoch": 0.17,
-      "grad_norm": 32.81066991350388,
+      "grad_norm": 6360.665892121058,
       "learning_rate": 4.931986719649298e-07,
-      "logits/chosen": -1.9151760339736938,
-      "logits/rejected": -1.802167534828186,
-      "logps/chosen": -356.9640808105469,
-      "logps/rejected": -325.5661315917969,
-      "loss": 0.5878,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.08058764040470123,
-      "rewards/margins": 0.30372124910354614,
-      "rewards/rejected": -0.38430893421173096,
+      "logits/chosen": -1.5932952165603638,
+      "logits/rejected": -1.5602772235870361,
+      "logps/chosen": -130.39671325683594,
+      "logps/rejected": -101.85746002197266,
+      "loss": 1.1921,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 100.63119506835938,
+      "rewards/margins": 181.93630981445312,
+      "rewards/rejected": -81.30511474609375,
       "step": 80
     },
     {
       "epoch": 0.19,
-      "grad_norm": 31.216011823878958,
+      "grad_norm": 12014.555301109034,
       "learning_rate": 4.883222001996351e-07,
-      "logits/chosen": -1.7752355337142944,
-      "logits/rejected": -1.7595351934432983,
-      "logps/chosen": -325.2732849121094,
-      "logps/rejected": -370.2122497558594,
-      "loss": 0.5628,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.44604864716529846,
-      "rewards/margins": 0.5209277272224426,
-      "rewards/rejected": -0.9669763445854187,
+      "logits/chosen": -1.6406339406967163,
+      "logits/rejected": -1.6412605047225952,
+      "logps/chosen": -138.44619750976562,
+      "logps/rejected": -112.9968032836914,
+      "loss": 0.9751,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": 107.64051818847656,
+      "rewards/margins": 218.6805419921875,
+      "rewards/rejected": -111.0400390625,
       "step": 90
     },
     {
       "epoch": 0.21,
-      "grad_norm": 30.757968765158964,
+      "grad_norm": 11340.576903436586,
       "learning_rate": 4.821741763807186e-07,
-      "logits/chosen": -1.8407386541366577,
-      "logits/rejected": -1.818633794784546,
-      "logps/chosen": -356.88604736328125,
-      "logps/rejected": -402.81597900390625,
-      "loss": 0.5759,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.7781068682670593,
-      "rewards/margins": 0.501887321472168,
-      "rewards/rejected": -1.279994249343872,
+      "logits/chosen": -1.6779956817626953,
+      "logits/rejected": -1.6324456930160522,
+      "logps/chosen": -118.78487396240234,
+      "logps/rejected": -101.80384826660156,
+      "loss": 1.1504,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 91.92900848388672,
+      "rewards/margins": 203.4224090576172,
+      "rewards/rejected": -111.493408203125,
       "step": 100
     },
     {
       "epoch": 0.21,
-      "eval_logits/chosen": -1.9150919914245605,
-      "eval_logits/rejected": -1.8988227844238281,
-      "eval_logps/chosen": -335.8161315917969,
-      "eval_logps/rejected": -395.6910095214844,
-      "eval_loss": 0.5723333954811096,
-      "eval_rewards/accuracies": 0.71484375,
-      "eval_rewards/chosen": -0.5418255925178528,
-      "eval_rewards/margins": 0.5374687910079956,
-      "eval_rewards/rejected": -1.0792944431304932,
-      "eval_runtime": 97.7859,
-      "eval_samples_per_second": 20.453,
+      "eval_logits/chosen": -1.724159836769104,
+      "eval_logits/rejected": -1.6941893100738525,
+      "eval_logps/chosen": -127.37677764892578,
+      "eval_logps/rejected": -104.87450408935547,
+      "eval_loss": 0.9783788323402405,
+      "eval_rewards/accuracies": 0.91015625,
+      "eval_rewards/chosen": 103.95471954345703,
+      "eval_rewards/margins": 220.29249572753906,
+      "eval_rewards/rejected": -116.33775329589844,
+      "eval_runtime": 97.7821,
+      "eval_samples_per_second": 20.454,
       "eval_steps_per_second": 0.327,
       "step": 100
     },
     {
       "epoch": 0.23,
-      "grad_norm": 31.68583269080582,
+      "grad_norm": 9309.434410308813,
       "learning_rate": 4.747874028753375e-07,
-      "logits/chosen": -1.8968604803085327,
-      "logits/rejected": -1.7719805240631104,
-      "logps/chosen": -367.7148742675781,
-      "logps/rejected": -371.1282653808594,
-      "loss": 0.5798,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.36616355180740356,
-      "rewards/margins": 0.49689429998397827,
-      "rewards/rejected": -0.8630577921867371,
+      "logits/chosen": -1.6036758422851562,
+      "logits/rejected": -1.651767373085022,
+      "logps/chosen": -123.1724624633789,
+      "logps/rejected": -111.15580749511719,
+      "loss": 1.3815,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 88.74246978759766,
+      "rewards/margins": 178.5576934814453,
+      "rewards/rejected": -89.81523132324219,
       "step": 110
     },
     {
       "epoch": 0.25,
-      "grad_norm": 30.73893119347065,
+      "grad_norm": 7646.143632789072,
       "learning_rate": 4.662012913161997e-07,
-      "logits/chosen": -1.7683579921722412,
-      "logits/rejected": -1.7642805576324463,
-      "logps/chosen": -332.87030029296875,
-      "logps/rejected": -358.53521728515625,
-      "loss": 0.547,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.2855382263660431,
-      "rewards/margins": 0.4636038839817047,
-      "rewards/rejected": -0.7491421103477478,
+      "logits/chosen": -1.6596767902374268,
+      "logits/rejected": -1.639947533607483,
+      "logps/chosen": -122.2258071899414,
+      "logps/rejected": -112.39066314697266,
+      "loss": 1.2948,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 106.12471008300781,
+      "rewards/margins": 216.6038818359375,
+      "rewards/rejected": -110.47917175292969,
       "step": 120
     },
     {
       "epoch": 0.27,
-      "grad_norm": 36.07936832382646,
+      "grad_norm": 5056.012586834783,
       "learning_rate": 4.5646165232345103e-07,
-      "logits/chosen": -1.8693767786026,
-      "logits/rejected": -1.794420599937439,
-      "logps/chosen": -345.6004638671875,
-      "logps/rejected": -402.54608154296875,
-      "loss": 0.5353,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.4933016300201416,
-      "rewards/margins": 0.5930127501487732,
-      "rewards/rejected": -1.0863142013549805,
+      "logits/chosen": -1.6654844284057617,
+      "logits/rejected": -1.6651356220245361,
+      "logps/chosen": -129.16343688964844,
+      "logps/rejected": -105.808837890625,
+      "loss": 1.5047,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 115.51092529296875,
+      "rewards/margins": 240.7315673828125,
+      "rewards/rejected": -125.22064208984375,
       "step": 130
     },
     {
       "epoch": 0.29,
-      "grad_norm": 39.43015574676247,
+      "grad_norm": 13712.265823185711,
       "learning_rate": 4.456204510851956e-07,
-      "logits/chosen": -1.7766962051391602,
-      "logits/rejected": -1.7008291482925415,
-      "logps/chosen": -394.8687744140625,
-      "logps/rejected": -457.8002014160156,
-      "loss": 0.5255,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.5899434089660645,
-      "rewards/margins": 0.7221861481666565,
-      "rewards/rejected": -1.3121296167373657,
+      "logits/chosen": -1.5624234676361084,
+      "logits/rejected": -1.5188586711883545,
+      "logps/chosen": -126.886474609375,
+      "logps/rejected": -106.10212707519531,
+      "loss": 2.4129,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 106.82698822021484,
+      "rewards/margins": 237.18234252929688,
+      "rewards/rejected": -130.3553924560547,
       "step": 140
     },
     {
       "epoch": 0.31,
-      "grad_norm": 48.649979456976475,
+      "grad_norm": 11133.14391408868,
       "learning_rate": 4.337355301007335e-07,
-      "logits/chosen": -1.7205531597137451,
-      "logits/rejected": -1.6475614309310913,
-      "logps/chosen": -382.6226806640625,
-      "logps/rejected": -433.13409423828125,
-      "loss": 0.5514,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.8077861666679382,
-      "rewards/margins": 0.5227850675582886,
-      "rewards/rejected": -1.3305714130401611,
+      "logits/chosen": -1.688401460647583,
+      "logits/rejected": -1.706859827041626,
+      "logps/chosen": -124.13720703125,
+      "logps/rejected": -103.48664855957031,
+      "loss": 1.5451,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 115.81596374511719,
+      "rewards/margins": 246.532958984375,
+      "rewards/rejected": -130.7169647216797,
       "step": 150
     },
     {
       "epoch": 0.33,
-      "grad_norm": 44.8404423652529,
+      "grad_norm": 13437.263938948909,
       "learning_rate": 4.2087030056579986e-07,
-      "logits/chosen": -1.698359727859497,
-      "logits/rejected": -1.583348035812378,
-      "logps/chosen": -338.5320129394531,
-      "logps/rejected": -390.748291015625,
-      "loss": 0.5328,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.4389611780643463,
-      "rewards/margins": 0.7732844352722168,
-      "rewards/rejected": -1.2122455835342407,
+      "logits/chosen": -1.5865412950515747,
+      "logits/rejected": -1.5408028364181519,
+      "logps/chosen": -131.3970489501953,
+      "logps/rejected": -111.11344909667969,
+      "loss": 1.9447,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 105.68785095214844,
+      "rewards/margins": 227.367919921875,
+      "rewards/rejected": -121.6800308227539,
       "step": 160
     },
     {
       "epoch": 0.36,
-      "grad_norm": 44.50813022956455,
+      "grad_norm": 12185.877768787304,
       "learning_rate": 4.070934040463998e-07,
-      "logits/chosen": -1.607373833656311,
-      "logits/rejected": -1.4848835468292236,
-      "logps/chosen": -323.30230712890625,
-      "logps/rejected": -363.1103515625,
-      "loss": 0.5341,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.5660237669944763,
-      "rewards/margins": 0.5307204723358154,
-      "rewards/rejected": -1.0967442989349365,
+      "logits/chosen": -1.8080990314483643,
+      "logits/rejected": -1.7689082622528076,
+      "logps/chosen": -127.97332763671875,
+      "logps/rejected": -110.8963623046875,
+      "loss": 2.3084,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 111.68205261230469,
+      "rewards/margins": 263.9295349121094,
+      "rewards/rejected": -152.2474822998047,
       "step": 170
     },
     {
       "epoch": 0.38,
-      "grad_norm": 36.42011775174975,
+      "grad_norm": 12397.612395894674,
       "learning_rate": 3.9247834624635404e-07,
-      "logits/chosen": -1.4740314483642578,
-      "logits/rejected": -1.3969428539276123,
-      "logps/chosen": -355.79022216796875,
-      "logps/rejected": -386.48846435546875,
-      "loss": 0.5082,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.8436854481697083,
-      "rewards/margins": 0.5756739377975464,
-      "rewards/rejected": -1.4193594455718994,
+      "logits/chosen": -1.7653987407684326,
+      "logits/rejected": -1.7487728595733643,
+      "logps/chosen": -130.79266357421875,
+      "logps/rejected": -106.92414855957031,
+      "loss": 2.1426,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 123.60539245605469,
+      "rewards/margins": 284.14984130859375,
+      "rewards/rejected": -160.54443359375,
       "step": 180
     },
     {
       "epoch": 0.4,
-      "grad_norm": 37.5415107419379,
+      "grad_norm": 8233.63739133568,
       "learning_rate": 3.7710310482256523e-07,
-      "logits/chosen": -1.5718328952789307,
-      "logits/rejected": -1.524204134941101,
-      "logps/chosen": -348.835693359375,
-      "logps/rejected": -401.84857177734375,
-      "loss": 0.5283,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.5492793321609497,
-      "rewards/margins": 0.6370285153388977,
-      "rewards/rejected": -1.186307668685913,
+      "logits/chosen": -1.803746223449707,
+      "logits/rejected": -1.7734184265136719,
+      "logps/chosen": -120.31190490722656,
+      "logps/rejected": -120.1562271118164,
+      "loss": 1.8983,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 114.87654876708984,
+      "rewards/margins": 295.48638916015625,
+      "rewards/rejected": -180.6098175048828,
       "step": 190
     },
     {
       "epoch": 0.42,
-      "grad_norm": 86.8216257987796,
+      "grad_norm": 8854.142242075086,
       "learning_rate": 3.610497133404795e-07,
-      "logits/chosen": -1.508467674255371,
-      "logits/rejected": -1.409771203994751,
-      "logps/chosen": -339.861328125,
-      "logps/rejected": -404.37200927734375,
-      "loss": 0.5391,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.6251331567764282,
-      "rewards/margins": 0.7106647491455078,
-      "rewards/rejected": -1.335797905921936,
+      "logits/chosen": -1.748956322669983,
+      "logits/rejected": -1.7438066005706787,
+      "logps/chosen": -126.25050354003906,
+      "logps/rejected": -105.51225280761719,
+      "loss": 2.8553,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 128.85885620117188,
+      "rewards/margins": 303.90032958984375,
+      "rewards/rejected": -175.04144287109375,
       "step": 200
     },
     {
       "epoch": 0.42,
-      "eval_logits/chosen": -1.6569650173187256,
-      "eval_logits/rejected": -1.577364206314087,
-      "eval_logps/chosen": -321.1681823730469,
-      "eval_logps/rejected": -405.005615234375,
-      "eval_loss": 0.5137878060340881,
-      "eval_rewards/accuracies": 0.74609375,
-      "eval_rewards/chosen": -0.3953460156917572,
-      "eval_rewards/margins": 0.777094841003418,
-      "eval_rewards/rejected": -1.172440767288208,
-      "eval_runtime": 97.4997,
-      "eval_samples_per_second": 20.513,
+      "eval_logits/chosen": -1.8518784046173096,
+      "eval_logits/rejected": -1.8462214469909668,
+      "eval_logps/chosen": -125.94612121582031,
+      "eval_logps/rejected": -111.28173828125,
+      "eval_loss": 1.8849064111709595,
+      "eval_rewards/accuracies": 0.91015625,
+      "eval_rewards/chosen": 118.2613525390625,
+      "eval_rewards/margins": 298.67144775390625,
+      "eval_rewards/rejected": -180.41009521484375,
+      "eval_runtime": 97.5342,
+      "eval_samples_per_second": 20.506,
       "eval_steps_per_second": 0.328,
       "step": 200
     },
     {
       "epoch": 0.44,
-      "grad_norm": 38.50117908255001,
+      "grad_norm": 11055.40272221904,
       "learning_rate": 3.4440382358952115e-07,
-      "logits/chosen": -1.443800687789917,
-      "logits/rejected": -1.3134807348251343,
-      "logps/chosen": -354.0191345214844,
-      "logps/rejected": -393.02264404296875,
-      "loss": 0.5328,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.40636110305786133,
-      "rewards/margins": 0.759956419467926,
-      "rewards/rejected": -1.1663174629211426,
+      "logits/chosen": -1.6907581090927124,
+      "logits/rejected": -1.6238548755645752,
+      "logps/chosen": -122.90483093261719,
+      "logps/rejected": -103.57493591308594,
+      "loss": 2.2063,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 96.85926818847656,
+      "rewards/margins": 236.5902862548828,
+      "rewards/rejected": -139.7310028076172,
       "step": 210
     },
     {
       "epoch": 0.46,
-      "grad_norm": 50.76318175549852,
+      "grad_norm": 8772.23523058132,
       "learning_rate": 3.272542485937368e-07,
-      "logits/chosen": -1.2096138000488281,
-      "logits/rejected": -1.064009189605713,
-      "logps/chosen": -339.54339599609375,
-      "logps/rejected": -401.81622314453125,
-      "loss": 0.5149,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.570379376411438,
-      "rewards/margins": 0.8607443571090698,
-      "rewards/rejected": -1.4311238527297974,
+      "logits/chosen": -1.832867980003357,
+      "logits/rejected": -1.8748031854629517,
+      "logps/chosen": -124.8982162475586,
+      "logps/rejected": -103.1186752319336,
+      "loss": 2.1666,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 115.77070617675781,
+      "rewards/margins": 271.67108154296875,
+      "rewards/rejected": -155.9003448486328,
       "step": 220
     },
     {
       "epoch": 0.48,
-      "grad_norm": 56.74639801937906,
+      "grad_norm": 7569.137122040314,
       "learning_rate": 3.096924887558854e-07,
-      "logits/chosen": -1.1264355182647705,
-      "logits/rejected": -1.0245306491851807,
-      "logps/chosen": -354.9466857910156,
-      "logps/rejected": -463.1206970214844,
-      "loss": 0.5149,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.8869966268539429,
-      "rewards/margins": 1.0312917232513428,
-      "rewards/rejected": -1.9182884693145752,
+      "logits/chosen": -1.7989298105239868,
+      "logits/rejected": -1.737357497215271,
+      "logps/chosen": -135.94210815429688,
+      "logps/rejected": -119.22425842285156,
+      "loss": 2.7122,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 117.66719818115234,
+      "rewards/margins": 311.7781677246094,
+      "rewards/rejected": -194.11097717285156,
       "step": 230
     },
     {
       "epoch": 0.5,
-      "grad_norm": 42.18851613816063,
+      "grad_norm": 9108.756414029493,
       "learning_rate": 2.9181224366319943e-07,
-      "logits/chosen": -1.2307146787643433,
-      "logits/rejected": -1.139776349067688,
-      "logps/chosen": -370.49853515625,
-      "logps/rejected": -442.11114501953125,
-      "loss": 0.487,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.8160694241523743,
-      "rewards/margins": 0.8950576782226562,
-      "rewards/rejected": -1.7111270427703857,
+      "logits/chosen": -1.831538438796997,
+      "logits/rejected": -1.8362337350845337,
+      "logps/chosen": -121.7720947265625,
+      "logps/rejected": -112.72883605957031,
+      "loss": 2.3272,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 101.53041076660156,
+      "rewards/margins": 252.52392578125,
+      "rewards/rejected": -150.99354553222656,
       "step": 240
     },
     {
       "epoch": 0.52,
-      "grad_norm": 42.578136485563924,
+      "grad_norm": 11190.763278546787,
       "learning_rate": 2.7370891215954565e-07,
-      "logits/chosen": -1.104014277458191,
-      "logits/rejected": -0.9260801076889038,
-      "logps/chosen": -386.5050964355469,
-      "logps/rejected": -449.59722900390625,
-      "loss": 0.5122,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.6448307633399963,
-      "rewards/margins": 0.9931491017341614,
-      "rewards/rejected": -1.6379798650741577,
+      "logits/chosen": -1.83551824092865,
+      "logits/rejected": -1.8649381399154663,
+      "logps/chosen": -125.136474609375,
+      "logps/rejected": -114.23868560791016,
+      "loss": 2.3585,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 122.786865234375,
+      "rewards/margins": 295.3399658203125,
+      "rewards/rejected": -172.55311584472656,
       "step": 250
     },
     {
       "epoch": 0.54,
-      "grad_norm": 45.2823901088297,
+      "grad_norm": 6158.166356558157,
       "learning_rate": 2.55479083351317e-07,
-      "logits/chosen": -0.9789173007011414,
-      "logits/rejected": -0.7768954634666443,
-      "logps/chosen": -395.30548095703125,
-      "logps/rejected": -449.7933044433594,
-      "loss": 0.505,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.7873316407203674,
-      "rewards/margins": 0.9275019764900208,
-      "rewards/rejected": -1.7148334980010986,
+      "logits/chosen": -1.8736432790756226,
+      "logits/rejected": -1.887500524520874,
+      "logps/chosen": -129.7669677734375,
+      "logps/rejected": -101.11165618896484,
+      "loss": 2.615,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 140.60031127929688,
+      "rewards/margins": 302.95391845703125,
+      "rewards/rejected": -162.3535919189453,
       "step": 260
     },
     {
       "epoch": 0.56,
-      "grad_norm": 44.49352654305102,
+      "grad_norm": 5058.294410517059,
       "learning_rate": 2.3722002126275822e-07,
-      "logits/chosen": -0.9071099162101746,
-      "logits/rejected": -0.8342393636703491,
-      "logps/chosen": -374.5643310546875,
-      "logps/rejected": -433.95098876953125,
-      "loss": 0.5205,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.729364275932312,
-      "rewards/margins": 0.7837658524513245,
-      "rewards/rejected": -1.5131301879882812,
+      "logits/chosen": -1.8732073307037354,
+      "logits/rejected": -1.817939043045044,
+      "logps/chosen": -120.4271011352539,
+      "logps/rejected": -108.13395690917969,
+      "loss": 2.1432,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 116.31379699707031,
+      "rewards/margins": 285.315185546875,
+      "rewards/rejected": -169.00140380859375,
       "step": 270
     },
     {
       "epoch": 0.59,
-      "grad_norm": 42.661081149871585,
+      "grad_norm": 8591.417699065232,
       "learning_rate": 2.19029145890313e-07,
-      "logits/chosen": -0.9833901524543762,
-      "logits/rejected": -0.7881035804748535,
-      "logps/chosen": -336.73388671875,
-      "logps/rejected": -407.16326904296875,
-      "loss": 0.5186,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.5388863682746887,
-      "rewards/margins": 0.8700063824653625,
-      "rewards/rejected": -1.4088926315307617,
+      "logits/chosen": -1.7311344146728516,
+      "logits/rejected": -1.812242865562439,
+      "logps/chosen": -126.07157897949219,
+      "logps/rejected": -120.28532409667969,
+      "loss": 3.1754,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 109.85076904296875,
+      "rewards/margins": 299.208740234375,
+      "rewards/rejected": -189.35797119140625,
       "step": 280
     },
     {
       "epoch": 0.61,
-      "grad_norm": 42.8181212203347,
+      "grad_norm": 10809.88836686454,
       "learning_rate": 2.0100351342479216e-07,
-      "logits/chosen": -1.1338340044021606,
-      "logits/rejected": -1.0056653022766113,
-      "logps/chosen": -327.16217041015625,
-      "logps/rejected": -406.1835021972656,
-      "loss": 0.5062,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.5975332260131836,
-      "rewards/margins": 0.854636549949646,
-      "rewards/rejected": -1.4521698951721191,
+      "logits/chosen": -1.8156871795654297,
+      "logits/rejected": -1.7812505960464478,
+      "logps/chosen": -116.09767150878906,
+      "logps/rejected": -107.4920883178711,
+      "loss": 2.8029,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 120.6517333984375,
+      "rewards/margins": 304.40826416015625,
+      "rewards/rejected": -183.7565460205078,
       "step": 290
     },
     {
       "epoch": 0.63,
-      "grad_norm": 43.557515582129994,
+      "grad_norm": 7050.7886997426285,
       "learning_rate": 1.8323929841460178e-07,
-      "logits/chosen": -1.1290686130523682,
-      "logits/rejected": -0.9314834475517273,
-      "logps/chosen": -405.9638671875,
-      "logps/rejected": -460.133544921875,
-      "loss": 0.4788,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.8333964347839355,
-      "rewards/margins": 0.9296092987060547,
-      "rewards/rejected": -1.7630058526992798,
+      "logits/chosen": -1.7682945728302002,
+      "logits/rejected": -1.7057702541351318,
+      "logps/chosen": -134.6074676513672,
+      "logps/rejected": -124.12396240234375,
+      "loss": 2.2897,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 118.32698059082031,
+      "rewards/margins": 297.16571044921875,
+      "rewards/rejected": -178.83876037597656,
       "step": 300
     },
     {
       "epoch": 0.63,
-      "eval_logits/chosen": -0.986372709274292,
-      "eval_logits/rejected": -0.8825947046279907,
-      "eval_logps/chosen": -367.72772216796875,
-      "eval_logps/rejected": -475.55322265625,
-      "eval_loss": 0.49388447403907776,
-      "eval_rewards/accuracies": 0.7578125,
-      "eval_rewards/chosen": -0.8609416484832764,
-      "eval_rewards/margins": 1.016974687576294,
-      "eval_rewards/rejected": -1.8779162168502808,
-      "eval_runtime": 97.7186,
-      "eval_samples_per_second": 20.467,
-      "eval_steps_per_second": 0.327,
+      "eval_logits/chosen": -1.868285059928894,
+      "eval_logits/rejected": -1.8641510009765625,
+      "eval_logps/chosen": -124.9817886352539,
+      "eval_logps/rejected": -112.93190002441406,
+      "eval_loss": 2.1029016971588135,
+      "eval_rewards/accuracies": 0.9140625,
+      "eval_rewards/chosen": 127.90460968017578,
+      "eval_rewards/margins": 324.8162536621094,
+      "eval_rewards/rejected": -196.91163635253906,
+      "eval_runtime": 97.6004,
+      "eval_samples_per_second": 20.492,
+      "eval_steps_per_second": 0.328,
       "step": 300
     },
     {
       "epoch": 0.65,
-      "grad_norm": 50.71884572569721,
+      "grad_norm": 11804.0120903992,
       "learning_rate": 1.6583128063291573e-07,
-      "logits/chosen": -0.8109232783317566,
-      "logits/rejected": -0.6804934144020081,
-      "logps/chosen": -421.7939453125,
-      "logps/rejected": -471.0542907714844,
-      "loss": 0.4943,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.045866847038269,
-      "rewards/margins": 0.8514853715896606,
-      "rewards/rejected": -1.8973522186279297,
+      "logits/chosen": -1.7608633041381836,
+      "logits/rejected": -1.7920173406600952,
+      "logps/chosen": -126.25215911865234,
+      "logps/rejected": -115.8753433227539,
+      "loss": 2.7724,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 130.85523986816406,
+      "rewards/margins": 304.2178039550781,
+      "rewards/rejected": -173.3625946044922,
       "step": 310
     },
     {
       "epoch": 0.67,
-      "grad_norm": 50.530325658635704,
+      "grad_norm": 10459.35107314203,
       "learning_rate": 1.488723393865766e-07,
-      "logits/chosen": -0.697489321231842,
-      "logits/rejected": -0.5368015170097351,
-      "logps/chosen": -416.4817810058594,
-      "logps/rejected": -457.8753356933594,
-      "loss": 0.4844,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.9900579452514648,
-      "rewards/margins": 0.9124795198440552,
-      "rewards/rejected": -1.9025375843048096,
+      "logits/chosen": -1.7917076349258423,
+      "logits/rejected": -1.7515103816986084,
+      "logps/chosen": -116.50152587890625,
+      "logps/rejected": -114.2782211303711,
+      "loss": 2.4757,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 142.95352172851562,
+      "rewards/margins": 323.181884765625,
+      "rewards/rejected": -180.22837829589844,
       "step": 320
     },
     {
       "epoch": 0.69,
-      "grad_norm": 51.29954417938159,
+      "grad_norm": 8336.1056366251,
       "learning_rate": 1.3245295796480788e-07,
-      "logits/chosen": -0.9539875984191895,
-      "logits/rejected": -0.7579910159111023,
-      "logps/chosen": -368.8915710449219,
-      "logps/rejected": -450.0838317871094,
-      "loss": 0.4783,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.8678079843521118,
-      "rewards/margins": 0.7714605331420898,
-      "rewards/rejected": -1.6392685174942017,
+      "logits/chosen": -1.7024962902069092,
+      "logits/rejected": -1.7793302536010742,
+      "logps/chosen": -126.26700592041016,
+      "logps/rejected": -110.26517486572266,
+      "loss": 2.626,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 119.15933990478516,
+      "rewards/margins": 284.33038330078125,
+      "rewards/rejected": -165.17105102539062,
       "step": 330
     },
     {
       "epoch": 0.71,
-      "grad_norm": 47.146951845139505,
+      "grad_norm": 6521.040226026619,
       "learning_rate": 1.1666074087171627e-07,
-      "logits/chosen": -0.9075170755386353,
-      "logits/rejected": -0.7373852133750916,
-      "logps/chosen": -388.6153259277344,
-      "logps/rejected": -477.04217529296875,
-      "loss": 0.4968,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.7279259562492371,
-      "rewards/margins": 1.0357873439788818,
-      "rewards/rejected": -1.7637134790420532,
+      "logits/chosen": -1.6726289987564087,
+      "logits/rejected": -1.7343635559082031,
+      "logps/chosen": -120.09101867675781,
+      "logps/rejected": -102.69850158691406,
+      "loss": 3.9728,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 103.992919921875,
+      "rewards/margins": 254.6125030517578,
+      "rewards/rejected": -150.61959838867188,
       "step": 340
     },
     {
       "epoch": 0.73,
-      "grad_norm": 40.51519007652658,
+      "grad_norm": 9653.282761957253,
       "learning_rate": 1.0157994641835734e-07,
-      "logits/chosen": -0.9085396528244019,
-      "logits/rejected": -0.7681713104248047,
-      "logps/chosen": -352.57073974609375,
-      "logps/rejected": -430.44903564453125,
-      "loss": 0.4725,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.7491118311882019,
-      "rewards/margins": 0.9513088464736938,
-      "rewards/rejected": -1.7004207372665405,
+      "logits/chosen": -1.7162805795669556,
+      "logits/rejected": -1.7199398279190063,
+      "logps/chosen": -120.1608657836914,
+      "logps/rejected": -121.0823974609375,
+      "loss": 2.2998,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 99.24105072021484,
+      "rewards/margins": 294.47723388671875,
+      "rewards/rejected": -195.23617553710938,
       "step": 350
     },
     {
       "epoch": 0.75,
-      "grad_norm": 44.22500120445586,
+      "grad_norm": 8483.170816558306,
       "learning_rate": 8.729103716819111e-08,
-      "logits/chosen": -1.0195329189300537,
-      "logits/rejected": -0.7282712459564209,
-      "logps/chosen": -405.8896179199219,
-      "logps/rejected": -458.53826904296875,
-      "loss": 0.5133,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.7919691801071167,
-      "rewards/margins": 0.9254425168037415,
-      "rewards/rejected": -1.7174116373062134,
+      "logits/chosen": -1.7057559490203857,
+      "logits/rejected": -1.7162882089614868,
+      "logps/chosen": -126.68985748291016,
+      "logps/rejected": -117.80401611328125,
+      "loss": 1.8659,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 114.60487365722656,
+      "rewards/margins": 299.46978759765625,
+      "rewards/rejected": -184.86489868164062,
       "step": 360
     },
     {
       "epoch": 0.77,
-      "grad_norm": 42.97964211979383,
+      "grad_norm": 16292.765448877528,
       "learning_rate": 7.387025063449081e-08,
-      "logits/chosen": -0.7996746301651001,
-      "logits/rejected": -0.5788342356681824,
-      "logps/chosen": -381.4356689453125,
-      "logps/rejected": -423.28570556640625,
-      "loss": 0.5002,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.8598394393920898,
-      "rewards/margins": 0.7896124124526978,
-      "rewards/rejected": -1.6494518518447876,
+      "logits/chosen": -1.8123562335968018,
+      "logits/rejected": -1.7508220672607422,
+      "logps/chosen": -116.35862731933594,
+      "logps/rejected": -107.92704010009766,
+      "loss": 2.4835,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 100.31108093261719,
+      "rewards/margins": 254.4406280517578,
+      "rewards/rejected": -154.12954711914062,
       "step": 370
     },
     {
       "epoch": 0.79,
-      "grad_norm": 40.60703994205003,
+      "grad_norm": 12994.314264805473,
       "learning_rate": 6.138919252022435e-08,
-      "logits/chosen": -0.7264416217803955,
-      "logits/rejected": -0.689493715763092,
-      "logps/chosen": -355.46893310546875,
-      "logps/rejected": -487.8539123535156,
-      "loss": 0.485,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.9955714344978333,
-      "rewards/margins": 1.0758744478225708,
-      "rewards/rejected": -2.071445941925049,
+      "logits/chosen": -1.84799063205719,
+      "logits/rejected": -1.857129693031311,
+      "logps/chosen": -124.62117004394531,
+      "logps/rejected": -118.90538024902344,
+      "loss": 2.1362,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 120.3254623413086,
+      "rewards/margins": 317.2894592285156,
+      "rewards/rejected": -196.96397399902344,
       "step": 380
     },
     {
       "epoch": 0.82,
-      "grad_norm": 46.93787360588045,
+      "grad_norm": 11506.166124614792,
       "learning_rate": 4.991445467064689e-08,
-      "logits/chosen": -0.8655691146850586,
-      "logits/rejected": -0.7559862732887268,
-      "logps/chosen": -414.95660400390625,
-      "logps/rejected": -487.86029052734375,
-      "loss": 0.4776,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.9033805727958679,
-      "rewards/margins": 0.8835199475288391,
-      "rewards/rejected": -1.786900281906128,
+      "logits/chosen": -1.7219364643096924,
+      "logits/rejected": -1.7155206203460693,
+      "logps/chosen": -119.84449768066406,
+      "logps/rejected": -109.42668151855469,
+      "loss": 2.056,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": 126.5132827758789,
+      "rewards/margins": 317.73858642578125,
+      "rewards/rejected": -191.22531127929688,
       "step": 390
     },
     {
       "epoch": 0.84,
-      "grad_norm": 44.554590531311376,
+      "grad_norm": 14299.186863928613,
       "learning_rate": 3.9507259776993954e-08,
-      "logits/chosen": -0.6984928846359253,
-      "logits/rejected": -0.5499908924102783,
-      "logps/chosen": -390.16436767578125,
-      "logps/rejected": -478.6742248535156,
-      "loss": 0.4937,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.9803212285041809,
-      "rewards/margins": 0.9358729124069214,
-      "rewards/rejected": -1.916194200515747,
+      "logits/chosen": -1.7028141021728516,
+      "logits/rejected": -1.7706788778305054,
+      "logps/chosen": -122.94708251953125,
+      "logps/rejected": -110.153564453125,
+      "loss": 2.2714,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 98.14391326904297,
+      "rewards/margins": 283.8785400390625,
+      "rewards/rejected": -185.73460388183594,
       "step": 400
     },
     {
       "epoch": 0.84,
-      "eval_logits/chosen": -0.9126077890396118,
-      "eval_logits/rejected": -0.760056734085083,
-      "eval_logps/chosen": -361.4586486816406,
-      "eval_logps/rejected": -470.95220947265625,
-      "eval_loss": 0.4905379116535187,
-      "eval_rewards/accuracies": 0.75390625,
-      "eval_rewards/chosen": -0.7982508540153503,
-      "eval_rewards/margins": 1.0336554050445557,
-      "eval_rewards/rejected": -1.8319063186645508,
-      "eval_runtime": 97.7293,
-      "eval_samples_per_second": 20.465,
-      "eval_steps_per_second": 0.327,
+      "eval_logits/chosen": -1.8724907636642456,
+      "eval_logits/rejected": -1.8708370923995972,
+      "eval_logps/chosen": -125.25865936279297,
+      "eval_logps/rejected": -113.72941589355469,
+      "eval_loss": 1.8651787042617798,
+      "eval_rewards/accuracies": 0.9140625,
+      "eval_rewards/chosen": 125.13589477539062,
+      "eval_rewards/margins": 330.022705078125,
+      "eval_rewards/rejected": -204.8868408203125,
+      "eval_runtime": 97.6945,
+      "eval_samples_per_second": 20.472,
+      "eval_steps_per_second": 0.328,
       "step": 400
     },
     {
       "epoch": 0.86,
-      "grad_norm": 39.65148650158285,
+      "grad_norm": 9838.615615925528,
       "learning_rate": 3.022313472693447e-08,
-      "logits/chosen": -0.938845157623291,
-      "logits/rejected": -0.6247340440750122,
-      "logps/chosen": -400.84222412109375,
-      "logps/rejected": -476.00018310546875,
-      "loss": 0.4863,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.8032888174057007,
-      "rewards/margins": 1.0230791568756104,
-      "rewards/rejected": -1.8263680934906006,
+      "logits/chosen": -1.7946465015411377,
+      "logits/rejected": -1.8112504482269287,
+      "logps/chosen": -134.29222106933594,
+      "logps/rejected": -112.4157943725586,
+      "loss": 2.3885,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 130.2279815673828,
+      "rewards/margins": 333.3272705078125,
+      "rewards/rejected": -203.0992889404297,
       "step": 410
     },
     {
       "epoch": 0.88,
-      "grad_norm": 42.93092965398867,
+      "grad_norm": 9593.489304491115,
       "learning_rate": 2.2111614344599684e-08,
-      "logits/chosen": -0.8916628956794739,
-      "logits/rejected": -0.7629222869873047,
-      "logps/chosen": -404.97088623046875,
-      "logps/rejected": -487.2880859375,
-      "loss": 0.4797,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.836542010307312,
-      "rewards/margins": 1.0462357997894287,
-      "rewards/rejected": -1.8827779293060303,
+      "logits/chosen": -1.7704830169677734,
+      "logits/rejected": -1.758772611618042,
+      "logps/chosen": -118.04278564453125,
+      "logps/rejected": -109.47645568847656,
+      "loss": 1.872,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 102.66886138916016,
+      "rewards/margins": 290.7878112792969,
+      "rewards/rejected": -188.11898803710938,
       "step": 420
     },
     {
       "epoch": 0.9,
-      "grad_norm": 50.64982947415171,
+      "grad_norm": 6763.365413342396,
       "learning_rate": 1.521597710086439e-08,
-      "logits/chosen": -0.7213121056556702,
-      "logits/rejected": -0.5332973003387451,
-      "logps/chosen": -401.74871826171875,
-      "logps/rejected": -478.0919494628906,
-      "loss": 0.4735,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.9987181425094604,
-      "rewards/margins": 0.9951263666152954,
-      "rewards/rejected": -1.9938443899154663,
+      "logits/chosen": -1.710999846458435,
+      "logits/rejected": -1.7392107248306274,
+      "logps/chosen": -134.11866760253906,
+      "logps/rejected": -109.66754150390625,
+      "loss": 2.0497,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 119.90589904785156,
+      "rewards/margins": 281.7261657714844,
+      "rewards/rejected": -161.82025146484375,
       "step": 430
     },
     {
       "epoch": 0.92,
-      "grad_norm": 45.64074270515632,
+      "grad_norm": 8451.951191648946,
       "learning_rate": 9.57301420397924e-09,
-      "logits/chosen": -0.8010743856430054,
-      "logits/rejected": -0.6153632998466492,
-      "logps/chosen": -389.0445251464844,
-      "logps/rejected": -466.53387451171875,
-      "loss": 0.4956,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.842692494392395,
-      "rewards/margins": 0.8713605999946594,
-      "rewards/rejected": -1.7140531539916992,
+      "logits/chosen": -1.852447748184204,
+      "logits/rejected": -1.8499510288238525,
+      "logps/chosen": -125.65571594238281,
+      "logps/rejected": -113.3338394165039,
+      "loss": 2.2476,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 133.1741485595703,
+      "rewards/margins": 332.48126220703125,
+      "rewards/rejected": -199.3070831298828,
       "step": 440
     },
     {
       "epoch": 0.94,
-      "grad_norm": 42.58278664732128,
+      "grad_norm": 9580.867576787692,
       "learning_rate": 5.212833302556258e-09,
-      "logits/chosen": -0.7098689675331116,
-      "logits/rejected": -0.6174753308296204,
-      "logps/chosen": -411.04345703125,
-      "logps/rejected": -515.0621948242188,
-      "loss": 0.4986,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.983228325843811,
-      "rewards/margins": 0.8576449155807495,
-      "rewards/rejected": -1.84087336063385,
+      "logits/chosen": -1.8617397546768188,
+      "logits/rejected": -1.8191306591033936,
+      "logps/chosen": -121.5869369506836,
+      "logps/rejected": -108.55745697021484,
+      "loss": 2.5789,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 137.1997528076172,
+      "rewards/margins": 339.0559387207031,
+      "rewards/rejected": -201.85618591308594,
       "step": 450
     },
     {
       "epoch": 0.96,
-      "grad_norm": 60.928874193313085,
+      "grad_norm": 4611.3001759975,
       "learning_rate": 2.158697848236607e-09,
-      "logits/chosen": -0.7517565488815308,
-      "logits/rejected": -0.5577678084373474,
-      "logps/chosen": -384.75677490234375,
-      "logps/rejected": -444.00030517578125,
-      "loss": 0.4922,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.859534740447998,
-      "rewards/margins": 0.9033729434013367,
-      "rewards/rejected": -1.76290762424469,
+      "logits/chosen": -1.7538158893585205,
+      "logits/rejected": -1.7651903629302979,
+      "logps/chosen": -126.25445556640625,
+      "logps/rejected": -113.3001937866211,
+      "loss": 1.6224,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 130.99368286132812,
+      "rewards/margins": 307.7095642089844,
+      "rewards/rejected": -176.7158660888672,
       "step": 460
     },
     {
       "epoch": 0.98,
-      "grad_norm": 45.081509845975475,
+      "grad_norm": 8182.843310129901,
       "learning_rate": 4.269029751107489e-10,
-      "logits/chosen": -0.7836319804191589,
-      "logits/rejected": -0.5721665620803833,
-      "logps/chosen": -376.28497314453125,
-      "logps/rejected": -472.9750061035156,
-      "loss": 0.4796,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.814776599407196,
-      "rewards/margins": 0.9141017198562622,
-      "rewards/rejected": -1.7288782596588135,
+      "logits/chosen": -1.728009581565857,
+      "logits/rejected": -1.7379541397094727,
+      "logps/chosen": -119.18067932128906,
+      "logps/rejected": -124.09730529785156,
+      "loss": 2.184,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 105.3095703125,
+      "rewards/margins": 308.53289794921875,
+      "rewards/rejected": -203.22329711914062,
       "step": 470
     },
     {
       "epoch": 1.0,
       "step": 478,
       "total_flos": 0.0,
-      "train_loss": 0.07911034258838479,
-      "train_runtime": 1140.8874,
-      "train_samples_per_second": 53.585,
-      "train_steps_per_second": 0.419
+      "train_loss": 2.0128297995323914,
+      "train_runtime": 7588.5519,
+      "train_samples_per_second": 8.056,
+      "train_steps_per_second": 0.063
     }
   ],
   "logging_steps": 10,