Model save

README.md

@@ -15,15 +15,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.1843
-- Rewards/chosen: -5.6098
-- Rewards/rejected: -5.9639
-- Rewards/accuracies: 0.5117
-- Rewards/margins: 0.3541
-- Logps/rejected: -1114.7808
-- Logps/chosen: -951.4574
-- Logits/rejected: -7.9900
-- Logits/chosen: -7.4446
+- Loss: 6.9230
+- Rewards/chosen: -4.5175
+- Rewards/rejected: 0.4288
+- Rewards/accuracies: 0.3164
+- Rewards/margins: -4.9464
+- Logps/rejected: -517.5300
+- Logps/chosen: -399.5095
+- Logits/rejected: -4.8908
+- Logits/chosen: -4.6604
 
 ## Model description
 
@@ -58,16 +58,16 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
-|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.4015        | 0.26  | 100  | 0.9856          | -4.3603        | -4.6945          | 0.5273             | 0.3341          | -987.8358      | -826.5081    | -6.7933         | -6.4109       |
-| 0.3649        | 0.53  | 200  | 1.1239          | -4.8760        | -5.1429          | 0.4883             | 0.2669          | -1032.6809     | -878.0756    | -7.6378         | -7.1525       |
-| 0.3506        | 0.79  | 300  | 1.1843          | -5.6098        | -5.9639          | 0.5117             | 0.3541          | -1114.7808     | -951.4574    | -7.9900         | -7.4446       |
+| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.5909        | 0.2635 | 100  | 6.6534          | -3.0090        | 2.8904           | 0.2773             | -5.8994         | -512.6068      | -396.4924    | -4.8508         | -4.6121       |
+| 0.7239        | 0.5270 | 200  | 8.0720          | -2.8191        | 3.5065           | 0.2734             | -6.3256         | -511.3747      | -396.1127    | -4.9896         | -4.7715       |
+| 0.5556        | 0.7905 | 300  | 6.9230          | -4.5175        | 0.4288           | 0.3164             | -4.9464         | -517.5300      | -399.5095    | -4.8908         | -4.6604       |
 
 
 ### Framework versions
 
-- Transformers 4.38.2
+- Transformers 4.41.1
 - Pytorch 2.1.2+cu118
 - Datasets 2.16.1
-- Tokenizers 0.15.2
+- Tokenizers 0.19.1

all_results.json

@@ -1,8 +1,9 @@
 {
-    "epoch": 1.0,
-    "train_loss": 0.39733357479830217,
-    "train_runtime": 5845.1464,
+    "epoch": 0.9986824769433466,
+    "total_flos": 0.0,
+    "train_loss": 0.5517442987587962,
+    "train_runtime": 6181.8185,
     "train_samples": 48530,
-    "train_samples_per_second": 8.303,
-    "train_steps_per_second": 0.065
+    "train_samples_per_second": 7.85,
+    "train_steps_per_second": 0.061
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 32000,
-  "transformers_version": "4.38.2"
+  "transformers_version": "4.41.1"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f0448d3619348a37bb87330d8a16f03f2db67a5e69f6d423351f4de49ba701b4
+oid sha256:fad6798eb6bb6153f6b5aff19d3bc5cffa1aeb40bf26f57d5a587788a415d08c
 size 4943178720

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0afb427ab7948f6418557c415a2087f26e4a2bcf506844122d917521dfff33e5
+oid sha256:20ca669573e568702fa6d40967a3d29846dc5ecf876326abd24dbc1904e69161
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a88ed491b35c8995c182b2bbbff2fb9fb8eab8e0c1d499eee5dfc7d7e795619f
+oid sha256:ddf6a93711a98049fe876e8b35960fb84be7446726bf529c7236bf8c1e7f0d9d
 size 4540532728

runs/May29_01-45-07_n136-082-130/events.out.tfevents.1716918651.n136-082-130.2665532.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3ddf24f553333828786988c3971d9d1022e8757f1230b262d1d8425c990e02ee
-size 28505
+oid sha256:2856f09a0f9bb2d9198cc98a815d742f99ea9ee604c8ae1f64256658c1973601
+size 33675

train_results.json

@@ -1,8 +1,9 @@
 {
-    "epoch": 1.0,
-    "train_loss": 0.39733357479830217,
-    "train_runtime": 5845.1464,
+    "epoch": 0.9986824769433466,
+    "total_flos": 0.0,
+    "train_loss": 0.5517442987587962,
+    "train_runtime": 6181.8185,
     "train_samples": 48530,
-    "train_samples_per_second": 8.303,
-    "train_steps_per_second": 0.065
+    "train_samples_per_second": 7.85,
+    "train_steps_per_second": 0.061
 }

trainer_state.json

@@ -9,8 +9,8 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0,
-      "grad_norm": 34.01362102288599,
+      "epoch": 0.002635046113306983,
+      "grad_norm": 1701.3284378032004,
       "learning_rate": 1.3157894736842104e-08,
       "logits/chosen": -4.685327529907227,
       "logits/rejected": -4.87608528137207,
@@ -24,616 +24,616 @@
       "step": 1
     },
     {
-      "epoch": 0.03,
-      "grad_norm": 33.835137410082986,
+      "epoch": 0.026350461133069828,
+      "grad_norm": 1584.9120962726856,
       "learning_rate": 1.3157894736842104e-07,
-      "logits/chosen": -4.499408721923828,
-      "logits/rejected": -4.84108829498291,
-      "logps/chosen": -223.5843048095703,
-      "logps/rejected": -160.73016357421875,
-      "loss": 0.6929,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.0008713232818990946,
-      "rewards/margins": 3.9665635995334014e-05,
-      "rewards/rejected": 0.000831657787784934,
+      "logits/chosen": -4.499300479888916,
+      "logits/rejected": -4.840802192687988,
+      "logps/chosen": -223.6631317138672,
+      "logps/rejected": -160.81097412109375,
+      "loss": 0.7136,
+      "rewards/accuracies": 0.4444444477558136,
+      "rewards/chosen": 0.004151582717895508,
+      "rewards/margins": 0.002958830911666155,
+      "rewards/rejected": 0.001192751806229353,
       "step": 10
     },
     {
-      "epoch": 0.05,
-      "grad_norm": 32.9122864599857,
+      "epoch": 0.052700922266139656,
+      "grad_norm": 955.29292445595,
       "learning_rate": 2.631578947368421e-07,
-      "logits/chosen": -4.521907806396484,
-      "logits/rejected": -4.8204779624938965,
-      "logps/chosen": -220.248779296875,
-      "logps/rejected": -173.30508422851562,
-      "loss": 0.6818,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": 0.01710951328277588,
-      "rewards/margins": 0.023774703964591026,
-      "rewards/rejected": -0.006665193475782871,
+      "logits/chosen": -4.518028259277344,
+      "logits/rejected": -4.817793846130371,
+      "logps/chosen": -220.1512451171875,
+      "logps/rejected": -172.69322204589844,
+      "loss": 0.4939,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.9042474627494812,
+      "rewards/margins": 0.9315685033798218,
+      "rewards/rejected": -0.02732105180621147,
       "step": 20
     },
     {
-      "epoch": 0.08,
-      "grad_norm": 32.91997500077958,
+      "epoch": 0.07905138339920949,
+      "grad_norm": 1078.6380127502835,
       "learning_rate": 3.9473684210526315e-07,
-      "logits/chosen": -4.713895320892334,
-      "logits/rejected": -5.012445449829102,
-      "logps/chosen": -223.3787841796875,
-      "logps/rejected": -201.9126434326172,
-      "loss": 0.6294,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.047253355383872986,
-      "rewards/margins": 0.1270204484462738,
-      "rewards/rejected": -0.1742737889289856,
+      "logits/chosen": -4.58280086517334,
+      "logits/rejected": -4.878857612609863,
+      "logps/chosen": -212.6325225830078,
+      "logps/rejected": -183.79238891601562,
+      "loss": 0.3906,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 3.010448932647705,
+      "rewards/margins": 2.6640021800994873,
+      "rewards/rejected": 0.346446692943573,
       "step": 30
     },
     {
-      "epoch": 0.11,
-      "grad_norm": 44.205273980146465,
+      "epoch": 0.10540184453227931,
+      "grad_norm": 1034.6942446334156,
       "learning_rate": 4.999575626062319e-07,
-      "logits/chosen": -4.862967491149902,
-      "logits/rejected": -5.199351787567139,
-      "logps/chosen": -284.15850830078125,
-      "logps/rejected": -260.1568298339844,
-      "loss": 0.5878,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.5039564371109009,
-      "rewards/margins": 0.3390721082687378,
-      "rewards/rejected": -0.8430284261703491,
+      "logits/chosen": -4.496463298797607,
+      "logits/rejected": -4.832797050476074,
+      "logps/chosen": -225.2804412841797,
+      "logps/rejected": -177.3509063720703,
+      "loss": 0.4804,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 4.241217136383057,
+      "rewards/margins": 4.989673614501953,
+      "rewards/rejected": -0.7484563589096069,
       "step": 40
     },
     {
-      "epoch": 0.13,
-      "grad_norm": 33.79573625552293,
+      "epoch": 0.13175230566534915,
+      "grad_norm": 1292.8962171873402,
       "learning_rate": 4.984737660598186e-07,
-      "logits/chosen": -4.860326766967773,
-      "logits/rejected": -5.1770920753479,
-      "logps/chosen": -287.49212646484375,
-      "logps/rejected": -275.1466979980469,
-      "loss": 0.544,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.6591841578483582,
-      "rewards/margins": 0.3825286030769348,
-      "rewards/rejected": -1.0417125225067139,
+      "logits/chosen": -4.517908573150635,
+      "logits/rejected": -4.786294937133789,
+      "logps/chosen": -214.01718139648438,
+      "logps/rejected": -174.09678649902344,
+      "loss": 0.5163,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 3.7782645225524902,
+      "rewards/margins": 5.338944435119629,
+      "rewards/rejected": -1.5606796741485596,
       "step": 50
     },
     {
-      "epoch": 0.16,
-      "grad_norm": 37.5548220875479,
+      "epoch": 0.15810276679841898,
+      "grad_norm": 1081.6576652859876,
       "learning_rate": 4.948824853131236e-07,
-      "logits/chosen": -5.250467777252197,
-      "logits/rejected": -5.646960258483887,
-      "logps/chosen": -355.8065185546875,
-      "logps/rejected": -370.16064453125,
-      "loss": 0.4873,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.3389923572540283,
-      "rewards/margins": 0.6164921522140503,
-      "rewards/rejected": -1.955484390258789,
+      "logits/chosen": -4.719171047210693,
+      "logits/rejected": -4.9818878173828125,
+      "logps/chosen": -215.00869750976562,
+      "logps/rejected": -180.1785430908203,
+      "loss": 0.4821,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 3.4492969512939453,
+      "rewards/margins": 6.232473373413086,
+      "rewards/rejected": -2.7831759452819824,
       "step": 60
     },
     {
-      "epoch": 0.18,
-      "grad_norm": 40.80845104625175,
+      "epoch": 0.1844532279314888,
+      "grad_norm": 780.7579097796144,
       "learning_rate": 4.892141805936084e-07,
-      "logits/chosen": -5.506089687347412,
-      "logits/rejected": -5.9388532638549805,
-      "logps/chosen": -373.0067443847656,
-      "logps/rejected": -409.2164001464844,
-      "loss": 0.4619,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.4274970293045044,
-      "rewards/margins": 0.8630696535110474,
-      "rewards/rejected": -2.290566921234131,
+      "logits/chosen": -4.691411018371582,
+      "logits/rejected": -4.9814910888671875,
+      "logps/chosen": -222.72189331054688,
+      "logps/rejected": -186.29788208007812,
+      "loss": 0.5005,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 3.7675652503967285,
+      "rewards/margins": 6.836648464202881,
+      "rewards/rejected": -3.0690836906433105,
       "step": 70
     },
     {
-      "epoch": 0.21,
-      "grad_norm": 62.51614299706751,
+      "epoch": 0.21080368906455862,
+      "grad_norm": 1751.5015463679365,
       "learning_rate": 4.81516928858564e-07,
-      "logits/chosen": -5.957489967346191,
-      "logits/rejected": -6.362034797668457,
-      "logps/chosen": -431.8949279785156,
-      "logps/rejected": -484.7469177246094,
-      "loss": 0.4382,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -2.0529427528381348,
-      "rewards/margins": 1.0657070875167847,
-      "rewards/rejected": -3.118650197982788,
+      "logits/chosen": -4.691437721252441,
+      "logits/rejected": -4.943267822265625,
+      "logps/chosen": -219.7614288330078,
+      "logps/rejected": -179.60899353027344,
+      "loss": 0.5615,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 3.4195969104766846,
+      "rewards/margins": 6.783135890960693,
+      "rewards/rejected": -3.363539457321167,
       "step": 80
     },
     {
-      "epoch": 0.24,
-      "grad_norm": 42.64563417225672,
+      "epoch": 0.23715415019762845,
+      "grad_norm": 1233.3333710128843,
       "learning_rate": 4.7185601601995784e-07,
-      "logits/chosen": -5.974350929260254,
-      "logits/rejected": -6.67104959487915,
-      "logps/chosen": -387.2274475097656,
-      "logps/rejected": -464.65826416015625,
-      "loss": 0.4236,
+      "logits/chosen": -4.569981575012207,
+      "logits/rejected": -4.954944610595703,
+      "logps/chosen": -212.4117431640625,
+      "logps/rejected": -173.86911010742188,
+      "loss": 0.4989,
       "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -1.6722873449325562,
-      "rewards/margins": 1.3039813041687012,
-      "rewards/rejected": -2.976268768310547,
+      "rewards/chosen": 3.7934730052948,
+      "rewards/margins": 7.21230936050415,
+      "rewards/rejected": -3.4188361167907715,
       "step": 90
     },
     {
-      "epoch": 0.26,
-      "grad_norm": 49.657859383445725,
+      "epoch": 0.2635046113306983,
+      "grad_norm": 1244.999056136522,
       "learning_rate": 4.603133832077953e-07,
-      "logits/chosen": -6.59436559677124,
-      "logits/rejected": -7.055686950683594,
-      "logps/chosen": -443.9141540527344,
-      "logps/rejected": -517.8334350585938,
-      "loss": 0.4015,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -2.304152011871338,
-      "rewards/margins": 1.163777232170105,
-      "rewards/rejected": -3.4679291248321533,
+      "logits/chosen": -4.672577381134033,
+      "logits/rejected": -4.9231157302856445,
+      "logps/chosen": -206.11221313476562,
+      "logps/rejected": -176.262451171875,
+      "loss": 0.5909,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 3.6933536529541016,
+      "rewards/margins": 6.304307460784912,
+      "rewards/rejected": -2.6109542846679688,
       "step": 100
     },
     {
-      "epoch": 0.26,
-      "eval_logits/chosen": -6.410887718200684,
-      "eval_logits/rejected": -6.793334007263184,
-      "eval_logps/chosen": -826.5081176757812,
-      "eval_logps/rejected": -987.8357543945312,
-      "eval_loss": 0.9856035113334656,
-      "eval_rewards/accuracies": 0.52734375,
-      "eval_rewards/chosen": -4.360336780548096,
-      "eval_rewards/margins": 0.3341439962387085,
-      "eval_rewards/rejected": -4.694480895996094,
-      "eval_runtime": 97.5721,
-      "eval_samples_per_second": 20.498,
-      "eval_steps_per_second": 0.328,
+      "epoch": 0.2635046113306983,
+      "eval_logits/chosen": -4.612120628356934,
+      "eval_logits/rejected": -4.85081672668457,
+      "eval_logps/chosen": -396.4924011230469,
+      "eval_logps/rejected": -512.6068115234375,
+      "eval_loss": 6.653407096862793,
+      "eval_rewards/accuracies": 0.27734375,
+      "eval_rewards/chosen": -3.0089728832244873,
+      "eval_rewards/margins": -5.8993988037109375,
+      "eval_rewards/rejected": 2.890425682067871,
+      "eval_runtime": 98.318,
+      "eval_samples_per_second": 20.342,
+      "eval_steps_per_second": 0.325,
       "step": 100
     },
     {
-      "epoch": 0.29,
-      "grad_norm": 59.660602657641974,
+      "epoch": 0.2898550724637681,
+      "grad_norm": 870.1595065644425,
       "learning_rate": 4.4698693176863316e-07,
-      "logits/chosen": -6.508014678955078,
-      "logits/rejected": -6.995993137359619,
-      "logps/chosen": -467.217041015625,
-      "logps/rejected": -571.2991943359375,
-      "loss": 0.3652,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -2.5293219089508057,
-      "rewards/margins": 1.5391861200332642,
-      "rewards/rejected": -4.068508148193359,
+      "logits/chosen": -4.71740198135376,
+      "logits/rejected": -4.952963829040527,
+      "logps/chosen": -208.0568084716797,
+      "logps/rejected": -172.92605590820312,
+      "loss": 0.4837,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 3.114009380340576,
+      "rewards/margins": 7.352834224700928,
+      "rewards/rejected": -4.238823890686035,
       "step": 110
     },
     {
-      "epoch": 0.32,
-      "grad_norm": 48.65436374701769,
+      "epoch": 0.31620553359683795,
+      "grad_norm": 1230.5660125025618,
       "learning_rate": 4.319896928940505e-07,
-      "logits/chosen": -6.631407737731934,
-      "logits/rejected": -7.224958896636963,
-      "logps/chosen": -449.9879455566406,
-      "logps/rejected": -551.552734375,
-      "loss": 0.3856,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -2.370181083679199,
-      "rewards/margins": 1.3927323818206787,
-      "rewards/rejected": -3.762913465499878,
+      "logits/chosen": -4.8249382972717285,
+      "logits/rejected": -5.139795780181885,
+      "logps/chosen": -204.9242401123047,
+      "logps/rejected": -181.16221618652344,
+      "loss": 0.6657,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 4.0227861404418945,
+      "rewards/margins": 6.973240852355957,
+      "rewards/rejected": -2.950455904006958,
       "step": 120
     },
     {
-      "epoch": 0.34,
-      "grad_norm": 41.65961698696447,
+      "epoch": 0.3425559947299078,
+      "grad_norm": 1400.2090737238004,
       "learning_rate": 4.1544886892205354e-07,
-      "logits/chosen": -6.516014099121094,
-      "logits/rejected": -7.032981872558594,
-      "logps/chosen": -490.3934020996094,
-      "logps/rejected": -610.3671264648438,
-      "loss": 0.3736,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -2.659696578979492,
-      "rewards/margins": 1.6181414127349854,
-      "rewards/rejected": -4.277838230133057,
+      "logits/chosen": -4.680369853973389,
+      "logits/rejected": -4.979363441467285,
+      "logps/chosen": -218.03897094726562,
+      "logps/rejected": -192.44757080078125,
+      "loss": 0.4732,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 3.1923911571502686,
+      "rewards/margins": 8.124491691589355,
+      "rewards/rejected": -4.932101249694824,
       "step": 130
     },
     {
-      "epoch": 0.37,
-      "grad_norm": 61.060769075088814,
+      "epoch": 0.3689064558629776,
+      "grad_norm": 1284.6303869923338,
       "learning_rate": 3.975047544428254e-07,
-      "logits/chosen": -6.8892974853515625,
-      "logits/rejected": -7.428150177001953,
-      "logps/chosen": -479.0992736816406,
-      "logps/rejected": -578.5732421875,
-      "loss": 0.3918,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -2.728147506713867,
-      "rewards/margins": 1.3954027891159058,
-      "rewards/rejected": -4.123549938201904,
+      "logits/chosen": -4.738985061645508,
+      "logits/rejected": -4.995828628540039,
+      "logps/chosen": -200.0480499267578,
+      "logps/rejected": -173.74069213867188,
+      "loss": 0.7249,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 3.118265151977539,
+      "rewards/margins": 6.879487037658691,
+      "rewards/rejected": -3.7612221240997314,
       "step": 140
     },
     {
-      "epoch": 0.4,
-      "grad_norm": 40.0921041292044,
+      "epoch": 0.3952569169960474,
+      "grad_norm": 1087.9280552765,
       "learning_rate": 3.78309546359696e-07,
-      "logits/chosen": -7.085695743560791,
-      "logits/rejected": -7.593710422515869,
-      "logps/chosen": -485.8174743652344,
-      "logps/rejected": -611.1568603515625,
-      "loss": 0.3821,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.8462297916412354,
-      "rewards/margins": 1.527190089225769,
-      "rewards/rejected": -4.373419761657715,
+      "logits/chosen": -4.816591739654541,
+      "logits/rejected": -5.046236991882324,
+      "logps/chosen": -197.75784301757812,
+      "logps/rejected": -187.12631225585938,
+      "loss": 0.6191,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 1.7183030843734741,
+      "rewards/margins": 8.374044418334961,
+      "rewards/rejected": -6.655740261077881,
       "step": 150
     },
     {
-      "epoch": 0.42,
-      "grad_norm": 45.94809301945095,
+      "epoch": 0.42160737812911725,
+      "grad_norm": 870.0436658351372,
       "learning_rate": 3.580260529980584e-07,
-      "logits/chosen": -6.79840087890625,
-      "logits/rejected": -7.57172155380249,
-      "logps/chosen": -476.08197021484375,
-      "logps/rejected": -613.65576171875,
-      "loss": 0.3533,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -2.6100144386291504,
-      "rewards/margins": 1.7850373983383179,
-      "rewards/rejected": -4.3950514793396,
+      "logits/chosen": -4.622679233551025,
+      "logits/rejected": -4.90102481842041,
+      "logps/chosen": -210.0752410888672,
+      "logps/rejected": -182.36378479003906,
+      "loss": 0.5743,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 2.5026469230651855,
+      "rewards/margins": 6.609257698059082,
+      "rewards/rejected": -4.106611251831055,
       "step": 160
     },
     {
-      "epoch": 0.45,
-      "grad_norm": 49.925985141071024,
+      "epoch": 0.4479578392621871,
+      "grad_norm": 1109.7545725706777,
       "learning_rate": 3.36826313211205e-07,
-      "logits/chosen": -7.27915096282959,
-      "logits/rejected": -7.999810695648193,
-      "logps/chosen": -464.27606201171875,
-      "logps/rejected": -618.3897705078125,
-      "loss": 0.3586,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.638946056365967,
-      "rewards/margins": 1.9340379238128662,
-      "rewards/rejected": -4.572983741760254,
+      "logits/chosen": -4.862623691558838,
+      "logits/rejected": -5.1108198165893555,
+      "logps/chosen": -195.3279266357422,
+      "logps/rejected": -170.62802124023438,
+      "loss": 0.6483,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": 2.5267701148986816,
+      "rewards/margins": 7.295065402984619,
+      "rewards/rejected": -4.768294811248779,
       "step": 170
     },
     {
-      "epoch": 0.47,
-      "grad_norm": 49.77263247276722,
+      "epoch": 0.4743083003952569,
+      "grad_norm": 1174.4955529247104,
       "learning_rate": 3.14890137195437e-07,
-      "logits/chosen": -7.062252998352051,
-      "logits/rejected": -7.757845878601074,
-      "logps/chosen": -499.0604553222656,
-      "logps/rejected": -631.4205322265625,
-      "loss": 0.35,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -2.841132640838623,
-      "rewards/margins": 1.6729885339736938,
-      "rewards/rejected": -4.514122009277344,
+      "logits/chosen": -4.831478595733643,
+      "logits/rejected": -5.037031173706055,
+      "logps/chosen": -211.2603302001953,
+      "logps/rejected": -190.32931518554688,
+      "loss": 0.5829,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 1.8433873653411865,
+      "rewards/margins": 7.003817558288574,
+      "rewards/rejected": -5.160429954528809,
       "step": 180
     },
     {
-      "epoch": 0.5,
-      "grad_norm": 47.32333832777734,
+      "epoch": 0.5006587615283268,
+      "grad_norm": 1405.2874598551575,
       "learning_rate": 2.9240358139084013e-07,
-      "logits/chosen": -7.432755947113037,
-      "logits/rejected": -8.177480697631836,
-      "logps/chosen": -571.7742919921875,
-      "logps/rejected": -753.3114013671875,
-      "loss": 0.3584,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -3.5975048542022705,
-      "rewards/margins": 2.1865601539611816,
-      "rewards/rejected": -5.784065246582031,
+      "logits/chosen": -4.9419050216674805,
+      "logits/rejected": -5.200289726257324,
+      "logps/chosen": -209.5860595703125,
+      "logps/rejected": -188.95069885253906,
+      "loss": 0.5485,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 1.218836784362793,
+      "rewards/margins": 8.241748809814453,
+      "rewards/rejected": -7.022912502288818,
       "step": 190
     },
     {
-      "epoch": 0.53,
-      "grad_norm": 47.64189353399637,
+      "epoch": 0.5270092226613966,
+      "grad_norm": 1006.939051019727,
       "learning_rate": 2.695573704031885e-07,
-      "logits/chosen": -6.939781188964844,
-      "logits/rejected": -7.71111536026001,
-      "logps/chosen": -500.54449462890625,
-      "logps/rejected": -651.9711303710938,
-      "loss": 0.3649,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -2.752312421798706,
-      "rewards/margins": 1.9267911911010742,
-      "rewards/rejected": -4.679104328155518,
+      "logits/chosen": -4.7473955154418945,
+      "logits/rejected": -5.000621795654297,
+      "logps/chosen": -217.6959686279297,
+      "logps/rejected": -190.3670196533203,
+      "loss": 0.7239,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 3.8086190223693848,
+      "rewards/margins": 6.961747646331787,
+      "rewards/rejected": -3.1531288623809814,
       "step": 200
     },
     {
-      "epoch": 0.53,
-      "eval_logits/chosen": -7.152472496032715,
-      "eval_logits/rejected": -7.637792110443115,
-      "eval_logps/chosen": -878.0755615234375,
-      "eval_logps/rejected": -1032.680908203125,
-      "eval_loss": 1.1239182949066162,
-      "eval_rewards/accuracies": 0.48828125,
-      "eval_rewards/chosen": -4.876009941101074,
-      "eval_rewards/margins": 0.26692283153533936,
-      "eval_rewards/rejected": -5.142932891845703,
-      "eval_runtime": 97.9349,
+      "epoch": 0.5270092226613966,
+      "eval_logits/chosen": -4.771502494812012,
+      "eval_logits/rejected": -4.989596843719482,
+      "eval_logps/chosen": -396.1126708984375,
+      "eval_logps/rejected": -511.37469482421875,
+      "eval_loss": 8.072031021118164,
+      "eval_rewards/accuracies": 0.2734375,
+      "eval_rewards/chosen": -2.8191022872924805,
+      "eval_rewards/margins": -6.325590133666992,
+      "eval_rewards/rejected": 3.5064878463745117,
+      "eval_runtime": 97.9327,
       "eval_samples_per_second": 20.422,
       "eval_steps_per_second": 0.327,
       "step": 200
     },
     {
-      "epoch": 0.55,
-      "grad_norm": 54.30113996467665,
+      "epoch": 0.5533596837944664,
+      "grad_norm": 1103.8948285514384,
       "learning_rate": 2.465452793317865e-07,
-      "logits/chosen": -6.918679237365723,
-      "logits/rejected": -7.81919002532959,
-      "logps/chosen": -490.4891662597656,
-      "logps/rejected": -645.8287353515625,
-      "loss": 0.351,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -2.56154727935791,
-      "rewards/margins": 1.9791193008422852,
-      "rewards/rejected": -4.5406670570373535,
+      "logits/chosen": -4.7866034507751465,
+      "logits/rejected": -5.081458568572998,
+      "logps/chosen": -228.73922729492188,
+      "logps/rejected": -201.91409301757812,
+      "loss": 0.5681,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 2.7976126670837402,
+      "rewards/margins": 7.8736419677734375,
+      "rewards/rejected": -5.076028823852539,
       "step": 210
     },
     {
-      "epoch": 0.58,
-      "grad_norm": 48.87773515687126,
+      "epoch": 0.5797101449275363,
+      "grad_norm": 715.6592810091468,
       "learning_rate": 2.2356249022388789e-07,
-      "logits/chosen": -7.1236982345581055,
-      "logits/rejected": -8.037015914916992,
-      "logps/chosen": -493.3589782714844,
-      "logps/rejected": -644.3180541992188,
-      "loss": 0.3503,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": -2.7893471717834473,
-      "rewards/margins": 2.016085386276245,
-      "rewards/rejected": -4.805432319641113,
+      "logits/chosen": -4.758913516998291,
+      "logits/rejected": -5.04398250579834,
+      "logps/chosen": -205.92568969726562,
+      "logps/rejected": -170.3586883544922,
+      "loss": 0.4895,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 4.249292373657227,
+      "rewards/margins": 7.541260719299316,
+      "rewards/rejected": -3.291968822479248,
       "step": 220
     },
     {
-      "epoch": 0.61,
-      "grad_norm": 42.654084351174774,
+      "epoch": 0.6060606060606061,
+      "grad_norm": 944.8747719910025,
       "learning_rate": 2.0080393659578038e-07,
-      "logits/chosen": -7.341279029846191,
-      "logits/rejected": -8.349458694458008,
-      "logps/chosen": -546.9940795898438,
-      "logps/rejected": -735.5299072265625,
-      "loss": 0.3336,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -3.2596652507781982,
-      "rewards/margins": 2.3632309436798096,
-      "rewards/rejected": -5.622895240783691,
+      "logits/chosen": -4.764594078063965,
+      "logits/rejected": -5.100175857543945,
+      "logps/chosen": -210.0486297607422,
+      "logps/rejected": -179.91928100585938,
+      "loss": 0.5197,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 5.489472389221191,
+      "rewards/margins": 8.828967094421387,
+      "rewards/rejected": -3.3394947052001953,
       "step": 230
     },
     {
-      "epoch": 0.63,
-      "grad_norm": 48.836991831948275,
+      "epoch": 0.6324110671936759,
+      "grad_norm": 1267.6123006349214,
       "learning_rate": 1.7846265006183976e-07,
-      "logits/chosen": -7.382364749908447,
-      "logits/rejected": -8.096385955810547,
-      "logps/chosen": -559.1055908203125,
-      "logps/rejected": -690.3636474609375,
-      "loss": 0.3239,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -3.478092670440674,
-      "rewards/margins": 1.7500022649765015,
-      "rewards/rejected": -5.228094577789307,
+      "logits/chosen": -4.821089744567871,
+      "logits/rejected": -5.0871124267578125,
+      "logps/chosen": -211.7589874267578,
+      "logps/rejected": -181.33819580078125,
+      "loss": 0.5444,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.23131565749645233,
+      "rewards/margins": 6.660643577575684,
+      "rewards/rejected": -6.891959190368652,
       "step": 240
     },
     {
-      "epoch": 0.66,
-      "grad_norm": 41.88272757196894,
+      "epoch": 0.6587615283267457,
+      "grad_norm": 911.8592996107284,
       "learning_rate": 1.5672812309497722e-07,
-      "logits/chosen": -7.3642730712890625,
-      "logits/rejected": -8.137662887573242,
-      "logps/chosen": -481.7484436035156,
-      "logps/rejected": -658.2447509765625,
-      "loss": 0.3435,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -2.742130756378174,
-      "rewards/margins": 2.204786777496338,
-      "rewards/rejected": -4.946917533874512,
+      "logits/chosen": -4.781493663787842,
+      "logits/rejected": -5.052074432373047,
+      "logps/chosen": -201.82290649414062,
+      "logps/rejected": -175.06719970703125,
+      "loss": 0.5887,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 2.856235980987549,
+      "rewards/margins": 8.613394737243652,
+      "rewards/rejected": -5.757159233093262,
       "step": 250
     },
     {
-      "epoch": 0.69,
-      "grad_norm": 42.22161557256122,
+      "epoch": 0.6851119894598156,
+      "grad_norm": 840.8020636445922,
       "learning_rate": 1.357847018050843e-07,
-      "logits/chosen": -7.285035133361816,
-      "logits/rejected": -8.04423999786377,
-      "logps/chosen": -495.21929931640625,
-      "logps/rejected": -637.4512939453125,
-      "loss": 0.3575,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -2.5677530765533447,
-      "rewards/margins": 1.859423041343689,
-      "rewards/rejected": -4.427175521850586,
+      "logits/chosen": -4.674568176269531,
+      "logits/rejected": -4.961749076843262,
+      "logps/chosen": -232.63119506835938,
+      "logps/rejected": -204.2981719970703,
+      "loss": 0.6421,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 2.906367778778076,
+      "rewards/margins": 7.688606262207031,
+      "rewards/rejected": -4.782238483428955,
       "step": 260
     },
     {
-      "epoch": 0.71,
-      "grad_norm": 36.62148112990462,
+      "epoch": 0.7114624505928854,
+      "grad_norm": 787.2425305884327,
       "learning_rate": 1.1581002236747328e-07,
-      "logits/chosen": -7.350560188293457,
-      "logits/rejected": -8.334383964538574,
-      "logps/chosen": -470.9756774902344,
-      "logps/rejected": -644.9085083007812,
-      "loss": 0.3419,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -2.7616236209869385,
-      "rewards/margins": 2.1581368446350098,
-      "rewards/rejected": -4.919760704040527,
+      "logits/chosen": -4.637770652770996,
+      "logits/rejected": -4.975251197814941,
+      "logps/chosen": -190.36253356933594,
+      "logps/rejected": -166.21788024902344,
+      "loss": 0.5113,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 2.2253878116607666,
+      "rewards/margins": 8.868097305297852,
+      "rewards/rejected": -6.6427106857299805,
       "step": 270
     },
     {
-      "epoch": 0.74,
-      "grad_norm": 42.46981787888043,
+      "epoch": 0.7378129117259552,
+      "grad_norm": 1064.0700290859438,
       "learning_rate": 9.697350436308427e-08,
-      "logits/chosen": -7.353733062744141,
-      "logits/rejected": -8.123218536376953,
-      "logps/chosen": -547.2511596679688,
-      "logps/rejected": -669.4581298828125,
-      "loss": 0.3277,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -3.1215298175811768,
-      "rewards/margins": 1.7408149242401123,
-      "rewards/rejected": -4.862344264984131,
+      "logits/chosen": -4.585692405700684,
+      "logits/rejected": -4.871885776519775,
+      "logps/chosen": -229.0143585205078,
+      "logps/rejected": -194.08761596679688,
+      "loss": 0.5252,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 3.0419421195983887,
+      "rewards/margins": 8.473891258239746,
+      "rewards/rejected": -5.431949138641357,
       "step": 280
     },
     {
-      "epoch": 0.76,
-      "grad_norm": 45.06982020026593,
+      "epoch": 0.764163372859025,
+      "grad_norm": 839.1911415542294,
       "learning_rate": 7.943491380952188e-08,
-      "logits/chosen": -7.6978254318237305,
-      "logits/rejected": -8.468889236450195,
-      "logps/chosen": -488.8667907714844,
-      "logps/rejected": -652.3041381835938,
-      "loss": 0.3357,
+      "logits/chosen": -4.828024864196777,
+      "logits/rejected": -5.060397148132324,
+      "logps/chosen": -196.42611694335938,
+      "logps/rejected": -171.01446533203125,
+      "loss": 0.4822,
       "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -2.835703134536743,
-      "rewards/margins": 2.056410789489746,
-      "rewards/rejected": -4.89211368560791,
+      "rewards/chosen": 4.435191631317139,
+      "rewards/margins": 8.396050453186035,
+      "rewards/rejected": -3.9608588218688965,
       "step": 290
     },
     {
-      "epoch": 0.79,
-      "grad_norm": 50.46942762667779,
+      "epoch": 0.7905138339920948,
+      "grad_norm": 887.4606387846492,
       "learning_rate": 6.334300807088508e-08,
-      "logits/chosen": -7.3240485191345215,
-      "logits/rejected": -8.329621315002441,
-      "logps/chosen": -509.5638122558594,
-      "logps/rejected": -673.7567749023438,
-      "loss": 0.3506,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -3.0604350566864014,
-      "rewards/margins": 2.1173126697540283,
-      "rewards/rejected": -5.1777472496032715,
+      "logits/chosen": -4.608688831329346,
+      "logits/rejected": -4.9593987464904785,
+      "logps/chosen": -195.76962280273438,
+      "logps/rejected": -164.97048950195312,
+      "loss": 0.5556,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 3.875316619873047,
+      "rewards/margins": 8.369561195373535,
+      "rewards/rejected": -4.4942450523376465,
       "step": 300
     },
     {
-      "epoch": 0.79,
-      "eval_logits/chosen": -7.444587707519531,
-      "eval_logits/rejected": -7.9900360107421875,
-      "eval_logps/chosen": -951.4573974609375,
-      "eval_logps/rejected": -1114.78076171875,
-      "eval_loss": 1.1842519044876099,
-      "eval_rewards/accuracies": 0.51171875,
-      "eval_rewards/chosen": -5.609828948974609,
-      "eval_rewards/margins": 0.35410135984420776,
-      "eval_rewards/rejected": -5.963930606842041,
-      "eval_runtime": 97.9073,
-      "eval_samples_per_second": 20.427,
-      "eval_steps_per_second": 0.327,
+      "epoch": 0.7905138339920948,
+      "eval_logits/chosen": -4.660388469696045,
+      "eval_logits/rejected": -4.890798091888428,
+      "eval_logps/chosen": -399.5094909667969,
+      "eval_logps/rejected": -517.530029296875,
+      "eval_loss": 6.923000335693359,
+      "eval_rewards/accuracies": 0.31640625,
+      "eval_rewards/chosen": -4.517519474029541,
+      "eval_rewards/margins": -4.946353435516357,
+      "eval_rewards/rejected": 0.42883408069610596,
+      "eval_runtime": 99.1917,
+      "eval_samples_per_second": 20.163,
+      "eval_steps_per_second": 0.323,
       "step": 300
     },
     {
-      "epoch": 0.82,
-      "grad_norm": 45.51913834484837,
+      "epoch": 0.8168642951251647,
+      "grad_norm": 1030.5057638496492,
       "learning_rate": 4.8834274139883084e-08,
-      "logits/chosen": -7.380696773529053,
-      "logits/rejected": -8.350247383117676,
-      "logps/chosen": -504.37054443359375,
-      "logps/rejected": -678.6265869140625,
-      "loss": 0.3248,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -2.946779727935791,
-      "rewards/margins": 2.2168285846710205,
-      "rewards/rejected": -5.163609027862549,
+      "logits/chosen": -4.663365840911865,
+      "logits/rejected": -5.023074150085449,
+      "logps/chosen": -201.54336547851562,
+      "logps/rejected": -171.71324157714844,
+      "loss": 0.533,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 4.074584007263184,
+      "rewards/margins": 8.798391342163086,
+      "rewards/rejected": -4.723808765411377,
       "step": 310
     },
     {
-      "epoch": 0.84,
-      "grad_norm": 53.031113784661194,
+      "epoch": 0.8432147562582345,
+      "grad_norm": 1279.040936993484,
       "learning_rate": 3.60317709937693e-08,
-      "logits/chosen": -7.483295440673828,
-      "logits/rejected": -8.33633804321289,
-      "logps/chosen": -519.6754150390625,
-      "logps/rejected": -667.0064697265625,
-      "loss": 0.334,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -2.902660369873047,
-      "rewards/margins": 2.0172414779663086,
-      "rewards/rejected": -4.919901371002197,
+      "logits/chosen": -4.707262992858887,
+      "logits/rejected": -5.011012077331543,
+      "logps/chosen": -223.0199432373047,
+      "logps/rejected": -183.51393127441406,
+      "loss": 0.5239,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": 3.1946961879730225,
+      "rewards/margins": 7.443505764007568,
+      "rewards/rejected": -4.248808860778809,
       "step": 320
     },
     {
-      "epoch": 0.87,
-      "grad_norm": 41.68995945520798,
+      "epoch": 0.8695652173913043,
+      "grad_norm": 790.9511672313881,
       "learning_rate": 2.5044085842905683e-08,
-      "logits/chosen": -7.596086025238037,
-      "logits/rejected": -8.42108154296875,
-      "logps/chosen": -542.1593627929688,
-      "logps/rejected": -761.1915893554688,
-      "loss": 0.3439,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": -3.237351894378662,
-      "rewards/margins": 2.630371332168579,
-      "rewards/rejected": -5.867722988128662,
+      "logits/chosen": -4.707150459289551,
+      "logits/rejected": -4.967694282531738,
+      "logps/chosen": -208.9776153564453,
+      "logps/rejected": -184.1087188720703,
+      "loss": 0.5674,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 4.723270893096924,
+      "rewards/margins": 9.567978858947754,
+      "rewards/rejected": -4.844708442687988,
       "step": 330
     },
     {
-      "epoch": 0.9,
-      "grad_norm": 44.38600213815557,
+      "epoch": 0.8959156785243741,
+      "grad_norm": 1075.929120672392,
       "learning_rate": 1.5964413124758493e-08,
-      "logits/chosen": -7.379315376281738,
-      "logits/rejected": -8.234747886657715,
-      "logps/chosen": -516.7742919921875,
-      "logps/rejected": -690.0197143554688,
-      "loss": 0.3394,
+      "logits/chosen": -4.645999431610107,
+      "logits/rejected": -4.947402477264404,
+      "logps/chosen": -212.19015502929688,
+      "logps/rejected": -185.76678466796875,
+      "loss": 0.5578,
       "rewards/accuracies": 0.875,
-      "rewards/chosen": -2.9615204334259033,
-      "rewards/margins": 2.160275936126709,
-      "rewards/rejected": -5.121796607971191,
+      "rewards/chosen": 4.216076374053955,
+      "rewards/margins": 8.17945671081543,
+      "rewards/rejected": -3.9633796215057373,
       "step": 340
     },
     {
-      "epoch": 0.92,
-      "grad_norm": 49.890826040889124,
+      "epoch": 0.922266139657444,
+      "grad_norm": 1055.0113319220882,
       "learning_rate": 8.869764055041501e-09,
-      "logits/chosen": -7.525488376617432,
-      "logits/rejected": -8.198974609375,
-      "logps/chosen": -535.8231201171875,
-      "logps/rejected": -737.1505737304688,
-      "loss": 0.3403,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -3.1437525749206543,
-      "rewards/margins": 2.3052563667297363,
-      "rewards/rejected": -5.449008941650391,
+      "logits/chosen": -4.714905738830566,
+      "logits/rejected": -4.937991619110107,
+      "logps/chosen": -215.5680389404297,
+      "logps/rejected": -200.6318359375,
+      "loss": 0.5113,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 2.9399425983428955,
+      "rewards/margins": 7.131015777587891,
+      "rewards/rejected": -4.191073417663574,
       "step": 350
     },
     {
-      "epoch": 0.95,
-      "grad_norm": 41.28866479778979,
+      "epoch": 0.9486166007905138,
+      "grad_norm": 1003.3923870155849,
       "learning_rate": 3.82031344036729e-09,
-      "logits/chosen": -7.295458793640137,
-      "logits/rejected": -8.147361755371094,
-      "logps/chosen": -541.268798828125,
-      "logps/rejected": -692.4953002929688,
-      "loss": 0.3191,
+      "logits/chosen": -4.614500999450684,
+      "logits/rejected": -4.9195661544799805,
+      "logps/chosen": -216.197998046875,
+      "logps/rejected": -186.8596649169922,
+      "loss": 0.4994,
       "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -3.197694778442383,
-      "rewards/margins": 1.9286365509033203,
-      "rewards/rejected": -5.126331329345703,
+      "rewards/chosen": 2.650672197341919,
+      "rewards/margins": 6.149393081665039,
+      "rewards/rejected": -3.49872088432312,
       "step": 360
     },
     {
-      "epoch": 0.97,
-      "grad_norm": 56.54843526627352,
+      "epoch": 0.9749670619235836,
+      "grad_norm": 1021.842139855702,
       "learning_rate": 8.588892925590063e-10,
-      "logits/chosen": -7.414445400238037,
-      "logits/rejected": -8.447690963745117,
-      "logps/chosen": -499.84197998046875,
-      "logps/rejected": -670.2615966796875,
-      "loss": 0.3342,
+      "logits/chosen": -4.69917631149292,
+      "logits/rejected": -5.054296016693115,
+      "logps/chosen": -212.79714965820312,
+      "logps/rejected": -171.7442169189453,
+      "loss": 0.5535,
       "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -2.8063206672668457,
-      "rewards/margins": 2.2892918586730957,
-      "rewards/rejected": -5.0956130027771,
+      "rewards/chosen": 3.2063796520233154,
+      "rewards/margins": 8.728368759155273,
+      "rewards/rejected": -5.521987438201904,
       "step": 370
     },
     {
-      "epoch": 1.0,
+      "epoch": 0.9986824769433466,
       "step": 379,
       "total_flos": 0.0,
-      "train_loss": 0.39733357479830217,
-      "train_runtime": 5845.1464,
-      "train_samples_per_second": 8.303,
-      "train_steps_per_second": 0.065
+      "train_loss": 0.5517442987587962,
+      "train_runtime": 6181.8185,
+      "train_samples_per_second": 7.85,
+      "train_steps_per_second": 0.061
     }
   ],
   "logging_steps": 10,
@@ -641,6 +641,18 @@
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
   "total_flos": 0.0,
   "train_batch_size": 8,
   "trial_name": null,