Model save

README.md

@@ -17,15 +17,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.0712
-- Rewards/chosen: -2.3718
-- Rewards/rejected: -2.8225
+- Loss: 0.0660
+- Rewards/chosen: -2.5606
+- Rewards/rejected: -2.9549
 - Rewards/accuracies: 0.625
-- Rewards/margins: 0.4507
-- Logps/rejected: -539.6053
-- Logps/chosen: -494.2236
-- Logits/rejected: -2.2822
-- Logits/chosen: -2.3030
+- Rewards/margins: 0.3944
+- Logps/rejected: -552.8470
+- Logps/chosen: -513.0960
+- Logits/rejected: -2.2459
+- Logits/chosen: -2.2708
 
 ## Model description
 
@@ -47,7 +47,7 @@ The following hyperparameters were used during training:
 - learning_rate: 5e-07
 - train_batch_size: 8
 - eval_batch_size: 8
-- seed: 3
+- seed: 4
 - distributed_type: multi-GPU
 - num_devices: 8
 - gradient_accumulation_steps: 2
@@ -62,10 +62,10 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.0594        | 0.25  | 100  | 0.1035          | -1.7191        | -1.9450          | 0.6172             | 0.2259          | -451.8574      | -428.9503    | -2.3270         | -2.3408       |
-| 0.0329        | 0.49  | 200  | 0.0693          | -2.4492        | -2.8068          | 0.6094             | 0.3576          | -538.0304      | -501.9568    | -2.2147         | -2.2352       |
-| 0.0312        | 0.74  | 300  | 0.0689          | -2.4412        | -2.8616          | 0.6133             | 0.4204          | -543.5178      | -501.1634    | -2.2721         | -2.2933       |
-| 0.0331        | 0.99  | 400  | 0.0712          | -2.3718        | -2.8225          | 0.625              | 0.4507          | -539.6053      | -494.2236    | -2.2822         | -2.3030       |
+| 0.0437        | 0.25  | 100  | 0.0824          | -2.2538        | -2.4741          | 0.5859             | 0.2203          | -504.7590      | -482.4154    | -2.3143         | -2.3260       |
+| 0.0258        | 0.49  | 200  | 0.0581          | -2.8677        | -3.2192          | 0.5977             | 0.3515          | -579.2755      | -543.8072    | -2.1155         | -2.1394       |
+| 0.0402        | 0.74  | 300  | 0.0837          | -2.0997        | -2.5006          | 0.6289             | 0.4009          | -507.4115      | -467.0057    | -2.2751         | -2.2980       |
+| 0.0288        | 0.99  | 400  | 0.0660          | -2.5606        | -2.9549          | 0.625              | 0.3944          | -552.8470      | -513.0960    | -2.2459         | -2.2708       |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.0722552685457983,
-    "train_runtime": 3732.8792,
+    "train_loss": 0.07151281171374851,
+    "train_runtime": 3738.25,
     "train_samples": 51894,
-    "train_samples_per_second": 13.902,
+    "train_samples_per_second": 13.882,
     "train_steps_per_second": 0.108
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df5f8551f34bd5fa2c36c62a9e1e02db72d830f8b080213c70f6615d9f81b129
+oid sha256:fb2741318d25fa010663fe61ed02f4f293fa8ad301934c24bbabaf6e60633fb3
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec4273703f96f7c3f2cf7aaa5e04be8cea024440c644d9e1a06f6ec8a234f06a
+oid sha256:0f559ba79771ca5e5cdbae085a55b0de304927c43a4793b3f8234d1f33152354
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9d1a5ffde550f3d000a899abb7c1f554363bee7053e537892516534d9b1b6cf9
+oid sha256:901f2833dc2e0f3adb4f4bd18d3a372877da1018c193c779ed31f78d98f1f0a4
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.0722552685457983,
-    "train_runtime": 3732.8792,
+    "train_loss": 0.07151281171374851,
+    "train_runtime": 3738.25,
     "train_samples": 51894,
-    "train_samples_per_second": 13.902,
+    "train_samples_per_second": 13.882,
     "train_steps_per_second": 0.108
 }

trainer_state.json

@@ -11,11 +11,11 @@
     {
       "epoch": 0.0,
       "learning_rate": 1.2195121951219512e-08,
-      "logits/chosen": -2.8088459968566895,
-      "logits/rejected": -2.7595884799957275,
-      "logps/chosen": -368.90777587890625,
-      "logps/rejected": -133.10202026367188,
-      "loss": 0.3669,
+      "logits/chosen": -2.8681135177612305,
+      "logits/rejected": -2.8858838081359863,
+      "logps/chosen": -518.1907958984375,
+      "logps/rejected": -109.31971740722656,
+      "loss": 0.3475,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,634 +25,634 @@
     {
       "epoch": 0.02,
       "learning_rate": 1.219512195121951e-07,
-      "logits/chosen": -2.838677406311035,
-      "logits/rejected": -2.8248190879821777,
-      "logps/chosen": -433.822265625,
-      "logps/rejected": -114.71543884277344,
-      "loss": 0.3373,
-      "rewards/accuracies": 0.5555555820465088,
-      "rewards/chosen": 0.0010175479110330343,
-      "rewards/margins": 0.0018583540804684162,
-      "rewards/rejected": -0.0008408060530200601,
+      "logits/chosen": -2.7986178398132324,
+      "logits/rejected": -2.752176284790039,
+      "logps/chosen": -434.208251953125,
+      "logps/rejected": -114.19618225097656,
+      "loss": 0.3394,
+      "rewards/accuracies": 0.4930555522441864,
+      "rewards/chosen": 0.0002649651141837239,
+      "rewards/margins": 0.0009347840095870197,
+      "rewards/rejected": -0.0006698188371956348,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.439024390243902e-07,
-      "logits/chosen": -2.798461437225342,
-      "logits/rejected": -2.765454053878784,
-      "logps/chosen": -436.7164001464844,
-      "logps/rejected": -109.3239517211914,
-      "loss": 0.3366,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": 0.020252179354429245,
-      "rewards/margins": 0.03614808991551399,
-      "rewards/rejected": -0.015895914286375046,
+      "logits/chosen": -2.8215415477752686,
+      "logits/rejected": -2.7983882427215576,
+      "logps/chosen": -417.2633361816406,
+      "logps/rejected": -118.0062026977539,
+      "loss": 0.3373,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.019945567473769188,
+      "rewards/margins": 0.03575458750128746,
+      "rewards/rejected": -0.015809018164873123,
       "step": 20
     },
     {
       "epoch": 0.07,
       "learning_rate": 3.6585365853658536e-07,
-      "logits/chosen": -2.7184653282165527,
-      "logits/rejected": -2.6913540363311768,
-      "logps/chosen": -422.36480712890625,
-      "logps/rejected": -127.92415618896484,
-      "loss": 0.3034,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": 0.06996239721775055,
-      "rewards/margins": 0.19669881463050842,
-      "rewards/rejected": -0.12673643231391907,
+      "logits/chosen": -2.6574862003326416,
+      "logits/rejected": -2.6451315879821777,
+      "logps/chosen": -398.87353515625,
+      "logps/rejected": -125.69970703125,
+      "loss": 0.3045,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.07569055259227753,
+      "rewards/margins": 0.19884702563285828,
+      "rewards/rejected": -0.12315647304058075,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.878048780487804e-07,
-      "logits/chosen": -2.592528820037842,
-      "logits/rejected": -2.5740997791290283,
-      "logps/chosen": -396.34332275390625,
-      "logps/rejected": -138.47140502929688,
-      "loss": 0.2563,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": 0.023515433073043823,
-      "rewards/margins": 0.41449323296546936,
-      "rewards/rejected": -0.39097777009010315,
+      "logits/chosen": -2.54256272315979,
+      "logits/rejected": -2.5281729698181152,
+      "logps/chosen": -384.5321044921875,
+      "logps/rejected": -168.55758666992188,
+      "loss": 0.2564,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.024631643667817116,
+      "rewards/margins": 0.41851943731307983,
+      "rewards/rejected": -0.39388787746429443,
       "step": 40
     },
     {
       "epoch": 0.12,
       "learning_rate": 4.992461696250783e-07,
-      "logits/chosen": -2.425698757171631,
-      "logits/rejected": -2.399880886077881,
-      "logps/chosen": -445.71978759765625,
-      "logps/rejected": -201.20761108398438,
-      "loss": 0.1773,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.034065067768096924,
-      "rewards/margins": 0.8275578618049622,
-      "rewards/rejected": -0.8616229295730591,
+      "logits/chosen": -2.4257261753082275,
+      "logits/rejected": -2.3928446769714355,
+      "logps/chosen": -436.45330810546875,
+      "logps/rejected": -219.0617218017578,
+      "loss": 0.1809,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.1671580970287323,
+      "rewards/margins": 0.7879143953323364,
+      "rewards/rejected": -0.9550724029541016,
       "step": 50
     },
     {
       "epoch": 0.15,
       "learning_rate": 4.966461721767899e-07,
-      "logits/chosen": -2.4016242027282715,
-      "logits/rejected": -2.3502964973449707,
-      "logps/chosen": -424.775390625,
-      "logps/rejected": -253.54776000976562,
-      "loss": 0.1294,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.3768869638442993,
-      "rewards/margins": 0.9074532389640808,
-      "rewards/rejected": -1.2843403816223145,
+      "logits/chosen": -2.3805835247039795,
+      "logits/rejected": -2.3364853858947754,
+      "logps/chosen": -437.4466857910156,
+      "logps/rejected": -240.6685791015625,
+      "loss": 0.1377,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.32454290986061096,
+      "rewards/margins": 0.9316140413284302,
+      "rewards/rejected": -1.2561569213867188,
       "step": 60
     },
     {
       "epoch": 0.17,
       "learning_rate": 4.922100518015975e-07,
-      "logits/chosen": -2.43666410446167,
-      "logits/rejected": -2.387927293777466,
-      "logps/chosen": -420.531494140625,
-      "logps/rejected": -273.5174255371094,
-      "loss": 0.1116,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.3666774034500122,
-      "rewards/margins": 1.1816540956497192,
-      "rewards/rejected": -1.548331618309021,
+      "logits/chosen": -2.3752458095550537,
+      "logits/rejected": -2.3281030654907227,
+      "logps/chosen": -419.6747131347656,
+      "logps/rejected": -264.75787353515625,
+      "loss": 0.103,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.3937300443649292,
+      "rewards/margins": 1.1917842626571655,
+      "rewards/rejected": -1.5855143070220947,
       "step": 70
     },
     {
       "epoch": 0.2,
       "learning_rate": 4.859708325770919e-07,
-      "logits/chosen": -2.37559175491333,
-      "logits/rejected": -2.327603816986084,
-      "logps/chosen": -472.6153259277344,
-      "logps/rejected": -317.5882873535156,
-      "loss": 0.0637,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.8155827522277832,
-      "rewards/margins": 1.3035672903060913,
-      "rewards/rejected": -2.119150161743164,
+      "logits/chosen": -2.4320530891418457,
+      "logits/rejected": -2.3738579750061035,
+      "logps/chosen": -472.10479736328125,
+      "logps/rejected": -330.32403564453125,
+      "loss": 0.0674,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -0.5765678286552429,
+      "rewards/margins": 1.5421369075775146,
+      "rewards/rejected": -2.1187047958374023,
       "step": 80
     },
     {
       "epoch": 0.22,
       "learning_rate": 4.779749614980225e-07,
-      "logits/chosen": -2.3662772178649902,
-      "logits/rejected": -2.3145246505737305,
-      "logps/chosen": -546.580810546875,
-      "logps/rejected": -391.6395263671875,
-      "loss": 0.0501,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -0.7051855325698853,
-      "rewards/margins": 1.912410020828247,
-      "rewards/rejected": -2.617595672607422,
+      "logits/chosen": -2.3991949558258057,
+      "logits/rejected": -2.357053279876709,
+      "logps/chosen": -487.83074951171875,
+      "logps/rejected": -349.1925354003906,
+      "loss": 0.0553,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.666537880897522,
+      "rewards/margins": 1.7182201147079468,
+      "rewards/rejected": -2.3847577571868896,
       "step": 90
     },
     {
       "epoch": 0.25,
       "learning_rate": 4.682819627081427e-07,
-      "logits/chosen": -2.3446455001831055,
-      "logits/rejected": -2.278437852859497,
-      "logps/chosen": -482.21063232421875,
-      "logps/rejected": -363.7936096191406,
-      "loss": 0.0594,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -0.6942282915115356,
-      "rewards/margins": 1.7591311931610107,
-      "rewards/rejected": -2.4533591270446777,
+      "logits/chosen": -2.3752927780151367,
+      "logits/rejected": -2.326216220855713,
+      "logps/chosen": -515.1549682617188,
+      "logps/rejected": -378.8877868652344,
+      "loss": 0.0437,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.6667075157165527,
+      "rewards/margins": 2.000246524810791,
+      "rewards/rejected": -2.666954278945923,
       "step": 100
     },
     {
       "epoch": 0.25,
-      "eval_logits/chosen": -2.340813636779785,
-      "eval_logits/rejected": -2.327035903930664,
-      "eval_logps/chosen": -428.9503173828125,
-      "eval_logps/rejected": -451.85736083984375,
-      "eval_loss": 0.10351637005805969,
-      "eval_rewards/accuracies": 0.6171875,
-      "eval_rewards/chosen": -1.7191063165664673,
-      "eval_rewards/margins": 0.22593416273593903,
-      "eval_rewards/rejected": -1.9450405836105347,
-      "eval_runtime": 53.3665,
-      "eval_samples_per_second": 37.477,
+      "eval_logits/chosen": -2.3259778022766113,
+      "eval_logits/rejected": -2.314302682876587,
+      "eval_logps/chosen": -482.4153747558594,
+      "eval_logps/rejected": -504.759033203125,
+      "eval_loss": 0.08243285864591599,
+      "eval_rewards/accuracies": 0.5859375,
+      "eval_rewards/chosen": -2.2537574768066406,
+      "eval_rewards/margins": 0.22029951214790344,
+      "eval_rewards/rejected": -2.4740567207336426,
+      "eval_runtime": 53.3582,
+      "eval_samples_per_second": 37.483,
       "eval_steps_per_second": 0.6,
       "step": 100
     },
     {
       "epoch": 0.27,
       "learning_rate": 4.569639943810477e-07,
-      "logits/chosen": -2.3267300128936768,
-      "logits/rejected": -2.256336212158203,
-      "logps/chosen": -502.18572998046875,
-      "logps/rejected": -387.1337890625,
-      "loss": 0.0472,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.9502252340316772,
-      "rewards/margins": 1.751552939414978,
-      "rewards/rejected": -2.7017781734466553,
+      "logits/chosen": -2.3144371509552,
+      "logits/rejected": -2.2340025901794434,
+      "logps/chosen": -490.12921142578125,
+      "logps/rejected": -419.07867431640625,
+      "loss": 0.0359,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.3260681629180908,
+      "rewards/margins": 1.7610738277435303,
+      "rewards/rejected": -3.0871422290802,
       "step": 110
     },
     {
       "epoch": 0.3,
       "learning_rate": 4.4410531154874543e-07,
-      "logits/chosen": -2.3445639610290527,
-      "logits/rejected": -2.2553389072418213,
-      "logps/chosen": -552.4199829101562,
-      "logps/rejected": -416.80755615234375,
-      "loss": 0.0477,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.0058166980743408,
-      "rewards/margins": 1.8569440841674805,
-      "rewards/rejected": -2.8627610206604004,
+      "logits/chosen": -2.3716444969177246,
+      "logits/rejected": -2.3235533237457275,
+      "logps/chosen": -466.01702880859375,
+      "logps/rejected": -356.8735046386719,
+      "loss": 0.0575,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.7180399298667908,
+      "rewards/margins": 1.6505486965179443,
+      "rewards/rejected": -2.368588924407959,
       "step": 120
     },
     {
       "epoch": 0.32,
       "learning_rate": 4.298016388768561e-07,
-      "logits/chosen": -2.396329641342163,
-      "logits/rejected": -2.322551727294922,
-      "logps/chosen": -542.0057373046875,
-      "logps/rejected": -407.68634033203125,
-      "loss": 0.0418,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -0.768031895160675,
-      "rewards/margins": 2.077030658721924,
-      "rewards/rejected": -2.845062017440796,
+      "logits/chosen": -2.3074584007263184,
+      "logits/rejected": -2.257930040359497,
+      "logps/chosen": -472.1845703125,
+      "logps/rejected": -373.66522216796875,
+      "loss": 0.0498,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.0088322162628174,
+      "rewards/margins": 1.6445964574813843,
+      "rewards/rejected": -2.653428792953491,
       "step": 130
     },
     {
       "epoch": 0.35,
       "learning_rate": 4.1415945805573005e-07,
-      "logits/chosen": -2.3263237476348877,
-      "logits/rejected": -2.2574667930603027,
-      "logps/chosen": -506.77471923828125,
-      "logps/rejected": -388.97479248046875,
-      "loss": 0.0506,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.8580313920974731,
-      "rewards/margins": 1.7057987451553345,
-      "rewards/rejected": -2.5638298988342285,
+      "logits/chosen": -2.225804328918457,
+      "logits/rejected": -2.15400767326355,
+      "logps/chosen": -534.1700439453125,
+      "logps/rejected": -430.3104553222656,
+      "loss": 0.0361,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.042690634727478,
+      "rewards/margins": 2.092653751373291,
+      "rewards/rejected": -3.1353445053100586,
       "step": 140
     },
     {
       "epoch": 0.37,
       "learning_rate": 3.972952151123984e-07,
-      "logits/chosen": -2.3322761058807373,
-      "logits/rejected": -2.2486355304718018,
-      "logps/chosen": -450.03778076171875,
-      "logps/rejected": -351.47064208984375,
-      "loss": 0.0528,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -0.7531972527503967,
-      "rewards/margins": 1.7522554397583008,
-      "rewards/rejected": -2.505452871322632,
+      "logits/chosen": -2.2562787532806396,
+      "logits/rejected": -2.164506673812866,
+      "logps/chosen": -522.7659912109375,
+      "logps/rejected": -425.18109130859375,
+      "loss": 0.0344,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.1226718425750732,
+      "rewards/margins": 2.0036892890930176,
+      "rewards/rejected": -3.12636137008667,
       "step": 150
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.793344535444142e-07,
-      "logits/chosen": -2.298706531524658,
-      "logits/rejected": -2.205777168273926,
-      "logps/chosen": -549.6655883789062,
-      "logps/rejected": -407.4877624511719,
-      "loss": 0.0361,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -0.8248310089111328,
-      "rewards/margins": 2.1384449005126953,
-      "rewards/rejected": -2.963275909423828,
+      "logits/chosen": -2.267565965652466,
+      "logits/rejected": -2.1969974040985107,
+      "logps/chosen": -530.3189086914062,
+      "logps/rejected": -426.72332763671875,
+      "loss": 0.0393,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.2129985094070435,
+      "rewards/margins": 1.8231168985366821,
+      "rewards/rejected": -3.0361156463623047,
       "step": 160
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.604108797288461e-07,
-      "logits/chosen": -2.301478862762451,
-      "logits/rejected": -2.199977397918701,
-      "logps/chosen": -550.0228271484375,
-      "logps/rejected": -447.4345703125,
-      "loss": 0.0349,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.1104724407196045,
-      "rewards/margins": 2.2591710090637207,
-      "rewards/rejected": -3.369643449783325,
+      "logits/chosen": -2.237342119216919,
+      "logits/rejected": -2.1961898803710938,
+      "logps/chosen": -448.13812255859375,
+      "logps/rejected": -372.9068298339844,
+      "loss": 0.0465,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.092370629310608,
+      "rewards/margins": 1.5985119342803955,
+      "rewards/rejected": -2.690882444381714,
       "step": 170
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.40665367563858e-07,
-      "logits/chosen": -2.2790443897247314,
-      "logits/rejected": -2.1830639839172363,
-      "logps/chosen": -540.7822265625,
-      "logps/rejected": -438.80816650390625,
-      "loss": 0.0358,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -1.3068325519561768,
-      "rewards/margins": 1.9258372783660889,
-      "rewards/rejected": -3.2326698303222656,
+      "logits/chosen": -2.2571911811828613,
+      "logits/rejected": -2.140353202819824,
+      "logps/chosen": -548.1529541015625,
+      "logps/rejected": -449.4532165527344,
+      "loss": 0.035,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -0.8807584643363953,
+      "rewards/margins": 2.355053663253784,
+      "rewards/rejected": -3.2358124256134033,
       "step": 180
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.202449097526798e-07,
-      "logits/chosen": -2.2940845489501953,
-      "logits/rejected": -2.213531732559204,
-      "logps/chosen": -518.0568237304688,
-      "logps/rejected": -424.33331298828125,
-      "loss": 0.0358,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -1.1591523885726929,
-      "rewards/margins": 2.0107340812683105,
-      "rewards/rejected": -3.169886350631714,
+      "logits/chosen": -2.1954236030578613,
+      "logits/rejected": -2.113832950592041,
+      "logps/chosen": -545.7277221679688,
+      "logps/rejected": -466.76580810546875,
+      "loss": 0.029,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.4056795835494995,
+      "rewards/margins": 2.1022555828094482,
+      "rewards/rejected": -3.5079357624053955,
       "step": 190
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.993015235369905e-07,
-      "logits/chosen": -2.2501273155212402,
-      "logits/rejected": -2.1389498710632324,
-      "logps/chosen": -568.6901245117188,
-      "logps/rejected": -470.89617919921875,
-      "loss": 0.0329,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.2941691875457764,
-      "rewards/margins": 2.236302375793457,
-      "rewards/rejected": -3.5304713249206543,
+      "logits/chosen": -2.1386027336120605,
+      "logits/rejected": -2.0572166442871094,
+      "logps/chosen": -560.2534790039062,
+      "logps/rejected": -491.8816833496094,
+      "loss": 0.0258,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.810624361038208,
+      "rewards/margins": 1.9691530466079712,
+      "rewards/rejected": -3.7797775268554688,
       "step": 200
     },
     {
       "epoch": 0.49,
-      "eval_logits/chosen": -2.2352473735809326,
-      "eval_logits/rejected": -2.214733362197876,
-      "eval_logps/chosen": -501.9567565917969,
-      "eval_logps/rejected": -538.0303955078125,
-      "eval_loss": 0.06932022422552109,
-      "eval_rewards/accuracies": 0.609375,
-      "eval_rewards/chosen": -2.449171304702759,
-      "eval_rewards/margins": 0.35759952664375305,
-      "eval_rewards/rejected": -2.8067705631256104,
-      "eval_runtime": 53.3061,
-      "eval_samples_per_second": 37.519,
-      "eval_steps_per_second": 0.6,
+      "eval_logits/chosen": -2.1394448280334473,
+      "eval_logits/rejected": -2.1155476570129395,
+      "eval_logps/chosen": -543.8071899414062,
+      "eval_logps/rejected": -579.2755126953125,
+      "eval_loss": 0.0581156425178051,
+      "eval_rewards/accuracies": 0.59765625,
+      "eval_rewards/chosen": -2.86767578125,
+      "eval_rewards/margins": 0.35154610872268677,
+      "eval_rewards/rejected": -3.219222068786621,
+      "eval_runtime": 53.2701,
+      "eval_samples_per_second": 37.545,
+      "eval_steps_per_second": 0.601,
       "step": 200
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7799111902582693e-07,
-      "logits/chosen": -2.2516720294952393,
-      "logits/rejected": -2.1468265056610107,
-      "logps/chosen": -544.9647216796875,
-      "logps/rejected": -425.84832763671875,
-      "loss": 0.0319,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.4447880983352661,
-      "rewards/margins": 1.7926721572875977,
-      "rewards/rejected": -3.2374606132507324,
+      "logits/chosen": -2.1782305240631104,
+      "logits/rejected": -2.044674873352051,
+      "logps/chosen": -579.908935546875,
+      "logps/rejected": -500.6641540527344,
+      "loss": 0.0219,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.408044695854187,
+      "rewards/margins": 2.4992563724517822,
+      "rewards/rejected": -3.9073009490966797,
       "step": 210
     },
     {
       "epoch": 0.54,
       "learning_rate": 2.564723385445869e-07,
-      "logits/chosen": -2.325510025024414,
-      "logits/rejected": -2.2458481788635254,
-      "logps/chosen": -532.0316772460938,
-      "logps/rejected": -426.2433166503906,
-      "loss": 0.0441,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.1441152095794678,
-      "rewards/margins": 1.8752161264419556,
-      "rewards/rejected": -3.019331455230713,
+      "logits/chosen": -2.2589755058288574,
+      "logits/rejected": -2.156228542327881,
+      "logps/chosen": -563.1976318359375,
+      "logps/rejected": -475.75030517578125,
+      "loss": 0.038,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.3078866004943848,
+      "rewards/margins": 2.1681323051452637,
+      "rewards/rejected": -3.4760184288024902,
       "step": 220
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3490537564442845e-07,
-      "logits/chosen": -2.3061037063598633,
-      "logits/rejected": -2.2063522338867188,
-      "logps/chosen": -515.2584228515625,
-      "logps/rejected": -387.2288818359375,
-      "loss": 0.0536,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.2331289052963257,
-      "rewards/margins": 1.573769211769104,
-      "rewards/rejected": -2.806898355484009,
+      "logits/chosen": -2.2288191318511963,
+      "logits/rejected": -2.136579751968384,
+      "logps/chosen": -507.54632568359375,
+      "logps/rejected": -419.88470458984375,
+      "loss": 0.0432,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.2000774145126343,
+      "rewards/margins": 1.7510545253753662,
+      "rewards/rejected": -2.951131820678711,
       "step": 230
     },
     {
       "epoch": 0.59,
       "learning_rate": 2.1345078256378801e-07,
-      "logits/chosen": -2.3259823322296143,
-      "logits/rejected": -2.232604503631592,
-      "logps/chosen": -529.44775390625,
-      "logps/rejected": -442.9454040527344,
-      "loss": 0.0384,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.2063531875610352,
-      "rewards/margins": 2.0420820713043213,
-      "rewards/rejected": -3.2484352588653564,
+      "logits/chosen": -2.282217264175415,
+      "logits/rejected": -2.1927459239959717,
+      "logps/chosen": -539.92822265625,
+      "logps/rejected": -433.8241271972656,
+      "loss": 0.0373,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.2784963846206665,
+      "rewards/margins": 1.8950881958007812,
+      "rewards/rejected": -3.1735846996307373,
       "step": 240
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.9226827501969865e-07,
-      "logits/chosen": -2.310181140899658,
-      "logits/rejected": -2.225755214691162,
-      "logps/chosen": -569.6714477539062,
-      "logps/rejected": -482.9613342285156,
-      "loss": 0.0368,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -1.2699750661849976,
-      "rewards/margins": 2.3776299953460693,
-      "rewards/rejected": -3.6476047039031982,
+      "logits/chosen": -2.2803092002868652,
+      "logits/rejected": -2.1990160942077637,
+      "logps/chosen": -537.9136962890625,
+      "logps/rejected": -442.28350830078125,
+      "loss": 0.04,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.2320274114608765,
+      "rewards/margins": 2.1027939319610596,
+      "rewards/rejected": -3.3348212242126465,
       "step": 250
     },
     {
       "epoch": 0.64,
       "learning_rate": 1.715155432264775e-07,
-      "logits/chosen": -2.3007090091705322,
-      "logits/rejected": -2.2159204483032227,
-      "logps/chosen": -574.6656494140625,
-      "logps/rejected": -473.60528564453125,
-      "loss": 0.0275,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -1.4263044595718384,
-      "rewards/margins": 2.146233081817627,
-      "rewards/rejected": -3.572537660598755,
+      "logits/chosen": -2.2646355628967285,
+      "logits/rejected": -2.14613676071167,
+      "logps/chosen": -502.49664306640625,
+      "logps/rejected": -420.11004638671875,
+      "loss": 0.0396,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.1264328956604004,
+      "rewards/margins": 2.006878614425659,
+      "rewards/rejected": -3.1333117485046387,
       "step": 260
     },
     {
       "epoch": 0.67,
       "learning_rate": 1.51347077992983e-07,
-      "logits/chosen": -2.280165195465088,
-      "logits/rejected": -2.1988308429718018,
-      "logps/chosen": -573.0145874023438,
-      "logps/rejected": -490.4935607910156,
-      "loss": 0.024,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -1.6931577920913696,
-      "rewards/margins": 1.988318681716919,
-      "rewards/rejected": -3.68147611618042,
+      "logits/chosen": -2.3088698387145996,
+      "logits/rejected": -2.2018628120422363,
+      "logps/chosen": -554.0256958007812,
+      "logps/rejected": -421.2101135253906,
+      "loss": 0.0375,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -1.10258948802948,
+      "rewards/margins": 1.9626919031143188,
+      "rewards/rejected": -3.065281391143799,
       "step": 270
     },
     {
       "epoch": 0.69,
       "learning_rate": 1.3191302063739906e-07,
-      "logits/chosen": -2.247427463531494,
-      "logits/rejected": -2.1717417240142822,
-      "logps/chosen": -552.9573364257812,
-      "logps/rejected": -480.90435791015625,
-      "loss": 0.0231,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.7376149892807007,
-      "rewards/margins": 1.9405027627944946,
-      "rewards/rejected": -3.678117275238037,
+      "logits/chosen": -2.310133457183838,
+      "logits/rejected": -2.216827392578125,
+      "logps/chosen": -522.3606567382812,
+      "logps/rejected": -438.058349609375,
+      "loss": 0.043,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.1809624433517456,
+      "rewards/margins": 1.9291051626205444,
+      "rewards/rejected": -3.110067844390869,
       "step": 280
     },
     {
       "epoch": 0.72,
       "learning_rate": 1.1335804528119475e-07,
-      "logits/chosen": -2.3430678844451904,
-      "logits/rejected": -2.2265610694885254,
-      "logps/chosen": -586.9962158203125,
-      "logps/rejected": -472.01611328125,
-      "loss": 0.0285,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.5123710632324219,
-      "rewards/margins": 2.2006583213806152,
-      "rewards/rejected": -3.713029384613037,
+      "logits/chosen": -2.3108785152435303,
+      "logits/rejected": -2.2141172885894775,
+      "logps/chosen": -544.7510986328125,
+      "logps/rejected": -427.60150146484375,
+      "loss": 0.044,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -1.00501549243927,
+      "rewards/margins": 2.146829605102539,
+      "rewards/rejected": -3.1518452167510986,
       "step": 290
     },
     {
       "epoch": 0.74,
       "learning_rate": 9.582028184286423e-08,
-      "logits/chosen": -2.2495548725128174,
-      "logits/rejected": -2.186642646789551,
-      "logps/chosen": -531.0364990234375,
-      "logps/rejected": -480.0726623535156,
-      "loss": 0.0312,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.7118114233016968,
-      "rewards/margins": 1.8730456829071045,
-      "rewards/rejected": -3.58485746383667,
+      "logits/chosen": -2.350487470626831,
+      "logits/rejected": -2.307096481323242,
+      "logps/chosen": -554.42529296875,
+      "logps/rejected": -470.14434814453125,
+      "loss": 0.0402,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.1662867069244385,
+      "rewards/margins": 2.156501531600952,
+      "rewards/rejected": -3.3227882385253906,
       "step": 300
     },
     {
       "epoch": 0.74,
-      "eval_logits/chosen": -2.2933216094970703,
-      "eval_logits/rejected": -2.2721123695373535,
-      "eval_logps/chosen": -501.1633605957031,
-      "eval_logps/rejected": -543.5177612304688,
-      "eval_loss": 0.06885366886854172,
-      "eval_rewards/accuracies": 0.61328125,
-      "eval_rewards/chosen": -2.441237449645996,
-      "eval_rewards/margins": 0.42040756344795227,
-      "eval_rewards/rejected": -2.861644983291626,
-      "eval_runtime": 53.2903,
-      "eval_samples_per_second": 37.53,
+      "eval_logits/chosen": -2.2979543209075928,
+      "eval_logits/rejected": -2.2751243114471436,
+      "eval_logps/chosen": -467.0057067871094,
+      "eval_logps/rejected": -507.4114685058594,
+      "eval_loss": 0.08367828279733658,
+      "eval_rewards/accuracies": 0.62890625,
+      "eval_rewards/chosen": -2.099660634994507,
+      "eval_rewards/margins": 0.4009218215942383,
+      "eval_rewards/rejected": -2.500582218170166,
+      "eval_runtime": 53.3734,
+      "eval_samples_per_second": 37.472,
       "eval_steps_per_second": 0.6,
       "step": 300
     },
     {
       "epoch": 0.76,
       "learning_rate": 7.943028774907065e-08,
-      "logits/chosen": -2.2719688415527344,
-      "logits/rejected": -2.1988675594329834,
-      "logps/chosen": -524.6929931640625,
-      "logps/rejected": -446.8042907714844,
-      "loss": 0.0349,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.4022165536880493,
-      "rewards/margins": 1.8806768655776978,
-      "rewards/rejected": -3.282893419265747,
+      "logits/chosen": -2.316253185272217,
+      "logits/rejected": -2.209606170654297,
+      "logps/chosen": -524.6145629882812,
+      "logps/rejected": -420.94671630859375,
+      "loss": 0.0324,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.0568145513534546,
+      "rewards/margins": 2.0644707679748535,
+      "rewards/rejected": -3.1212852001190186,
       "step": 310
     },
     {
       "epoch": 0.79,
       "learning_rate": 6.431007601814637e-08,
-      "logits/chosen": -2.2960824966430664,
-      "logits/rejected": -2.2386252880096436,
-      "logps/chosen": -477.001953125,
-      "logps/rejected": -436.0245666503906,
-      "loss": 0.0298,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -1.4929635524749756,
-      "rewards/margins": 1.7944204807281494,
-      "rewards/rejected": -3.287383556365967,
+      "logits/chosen": -2.2733869552612305,
+      "logits/rejected": -2.169506549835205,
+      "logps/chosen": -532.5906982421875,
+      "logps/rejected": -450.932373046875,
+      "loss": 0.0316,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.5221502780914307,
+      "rewards/margins": 1.9199508428573608,
+      "rewards/rejected": -3.442101001739502,
       "step": 320
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.0572206951246e-08,
-      "logits/chosen": -2.277937650680542,
-      "logits/rejected": -2.1940300464630127,
-      "logps/chosen": -516.416015625,
-      "logps/rejected": -444.90032958984375,
-      "loss": 0.0329,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.4886820316314697,
-      "rewards/margins": 1.8972896337509155,
-      "rewards/rejected": -3.385971784591675,
+      "logits/chosen": -2.290539503097534,
+      "logits/rejected": -2.193920850753784,
+      "logps/chosen": -562.043701171875,
+      "logps/rejected": -479.5208435058594,
+      "loss": 0.0278,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.3767458200454712,
+      "rewards/margins": 2.1372973918914795,
+      "rewards/rejected": -3.514043092727661,
       "step": 330
     },
     {
       "epoch": 0.84,
       "learning_rate": 3.831895019292897e-08,
-      "logits/chosen": -2.3472743034362793,
-      "logits/rejected": -2.266993999481201,
-      "logps/chosen": -560.1998291015625,
-      "logps/rejected": -486.14801025390625,
-      "loss": 0.0324,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -1.25786554813385,
-      "rewards/margins": 2.4262924194335938,
-      "rewards/rejected": -3.6841578483581543,
+      "logits/chosen": -2.3263535499572754,
+      "logits/rejected": -2.207899570465088,
+      "logps/chosen": -619.2625122070312,
+      "logps/rejected": -520.6148071289062,
+      "loss": 0.0305,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.5666420459747314,
+      "rewards/margins": 2.300938844680786,
+      "rewards/rejected": -3.8675804138183594,
       "step": 340
     },
     {
       "epoch": 0.86,
       "learning_rate": 2.764152339909756e-08,
-      "logits/chosen": -2.2894670963287354,
-      "logits/rejected": -2.2070441246032715,
-      "logps/chosen": -551.2086181640625,
-      "logps/rejected": -415.3118591308594,
-      "loss": 0.0328,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.2593928575515747,
-      "rewards/margins": 1.9064128398895264,
-      "rewards/rejected": -3.1658055782318115,
+      "logits/chosen": -2.305875539779663,
+      "logits/rejected": -2.1924188137054443,
+      "logps/chosen": -568.1319580078125,
+      "logps/rejected": -475.6539611816406,
+      "loss": 0.0245,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.341552972793579,
+      "rewards/margins": 2.3198726177215576,
+      "rewards/rejected": -3.661425828933716,
       "step": 350
     },
     {
       "epoch": 0.89,
       "learning_rate": 1.861941317991664e-08,
-      "logits/chosen": -2.3396449089050293,
-      "logits/rejected": -2.227651834487915,
-      "logps/chosen": -571.0888061523438,
-      "logps/rejected": -453.03277587890625,
-      "loss": 0.0325,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -1.14793860912323,
-      "rewards/margins": 2.2367420196533203,
-      "rewards/rejected": -3.3846805095672607,
+      "logits/chosen": -2.31453800201416,
+      "logits/rejected": -2.209552049636841,
+      "logps/chosen": -574.0198974609375,
+      "logps/rejected": -498.809326171875,
+      "loss": 0.0246,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.3858083486557007,
+      "rewards/margins": 2.422987699508667,
+      "rewards/rejected": -3.80879545211792,
       "step": 360
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.13197833728636e-08,
-      "logits/chosen": -2.2972564697265625,
-      "logits/rejected": -2.215446710586548,
-      "logps/chosen": -527.4664306640625,
-      "logps/rejected": -465.6924743652344,
-      "loss": 0.0288,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -1.247899055480957,
-      "rewards/margins": 2.289482355117798,
-      "rewards/rejected": -3.537381410598755,
+      "logits/chosen": -2.2876980304718018,
+      "logits/rejected": -2.1881823539733887,
+      "logps/chosen": -583.4609985351562,
+      "logps/rejected": -515.4216918945312,
+      "loss": 0.0274,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.2713569402694702,
+      "rewards/margins": 2.548645496368408,
+      "rewards/rejected": -3.820002317428589,
       "step": 370
     },
     {
       "epoch": 0.94,
       "learning_rate": 5.79697505093521e-09,
-      "logits/chosen": -2.293482542037964,
-      "logits/rejected": -2.2097363471984863,
-      "logps/chosen": -540.6966552734375,
-      "logps/rejected": -439.814697265625,
-      "loss": 0.0375,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.383996605873108,
-      "rewards/margins": 1.9607197046279907,
-      "rewards/rejected": -3.3447163105010986,
+      "logits/chosen": -2.2938995361328125,
+      "logits/rejected": -2.161371946334839,
+      "logps/chosen": -567.2229614257812,
+      "logps/rejected": -493.6429138183594,
+      "loss": 0.0339,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.495012879371643,
+      "rewards/margins": 2.173337697982788,
+      "rewards/rejected": -3.6683506965637207,
       "step": 380
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.092101988131256e-09,
-      "logits/chosen": -2.346567153930664,
-      "logits/rejected": -2.220730781555176,
-      "logps/chosen": -575.7041625976562,
-      "logps/rejected": -463.69427490234375,
-      "loss": 0.0315,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -1.1256561279296875,
-      "rewards/margins": 2.420063018798828,
-      "rewards/rejected": -3.5457186698913574,
+      "logits/chosen": -2.3137059211730957,
+      "logits/rejected": -2.1986048221588135,
+      "logps/chosen": -600.1227416992188,
+      "logps/rejected": -496.6559143066406,
+      "loss": 0.0258,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.368238925933838,
+      "rewards/margins": 2.388805389404297,
+      "rewards/rejected": -3.7570443153381348,
       "step": 390
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.327445937151673e-10,
-      "logits/chosen": -2.3339614868164062,
-      "logits/rejected": -2.2517640590667725,
-      "logps/chosen": -568.7457275390625,
-      "logps/rejected": -479.13653564453125,
-      "loss": 0.0331,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.2012748718261719,
-      "rewards/margins": 2.3051795959472656,
-      "rewards/rejected": -3.5064544677734375,
+      "logits/chosen": -2.316849708557129,
+      "logits/rejected": -2.1959729194641113,
+      "logps/chosen": -559.0263061523438,
+      "logps/rejected": -482.11773681640625,
+      "loss": 0.0288,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.659148931503296,
+      "rewards/margins": 2.0086750984191895,
+      "rewards/rejected": -3.6678237915039062,
       "step": 400
     },
     {
       "epoch": 0.99,
-      "eval_logits/chosen": -2.3029849529266357,
-      "eval_logits/rejected": -2.282188892364502,
-      "eval_logps/chosen": -494.22357177734375,
-      "eval_logps/rejected": -539.6053466796875,
-      "eval_loss": 0.07123579829931259,
+      "eval_logits/chosen": -2.2708253860473633,
+      "eval_logits/rejected": -2.245922565460205,
+      "eval_logps/chosen": -513.0960083007812,
+      "eval_logps/rejected": -552.8470458984375,
+      "eval_loss": 0.06599809229373932,
       "eval_rewards/accuracies": 0.625,
-      "eval_rewards/chosen": -2.3718395233154297,
-      "eval_rewards/margins": 0.45068085193634033,
-      "eval_rewards/rejected": -2.8225200176239014,
-      "eval_runtime": 53.2767,
-      "eval_samples_per_second": 37.54,
-      "eval_steps_per_second": 0.601,
+      "eval_rewards/chosen": -2.560563564300537,
+      "eval_rewards/margins": 0.3943747282028198,
+      "eval_rewards/rejected": -2.9549384117126465,
+      "eval_runtime": 53.3482,
+      "eval_samples_per_second": 37.49,
+      "eval_steps_per_second": 0.6,
       "step": 400
     },
     {
       "epoch": 1.0,
       "step": 405,
       "total_flos": 0.0,
-      "train_loss": 0.0722552685457983,
-      "train_runtime": 3732.8792,
-      "train_samples_per_second": 13.902,
+      "train_loss": 0.07151281171374851,
+      "train_runtime": 3738.25,
+      "train_samples_per_second": 13.882,
       "train_steps_per_second": 0.108
     }
   ],