Model save

README.md

@@ -2,6 +2,7 @@
 tags:
 - trl
 - dpo
+- alignment-handbook
 - generated_from_trainer
 model-index:
 - name: zephyr-7b-dpo-full
@@ -15,15 +16,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4958
-- Rewards/chosen: -0.9104
-- Rewards/rejected: -1.8922
-- Rewards/accuracies: 0.7695
-- Rewards/margins: 0.9818
-- Logps/rejected: -476.9780
-- Logps/chosen: -372.6729
-- Logits/rejected: -0.9140
-- Logits/chosen: -1.0283
+- Logits/chosen: -0.9126
+- Logits/rejected: -0.7601
+- Logps/chosen: -361.4586
+- Logps/rejected: -470.9522
+- Loss: 0.4905
+- Rewards/accuracies: 0.7539
+- Rewards/chosen: -0.7983
+- Rewards/margins: 1.0337
+- Rewards/rejected: -1.8319
 
 ## Model description
 
@@ -58,12 +59,12 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
-|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.5659        | 0.21  | 100  | 0.5687          | -0.3404        | -0.8867          | 0.7305             | 0.5463          | -376.4286      | -315.6726    | -1.8735         | -1.8896       |
-| 0.5452        | 0.42  | 200  | 0.5199          | -0.7424        | -1.5558          | 0.7656             | 0.8134          | -443.3397      | -355.8689    | -1.6298         | -1.6748       |
-| 0.4825        | 0.63  | 300  | 0.5001          | -1.0399        | -2.0305          | 0.75               | 0.9906          | -490.8157      | -385.6247    | -1.1553         | -1.2376       |
-| 0.4965        | 0.84  | 400  | 0.4958          | -0.9104        | -1.8922          | 0.7695             | 0.9818          | -476.9780      | -372.6729    | -0.9140         | -1.0283       |
+| Training Loss | Epoch | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
+|:-------------:|:-----:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
+| 0.5759        | 0.21  | 100  | -1.9151       | -1.8988         | -335.8161    | -395.6910      | 0.5723          | 0.7148             | -0.5418        | 0.5375          | -1.0793          |
+| 0.5391        | 0.42  | 200  | -1.6570       | -1.5774         | -321.1682    | -405.0056      | 0.5138          | 0.7461             | -0.3953        | 0.7771          | -1.1724          |
+| 0.4788        | 0.63  | 300  | -0.9864       | -0.8826         | -367.7277    | -475.5532      | 0.4939          | 0.7578             | -0.8609        | 1.0170          | -1.8779          |
+| 0.4937        | 0.84  | 400  | -0.9126       | -0.7601         | -361.4586    | -470.9522      | 0.4905          | 0.7539             | -0.7983        | 1.0337          | -1.8319          |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,21 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5343728439578451,
-    "train_runtime": 7644.6536,
+    "eval_logits/chosen": -0.8698186874389648,
+    "eval_logits/rejected": -0.7152774930000305,
+    "eval_logps/chosen": -366.0130310058594,
+    "eval_logps/rejected": -477.25433349609375,
+    "eval_loss": 0.48990052938461304,
+    "eval_rewards/accuracies": 0.765625,
+    "eval_rewards/chosen": -0.8437946438789368,
+    "eval_rewards/margins": 1.0511335134506226,
+    "eval_rewards/rejected": -1.894927978515625,
+    "eval_runtime": 97.4253,
+    "eval_samples": 2000,
+    "eval_samples_per_second": 20.529,
+    "eval_steps_per_second": 0.328,
+    "train_loss": 0.07911034258838479,
+    "train_runtime": 1140.8874,
     "train_samples": 61134,
-    "train_samples_per_second": 7.997,
-    "train_steps_per_second": 0.063
+    "train_samples_per_second": 53.585,
+    "train_steps_per_second": 0.419
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.4783902168273926,
-    "eval_logits/rejected": -2.4373888969421387,
-    "eval_logps/chosen": -251.0624237060547,
-    "eval_logps/rejected": -269.08074951171875,
-    "eval_loss": 0.5351515412330627,
-    "eval_rewards/accuracies": 0.76171875,
-    "eval_rewards/chosen": 0.5765520930290222,
-    "eval_rewards/margins": 0.8972306251525879,
-    "eval_rewards/rejected": -0.3206784725189209,
-    "eval_runtime": 96.1948,
+    "eval_logits/chosen": -0.8698186874389648,
+    "eval_logits/rejected": -0.7152774930000305,
+    "eval_logps/chosen": -366.0130310058594,
+    "eval_logps/rejected": -477.25433349609375,
+    "eval_loss": 0.48990052938461304,
+    "eval_rewards/accuracies": 0.765625,
+    "eval_rewards/chosen": -0.8437946438789368,
+    "eval_rewards/margins": 1.0511335134506226,
+    "eval_rewards/rejected": -1.894927978515625,
+    "eval_runtime": 97.4253,
     "eval_samples": 2000,
-    "eval_samples_per_second": 20.791,
-    "eval_steps_per_second": 0.333
+    "eval_samples_per_second": 20.529,
+    "eval_steps_per_second": 0.328
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:516514bc4b6633ccbe70bca9426bc217dd255127bfbd089e8e3cec35b6ca7fdb
+oid sha256:8993f207919853c6077676c36bf82423d62894d8b5f9f43eb1749a3ca762a15e
 size 4943178720

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b2ff5e67250cf98084bd9c68b114a890bab93ca22aca884e3f270b315be30900
+oid sha256:cac8f4af54fcab2456b0f93ada8fead3c13b06c0eacdee7245d8ad8692e950bf
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4a114f10b8e0e98de3f48e85e83aab303bf26c1d684935cd124820f266f747e5
+oid sha256:d4c13bcd7d01eec2d330950d0b4cba5bff9d8afbdf18f3ca6d3892559af67e17
 size 4540532728

runs/May11_00-51-53_n136-101-036/events.out.tfevents.1715368439.n136-101-036.30703.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d8dfd8a88b760fbcd52c5d43ba6361ea6633cb25ca5dd7920303532c663ea4b
+size 828

runs/May11_08-26-04_n136-101-036/events.out.tfevents.1715387281.n136-101-036.245338.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:512d868b6dc516de8d7b033f1918ce2d6d3f3be53ebc9cf7adcc94520ff9a218
+size 10015

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5343728439578451,
-    "train_runtime": 7644.6536,
+    "train_loss": 0.07911034258838479,
+    "train_runtime": 1140.8874,
     "train_samples": 61134,
-    "train_samples_per_second": 7.997,
-    "train_steps_per_second": 0.063
+    "train_samples_per_second": 53.585,
+    "train_steps_per_second": 0.419
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 27.380286346585322,
+      "grad_norm": 27.376848082151543,
       "learning_rate": 1.0416666666666666e-08,
       "logits/chosen": -1.8783892393112183,
       "logits/rejected": -1.8756425380706787,
@@ -25,781 +25,781 @@
     },
     {
       "epoch": 0.02,
-      "grad_norm": 25.334245571211405,
+      "grad_norm": 25.183613892194092,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -1.7500182390213013,
-      "logits/rejected": -1.7071739435195923,
-      "logps/chosen": -280.65020751953125,
-      "logps/rejected": -271.91290283203125,
-      "loss": 0.6933,
-      "rewards/accuracies": 0.3958333432674408,
-      "rewards/chosen": -0.0017088722670450807,
-      "rewards/margins": -0.0012201463105157018,
-      "rewards/rejected": -0.0004887260147370398,
+      "logits/chosen": -1.7508937120437622,
+      "logits/rejected": -1.7071995735168457,
+      "logps/chosen": -280.5123596191406,
+      "logps/rejected": -271.9100341796875,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4791666567325592,
+      "rewards/chosen": -0.00033082481240853667,
+      "rewards/margins": 0.0001291988301090896,
+      "rewards/rejected": -0.00046002367162145674,
       "step": 10
     },
     {
       "epoch": 0.04,
-      "grad_norm": 23.080460659897174,
+      "grad_norm": 23.230240384482265,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -1.8319015502929688,
-      "logits/rejected": -1.7260072231292725,
-      "logps/chosen": -298.88006591796875,
-      "logps/rejected": -320.7435302734375,
-      "loss": 0.6918,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.009266598150134087,
-      "rewards/margins": 0.00015246197290252894,
-      "rewards/rejected": 0.00911413598805666,
+      "logits/chosen": -1.832267165184021,
+      "logits/rejected": -1.7261158227920532,
+      "logps/chosen": -298.87811279296875,
+      "logps/rejected": -320.8003845214844,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.009286092594265938,
+      "rewards/margins": 0.0007404378848150373,
+      "rewards/rejected": 0.008545654825866222,
       "step": 20
     },
     {
       "epoch": 0.06,
-      "grad_norm": 22.85974921125575,
+      "grad_norm": 22.82438334979188,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -1.8605388402938843,
-      "logits/rejected": -1.8093788623809814,
-      "logps/chosen": -314.97576904296875,
-      "logps/rejected": -281.7659606933594,
-      "loss": 0.6844,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.06469086557626724,
-      "rewards/margins": 0.02124236896634102,
-      "rewards/rejected": 0.04344850033521652,
+      "logits/chosen": -1.8619463443756104,
+      "logits/rejected": -1.8105783462524414,
+      "logps/chosen": -315.0911865234375,
+      "logps/rejected": -281.7936096191406,
+      "loss": 0.6843,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.06353694945573807,
+      "rewards/margins": 0.020365100353956223,
+      "rewards/rejected": 0.043171852827072144,
       "step": 30
     },
     {
       "epoch": 0.08,
-      "grad_norm": 20.24451802630232,
+      "grad_norm": 20.30578423378571,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -1.7260879278182983,
-      "logits/rejected": -1.6905444860458374,
-      "logps/chosen": -269.099365234375,
-      "logps/rejected": -258.0982971191406,
-      "loss": 0.6708,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.15108337998390198,
-      "rewards/margins": 0.05832190066576004,
-      "rewards/rejected": 0.09276147186756134,
+      "logits/chosen": -1.7271950244903564,
+      "logits/rejected": -1.6917632818222046,
+      "logps/chosen": -269.0830078125,
+      "logps/rejected": -258.13482666015625,
+      "loss": 0.6707,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.15124677121639252,
+      "rewards/margins": 0.058850765228271484,
+      "rewards/rejected": 0.09239600598812103,
       "step": 40
     },
     {
       "epoch": 0.1,
-      "grad_norm": 18.927988004439843,
+      "grad_norm": 18.991767469871117,
       "learning_rate": 4.999733114418725e-07,
-      "logits/chosen": -1.7597843408584595,
-      "logits/rejected": -1.747876763343811,
-      "logps/chosen": -274.760009765625,
-      "logps/rejected": -298.1368408203125,
-      "loss": 0.6569,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.22397354245185852,
-      "rewards/margins": 0.0527268722653389,
-      "rewards/rejected": 0.17124666273593903,
+      "logits/chosen": -1.7609294652938843,
+      "logits/rejected": -1.749925971031189,
+      "logps/chosen": -274.7580261230469,
+      "logps/rejected": -298.2468566894531,
+      "loss": 0.6568,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.22399310767650604,
+      "rewards/margins": 0.05384649708867073,
+      "rewards/rejected": 0.1701466143131256,
       "step": 50
     },
     {
       "epoch": 0.13,
-      "grad_norm": 22.49423236260695,
+      "grad_norm": 22.512008467267577,
       "learning_rate": 4.990398100856366e-07,
-      "logits/chosen": -1.8483797311782837,
-      "logits/rejected": -1.8096563816070557,
-      "logps/chosen": -268.5242614746094,
-      "logps/rejected": -318.25177001953125,
-      "loss": 0.6421,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": 0.22741086781024933,
-      "rewards/margins": 0.11925415694713593,
-      "rewards/rejected": 0.1081567034125328,
+      "logits/chosen": -1.8478819131851196,
+      "logits/rejected": -1.8104110956192017,
+      "logps/chosen": -268.5568542480469,
+      "logps/rejected": -318.2205505371094,
+      "loss": 0.6419,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.22708478569984436,
+      "rewards/margins": 0.11861655861139297,
+      "rewards/rejected": 0.108468197286129,
       "step": 60
     },
     {
       "epoch": 0.15,
-      "grad_norm": 24.198577605355105,
+      "grad_norm": 24.33517976511266,
       "learning_rate": 4.967775735898179e-07,
-      "logits/chosen": -1.6724300384521484,
-      "logits/rejected": -1.6896724700927734,
-      "logps/chosen": -274.63629150390625,
-      "logps/rejected": -290.02655029296875,
-      "loss": 0.612,
+      "logits/chosen": -1.6735728979110718,
+      "logits/rejected": -1.6890596151351929,
+      "logps/chosen": -274.67156982421875,
+      "logps/rejected": -289.94256591796875,
+      "loss": 0.6121,
       "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": 0.11364272981882095,
-      "rewards/margins": 0.2259376049041748,
-      "rewards/rejected": -0.11229487508535385,
+      "rewards/chosen": 0.11328981816768646,
+      "rewards/margins": 0.22474534809589386,
+      "rewards/rejected": -0.11145554482936859,
       "step": 70
     },
     {
       "epoch": 0.17,
-      "grad_norm": 32.651664968160794,
+      "grad_norm": 32.81066991350388,
       "learning_rate": 4.931986719649298e-07,
-      "logits/chosen": -1.907275915145874,
-      "logits/rejected": -1.7932302951812744,
-      "logps/chosen": -356.7095947265625,
-      "logps/rejected": -325.30950927734375,
-      "loss": 0.588,
+      "logits/chosen": -1.9151760339736938,
+      "logits/rejected": -1.802167534828186,
+      "logps/chosen": -356.9640808105469,
+      "logps/rejected": -325.5661315917969,
+      "loss": 0.5878,
       "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.07804231345653534,
-      "rewards/margins": 0.30370011925697327,
-      "rewards/rejected": -0.381742388010025,
+      "rewards/chosen": -0.08058764040470123,
+      "rewards/margins": 0.30372124910354614,
+      "rewards/rejected": -0.38430893421173096,
       "step": 80
     },
     {
       "epoch": 0.19,
-      "grad_norm": 33.5048018608096,
+      "grad_norm": 31.216011823878958,
       "learning_rate": 4.883222001996351e-07,
-      "logits/chosen": -1.7514793872833252,
-      "logits/rejected": -1.7362706661224365,
-      "logps/chosen": -325.14605712890625,
-      "logps/rejected": -369.40643310546875,
-      "loss": 0.5632,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.4447757601737976,
-      "rewards/margins": 0.5141423940658569,
-      "rewards/rejected": -0.9589180946350098,
+      "logits/chosen": -1.7752355337142944,
+      "logits/rejected": -1.7595351934432983,
+      "logps/chosen": -325.2732849121094,
+      "logps/rejected": -370.2122497558594,
+      "loss": 0.5628,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.44604864716529846,
+      "rewards/margins": 0.5209277272224426,
+      "rewards/rejected": -0.9669763445854187,
       "step": 90
     },
     {
       "epoch": 0.21,
-      "grad_norm": 30.80051460186291,
+      "grad_norm": 30.757968765158964,
       "learning_rate": 4.821741763807186e-07,
-      "logits/chosen": -1.7818742990493774,
-      "logits/rejected": -1.7546179294586182,
-      "logps/chosen": -338.099853515625,
-      "logps/rejected": -388.9416809082031,
-      "loss": 0.5659,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.590244472026825,
-      "rewards/margins": 0.5510064363479614,
-      "rewards/rejected": -1.1412509679794312,
+      "logits/chosen": -1.8407386541366577,
+      "logits/rejected": -1.818633794784546,
+      "logps/chosen": -356.88604736328125,
+      "logps/rejected": -402.81597900390625,
+      "loss": 0.5759,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.7781068682670593,
+      "rewards/margins": 0.501887321472168,
+      "rewards/rejected": -1.279994249343872,
       "step": 100
     },
     {
       "epoch": 0.21,
-      "eval_logits/chosen": -1.8896375894546509,
-      "eval_logits/rejected": -1.8735315799713135,
-      "eval_logps/chosen": -315.6726379394531,
-      "eval_logps/rejected": -376.4285583496094,
-      "eval_loss": 0.568724513053894,
-      "eval_rewards/accuracies": 0.73046875,
-      "eval_rewards/chosen": -0.3403905928134918,
-      "eval_rewards/margins": 0.5462794899940491,
-      "eval_rewards/rejected": -0.8866700530052185,
-      "eval_runtime": 97.7372,
-      "eval_samples_per_second": 20.463,
+      "eval_logits/chosen": -1.9150919914245605,
+      "eval_logits/rejected": -1.8988227844238281,
+      "eval_logps/chosen": -335.8161315917969,
+      "eval_logps/rejected": -395.6910095214844,
+      "eval_loss": 0.5723333954811096,
+      "eval_rewards/accuracies": 0.71484375,
+      "eval_rewards/chosen": -0.5418255925178528,
+      "eval_rewards/margins": 0.5374687910079956,
+      "eval_rewards/rejected": -1.0792944431304932,
+      "eval_runtime": 97.7859,
+      "eval_samples_per_second": 20.453,
       "eval_steps_per_second": 0.327,
       "step": 100
     },
     {
       "epoch": 0.23,
-      "grad_norm": 46.20168597373281,
+      "grad_norm": 31.68583269080582,
       "learning_rate": 4.747874028753375e-07,
-      "logits/chosen": -1.9444591999053955,
-      "logits/rejected": -1.8265165090560913,
-      "logps/chosen": -366.3944396972656,
-      "logps/rejected": -371.1445007324219,
-      "loss": 0.5753,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.3529593050479889,
-      "rewards/margins": 0.5102607607841492,
-      "rewards/rejected": -0.8632200956344604,
+      "logits/chosen": -1.8968604803085327,
+      "logits/rejected": -1.7719805240631104,
+      "logps/chosen": -367.7148742675781,
+      "logps/rejected": -371.1282653808594,
+      "loss": 0.5798,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.36616355180740356,
+      "rewards/margins": 0.49689429998397827,
+      "rewards/rejected": -0.8630577921867371,
       "step": 110
     },
     {
       "epoch": 0.25,
-      "grad_norm": 32.34008862326302,
+      "grad_norm": 30.73893119347065,
       "learning_rate": 4.662012913161997e-07,
-      "logits/chosen": -1.8391367197036743,
-      "logits/rejected": -1.8425571918487549,
-      "logps/chosen": -344.04052734375,
-      "logps/rejected": -375.23760986328125,
-      "loss": 0.5447,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.39724019169807434,
-      "rewards/margins": 0.5189257264137268,
-      "rewards/rejected": -0.9161659479141235,
+      "logits/chosen": -1.7683579921722412,
+      "logits/rejected": -1.7642805576324463,
+      "logps/chosen": -332.87030029296875,
+      "logps/rejected": -358.53521728515625,
+      "loss": 0.547,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.2855382263660431,
+      "rewards/margins": 0.4636038839817047,
+      "rewards/rejected": -0.7491421103477478,
       "step": 120
     },
     {
       "epoch": 0.27,
-      "grad_norm": 82.35253906426779,
+      "grad_norm": 36.07936832382646,
       "learning_rate": 4.5646165232345103e-07,
-      "logits/chosen": -1.8358303308486938,
-      "logits/rejected": -1.795000672340393,
-      "logps/chosen": -347.0339050292969,
-      "logps/rejected": -402.7286071777344,
-      "loss": 0.5339,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.5076359510421753,
-      "rewards/margins": 0.5805034041404724,
-      "rewards/rejected": -1.088139533996582,
+      "logits/chosen": -1.8693767786026,
+      "logits/rejected": -1.794420599937439,
+      "logps/chosen": -345.6004638671875,
+      "logps/rejected": -402.54608154296875,
+      "loss": 0.5353,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.4933016300201416,
+      "rewards/margins": 0.5930127501487732,
+      "rewards/rejected": -1.0863142013549805,
       "step": 130
     },
     {
       "epoch": 0.29,
-      "grad_norm": 42.708546895034445,
+      "grad_norm": 39.43015574676247,
       "learning_rate": 4.456204510851956e-07,
-      "logits/chosen": -1.754401445388794,
-      "logits/rejected": -1.7028589248657227,
-      "logps/chosen": -391.20172119140625,
-      "logps/rejected": -456.5897521972656,
-      "loss": 0.5274,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.5532731413841248,
-      "rewards/margins": 0.7467519044876099,
-      "rewards/rejected": -1.3000251054763794,
+      "logits/chosen": -1.7766962051391602,
+      "logits/rejected": -1.7008291482925415,
+      "logps/chosen": -394.8687744140625,
+      "logps/rejected": -457.8002014160156,
+      "loss": 0.5255,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.5899434089660645,
+      "rewards/margins": 0.7221861481666565,
+      "rewards/rejected": -1.3121296167373657,
       "step": 140
     },
     {
       "epoch": 0.31,
-      "grad_norm": 46.79065019948174,
+      "grad_norm": 48.649979456976475,
       "learning_rate": 4.337355301007335e-07,
-      "logits/chosen": -1.665116310119629,
-      "logits/rejected": -1.5959926843643188,
-      "logps/chosen": -376.63690185546875,
-      "logps/rejected": -426.26287841796875,
-      "loss": 0.5535,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.7479280233383179,
-      "rewards/margins": 0.5139313340187073,
-      "rewards/rejected": -1.2618591785430908,
+      "logits/chosen": -1.7205531597137451,
+      "logits/rejected": -1.6475614309310913,
+      "logps/chosen": -382.6226806640625,
+      "logps/rejected": -433.13409423828125,
+      "loss": 0.5514,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.8077861666679382,
+      "rewards/margins": 0.5227850675582886,
+      "rewards/rejected": -1.3305714130401611,
       "step": 150
     },
     {
       "epoch": 0.33,
-      "grad_norm": 44.0166823935413,
+      "grad_norm": 44.8404423652529,
       "learning_rate": 4.2087030056579986e-07,
-      "logits/chosen": -1.6230976581573486,
-      "logits/rejected": -1.5330790281295776,
-      "logps/chosen": -348.4312438964844,
-      "logps/rejected": -400.83477783203125,
-      "loss": 0.5425,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.5379534363746643,
-      "rewards/margins": 0.7751569747924805,
-      "rewards/rejected": -1.3131104707717896,
+      "logits/chosen": -1.698359727859497,
+      "logits/rejected": -1.583348035812378,
+      "logps/chosen": -338.5320129394531,
+      "logps/rejected": -390.748291015625,
+      "loss": 0.5328,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.4389611780643463,
+      "rewards/margins": 0.7732844352722168,
+      "rewards/rejected": -1.2122455835342407,
       "step": 160
     },
     {
       "epoch": 0.36,
-      "grad_norm": 42.96060174901536,
+      "grad_norm": 44.50813022956455,
       "learning_rate": 4.070934040463998e-07,
-      "logits/chosen": -1.555373191833496,
-      "logits/rejected": -1.45396089553833,
-      "logps/chosen": -341.1943054199219,
-      "logps/rejected": -384.02294921875,
-      "loss": 0.5327,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.7449436187744141,
-      "rewards/margins": 0.5609264373779297,
-      "rewards/rejected": -1.3058700561523438,
+      "logits/chosen": -1.607373833656311,
+      "logits/rejected": -1.4848835468292236,
+      "logps/chosen": -323.30230712890625,
+      "logps/rejected": -363.1103515625,
+      "loss": 0.5341,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.5660237669944763,
+      "rewards/margins": 0.5307204723358154,
+      "rewards/rejected": -1.0967442989349365,
       "step": 170
     },
     {
       "epoch": 0.38,
-      "grad_norm": 42.12038704836531,
+      "grad_norm": 36.42011775174975,
       "learning_rate": 3.9247834624635404e-07,
-      "logits/chosen": -1.3416413068771362,
-      "logits/rejected": -1.2860534191131592,
-      "logps/chosen": -338.95025634765625,
-      "logps/rejected": -377.64208984375,
-      "loss": 0.508,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.6752852201461792,
-      "rewards/margins": 0.655610203742981,
-      "rewards/rejected": -1.3308954238891602,
+      "logits/chosen": -1.4740314483642578,
+      "logits/rejected": -1.3969428539276123,
+      "logps/chosen": -355.79022216796875,
+      "logps/rejected": -386.48846435546875,
+      "loss": 0.5082,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.8436854481697083,
+      "rewards/margins": 0.5756739377975464,
+      "rewards/rejected": -1.4193594455718994,
       "step": 180
     },
     {
       "epoch": 0.4,
-      "grad_norm": 32.57639205791924,
+      "grad_norm": 37.5415107419379,
       "learning_rate": 3.7710310482256523e-07,
-      "logits/chosen": -1.5504372119903564,
-      "logits/rejected": -1.4975203275680542,
-      "logps/chosen": -332.02703857421875,
-      "logps/rejected": -379.094970703125,
-      "loss": 0.5348,
+      "logits/chosen": -1.5718328952789307,
+      "logits/rejected": -1.524204134941101,
+      "logps/chosen": -348.835693359375,
+      "logps/rejected": -401.84857177734375,
+      "loss": 0.5283,
       "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.38119250535964966,
-      "rewards/margins": 0.5775793194770813,
-      "rewards/rejected": -0.958771824836731,
+      "rewards/chosen": -0.5492793321609497,
+      "rewards/margins": 0.6370285153388977,
+      "rewards/rejected": -1.186307668685913,
       "step": 190
     },
     {
       "epoch": 0.42,
-      "grad_norm": 87.04205573618253,
+      "grad_norm": 86.8216257987796,
       "learning_rate": 3.610497133404795e-07,
-      "logits/chosen": -1.5911279916763306,
-      "logits/rejected": -1.4979264736175537,
-      "logps/chosen": -351.78826904296875,
-      "logps/rejected": -416.9830627441406,
-      "loss": 0.5452,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.7444024085998535,
-      "rewards/margins": 0.7175058722496033,
-      "rewards/rejected": -1.461908221244812,
+      "logits/chosen": -1.508467674255371,
+      "logits/rejected": -1.409771203994751,
+      "logps/chosen": -339.861328125,
+      "logps/rejected": -404.37200927734375,
+      "loss": 0.5391,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.6251331567764282,
+      "rewards/margins": 0.7106647491455078,
+      "rewards/rejected": -1.335797905921936,
       "step": 200
     },
     {
       "epoch": 0.42,
-      "eval_logits/chosen": -1.6748462915420532,
-      "eval_logits/rejected": -1.629786491394043,
-      "eval_logps/chosen": -355.8689270019531,
-      "eval_logps/rejected": -443.3397216796875,
-      "eval_loss": 0.5198626518249512,
-      "eval_rewards/accuracies": 0.765625,
-      "eval_rewards/chosen": -0.7423534989356995,
-      "eval_rewards/margins": 0.8134280443191528,
-      "eval_rewards/rejected": -1.555781602859497,
-      "eval_runtime": 97.6582,
-      "eval_samples_per_second": 20.48,
+      "eval_logits/chosen": -1.6569650173187256,
+      "eval_logits/rejected": -1.577364206314087,
+      "eval_logps/chosen": -321.1681823730469,
+      "eval_logps/rejected": -405.005615234375,
+      "eval_loss": 0.5137878060340881,
+      "eval_rewards/accuracies": 0.74609375,
+      "eval_rewards/chosen": -0.3953460156917572,
+      "eval_rewards/margins": 0.777094841003418,
+      "eval_rewards/rejected": -1.172440767288208,
+      "eval_runtime": 97.4997,
+      "eval_samples_per_second": 20.513,
       "eval_steps_per_second": 0.328,
       "step": 200
     },
     {
       "epoch": 0.44,
-      "grad_norm": 45.47296213329607,
+      "grad_norm": 38.50117908255001,
       "learning_rate": 3.4440382358952115e-07,
-      "logits/chosen": -1.4384346008300781,
-      "logits/rejected": -1.354467511177063,
-      "logps/chosen": -392.1740417480469,
-      "logps/rejected": -434.571044921875,
-      "loss": 0.5269,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.7879104018211365,
-      "rewards/margins": 0.7938907742500305,
-      "rewards/rejected": -1.581801176071167,
+      "logits/chosen": -1.443800687789917,
+      "logits/rejected": -1.3134807348251343,
+      "logps/chosen": -354.0191345214844,
+      "logps/rejected": -393.02264404296875,
+      "loss": 0.5328,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.40636110305786133,
+      "rewards/margins": 0.759956419467926,
+      "rewards/rejected": -1.1663174629211426,
       "step": 210
     },
     {
       "epoch": 0.46,
-      "grad_norm": 61.47239109656524,
+      "grad_norm": 50.76318175549852,
       "learning_rate": 3.272542485937368e-07,
-      "logits/chosen": -1.2455013990402222,
-      "logits/rejected": -1.1706390380859375,
-      "logps/chosen": -361.58245849609375,
-      "logps/rejected": -426.8412170410156,
-      "loss": 0.5093,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.7907696962356567,
-      "rewards/margins": 0.8906044960021973,
-      "rewards/rejected": -1.6813743114471436,
+      "logits/chosen": -1.2096138000488281,
+      "logits/rejected": -1.064009189605713,
+      "logps/chosen": -339.54339599609375,
+      "logps/rejected": -401.81622314453125,
+      "loss": 0.5149,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.570379376411438,
+      "rewards/margins": 0.8607443571090698,
+      "rewards/rejected": -1.4311238527297974,
       "step": 220
     },
     {
       "epoch": 0.48,
-      "grad_norm": 49.00298937560116,
+      "grad_norm": 56.74639801937906,
       "learning_rate": 3.096924887558854e-07,
-      "logits/chosen": -1.2810494899749756,
-      "logits/rejected": -1.1935176849365234,
-      "logps/chosen": -353.9776306152344,
-      "logps/rejected": -455.53460693359375,
-      "loss": 0.5159,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.8773061633110046,
-      "rewards/margins": 0.965121865272522,
-      "rewards/rejected": -1.842428207397461,
+      "logits/chosen": -1.1264355182647705,
+      "logits/rejected": -1.0245306491851807,
+      "logps/chosen": -354.9466857910156,
+      "logps/rejected": -463.1206970214844,
+      "loss": 0.5149,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.8869966268539429,
+      "rewards/margins": 1.0312917232513428,
+      "rewards/rejected": -1.9182884693145752,
       "step": 230
     },
     {
       "epoch": 0.5,
-      "grad_norm": 50.57344886940067,
+      "grad_norm": 42.18851613816063,
       "learning_rate": 2.9181224366319943e-07,
-      "logits/chosen": -1.463232398033142,
-      "logits/rejected": -1.3980356454849243,
-      "logps/chosen": -357.5630798339844,
-      "logps/rejected": -425.10345458984375,
-      "loss": 0.492,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.6867150664329529,
-      "rewards/margins": 0.8543354868888855,
-      "rewards/rejected": -1.5410505533218384,
+      "logits/chosen": -1.2307146787643433,
+      "logits/rejected": -1.139776349067688,
+      "logps/chosen": -370.49853515625,
+      "logps/rejected": -442.11114501953125,
+      "loss": 0.487,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.8160694241523743,
+      "rewards/margins": 0.8950576782226562,
+      "rewards/rejected": -1.7111270427703857,
       "step": 240
     },
     {
       "epoch": 0.52,
-      "grad_norm": 46.88594336761054,
+      "grad_norm": 42.578136485563924,
       "learning_rate": 2.7370891215954565e-07,
-      "logits/chosen": -1.3949867486953735,
-      "logits/rejected": -1.2598216533660889,
-      "logps/chosen": -396.63641357421875,
-      "logps/rejected": -457.6905822753906,
-      "loss": 0.5209,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.7461437582969666,
-      "rewards/margins": 0.9727694392204285,
-      "rewards/rejected": -1.7189133167266846,
+      "logits/chosen": -1.104014277458191,
+      "logits/rejected": -0.9260801076889038,
+      "logps/chosen": -386.5050964355469,
+      "logps/rejected": -449.59722900390625,
+      "loss": 0.5122,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.6448307633399963,
+      "rewards/margins": 0.9931491017341614,
+      "rewards/rejected": -1.6379798650741577,
       "step": 250
     },
     {
       "epoch": 0.54,
-      "grad_norm": 58.910993035030984,
+      "grad_norm": 45.2823901088297,
       "learning_rate": 2.55479083351317e-07,
-      "logits/chosen": -1.3302087783813477,
-      "logits/rejected": -1.1668974161148071,
-      "logps/chosen": -409.41571044921875,
-      "logps/rejected": -466.9542541503906,
-      "loss": 0.5097,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.9284337162971497,
-      "rewards/margins": 0.9580095410346985,
-      "rewards/rejected": -1.8864428997039795,
+      "logits/chosen": -0.9789173007011414,
+      "logits/rejected": -0.7768954634666443,
+      "logps/chosen": -395.30548095703125,
+      "logps/rejected": -449.7933044433594,
+      "loss": 0.505,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.7873316407203674,
+      "rewards/margins": 0.9275019764900208,
+      "rewards/rejected": -1.7148334980010986,
       "step": 260
     },
     {
       "epoch": 0.56,
-      "grad_norm": 53.57111635571251,
+      "grad_norm": 44.49352654305102,
       "learning_rate": 2.3722002126275822e-07,
-      "logits/chosen": -1.1764518022537231,
-      "logits/rejected": -1.0825382471084595,
-      "logps/chosen": -409.5778503417969,
-      "logps/rejected": -470.2406311035156,
-      "loss": 0.517,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.079499363899231,
-      "rewards/margins": 0.7965269684791565,
-      "rewards/rejected": -1.8760263919830322,
+      "logits/chosen": -0.9071099162101746,
+      "logits/rejected": -0.8342393636703491,
+      "logps/chosen": -374.5643310546875,
+      "logps/rejected": -433.95098876953125,
+      "loss": 0.5205,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.729364275932312,
+      "rewards/margins": 0.7837658524513245,
+      "rewards/rejected": -1.5131301879882812,
       "step": 270
     },
     {
       "epoch": 0.59,
-      "grad_norm": 47.041203376030666,
+      "grad_norm": 42.661081149871585,
       "learning_rate": 2.19029145890313e-07,
-      "logits/chosen": -1.1975743770599365,
-      "logits/rejected": -1.022172451019287,
-      "logps/chosen": -365.8079528808594,
-      "logps/rejected": -442.7471618652344,
-      "loss": 0.513,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.8296273350715637,
-      "rewards/margins": 0.9351041913032532,
-      "rewards/rejected": -1.7647316455841064,
+      "logits/chosen": -0.9833901524543762,
+      "logits/rejected": -0.7881035804748535,
+      "logps/chosen": -336.73388671875,
+      "logps/rejected": -407.16326904296875,
+      "loss": 0.5186,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.5388863682746887,
+      "rewards/margins": 0.8700063824653625,
+      "rewards/rejected": -1.4088926315307617,
       "step": 280
     },
     {
       "epoch": 0.61,
-      "grad_norm": 62.17513110089864,
+      "grad_norm": 42.8181212203347,
       "learning_rate": 2.0100351342479216e-07,
-      "logits/chosen": -1.2788095474243164,
-      "logits/rejected": -1.2279428243637085,
-      "logps/chosen": -350.14013671875,
-      "logps/rejected": -429.78631591796875,
-      "loss": 0.5036,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.827313244342804,
-      "rewards/margins": 0.8608844876289368,
-      "rewards/rejected": -1.6881977319717407,
+      "logits/chosen": -1.1338340044021606,
+      "logits/rejected": -1.0056653022766113,
+      "logps/chosen": -327.16217041015625,
+      "logps/rejected": -406.1835021972656,
+      "loss": 0.5062,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.5975332260131836,
+      "rewards/margins": 0.854636549949646,
+      "rewards/rejected": -1.4521698951721191,
       "step": 290
     },
     {
       "epoch": 0.63,
-      "grad_norm": 41.394100435046234,
+      "grad_norm": 43.557515582129994,
       "learning_rate": 1.8323929841460178e-07,
-      "logits/chosen": -1.3548160791397095,
-      "logits/rejected": -1.1741359233856201,
-      "logps/chosen": -427.8033142089844,
-      "logps/rejected": -481.6588439941406,
-      "loss": 0.4825,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.0517909526824951,
-      "rewards/margins": 0.926468014717102,
-      "rewards/rejected": -1.9782588481903076,
+      "logits/chosen": -1.1290686130523682,
+      "logits/rejected": -0.9314834475517273,
+      "logps/chosen": -405.9638671875,
+      "logps/rejected": -460.133544921875,
+      "loss": 0.4788,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8333964347839355,
+      "rewards/margins": 0.9296092987060547,
+      "rewards/rejected": -1.7630058526992798,
       "step": 300
     },
     {
       "epoch": 0.63,
-      "eval_logits/chosen": -1.2376099824905396,
-      "eval_logits/rejected": -1.1552666425704956,
-      "eval_logps/chosen": -385.6247253417969,
-      "eval_logps/rejected": -490.81573486328125,
-      "eval_loss": 0.5000515580177307,
-      "eval_rewards/accuracies": 0.75,
-      "eval_rewards/chosen": -1.0399118661880493,
-      "eval_rewards/margins": 0.9906294345855713,
-      "eval_rewards/rejected": -2.03054141998291,
-      "eval_runtime": 97.49,
-      "eval_samples_per_second": 20.515,
-      "eval_steps_per_second": 0.328,
+      "eval_logits/chosen": -0.986372709274292,
+      "eval_logits/rejected": -0.8825947046279907,
+      "eval_logps/chosen": -367.72772216796875,
+      "eval_logps/rejected": -475.55322265625,
+      "eval_loss": 0.49388447403907776,
+      "eval_rewards/accuracies": 0.7578125,
+      "eval_rewards/chosen": -0.8609416484832764,
+      "eval_rewards/margins": 1.016974687576294,
+      "eval_rewards/rejected": -1.8779162168502808,
+      "eval_runtime": 97.7186,
+      "eval_samples_per_second": 20.467,
+      "eval_steps_per_second": 0.327,
       "step": 300
     },
     {
       "epoch": 0.65,
-      "grad_norm": 42.35165195667388,
+      "grad_norm": 50.71884572569721,
       "learning_rate": 1.6583128063291573e-07,
-      "logits/chosen": -1.0750070810317993,
-      "logits/rejected": -0.9531829953193665,
-      "logps/chosen": -435.00848388671875,
-      "logps/rejected": -489.2622985839844,
-      "loss": 0.4864,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.1780123710632324,
-      "rewards/margins": 0.901419997215271,
-      "rewards/rejected": -2.079432725906372,
+      "logits/chosen": -0.8109232783317566,
+      "logits/rejected": -0.6804934144020081,
+      "logps/chosen": -421.7939453125,
+      "logps/rejected": -471.0542907714844,
+      "loss": 0.4943,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.045866847038269,
+      "rewards/margins": 0.8514853715896606,
+      "rewards/rejected": -1.8973522186279297,
       "step": 310
     },
     {
       "epoch": 0.67,
-      "grad_norm": 55.978319290286166,
+      "grad_norm": 50.530325658635704,
       "learning_rate": 1.488723393865766e-07,
-      "logits/chosen": -0.9039633870124817,
-      "logits/rejected": -0.7552443146705627,
-      "logps/chosen": -431.10089111328125,
-      "logps/rejected": -470.2948303222656,
-      "loss": 0.474,
+      "logits/chosen": -0.697489321231842,
+      "logits/rejected": -0.5368015170097351,
+      "logps/chosen": -416.4817810058594,
+      "logps/rejected": -457.8753356933594,
+      "loss": 0.4844,
       "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.13624906539917,
-      "rewards/margins": 0.8904827237129211,
-      "rewards/rejected": -2.0267317295074463,
+      "rewards/chosen": -0.9900579452514648,
+      "rewards/margins": 0.9124795198440552,
+      "rewards/rejected": -1.9025375843048096,
       "step": 320
     },
     {
       "epoch": 0.69,
-      "grad_norm": 48.79887676772944,
+      "grad_norm": 51.29954417938159,
       "learning_rate": 1.3245295796480788e-07,
-      "logits/chosen": -1.0389103889465332,
-      "logits/rejected": -0.8908211588859558,
-      "logps/chosen": -379.30291748046875,
-      "logps/rejected": -461.28790283203125,
-      "loss": 0.4778,
+      "logits/chosen": -0.9539875984191895,
+      "logits/rejected": -0.7579910159111023,
+      "logps/chosen": -368.8915710449219,
+      "logps/rejected": -450.0838317871094,
+      "loss": 0.4783,
       "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.9719207882881165,
-      "rewards/margins": 0.7793878316879272,
-      "rewards/rejected": -1.7513086795806885,
+      "rewards/chosen": -0.8678079843521118,
+      "rewards/margins": 0.7714605331420898,
+      "rewards/rejected": -1.6392685174942017,
       "step": 330
     },
     {
       "epoch": 0.71,
-      "grad_norm": 52.31506719014817,
+      "grad_norm": 47.146951845139505,
       "learning_rate": 1.1666074087171627e-07,
-      "logits/chosen": -0.9594858288764954,
-      "logits/rejected": -0.8123551607131958,
-      "logps/chosen": -412.3460998535156,
-      "logps/rejected": -500.8518981933594,
-      "loss": 0.507,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.9652332067489624,
-      "rewards/margins": 1.036577820777893,
-      "rewards/rejected": -2.0018107891082764,
+      "logits/chosen": -0.9075170755386353,
+      "logits/rejected": -0.7373852133750916,
+      "logps/chosen": -388.6153259277344,
+      "logps/rejected": -477.04217529296875,
+      "loss": 0.4968,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.7279259562492371,
+      "rewards/margins": 1.0357873439788818,
+      "rewards/rejected": -1.7637134790420532,
       "step": 340
     },
     {
       "epoch": 0.73,
-      "grad_norm": 43.461488612565205,
+      "grad_norm": 40.51519007652658,
       "learning_rate": 1.0157994641835734e-07,
-      "logits/chosen": -0.8856803178787231,
-      "logits/rejected": -0.7320196628570557,
-      "logps/chosen": -378.4570617675781,
-      "logps/rejected": -464.3495178222656,
-      "loss": 0.4699,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.007974624633789,
-      "rewards/margins": 1.0314509868621826,
-      "rewards/rejected": -2.039425849914551,
+      "logits/chosen": -0.9085396528244019,
+      "logits/rejected": -0.7681713104248047,
+      "logps/chosen": -352.57073974609375,
+      "logps/rejected": -430.44903564453125,
+      "loss": 0.4725,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.7491118311882019,
+      "rewards/margins": 0.9513088464736938,
+      "rewards/rejected": -1.7004207372665405,
       "step": 350
     },
     {
       "epoch": 0.75,
-      "grad_norm": 43.49572817733842,
+      "grad_norm": 44.22500120445586,
       "learning_rate": 8.729103716819111e-08,
-      "logits/chosen": -1.0236610174179077,
-      "logits/rejected": -0.790179967880249,
-      "logps/chosen": -434.175537109375,
-      "logps/rejected": -491.8944396972656,
-      "loss": 0.5145,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.0748282670974731,
-      "rewards/margins": 0.9761450886726379,
-      "rewards/rejected": -2.050973415374756,
+      "logits/chosen": -1.0195329189300537,
+      "logits/rejected": -0.7282712459564209,
+      "logps/chosen": -405.8896179199219,
+      "logps/rejected": -458.53826904296875,
+      "loss": 0.5133,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.7919691801071167,
+      "rewards/margins": 0.9254425168037415,
+      "rewards/rejected": -1.7174116373062134,
       "step": 360
     },
     {
       "epoch": 0.77,
-      "grad_norm": 39.78731600588934,
+      "grad_norm": 42.97964211979383,
       "learning_rate": 7.387025063449081e-08,
-      "logits/chosen": -0.8810917735099792,
-      "logits/rejected": -0.7196478247642517,
-      "logps/chosen": -407.5464782714844,
-      "logps/rejected": -448.7613220214844,
-      "loss": 0.5004,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.1209478378295898,
-      "rewards/margins": 0.7832604646682739,
-      "rewards/rejected": -1.9042081832885742,
+      "logits/chosen": -0.7996746301651001,
+      "logits/rejected": -0.5788342356681824,
+      "logps/chosen": -381.4356689453125,
+      "logps/rejected": -423.28570556640625,
+      "loss": 0.5002,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8598394393920898,
+      "rewards/margins": 0.7896124124526978,
+      "rewards/rejected": -1.6494518518447876,
       "step": 370
     },
     {
       "epoch": 0.79,
-      "grad_norm": 39.447201988242654,
+      "grad_norm": 40.60703994205003,
       "learning_rate": 6.138919252022435e-08,
-      "logits/chosen": -0.8456799387931824,
-      "logits/rejected": -0.7950155138969421,
-      "logps/chosen": -373.51910400390625,
-      "logps/rejected": -497.63751220703125,
-      "loss": 0.4898,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.1760728359222412,
-      "rewards/margins": 0.9932082891464233,
-      "rewards/rejected": -2.169281244277954,
+      "logits/chosen": -0.7264416217803955,
+      "logits/rejected": -0.689493715763092,
+      "logps/chosen": -355.46893310546875,
+      "logps/rejected": -487.8539123535156,
+      "loss": 0.485,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.9955714344978333,
+      "rewards/margins": 1.0758744478225708,
+      "rewards/rejected": -2.071445941925049,
       "step": 380
     },
     {
       "epoch": 0.82,
-      "grad_norm": 42.50193660041201,
+      "grad_norm": 46.93787360588045,
       "learning_rate": 4.991445467064689e-08,
-      "logits/chosen": -0.9549353718757629,
-      "logits/rejected": -0.8842083215713501,
-      "logps/chosen": -428.71875,
-      "logps/rejected": -496.11370849609375,
-      "loss": 0.4858,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.0410022735595703,
-      "rewards/margins": 0.8284327387809753,
-      "rewards/rejected": -1.8694350719451904,
+      "logits/chosen": -0.8655691146850586,
+      "logits/rejected": -0.7559862732887268,
+      "logps/chosen": -414.95660400390625,
+      "logps/rejected": -487.86029052734375,
+      "loss": 0.4776,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.9033805727958679,
+      "rewards/margins": 0.8835199475288391,
+      "rewards/rejected": -1.786900281906128,
       "step": 390
     },
     {
       "epoch": 0.84,
-      "grad_norm": 46.58374199750293,
+      "grad_norm": 44.554590531311376,
       "learning_rate": 3.9507259776993954e-08,
-      "logits/chosen": -0.8936346173286438,
-      "logits/rejected": -0.6968635320663452,
-      "logps/chosen": -398.35504150390625,
-      "logps/rejected": -488.99212646484375,
-      "loss": 0.4965,
+      "logits/chosen": -0.6984928846359253,
+      "logits/rejected": -0.5499908924102783,
+      "logps/chosen": -390.16436767578125,
+      "logps/rejected": -478.6742248535156,
+      "loss": 0.4937,
       "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.062227725982666,
-      "rewards/margins": 0.9571460485458374,
-      "rewards/rejected": -2.019374132156372,
+      "rewards/chosen": -0.9803212285041809,
+      "rewards/margins": 0.9358729124069214,
+      "rewards/rejected": -1.916194200515747,
       "step": 400
     },
     {
       "epoch": 0.84,
-      "eval_logits/chosen": -1.0282753705978394,
-      "eval_logits/rejected": -0.9140445590019226,
-      "eval_logps/chosen": -372.67291259765625,
-      "eval_logps/rejected": -476.9779968261719,
-      "eval_loss": 0.495796263217926,
-      "eval_rewards/accuracies": 0.76953125,
-      "eval_rewards/chosen": -0.9103929996490479,
-      "eval_rewards/margins": 0.9817712306976318,
-      "eval_rewards/rejected": -1.8921643495559692,
-      "eval_runtime": 97.5531,
-      "eval_samples_per_second": 20.502,
-      "eval_steps_per_second": 0.328,
+      "eval_logits/chosen": -0.9126077890396118,
+      "eval_logits/rejected": -0.760056734085083,
+      "eval_logps/chosen": -361.4586486816406,
+      "eval_logps/rejected": -470.95220947265625,
+      "eval_loss": 0.4905379116535187,
+      "eval_rewards/accuracies": 0.75390625,
+      "eval_rewards/chosen": -0.7982508540153503,
+      "eval_rewards/margins": 1.0336554050445557,
+      "eval_rewards/rejected": -1.8319063186645508,
+      "eval_runtime": 97.7293,
+      "eval_samples_per_second": 20.465,
+      "eval_steps_per_second": 0.327,
       "step": 400
     },
     {
       "epoch": 0.86,
-      "grad_norm": 47.55530045158415,
+      "grad_norm": 39.65148650158285,
       "learning_rate": 3.022313472693447e-08,
-      "logits/chosen": -1.049152135848999,
-      "logits/rejected": -0.7811739444732666,
-      "logps/chosen": -410.12506103515625,
-      "logps/rejected": -482.9457092285156,
-      "loss": 0.491,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.8961167335510254,
-      "rewards/margins": 0.9997068643569946,
-      "rewards/rejected": -1.8958238363265991,
+      "logits/chosen": -0.938845157623291,
+      "logits/rejected": -0.6247340440750122,
+      "logps/chosen": -400.84222412109375,
+      "logps/rejected": -476.00018310546875,
+      "loss": 0.4863,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.8032888174057007,
+      "rewards/margins": 1.0230791568756104,
+      "rewards/rejected": -1.8263680934906006,
       "step": 410
     },
     {
       "epoch": 0.88,
-      "grad_norm": 44.81818011957173,
+      "grad_norm": 42.93092965398867,
       "learning_rate": 2.2111614344599684e-08,
-      "logits/chosen": -1.0166425704956055,
-      "logits/rejected": -0.9132588505744934,
-      "logps/chosen": -412.86907958984375,
-      "logps/rejected": -484.56207275390625,
-      "loss": 0.4841,
+      "logits/chosen": -0.8916628956794739,
+      "logits/rejected": -0.7629222869873047,
+      "logps/chosen": -404.97088623046875,
+      "logps/rejected": -487.2880859375,
+      "loss": 0.4797,
       "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.915523886680603,
-      "rewards/margins": 0.9399937391281128,
-      "rewards/rejected": -1.8555176258087158,
+      "rewards/chosen": -0.836542010307312,
+      "rewards/margins": 1.0462357997894287,
+      "rewards/rejected": -1.8827779293060303,
       "step": 420
     },
     {
       "epoch": 0.9,
-      "grad_norm": 47.61958744158691,
+      "grad_norm": 50.64982947415171,
       "learning_rate": 1.521597710086439e-08,
-      "logits/chosen": -0.8952162861824036,
-      "logits/rejected": -0.7479010224342346,
-      "logps/chosen": -406.29949951171875,
-      "logps/rejected": -476.6239318847656,
-      "loss": 0.4761,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.0442266464233398,
-      "rewards/margins": 0.9349378347396851,
-      "rewards/rejected": -1.9791643619537354,
+      "logits/chosen": -0.7213121056556702,
+      "logits/rejected": -0.5332973003387451,
+      "logps/chosen": -401.74871826171875,
+      "logps/rejected": -478.0919494628906,
+      "loss": 0.4735,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.9987181425094604,
+      "rewards/margins": 0.9951263666152954,
+      "rewards/rejected": -1.9938443899154663,
       "step": 430
     },
     {
       "epoch": 0.92,
-      "grad_norm": 44.69602239358015,
+      "grad_norm": 45.64074270515632,
       "learning_rate": 9.57301420397924e-09,
-      "logits/chosen": -0.9741517305374146,
-      "logits/rejected": -0.8120586276054382,
-      "logps/chosen": -396.1504821777344,
-      "logps/rejected": -471.24591064453125,
-      "loss": 0.4934,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.9137523770332336,
-      "rewards/margins": 0.8474210500717163,
-      "rewards/rejected": -1.7611732482910156,
+      "logits/chosen": -0.8010743856430054,
+      "logits/rejected": -0.6153632998466492,
+      "logps/chosen": -389.0445251464844,
+      "logps/rejected": -466.53387451171875,
+      "loss": 0.4956,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.842692494392395,
+      "rewards/margins": 0.8713605999946594,
+      "rewards/rejected": -1.7140531539916992,
       "step": 440
     },
     {
       "epoch": 0.94,
-      "grad_norm": 42.566625000272865,
+      "grad_norm": 42.58278664732128,
       "learning_rate": 5.212833302556258e-09,
-      "logits/chosen": -0.8404706120491028,
-      "logits/rejected": -0.7910974025726318,
-      "logps/chosen": -417.85894775390625,
-      "logps/rejected": -517.82861328125,
-      "loss": 0.4938,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.0513834953308105,
-      "rewards/margins": 0.8171535730361938,
-      "rewards/rejected": -1.868537187576294,
+      "logits/chosen": -0.7098689675331116,
+      "logits/rejected": -0.6174753308296204,
+      "logps/chosen": -411.04345703125,
+      "logps/rejected": -515.0621948242188,
+      "loss": 0.4986,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.983228325843811,
+      "rewards/margins": 0.8576449155807495,
+      "rewards/rejected": -1.84087336063385,
       "step": 450
     },
     {
       "epoch": 0.96,
-      "grad_norm": 48.01378818460922,
+      "grad_norm": 60.928874193313085,
       "learning_rate": 2.158697848236607e-09,
-      "logits/chosen": -0.9327942728996277,
-      "logits/rejected": -0.7448553442955017,
-      "logps/chosen": -391.20928955078125,
-      "logps/rejected": -446.97723388671875,
-      "loss": 0.4927,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.9240593910217285,
-      "rewards/margins": 0.8686177134513855,
-      "rewards/rejected": -1.7926772832870483,
+      "logits/chosen": -0.7517565488815308,
+      "logits/rejected": -0.5577678084373474,
+      "logps/chosen": -384.75677490234375,
+      "logps/rejected": -444.00030517578125,
+      "loss": 0.4922,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.859534740447998,
+      "rewards/margins": 0.9033729434013367,
+      "rewards/rejected": -1.76290762424469,
       "step": 460
     },
     {
       "epoch": 0.98,
-      "grad_norm": 39.931022031228444,
+      "grad_norm": 45.081509845975475,
       "learning_rate": 4.269029751107489e-10,
-      "logits/chosen": -0.9641515016555786,
-      "logits/rejected": -0.7805702686309814,
-      "logps/chosen": -382.63818359375,
-      "logps/rejected": -478.60125732421875,
-      "loss": 0.4822,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.8783089518547058,
-      "rewards/margins": 0.9068325757980347,
-      "rewards/rejected": -1.7851415872573853,
+      "logits/chosen": -0.7836319804191589,
+      "logits/rejected": -0.5721665620803833,
+      "logps/chosen": -376.28497314453125,
+      "logps/rejected": -472.9750061035156,
+      "loss": 0.4796,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.814776599407196,
+      "rewards/margins": 0.9141017198562622,
+      "rewards/rejected": -1.7288782596588135,
       "step": 470
     },
     {
       "epoch": 1.0,
       "step": 478,
       "total_flos": 0.0,
-      "train_loss": 0.5343728439578451,
-      "train_runtime": 7644.6536,
-      "train_samples_per_second": 7.997,
-      "train_steps_per_second": 0.063
+      "train_loss": 0.07911034258838479,
+      "train_runtime": 1140.8874,
+      "train_samples_per_second": 53.585,
+      "train_steps_per_second": 0.419
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:871e8405759b32870571027069a523e6d82f4f33e7b8c54225735aff5fd7b74f
+oid sha256:868f154b678596a33694b41603d52d03ffdc90d23fb450de0a0ba992ad4dccd0
 size 6200