Model save

README.md

@@ -13,7 +13,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model was trained from scratch on an unknown dataset.
+This model was trained from scratch on the None dataset.
 
 ## Model description
 
@@ -32,7 +32,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 3e-07
+- learning_rate: 5e-07
 - train_batch_size: 4
 - eval_batch_size: 4
 - seed: 42
@@ -54,5 +54,5 @@ The following hyperparameters were used during training:
 
 - Transformers 4.39.3
 - Pytorch 2.1.2+cu118
-- Datasets 2.19.1
+- Datasets 2.16.1
 - Tokenizers 0.15.2

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.20326648155848184,
-    "train_runtime": 5896.1189,
+    "train_loss": 0.4513903107398596,
+    "train_runtime": 5854.463,
     "train_samples": 50000,
-    "train_samples_per_second": 8.48,
-    "train_steps_per_second": 0.066
+    "train_samples_per_second": 8.54,
+    "train_steps_per_second": 0.067
 }

config.json

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.41.1",
+  "transformers_version": "4.39.3",
   "use_cache": false,
   "vocab_size": 32000
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:58bde9bfb67010c336cace37ab13ac39da6af79040fcdbdabd0a04935b66a870
+oid sha256:04f2436f631684bd85d9ee0f015c2a6d18e79bd1c5a33dd8503659b12524b028
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:35d424b1b9f269fcdb54ade09434feadf354ef611fe440a2d936528908734919
+oid sha256:70de4abb0b18d600edfa17cb8203a64d82cb5cd93011b4c04bcdbe4e85b25295
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:073b6342526b72183cb12d771d63bd08d1440bc35898cc6d740a2bf32a97585b
+oid sha256:3f381addd443a7edabe9d8200e1356682fb04f61fecc2810526dc1a4c51d7868
 size 4540516344

runs/Jun22_08-38-32_n136-100-194/events.out.tfevents.1719016734.n136-100-194.1556408.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a680a3693b3c5f01d4a780663ddb11807a5a69d4f00219979400f7bc2764c809
-size 26098
+oid sha256:ffe1de74fe45221dbe6392c3d7e72da21567f3fa1c45a226d48ccd3eb6bd3501
+size 32644

tokenizer.json

@@ -134,7 +134,6 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
-    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.20326648155848184,
-    "train_runtime": 5896.1189,
+    "train_loss": 0.4513903107398596,
+    "train_runtime": 5854.463,
     "train_samples": 50000,
-    "train_samples_per_second": 8.48,
-    "train_steps_per_second": 0.066
+    "train_samples_per_second": 8.54,
+    "train_steps_per_second": 0.067
 }

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 747.0977926336889,
-      "learning_rate": 7.692307692307691e-09,
-      "logits/chosen": -2.5617921352386475,
-      "logits/rejected": -2.415619373321533,
+      "grad_norm": 1292.2103490533002,
+      "learning_rate": 1.282051282051282e-08,
+      "logits/chosen": -2.5583817958831787,
+      "logits/rejected": -2.4487552642822266,
       "logps/chosen": -258.1644592285156,
-      "logps/rejected": -191.65736389160156,
-      "loss": 0.6931,
+      "logps/rejected": -216.25729370117188,
+      "loss": 0.6964,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,597 +25,597 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 746.141595669296,
-      "learning_rate": 7.692307692307691e-08,
-      "logits/chosen": -2.6110925674438477,
-      "logits/rejected": -2.524423122406006,
-      "logps/chosen": -267.3368225097656,
-      "logps/rejected": -198.19520568847656,
-      "loss": 0.6963,
-      "rewards/accuracies": 0.3888888955116272,
-      "rewards/chosen": -0.02406422607600689,
-      "rewards/margins": -0.021091409027576447,
-      "rewards/rejected": -0.0029728179797530174,
+      "grad_norm": 1261.1198733310573,
+      "learning_rate": 1.2820512820512818e-07,
+      "logits/chosen": -2.6061007976531982,
+      "logits/rejected": -2.553147315979004,
+      "logps/chosen": -267.5506591796875,
+      "logps/rejected": -217.63583374023438,
+      "loss": 0.7045,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.012439604848623276,
+      "rewards/margins": 0.01009546872228384,
+      "rewards/rejected": 0.002344133099541068,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 433.45696577907285,
-      "learning_rate": 1.5384615384615382e-07,
-      "logits/chosen": -2.628641128540039,
-      "logits/rejected": -2.5271899700164795,
-      "logps/chosen": -260.9211120605469,
-      "logps/rejected": -198.38711547851562,
-      "loss": 0.5697,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": 0.20215623080730438,
-      "rewards/margins": 0.35489505529403687,
-      "rewards/rejected": -0.1527387946844101,
+      "grad_norm": 853.2707411908161,
+      "learning_rate": 2.5641025641025636e-07,
+      "logits/chosen": -2.629751443862915,
+      "logits/rejected": -2.5669989585876465,
+      "logps/chosen": -260.5412292480469,
+      "logps/rejected": -207.0039825439453,
+      "loss": 0.5192,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.6866833567619324,
+      "rewards/margins": 0.6415502429008484,
+      "rewards/rejected": 0.045133065432310104,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "grad_norm": 466.53027664588734,
-      "learning_rate": 2.3076923076923078e-07,
-      "logits/chosen": -2.6438043117523193,
-      "logits/rejected": -2.5429482460021973,
-      "logps/chosen": -252.55557250976562,
-      "logps/rejected": -195.11138916015625,
-      "loss": 0.306,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 1.301897406578064,
-      "rewards/margins": 1.880059838294983,
-      "rewards/rejected": -0.5781622529029846,
+      "grad_norm": 1142.6320871817904,
+      "learning_rate": 3.8461538461538463e-07,
+      "logits/chosen": -2.6441783905029297,
+      "logits/rejected": -2.5700392723083496,
+      "logps/chosen": -251.12313842773438,
+      "logps/rejected": -198.34071350097656,
+      "loss": 0.3383,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 3.621673583984375,
+      "rewards/margins": 3.1466643810272217,
+      "rewards/rejected": 0.47500935196876526,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 451.3374046706128,
-      "learning_rate": 2.999939918069778e-07,
-      "logits/chosen": -2.6604888439178467,
-      "logits/rejected": -2.5366878509521484,
-      "logps/chosen": -244.97402954101562,
-      "logps/rejected": -199.42355346679688,
-      "loss": 0.2052,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 2.6914100646972656,
-      "rewards/margins": 3.822247266769409,
-      "rewards/rejected": -1.130837321281433,
+      "grad_norm": 498.4795422948633,
+      "learning_rate": 4.99989986344963e-07,
+      "logits/chosen": -2.6392924785614014,
+      "logits/rejected": -2.561156749725342,
+      "logps/chosen": -243.8309783935547,
+      "logps/rejected": -192.75845336914062,
+      "loss": 0.3207,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 5.866530418395996,
+      "rewards/margins": 5.031473636627197,
+      "rewards/rejected": 0.8350569009780884,
       "step": 40
     },
     {
       "epoch": 0.13,
-      "grad_norm": 268.00932309883206,
-      "learning_rate": 2.9927359084964875e-07,
-      "logits/chosen": -2.7027461528778076,
-      "logits/rejected": -2.5904927253723145,
-      "logps/chosen": -258.02886962890625,
-      "logps/rejected": -204.94631958007812,
-      "loss": 0.2326,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 4.602013111114502,
-      "rewards/margins": 5.5924835205078125,
-      "rewards/rejected": -0.9904701113700867,
+      "grad_norm": 687.258084457276,
+      "learning_rate": 4.987893180827479e-07,
+      "logits/chosen": -2.6534440517425537,
+      "logits/rejected": -2.5842742919921875,
+      "logps/chosen": -258.9037170410156,
+      "logps/rejected": -203.41665649414062,
+      "loss": 0.3666,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 8.400110244750977,
+      "rewards/margins": 6.8410444259643555,
+      "rewards/rejected": 1.559065818786621,
       "step": 50
     },
     {
       "epoch": 0.15,
-      "grad_norm": 340.360656955259,
-      "learning_rate": 2.9735816061234966e-07,
-      "logits/chosen": -2.6768908500671387,
-      "logits/rejected": -2.557954788208008,
-      "logps/chosen": -259.3023681640625,
-      "logps/rejected": -192.10040283203125,
-      "loss": 0.1987,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 4.70054817199707,
-      "rewards/margins": 6.1066999435424805,
-      "rewards/rejected": -1.4061520099639893,
+      "grad_norm": 759.1516730607283,
+      "learning_rate": 4.955969343539162e-07,
+      "logits/chosen": -2.616115093231201,
+      "logits/rejected": -2.5453152656555176,
+      "logps/chosen": -262.9014892578125,
+      "logps/rejected": -209.2264404296875,
+      "loss": 0.3916,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 5.333600044250488,
+      "rewards/margins": 6.3847455978393555,
+      "rewards/rejected": -1.0511456727981567,
       "step": 60
     },
     {
       "epoch": 0.18,
-      "grad_norm": 344.3937897868754,
-      "learning_rate": 2.942630353226844e-07,
-      "logits/chosen": -2.650172472000122,
-      "logits/rejected": -2.500756025314331,
-      "logps/chosen": -288.3818054199219,
-      "logps/rejected": -222.72811889648438,
-      "loss": 0.1904,
+      "grad_norm": 560.0502144214485,
+      "learning_rate": 4.90438392204474e-07,
+      "logits/chosen": -2.590355634689331,
+      "logits/rejected": -2.5212607383728027,
+      "logps/chosen": -292.03955078125,
+      "logps/rejected": -227.56900024414062,
+      "loss": 0.366,
       "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 4.768267631530762,
-      "rewards/margins": 7.4695234298706055,
-      "rewards/rejected": -2.7012553215026855,
+      "rewards/chosen": 5.28397798538208,
+      "rewards/margins": 7.424477577209473,
+      "rewards/rejected": -2.1404995918273926,
       "step": 70
     },
     {
       "epoch": 0.2,
-      "grad_norm": 453.28474416843744,
-      "learning_rate": 2.900129934114876e-07,
-      "logits/chosen": -2.5939252376556396,
-      "logits/rejected": -2.458728790283203,
-      "logps/chosen": -260.589111328125,
-      "logps/rejected": -212.63925170898438,
-      "loss": 0.1791,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 3.8766517639160156,
-      "rewards/margins": 7.434275150299072,
-      "rewards/rejected": -3.5576236248016357,
+      "grad_norm": 1037.6404299504059,
+      "learning_rate": 4.83354989019146e-07,
+      "logits/chosen": -2.5414998531341553,
+      "logits/rejected": -2.468327283859253,
+      "logps/chosen": -260.0003356933594,
+      "logps/rejected": -203.8599090576172,
+      "loss": 0.3627,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 6.802065372467041,
+      "rewards/margins": 7.507330894470215,
+      "rewards/rejected": -0.705264687538147,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 598.8098573854415,
-      "learning_rate": 2.8464205914585213e-07,
-      "logits/chosen": -2.6328907012939453,
-      "logits/rejected": -2.507690906524658,
-      "logps/chosen": -253.75570678710938,
-      "logps/rejected": -193.23147583007812,
-      "loss": 0.19,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 3.2832133769989014,
-      "rewards/margins": 6.802037715911865,
-      "rewards/rejected": -3.518825054168701,
+      "grad_norm": 1095.0164975833522,
+      "learning_rate": 4.7440343190975353e-07,
+      "logits/chosen": -2.5701346397399902,
+      "logits/rejected": -2.513249397277832,
+      "logps/chosen": -257.8319396972656,
+      "logps/rejected": -217.47695922851562,
+      "loss": 0.3529,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 3.5220108032226562,
+      "rewards/margins": 6.205338478088379,
+      "rewards/rejected": -2.683328151702881,
       "step": 90
     },
     {
       "epoch": 0.26,
-      "grad_norm": 242.78091859451033,
-      "learning_rate": 2.78193230243403e-07,
-      "logits/chosen": -2.6586058139801025,
-      "logits/rejected": -2.52885103225708,
-      "logps/chosen": -234.36068725585938,
-      "logps/rejected": -197.55322265625,
-      "loss": 0.1766,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 4.212424278259277,
-      "rewards/margins": 7.145503997802734,
-      "rewards/rejected": -2.933079242706299,
+      "grad_norm": 493.8561719761038,
+      "learning_rate": 4.6365538373900506e-07,
+      "logits/chosen": -2.624537944793701,
+      "logits/rejected": -2.5495262145996094,
+      "logps/chosen": -237.0422821044922,
+      "logps/rejected": -201.1056671142578,
+      "loss": 0.6296,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 4.613960266113281,
+      "rewards/margins": 6.805362701416016,
+      "rewards/rejected": -2.191401958465576,
       "step": 100
     },
     {
       "epoch": 0.28,
-      "grad_norm": 288.7802754186709,
-      "learning_rate": 2.707181336484383e-07,
-      "logits/chosen": -2.6430606842041016,
-      "logits/rejected": -2.5142664909362793,
-      "logps/chosen": -253.27975463867188,
-      "logps/rejected": -192.4219207763672,
-      "loss": 0.1616,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 5.699277400970459,
-      "rewards/margins": 7.844499111175537,
-      "rewards/rejected": -2.1452219486236572,
+      "grad_norm": 783.3339585792174,
+      "learning_rate": 4.5119688941406386e-07,
+      "logits/chosen": -2.6185028553009033,
+      "logits/rejected": -2.5373358726501465,
+      "logps/chosen": -258.2229919433594,
+      "logps/rejected": -209.95010375976562,
+      "loss": 0.5085,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 6.706292629241943,
+      "rewards/margins": 8.07982349395752,
+      "rewards/rejected": -1.3735301494598389,
       "step": 110
     },
     {
       "epoch": 0.31,
-      "grad_norm": 242.33756453841693,
-      "learning_rate": 2.622766122256652e-07,
-      "logits/chosen": -2.6139981746673584,
-      "logits/rejected": -2.516448497772217,
-      "logps/chosen": -256.9673767089844,
-      "logps/rejected": -211.84988403320312,
-      "loss": 0.1786,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 5.386040210723877,
-      "rewards/margins": 8.150399208068848,
-      "rewards/rejected": -2.7643585205078125,
+      "grad_norm": 1013.1425815966796,
+      "learning_rate": 4.3712768704277524e-07,
+      "logits/chosen": -2.5965781211853027,
+      "logits/rejected": -2.5263915061950684,
+      "logps/chosen": -263.1861572265625,
+      "logps/rejected": -208.8050079345703,
+      "loss": 0.4699,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 5.6486382484436035,
+      "rewards/margins": 7.843411445617676,
+      "rewards/rejected": -2.194772720336914,
       "step": 120
     },
     {
       "epoch": 0.33,
-      "grad_norm": 181.17712779023478,
-      "learning_rate": 2.5293624568031005e-07,
-      "logits/chosen": -2.5844571590423584,
-      "logits/rejected": -2.4461381435394287,
-      "logps/chosen": -248.85513305664062,
-      "logps/rejected": -187.93716430664062,
-      "loss": 0.172,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 4.1653828620910645,
-      "rewards/margins": 7.870238304138184,
-      "rewards/rejected": -3.7048561573028564,
+      "grad_norm": 659.1716861853262,
+      "learning_rate": 4.2156040946718343e-07,
+      "logits/chosen": -2.571542263031006,
+      "logits/rejected": -2.502825975418091,
+      "logps/chosen": -252.36941528320312,
+      "logps/rejected": -197.37109375,
+      "loss": 0.4289,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 4.440427303314209,
+      "rewards/margins": 7.714503288269043,
+      "rewards/rejected": -3.274075746536255,
       "step": 130
     },
     {
       "epoch": 0.36,
-      "grad_norm": 386.4455316752931,
-      "learning_rate": 2.4277180953993823e-07,
-      "logits/chosen": -2.6205365657806396,
-      "logits/rejected": -2.5077974796295166,
-      "logps/chosen": -269.39251708984375,
-      "logps/rejected": -207.6556854248047,
-      "loss": 0.2112,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 3.9067764282226562,
-      "rewards/margins": 7.915855407714844,
-      "rewards/rejected": -4.0090789794921875,
+      "grad_norm": 706.862930333198,
+      "learning_rate": 4.046196825665637e-07,
+      "logits/chosen": -2.5883898735046387,
+      "logits/rejected": -2.5185062885284424,
+      "logps/chosen": -271.88494873046875,
+      "logps/rejected": -218.22323608398438,
+      "loss": 0.4943,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 2.9622278213500977,
+      "rewards/margins": 7.161763668060303,
+      "rewards/rejected": -4.199534893035889,
       "step": 140
     },
     {
       "epoch": 0.38,
-      "grad_norm": 264.0646768310831,
-      "learning_rate": 2.3186467652917566e-07,
-      "logits/chosen": -2.607632875442505,
-      "logits/rejected": -2.5044326782226562,
-      "logps/chosen": -261.9036560058594,
-      "logps/rejected": -212.12777709960938,
-      "loss": 0.2286,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 4.435807704925537,
-      "rewards/margins": 8.641576766967773,
-      "rewards/rejected": -4.2057695388793945,
+      "grad_norm": 715.990249793818,
+      "learning_rate": 3.864411275486261e-07,
+      "logits/chosen": -2.5699923038482666,
+      "logits/rejected": -2.502592086791992,
+      "logps/chosen": -264.418212890625,
+      "logps/rejected": -213.1693115234375,
+      "loss": 0.5111,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 5.556182384490967,
+      "rewards/margins": 8.052742004394531,
+      "rewards/rejected": -2.4965591430664062,
       "step": 150
     },
     {
       "epoch": 0.41,
-      "grad_norm": 571.6419774908388,
-      "learning_rate": 2.2030216512970552e-07,
-      "logits/chosen": -2.5919671058654785,
-      "logits/rejected": -2.48041033744812,
-      "logps/chosen": -240.66891479492188,
-      "logps/rejected": -183.67489624023438,
-      "loss": 0.2024,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 4.69089412689209,
-      "rewards/margins": 7.830643653869629,
-      "rewards/rejected": -3.139749526977539,
+      "grad_norm": 710.6520782907305,
+      "learning_rate": 3.671702752161759e-07,
+      "logits/chosen": -2.559586763381958,
+      "logits/rejected": -2.4897525310516357,
+      "logps/chosen": -245.1999969482422,
+      "logps/rejected": -198.19442749023438,
+      "loss": 0.4696,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 3.364131212234497,
+      "rewards/margins": 8.019769668579102,
+      "rewards/rejected": -4.655638694763184,
       "step": 160
     },
     {
       "epoch": 0.44,
-      "grad_norm": 351.0335983478419,
-      "learning_rate": 2.0817684054072823e-07,
-      "logits/chosen": -2.594038724899292,
-      "logits/rejected": -2.479696273803711,
-      "logps/chosen": -259.5168151855469,
-      "logps/rejected": -199.40609741210938,
-      "loss": 0.1492,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 4.83851432800293,
-      "rewards/margins": 7.966272830963135,
-      "rewards/rejected": -3.127758741378784,
+      "grad_norm": 865.8215089865342,
+      "learning_rate": 3.4696140090121375e-07,
+      "logits/chosen": -2.57694935798645,
+      "logits/rejected": -2.5136027336120605,
+      "logps/chosen": -266.91412353515625,
+      "logps/rejected": -211.6914825439453,
+      "loss": 0.397,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 3.399867296218872,
+      "rewards/margins": 8.182429313659668,
+      "rewards/rejected": -4.782561302185059,
       "step": 170
     },
     {
       "epoch": 0.46,
-      "grad_norm": 332.9543090868915,
-      "learning_rate": 1.9558577363613703e-07,
-      "logits/chosen": -2.643381357192993,
-      "logits/rejected": -2.5339646339416504,
-      "logps/chosen": -235.37637329101562,
-      "logps/rejected": -190.86380004882812,
-      "loss": 0.1789,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 3.2347798347473145,
-      "rewards/margins": 7.21783971786499,
-      "rewards/rejected": -3.983060359954834,
+      "grad_norm": 658.5054936178984,
+      "learning_rate": 3.259762893935617e-07,
+      "logits/chosen": -2.6607277393341064,
+      "logits/rejected": -2.581068515777588,
+      "logps/chosen": -239.2282257080078,
+      "logps/rejected": -188.63772583007812,
+      "loss": 0.4691,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 2.5246458053588867,
+      "rewards/margins": 7.373200416564941,
+      "rewards/rejected": -4.848554611206055,
       "step": 180
     },
     {
       "epoch": 0.49,
-      "grad_norm": 413.79495264463003,
-      "learning_rate": 1.826297638509251e-07,
-      "logits/chosen": -2.595829725265503,
-      "logits/rejected": -2.500739812850952,
-      "logps/chosen": -256.819580078125,
-      "logps/rejected": -206.32376098632812,
-      "loss": 0.179,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 3.607396364212036,
-      "rewards/margins": 7.752522945404053,
-      "rewards/rejected": -4.145126819610596,
+      "grad_norm": 746.921617266345,
+      "learning_rate": 3.0438293975154184e-07,
+      "logits/chosen": -2.6273646354675293,
+      "logits/rejected": -2.5594921112060547,
+      "logps/chosen": -263.73443603515625,
+      "logps/rejected": -207.8045654296875,
+      "loss": 0.4399,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 0.9775351285934448,
+      "rewards/margins": 8.604721069335938,
+      "rewards/rejected": -7.6271867752075195,
       "step": 190
     },
     {
       "epoch": 0.51,
-      "grad_norm": 458.13511698305706,
-      "learning_rate": 1.694125322181083e-07,
-      "logits/chosen": -2.63800311088562,
-      "logits/rejected": -2.5028953552246094,
-      "logps/chosen": -269.77081298828125,
-      "logps/rejected": -201.87176513671875,
-      "loss": 0.1642,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 5.391812801361084,
-      "rewards/margins": 9.271406173706055,
-      "rewards/rejected": -3.8795933723449707,
+      "grad_norm": 859.0725144089504,
+      "learning_rate": 2.823542203635138e-07,
+      "logits/chosen": -2.6617209911346436,
+      "logits/rejected": -2.5823440551757812,
+      "logps/chosen": -278.6440734863281,
+      "logps/rejected": -222.3423309326172,
+      "loss": 0.493,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 1.6911423206329346,
+      "rewards/margins": 9.542765617370605,
+      "rewards/rejected": -7.851624488830566,
       "step": 200
     },
     {
       "epoch": 0.54,
-      "grad_norm": 449.95115191482023,
-      "learning_rate": 1.5603989101641228e-07,
-      "logits/chosen": -2.620668888092041,
-      "logits/rejected": -2.5066120624542236,
-      "logps/chosen": -262.0960693359375,
-      "logps/rejected": -210.2143096923828,
-      "loss": 0.158,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 4.4803595542907715,
-      "rewards/margins": 8.733253479003906,
-      "rewards/rejected": -4.252893924713135,
+      "grad_norm": 1073.7778122145994,
+      "learning_rate": 2.600664850273538e-07,
+      "logits/chosen": -2.6473941802978516,
+      "logits/rejected": -2.581458568572998,
+      "logps/chosen": -270.93621826171875,
+      "logps/rejected": -214.9030303955078,
+      "loss": 0.7378,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 0.10583686828613281,
+      "rewards/margins": 7.7938642501831055,
+      "rewards/rejected": -7.688027858734131,
       "step": 210
     },
     {
       "epoch": 0.56,
-      "grad_norm": 386.8925888622673,
-      "learning_rate": 1.4261889667621828e-07,
-      "logits/chosen": -2.623037815093994,
-      "logits/rejected": -2.5237972736358643,
-      "logps/chosen": -254.78390502929688,
-      "logps/rejected": -206.7708282470703,
-      "loss": 0.2113,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 3.867889881134033,
-      "rewards/margins": 8.484498023986816,
-      "rewards/rejected": -4.616608142852783,
+      "grad_norm": 783.9779969138841,
+      "learning_rate": 2.3769816112703045e-07,
+      "logits/chosen": -2.6640350818634033,
+      "logits/rejected": -2.606898307800293,
+      "logps/chosen": -260.6977233886719,
+      "logps/rejected": -216.708740234375,
+      "loss": 0.5298,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 0.881191611289978,
+      "rewards/margins": 7.128912448883057,
+      "rewards/rejected": -6.247720241546631,
       "step": 220
     },
     {
       "epoch": 0.59,
-      "grad_norm": 264.4396878771059,
-      "learning_rate": 1.2925699272529007e-07,
-      "logits/chosen": -2.6517319679260254,
-      "logits/rejected": -2.5196144580841064,
-      "logps/chosen": -263.8755798339844,
-      "logps/rejected": -206.9174346923828,
-      "loss": 0.1518,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 4.761946201324463,
-      "rewards/margins": 8.56539249420166,
-      "rewards/rejected": -3.8034462928771973,
+      "grad_norm": 656.1996441950056,
+      "learning_rate": 2.1542832120881677e-07,
+      "logits/chosen": -2.6948089599609375,
+      "logits/rejected": -2.615889072418213,
+      "logps/chosen": -270.77557373046875,
+      "logps/rejected": -219.38320922851562,
+      "loss": 0.4547,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 2.3647713661193848,
+      "rewards/margins": 8.186816215515137,
+      "rewards/rejected": -5.82204532623291,
       "step": 230
     },
     {
       "epoch": 0.61,
-      "grad_norm": 419.86487391831014,
-      "learning_rate": 1.160611496355417e-07,
-      "logits/chosen": -2.6276602745056152,
-      "logits/rejected": -2.5167899131774902,
-      "logps/chosen": -257.7350769042969,
-      "logps/rejected": -210.53292846679688,
-      "loss": 0.1539,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 4.4489850997924805,
-      "rewards/margins": 9.010086059570312,
-      "rewards/rejected": -4.561100482940674,
+      "grad_norm": 1015.3762946599339,
+      "learning_rate": 1.934352493925695e-07,
+      "logits/chosen": -2.6751513481140137,
+      "logits/rejected": -2.6343884468078613,
+      "logps/chosen": -266.14263916015625,
+      "logps/rejected": -223.5273895263672,
+      "loss": 0.429,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 1.4297797679901123,
+      "rewards/margins": 10.390259742736816,
+      "rewards/rejected": -8.960479736328125,
       "step": 240
     },
     {
       "epoch": 0.64,
-      "grad_norm": 233.18563601164297,
-      "learning_rate": 1.0313700845691635e-07,
-      "logits/chosen": -2.6395657062530518,
-      "logits/rejected": -2.5249786376953125,
-      "logps/chosen": -263.78375244140625,
-      "logps/rejected": -216.53323364257812,
-      "loss": 0.1755,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 4.129330158233643,
-      "rewards/margins": 8.881619453430176,
-      "rewards/rejected": -4.752288341522217,
+      "grad_norm": 586.0366917195352,
+      "learning_rate": 1.7189501409486059e-07,
+      "logits/chosen": -2.6815237998962402,
+      "logits/rejected": -2.6186068058013916,
+      "logps/chosen": -271.32818603515625,
+      "logps/rejected": -225.2965545654297,
+      "loss": 0.4243,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 0.4054955542087555,
+      "rewards/margins": 8.429037094116211,
+      "rewards/rejected": -8.023542404174805,
       "step": 250
     },
     {
       "epoch": 0.67,
-      "grad_norm": 301.32208500782895,
-      "learning_rate": 9.058803509412647e-08,
-      "logits/chosen": -2.63856840133667,
-      "logits/rejected": -2.5386815071105957,
-      "logps/chosen": -257.0624084472656,
-      "logps/rejected": -209.1647186279297,
-      "loss": 0.1345,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 4.407042503356934,
-      "rewards/margins": 8.488649368286133,
-      "rewards/rejected": -4.081605434417725,
+      "grad_norm": 567.2324686905487,
+      "learning_rate": 1.5098005849021078e-07,
+      "logits/chosen": -2.6706247329711914,
+      "logits/rejected": -2.6183619499206543,
+      "logps/chosen": -265.3566589355469,
+      "logps/rejected": -211.55789184570312,
+      "loss": 0.4026,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 0.452970027923584,
+      "rewards/margins": 7.969753265380859,
+      "rewards/rejected": -7.516783237457275,
       "step": 260
     },
     {
       "epoch": 0.69,
-      "grad_norm": 262.72645974554706,
-      "learning_rate": 7.851469199680381e-08,
-      "logits/chosen": -2.6127829551696777,
-      "logits/rejected": -2.4841771125793457,
-      "logps/chosen": -268.802490234375,
-      "logps/rejected": -217.36740112304688,
-      "loss": 0.1877,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 4.018976211547852,
-      "rewards/margins": 9.022808074951172,
-      "rewards/rejected": -5.00383186340332,
+      "grad_norm": 651.6922692823155,
+      "learning_rate": 1.30857819994673e-07,
+      "logits/chosen": -2.648026943206787,
+      "logits/rejected": -2.5697438716888428,
+      "logps/chosen": -276.09661865234375,
+      "logps/rejected": -233.24966430664062,
+      "loss": 0.4615,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.4385630190372467,
+      "rewards/margins": 11.782003402709961,
+      "rewards/rejected": -11.343441009521484,
       "step": 270
     },
     {
       "epoch": 0.72,
-      "grad_norm": 216.84234596530857,
-      "learning_rate": 6.701363389420295e-08,
-      "logits/chosen": -2.649754762649536,
-      "logits/rejected": -2.5276429653167725,
-      "logps/chosen": -265.546875,
-      "logps/rejected": -207.27426147460938,
-      "loss": 0.1499,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 4.076898574829102,
-      "rewards/margins": 9.239118576049805,
-      "rewards/rejected": -5.1622209548950195,
+      "grad_norm": 492.898374979646,
+      "learning_rate": 1.116893898236716e-07,
+      "logits/chosen": -2.6764636039733887,
+      "logits/rejected": -2.6275370121002197,
+      "logps/chosen": -273.55120849609375,
+      "logps/rejected": -222.28256225585938,
+      "loss": 0.431,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 0.1396121382713318,
+      "rewards/margins": 8.543681144714355,
+      "rewards/rejected": -8.404068946838379,
       "step": 280
     },
     {
       "epoch": 0.74,
-      "grad_norm": 267.2442999173217,
-      "learning_rate": 5.617693401310837e-08,
-      "logits/chosen": -2.6028785705566406,
-      "logits/rejected": -2.526676654815674,
-      "logps/chosen": -266.23406982421875,
-      "logps/rejected": -225.2560577392578,
-      "loss": 0.1785,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 3.791701555252075,
-      "rewards/margins": 7.995067596435547,
-      "rewards/rejected": -4.203365802764893,
+      "grad_norm": 601.4691313440442,
+      "learning_rate": 9.362822335518062e-08,
+      "logits/chosen": -2.639714241027832,
+      "logits/rejected": -2.6006019115448,
+      "logps/chosen": -271.51104736328125,
+      "logps/rejected": -219.78646850585938,
+      "loss": 0.3785,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 0.39154329895973206,
+      "rewards/margins": 8.193072319030762,
+      "rewards/rejected": -7.8015289306640625,
       "step": 290
     },
     {
       "epoch": 0.77,
-      "grad_norm": 304.40926278591246,
-      "learning_rate": 4.609134697356009e-08,
-      "logits/chosen": -2.6325907707214355,
-      "logits/rejected": -2.513869524002075,
-      "logps/chosen": -270.08087158203125,
-      "logps/rejected": -216.07247924804688,
-      "loss": 0.1539,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 4.105973720550537,
-      "rewards/margins": 9.096407890319824,
-      "rewards/rejected": -4.990433692932129,
+      "grad_norm": 876.6568683843423,
+      "learning_rate": 7.681891162260015e-08,
+      "logits/chosen": -2.6577625274658203,
+      "logits/rejected": -2.610130786895752,
+      "logps/chosen": -278.3086853027344,
+      "logps/rejected": -223.9765625,
+      "loss": 0.4175,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.23941349983215332,
+      "rewards/margins": 8.298933029174805,
+      "rewards/rejected": -8.059518814086914,
       "step": 300
     },
     {
       "epoch": 0.79,
-      "grad_norm": 183.46140347071307,
-      "learning_rate": 3.683761426338148e-08,
-      "logits/chosen": -2.5910542011260986,
-      "logits/rejected": -2.4951541423797607,
-      "logps/chosen": -275.2190246582031,
-      "logps/rejected": -210.9887237548828,
-      "loss": 0.1539,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 4.16552209854126,
-      "rewards/margins": 8.900744438171387,
-      "rewards/rejected": -4.735221862792969,
+      "grad_norm": 491.2362210232499,
+      "learning_rate": 6.139602377230247e-08,
+      "logits/chosen": -2.623286724090576,
+      "logits/rejected": -2.5633485317230225,
+      "logps/chosen": -281.3511047363281,
+      "logps/rejected": -218.5331268310547,
+      "loss": 0.4291,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.8305476903915405,
+      "rewards/margins": 8.879867553710938,
+      "rewards/rejected": -8.049318313598633,
       "step": 310
     },
     {
       "epoch": 0.82,
-      "grad_norm": 336.8108974655118,
-      "learning_rate": 2.8489817851625024e-08,
-      "logits/chosen": -2.6144814491271973,
-      "logits/rejected": -2.5162534713745117,
-      "logps/chosen": -258.4134826660156,
-      "logps/rejected": -210.2257080078125,
-      "loss": 0.1866,
+      "grad_norm": 762.3598954431114,
+      "learning_rate": 4.748302975270837e-08,
+      "logits/chosen": -2.6497387886047363,
+      "logits/rejected": -2.610168933868408,
+      "logps/chosen": -264.21588134765625,
+      "logps/rejected": -206.8611602783203,
+      "loss": 0.4476,
       "rewards/accuracies": 0.875,
-      "rewards/chosen": 3.252080202102661,
-      "rewards/margins": 8.461040496826172,
-      "rewards/rejected": -5.20896053314209,
+      "rewards/chosen": 0.6263083219528198,
+      "rewards/margins": 7.853513240814209,
+      "rewards/rejected": -7.227204322814941,
       "step": 320
     },
     {
       "epoch": 0.84,
-      "grad_norm": 222.30131769249033,
-      "learning_rate": 2.1114787115667477e-08,
-      "logits/chosen": -2.634732484817505,
-      "logits/rejected": -2.5385124683380127,
-      "logps/chosen": -260.74530029296875,
-      "logps/rejected": -209.8925323486328,
-      "loss": 0.1579,
-      "rewards/accuracies": 0.981249988079071,
-      "rewards/chosen": 4.701260566711426,
-      "rewards/margins": 9.207574844360352,
-      "rewards/rejected": -4.506315231323242,
+      "grad_norm": 668.8344256472935,
+      "learning_rate": 3.5191311859445795e-08,
+      "logits/chosen": -2.6693060398101807,
+      "logits/rejected": -2.6197760105133057,
+      "logps/chosen": -268.38543701171875,
+      "logps/rejected": -220.74081420898438,
+      "loss": 0.4113,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 1.1819716691970825,
+      "rewards/margins": 8.392292976379395,
+      "rewards/rejected": -7.210320949554443,
       "step": 330
     },
     {
       "epoch": 0.87,
-      "grad_norm": 298.91033859930906,
-      "learning_rate": 1.4771563829877598e-08,
-      "logits/chosen": -2.620940685272217,
-      "logits/rejected": -2.5207433700561523,
-      "logps/chosen": -256.2254638671875,
-      "logps/rejected": -197.62445068359375,
-      "loss": 0.1936,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 4.292969703674316,
-      "rewards/margins": 9.223516464233398,
-      "rewards/rejected": -4.930546760559082,
+      "grad_norm": 600.9418350776771,
+      "learning_rate": 2.4619273049795996e-08,
+      "logits/chosen": -2.653104066848755,
+      "logits/rejected": -2.6034810543060303,
+      "logps/chosen": -263.5737609863281,
+      "logps/rejected": -213.79544067382812,
+      "loss": 0.3933,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 1.1136972904205322,
+      "rewards/margins": 9.145872116088867,
+      "rewards/rejected": -8.032175064086914,
       "step": 340
     },
     {
       "epoch": 0.9,
-      "grad_norm": 322.2905254048829,
-      "learning_rate": 9.510929498959268e-09,
-      "logits/chosen": -2.636793851852417,
-      "logits/rejected": -2.522016763687134,
-      "logps/chosen": -265.2098388671875,
-      "logps/rejected": -215.28469848632812,
-      "loss": 0.1724,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 3.8617148399353027,
-      "rewards/margins": 8.617898941040039,
-      "rewards/rejected": -4.7561845779418945,
+      "grad_norm": 448.6960620183454,
+      "learning_rate": 1.5851549164932115e-08,
+      "logits/chosen": -2.663357734680176,
+      "logits/rejected": -2.6194939613342285,
+      "logps/chosen": -272.70233154296875,
+      "logps/rejected": -229.30697631835938,
+      "loss": 0.4249,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 1.3936760425567627,
+      "rewards/margins": 8.432661056518555,
+      "rewards/rejected": -7.0389862060546875,
       "step": 350
     },
     {
       "epoch": 0.92,
-      "grad_norm": 389.7582997593581,
-      "learning_rate": 5.374998819965654e-09,
-      "logits/chosen": -2.6430556774139404,
-      "logits/rejected": -2.5382204055786133,
-      "logps/chosen": -270.87615966796875,
-      "logps/rejected": -212.25949096679688,
-      "loss": 0.1727,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 4.255101680755615,
-      "rewards/margins": 8.951577186584473,
-      "rewards/rejected": -4.696475028991699,
+      "grad_norm": 606.8337189375926,
+      "learning_rate": 8.958331366609423e-09,
+      "logits/chosen": -2.666874885559082,
+      "logits/rejected": -2.6079227924346924,
+      "logps/chosen": -278.23712158203125,
+      "logps/rejected": -222.1784210205078,
+      "loss": 0.4777,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 0.8818317651748657,
+      "rewards/margins": 8.486412048339844,
+      "rewards/rejected": -7.604579925537109,
       "step": 360
     },
     {
       "epoch": 0.95,
-      "grad_norm": 224.62737396153923,
-      "learning_rate": 2.396882527576477e-09,
-      "logits/chosen": -2.5986270904541016,
-      "logits/rejected": -2.492842197418213,
-      "logps/chosen": -269.3092346191406,
-      "logps/rejected": -214.992919921875,
-      "loss": 0.1462,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 4.158980846405029,
-      "rewards/margins": 8.108312606811523,
-      "rewards/rejected": -3.9493324756622314,
+      "grad_norm": 565.1329564450134,
+      "learning_rate": 3.994804212627461e-09,
+      "logits/chosen": -2.626528739929199,
+      "logits/rejected": -2.5960590839385986,
+      "logps/chosen": -277.90789794921875,
+      "logps/rejected": -232.6277313232422,
+      "loss": 0.4164,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 1.226670503616333,
+      "rewards/margins": 8.475992202758789,
+      "rewards/rejected": -7.249321937561035,
       "step": 370
     },
     {
       "epoch": 0.97,
-      "grad_norm": 174.48647090969504,
-      "learning_rate": 6.004223217757509e-10,
-      "logits/chosen": -2.653160572052002,
-      "logits/rejected": -2.5599982738494873,
-      "logps/chosen": -260.55279541015625,
-      "logps/rejected": -219.11105346679688,
-      "loss": 0.1559,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 4.291203498840332,
-      "rewards/margins": 8.267863273620605,
-      "rewards/rejected": -3.9766602516174316,
+      "grad_norm": 494.78405010061283,
+      "learning_rate": 1.0007038696262516e-09,
+      "logits/chosen": -2.672760486602783,
+      "logits/rejected": -2.6364920139312744,
+      "logps/chosen": -266.59918212890625,
+      "logps/rejected": -233.4165496826172,
+      "loss": 0.4064,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 1.6075836420059204,
+      "rewards/margins": 8.459333419799805,
+      "rewards/rejected": -6.851749420166016,
       "step": 380
     },
     {
       "epoch": 1.0,
-      "grad_norm": 384.8395002541089,
+      "grad_norm": 917.0969156966863,
       "learning_rate": 0.0,
-      "logits/chosen": -2.6584863662719727,
-      "logits/rejected": -2.5489349365234375,
-      "logps/chosen": -248.6810302734375,
-      "logps/rejected": -207.1171875,
-      "loss": 0.1633,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 3.9972071647644043,
-      "rewards/margins": 9.053262710571289,
-      "rewards/rejected": -5.056054592132568,
+      "logits/chosen": -2.6849241256713867,
+      "logits/rejected": -2.630876302719116,
+      "logps/chosen": -254.27664184570312,
+      "logps/rejected": -213.14102172851562,
+      "loss": 0.4983,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 0.6460372805595398,
+      "rewards/margins": 8.144643783569336,
+      "rewards/rejected": -7.498606204986572,
       "step": 390
     },
     {
       "epoch": 1.0,
       "step": 390,
       "total_flos": 0.0,
-      "train_loss": 0.20326648155848184,
-      "train_runtime": 5896.1189,
-      "train_samples_per_second": 8.48,
-      "train_steps_per_second": 0.066
+      "train_loss": 0.4513903107398596,
+      "train_runtime": 5854.463,
+      "train_samples_per_second": 8.54,
+      "train_steps_per_second": 0.067
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6819c1dd4414cf3e1008e039028d4a09c333504f352311e2138049a4cec39cd4
-size 6520
+oid sha256:b7346ad29a6b9b0903d845abfe58994ac8a80348e425e0e321ee83abcfcb035e
+size 6264