Model save

README.md

@@ -13,7 +13,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model was trained from scratch on an unknown dataset.
+This model was trained from scratch on the None dataset.
 
 ## Model description
 
@@ -32,7 +32,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 2e-07
+- learning_rate: 5e-07
 - train_batch_size: 4
 - eval_batch_size: 4
 - seed: 42
@@ -54,5 +54,5 @@ The following hyperparameters were used during training:
 
 - Transformers 4.39.3
 - Pytorch 2.1.2+cu118
-- Datasets 2.19.1
+- Datasets 2.16.1
 - Tokenizers 0.15.2

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.19850983711389394,
-    "train_runtime": 5891.9709,
+    "train_loss": 0.4220164916454217,
+    "train_runtime": 5868.9984,
     "train_samples": 50000,
-    "train_samples_per_second": 8.486,
+    "train_samples_per_second": 8.519,
     "train_steps_per_second": 0.066
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:54769bdbecf4786a63d2238e3ee62adeef4bf0b3137da2a6064f572ac502b5ba
+oid sha256:9659b2fe2863657b54319221d7173546de29af41278d41231f8673a897bd46b8
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:afed13f2fb9196ba7a439af3fcd67e27fa09aae99b793638e19395a092d9769c
+oid sha256:905bca6862e8610633d5462ea02e6f961ea69129ab2b9bca33dfe4903ff2404d
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6c7933f77356443d56777d8b1904dbda1adb1d3912d8ea0bbc0ed445713ff355
+oid sha256:d08b6e10c0f0d8381acac257253dd03a2544f83d61d9e17d65d887493bf482fa
 size 4540516344

runs/Jun22_05-26-45_n136-129-074/events.out.tfevents.1719005226.n136-129-074.366828.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6dc853f4da6780b28efae8568a6d4f95e2f7e7bbe28ca8d16b5307bd9314219a
-size 26099
+oid sha256:ee3fc26f8cbe7fb814b0bb7c25a3b94747f5f46956e1fee451a9c8cf05ea09d5
+size 32645

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.19850983711389394,
-    "train_runtime": 5891.9709,
+    "train_loss": 0.4220164916454217,
+    "train_runtime": 5868.9984,
     "train_samples": 50000,
-    "train_samples_per_second": 8.486,
+    "train_samples_per_second": 8.519,
     "train_steps_per_second": 0.066
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_metric": null,
   "best_model_checkpoint": null,
   "epoch": 0.9980806142034548,
-  "eval_steps": 1000000,
+  "eval_steps": 10000000,
   "global_step": 390,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
@@ -10,12 +10,12 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 747.1989420753074,
-      "learning_rate": 5.128205128205128e-09,
-      "logits/chosen": -2.5617921352386475,
-      "logits/rejected": -2.415619373321533,
+      "grad_norm": 1135.2510024076782,
+      "learning_rate": 1.282051282051282e-08,
+      "logits/chosen": -2.5583817958831787,
+      "logits/rejected": -2.4487552642822266,
       "logps/chosen": -258.1644592285156,
-      "logps/rejected": -191.65736389160156,
+      "logps/rejected": -216.25729370117188,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,596 +25,596 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 772.6555707154646,
-      "learning_rate": 5.128205128205127e-08,
-      "logits/chosen": -2.610959053039551,
-      "logits/rejected": -2.52398681640625,
-      "logps/chosen": -267.3009948730469,
-      "logps/rejected": -198.18966674804688,
-      "loss": 0.7062,
+      "grad_norm": 1064.195577422658,
+      "learning_rate": 1.2820512820512818e-07,
+      "logits/chosen": -2.606004476547241,
+      "logits/rejected": -2.553109884262085,
+      "logps/chosen": -267.5234680175781,
+      "logps/rejected": -217.6415557861328,
+      "loss": 0.7054,
       "rewards/accuracies": 0.4444444477558136,
-      "rewards/chosen": -0.006153374910354614,
-      "rewards/margins": -0.0059446971863508224,
-      "rewards/rejected": -0.0002086775202769786,
+      "rewards/chosen": 0.03280753642320633,
+      "rewards/margins": 0.0353083573281765,
+      "rewards/rejected": -0.002500815549865365,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 568.2747762208306,
-      "learning_rate": 1.0256410256410255e-07,
-      "logits/chosen": -2.626389980316162,
-      "logits/rejected": -2.5251834392547607,
-      "logps/chosen": -261.1226501464844,
-      "logps/rejected": -198.20663452148438,
-      "loss": 0.6286,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": 0.1013670414686203,
-      "rewards/margins": 0.16386722028255463,
-      "rewards/rejected": -0.06250017881393433,
+      "grad_norm": 736.2634036624544,
+      "learning_rate": 2.5641025641025636e-07,
+      "logits/chosen": -2.630505323410034,
+      "logits/rejected": -2.5676522254943848,
+      "logps/chosen": -260.584716796875,
+      "logps/rejected": -207.07144165039062,
+      "loss": 0.5213,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.5755742788314819,
+      "rewards/margins": 0.5894275903701782,
+      "rewards/rejected": -0.013853324577212334,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "grad_norm": 494.0619878155869,
-      "learning_rate": 1.5384615384615385e-07,
-      "logits/chosen": -2.631648302078247,
-      "logits/rejected": -2.533083915710449,
-      "logps/chosen": -253.81259155273438,
-      "logps/rejected": -194.48611450195312,
-      "loss": 0.4093,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.6733725666999817,
-      "rewards/margins": 0.9389022588729858,
-      "rewards/rejected": -0.2655297815799713,
+      "grad_norm": 1076.3695793406284,
+      "learning_rate": 3.8461538461538463e-07,
+      "logits/chosen": -2.6462178230285645,
+      "logits/rejected": -2.571561336517334,
+      "logps/chosen": -250.9139862060547,
+      "logps/rejected": -198.4534912109375,
+      "loss": 0.3324,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 3.3866074085235596,
+      "rewards/margins": 3.0545947551727295,
+      "rewards/rejected": 0.3320125639438629,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 479.5871736251968,
-      "learning_rate": 1.999959945379852e-07,
-      "logits/chosen": -2.6485562324523926,
-      "logits/rejected": -2.5283126831054688,
-      "logps/chosen": -247.01113891601562,
-      "logps/rejected": -198.84341430664062,
-      "loss": 0.2427,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 1.6728582382202148,
-      "rewards/margins": 2.513617753982544,
-      "rewards/rejected": -0.84075927734375,
+      "grad_norm": 418.3228099023361,
+      "learning_rate": 4.99989986344963e-07,
+      "logits/chosen": -2.6392903327941895,
+      "logits/rejected": -2.5602712631225586,
+      "logps/chosen": -243.54013061523438,
+      "logps/rejected": -192.9114227294922,
+      "loss": 0.3161,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 5.447351455688477,
+      "rewards/margins": 4.827452182769775,
+      "rewards/rejected": 0.6198989748954773,
       "step": 40
     },
     {
       "epoch": 0.13,
-      "grad_norm": 268.0648126747043,
-      "learning_rate": 1.9951572723309917e-07,
-      "logits/chosen": -2.701251983642578,
-      "logits/rejected": -2.596193790435791,
-      "logps/chosen": -260.8350830078125,
-      "logps/rejected": -204.7702178955078,
-      "loss": 0.2435,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 3.198915481567383,
-      "rewards/margins": 4.101343631744385,
-      "rewards/rejected": -0.902428150177002,
+      "grad_norm": 630.2703390024756,
+      "learning_rate": 4.987893180827479e-07,
+      "logits/chosen": -2.651214361190796,
+      "logits/rejected": -2.57964825630188,
+      "logps/chosen": -258.42962646484375,
+      "logps/rejected": -203.57992553710938,
+      "loss": 0.366,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 7.846033573150635,
+      "rewards/margins": 6.590806007385254,
+      "rewards/rejected": 1.255226731300354,
       "step": 50
     },
     {
       "epoch": 0.15,
-      "grad_norm": 321.17307426963725,
-      "learning_rate": 1.9823877374156647e-07,
-      "logits/chosen": -2.6826422214508057,
-      "logits/rejected": -2.5699238777160645,
-      "logps/chosen": -260.90740966796875,
-      "logps/rejected": -190.84542846679688,
-      "loss": 0.2062,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 3.89800763130188,
-      "rewards/margins": 4.676665306091309,
-      "rewards/rejected": -0.7786582112312317,
+      "grad_norm": 655.8352889546771,
+      "learning_rate": 4.955969343539162e-07,
+      "logits/chosen": -2.60957932472229,
+      "logits/rejected": -2.5362067222595215,
+      "logps/chosen": -262.3640441894531,
+      "logps/rejected": -209.32199096679688,
+      "loss": 0.3453,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 5.170942306518555,
+      "rewards/margins": 6.18172550201416,
+      "rewards/rejected": -1.0107834339141846,
       "step": 60
     },
     {
       "epoch": 0.18,
-      "grad_norm": 341.66511985659804,
-      "learning_rate": 1.9617535688178958e-07,
-      "logits/chosen": -2.664588451385498,
-      "logits/rejected": -2.5252487659454346,
-      "logps/chosen": -289.1197204589844,
-      "logps/rejected": -219.9818572998047,
-      "loss": 0.1912,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 4.399308204650879,
-      "rewards/margins": 5.727428436279297,
-      "rewards/rejected": -1.3281205892562866,
+      "grad_norm": 456.9589116841801,
+      "learning_rate": 4.90438392204474e-07,
+      "logits/chosen": -2.5825228691101074,
+      "logits/rejected": -2.5089833736419678,
+      "logps/chosen": -291.7918395996094,
+      "logps/rejected": -227.83432006835938,
+      "loss": 0.3454,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 4.895013809204102,
+      "rewards/margins": 7.00995397567749,
+      "rewards/rejected": -2.1149401664733887,
       "step": 70
     },
     {
       "epoch": 0.2,
-      "grad_norm": 345.50558992425647,
-      "learning_rate": 1.9334199560765839e-07,
-      "logits/chosen": -2.612795352935791,
-      "logits/rejected": -2.4848904609680176,
-      "logps/chosen": -260.74462890625,
-      "logps/rejected": -209.5861053466797,
-      "loss": 0.1745,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 3.798893690109253,
-      "rewards/margins": 5.829946994781494,
-      "rewards/rejected": -2.031052827835083,
+      "grad_norm": 816.8720109326792,
+      "learning_rate": 4.83354989019146e-07,
+      "logits/chosen": -2.5420753955841064,
+      "logits/rejected": -2.467258930206299,
+      "logps/chosen": -259.6270446777344,
+      "logps/rejected": -204.15179443359375,
+      "loss": 0.3311,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 6.344871997833252,
+      "rewards/margins": 7.2052764892578125,
+      "rewards/rejected": -0.860403835773468,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 623.1485221220804,
-      "learning_rate": 1.897613727639014e-07,
-      "logits/chosen": -2.6385045051574707,
-      "logits/rejected": -2.5179295539855957,
-      "logps/chosen": -254.0596160888672,
-      "logps/rejected": -191.18374633789062,
-      "loss": 0.1808,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 3.131239414215088,
-      "rewards/margins": 5.626199722290039,
-      "rewards/rejected": -2.494959831237793,
+      "grad_norm": 922.6738539012168,
+      "learning_rate": 4.7440343190975353e-07,
+      "logits/chosen": -2.5713560581207275,
+      "logits/rejected": -2.513441801071167,
+      "logps/chosen": -257.0751037597656,
+      "logps/rejected": -217.1184844970703,
+      "loss": 0.3343,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 3.73614239692688,
+      "rewards/margins": 5.834546089172363,
+      "rewards/rejected": -2.0984034538269043,
       "step": 90
     },
     {
       "epoch": 0.26,
-      "grad_norm": 253.52286816658938,
-      "learning_rate": 1.8546215349560202e-07,
-      "logits/chosen": -2.6612112522125244,
-      "logits/rejected": -2.533451795578003,
-      "logps/chosen": -235.6942138671875,
-      "logps/rejected": -196.51657104492188,
-      "loss": 0.1619,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 3.5456454753875732,
-      "rewards/margins": 5.960389614105225,
-      "rewards/rejected": -2.4147439002990723,
+      "grad_norm": 406.82707972381877,
+      "learning_rate": 4.6365538373900506e-07,
+      "logits/chosen": -2.6249356269836426,
+      "logits/rejected": -2.5500850677490234,
+      "logps/chosen": -236.4239501953125,
+      "logps/rejected": -200.73150634765625,
+      "loss": 0.5974,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 4.595959663391113,
+      "rewards/margins": 6.244544506072998,
+      "rewards/rejected": -1.648585557937622,
       "step": 100
     },
     {
       "epoch": 0.28,
-      "grad_norm": 266.28422663795834,
-      "learning_rate": 1.8047875576562553e-07,
-      "logits/chosen": -2.6504344940185547,
-      "logits/rejected": -2.52449369430542,
-      "logps/chosen": -254.90847778320312,
-      "logps/rejected": -191.41806030273438,
-      "loss": 0.1662,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 4.884920597076416,
-      "rewards/margins": 6.528225898742676,
-      "rewards/rejected": -1.6433048248291016,
+      "grad_norm": 656.3071663391811,
+      "learning_rate": 4.5119688941406386e-07,
+      "logits/chosen": -2.618974208831787,
+      "logits/rejected": -2.5380780696868896,
+      "logps/chosen": -257.79248046875,
+      "logps/rejected": -209.8715362548828,
+      "loss": 0.4404,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 6.305555820465088,
+      "rewards/margins": 7.463587760925293,
+      "rewards/rejected": -1.158031940460205,
       "step": 110
     },
     {
       "epoch": 0.31,
-      "grad_norm": 248.5491633368023,
-      "learning_rate": 1.748510748171101e-07,
-      "logits/chosen": -2.626223564147949,
-      "logits/rejected": -2.531078815460205,
-      "logps/chosen": -258.3885803222656,
-      "logps/rejected": -210.59378051757812,
-      "loss": 0.1688,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 4.675450801849365,
-      "rewards/margins": 6.811753273010254,
-      "rewards/rejected": -2.1363017559051514,
+      "grad_norm": 810.7648282749318,
+      "learning_rate": 4.3712768704277524e-07,
+      "logits/chosen": -2.5895957946777344,
+      "logits/rejected": -2.519530773162842,
+      "logps/chosen": -262.7950134277344,
+      "logps/rejected": -208.9604949951172,
+      "loss": 0.438,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 5.333884239196777,
+      "rewards/margins": 7.409175872802734,
+      "rewards/rejected": -2.075291156768799,
       "step": 120
     },
     {
       "epoch": 0.33,
-      "grad_norm": 219.45295559501494,
-      "learning_rate": 1.6862416378687336e-07,
-      "logits/chosen": -2.6003150939941406,
-      "logits/rejected": -2.465550661087036,
-      "logps/chosen": -249.7939910888672,
-      "logps/rejected": -186.2041778564453,
-      "loss": 0.1744,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 3.695958375930786,
-      "rewards/margins": 6.534309387207031,
-      "rewards/rejected": -2.838351011276245,
+      "grad_norm": 513.4959841183485,
+      "learning_rate": 4.2156040946718343e-07,
+      "logits/chosen": -2.5553436279296875,
+      "logits/rejected": -2.487457752227783,
+      "logps/chosen": -251.7507781982422,
+      "logps/rejected": -197.44088745117188,
+      "loss": 0.4027,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 4.441976070404053,
+      "rewards/margins": 7.408116340637207,
+      "rewards/rejected": -2.966140031814575,
       "step": 130
     },
     {
       "epoch": 0.36,
-      "grad_norm": 335.11070283406536,
-      "learning_rate": 1.6184787302662547e-07,
-      "logits/chosen": -2.6305205821990967,
-      "logits/rejected": -2.519258975982666,
-      "logps/chosen": -269.74249267578125,
-      "logps/rejected": -205.24441528320312,
-      "loss": 0.1847,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 3.7317748069763184,
-      "rewards/margins": 6.535207271575928,
-      "rewards/rejected": -2.8034329414367676,
+      "grad_norm": 650.5511601275197,
+      "learning_rate": 4.046196825665637e-07,
+      "logits/chosen": -2.5706536769866943,
+      "logits/rejected": -2.500262498855591,
+      "logps/chosen": -270.2043762207031,
+      "logps/rejected": -217.0515594482422,
+      "loss": 0.4293,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 3.977551221847534,
+      "rewards/margins": 6.7731499671936035,
+      "rewards/rejected": -2.7955987453460693,
       "step": 140
     },
     {
       "epoch": 0.38,
-      "grad_norm": 237.3062963631657,
-      "learning_rate": 1.5457645101945046e-07,
-      "logits/chosen": -2.613662004470825,
-      "logits/rejected": -2.50919771194458,
-      "logps/chosen": -262.550537109375,
-      "logps/rejected": -209.60781860351562,
-      "loss": 0.1987,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 4.1123762130737305,
-      "rewards/margins": 7.0581793785095215,
-      "rewards/rejected": -2.945803165435791,
+      "grad_norm": 530.5799871161138,
+      "learning_rate": 3.864411275486261e-07,
+      "logits/chosen": -2.5574281215667725,
+      "logits/rejected": -2.488007068634033,
+      "logps/chosen": -263.3489685058594,
+      "logps/rejected": -212.54638671875,
+      "loss": 0.4583,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 5.79421329498291,
+      "rewards/margins": 7.515044212341309,
+      "rewards/rejected": -1.720831274986267,
       "step": 150
     },
     {
       "epoch": 0.41,
-      "grad_norm": 781.538478360743,
-      "learning_rate": 1.4686811008647035e-07,
-      "logits/chosen": -2.6011874675750732,
-      "logits/rejected": -2.4891130924224854,
-      "logps/chosen": -241.5615692138672,
-      "logps/rejected": -182.00228881835938,
-      "loss": 0.1643,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 4.244563102722168,
-      "rewards/margins": 6.548022270202637,
-      "rewards/rejected": -2.3034589290618896,
+      "grad_norm": 600.6086946072276,
+      "learning_rate": 3.671702752161759e-07,
+      "logits/chosen": -2.563870906829834,
+      "logits/rejected": -2.493649482727051,
+      "logps/chosen": -244.5281219482422,
+      "logps/rejected": -198.3011474609375,
+      "loss": 0.4465,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 3.527863025665283,
+      "rewards/margins": 7.751578330993652,
+      "rewards/rejected": -4.223715782165527,
       "step": 160
     },
     {
       "epoch": 0.44,
-      "grad_norm": 420.43337815116814,
-      "learning_rate": 1.387845603604855e-07,
-      "logits/chosen": -2.6014420986175537,
-      "logits/rejected": -2.4862489700317383,
-      "logps/chosen": -260.2299499511719,
-      "logps/rejected": -197.31021118164062,
-      "loss": 0.1451,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 4.481954574584961,
-      "rewards/margins": 6.5617804527282715,
-      "rewards/rejected": -2.0798258781433105,
+      "grad_norm": 753.6856997505446,
+      "learning_rate": 3.4696140090121375e-07,
+      "logits/chosen": -2.5673775672912598,
+      "logits/rejected": -2.500842571258545,
+      "logps/chosen": -265.5797119140625,
+      "logps/rejected": -211.0306854248047,
+      "loss": 0.3547,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 4.089644908905029,
+      "rewards/margins": 7.812180519104004,
+      "rewards/rejected": -3.7225348949432373,
       "step": 170
     },
     {
       "epoch": 0.46,
-      "grad_norm": 219.52091746145038,
-      "learning_rate": 1.3039051575742468e-07,
-      "logits/chosen": -2.6457467079162598,
-      "logits/rejected": -2.5342042446136475,
-      "logps/chosen": -234.87063598632812,
-      "logps/rejected": -187.8373565673828,
-      "loss": 0.1641,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 3.4876370429992676,
-      "rewards/margins": 5.9574689865112305,
-      "rewards/rejected": -2.469831943511963,
+      "grad_norm": 645.3967547220625,
+      "learning_rate": 3.259762893935617e-07,
+      "logits/chosen": -2.6238903999328613,
+      "logits/rejected": -2.534097194671631,
+      "logps/chosen": -236.9849395751953,
+      "logps/rejected": -186.6522674560547,
+      "loss": 0.4499,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 4.038764953613281,
+      "rewards/margins": 6.760235786437988,
+      "rewards/rejected": -2.721470594406128,
       "step": 180
     },
     {
       "epoch": 0.49,
-      "grad_norm": 391.30420402591136,
-      "learning_rate": 1.2175317590061675e-07,
-      "logits/chosen": -2.598358154296875,
-      "logits/rejected": -2.5016226768493652,
-      "logps/chosen": -256.7655334472656,
-      "logps/rejected": -203.38717651367188,
-      "loss": 0.1704,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 3.634434223175049,
-      "rewards/margins": 6.311253547668457,
-      "rewards/rejected": -2.6768198013305664,
+      "grad_norm": 707.705744532387,
+      "learning_rate": 3.0438293975154184e-07,
+      "logits/chosen": -2.582486867904663,
+      "logits/rejected": -2.5034093856811523,
+      "logps/chosen": -261.0556945800781,
+      "logps/rejected": -205.6962890625,
+      "loss": 0.3591,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 3.011924982070923,
+      "rewards/margins": 8.104998588562012,
+      "rewards/rejected": -5.093073844909668,
       "step": 190
     },
     {
       "epoch": 0.51,
-      "grad_norm": 573.9951634797832,
-      "learning_rate": 1.1294168814540553e-07,
-      "logits/chosen": -2.640943765640259,
-      "logits/rejected": -2.5037968158721924,
-      "logps/chosen": -270.6221923828125,
-      "logps/rejected": -199.61119079589844,
-      "loss": 0.1514,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 4.9661078453063965,
-      "rewards/margins": 7.715418338775635,
-      "rewards/rejected": -2.749310255050659,
+      "grad_norm": 822.8629977119059,
+      "learning_rate": 2.823542203635138e-07,
+      "logits/chosen": -2.615396499633789,
+      "logits/rejected": -2.5223731994628906,
+      "logps/chosen": -277.3884582519531,
+      "logps/rejected": -221.803466796875,
+      "loss": 0.4468,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.5077309608459473,
+      "rewards/margins": 9.055838584899902,
+      "rewards/rejected": -6.548108100891113,
       "step": 200
     },
     {
       "epoch": 0.54,
-      "grad_norm": 383.73670531019474,
-      "learning_rate": 1.0402659401094151e-07,
-      "logits/chosen": -2.6191225051879883,
-      "logits/rejected": -2.5026001930236816,
-      "logps/chosen": -262.648681640625,
-      "logps/rejected": -207.82363891601562,
-      "loss": 0.1405,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 4.20405912399292,
-      "rewards/margins": 7.261610507965088,
-      "rewards/rejected": -3.057551383972168,
+      "grad_norm": 954.7034527431528,
+      "learning_rate": 2.600664850273538e-07,
+      "logits/chosen": -2.603569269180298,
+      "logits/rejected": -2.5283331871032715,
+      "logps/chosen": -269.19873046875,
+      "logps/rejected": -213.823974609375,
+      "loss": 0.6013,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 1.4840681552886963,
+      "rewards/margins": 7.454611778259277,
+      "rewards/rejected": -5.97054386138916,
       "step": 210
     },
     {
       "epoch": 0.56,
-      "grad_norm": 271.1779091479927,
-      "learning_rate": 9.507926445081218e-08,
-      "logits/chosen": -2.618332862854004,
-      "logits/rejected": -2.5180063247680664,
-      "logps/chosen": -255.3096160888672,
-      "logps/rejected": -204.3534698486328,
-      "loss": 0.1701,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 3.6050515174865723,
-      "rewards/margins": 7.0129852294921875,
-      "rewards/rejected": -3.4079346656799316,
+      "grad_norm": 685.0955562473252,
+      "learning_rate": 2.3769816112703045e-07,
+      "logits/chosen": -2.6224589347839355,
+      "logits/rejected": -2.55679988861084,
+      "logps/chosen": -257.71661376953125,
+      "logps/rejected": -214.28329467773438,
+      "loss": 0.4806,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 3.168187141418457,
+      "rewards/margins": 6.781345367431641,
+      "rewards/rejected": -3.6131577491760254,
       "step": 220
     },
     {
       "epoch": 0.59,
-      "grad_norm": 290.4603808348808,
-      "learning_rate": 8.61713284835267e-08,
-      "logits/chosen": -2.646491050720215,
-      "logits/rejected": -2.51234769821167,
-      "logps/chosen": -264.74420166015625,
-      "logps/rejected": -205.0327911376953,
-      "loss": 0.1425,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 4.327627182006836,
-      "rewards/margins": 7.188753604888916,
-      "rewards/rejected": -2.8611254692077637,
+      "grad_norm": 568.8894162951807,
+      "learning_rate": 2.1542832120881677e-07,
+      "logits/chosen": -2.664320945739746,
+      "logits/rejected": -2.5764544010162354,
+      "logps/chosen": -266.98114013671875,
+      "logps/rejected": -216.44894409179688,
+      "loss": 0.4149,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 5.137583255767822,
+      "rewards/margins": 7.965329647064209,
+      "rewards/rejected": -2.827746629714966,
       "step": 230
     },
     {
       "epoch": 0.61,
-      "grad_norm": 422.65919601372,
-      "learning_rate": 7.73740997570278e-08,
-      "logits/chosen": -2.6252548694610596,
-      "logits/rejected": -2.5125999450683594,
-      "logps/chosen": -258.0855712890625,
-      "logps/rejected": -207.72213745117188,
-      "loss": 0.1341,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 4.273736953735352,
-      "rewards/margins": 7.429440498352051,
-      "rewards/rejected": -3.155702829360962,
+      "grad_norm": 923.147651672606,
+      "learning_rate": 1.934352493925695e-07,
+      "logits/chosen": -2.6468780040740967,
+      "logits/rejected": -2.5980067253112793,
+      "logps/chosen": -262.94610595703125,
+      "logps/rejected": -220.69448852539062,
+      "loss": 0.3991,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 3.828115463256836,
+      "rewards/margins": 9.526643753051758,
+      "rewards/rejected": -5.69852876663208,
       "step": 240
     },
     {
       "epoch": 0.64,
-      "grad_norm": 236.70780195109265,
-      "learning_rate": 6.875800563794424e-08,
-      "logits/chosen": -2.637636184692383,
-      "logits/rejected": -2.522014856338501,
-      "logps/chosen": -264.11163330078125,
-      "logps/rejected": -213.34262084960938,
-      "loss": 0.1561,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 3.965393543243408,
-      "rewards/margins": 7.122384548187256,
-      "rewards/rejected": -3.1569907665252686,
+      "grad_norm": 615.4120078013015,
+      "learning_rate": 1.7189501409486059e-07,
+      "logits/chosen": -2.656362533569336,
+      "logits/rejected": -2.584864616394043,
+      "logps/chosen": -267.7325439453125,
+      "logps/rejected": -222.2632293701172,
+      "loss": 0.4004,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 3.2369320392608643,
+      "rewards/margins": 7.942319393157959,
+      "rewards/rejected": -4.705387115478516,
       "step": 250
     },
     {
       "epoch": 0.67,
-      "grad_norm": 261.4238126511671,
-      "learning_rate": 6.039202339608431e-08,
-      "logits/chosen": -2.6351547241210938,
-      "logits/rejected": -2.5343315601348877,
-      "logps/chosen": -257.4027099609375,
-      "logps/rejected": -206.6591033935547,
-      "loss": 0.1287,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 4.236881732940674,
-      "rewards/margins": 7.065675258636475,
-      "rewards/rejected": -2.828793525695801,
+      "grad_norm": 488.0068782741624,
+      "learning_rate": 1.5098005849021078e-07,
+      "logits/chosen": -2.64605450630188,
+      "logits/rejected": -2.586585283279419,
+      "logps/chosen": -261.89093017578125,
+      "logps/rejected": -208.77493286132812,
+      "loss": 0.3817,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 3.175231456756592,
+      "rewards/margins": 7.630448818206787,
+      "rewards/rejected": -4.455216884613037,
       "step": 260
     },
     {
       "epoch": 0.69,
-      "grad_norm": 290.68123700235606,
-      "learning_rate": 5.2343127997869205e-08,
-      "logits/chosen": -2.610731840133667,
-      "logits/rejected": -2.4807095527648926,
-      "logps/chosen": -270.530517578125,
-      "logps/rejected": -215.2696533203125,
-      "loss": 0.2282,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 3.1549553871154785,
-      "rewards/margins": 7.109915733337402,
-      "rewards/rejected": -3.9549612998962402,
+      "grad_norm": 559.4430135222711,
+      "learning_rate": 1.30857819994673e-07,
+      "logits/chosen": -2.6208698749542236,
+      "logits/rejected": -2.5371921062469482,
+      "logps/chosen": -274.78753662109375,
+      "logps/rejected": -230.4307861328125,
+      "loss": 0.5355,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 1.437089204788208,
+      "rewards/margins": 9.265036582946777,
+      "rewards/rejected": -7.82794713973999,
       "step": 270
     },
     {
       "epoch": 0.72,
-      "grad_norm": 212.4315878069041,
-      "learning_rate": 4.4675755929468636e-08,
-      "logits/chosen": -2.6439692974090576,
-      "logits/rejected": -2.521467447280884,
-      "logps/chosen": -266.0245056152344,
-      "logps/rejected": -204.48606872558594,
-      "loss": 0.139,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 3.838078737258911,
-      "rewards/margins": 7.6062164306640625,
-      "rewards/rejected": -3.7681381702423096,
+      "grad_norm": 432.8210354095987,
+      "learning_rate": 1.116893898236716e-07,
+      "logits/chosen": -2.654949426651001,
+      "logits/rejected": -2.5985524654388428,
+      "logps/chosen": -270.3836975097656,
+      "logps/rejected": -219.8002471923828,
+      "loss": 0.3718,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 2.6581013202667236,
+      "rewards/margins": 8.142509460449219,
+      "rewards/rejected": -5.484408378601074,
       "step": 280
     },
     {
       "epoch": 0.74,
-      "grad_norm": 229.50979085481035,
-      "learning_rate": 3.745128934207224e-08,
-      "logits/chosen": -2.598167657852173,
-      "logits/rejected": -2.5218276977539062,
-      "logps/chosen": -266.5920104980469,
-      "logps/rejected": -222.931884765625,
-      "loss": 0.1507,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 3.6127426624298096,
-      "rewards/margins": 6.654023170471191,
-      "rewards/rejected": -3.0412800312042236,
+      "grad_norm": 482.2442984028295,
+      "learning_rate": 9.362822335518062e-08,
+      "logits/chosen": -2.6166903972625732,
+      "logits/rejected": -2.5696167945861816,
+      "logps/chosen": -268.19140625,
+      "logps/rejected": -216.9479522705078,
+      "loss": 0.3568,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 3.0037200450897217,
+      "rewards/margins": 7.667593479156494,
+      "rewards/rejected": -4.663873195648193,
       "step": 290
     },
     {
       "epoch": 0.77,
-      "grad_norm": 321.80727834811256,
-      "learning_rate": 3.0727564649040063e-08,
-      "logits/chosen": -2.6253504753112793,
-      "logits/rejected": -2.5061511993408203,
-      "logps/chosen": -270.4463806152344,
-      "logps/rejected": -213.0215301513672,
-      "loss": 0.141,
-      "rewards/accuracies": 0.956250011920929,
-      "rewards/chosen": 3.923215389251709,
-      "rewards/margins": 7.3881683349609375,
-      "rewards/rejected": -3.464953660964966,
+      "grad_norm": 492.9163861530474,
+      "learning_rate": 7.681891162260015e-08,
+      "logits/chosen": -2.636460781097412,
+      "logits/rejected": -2.580770254135132,
+      "logps/chosen": -274.6198425292969,
+      "logps/rejected": -220.8531951904297,
+      "loss": 0.3983,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 3.163914680480957,
+      "rewards/margins": 7.829231262207031,
+      "rewards/rejected": -4.665315628051758,
       "step": 300
     },
     {
       "epoch": 0.79,
-      "grad_norm": 171.80295572629132,
-      "learning_rate": 2.4558409508920985e-08,
-      "logits/chosen": -2.5865883827209473,
-      "logits/rejected": -2.491208553314209,
-      "logps/chosen": -275.579345703125,
-      "logps/rejected": -208.02297973632812,
-      "loss": 0.1401,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 3.985370635986328,
-      "rewards/margins": 7.237724304199219,
-      "rewards/rejected": -3.2523536682128906,
+      "grad_norm": 437.9917779014462,
+      "learning_rate": 6.139602377230247e-08,
+      "logits/chosen": -2.6010611057281494,
+      "logits/rejected": -2.532543897628784,
+      "logps/chosen": -278.3953552246094,
+      "logps/rejected": -215.9014129638672,
+      "loss": 0.4376,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 3.1028757095336914,
+      "rewards/margins": 8.152434349060059,
+      "rewards/rejected": -5.049559593200684,
       "step": 310
     },
     {
       "epoch": 0.82,
-      "grad_norm": 294.8015156484179,
-      "learning_rate": 1.899321190108335e-08,
-      "logits/chosen": -2.6083686351776123,
-      "logits/rejected": -2.508666515350342,
-      "logps/chosen": -258.6110534667969,
-      "logps/rejected": -207.57406616210938,
-      "loss": 0.1709,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 3.153282642364502,
-      "rewards/margins": 7.0364227294921875,
-      "rewards/rejected": -3.8831400871276855,
+      "grad_norm": 649.8222699481745,
+      "learning_rate": 4.748302975270837e-08,
+      "logits/chosen": -2.6264309883117676,
+      "logits/rejected": -2.5793588161468506,
+      "logps/chosen": -261.37890625,
+      "logps/rejected": -204.51773071289062,
+      "loss": 0.405,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 2.8262996673583984,
+      "rewards/margins": 7.375731468200684,
+      "rewards/rejected": -4.549432277679443,
       "step": 320
     },
     {
       "epoch": 0.84,
-      "grad_norm": 262.1222746234904,
-      "learning_rate": 1.4076524743778317e-08,
-      "logits/chosen": -2.6292643547058105,
-      "logits/rejected": -2.531942129135132,
-      "logps/chosen": -261.5999450683594,
-      "logps/rejected": -207.26699829101562,
-      "loss": 0.1437,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": 4.273932456970215,
-      "rewards/margins": 7.467469215393066,
-      "rewards/rejected": -3.1935369968414307,
+      "grad_norm": 583.9617574483902,
+      "learning_rate": 3.5191311859445795e-08,
+      "logits/chosen": -2.6449975967407227,
+      "logits/rejected": -2.586719512939453,
+      "logps/chosen": -264.58428955078125,
+      "logps/rejected": -217.4517364501953,
+      "loss": 0.3924,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 4.091521263122559,
+      "rewards/margins": 7.869417667388916,
+      "rewards/rejected": -3.7778968811035156,
       "step": 330
     },
     {
       "epoch": 0.87,
-      "grad_norm": 299.7750791753028,
-      "learning_rate": 9.847709219918398e-09,
-      "logits/chosen": -2.6148436069488525,
-      "logits/rejected": -2.5132031440734863,
-      "logps/chosen": -256.66473388671875,
-      "logps/rejected": -194.7910919189453,
-      "loss": 0.1894,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/chosen": 4.073337078094482,
-      "rewards/margins": 7.587210655212402,
-      "rewards/rejected": -3.51387357711792,
+      "grad_norm": 516.763098966226,
+      "learning_rate": 2.4619273049795996e-08,
+      "logits/chosen": -2.631946563720703,
+      "logits/rejected": -2.5740180015563965,
+      "logps/chosen": -260.0722961425781,
+      "logps/rejected": -210.775146484375,
+      "loss": 0.3558,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 3.791111469268799,
+      "rewards/margins": 8.514566421508789,
+      "rewards/rejected": -4.723455905914307,
       "step": 340
     },
     {
       "epoch": 0.9,
-      "grad_norm": 346.7439242044206,
-      "learning_rate": 6.340619665972846e-09,
-      "logits/chosen": -2.630897045135498,
-      "logits/rejected": -2.5146021842956543,
-      "logps/chosen": -265.88916015625,
-      "logps/rejected": -212.77487182617188,
-      "loss": 0.1582,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 3.5220444202423096,
-      "rewards/margins": 7.0233283042907715,
-      "rewards/rejected": -3.501283645629883,
+      "grad_norm": 434.316228593937,
+      "learning_rate": 1.5851549164932115e-08,
+      "logits/chosen": -2.641859531402588,
+      "logits/rejected": -2.592379093170166,
+      "logps/chosen": -269.5948181152344,
+      "logps/rejected": -226.536865234375,
+      "loss": 0.382,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 3.7248435020446777,
+      "rewards/margins": 7.7656402587890625,
+      "rewards/rejected": -4.040797233581543,
       "step": 350
     },
     {
       "epoch": 0.92,
-      "grad_norm": 307.153905111391,
-      "learning_rate": 3.583332546643769e-09,
-      "logits/chosen": -2.6361472606658936,
-      "logits/rejected": -2.5298829078674316,
-      "logps/chosen": -271.5682067871094,
-      "logps/rejected": -209.4921417236328,
-      "loss": 0.1547,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 3.9090733528137207,
-      "rewards/margins": 7.22186279296875,
-      "rewards/rejected": -3.3127894401550293,
+      "grad_norm": 570.6334718025578,
+      "learning_rate": 8.958331366609423e-09,
+      "logits/chosen": -2.6432430744171143,
+      "logits/rejected": -2.574936628341675,
+      "logps/chosen": -275.0256652832031,
+      "logps/rejected": -219.6584014892578,
+      "loss": 0.4253,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 3.3530006408691406,
+      "rewards/margins": 8.096589088439941,
+      "rewards/rejected": -4.743588447570801,
       "step": 360
     },
     {
       "epoch": 0.95,
-      "grad_norm": 183.78861249224778,
-      "learning_rate": 1.5979216850509847e-09,
-      "logits/chosen": -2.593644618988037,
-      "logits/rejected": -2.4859519004821777,
-      "logps/chosen": -270.0194396972656,
-      "logps/rejected": -212.9616241455078,
-      "loss": 0.1314,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/chosen": 3.80389142036438,
-      "rewards/margins": 6.737584114074707,
-      "rewards/rejected": -2.933692216873169,
+      "grad_norm": 877.4134874498682,
+      "learning_rate": 3.994804212627461e-09,
+      "logits/chosen": -2.6024394035339355,
+      "logits/rejected": -2.5662083625793457,
+      "logps/chosen": -273.9478454589844,
+      "logps/rejected": -229.1957550048828,
+      "loss": 0.4977,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 4.258389472961426,
+      "rewards/margins": 7.956662178039551,
+      "rewards/rejected": -3.698272705078125,
       "step": 370
     },
     {
       "epoch": 0.97,
-      "grad_norm": 201.1703750461263,
-      "learning_rate": 4.002815478505006e-10,
-      "logits/chosen": -2.6470093727111816,
-      "logits/rejected": -2.552563428878784,
-      "logps/chosen": -261.07574462890625,
-      "logps/rejected": -216.7428741455078,
-      "loss": 0.1449,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 4.029725074768066,
-      "rewards/margins": 6.822300910949707,
-      "rewards/rejected": -2.7925755977630615,
+      "grad_norm": 416.60583937652194,
+      "learning_rate": 1.0007038696262516e-09,
+      "logits/chosen": -2.651128053665161,
+      "logits/rejected": -2.610159397125244,
+      "logps/chosen": -263.07269287109375,
+      "logps/rejected": -230.61502075195312,
+      "loss": 0.3902,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 4.250136375427246,
+      "rewards/margins": 8.099352836608887,
+      "rewards/rejected": -3.8492164611816406,
       "step": 380
     },
     {
       "epoch": 1.0,
-      "grad_norm": 359.6013698550439,
+      "grad_norm": 678.8175373396961,
       "learning_rate": 0.0,
-      "logits/chosen": -2.6531007289886475,
-      "logits/rejected": -2.540839195251465,
-      "logps/chosen": -249.09298706054688,
-      "logps/rejected": -204.24765014648438,
-      "loss": 0.146,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 3.7912139892578125,
-      "rewards/margins": 7.412507057189941,
-      "rewards/rejected": -3.621293306350708,
+      "logits/chosen": -2.6594204902648926,
+      "logits/rejected": -2.5979819297790527,
+      "logps/chosen": -250.8957977294922,
+      "logps/rejected": -210.31497192382812,
+      "loss": 0.4132,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 3.278926134109497,
+      "rewards/margins": 7.683538913726807,
+      "rewards/rejected": -4.4046125411987305,
       "step": 390
     },
     {
       "epoch": 1.0,
       "step": 390,
       "total_flos": 0.0,
-      "train_loss": 0.19850983711389394,
-      "train_runtime": 5891.9709,
-      "train_samples_per_second": 8.486,
+      "train_loss": 0.4220164916454217,
+      "train_runtime": 5868.9984,
+      "train_samples_per_second": 8.519,
       "train_steps_per_second": 0.066
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d90343e793e5916c6afac01760eb7b5a30707ec90a3ec177dfdeda931df024a
-size 6328
+oid sha256:47a21b5563dec2beb9a160ae932f9047a37bb854766f66095441d6c75a856221
+size 6264