Model save

README.md

@@ -13,7 +13,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model was trained from scratch on the None dataset.
+This model was trained from scratch on an unknown dataset.
 
 ## Model description
 
@@ -52,7 +52,7 @@ The following hyperparameters were used during training:
 
 ### Framework versions
 
-- Transformers 4.41.1
+- Transformers 4.39.3
 - Pytorch 2.1.2+cu118
-- Datasets 2.16.1
-- Tokenizers 0.19.1
+- Datasets 2.19.1
+- Tokenizers 0.15.2

all_results.json

@@ -1,9 +1,8 @@
 {
-    "epoch": 0.9980806142034548,
-    "total_flos": 0.0,
-    "train_loss": 0.29645214692140237,
-    "train_runtime": 6440.0874,
+    "epoch": 1.0,
+    "train_loss": 0.20326648155848184,
+    "train_runtime": 5896.1189,
     "train_samples": 50000,
-    "train_samples_per_second": 7.764,
-    "train_steps_per_second": 0.061
+    "train_samples_per_second": 8.48,
+    "train_steps_per_second": 0.066
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "transformers_version": "4.41.1"
+  "transformers_version": "4.39.3"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec9240d64ac00cd6eb3d828f3a6b877927f4e73531e0292161f3a9fb63dfbe81
+oid sha256:58bde9bfb67010c336cace37ab13ac39da6af79040fcdbdabd0a04935b66a870
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f71eb1b478b34a5d188dd277f1c68954a3090f452264f7aee953e94fffa874f7
+oid sha256:35d424b1b9f269fcdb54ade09434feadf354ef611fe440a2d936528908734919
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cadb0e9982bf9b0f8f3048618cb4654101dc9b30be3d02283ea1fc069abbe2d3
+oid sha256:073b6342526b72183cb12d771d63bd08d1440bc35898cc6d740a2bf32a97585b
 size 4540516344

runs/Jun22_07-28-38_n136-112-146/events.out.tfevents.1719013199.n136-112-146.3352140.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a78c114b19fab1e039d3a4be75335ff74ec60afbf850c3d771699bbeb6953590
-size 26098
+oid sha256:8ab2372bceb2d6c5b46f639ae89e3d4ccce0ce757ac17c449bfd5ab2298930cb
+size 32644

train_results.json

@@ -1,9 +1,8 @@
 {
-    "epoch": 0.9980806142034548,
-    "total_flos": 0.0,
-    "train_loss": 0.29645214692140237,
-    "train_runtime": 6440.0874,
+    "epoch": 1.0,
+    "train_loss": 0.20326648155848184,
+    "train_runtime": 5896.1189,
     "train_samples": 50000,
-    "train_samples_per_second": 7.764,
-    "train_steps_per_second": 0.061
+    "train_samples_per_second": 8.48,
+    "train_steps_per_second": 0.066
 }

trainer_state.json

@@ -9,13 +9,13 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0025591810620601407,
-      "grad_norm": 709.6283481081026,
+      "epoch": 0.0,
+      "grad_norm": 747.0977926336889,
       "learning_rate": 7.692307692307691e-09,
-      "logits/chosen": -2.5583817958831787,
-      "logits/rejected": -2.4487552642822266,
+      "logits/chosen": -2.5617921352386475,
+      "logits/rejected": -2.415619373321533,
       "logps/chosen": -258.1644592285156,
-      "logps/rejected": -216.25729370117188,
+      "logps/rejected": -191.65736389160156,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -24,598 +24,598 @@
       "step": 1
     },
     {
-      "epoch": 0.025591810620601407,
-      "grad_norm": 680.9594369005476,
+      "epoch": 0.03,
+      "grad_norm": 746.141595669296,
       "learning_rate": 7.692307692307691e-08,
-      "logits/chosen": -2.605868101119995,
-      "logits/rejected": -2.5530831813812256,
-      "logps/chosen": -267.5987548828125,
-      "logps/rejected": -217.66183471679688,
-      "loss": 0.698,
-      "rewards/accuracies": 0.4444444477558136,
-      "rewards/chosen": -0.01713324338197708,
-      "rewards/margins": -0.0054442849941551685,
-      "rewards/rejected": -0.011688957922160625,
+      "logits/chosen": -2.6110925674438477,
+      "logits/rejected": -2.524423122406006,
+      "logps/chosen": -267.3368225097656,
+      "logps/rejected": -198.19520568847656,
+      "loss": 0.6963,
+      "rewards/accuracies": 0.3888888955116272,
+      "rewards/chosen": -0.02406422607600689,
+      "rewards/margins": -0.021091409027576447,
+      "rewards/rejected": -0.0029728179797530174,
       "step": 10
     },
     {
-      "epoch": 0.05118362124120281,
-      "grad_norm": 521.8729090609377,
+      "epoch": 0.05,
+      "grad_norm": 433.45696577907285,
       "learning_rate": 1.5384615384615382e-07,
-      "logits/chosen": -2.6261820793151855,
-      "logits/rejected": -2.563920497894287,
-      "logps/chosen": -260.98382568359375,
-      "logps/rejected": -207.09121704101562,
-      "loss": 0.6233,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.16019153594970703,
-      "rewards/margins": 0.1787206381559372,
-      "rewards/rejected": -0.018529098480939865,
+      "logits/chosen": -2.628641128540039,
+      "logits/rejected": -2.5271899700164795,
+      "logps/chosen": -260.9211120605469,
+      "logps/rejected": -198.38711547851562,
+      "loss": 0.5697,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.20215623080730438,
+      "rewards/margins": 0.35489505529403687,
+      "rewards/rejected": -0.1527387946844101,
       "step": 20
     },
     {
-      "epoch": 0.07677543186180422,
-      "grad_norm": 587.1314648207136,
+      "epoch": 0.08,
+      "grad_norm": 466.53027664588734,
       "learning_rate": 2.3076923076923078e-07,
-      "logits/chosen": -2.637615919113159,
-      "logits/rejected": -2.5644373893737793,
-      "logps/chosen": -252.89340209960938,
-      "logps/rejected": -198.89572143554688,
-      "loss": 0.4258,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 1.126920461654663,
-      "rewards/margins": 1.1405318975448608,
-      "rewards/rejected": -0.013611525297164917,
+      "logits/chosen": -2.6438043117523193,
+      "logits/rejected": -2.5429482460021973,
+      "logps/chosen": -252.55557250976562,
+      "logps/rejected": -195.11138916015625,
+      "loss": 0.306,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 1.301897406578064,
+      "rewards/margins": 1.880059838294983,
+      "rewards/rejected": -0.5781622529029846,
       "step": 30
     },
     {
-      "epoch": 0.10236724248240563,
-      "grad_norm": 348.96467920767,
+      "epoch": 0.1,
+      "grad_norm": 451.3374046706128,
       "learning_rate": 2.999939918069778e-07,
-      "logits/chosen": -2.648346424102783,
-      "logits/rejected": -2.5705723762512207,
-      "logps/chosen": -245.802001953125,
-      "logps/rejected": -193.64395141601562,
-      "loss": 0.3346,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 2.2736613750457764,
-      "rewards/margins": 2.252485513687134,
-      "rewards/rejected": 0.021175961941480637,
+      "logits/chosen": -2.6604888439178467,
+      "logits/rejected": -2.5366878509521484,
+      "logps/chosen": -244.97402954101562,
+      "logps/rejected": -199.42355346679688,
+      "loss": 0.2052,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.6914100646972656,
+      "rewards/margins": 3.822247266769409,
+      "rewards/rejected": -1.130837321281433,
       "step": 40
     },
     {
-      "epoch": 0.12795905310300704,
-      "grad_norm": 372.4722713208355,
+      "epoch": 0.13,
+      "grad_norm": 268.00932309883206,
       "learning_rate": 2.9927359084964875e-07,
-      "logits/chosen": -2.681694746017456,
-      "logits/rejected": -2.61210560798645,
-      "logps/chosen": -259.555908203125,
-      "logps/rejected": -203.13912963867188,
-      "loss": 0.3226,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 4.340622901916504,
-      "rewards/margins": 3.3357081413269043,
-      "rewards/rejected": 1.0049149990081787,
+      "logits/chosen": -2.7027461528778076,
+      "logits/rejected": -2.5904927253723145,
+      "logps/chosen": -258.02886962890625,
+      "logps/rejected": -204.94631958007812,
+      "loss": 0.2326,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 4.602013111114502,
+      "rewards/margins": 5.5924835205078125,
+      "rewards/rejected": -0.9904701113700867,
       "step": 50
     },
     {
-      "epoch": 0.15355086372360843,
-      "grad_norm": 406.3298611207908,
+      "epoch": 0.15,
+      "grad_norm": 340.360656955259,
       "learning_rate": 2.9735816061234966e-07,
-      "logits/chosen": -2.6559042930603027,
-      "logits/rejected": -2.586275577545166,
-      "logps/chosen": -260.26336669921875,
-      "logps/rejected": -206.2169647216797,
-      "loss": 0.3104,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 4.282172203063965,
-      "rewards/margins": 3.3613991737365723,
-      "rewards/rejected": 0.9207728505134583,
+      "logits/chosen": -2.6768908500671387,
+      "logits/rejected": -2.557954788208008,
+      "logps/chosen": -259.3023681640625,
+      "logps/rejected": -192.10040283203125,
+      "loss": 0.1987,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 4.70054817199707,
+      "rewards/margins": 6.1066999435424805,
+      "rewards/rejected": -1.4061520099639893,
       "step": 60
     },
     {
-      "epoch": 0.17914267434420986,
-      "grad_norm": 326.0546143118845,
+      "epoch": 0.18,
+      "grad_norm": 344.3937897868754,
       "learning_rate": 2.942630353226844e-07,
-      "logits/chosen": -2.6311216354370117,
-      "logits/rejected": -2.563368320465088,
-      "logps/chosen": -289.8287048339844,
-      "logps/rejected": -225.3174591064453,
-      "loss": 0.2976,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 4.040958404541016,
-      "rewards/margins": 4.104364395141602,
-      "rewards/rejected": -0.0634058266878128,
+      "logits/chosen": -2.650172472000122,
+      "logits/rejected": -2.500756025314331,
+      "logps/chosen": -288.3818054199219,
+      "logps/rejected": -222.72811889648438,
+      "loss": 0.1904,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 4.768267631530762,
+      "rewards/margins": 7.4695234298706055,
+      "rewards/rejected": -2.7012553215026855,
       "step": 70
     },
     {
-      "epoch": 0.20473448496481125,
-      "grad_norm": 419.2531192048535,
+      "epoch": 0.2,
+      "grad_norm": 453.28474416843744,
       "learning_rate": 2.900129934114876e-07,
-      "logits/chosen": -2.576873302459717,
-      "logits/rejected": -2.505291700363159,
-      "logps/chosen": -260.63385009765625,
-      "logps/rejected": -204.9154510498047,
-      "loss": 0.255,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 3.4621634483337402,
-      "rewards/margins": 4.38175106048584,
-      "rewards/rejected": -0.9195877909660339,
+      "logits/chosen": -2.5939252376556396,
+      "logits/rejected": -2.458728790283203,
+      "logps/chosen": -260.589111328125,
+      "logps/rejected": -212.63925170898438,
+      "loss": 0.1791,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 3.8766517639160156,
+      "rewards/margins": 7.434275150299072,
+      "rewards/rejected": -3.5576236248016357,
       "step": 80
     },
     {
-      "epoch": 0.23032629558541268,
-      "grad_norm": 526.7607200099269,
+      "epoch": 0.23,
+      "grad_norm": 598.8098573854415,
       "learning_rate": 2.8464205914585213e-07,
-      "logits/chosen": -2.578784942626953,
-      "logits/rejected": -2.520214080810547,
-      "logps/chosen": -256.7921447753906,
-      "logps/rejected": -216.86544799804688,
-      "loss": 0.2605,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": 2.476574182510376,
-      "rewards/margins": 3.6615688800811768,
-      "rewards/rejected": -1.1849944591522217,
+      "logits/chosen": -2.6328907012939453,
+      "logits/rejected": -2.507690906524658,
+      "logps/chosen": -253.75570678710938,
+      "logps/rejected": -193.23147583007812,
+      "loss": 0.19,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 3.2832133769989014,
+      "rewards/margins": 6.802037715911865,
+      "rewards/rejected": -3.518825054168701,
       "step": 90
     },
     {
-      "epoch": 0.2559181062060141,
-      "grad_norm": 765.903861721633,
+      "epoch": 0.26,
+      "grad_norm": 242.78091859451033,
       "learning_rate": 2.78193230243403e-07,
-      "logits/chosen": -2.6087539196014404,
-      "logits/rejected": -2.529031276702881,
-      "logps/chosen": -236.2876434326172,
-      "logps/rejected": -200.96026611328125,
-      "loss": 0.3552,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 2.9406275749206543,
-      "rewards/margins": 4.085367679595947,
-      "rewards/rejected": -1.1447399854660034,
+      "logits/chosen": -2.6586058139801025,
+      "logits/rejected": -2.52885103225708,
+      "logps/chosen": -234.36068725585938,
+      "logps/rejected": -197.55322265625,
+      "loss": 0.1766,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 4.212424278259277,
+      "rewards/margins": 7.145503997802734,
+      "rewards/rejected": -2.933079242706299,
       "step": 100
     },
     {
-      "epoch": 0.28150991682661547,
-      "grad_norm": 341.8125232177188,
+      "epoch": 0.28,
+      "grad_norm": 288.7802754186709,
       "learning_rate": 2.707181336484383e-07,
-      "logits/chosen": -2.6070523262023926,
-      "logits/rejected": -2.5225558280944824,
-      "logps/chosen": -257.5608825683594,
-      "logps/rejected": -209.5845489501953,
-      "loss": 0.2939,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 4.056784629821777,
-      "rewards/margins": 4.63706636428833,
-      "rewards/rejected": -0.5802817344665527,
+      "logits/chosen": -2.6430606842041016,
+      "logits/rejected": -2.5142664909362793,
+      "logps/chosen": -253.27975463867188,
+      "logps/rejected": -192.4219207763672,
+      "loss": 0.1616,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 5.699277400970459,
+      "rewards/margins": 7.844499111175537,
+      "rewards/rejected": -2.1452219486236572,
       "step": 110
     },
     {
-      "epoch": 0.30710172744721687,
-      "grad_norm": 404.74851579206114,
+      "epoch": 0.31,
+      "grad_norm": 242.33756453841693,
       "learning_rate": 2.622766122256652e-07,
-      "logits/chosen": -2.597041606903076,
-      "logits/rejected": -2.524170398712158,
-      "logps/chosen": -261.7186279296875,
-      "logps/rejected": -207.9816436767578,
-      "loss": 0.2833,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 3.8718819618225098,
-      "rewards/margins": 4.679508686065674,
-      "rewards/rejected": -0.8076267242431641,
+      "logits/chosen": -2.6139981746673584,
+      "logits/rejected": -2.516448497772217,
+      "logps/chosen": -256.9673767089844,
+      "logps/rejected": -211.84988403320312,
+      "loss": 0.1786,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 5.386040210723877,
+      "rewards/margins": 8.150399208068848,
+      "rewards/rejected": -2.7643585205078125,
       "step": 120
     },
     {
-      "epoch": 0.3326935380678183,
-      "grad_norm": 236.84495240691993,
+      "epoch": 0.33,
+      "grad_norm": 181.17712779023478,
       "learning_rate": 2.5293624568031005e-07,
-      "logits/chosen": -2.575899600982666,
-      "logits/rejected": -2.50651216506958,
-      "logps/chosen": -250.2094268798828,
-      "logps/rejected": -195.43930053710938,
-      "loss": 0.2701,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": 3.546905994415283,
-      "rewards/margins": 4.399945259094238,
-      "rewards/rejected": -0.8530394434928894,
+      "logits/chosen": -2.5844571590423584,
+      "logits/rejected": -2.4461381435394287,
+      "logps/chosen": -248.85513305664062,
+      "logps/rejected": -187.93716430664062,
+      "loss": 0.172,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 4.1653828620910645,
+      "rewards/margins": 7.870238304138184,
+      "rewards/rejected": -3.7048561573028564,
       "step": 130
     },
     {
-      "epoch": 0.3582853486884197,
-      "grad_norm": 316.10574756623254,
+      "epoch": 0.36,
+      "grad_norm": 386.4455316752931,
       "learning_rate": 2.4277180953993823e-07,
-      "logits/chosen": -2.597114086151123,
-      "logits/rejected": -2.526017904281616,
-      "logps/chosen": -267.56597900390625,
-      "logps/rejected": -214.47607421875,
-      "loss": 0.3022,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 3.8051650524139404,
-      "rewards/margins": 4.264659881591797,
-      "rewards/rejected": -0.4594948887825012,
+      "logits/chosen": -2.6205365657806396,
+      "logits/rejected": -2.5077974796295166,
+      "logps/chosen": -269.39251708984375,
+      "logps/rejected": -207.6556854248047,
+      "loss": 0.2112,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 3.9067764282226562,
+      "rewards/margins": 7.915855407714844,
+      "rewards/rejected": -4.0090789794921875,
       "step": 140
     },
     {
-      "epoch": 0.3838771593090211,
-      "grad_norm": 309.0947994138828,
+      "epoch": 0.38,
+      "grad_norm": 264.0646768310831,
       "learning_rate": 2.3186467652917566e-07,
-      "logits/chosen": -2.5764989852905273,
-      "logits/rejected": -2.5064079761505127,
-      "logps/chosen": -261.96685791015625,
-      "logps/rejected": -211.4801025390625,
-      "loss": 0.2803,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 4.31245756149292,
-      "rewards/margins": 4.854833126068115,
-      "rewards/rejected": -0.542374849319458,
+      "logits/chosen": -2.607632875442505,
+      "logits/rejected": -2.5044326782226562,
+      "logps/chosen": -261.9036560058594,
+      "logps/rejected": -212.12777709960938,
+      "loss": 0.2286,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 4.435807704925537,
+      "rewards/margins": 8.641576766967773,
+      "rewards/rejected": -4.2057695388793945,
       "step": 150
     },
     {
-      "epoch": 0.4094689699296225,
-      "grad_norm": 269.7166459422843,
+      "epoch": 0.41,
+      "grad_norm": 571.6419774908388,
       "learning_rate": 2.2030216512970552e-07,
-      "logits/chosen": -2.574291467666626,
-      "logits/rejected": -2.503854274749756,
-      "logps/chosen": -241.0322723388672,
-      "logps/rejected": -194.2302703857422,
-      "loss": 0.2695,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 3.9528305530548096,
-      "rewards/margins": 4.557216167449951,
-      "rewards/rejected": -0.604385256767273,
+      "logits/chosen": -2.5919671058654785,
+      "logits/rejected": -2.48041033744812,
+      "logps/chosen": -240.66891479492188,
+      "logps/rejected": -183.67489624023438,
+      "loss": 0.2024,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": 4.69089412689209,
+      "rewards/margins": 7.830643653869629,
+      "rewards/rejected": -3.139749526977539,
       "step": 160
     },
     {
-      "epoch": 0.4350607805502239,
-      "grad_norm": 316.51801045522615,
+      "epoch": 0.44,
+      "grad_norm": 351.0335983478419,
       "learning_rate": 2.0817684054072823e-07,
-      "logits/chosen": -2.578244924545288,
-      "logits/rejected": -2.512073040008545,
-      "logps/chosen": -262.7640075683594,
-      "logps/rejected": -207.98153686523438,
-      "loss": 0.2347,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 3.963876247406006,
-      "rewards/margins": 4.765882968902588,
-      "rewards/rejected": -0.8020064234733582,
+      "logits/chosen": -2.594038724899292,
+      "logits/rejected": -2.479696273803711,
+      "logps/chosen": -259.5168151855469,
+      "logps/rejected": -199.40609741210938,
+      "loss": 0.1492,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 4.83851432800293,
+      "rewards/margins": 7.966272830963135,
+      "rewards/rejected": -3.127758741378784,
       "step": 170
     },
     {
-      "epoch": 0.46065259117082535,
-      "grad_norm": 308.7222349543346,
+      "epoch": 0.46,
+      "grad_norm": 332.9543090868915,
       "learning_rate": 1.9558577363613703e-07,
-      "logits/chosen": -2.6352927684783936,
-      "logits/rejected": -2.549041271209717,
-      "logps/chosen": -235.12112426757812,
-      "logps/rejected": -184.7916717529297,
-      "loss": 0.2985,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 3.456139326095581,
-      "rewards/margins": 4.226747035980225,
-      "rewards/rejected": -0.770608127117157,
+      "logits/chosen": -2.643381357192993,
+      "logits/rejected": -2.5339646339416504,
+      "logps/chosen": -235.37637329101562,
+      "logps/rejected": -190.86380004882812,
+      "loss": 0.1789,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 3.2347798347473145,
+      "rewards/margins": 7.21783971786499,
+      "rewards/rejected": -3.983060359954834,
       "step": 180
     },
     {
-      "epoch": 0.48624440179142675,
-      "grad_norm": 258.43311774583907,
+      "epoch": 0.49,
+      "grad_norm": 413.79495264463003,
       "learning_rate": 1.826297638509251e-07,
-      "logits/chosen": -2.5956501960754395,
-      "logits/rejected": -2.5205705165863037,
-      "logps/chosen": -257.1447448730469,
-      "logps/rejected": -201.02076721191406,
-      "loss": 0.2441,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 3.8379158973693848,
-      "rewards/margins": 4.68334436416626,
-      "rewards/rejected": -0.8454282879829407,
+      "logits/chosen": -2.595829725265503,
+      "logits/rejected": -2.500739812850952,
+      "logps/chosen": -256.819580078125,
+      "logps/rejected": -206.32376098632812,
+      "loss": 0.179,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 3.607396364212036,
+      "rewards/margins": 7.752522945404053,
+      "rewards/rejected": -4.145126819610596,
       "step": 190
     },
     {
-      "epoch": 0.5118362124120281,
-      "grad_norm": 345.320567765688,
+      "epoch": 0.51,
+      "grad_norm": 458.13511698305706,
       "learning_rate": 1.694125322181083e-07,
-      "logits/chosen": -2.6293721199035645,
-      "logits/rejected": -2.5405123233795166,
-      "logps/chosen": -272.0235290527344,
-      "logps/rejected": -215.8284912109375,
-      "loss": 0.2397,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 4.249785423278809,
-      "rewards/margins": 5.354866981506348,
-      "rewards/rejected": -1.1050812005996704,
+      "logits/chosen": -2.63800311088562,
+      "logits/rejected": -2.5028953552246094,
+      "logps/chosen": -269.77081298828125,
+      "logps/rejected": -201.87176513671875,
+      "loss": 0.1642,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 5.391812801361084,
+      "rewards/margins": 9.271406173706055,
+      "rewards/rejected": -3.8795933723449707,
       "step": 200
     },
     {
-      "epoch": 0.5374280230326296,
-      "grad_norm": 544.5614360009855,
+      "epoch": 0.54,
+      "grad_norm": 449.95115191482023,
       "learning_rate": 1.5603989101641228e-07,
-      "logits/chosen": -2.6099467277526855,
-      "logits/rejected": -2.5365748405456543,
-      "logps/chosen": -264.88690185546875,
-      "logps/rejected": -209.3201141357422,
-      "loss": 0.3574,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 3.0834546089172363,
-      "rewards/margins": 4.563128471374512,
-      "rewards/rejected": -1.4796737432479858,
+      "logits/chosen": -2.620668888092041,
+      "logits/rejected": -2.5066120624542236,
+      "logps/chosen": -262.0960693359375,
+      "logps/rejected": -210.2143096923828,
+      "loss": 0.158,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 4.4803595542907715,
+      "rewards/margins": 8.733253479003906,
+      "rewards/rejected": -4.252893924713135,
       "step": 210
     },
     {
-      "epoch": 0.5630198336532309,
-      "grad_norm": 355.4652179934555,
+      "epoch": 0.56,
+      "grad_norm": 386.8925888622673,
       "learning_rate": 1.4261889667621828e-07,
-      "logits/chosen": -2.620954751968384,
-      "logits/rejected": -2.556044578552246,
-      "logps/chosen": -255.65048217773438,
-      "logps/rejected": -211.9694366455078,
-      "loss": 0.2686,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 3.0131924152374268,
-      "rewards/margins": 4.114481449127197,
-      "rewards/rejected": -1.1012890338897705,
+      "logits/chosen": -2.623037815093994,
+      "logits/rejected": -2.5237972736358643,
+      "logps/chosen": -254.78390502929688,
+      "logps/rejected": -206.7708282470703,
+      "loss": 0.2113,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 3.867889881134033,
+      "rewards/margins": 8.484498023986816,
+      "rewards/rejected": -4.616608142852783,
       "step": 220
     },
     {
-      "epoch": 0.5886116442738324,
-      "grad_norm": 308.3809131002459,
+      "epoch": 0.59,
+      "grad_norm": 264.4396878771059,
       "learning_rate": 1.2925699272529007e-07,
-      "logits/chosen": -2.6484158039093018,
-      "logits/rejected": -2.5617594718933105,
-      "logps/chosen": -265.15606689453125,
-      "logps/rejected": -213.99850463867188,
-      "loss": 0.253,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 4.123552322387695,
-      "rewards/margins": 4.665673732757568,
-      "rewards/rejected": -0.542121171951294,
+      "logits/chosen": -2.6517319679260254,
+      "logits/rejected": -2.5196144580841064,
+      "logps/chosen": -263.8755798339844,
+      "logps/rejected": -206.9174346923828,
+      "loss": 0.1518,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 4.761946201324463,
+      "rewards/margins": 8.56539249420166,
+      "rewards/rejected": -3.8034462928771973,
       "step": 230
     },
     {
-      "epoch": 0.6142034548944337,
-      "grad_norm": 534.1086121087117,
+      "epoch": 0.61,
+      "grad_norm": 419.86487391831014,
       "learning_rate": 1.160611496355417e-07,
-      "logits/chosen": -2.634096145629883,
-      "logits/rejected": -2.5853183269500732,
-      "logps/chosen": -259.93743896484375,
-      "logps/rejected": -216.4737548828125,
-      "loss": 0.2542,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 3.896925687789917,
-      "rewards/margins": 5.348150730133057,
-      "rewards/rejected": -1.451224684715271,
+      "logits/chosen": -2.6276602745056152,
+      "logits/rejected": -2.5167899131774902,
+      "logps/chosen": -257.7350769042969,
+      "logps/rejected": -210.53292846679688,
+      "loss": 0.1539,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 4.4489850997924805,
+      "rewards/margins": 9.010086059570312,
+      "rewards/rejected": -4.561100482940674,
       "step": 240
     },
     {
-      "epoch": 0.6397952655150352,
-      "grad_norm": 369.496952926306,
+      "epoch": 0.64,
+      "grad_norm": 233.18563601164297,
       "learning_rate": 1.0313700845691635e-07,
-      "logits/chosen": -2.64689040184021,
-      "logits/rejected": -2.5748291015625,
-      "logps/chosen": -265.5174255371094,
-      "logps/rejected": -219.63558959960938,
-      "loss": 0.263,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 3.130638837814331,
-      "rewards/margins": 4.757687568664551,
-      "rewards/rejected": -1.6270482540130615,
+      "logits/chosen": -2.6395657062530518,
+      "logits/rejected": -2.5249786376953125,
+      "logps/chosen": -263.78375244140625,
+      "logps/rejected": -216.53323364257812,
+      "loss": 0.1755,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 4.129330158233643,
+      "rewards/margins": 8.881619453430176,
+      "rewards/rejected": -4.752288341522217,
       "step": 250
     },
     {
-      "epoch": 0.6653870761356366,
-      "grad_norm": 277.7533336484834,
+      "epoch": 0.67,
+      "grad_norm": 301.32208500782895,
       "learning_rate": 9.058803509412647e-08,
-      "logits/chosen": -2.639646530151367,
-      "logits/rejected": -2.577847719192505,
-      "logps/chosen": -259.77166748046875,
-      "logps/rejected": -206.2452392578125,
-      "loss": 0.2401,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 3.0441479682922363,
-      "rewards/margins": 4.5638251304626465,
-      "rewards/rejected": -1.5196778774261475,
+      "logits/chosen": -2.63856840133667,
+      "logits/rejected": -2.5386815071105957,
+      "logps/chosen": -257.0624084472656,
+      "logps/rejected": -209.1647186279297,
+      "loss": 0.1345,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 4.407042503356934,
+      "rewards/margins": 8.488649368286133,
+      "rewards/rejected": -4.081605434417725,
       "step": 260
     },
     {
-      "epoch": 0.690978886756238,
-      "grad_norm": 399.30433913889635,
+      "epoch": 0.69,
+      "grad_norm": 262.72645974554706,
       "learning_rate": 7.851469199680381e-08,
-      "logits/chosen": -2.6187744140625,
-      "logits/rejected": -2.5289955139160156,
-      "logps/chosen": -270.216796875,
-      "logps/rejected": -225.6618194580078,
-      "loss": 0.267,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 3.1835665702819824,
-      "rewards/margins": 5.6915483474731445,
-      "rewards/rejected": -2.507981777191162,
+      "logits/chosen": -2.6127829551696777,
+      "logits/rejected": -2.4841771125793457,
+      "logps/chosen": -268.802490234375,
+      "logps/rejected": -217.36740112304688,
+      "loss": 0.1877,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 4.018976211547852,
+      "rewards/margins": 9.022808074951172,
+      "rewards/rejected": -5.00383186340332,
       "step": 270
     },
     {
-      "epoch": 0.7165706973768394,
-      "grad_norm": 272.0486827380912,
+      "epoch": 0.72,
+      "grad_norm": 216.84234596530857,
       "learning_rate": 6.701363389420295e-08,
-      "logits/chosen": -2.643656015396118,
-      "logits/rejected": -2.585157871246338,
-      "logps/chosen": -267.4632568359375,
-      "logps/rejected": -216.541015625,
-      "loss": 0.2361,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 3.1215426921844482,
-      "rewards/margins": 4.919692039489746,
-      "rewards/rejected": -1.7981488704681396,
+      "logits/chosen": -2.649754762649536,
+      "logits/rejected": -2.5276429653167725,
+      "logps/chosen": -265.546875,
+      "logps/rejected": -207.27426147460938,
+      "loss": 0.1499,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 4.076898574829102,
+      "rewards/margins": 9.239118576049805,
+      "rewards/rejected": -5.1622209548950195,
       "step": 280
     },
     {
-      "epoch": 0.7421625079974408,
-      "grad_norm": 328.48611808044626,
+      "epoch": 0.74,
+      "grad_norm": 267.2442999173217,
       "learning_rate": 5.617693401310837e-08,
-      "logits/chosen": -2.6063172817230225,
-      "logits/rejected": -2.5578410625457764,
-      "logps/chosen": -265.97723388671875,
-      "logps/rejected": -214.73867797851562,
-      "loss": 0.234,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 2.9844164848327637,
-      "rewards/margins": 4.7947001457214355,
-      "rewards/rejected": -1.810283899307251,
+      "logits/chosen": -2.6028785705566406,
+      "logits/rejected": -2.526676654815674,
+      "logps/chosen": -266.23406982421875,
+      "logps/rejected": -225.2560577392578,
+      "loss": 0.1785,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 3.791701555252075,
+      "rewards/margins": 7.995067596435547,
+      "rewards/rejected": -4.203365802764893,
       "step": 290
     },
     {
-      "epoch": 0.7677543186180422,
-      "grad_norm": 485.80687185291606,
+      "epoch": 0.77,
+      "grad_norm": 304.40926278591246,
       "learning_rate": 4.609134697356009e-08,
-      "logits/chosen": -2.6260290145874023,
-      "logits/rejected": -2.570155620574951,
-      "logps/chosen": -272.4629211425781,
-      "logps/rejected": -218.36135864257812,
-      "loss": 0.2579,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 3.055912971496582,
-      "rewards/margins": 4.725813388824463,
-      "rewards/rejected": -1.66990065574646,
+      "logits/chosen": -2.6325907707214355,
+      "logits/rejected": -2.513869524002075,
+      "logps/chosen": -270.08087158203125,
+      "logps/rejected": -216.07247924804688,
+      "loss": 0.1539,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": 4.105973720550537,
+      "rewards/margins": 9.096407890319824,
+      "rewards/rejected": -4.990433692932129,
       "step": 300
     },
     {
-      "epoch": 0.7933461292386437,
-      "grad_norm": 242.33931597612025,
+      "epoch": 0.79,
+      "grad_norm": 183.46140347071307,
       "learning_rate": 3.683761426338148e-08,
-      "logits/chosen": -2.5885746479034424,
-      "logits/rejected": -2.5180420875549316,
-      "logps/chosen": -275.88812255859375,
-      "logps/rejected": -213.2559814453125,
-      "loss": 0.2684,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 3.1929287910461426,
-      "rewards/margins": 5.026190757751465,
-      "rewards/rejected": -1.8332622051239014,
+      "logits/chosen": -2.5910542011260986,
+      "logits/rejected": -2.4951541423797607,
+      "logps/chosen": -275.2190246582031,
+      "logps/rejected": -210.9887237548828,
+      "loss": 0.1539,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 4.16552209854126,
+      "rewards/margins": 8.900744438171387,
+      "rewards/rejected": -4.735221862792969,
       "step": 310
     },
     {
-      "epoch": 0.818937939859245,
-      "grad_norm": 363.0806683357755,
+      "epoch": 0.82,
+      "grad_norm": 336.8108974655118,
       "learning_rate": 2.8489817851625024e-08,
-      "logits/chosen": -2.610605001449585,
-      "logits/rejected": -2.562117099761963,
-      "logps/chosen": -259.27264404296875,
-      "logps/rejected": -202.4815673828125,
-      "loss": 0.2424,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 2.8195748329162598,
-      "rewards/margins": 4.644892692565918,
-      "rewards/rejected": -1.8253180980682373,
+      "logits/chosen": -2.6144814491271973,
+      "logits/rejected": -2.5162534713745117,
+      "logps/chosen": -258.4134826660156,
+      "logps/rejected": -210.2257080078125,
+      "loss": 0.1866,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 3.252080202102661,
+      "rewards/margins": 8.461040496826172,
+      "rewards/rejected": -5.20896053314209,
       "step": 320
     },
     {
-      "epoch": 0.8445297504798465,
-      "grad_norm": 367.20877637580975,
+      "epoch": 0.84,
+      "grad_norm": 222.30131769249033,
       "learning_rate": 2.1114787115667477e-08,
-      "logits/chosen": -2.633531332015991,
-      "logits/rejected": -2.5729198455810547,
-      "logps/chosen": -262.47796630859375,
-      "logps/rejected": -215.085205078125,
-      "loss": 0.2445,
-      "rewards/accuracies": 0.9125000238418579,
-      "rewards/chosen": 3.610386610031128,
-      "rewards/margins": 4.788306713104248,
-      "rewards/rejected": -1.177919864654541,
+      "logits/chosen": -2.634732484817505,
+      "logits/rejected": -2.5385124683380127,
+      "logps/chosen": -260.74530029296875,
+      "logps/rejected": -209.8925323486328,
+      "loss": 0.1579,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": 4.701260566711426,
+      "rewards/margins": 9.207574844360352,
+      "rewards/rejected": -4.506315231323242,
       "step": 330
     },
     {
-      "epoch": 0.8701215611004478,
-      "grad_norm": 293.8708439770587,
+      "epoch": 0.87,
+      "grad_norm": 298.91033859930906,
       "learning_rate": 1.4771563829877598e-08,
-      "logits/chosen": -2.6198182106018066,
-      "logits/rejected": -2.561091661453247,
-      "logps/chosen": -258.06500244140625,
-      "logps/rejected": -208.5477752685547,
-      "loss": 0.2419,
-      "rewards/accuracies": 0.9437500238418579,
-      "rewards/chosen": 3.373103380203247,
-      "rewards/margins": 5.2115888595581055,
-      "rewards/rejected": -1.8384857177734375,
+      "logits/chosen": -2.620940685272217,
+      "logits/rejected": -2.5207433700561523,
+      "logps/chosen": -256.2254638671875,
+      "logps/rejected": -197.62445068359375,
+      "loss": 0.1936,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/chosen": 4.292969703674316,
+      "rewards/margins": 9.223516464233398,
+      "rewards/rejected": -4.930546760559082,
       "step": 340
     },
     {
-      "epoch": 0.8957133717210493,
-      "grad_norm": 723.7045832401957,
+      "epoch": 0.9,
+      "grad_norm": 322.2905254048829,
       "learning_rate": 9.510929498959268e-09,
-      "logits/chosen": -2.6300716400146484,
-      "logits/rejected": -2.58100962638855,
-      "logps/chosen": -267.67926025390625,
-      "logps/rejected": -224.3400421142578,
-      "loss": 0.2628,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 3.285783290863037,
-      "rewards/margins": 4.712864398956299,
-      "rewards/rejected": -1.4270811080932617,
+      "logits/chosen": -2.636793851852417,
+      "logits/rejected": -2.522016763687134,
+      "logps/chosen": -265.2098388671875,
+      "logps/rejected": -215.28469848632812,
+      "loss": 0.1724,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 3.8617148399353027,
+      "rewards/margins": 8.617898941040039,
+      "rewards/rejected": -4.7561845779418945,
       "step": 350
     },
     {
-      "epoch": 0.9213051823416507,
-      "grad_norm": 345.1677659124253,
+      "epoch": 0.92,
+      "grad_norm": 389.7582997593581,
       "learning_rate": 5.374998819965654e-09,
-      "logits/chosen": -2.6322882175445557,
-      "logits/rejected": -2.561767101287842,
-      "logps/chosen": -272.7427673339844,
-      "logps/rejected": -216.9067840576172,
-      "loss": 0.291,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 3.2370834350585938,
-      "rewards/margins": 4.826039791107178,
-      "rewards/rejected": -1.5889561176300049,
+      "logits/chosen": -2.6430556774139404,
+      "logits/rejected": -2.5382204055786133,
+      "logps/chosen": -270.87615966796875,
+      "logps/rejected": -212.25949096679688,
+      "loss": 0.1727,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 4.255101680755615,
+      "rewards/margins": 8.951577186584473,
+      "rewards/rejected": -4.696475028991699,
       "step": 360
     },
     {
-      "epoch": 0.946896992962252,
-      "grad_norm": 308.7073524078868,
+      "epoch": 0.95,
+      "grad_norm": 224.62737396153923,
       "learning_rate": 2.396882527576477e-09,
-      "logits/chosen": -2.5925230979919434,
-      "logits/rejected": -2.5545706748962402,
-      "logps/chosen": -271.9848327636719,
-      "logps/rejected": -226.7042999267578,
-      "loss": 0.2664,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 3.643012523651123,
-      "rewards/margins": 4.708712577819824,
-      "rewards/rejected": -1.065699815750122,
+      "logits/chosen": -2.5986270904541016,
+      "logits/rejected": -2.492842197418213,
+      "logps/chosen": -269.3092346191406,
+      "logps/rejected": -214.992919921875,
+      "loss": 0.1462,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 4.158980846405029,
+      "rewards/margins": 8.108312606811523,
+      "rewards/rejected": -3.9493324756622314,
       "step": 370
     },
     {
-      "epoch": 0.9724888035828535,
-      "grad_norm": 182.6217744497864,
+      "epoch": 0.97,
+      "grad_norm": 174.48647090969504,
       "learning_rate": 6.004223217757509e-10,
-      "logits/chosen": -2.6407010555267334,
-      "logits/rejected": -2.5988292694091797,
-      "logps/chosen": -261.07269287109375,
-      "logps/rejected": -228.239013671875,
-      "loss": 0.2581,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 3.656334638595581,
-      "rewards/margins": 4.874091148376465,
-      "rewards/rejected": -1.2177565097808838,
+      "logits/chosen": -2.653160572052002,
+      "logits/rejected": -2.5599982738494873,
+      "logps/chosen": -260.55279541015625,
+      "logps/rejected": -219.11105346679688,
+      "loss": 0.1559,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 4.291203498840332,
+      "rewards/margins": 8.267863273620605,
+      "rewards/rejected": -3.9766602516174316,
       "step": 380
     },
     {
-      "epoch": 0.9980806142034548,
-      "grad_norm": 472.83951094562434,
+      "epoch": 1.0,
+      "grad_norm": 384.8395002541089,
       "learning_rate": 0.0,
-      "logits/chosen": -2.653154134750366,
-      "logits/rejected": -2.5895044803619385,
-      "logps/chosen": -248.54092407226562,
-      "logps/rejected": -207.9895477294922,
-      "loss": 0.2561,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 3.2267730236053467,
-      "rewards/margins": 4.816933631896973,
-      "rewards/rejected": -1.5901600122451782,
+      "logits/chosen": -2.6584863662719727,
+      "logits/rejected": -2.5489349365234375,
+      "logps/chosen": -248.6810302734375,
+      "logps/rejected": -207.1171875,
+      "loss": 0.1633,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 3.9972071647644043,
+      "rewards/margins": 9.053262710571289,
+      "rewards/rejected": -5.056054592132568,
       "step": 390
     },
     {
-      "epoch": 0.9980806142034548,
+      "epoch": 1.0,
       "step": 390,
       "total_flos": 0.0,
-      "train_loss": 0.29645214692140237,
-      "train_runtime": 6440.0874,
-      "train_samples_per_second": 7.764,
-      "train_steps_per_second": 0.061
+      "train_loss": 0.20326648155848184,
+      "train_runtime": 5896.1189,
+      "train_samples_per_second": 8.48,
+      "train_steps_per_second": 0.066
     }
   ],
   "logging_steps": 10,
@@ -623,18 +623,6 @@
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
-  "stateful_callbacks": {
-    "TrainerControl": {
-      "args": {
-        "should_epoch_stop": false,
-        "should_evaluate": false,
-        "should_log": false,
-        "should_save": true,
-        "should_training_stop": false
-      },
-      "attributes": {}
-    }
-  },
   "total_flos": 0.0,
   "train_batch_size": 4,
   "trial_name": null,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b7346ad29a6b9b0903d845abfe58994ac8a80348e425e0e321ee83abcfcb035e
-size 6264
+oid sha256:7d90343e793e5916c6afac01760eb7b5a30707ec90a3ec177dfdeda931df024a
+size 6328