Model save

README.md

@@ -13,7 +13,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model was trained from scratch on an unknown dataset.
+This model was trained from scratch on the None dataset.
 
 ## Model description
 
@@ -52,7 +52,7 @@ The following hyperparameters were used during training:
 
 ### Framework versions
 
-- Transformers 4.40.2
+- Transformers 4.41.1
 - Pytorch 2.1.2+cu118
-- Datasets 2.19.1
+- Datasets 2.16.1
 - Tokenizers 0.19.1

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9984,
     "total_flos": 0.0,
-    "train_loss": 0.3884877807054764,
-    "train_runtime": 4677.6403,
+    "train_loss": 0.4014626894241724,
+    "train_runtime": 5052.0322,
     "train_samples": 39942,
-    "train_samples_per_second": 8.539,
-    "train_steps_per_second": 0.033
+    "train_samples_per_second": 7.906,
+    "train_steps_per_second": 0.031
 }

config.json

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.40.2",
+  "transformers_version": "4.41.1",
   "use_cache": false,
   "vocab_size": 32002
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 32000,
-  "transformers_version": "4.40.2"
+  "transformers_version": "4.41.1"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9dba7919b04fd1d70e11a29220f5eaf634bb315409f9a94f7a263955001973c6
+oid sha256:7f512a730a26a61a1c16526c02b35b83222d091cd9296f1da7e71affd7995410
 size 4943178720

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a84b4be1c4d4aad2ce051a91acda1c33720055694a806a6fb36a24f1b682e2c3
+oid sha256:ac0006e40eab93ba96e9c4ac961c6bd73d92585b3f54f427b278b0fb7b9767b2
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:509b7c42a2681737189e49583a27d65db54dd70bda3ef6f8890b6b3e93dca2ff
+oid sha256:a220e930529efb9b6318941d96516bbbb5fa2c9d14cf80d2860389fe098f2e86
 size 4540532728

runs/Jun05_23-17-05_n136-082-130/events.out.tfevents.1717600754.n136-082-130.1889466.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eb020d37b9be1057d9fcbd66defea51b3268e8b466994f77a5e0067c756f0b28
-size 12591
+oid sha256:38495588af7a3d409c5c24789e209e8c63e45c66ecbc5399988e055f803c60c7
+size 16361

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9984,
     "total_flos": 0.0,
-    "train_loss": 0.3884877807054764,
-    "train_runtime": 4677.6403,
+    "train_loss": 0.4014626894241724,
+    "train_runtime": 5052.0322,
     "train_samples": 39942,
-    "train_samples_per_second": 8.539,
-    "train_steps_per_second": 0.033
+    "train_samples_per_second": 7.906,
+    "train_steps_per_second": 0.031
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0064,
-      "grad_norm": 1341.8773394764246,
+      "grad_norm": 1341.948853106591,
       "learning_rate": 3.125e-09,
       "logits/chosen": -3.9499800205230713,
       "logits/rejected": -4.237819194793701,
@@ -25,237 +25,237 @@
     },
     {
       "epoch": 0.064,
-      "grad_norm": 1342.2810836893796,
+      "grad_norm": 1338.549586928997,
       "learning_rate": 3.125e-08,
-      "logits/chosen": -4.129705905914307,
-      "logits/rejected": -4.352028846740723,
-      "logps/chosen": -351.5079650878906,
-      "logps/rejected": -308.8138427734375,
-      "loss": 0.7326,
-      "rewards/accuracies": 0.3680555522441864,
-      "rewards/chosen": -0.04078766331076622,
-      "rewards/margins": -0.11378024518489838,
-      "rewards/rejected": 0.07299260050058365,
+      "logits/chosen": -4.129465579986572,
+      "logits/rejected": -4.351754188537598,
+      "logps/chosen": -351.4585266113281,
+      "logps/rejected": -308.89202880859375,
+      "loss": 0.7278,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.016052477061748505,
+      "rewards/margins": -0.04994054138660431,
+      "rewards/rejected": 0.033888060599565506,
       "step": 10
     },
     {
       "epoch": 0.128,
-      "grad_norm": 1252.3965895279962,
+      "grad_norm": 1285.3502810697903,
       "learning_rate": 4.9899357349880975e-08,
-      "logits/chosen": -4.194980144500732,
-      "logits/rejected": -4.382790565490723,
-      "logps/chosen": -334.9039001464844,
-      "logps/rejected": -293.8416748046875,
-      "loss": 0.683,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": 0.22410114109516144,
-      "rewards/margins": 0.11712154000997543,
-      "rewards/rejected": 0.10697959363460541,
+      "logits/chosen": -4.195886611938477,
+      "logits/rejected": -4.383217811584473,
+      "logps/chosen": -334.9483337402344,
+      "logps/rejected": -293.82867431640625,
+      "loss": 0.6803,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.20189008116722107,
+      "rewards/margins": 0.08841639757156372,
+      "rewards/rejected": 0.11347369104623795,
       "step": 20
     },
     {
       "epoch": 0.192,
-      "grad_norm": 904.3776918610464,
+      "grad_norm": 938.2936106934807,
       "learning_rate": 4.877641290737884e-08,
-      "logits/chosen": -4.230466365814209,
-      "logits/rejected": -4.363996505737305,
-      "logps/chosen": -327.71453857421875,
-      "logps/rejected": -295.3287658691406,
-      "loss": 0.5498,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 0.9708820581436157,
-      "rewards/margins": 0.5084127187728882,
-      "rewards/rejected": 0.46246927976608276,
+      "logits/chosen": -4.2334418296813965,
+      "logits/rejected": -4.3670477867126465,
+      "logps/chosen": -327.77716064453125,
+      "logps/rejected": -295.27203369140625,
+      "loss": 0.5536,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.9395688772201538,
+      "rewards/margins": 0.4487342834472656,
+      "rewards/rejected": 0.49083465337753296,
       "step": 30
     },
     {
       "epoch": 0.256,
-      "grad_norm": 894.6327423356746,
+      "grad_norm": 899.5533271284629,
       "learning_rate": 4.646121984004665e-08,
-      "logits/chosen": -4.1493096351623535,
-      "logits/rejected": -4.351648807525635,
-      "logps/chosen": -330.09368896484375,
-      "logps/rejected": -288.2974853515625,
-      "loss": 0.4125,
+      "logits/chosen": -4.14696741104126,
+      "logits/rejected": -4.348996162414551,
+      "logps/chosen": -330.3467712402344,
+      "logps/rejected": -288.37066650390625,
+      "loss": 0.4332,
       "rewards/accuracies": 0.8218749761581421,
-      "rewards/chosen": 1.9414455890655518,
-      "rewards/margins": 1.1434320211410522,
-      "rewards/rejected": 0.7980135083198547,
+      "rewards/chosen": 1.8148998022079468,
+      "rewards/margins": 1.0534685850143433,
+      "rewards/rejected": 0.7614310383796692,
       "step": 40
     },
     {
       "epoch": 0.32,
-      "grad_norm": 706.4309708182283,
+      "grad_norm": 737.601892593307,
       "learning_rate": 4.3069871595684784e-08,
-      "logits/chosen": -4.244365215301514,
-      "logits/rejected": -4.423664093017578,
-      "logps/chosen": -329.6412353515625,
-      "logps/rejected": -291.22528076171875,
-      "loss": 0.3694,
-      "rewards/accuracies": 0.840624988079071,
-      "rewards/chosen": 2.6057987213134766,
-      "rewards/margins": 1.537340521812439,
-      "rewards/rejected": 1.068458080291748,
+      "logits/chosen": -4.238839149475098,
+      "logits/rejected": -4.417731285095215,
+      "logps/chosen": -329.99560546875,
+      "logps/rejected": -291.286865234375,
+      "loss": 0.3783,
+      "rewards/accuracies": 0.8531249761581421,
+      "rewards/chosen": 2.4285898208618164,
+      "rewards/margins": 1.3909423351287842,
+      "rewards/rejected": 1.0376476049423218,
       "step": 50
     },
     {
       "epoch": 0.384,
-      "grad_norm": 679.6447682422123,
+      "grad_norm": 725.6613657693058,
       "learning_rate": 3.8772424536302564e-08,
-      "logits/chosen": -4.262530326843262,
-      "logits/rejected": -4.4340620040893555,
-      "logps/chosen": -320.7197570800781,
-      "logps/rejected": -291.15264892578125,
-      "loss": 0.3459,
-      "rewards/accuracies": 0.8343750238418579,
-      "rewards/chosen": 3.022132158279419,
-      "rewards/margins": 1.8344866037368774,
-      "rewards/rejected": 1.187645673751831,
+      "logits/chosen": -4.256905555725098,
+      "logits/rejected": -4.42824125289917,
+      "logps/chosen": -321.1641845703125,
+      "logps/rejected": -291.2255859375,
+      "loss": 0.3554,
+      "rewards/accuracies": 0.846875011920929,
+      "rewards/chosen": 2.799910306930542,
+      "rewards/margins": 1.64876389503479,
+      "rewards/rejected": 1.151146411895752,
       "step": 60
     },
     {
       "epoch": 0.448,
-      "grad_norm": 600.9568341116722,
+      "grad_norm": 650.256008502672,
       "learning_rate": 3.378437060203357e-08,
-      "logits/chosen": -4.188047885894775,
-      "logits/rejected": -4.377224445343018,
-      "logps/chosen": -320.23345947265625,
-      "logps/rejected": -288.5027770996094,
-      "loss": 0.3189,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": 3.3037331104278564,
-      "rewards/margins": 2.1254096031188965,
-      "rewards/rejected": 1.1783230304718018,
+      "logits/chosen": -4.1877875328063965,
+      "logits/rejected": -4.377414703369141,
+      "logps/chosen": -320.77117919921875,
+      "logps/rejected": -288.512451171875,
+      "loss": 0.3377,
+      "rewards/accuracies": 0.7906249761581421,
+      "rewards/chosen": 3.0348763465881348,
+      "rewards/margins": 1.8613855838775635,
+      "rewards/rejected": 1.1734905242919922,
       "step": 70
     },
     {
       "epoch": 0.512,
-      "grad_norm": 654.7049863576665,
+      "grad_norm": 643.4652639779088,
       "learning_rate": 2.8355831645441387e-08,
-      "logits/chosen": -4.0522565841674805,
-      "logits/rejected": -4.341280937194824,
-      "logps/chosen": -345.8344421386719,
-      "logps/rejected": -307.4328918457031,
-      "loss": 0.3105,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 3.7246456146240234,
-      "rewards/margins": 2.5337729454040527,
-      "rewards/rejected": 1.1908724308013916,
+      "logits/chosen": -4.054490089416504,
+      "logits/rejected": -4.3433709144592285,
+      "logps/chosen": -346.53253173828125,
+      "logps/rejected": -307.34930419921875,
+      "loss": 0.3369,
+      "rewards/accuracies": 0.8656250238418579,
+      "rewards/chosen": 3.375593662261963,
+      "rewards/margins": 2.1429190635681152,
+      "rewards/rejected": 1.2326747179031372,
       "step": 80
     },
     {
       "epoch": 0.576,
-      "grad_norm": 638.1282144295093,
+      "grad_norm": 658.4601656800384,
       "learning_rate": 2.2759017277414164e-08,
-      "logits/chosen": -4.180428504943848,
-      "logits/rejected": -4.390549659729004,
-      "logps/chosen": -332.82275390625,
-      "logps/rejected": -295.1810607910156,
-      "loss": 0.3099,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 3.2552542686462402,
-      "rewards/margins": 2.3172354698181152,
-      "rewards/rejected": 0.9380186796188354,
+      "logits/chosen": -4.182233810424805,
+      "logits/rejected": -4.391860485076904,
+      "logps/chosen": -333.4163513183594,
+      "logps/rejected": -295.03515625,
+      "loss": 0.3301,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 2.9584574699401855,
+      "rewards/margins": 1.9474881887435913,
+      "rewards/rejected": 1.0109691619873047,
       "step": 90
     },
     {
       "epoch": 0.64,
-      "grad_norm": 680.3285346474286,
+      "grad_norm": 716.6222925151064,
       "learning_rate": 1.7274575140626317e-08,
-      "logits/chosen": -4.167009353637695,
-      "logits/rejected": -4.386021614074707,
-      "logps/chosen": -330.049560546875,
-      "logps/rejected": -285.8011169433594,
-      "loss": 0.3123,
-      "rewards/accuracies": 0.8843749761581421,
-      "rewards/chosen": 3.6218514442443848,
-      "rewards/margins": 2.723836898803711,
-      "rewards/rejected": 0.8980148434638977,
+      "logits/chosen": -4.168593406677246,
+      "logits/rejected": -4.387479305267334,
+      "logps/chosen": -330.75775146484375,
+      "logps/rejected": -285.5531311035156,
+      "loss": 0.3315,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 3.267747163772583,
+      "rewards/margins": 2.245748281478882,
+      "rewards/rejected": 1.0219987630844116,
       "step": 100
     },
     {
       "epoch": 0.704,
-      "grad_norm": 616.2712616857408,
+      "grad_norm": 660.4496184068971,
       "learning_rate": 1.217751806485235e-08,
-      "logits/chosen": -4.145500183105469,
-      "logits/rejected": -4.386542320251465,
-      "logps/chosen": -311.7583923339844,
-      "logps/rejected": -276.3233947753906,
-      "loss": 0.3022,
-      "rewards/accuracies": 0.8843749761581421,
-      "rewards/chosen": 3.584909439086914,
-      "rewards/margins": 2.6118006706237793,
-      "rewards/rejected": 0.9731090664863586,
+      "logits/chosen": -4.1479597091674805,
+      "logits/rejected": -4.388964653015137,
+      "logps/chosen": -312.3727722167969,
+      "logps/rejected": -276.13372802734375,
+      "loss": 0.3225,
+      "rewards/accuracies": 0.8656250238418579,
+      "rewards/chosen": 3.277716875076294,
+      "rewards/margins": 2.2097737789154053,
+      "rewards/rejected": 1.0679429769515991,
       "step": 110
     },
     {
       "epoch": 0.768,
-      "grad_norm": 649.1888991009114,
+      "grad_norm": 637.7261383555538,
       "learning_rate": 7.723433775328384e-09,
-      "logits/chosen": -4.141805171966553,
-      "logits/rejected": -4.35054874420166,
-      "logps/chosen": -325.5559997558594,
-      "logps/rejected": -280.5980529785156,
-      "loss": 0.3033,
-      "rewards/accuracies": 0.8656250238418579,
-      "rewards/chosen": 3.6838138103485107,
-      "rewards/margins": 2.6417319774627686,
-      "rewards/rejected": 1.0420820713043213,
+      "logits/chosen": -4.145724296569824,
+      "logits/rejected": -4.354761123657227,
+      "logps/chosen": -326.25091552734375,
+      "logps/rejected": -280.4151916503906,
+      "loss": 0.3134,
+      "rewards/accuracies": 0.8531249761581421,
+      "rewards/chosen": 3.3363654613494873,
+      "rewards/margins": 2.202853202819824,
+      "rewards/rejected": 1.1335121393203735,
       "step": 120
     },
     {
       "epoch": 0.832,
-      "grad_norm": 747.4298760038148,
+      "grad_norm": 768.5710075733069,
       "learning_rate": 4.135668656967433e-09,
-      "logits/chosen": -4.228358268737793,
-      "logits/rejected": -4.38976526260376,
-      "logps/chosen": -331.02642822265625,
-      "logps/rejected": -286.7439880371094,
-      "loss": 0.3064,
+      "logits/chosen": -4.2324018478393555,
+      "logits/rejected": -4.393607139587402,
+      "logps/chosen": -331.71051025390625,
+      "logps/rejected": -286.67938232421875,
+      "loss": 0.3203,
       "rewards/accuracies": 0.875,
-      "rewards/chosen": 3.7264277935028076,
-      "rewards/margins": 2.6530587673187256,
-      "rewards/rejected": 1.073369026184082,
+      "rewards/chosen": 3.384364604949951,
+      "rewards/margins": 2.2786831855773926,
+      "rewards/rejected": 1.1056816577911377,
       "step": 130
     },
     {
       "epoch": 0.896,
-      "grad_norm": 697.5841535989922,
+      "grad_norm": 742.2155895768275,
       "learning_rate": 1.5941282340065698e-09,
-      "logits/chosen": -4.18213415145874,
-      "logits/rejected": -4.3970947265625,
-      "logps/chosen": -332.56500244140625,
-      "logps/rejected": -303.63543701171875,
-      "loss": 0.3069,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 3.5617058277130127,
-      "rewards/margins": 2.6050186157226562,
-      "rewards/rejected": 0.9566874504089355,
+      "logits/chosen": -4.18549919128418,
+      "logits/rejected": -4.4001359939575195,
+      "logps/chosen": -333.18035888671875,
+      "logps/rejected": -303.45220947265625,
+      "loss": 0.3282,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 3.2540221214294434,
+      "rewards/margins": 2.2057249546051025,
+      "rewards/rejected": 1.048297643661499,
       "step": 140
     },
     {
       "epoch": 0.96,
-      "grad_norm": 567.1610784183449,
+      "grad_norm": 668.7483012920786,
       "learning_rate": 2.262559558016325e-10,
-      "logits/chosen": -4.118973731994629,
-      "logits/rejected": -4.348026752471924,
-      "logps/chosen": -339.0107116699219,
-      "logps/rejected": -295.09564208984375,
-      "loss": 0.3078,
-      "rewards/accuracies": 0.8656250238418579,
-      "rewards/chosen": 3.7477049827575684,
-      "rewards/margins": 2.61022686958313,
-      "rewards/rejected": 1.1374781131744385,
+      "logits/chosen": -4.1221604347229,
+      "logits/rejected": -4.351316452026367,
+      "logps/chosen": -339.6897888183594,
+      "logps/rejected": -295.03692626953125,
+      "loss": 0.3208,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 3.4081547260284424,
+      "rewards/margins": 2.2413132190704346,
+      "rewards/rejected": 1.1668416261672974,
       "step": 150
     },
     {
       "epoch": 0.9984,
       "step": 156,
       "total_flos": 0.0,
-      "train_loss": 0.3884877807054764,
-      "train_runtime": 4677.6403,
-      "train_samples_per_second": 8.539,
-      "train_steps_per_second": 0.033
+      "train_loss": 0.4014626894241724,
+      "train_runtime": 5052.0322,
+      "train_samples_per_second": 7.906,
+      "train_steps_per_second": 0.031
     }
   ],
   "logging_steps": 10,
@@ -263,6 +263,18 @@
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
   "total_flos": 0.0,
   "train_batch_size": 8,
   "trial_name": null,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:44368936e8a5f160f38c764acd78a1ed87cb99f1b31bc5a44994e052788c660a
-size 6392
+oid sha256:2d92a249952e57df29e57ec30d0088226bf91a9212b8928552a333e4b9069864
+size 6520