Model save

README.md

@@ -13,7 +13,7 @@ model-index:
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/sanqiang/wdpo/runs/i9fm0xk3)
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/sanqiang/wdpo/runs/sypbih9g)
 # zephyr-7b-dpo-full
 
 This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.20092295110225677,
-    "train_runtime": 384.8987,
+    "train_loss": 0.2009538251836345,
+    "train_runtime": 384.7218,
     "train_samples": 6750,
-    "train_samples_per_second": 17.537,
+    "train_samples_per_second": 17.545,
     "train_steps_per_second": 0.138
 }

config.json

@@ -3,6 +3,7 @@
   "architectures": [
     "MistralForCausalLM"
   ],
+  "attention_dropout": 0.0,
   "bos_token_id": 1,
   "eos_token_id": 2,
   "hidden_act": "silu",
@@ -19,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.35.2",
+  "transformers_version": "4.41.0.dev0",
   "use_cache": false,
   "vocab_size": 32000
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a46cc3b5f84d4d99bb5b73eed876f0b5578ddd4737df15a4e469e33c95c08170
+oid sha256:64ac0bec7ee02cdce6549e87eb5021fadfbc68acc6097efe408ba777bd78ded0
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cde9eb125e0d5a3b3305ef0cdb4bea397e66b0d0622e3a57126bb6a5687634fa
+oid sha256:3aa2404e9e95bdb4831267fbcb3c05850f0ff8f8576225b95f5aadc01aeb29fb
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3c0430eff547cb6ba5d3819817467987ed432cfa8a3a5fcd748e54a73cb5ddc4
+oid sha256:03bfb347f7333d6cdebfaf4c3a4baf4be3d0a11a34389519ee0cf86eb26080c2
 size 4540516344

tokenizer.json

@@ -134,6 +134,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
+    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

tokenizer_config.json

@@ -1,4 +1,6 @@
 {
+  "add_bos_token": true,
+  "add_eos_token": false,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
@@ -34,7 +36,6 @@
   "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
-  "legacy": true,
   "model_max_length": 2048,
   "pad_token": "</s>",
   "sp_model_kwargs": {},

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.20092295110225677,
-    "train_runtime": 384.8987,
+    "train_loss": 0.2009538251836345,
+    "train_runtime": 384.7218,
     "train_samples": 6750,
-    "train_samples_per_second": 17.537,
+    "train_samples_per_second": 17.545,
     "train_steps_per_second": 0.138
 }

trainer_state.json

@@ -13,7 +13,7 @@
       "debug/policy_weights": 0.24804016947746277,
       "debug/raw_losses": 0.6931471824645996,
       "epoch": 0.018867924528301886,
-      "grad_norm": 3.0791833143219045,
+      "grad_norm": 3.079222482298495,
       "learning_rate": 8.333333333333333e-08,
       "logits/chosen": -2.855412006378174,
       "logits/rejected": -2.8797199726104736,
@@ -27,102 +27,102 @@
       "step": 1
     },
     {
-      "debug/losses": 0.20367620885372162,
-      "debug/policy_weights": 0.2948996424674988,
-      "debug/raw_losses": 0.6906173229217529,
+      "debug/losses": 0.20364493131637573,
+      "debug/policy_weights": 0.2948070466518402,
+      "debug/raw_losses": 0.6907545924186707,
       "epoch": 0.18867924528301888,
-      "grad_norm": 3.2025034006962603,
+      "grad_norm": 3.193404567956391,
       "learning_rate": 4.911172937635942e-07,
-      "logits/chosen": -2.8658909797668457,
-      "logits/rejected": -2.8917548656463623,
-      "logps/chosen": -305.7406005859375,
-      "logps/rejected": -332.2490234375,
+      "logits/chosen": -2.865929126739502,
+      "logits/rejected": -2.8918821811676025,
+      "logps/chosen": -305.75299072265625,
+      "logps/rejected": -332.2339172363281,
       "loss": 0.2033,
-      "rewards/accuracies": 0.4791666567325592,
-      "rewards/chosen": 0.0008342999499291182,
-      "rewards/margins": 0.005153011996299028,
-      "rewards/rejected": -0.004318712279200554,
+      "rewards/accuracies": 0.5069444179534912,
+      "rewards/chosen": 0.0007104460964910686,
+      "rewards/margins": 0.004878002218902111,
+      "rewards/rejected": -0.0041675567626953125,
       "step": 10
     },
     {
-      "debug/losses": 0.20357565581798553,
-      "debug/policy_weights": 0.30120497941970825,
-      "debug/raw_losses": 0.675395131111145,
+      "debug/losses": 0.2036052942276001,
+      "debug/policy_weights": 0.3014451563358307,
+      "debug/raw_losses": 0.6749827265739441,
       "epoch": 0.37735849056603776,
-      "grad_norm": 3.097721440067098,
+      "grad_norm": 3.0715644454401403,
       "learning_rate": 3.982949361823388e-07,
-      "logits/chosen": -2.855691432952881,
-      "logits/rejected": -2.8575425148010254,
-      "logps/chosen": -324.3226623535156,
-      "logps/rejected": -331.01007080078125,
-      "loss": 0.2068,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.0029448498971760273,
-      "rewards/margins": 0.03993762657046318,
-      "rewards/rejected": -0.03699277713894844,
+      "logits/chosen": -2.8559136390686035,
+      "logits/rejected": -2.8577218055725098,
+      "logps/chosen": -324.2321472167969,
+      "logps/rejected": -331.00140380859375,
+      "loss": 0.2069,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.003850350622087717,
+      "rewards/margins": 0.040756504982709885,
+      "rewards/rejected": -0.036906156688928604,
       "step": 20
     },
     {
-      "debug/losses": 0.20828036963939667,
-      "debug/policy_weights": 0.3203299641609192,
-      "debug/raw_losses": 0.6491612195968628,
+      "debug/losses": 0.2083912342786789,
+      "debug/policy_weights": 0.3206340968608856,
+      "debug/raw_losses": 0.6488931775093079,
       "epoch": 0.5660377358490566,
-      "grad_norm": 3.2758930059808353,
+      "grad_norm": 3.2842987687808507,
       "learning_rate": 2.416462557480814e-07,
-      "logits/chosen": -2.839444398880005,
-      "logits/rejected": -2.842738628387451,
-      "logps/chosen": -298.2297668457031,
-      "logps/rejected": -313.391845703125,
-      "loss": 0.2007,
+      "logits/chosen": -2.839723825454712,
+      "logits/rejected": -2.842979907989502,
+      "logps/chosen": -298.1597595214844,
+      "logps/rejected": -313.3924560546875,
+      "loss": 0.2009,
       "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.014156119897961617,
-      "rewards/margins": 0.11547034978866577,
-      "rewards/rejected": -0.12962646782398224,
+      "rewards/chosen": -0.01345623005181551,
+      "rewards/margins": 0.11617596447467804,
+      "rewards/rejected": -0.12963220477104187,
       "step": 30
     },
     {
-      "debug/losses": 0.18270191550254822,
-      "debug/policy_weights": 0.30252760648727417,
-      "debug/raw_losses": 0.584720253944397,
+      "debug/losses": 0.18273566663265228,
+      "debug/policy_weights": 0.3026610016822815,
+      "debug/raw_losses": 0.5847629904747009,
       "epoch": 0.7547169811320755,
-      "grad_norm": 2.7163808332522805,
+      "grad_norm": 2.7988373684679493,
       "learning_rate": 8.859303711029939e-08,
-      "logits/chosen": -2.842470645904541,
-      "logits/rejected": -2.8394596576690674,
-      "logps/chosen": -290.64208984375,
-      "logps/rejected": -320.837890625,
-      "loss": 0.1963,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.011087085120379925,
-      "rewards/margins": 0.30588188767433167,
-      "rewards/rejected": -0.31696897745132446,
+      "logits/chosen": -2.8427796363830566,
+      "logits/rejected": -2.839672327041626,
+      "logps/chosen": -290.6694641113281,
+      "logps/rejected": -320.8838806152344,
+      "loss": 0.1964,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.011360697448253632,
+      "rewards/margins": 0.3060683310031891,
+      "rewards/rejected": -0.3174290060997009,
       "step": 40
     },
     {
-      "debug/losses": 0.23133957386016846,
-      "debug/policy_weights": 0.3536807894706726,
-      "debug/raw_losses": 0.6469973921775818,
+      "debug/losses": 0.23139193654060364,
+      "debug/policy_weights": 0.35352057218551636,
+      "debug/raw_losses": 0.6475176811218262,
       "epoch": 0.9433962264150944,
-      "grad_norm": 3.004599614335292,
+      "grad_norm": 3.0097008369658336,
       "learning_rate": 5.009573740853313e-09,
-      "logits/chosen": -2.8772711753845215,
-      "logits/rejected": -2.864253520965576,
-      "logps/chosen": -282.04022216796875,
-      "logps/rejected": -316.0839538574219,
-      "loss": 0.1939,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.027098428457975388,
-      "rewards/margins": 0.1554117500782013,
-      "rewards/rejected": -0.1825101673603058,
+      "logits/chosen": -2.877237319946289,
+      "logits/rejected": -2.864227533340454,
+      "logps/chosen": -282.114501953125,
+      "logps/rejected": -316.05438232421875,
+      "loss": 0.1937,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.02784130536019802,
+      "rewards/margins": 0.15437331795692444,
+      "rewards/rejected": -0.1822146326303482,
       "step": 50
     },
     {
       "epoch": 1.0,
       "step": 53,
       "total_flos": 0.0,
-      "train_loss": 0.20092295110225677,
-      "train_runtime": 384.8987,
-      "train_samples_per_second": 17.537,
+      "train_loss": 0.2009538251836345,
+      "train_runtime": 384.7218,
+      "train_samples_per_second": 17.545,
       "train_steps_per_second": 0.138
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8c151d7becb900bed631e41dff74cbdb1243adce5d3b22205a355b75f2b0912
-size 5944
+oid sha256:3cd92261bd298819f5c7cf60f5729c763c4e3f4d7cdf1279294fbc67f09e3a3b
+size 6456