Model save

README.md

@@ -2,15 +2,11 @@
 license: apache-2.0
 base_model: Qwen/Qwen2-0.5B
 tags:
-- alignment-handbook
-- trl
-- sft
-- generated_from_trainer
 - trl
 - sft
 - generated_from_trainer
 datasets:
-- HuggingFaceH4/ultrachat_200k
+- generator
 model-index:
 - name: qwen2-0.5b-sft
   results: []
@@ -19,12 +15,12 @@ model-index:
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/zhaoyang1/huggingface/runs/h6gigsup)
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/zhaoyang1/huggingface/runs/n1cgl3ti)
 # qwen2-0.5b-sft
 
-This model is a fine-tuned version of [Qwen/Qwen2-0.5B](https://huggingface.co/Qwen/Qwen2-0.5B) on the HuggingFaceH4/ultrachat_200k dataset.
+This model is a fine-tuned version of [Qwen/Qwen2-0.5B](https://huggingface.co/Qwen/Qwen2-0.5B) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.5079
+- Loss: 1.5269
 
 ## Model description
 
@@ -62,7 +58,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss |
 |:-------------:|:------:|:----:|:---------------:|
-| 1.5055        | 0.9997 | 1911 | 1.5079          |
+| 1.524         | 0.9999 | 1888 | 1.5269          |
 
 
 ### Framework versions

all_results.json

@@ -1,14 +1,9 @@
 {
-    "epoch": 0.999738425320429,
-    "eval_loss": 1.5079056024551392,
-    "eval_runtime": 417.8091,
-    "eval_samples": 23109,
-    "eval_samples_per_second": 64.802,
-    "eval_steps_per_second": 4.052,
-    "total_flos": 107512363745280.0,
-    "train_loss": 1.5265680355653908,
-    "train_runtime": 15553.3872,
+    "epoch": 0.9998676022772408,
+    "total_flos": 106218135748608.0,
+    "train_loss": 1.5447227579809852,
+    "train_runtime": 15813.94,
     "train_samples": 207864,
-    "train_samples_per_second": 15.73,
-    "train_steps_per_second": 0.123
+    "train_samples_per_second": 15.283,
+    "train_steps_per_second": 0.119
 }

config.json

@@ -4,8 +4,8 @@
     "Qwen2ForCausalLM"
   ],
   "attention_dropout": 0.0,
-  "bos_token_id": 151643,
-  "eos_token_id": 151643,
+  "bos_token_id": 151644,
+  "eos_token_id": 151645,
   "hidden_act": "silu",
   "hidden_size": 896,
   "initializer_range": 0.02,
@@ -16,13 +16,14 @@
   "num_attention_heads": 14,
   "num_hidden_layers": 24,
   "num_key_value_heads": 2,
+  "pad_token_id": 151645,
   "rms_norm_eps": 1e-06,
   "rope_theta": 1000000.0,
   "sliding_window": 131072,
   "tie_word_embeddings": true,
   "torch_dtype": "float16",
   "transformers_version": "4.42.0",
-  "use_cache": true,
+  "use_cache": false,
   "use_sliding_window": false,
-  "vocab_size": 151936
+  "vocab_size": 151646
 }

generation_config.json

@@ -1,6 +1,7 @@
 {
-  "bos_token_id": 151643,
-  "eos_token_id": 151643,
+  "bos_token_id": 151644,
+  "eos_token_id": 151645,
   "max_new_tokens": 2048,
+  "pad_token_id": 151645,
   "transformers_version": "4.42.0"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b385b5be4285f65f53e7bbddf58fecb4441ec7b33cdee2abd225ff77e34f2f5a
-size 988097536
+oid sha256:4828a8f271d2dfaa510a80ad20ce65d9f92dcdcbd95b7c6dfad5614cfeacb97b
+size 987577856

special_tokens_map.json

@@ -1,20 +1,21 @@
 {
   "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>"
+    {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
   ],
-  "eos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
+  "bos_token": "<|im_start|>",
+  "eos_token": "<|im_end|>",
+  "pad_token": "<|im_end|>"
 }

tokenizer_config.json

@@ -30,13 +30,13 @@
     "<|im_start|>",
     "<|im_end|>"
   ],
-  "bos_token": null,
-  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|endoftext|>",
+  "eos_token": "<|im_end|>",
   "errors": "replace",
   "model_max_length": 32768,
-  "pad_token": "<|endoftext|>",
+  "pad_token": "<|im_end|>",
   "split_special_tokens": false,
   "tokenizer_class": "Qwen2Tokenizer",
   "unk_token": null

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 0.999738425320429,
-    "total_flos": 107512363745280.0,
-    "train_loss": 1.5265680355653908,
-    "train_runtime": 15553.3872,
+    "epoch": 0.9998676022772408,
+    "total_flos": 106218135748608.0,
+    "train_loss": 1.5447227579809852,
+    "train_runtime": 15813.94,
     "train_samples": 207864,
-    "train_samples_per_second": 15.73,
-    "train_steps_per_second": 0.123
+    "train_samples_per_second": 15.283,
+    "train_steps_per_second": 0.119
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f492d9e15d1c86f0ab5946584cc5b2239088843e04a1809e6dcc8352888ca551
+oid sha256:71a42492410252f3c7b6d2b0c63f745b17658742bce5e32808b2a14e95017cdb
 size 7032