Upload folder using huggingface_hub

xDAN L3 100B : xDAN-APUS4-MoE-v3.1-0410

check.py

@@ -0,0 +1,67 @@
+import os
+import argparse
+from transformers import LlamaTokenizer
+import sentencepiece as spm
+
+# 设置环境变量
+os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
+
+def update_tokenizer(set_chatml, output_hf_dir="./tokenizer.model"):
+    # 从预训练路径重新加载tokenizer
+    custom_tokenizer = LlamaTokenizer.from_pretrained(output_hf_dir)
+
+    # 如果启用了set_chatml，更新bos_token和eos_token
+
+    print(f"当前bos标记:   (id: {custom_tokenizer.bos_token})")
+    print(f"当前eos标记:   (id: {custom_tokenizer.eos_token})")
+    # 统计总词汇量
+    vocab_size = len(custom_tokenizer)
+    print(f"总词汇量: {vocab_size}")
+
+
+    # 根据 --set_chatml 参数设置 bos_token 和 eos_token
+    if set_chatml:
+        print("设置Chatml格式为EOS/BOS")
+        custom_tokenizer.bos_token = "<|im_start|>"
+        custom_tokenizer.eos_token = "<|im_end|>"
+        print("BOS and EOS tokens set to '<|im_start|>' and '<|im_end|>' for ChatML.")
+    else:
+        print(f"Default BOS token: {custom_tokenizer.bos_token}, EOS token: {custom_tokenizer.eos_token}")
+    
+
+    startoftext_id = custom_tokenizer.convert_tokens_to_ids('<|startoftext|>')
+    endoftext_id = custom_tokenizer.convert_tokens_to_ids('<|endoftext|>')
+
+    print(f"当前 <|startoftext|> 索引: {startoftext_id}")
+    print(f"当前 <|endoftext|> 索引: {endoftext_id}")
+
+
+    startofs_id = custom_tokenizer.convert_tokens_to_ids('<s>')
+    endofs_id = custom_tokenizer.convert_tokens_to_ids('</s>')
+
+    print(f"当前 <s> 索引: {startofs_id}")
+    print(f"当前 </s>索引: {endofs_id}")
+
+    # 统计bos和eos标记及其数量
+    bos_id = custom_tokenizer.bos_token_id
+    eos_id = custom_tokenizer.eos_token_id
+    bos_token = custom_tokenizer.convert_ids_to_tokens(bos_id)
+    eos_token = custom_tokenizer.convert_ids_to_tokens(eos_id)
+    print(f"最终bos标记: {bos_token} (id: {custom_tokenizer.bos_token})")
+    print(f"最终eos标记: {eos_token} (id: {custom_tokenizer.eos_token})")
+
+    print("all_special_tokens: \n\n",custom_tokenizer.all_special_tokens)
+    print("all_special_ids: \n\n",custom_tokenizer.all_special_ids)
+    print("special_tokens_map: \n\n",custom_tokenizer.special_tokens_map)
+    output_hf_dir ="./save_tokenizer"
+    custom_tokenizer.save_pretrained(output_hf_dir)
+    print(f" Tokenizer has been saved to {output_hf_dir}")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="更新Tokenizer，根据需要设置BOS/EOS为ChatML格式")
+    parser.add_argument("--set_chatml", action="store_true", help="如果设置，将BOS和EOS标记用于ChatML。")
+    args = parser.parse_args()
+    
+    # 根据命令行参数更新tokenizer
+    update_tokenizer(args.set_chatml)
+

config.json

@@ -0,0 +1,34 @@
+{
+  "_name_or_path": "xDAN2099/xDAN-L2-RL-Chat-Base-v1.2",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 6,
+  "eos_token_id": 7,
+  "hidden_act": "silu",
+  "hidden_size": 7168,
+  "initializer_range": 0.02,
+  "intermediate_size": 20480,
+  "max_position_embeddings": 32768,
+  "model_type": "mixtral",
+  "num_attention_heads": 56,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 60,
+  "num_key_value_heads": 8,
+  "num_local_experts": 4,
+  "output_router_logits": false,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000000.0,
+  "router_aux_loss_coef": 0.001,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.39.2",
+  "use_cache": false,
+  "vocab_size": 64000
+}

model-00001-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e7f30c6f4cfcbd40950421fc33fde3a72840bdc88539121c1ba1627e63a971e
+size 9879789120

model-00002-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbbaea93743fd6c576eaad3224015d8bdfee7b03368b89e7583b658befc12ad2
+size 9865065968

model-00003-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad4c01858a055cc7960995778cc71925d3dc94599492a0ad0ef47f6a229594b4
+size 9806374952

model-00004-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75408df210a2387b5ca567940b458d59798a17989eb382f3e334b91bfc856feb
+size 9997201336

model-00005-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ada54dac2b9de4f5947af329d2077e20a29ec5c68764e9a11abff7218fd3f1a
+size 9967841040

model-00006-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:482cf8bcd637dc6657337a14c62e64aceed3b1c7cafe7fd460af39e117f0d0e6
+size 9806375000

model-00007-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0159ffe8ac1c41f361dc155512870276f9d49441b5fb2fe5ce2747d63b7b358a
+size 9865066016

model-00008-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d09f79b3f16853164d224cce9f9c653acca3772888ca881bce46fb5c0144c8e
+size 9806375000

model-00009-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a76dde992ac9fff0872fb85e30cd2b3780ebf30f1d7296147c7260609160a8a
+size 9806375000

model-00010-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bf5e1663e4c5ae5bd4e2fda2361e9ac837e5ad13d28be1eaaf68fae732f8d72
+size 9865066016

model-00011-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca16eb813b303abf74b01b310707cd9127945ffd6ba1583ffe54c1d90d8d1da8
+size 9806375000

model-00012-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f35c0721bd26c13dfe69144c2e4d9312f6d47fc1bcdd952367c00443ca8d0e9
+size 9997201392

model-00013-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e895d9257b9903c04e7cd18c4e325addd8e1b7a99c92380deb34c19cc40757e7
+size 9967841040

model-00014-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e607969e5a2d24eeb17189810c0a498d1df9db1a9ce117b6705691cdd61ed78
+size 9806375000

model-00015-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47c9dd3508df9cfdb3b84226814c184853df94b281282aec9724147da62aa4c9
+size 9865066016

model-00016-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a941fb43fac5cc2cd089dae7dafbb6ed99d7a6851878b664914cbcec0037f0d7
+size 9806375000

model-00017-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:541a1009d2097d35995921abcbab4a336c027d85b7ea7ba003d47f18c0b92565
+size 9806375000

model-00018-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0fd2cd18bd17bb902929954b6151404fbda56710a03980ce482a54d069d5b0c
+size 9865066016

model-00019-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbfd0c19d0b26d3ffabbecfc96e1a0a9d2d6843f956b92530addd9253f891c67
+size 9806375000

model-00020-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7c37309b1d4372ac6791eabc7ddbb018ba9865b32b318bb2901c4b959510286
+size 9997201392

model-00021-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e6bb8c4310d4bf19a6e91c467b694679b16f4b8bac0897a632a383e71645cf0
+size 9967841040

model-00022-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acd802426f28663efbdff50c5d0198e5493fcbc2f2b469e441e35b75c3c69e1a
+size 9806375000

model-00023-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93d93d5686d2b06c1ae8e5130ac68dfb93b742bd2529155633d0bba1f9319253
+size 9865066016

model-00024-of-00024.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4f709f13dfe9c5f17a223874b58feab51f88041cd1bf81b3abea7280a0528e3
+size 297048896

save_tokenizer/added_tokens.json

@@ -0,0 +1,4 @@
+{
+  "</s>": 64001,
+  "<s>": 64000
+}

save_tokenizer/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

save_tokenizer/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:386c49cf943d71aa110361135338c50e38beeff0a66593480421f37b319e1a39
+size 1033105

save_tokenizer/tokenizer_config.json

@@ -0,0 +1,42 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64000": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64001": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

special_tokens_map.json

@@ -0,0 +1,27 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>"
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|im_start|>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:386c49cf943d71aa110361135338c50e38beeff0a66593480421f37b319e1a39
+size 1033105

tokenizer_config.json

@@ -0,0 +1,65 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|startoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ '<|im_start|>system\\n' + system_message + '<|im_end|>\\n' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|im_start|>user\\n' + content + '<|im_end|>\\n<|im_start|>assistant\\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|im_end|>' + '\\n' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "legacy": true,
+  "model_max_length": 200000,
+  "pad_token": "<|im_start|>",
+  "padding_side": "left",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false,
+  "use_fast": true
+}