update

README.md

@@ -15,7 +15,10 @@ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-
 ## ss
 
 
-## ss
+## TODO
+
+
+'MossTokenizer' object has no attribute 'encoder'
 
 
 

vocab/moss/__init__.py

@@ -2,10 +2,11 @@
 import os
 from transformers import AutoTokenizer, BloomTokenizerFast
 
-CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
-TOKENIZER_DIR = os.path.join(CURRENT_DIR, "moss-moon-003-sft")
+# CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+# TOKENIZER_DIR = os.path.join(CURRENT_DIR, "moss-moon-003-sft")
+# tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR, trust_remote_code=True)
 
-tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR, trust_remote_code=True)
+tokenizer = AutoTokenizer.from_pretrained("fnlp/moss-moon-003-sft", trust_remote_code=True)
 
 # vocab_size = len(tokenizer.get_vocab())
 # vocab_size = tokenizer.vocab_size

vocab/qwen_7b_chat/__init__.py

@@ -9,12 +9,13 @@ https://github.com/QwenLM/Qwen/blob/main/tokenization_note_zh.md
 
 import os
 from transformers import AutoTokenizer
-CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
-TOKENIZER_DIR = os.path.join(CURRENT_DIR, "Qwen-7B-Chat")
 
 # 请注意：分词器默认行为已更改为默认关闭特殊token攻击防护。
-# tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen-VL-Chat", trust_remote_code=True)
-tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR, trust_remote_code=True)
+tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen-7B-Chat", trust_remote_code=True)
+
+# CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+# TOKENIZER_DIR = os.path.join(CURRENT_DIR, "Qwen-7B-Chat")
+# tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR, trust_remote_code=True)
 
 tokenizer.comments = "在gpt4词典基础上，删除了100个多数字token，增加10000中文词token；并优化了special_token的分词"
 

vocab/skywork_13b_base/__init__.py

@@ -0,0 +1,6 @@
+
+from transformers import AutoTokenizer
+
+
+
+tokenizer = AutoTokenizer.from_pretrained("Skywork/Skywork-13B-base", trust_remote_code=True)

vocab/skywork_13b_math/__init__.py

@@ -0,0 +1,4 @@
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("Skywork/Skywork-13B-Math", trust_remote_code=True)