update

vocab/moss/README.md

@@ -25,3 +25,9 @@ moss的
         text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors)
         return text
 ```
+
+
+## troubleshooting
+
+
+

vocab/moss/__init__.py

@@ -2,11 +2,11 @@
 import os
 from transformers import AutoTokenizer, BloomTokenizerFast
 
-# CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
-# TOKENIZER_DIR = os.path.join(CURRENT_DIR, "moss-moon-003-sft")
-# tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR, trust_remote_code=True)
+CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+TOKENIZER_DIR = os.path.join(CURRENT_DIR, "moss-moon-003-sft")
+tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR, trust_remote_code=True)
 
-tokenizer = AutoTokenizer.from_pretrained("fnlp/moss-moon-003-sft", trust_remote_code=True)
+# tokenizer = AutoTokenizer.from_pretrained("fnlp/moss-moon-003-sft", trust_remote_code=True)
 
 # vocab_size = len(tokenizer.get_vocab())
 # vocab_size = tokenizer.vocab_size