fix moss

vocab/moss/moss-moon-003-sft/tokenization_moss.py

@@ -146,6 +146,11 @@ class MossTokenizer(PreTrainedTokenizer):
         eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
         unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token
         pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
+        self.add_bos_token = add_bos_token
+
+        with open(vocab_file, encoding="utf-8") as vocab_handle:
+            self.encoder = json.load(vocab_handle)
+
         super().__init__(
             errors=errors,
             unk_token=unk_token,
@@ -156,10 +161,7 @@ class MossTokenizer(PreTrainedTokenizer):
             add_bos_token=add_bos_token,
             **kwargs,
         )
-        self.add_bos_token = add_bos_token
 
-        with open(vocab_file, encoding="utf-8") as vocab_handle:
-            self.encoder = json.load(vocab_handle)
         self.decoder = {v: k for k, v in self.encoder.items()}
         self.errors = errors  # how to handle errors in decoding
         self.byte_encoder = bytes_to_unicode()