update tokenizer according to https://huggingface.co/baichuan-inc/Baichuan2-13B-Chat/discussions/18

tokenization_baichuan.py

@@ -68,6 +68,11 @@ class BaichuanTokenizer(PreTrainedTokenizer):
             if isinstance(pad_token, str)
             else pad_token
         )
+        self.vocab_file = vocab_file
+        self.add_bos_token = add_bos_token
+        self.add_eos_token = add_eos_token
+        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
+        self.sp_model.Load(vocab_file)
         super().__init__(
             bos_token=bos_token,
             eos_token=eos_token,
@@ -79,11 +84,6 @@ class BaichuanTokenizer(PreTrainedTokenizer):
             clean_up_tokenization_spaces=clean_up_tokenization_spaces,
             **kwargs,
         )
-        self.vocab_file = vocab_file
-        self.add_bos_token = add_bos_token
-        self.add_eos_token = add_eos_token
-        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
-        self.sp_model.Load(vocab_file)
 
     def __getstate__(self):
         state = self.__dict__.copy()