Update tokenization_InternLM_XComposer.py

tokenization_InternLM_XComposer.py

@@ -63,7 +63,13 @@ class InternLMXComposerTokenizer(PreTrainedTokenizer):
         **kwargs,
     ):
         self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
-        
+        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
+        self.sp_model.Load(vocab_file)
+        self.vocab_file = vocab_file
+        self.add_bos_token = add_bos_token
+        self.add_eos_token = add_eos_token
+        self.decode_with_prefix_space = decode_with_prefix_space        
+        self._no_prefix_space_tokens = None
         
         super().__init__(
             bos_token=bos_token,
@@ -73,13 +79,7 @@ class InternLMXComposerTokenizer(PreTrainedTokenizer):
             clean_up_tokenization_spaces=clean_up_tokenization_spaces,
             **kwargs,
         )
-        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
-        self.sp_model.Load(vocab_file)
-        self.vocab_file = vocab_file
-        self.add_bos_token = add_bos_token
-        self.add_eos_token = add_eos_token
-        self.decode_with_prefix_space = decode_with_prefix_space        
-        self._no_prefix_space_tokens = None
+        
 
         """ Initialisation"""