gokaygokay commited on
Commit
9e0eb41
1 Parent(s): 603fd34

Update tokenization_InternLM_XComposer.py

Browse files
Files changed (1) hide show
  1. tokenization_InternLM_XComposer.py +2 -1
tokenization_InternLM_XComposer.py CHANGED
@@ -63,7 +63,7 @@ class InternLMXComposerTokenizer(PreTrainedTokenizer):
63
  **kwargs,
64
  ):
65
  self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
66
- self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
67
 
68
  super().__init__(
69
  bos_token=bos_token,
@@ -73,6 +73,7 @@ class InternLMXComposerTokenizer(PreTrainedTokenizer):
73
  clean_up_tokenization_spaces=clean_up_tokenization_spaces,
74
  **kwargs,
75
  )
 
76
  self.sp_model.Load(vocab_file)
77
  self.vocab_file = vocab_file
78
  self.add_bos_token = add_bos_token
 
63
  **kwargs,
64
  ):
65
  self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
66
+
67
 
68
  super().__init__(
69
  bos_token=bos_token,
 
73
  clean_up_tokenization_spaces=clean_up_tokenization_spaces,
74
  **kwargs,
75
  )
76
+ self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
77
  self.sp_model.Load(vocab_file)
78
  self.vocab_file = vocab_file
79
  self.add_bos_token = add_bos_token