gokaygokay
commited on
Commit
•
a66f4c7
1
Parent(s):
b656627
Update tokenization_InternLM_XComposer.py
Browse files
tokenization_InternLM_XComposer.py
CHANGED
@@ -63,6 +63,8 @@ class InternLMXComposerTokenizer(PreTrainedTokenizer):
|
|
63 |
**kwargs,
|
64 |
):
|
65 |
self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
|
|
|
|
|
66 |
super().__init__(
|
67 |
bos_token=bos_token,
|
68 |
eos_token=eos_token,
|
@@ -74,9 +76,7 @@ class InternLMXComposerTokenizer(PreTrainedTokenizer):
|
|
74 |
self.vocab_file = vocab_file
|
75 |
self.add_bos_token = add_bos_token
|
76 |
self.add_eos_token = add_eos_token
|
77 |
-
self.decode_with_prefix_space = decode_with_prefix_space
|
78 |
-
self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
|
79 |
-
self.sp_model.Load(vocab_file)
|
80 |
self._no_prefix_space_tokens = None
|
81 |
|
82 |
""" Initialisation"""
|
|
|
63 |
**kwargs,
|
64 |
):
|
65 |
self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
|
66 |
+
self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
|
67 |
+
self.sp_model.Load(vocab_file)
|
68 |
super().__init__(
|
69 |
bos_token=bos_token,
|
70 |
eos_token=eos_token,
|
|
|
76 |
self.vocab_file = vocab_file
|
77 |
self.add_bos_token = add_bos_token
|
78 |
self.add_eos_token = add_eos_token
|
79 |
+
self.decode_with_prefix_space = decode_with_prefix_space
|
|
|
|
|
80 |
self._no_prefix_space_tokens = None
|
81 |
|
82 |
""" Initialisation"""
|