Update tokenization_rwkv_world.py
Browse files
tokenization_rwkv_world.py
CHANGED
@@ -149,7 +149,7 @@ class RWKVWorldTokenizer(PreTrainedTokenizer):
|
|
149 |
def get_vocab(self):
|
150 |
return dict(self.encoder, **self.added_tokens_encoder)
|
151 |
|
152 |
-
def add_tokens(self, new_tokens):
|
153 |
for token in new_tokens:
|
154 |
token_id = self.convert_tokens_to_ids(token)
|
155 |
self.added_tokens_decoder[token_id] = token
|
|
|
149 |
def get_vocab(self):
|
150 |
return dict(self.encoder, **self.added_tokens_encoder)
|
151 |
|
152 |
+
def add_tokens(self, new_tokens, special_tokens: bool = False):
|
153 |
for token in new_tokens:
|
154 |
token_id = self.convert_tokens_to_ids(token)
|
155 |
self.added_tokens_decoder[token_id] = token
|