BBuf commited on
Commit
4f41121
1 Parent(s): e6b8584

Update tokenization_rwkv_world.py

Browse files
Files changed (1) hide show
  1. tokenization_rwkv_world.py +1 -1
tokenization_rwkv_world.py CHANGED
@@ -149,7 +149,7 @@ class RWKVWorldTokenizer(PreTrainedTokenizer):
149
  def get_vocab(self):
150
  return dict(self.encoder, **self.added_tokens_encoder)
151
 
152
- def add_tokens(self, new_tokens):
153
  for token in new_tokens:
154
  token_id = self.convert_tokens_to_ids(token)
155
  self.added_tokens_decoder[token_id] = token
 
149
  def get_vocab(self):
150
  return dict(self.encoder, **self.added_tokens_encoder)
151
 
152
+ def add_tokens(self, new_tokens, special_tokens: bool = False):
153
  for token in new_tokens:
154
  token_id = self.convert_tokens_to_ids(token)
155
  self.added_tokens_decoder[token_id] = token