Qubitium commited on
Commit
5135d75
1 Parent(s): 7fb86d8

Update tiktoken.py (#4)

Browse files

- Update tiktoken.py (345342df4cc7b09196ac8c6d8de5615037638ceb)

Files changed (1) hide show
  1. tiktoken.py +2 -2
tiktoken.py CHANGED
@@ -247,7 +247,7 @@ class TiktokenTokenizerWrapper(PreTrainedTokenizer):
247
  # Get an index to add and add the item
248
  vocab_clone[candidate_extra_id] = index_to_add
249
 
250
- return vocab_clone
251
 
252
  def _tokenize(self, text: str) -> List[str]:
253
  """Returns a tokenized string."""
@@ -371,4 +371,4 @@ class TiktokenTokenizerWrapper(PreTrainedTokenizer):
371
  if len(encoded) > 1:
372
  actual_new_tokens.append(token)
373
 
374
- return self.add_tokens(actual_new_tokens, special_tokens=True)
 
247
  # Get an index to add and add the item
248
  vocab_clone[candidate_extra_id] = index_to_add
249
 
250
+ return dict(vocab_clone, **self.added_tokens_encoder)
251
 
252
  def _tokenize(self, text: str) -> List[str]:
253
  """Returns a tokenized string."""
 
371
  if len(encoded) > 1:
372
  actual_new_tokens.append(token)
373
 
374
+ return self.add_tokens(actual_new_tokens, special_tokens=True)