KaleiNeely commited on
Commit
8c48767
1 Parent(s): 2d52c88

Update tokenization_rwkv_world.py

Browse files
Files changed (1) hide show
  1. tokenization_rwkv_world.py +2 -1
tokenization_rwkv_world.py CHANGED
@@ -202,7 +202,8 @@ class RWKVWorldTokenizer(PreTrainedTokenizer):
202
  return tokens
203
 
204
  def decodeBytes(self, tokens):
205
- return b''.join(map(lambda i: self.encoder[i], tokens))
 
206
 
207
  def _tokenize(self, text, **kwargs):
208
  """Tokenize a string."""
 
202
  return tokens
203
 
204
  def decodeBytes(self, tokens):
205
+ byte_sequence = [self.encoder[i] for i in tokens if i != 0]
206
+ return b''.join(byte_sequence)
207
 
208
  def _tokenize(self, text, **kwargs):
209
  """Tokenize a string."""