KaleiNeely commited on
Commit
d51b8af
1 Parent(s): 9f2aa8f

Update tokenization_rwkv_world.py

Browse files
Files changed (1) hide show
  1. tokenization_rwkv_world.py +5 -1
tokenization_rwkv_world.py CHANGED
@@ -281,7 +281,11 @@ class RWKVWorldTokenizer(PreTrainedTokenizer):
281
  )
282
 
283
  with open(vocab_file, "w", encoding="utf-8") as f:
284
- f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")
 
 
 
 
285
 
286
  return (vocab_file,)
287
 
 
281
  )
282
 
283
  with open(vocab_file, "w", encoding="utf-8") as f:
284
+ for idx, x in self.encoder.items():
285
+ if isinstance(x, str):
286
+ x = x.decode("utf-8")
287
+ line = f"{idx} {repr(x)} {len(x)}\n"
288
+ f.write(line)
289
 
290
  return (vocab_file,)
291