File size: 199 Bytes
f4973d4 |
1 2 3 4 5 6 7 8 9 10 11 |
from vocab.gpt_35_turbo import tokenizer
print(tokenizer.get_vocab())
text = "中"
token_ids = tokenizer.encode(text)
decode_str = tokenizer.convert_tokens_to_string(token_ids)
print(decode_str)
|