vocab/gpt_35_turbo/decode_test.py

from vocab.gpt_35_turbo import tokenizer


text = "你好，请告诉我聚乙烯是什么"
encoding = tokenizer.encode(text)


print(tokenizer.decode([6744]))
print(tokenizer.convert_ids_to_tokens([6744]))

print(tokenizer.decode([100256]))   # 是没有这个token吗？
print(tokenizer.convert_ids_to_tokens([100256]))


print(tokenizer.decode([100262]))
print(tokenizer.convert_ids_to_tokens([100262]))

print(tokenizer.decode([100273]))
print(tokenizer.convert_ids_to_tokens([100273]))