| """ | |
| """ | |
| import tiktoken | |
| import tokenizer.tiktoken_patch | |
| tokenizer = tiktoken.encoding_for_model('gpt-3.5-turbo') | |
| tokenizer.vocab_size = tokenizer.n_vocab | |
| tokenizer.comments = "tiktoken is a fast BPE tokeniser for use with OpenAI's models. There are 16 tokens KeyError" | |
| tokenizer.reversible = True # It's reversible and lossless, so you can convert tokens back into the original text | |