vocab/gpt_35_turbo/__init__.py

"""

"""

import tiktoken
from patcher import tiktoken_patch

tokenizer = tiktoken.encoding_for_model('gpt-3.5-turbo')

tokenizer.comments = "tiktoken is a fast BPE tokeniser for use with OpenAI's models. There are 16 tokens KeyError"
tokenizer.reversible = True  # It's reversible and lossless, so you can convert tokens back into the original text