xu-song's picture
add more tokenizers
f4973d4
raw
history blame
204 Bytes
import os
from transformers import AutoTokenizer
os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm3-6b", trust_remote_code=True)