alespalla / tokenizer.py
GPT-14's picture
first commit
6afee81
raw
history blame contribute delete
No virus
122 Bytes
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("gpt2")
tokenizer.save_pretrained("./")