from transformers import AutoTokenizer | |
tokenizer = AutoTokenizer.from_pretrained("your-model-name") | |
tokenizer.save_pretrained("model_directory") |
from transformers import AutoTokenizer | |
tokenizer = AutoTokenizer.from_pretrained("your-model-name") | |
tokenizer.save_pretrained("model_directory") |