| from transformers import AutoTokenizer | |
| import argparse | |
| import os | |
| os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python" | |
| parser = argparse.ArgumentParser(description='convert slow tokenizer to fast') | |
| parser.add_argument('model_dir', type=str, help='Local model folder') | |
| args = parser.parse_args() | |
| tokenizer = AutoTokenizer.from_pretrained(args.model_dir) | |
| tokenizer.save_pretrained(args.model_dir) | |