| import os | |
| from transformers import GPT2Tokenizer | |
| from vocab import TokenizerType, TokenizerImpl | |
| # CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) | |
| # TOKENIZER_DIR = os.path.join(CURRENT_DIR, "tokenizer") | |
| # tokenizer = GPT2Tokenizer.from_pretrained(TOKENIZER_DIR) | |
| tokenizer = GPT2Tokenizer.from_pretrained("openai-community/gpt2") | |
| # tokenizer.type = TokenizerType. | |
| # 源码 https://github.com/huggingface/transformers/blob/main/src/transformers/models/gpt2/tokenization_gpt2.py | |