vocab/baichuan2/__init__.py

import os
import config
from transformers import AutoTokenizer
from vocab import TokenizerType





if config.USE_REMOTE:
    tokenizer = AutoTokenizer.from_pretrained("baichuan-inc/Baichuan2-7B-Chat", trust_remote_code=True)
else:
    CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
    TOKENIZER_DIR = os.path.join(CURRENT_DIR, "Baichuan2-7B-Chat")
    tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR, trust_remote_code=True)


# byte-bpe  sentencepiece
tokenizer.type = TokenizerType.ByteBPE

tokenizer.comments = "expand the vocabulary size from 64000 in Baichuan1 to 125696"