vocab/chinese_llama2/demo.py

from vocab.chinese_llama2 import tokenizer

encoding = tokenizer.encode("<s>开始</s>站位符<pad>试试<unk>")
print(encoding)