File size: 132 Bytes
f4973d4 |
1 2 3 4 5 6 7 |
from vocab.chinese_llama2 import tokenizer
encoding = tokenizer.encode("<s>开始</s>站位符<pad>试试<unk>")
print(encoding)
|
f4973d4 |
1 2 3 4 5 6 7 |
from vocab.chinese_llama2 import tokenizer
encoding = tokenizer.encode("<s>开始</s>站位符<pad>试试<unk>")
print(encoding)
|