| from transformers import LlamaTokenizer | |
| tokenizer = LlamaTokenizer.from_pretrained("ziqingyang/chinese-llama-2-7b") | |
| tokenizer.comments = "重新设计了新词表（大小：55296），进一步提升了中文字词的覆盖程度" | |
| from transformers import LlamaTokenizer | |
| tokenizer = LlamaTokenizer.from_pretrained("ziqingyang/chinese-llama-2-7b") | |
| tokenizer.comments = "重新设计了新词表（大小：55296），进一步提升了中文字词的覆盖程度" | |