ocisd4
/

openllama_tokenizer_ext_zh

Model card Files Files and versions Community

samleeasus commited on Jun 2, 2023

Commit

e119f9b

•

1 Parent(s): bccccd8

Update README.md

Files changed (1) hide show

README.md +1 -1

README.md CHANGED Viewed

@@ -4,7 +4,7 @@ from transformers import LlamaTokenizer
 tokenizer = LlamaTokenizer.from_pretrained(
         'ocisd4/openllama_tokenizer_ext_zh',
         pad_token="<pad>",
-        add_bos_token=False,
         add_eos_token=True,
         use_auth_token='True',
 )

 tokenizer = LlamaTokenizer.from_pretrained(
         'ocisd4/openllama_tokenizer_ext_zh',
         pad_token="<pad>",
+        add_bos_token=True,
         add_eos_token=True,
         use_auth_token='True',
 )