Chaewon commited on
Commit
7288717
1 Parent(s): 9f0e2a9

add tokenizer

Browse files
Files changed (3) hide show
  1. added_tokens.json +1 -1
  2. special_tokens_map.json +1 -1
  3. tokenizer.json +0 -0
added_tokens.json CHANGED
@@ -1 +1 @@
1
- {"<pad>": 50259, "<s>": 50257, "</s>": 50258}
 
1
+ {"<pad>": 50259, "</s>": 50258, "<s>": 50257}
special_tokens_map.json CHANGED
@@ -1 +1 @@
1
- {"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>"}
 
1
+ {"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<|endoftext|>", "pad_token": "<pad>", "additional_special_tokens": ["<s>", "</s>", "<pad>"]}
tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff