danielhanchen commited on
Commit
1d9acd8
1 Parent(s): 07eb824

Upload tokenizer

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +2 -2
  2. tokenizer_config.json +1 -1
special_tokens_map.json CHANGED
@@ -14,10 +14,10 @@
14
  "single_word": false
15
  },
16
  "pad_token": {
17
- "content": "<|endoftext|>",
18
  "lstrip": false,
19
  "normalized": false,
20
- "rstrip": false,
21
  "single_word": false
22
  },
23
  "unk_token": {
 
14
  "single_word": false
15
  },
16
  "pad_token": {
17
+ "content": "<|placeholder6|>",
18
  "lstrip": false,
19
  "normalized": false,
20
+ "rstrip": true,
21
  "single_word": false
22
  },
23
  "unk_token": {
tokenizer_config.json CHANGED
@@ -121,7 +121,7 @@
121
  "eos_token": "<|endoftext|>",
122
  "legacy": false,
123
  "model_max_length": 4096,
124
- "pad_token": "<|endoftext|>",
125
  "padding_side": "left",
126
  "sp_model_kwargs": {},
127
  "tokenizer_class": "LlamaTokenizer",
 
121
  "eos_token": "<|endoftext|>",
122
  "legacy": false,
123
  "model_max_length": 4096,
124
+ "pad_token": "<|placeholder6|>",
125
  "padding_side": "left",
126
  "sp_model_kwargs": {},
127
  "tokenizer_class": "LlamaTokenizer",