hadiqa123 commited on
Commit
2d2f570
1 Parent(s): 21f6f5f

Upload tokenizer

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +1 -7
  2. tokenizer_config.json +0 -0
special_tokens_map.json CHANGED
@@ -122,13 +122,7 @@
122
  "rstrip": false,
123
  "single_word": false
124
  },
125
- "pad_token": {
126
- "content": "<|endoftext|>",
127
- "lstrip": false,
128
- "normalized": false,
129
- "rstrip": false,
130
- "single_word": false
131
- },
132
  "unk_token": {
133
  "content": "<|endoftext|>",
134
  "lstrip": false,
 
122
  "rstrip": false,
123
  "single_word": false
124
  },
125
+ "pad_token": "<|endoftext|>",
 
 
 
 
 
 
126
  "unk_token": {
127
  "content": "<|endoftext|>",
128
  "lstrip": false,
tokenizer_config.json CHANGED
The diff for this file is too large to render. See raw diff