emonidi commited on
Commit
e6be2a2
·
1 Parent(s): 17d28b0

Upload tokenizer

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +2 -2
special_tokens_map.json CHANGED
@@ -125,14 +125,14 @@
125
  "pad_token": {
126
  "content": "<|endoftext|>",
127
  "lstrip": false,
128
- "normalized": true,
129
  "rstrip": false,
130
  "single_word": false
131
  },
132
  "unk_token": {
133
  "content": "",
134
  "lstrip": false,
135
- "normalized": false,
136
  "rstrip": false,
137
  "single_word": false
138
  }
 
125
  "pad_token": {
126
  "content": "<|endoftext|>",
127
  "lstrip": false,
128
+ "normalized": false,
129
  "rstrip": false,
130
  "single_word": false
131
  },
132
  "unk_token": {
133
  "content": "",
134
  "lstrip": false,
135
+ "normalized": true,
136
  "rstrip": false,
137
  "single_word": false
138
  }