Upload tokenizer.json (#28)
Browse files- Upload tokenizer.json (d749371bfe914cc1a3a9e6bee38a38825a3c697b)
- tokenizer.json +1 -1
tokenizer.json
CHANGED
@@ -203,7 +203,7 @@
|
|
203 |
},
|
204 |
{
|
205 |
"id": 50279,
|
206 |
-
"content": "<|
|
207 |
"single_word": false,
|
208 |
"lstrip": false,
|
209 |
"rstrip": false,
|
|
|
203 |
},
|
204 |
{
|
205 |
"id": 50279,
|
206 |
+
"content": "<|he|>",
|
207 |
"single_word": false,
|
208 |
"lstrip": false,
|
209 |
"rstrip": false,
|