Upload tokenizer.json with huggingface_hub
Browse files- tokenizer.json +27 -1
tokenizer.json
CHANGED
|
@@ -299,6 +299,33 @@
|
|
| 299 |
"rstrip": false,
|
| 300 |
"normalized": false,
|
| 301 |
"special": true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 302 |
}
|
| 303 |
],
|
| 304 |
"normalizer": null,
|
|
@@ -336,7 +363,6 @@
|
|
| 336 |
"end_of_word_suffix": "",
|
| 337 |
"fuse_unk": false,
|
| 338 |
"byte_fallback": false,
|
| 339 |
-
"ignore_merges": false,
|
| 340 |
"vocab": {
|
| 341 |
"!": 0,
|
| 342 |
"\"": 1,
|
|
|
|
| 299 |
"rstrip": false,
|
| 300 |
"normalized": false,
|
| 301 |
"special": true
|
| 302 |
+
},
|
| 303 |
+
{
|
| 304 |
+
"id": 100289,
|
| 305 |
+
"content": "<s>",
|
| 306 |
+
"single_word": false,
|
| 307 |
+
"lstrip": false,
|
| 308 |
+
"rstrip": false,
|
| 309 |
+
"normalized": false,
|
| 310 |
+
"special": true
|
| 311 |
+
},
|
| 312 |
+
{
|
| 313 |
+
"id": 100290,
|
| 314 |
+
"content": "</s>",
|
| 315 |
+
"single_word": false,
|
| 316 |
+
"lstrip": false,
|
| 317 |
+
"rstrip": false,
|
| 318 |
+
"normalized": false,
|
| 319 |
+
"special": true
|
| 320 |
+
},
|
| 321 |
+
{
|
| 322 |
+
"id": 100291,
|
| 323 |
+
"content": "<unk>",
|
| 324 |
+
"single_word": false,
|
| 325 |
+
"lstrip": false,
|
| 326 |
+
"rstrip": false,
|
| 327 |
+
"normalized": false,
|
| 328 |
+
"special": true
|
| 329 |
}
|
| 330 |
],
|
| 331 |
"normalizer": null,
|
|
|
|
| 363 |
"end_of_word_suffix": "",
|
| 364 |
"fuse_unk": false,
|
| 365 |
"byte_fallback": false,
|
|
|
|
| 366 |
"vocab": {
|
| 367 |
"!": 0,
|
| 368 |
"\"": 1,
|