Upload tokenizer.json
Browse files- tokenizer.json +1 -27
tokenizer.json
CHANGED
|
@@ -2324,33 +2324,6 @@
|
|
| 2324 |
"rstrip": false,
|
| 2325 |
"normalized": false,
|
| 2326 |
"special": true
|
| 2327 |
-
},
|
| 2328 |
-
{
|
| 2329 |
-
"id": 128000,
|
| 2330 |
-
"content": "<|user|>",
|
| 2331 |
-
"single_word": false,
|
| 2332 |
-
"lstrip": false,
|
| 2333 |
-
"rstrip": false,
|
| 2334 |
-
"normalized": false,
|
| 2335 |
-
"special": true
|
| 2336 |
-
},
|
| 2337 |
-
{
|
| 2338 |
-
"id": 128001,
|
| 2339 |
-
"content": "<|system|>",
|
| 2340 |
-
"single_word": false,
|
| 2341 |
-
"lstrip": false,
|
| 2342 |
-
"rstrip": false,
|
| 2343 |
-
"normalized": false,
|
| 2344 |
-
"special": true
|
| 2345 |
-
},
|
| 2346 |
-
{
|
| 2347 |
-
"id": 128002,
|
| 2348 |
-
"content": "<|assistant|>",
|
| 2349 |
-
"single_word": false,
|
| 2350 |
-
"lstrip": false,
|
| 2351 |
-
"rstrip": false,
|
| 2352 |
-
"normalized": false,
|
| 2353 |
-
"special": true
|
| 2354 |
}
|
| 2355 |
],
|
| 2356 |
"normalizer": null,
|
|
@@ -2393,6 +2366,7 @@
|
|
| 2393 |
"end_of_word_suffix": null,
|
| 2394 |
"fuse_unk": false,
|
| 2395 |
"byte_fallback": false,
|
|
|
|
| 2396 |
"vocab": {
|
| 2397 |
"<|begin_of_text|>": 0,
|
| 2398 |
"<|end_of_text|>": 1,
|
|
|
|
| 2324 |
"rstrip": false,
|
| 2325 |
"normalized": false,
|
| 2326 |
"special": true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2327 |
}
|
| 2328 |
],
|
| 2329 |
"normalizer": null,
|
|
|
|
| 2366 |
"end_of_word_suffix": null,
|
| 2367 |
"fuse_unk": false,
|
| 2368 |
"byte_fallback": false,
|
| 2369 |
+
"ignore_merges": true,
|
| 2370 |
"vocab": {
|
| 2371 |
"<|begin_of_text|>": 0,
|
| 2372 |
"<|end_of_text|>": 1,
|