Upload tokenizer.json
Browse files- tokenizer.json +0 -27
tokenizer.json
CHANGED
|
@@ -2324,33 +2324,6 @@
|
|
| 2324 |
"rstrip": false,
|
| 2325 |
"normalized": false,
|
| 2326 |
"special": true
|
| 2327 |
-
},
|
| 2328 |
-
{
|
| 2329 |
-
"id": 128000,
|
| 2330 |
-
"content": "<|user|>",
|
| 2331 |
-
"single_word": false,
|
| 2332 |
-
"lstrip": false,
|
| 2333 |
-
"rstrip": false,
|
| 2334 |
-
"normalized": false,
|
| 2335 |
-
"special": true
|
| 2336 |
-
},
|
| 2337 |
-
{
|
| 2338 |
-
"id": 128001,
|
| 2339 |
-
"content": "<|system|>",
|
| 2340 |
-
"single_word": false,
|
| 2341 |
-
"lstrip": false,
|
| 2342 |
-
"rstrip": false,
|
| 2343 |
-
"normalized": false,
|
| 2344 |
-
"special": true
|
| 2345 |
-
},
|
| 2346 |
-
{
|
| 2347 |
-
"id": 128002,
|
| 2348 |
-
"content": "<|assistant|>",
|
| 2349 |
-
"single_word": false,
|
| 2350 |
-
"lstrip": false,
|
| 2351 |
-
"rstrip": false,
|
| 2352 |
-
"normalized": false,
|
| 2353 |
-
"special": true
|
| 2354 |
}
|
| 2355 |
],
|
| 2356 |
"normalizer": null,
|
|
|
|
| 2324 |
"rstrip": false,
|
| 2325 |
"normalized": false,
|
| 2326 |
"special": true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2327 |
}
|
| 2328 |
],
|
| 2329 |
"normalizer": null,
|