Upload tokenizer.json with huggingface_hub
Browse files- tokenizer.json +6 -28
tokenizer.json
CHANGED
|
@@ -1,6 +1,11 @@
|
|
| 1 |
{
|
| 2 |
"version": "1.0",
|
| 3 |
-
"truncation":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
"padding": null,
|
| 5 |
"added_tokens": [
|
| 6 |
{
|
|
@@ -299,33 +304,6 @@
|
|
| 299 |
"rstrip": false,
|
| 300 |
"normalized": false,
|
| 301 |
"special": true
|
| 302 |
-
},
|
| 303 |
-
{
|
| 304 |
-
"id": 100289,
|
| 305 |
-
"content": "<s>",
|
| 306 |
-
"single_word": false,
|
| 307 |
-
"lstrip": false,
|
| 308 |
-
"rstrip": false,
|
| 309 |
-
"normalized": false,
|
| 310 |
-
"special": true
|
| 311 |
-
},
|
| 312 |
-
{
|
| 313 |
-
"id": 100290,
|
| 314 |
-
"content": "</s>",
|
| 315 |
-
"single_word": false,
|
| 316 |
-
"lstrip": false,
|
| 317 |
-
"rstrip": false,
|
| 318 |
-
"normalized": false,
|
| 319 |
-
"special": true
|
| 320 |
-
},
|
| 321 |
-
{
|
| 322 |
-
"id": 100291,
|
| 323 |
-
"content": "<unk>",
|
| 324 |
-
"single_word": false,
|
| 325 |
-
"lstrip": false,
|
| 326 |
-
"rstrip": false,
|
| 327 |
-
"normalized": false,
|
| 328 |
-
"special": true
|
| 329 |
}
|
| 330 |
],
|
| 331 |
"normalizer": null,
|
|
|
|
| 1 |
{
|
| 2 |
"version": "1.0",
|
| 3 |
+
"truncation": {
|
| 4 |
+
"direction": "Right",
|
| 5 |
+
"max_length": 2048,
|
| 6 |
+
"strategy": "LongestFirst",
|
| 7 |
+
"stride": 0
|
| 8 |
+
},
|
| 9 |
"padding": null,
|
| 10 |
"added_tokens": [
|
| 11 |
{
|
|
|
|
| 304 |
"rstrip": false,
|
| 305 |
"normalized": false,
|
| 306 |
"special": true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 307 |
}
|
| 308 |
],
|
| 309 |
"normalizer": null,
|