root
commited on
Commit
•
7751418
1
Parent(s):
c88fe71
added truncation
Browse files- tokenizer_config.json +2 -1
tokenizer_config.json
CHANGED
@@ -15,5 +15,6 @@
|
|
15 |
"pad_token": "<pad>",
|
16 |
"sep_token": "</s>",
|
17 |
"tokenizer_class": "XLMRobertaTokenizer",
|
18 |
-
"unk_token": "<unk>"
|
|
|
19 |
}
|
|
|
15 |
"pad_token": "<pad>",
|
16 |
"sep_token": "</s>",
|
17 |
"tokenizer_class": "XLMRobertaTokenizer",
|
18 |
+
"unk_token": "<unk>",
|
19 |
+
"truncation": true
|
20 |
}
|