add decoder to tokenizer config
Browse files- tokenizer.json +2 -2
tokenizer.json
CHANGED
@@ -19,7 +19,7 @@
|
|
19 |
"special": true,
|
20 |
"content": "<mask>",
|
21 |
"single_word": false,
|
22 |
-
"lstrip":
|
23 |
"rstrip": false,
|
24 |
"normalized": false
|
25 |
},
|
@@ -81,7 +81,7 @@
|
|
81 |
"trim_offsets": true,
|
82 |
"add_prefix_space": true
|
83 |
},
|
84 |
-
"decoder":
|
85 |
"model": {
|
86 |
"type": "BPE",
|
87 |
"dropout": null,
|
|
|
19 |
"special": true,
|
20 |
"content": "<mask>",
|
21 |
"single_word": false,
|
22 |
+
"lstrip": true,
|
23 |
"rstrip": false,
|
24 |
"normalized": false
|
25 |
},
|
|
|
81 |
"trim_offsets": true,
|
82 |
"add_prefix_space": true
|
83 |
},
|
84 |
+
"decoder":{"type":"ByteLevel","add_prefix_space":true,"trim_offsets":true},
|
85 |
"model": {
|
86 |
"type": "BPE",
|
87 |
"dropout": null,
|