Fix the arguments
Browse files- tokenizer.json +41 -3
tokenizer.json
CHANGED
@@ -216,11 +216,49 @@
|
|
216 |
"post_processor": {
|
217 |
"type": "TemplateProcessing",
|
218 |
"single": [
|
219 |
-
|
220 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
221 |
],
|
222 |
"special_tokens": {
|
223 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
224 |
}
|
225 |
},
|
226 |
"decoder": {
|
|
|
216 |
"post_processor": {
|
217 |
"type": "TemplateProcessing",
|
218 |
"single": [
|
219 |
+
{
|
220 |
+
"SpecialToken": {
|
221 |
+
"id": "<|begin_of_text|>",
|
222 |
+
"type_id": 0
|
223 |
+
}
|
224 |
+
},
|
225 |
+
{
|
226 |
+
"Sequence": {
|
227 |
+
"id": "A",
|
228 |
+
"type_id": 0
|
229 |
+
}
|
230 |
+
}
|
231 |
+
],
|
232 |
+
"pair": [
|
233 |
+
{
|
234 |
+
"SpecialToken": {
|
235 |
+
"id": "<|begin_of_text|>",
|
236 |
+
"type_id": 0
|
237 |
+
}
|
238 |
+
},
|
239 |
+
{
|
240 |
+
"Sequence": {
|
241 |
+
"id": "A",
|
242 |
+
"type_id": 0
|
243 |
+
}
|
244 |
+
},
|
245 |
+
{
|
246 |
+
"Sequence": {
|
247 |
+
"id": "B",
|
248 |
+
"type_id": 1
|
249 |
+
}
|
250 |
+
}
|
251 |
],
|
252 |
"special_tokens": {
|
253 |
+
"<|begin_of_text|>": {
|
254 |
+
"id": "<|begin_of_text|>",
|
255 |
+
"ids": [
|
256 |
+
100256
|
257 |
+
],
|
258 |
+
"tokens": [
|
259 |
+
"<|begin_of_text|>"
|
260 |
+
]
|
261 |
+
}
|
262 |
}
|
263 |
},
|
264 |
"decoder": {
|