phueb commited on
Commit
b74aa12
1 Parent(s): 1c0595e

add decoder to tokenizer config

Browse files
Files changed (1) hide show
  1. tokenizer.json +2 -2
tokenizer.json CHANGED
@@ -19,7 +19,7 @@
19
  "special": true,
20
  "content": "<mask>",
21
  "single_word": false,
22
- "lstrip": false,
23
  "rstrip": false,
24
  "normalized": false
25
  },
@@ -81,7 +81,7 @@
81
  "trim_offsets": true,
82
  "add_prefix_space": true
83
  },
84
- "decoder": null,
85
  "model": {
86
  "type": "BPE",
87
  "dropout": null,
 
19
  "special": true,
20
  "content": "<mask>",
21
  "single_word": false,
22
+ "lstrip": true,
23
  "rstrip": false,
24
  "normalized": false
25
  },
 
81
  "trim_offsets": true,
82
  "add_prefix_space": true
83
  },
84
+ "decoder":{"type":"ByteLevel","add_prefix_space":true,"trim_offsets":true},
85
  "model": {
86
  "type": "BPE",
87
  "dropout": null,