CharlieFRuan
commited on
Commit
•
de7411b
1
Parent(s):
8592f44
Update mlc-chat-config.json
Browse files- mlc-chat-config.json +5 -0
mlc-chat-config.json
CHANGED
@@ -71,5 +71,10 @@
|
|
71 |
"tokenizer.json",
|
72 |
"tokenizer_config.json"
|
73 |
],
|
|
|
|
|
|
|
|
|
|
|
74 |
"version": "0.1.0"
|
75 |
}
|
|
|
71 |
"tokenizer.json",
|
72 |
"tokenizer_config.json"
|
73 |
],
|
74 |
+
"tokenizer_info": {
|
75 |
+
"token_postproc_method": "byte_level",
|
76 |
+
"prepend_space_in_encode": false,
|
77 |
+
"strip_space_in_decode": false
|
78 |
+
},
|
79 |
"version": "0.1.0"
|
80 |
}
|