CharlieFRuan commited on
Commit
f35dfac
1 Parent(s): 91b7588

Update mlc-chat-config.json

Browse files
Files changed (1) hide show
  1. mlc-chat-config.json +5 -0
mlc-chat-config.json CHANGED
@@ -71,5 +71,10 @@
71
  "tokenizer.json",
72
  "tokenizer_config.json"
73
  ],
 
 
 
 
 
74
  "version": "0.1.0"
75
  }
 
71
  "tokenizer.json",
72
  "tokenizer_config.json"
73
  ],
74
+ "tokenizer_info": {
75
+ "token_postproc_method": "byte_level",
76
+ "prepend_space_in_encode": false,
77
+ "strip_space_in_decode": false
78
+ },
79
  "version": "0.1.0"
80
  }