plutokokoa commited on
Commit
2e00ae8
1 Parent(s): f9afb3b

change tokenizer config to dwdcth/chatglm3-6b-int4

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +8 -13
tokenizer_config.json CHANGED
@@ -1,17 +1,12 @@
1
  {
2
- "added_tokens_decoder": {},
 
 
 
3
  "auto_map": {
4
  "AutoTokenizer": [
5
- "dwdcth/chatglm3-6b-int4--tokenization_chatglm.ChatGLMTokenizer",
6
  null
7
- ]
8
- },
9
- "clean_up_tokenization_spaces": false,
10
- "do_lower_case": false,
11
- "model_max_length": 1000000000000000019884624838656,
12
- "pad_token": "<unk>",
13
- "padding_side": "left",
14
- "remove_space": false,
15
- "tokenizer_class": "ChatGLMTokenizer",
16
- "unk_token": "<unk>"
17
- }
 
1
  {
2
+ "name_or_path": "THUDM/chatglm3-6b",
3
+ "remove_space": false,
4
+ "do_lower_case": false,
5
+ "tokenizer_class": "ChatGLMTokenizer",
6
  "auto_map": {
7
  "AutoTokenizer": [
8
+ "tokenization_chatglm.ChatGLMTokenizer",
9
  null
10
+ ]
11
+ }
12
+ }