winglian commited on
Commit
1757241
1 Parent(s): 1beff82

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +7 -8
tokenizer_config.json CHANGED
@@ -28,32 +28,31 @@
28
  },
29
  "32000": {
30
  "content": "<|im_end|>",
31
- "lstrip": true,
32
- "normalized": true,
33
- "rstrip": true,
34
  "single_word": false,
35
- "special": false
36
  },
37
  "32001": {
38
  "content": "<|im_start|>",
39
  "lstrip": true,
40
- "normalized": true,
41
  "rstrip": true,
42
  "single_word": false,
43
- "special": false
44
  }
45
  },
46
  "additional_special_tokens": [],
47
  "bos_token": "<s>",
48
  "clean_up_tokenization_spaces": false,
49
- "eos_token": "</s>",
50
  "legacy": true,
51
  "model_max_length": 1000000000000000019884624838656,
52
  "pad_token": null,
53
  "sp_model_kwargs": {},
54
  "spaces_between_special_tokens": false,
55
  "tokenizer_class": "LlamaTokenizer",
56
- "tokenizer_file": "/home/wing/.cache/huggingface/hub/models--mistralai--Mistral-7B-v0.1/snapshots/78814a934d8ce59ee93599378f3e929ff137da06/tokenizer.json",
57
  "trust_remote_code": false,
58
  "unk_token": "<unk>",
59
  "use_default_system_prompt": true,
 
28
  },
29
  "32000": {
30
  "content": "<|im_end|>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
  "single_word": false,
35
+ "special": true
36
  },
37
  "32001": {
38
  "content": "<|im_start|>",
39
  "lstrip": true,
40
+ "normalized": false,
41
  "rstrip": true,
42
  "single_word": false,
43
+ "special": true
44
  }
45
  },
46
  "additional_special_tokens": [],
47
  "bos_token": "<s>",
48
  "clean_up_tokenization_spaces": false,
49
+ "eos_token": "<|im_end|>",
50
  "legacy": true,
51
  "model_max_length": 1000000000000000019884624838656,
52
  "pad_token": null,
53
  "sp_model_kwargs": {},
54
  "spaces_between_special_tokens": false,
55
  "tokenizer_class": "LlamaTokenizer",
 
56
  "trust_remote_code": false,
57
  "unk_token": "<unk>",
58
  "use_default_system_prompt": true,