saattrupdan commited on
Commit
ee571a1
1 Parent(s): cc97a97

Update tokenizer_config.json

Browse files

This uses the tokenizer config from [NeuralBeagle](https://huggingface.co/mlabonne/NeuralBeagle14-7B), with updated `model_max_length` to Mistral's 32,768 context length.

Files changed (1) hide show
  1. tokenizer_config.json +25 -6
tokenizer_config.json CHANGED
@@ -11,7 +11,7 @@
11
  "special": true
12
  },
13
  "1": {
14
- "content": "<s>",
15
  "lstrip": false,
16
  "normalized": false,
17
  "rstrip": false,
@@ -19,7 +19,23 @@
19
  "special": true
20
  },
21
  "2": {
22
- "content": "</s>",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
23
  "lstrip": false,
24
  "normalized": false,
25
  "rstrip": false,
@@ -28,15 +44,18 @@
28
  }
29
  },
30
  "additional_special_tokens": [],
31
- "bos_token": "<s>",
 
32
  "clean_up_tokenization_spaces": false,
33
- "eos_token": "</s>",
34
  "legacy": true,
35
- "model_max_length": 1000000000000000019884624838656,
36
  "pad_token": null,
37
  "sp_model_kwargs": {},
38
  "spaces_between_special_tokens": false,
39
  "tokenizer_class": "LlamaTokenizer",
 
40
  "unk_token": "<unk>",
41
- "use_default_system_prompt": false
 
42
  }
 
11
  "special": true
12
  },
13
  "1": {
14
+ "content": "<|im_start|>",
15
  "lstrip": false,
16
  "normalized": false,
17
  "rstrip": false,
 
19
  "special": true
20
  },
21
  "2": {
22
+ "content": "<|im_end|>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "32000": {
30
+ "content": "<|im_end|>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "32001": {
38
+ "content": "<|im_start|>",
39
  "lstrip": false,
40
  "normalized": false,
41
  "rstrip": false,
 
44
  }
45
  },
46
  "additional_special_tokens": [],
47
+ "bos_token": "<|im_start|>",
48
+ "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
49
  "clean_up_tokenization_spaces": false,
50
+ "eos_token": "<|im_end|>",
51
  "legacy": true,
52
+ "model_max_length": 32768,
53
  "pad_token": null,
54
  "sp_model_kwargs": {},
55
  "spaces_between_special_tokens": false,
56
  "tokenizer_class": "LlamaTokenizer",
57
+ "trust_remote_code": false,
58
  "unk_token": "<unk>",
59
+ "use_default_system_prompt": true,
60
+ "use_fast": true
61
  }