teknium commited on
Commit
576f17d
1 Parent(s): ff2b857

Upload tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +11 -9
tokenizer_config.json CHANGED
@@ -17,7 +17,7 @@
17
  "special": true
18
  },
19
  "128002": {
20
- "content": "<|reserved_special_token_0|>",
21
  "lstrip": false,
22
  "normalized": false,
23
  "rstrip": false,
@@ -25,7 +25,7 @@
25
  "special": true
26
  },
27
  "128003": {
28
- "content": "<|reserved_special_token_1|>",
29
  "lstrip": false,
30
  "normalized": false,
31
  "rstrip": false,
@@ -33,7 +33,7 @@
33
  "special": true
34
  },
35
  "128004": {
36
- "content": "<|reserved_special_token_2|>",
37
  "lstrip": false,
38
  "normalized": false,
39
  "rstrip": false,
@@ -41,7 +41,7 @@
41
  "special": true
42
  },
43
  "128005": {
44
- "content": "<|reserved_special_token_3|>",
45
  "lstrip": false,
46
  "normalized": false,
47
  "rstrip": false,
@@ -65,7 +65,7 @@
65
  "special": true
66
  },
67
  "128008": {
68
- "content": "<|reserved_special_token_4|>",
69
  "lstrip": false,
70
  "normalized": false,
71
  "rstrip": false,
@@ -81,7 +81,7 @@
81
  "special": true
82
  },
83
  "128010": {
84
- "content": "<|reserved_special_token_5|>",
85
  "lstrip": false,
86
  "normalized": false,
87
  "rstrip": false,
@@ -89,7 +89,7 @@
89
  "special": true
90
  },
91
  "128011": {
92
- "content": "<|reserved_special_token_6|>",
93
  "lstrip": false,
94
  "normalized": false,
95
  "rstrip": false,
@@ -97,7 +97,7 @@
97
  "special": true
98
  },
99
  "128012": {
100
- "content": "<|reserved_special_token_7|>",
101
  "lstrip": false,
102
  "normalized": false,
103
  "rstrip": false,
@@ -2050,12 +2050,14 @@
2050
  }
2051
  },
2052
  "bos_token": "<|begin_of_text|>",
 
2053
  "clean_up_tokenization_spaces": true,
2054
- "eos_token": "<|end_of_text|>",
2055
  "model_input_names": [
2056
  "input_ids",
2057
  "attention_mask"
2058
  ],
2059
  "model_max_length": 1000000000000000019884624838656,
 
2060
  "tokenizer_class": "PreTrainedTokenizerFast"
2061
  }
 
17
  "special": true
18
  },
19
  "128002": {
20
+ "content": "<|im_start|>",
21
  "lstrip": false,
22
  "normalized": false,
23
  "rstrip": false,
 
25
  "special": true
26
  },
27
  "128003": {
28
+ "content": "<|im_end|>",
29
  "lstrip": false,
30
  "normalized": false,
31
  "rstrip": false,
 
33
  "special": true
34
  },
35
  "128004": {
36
+ "content": "<tool_call>",
37
  "lstrip": false,
38
  "normalized": false,
39
  "rstrip": false,
 
41
  "special": true
42
  },
43
  "128005": {
44
+ "content": "<tool_response>",
45
  "lstrip": false,
46
  "normalized": false,
47
  "rstrip": false,
 
65
  "special": true
66
  },
67
  "128008": {
68
+ "content": "<tools>",
69
  "lstrip": false,
70
  "normalized": false,
71
  "rstrip": false,
 
81
  "special": true
82
  },
83
  "128010": {
84
+ "content": "</tools>",
85
  "lstrip": false,
86
  "normalized": false,
87
  "rstrip": false,
 
89
  "special": true
90
  },
91
  "128011": {
92
+ "content": "</tool_call>",
93
  "lstrip": false,
94
  "normalized": false,
95
  "rstrip": false,
 
97
  "special": true
98
  },
99
  "128012": {
100
+ "content": "</tool_response>",
101
  "lstrip": false,
102
  "normalized": false,
103
  "rstrip": false,
 
2050
  }
2051
  },
2052
  "bos_token": "<|begin_of_text|>",
2053
+ "chat_template": "{{bos_token}}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
2054
  "clean_up_tokenization_spaces": true,
2055
+ "eos_token": "<|im_end|>",
2056
  "model_input_names": [
2057
  "input_ids",
2058
  "attention_mask"
2059
  ],
2060
  "model_max_length": 1000000000000000019884624838656,
2061
+ "pad_token": "<|end_of_text|>",
2062
  "tokenizer_class": "PreTrainedTokenizerFast"
2063
  }