JordiBayarri commited on
Commit
27297ce
1 Parent(s): b002e98

correct function calling token ids

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +5 -21
tokenizer_config.json CHANGED
@@ -17,20 +17,20 @@
17
  "special": true
18
  },
19
  "128002": {
20
- "content": "<|reserved_special_token_0|>",
21
  "lstrip": false,
22
  "normalized": false,
23
  "rstrip": false,
24
  "single_word": false,
25
- "special": true
26
  },
27
  "128003": {
28
- "content": "<|reserved_special_token_1|>",
29
  "lstrip": false,
30
  "normalized": false,
31
  "rstrip": false,
32
  "single_word": false,
33
- "special": true
34
  },
35
  "128004": {
36
  "content": "<|finetune_right_pad_id|>",
@@ -2047,22 +2047,6 @@
2047
  "rstrip": false,
2048
  "single_word": false,
2049
  "special": true
2050
- },
2051
- "128256": {
2052
- "content": "<tool_call>",
2053
- "lstrip": false,
2054
- "normalized": false,
2055
- "rstrip": false,
2056
- "single_word": false,
2057
- "special": false
2058
- },
2059
- "128257": {
2060
- "content": "<tool_response>",
2061
- "lstrip": false,
2062
- "normalized": false,
2063
- "rstrip": false,
2064
- "single_word": false,
2065
- "special": false
2066
  }
2067
  },
2068
  "bos_token": "<|begin_of_text|>",
@@ -2076,4 +2060,4 @@
2076
  ],
2077
  "model_max_length": 131072,
2078
  "tokenizer_class": "PreTrainedTokenizerFast"
2079
- }
 
17
  "special": true
18
  },
19
  "128002": {
20
+ "content": "<tool_call>",
21
  "lstrip": false,
22
  "normalized": false,
23
  "rstrip": false,
24
  "single_word": false,
25
+ "special": false
26
  },
27
  "128003": {
28
+ "content": "<tool_response>",
29
  "lstrip": false,
30
  "normalized": false,
31
  "rstrip": false,
32
  "single_word": false,
33
+ "special": false
34
  },
35
  "128004": {
36
  "content": "<|finetune_right_pad_id|>",
 
2047
  "rstrip": false,
2048
  "single_word": false,
2049
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2050
  }
2051
  },
2052
  "bos_token": "<|begin_of_text|>",
 
2060
  ],
2061
  "model_max_length": 131072,
2062
  "tokenizer_class": "PreTrainedTokenizerFast"
2063
+ }