Locutusque commited on
Commit
8fe538e
1 Parent(s): 71ab8a5

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -7
tokenizer_config.json CHANGED
@@ -27,12 +27,6 @@
27
  "special": true
28
  }
29
  },
30
- "auto_map": {
31
- "AutoTokenizer": [
32
- "chargoddard/Yi-34B-Llama--tokenization_yi.YiTokenizer",
33
- null
34
- ]
35
- },
36
  "bos_token": "<|startoftext|>",
37
  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
38
  "clean_up_tokenization_spaces": false,
@@ -40,6 +34,6 @@
40
  "model_max_length": 4096,
41
  "pad_token": "<|endoftext|>",
42
  "sp_model_kwargs": {},
43
- "tokenizer_class": "YiTokenizer",
44
  "unk_token": "<unk>"
45
  }
 
27
  "special": true
28
  }
29
  },
 
 
 
 
 
 
30
  "bos_token": "<|startoftext|>",
31
  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
32
  "clean_up_tokenization_spaces": false,
 
34
  "model_max_length": 4096,
35
  "pad_token": "<|endoftext|>",
36
  "sp_model_kwargs": {},
37
+ "tokenizer_class": "LlamaTokenizer",
38
  "unk_token": "<unk>"
39
  }