yihanwang617 commited on
Commit
0085161
·
verified ·
1 Parent(s): 7a3ad73

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +3 -2
tokenizer_config.json CHANGED
@@ -2066,13 +2066,14 @@
2066
  }
2067
  },
2068
  "bos_token": "<|begin_of_text|>",
 
2069
  "clean_up_tokenization_spaces": true,
2070
  "eos_token": "<|end_of_text|>",
2071
  "model_input_names": [
2072
  "input_ids",
2073
  "attention_mask"
2074
  ],
2075
- "model_max_length": 1000000000000000019884624838656,
2076
  "pad_token": "[PAD]",
2077
  "tokenizer_class": "PreTrainedTokenizerFast"
2078
- }
 
2066
  }
2067
  },
2068
  "bos_token": "<|begin_of_text|>",
2069
+ "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
2070
  "clean_up_tokenization_spaces": true,
2071
  "eos_token": "<|end_of_text|>",
2072
  "model_input_names": [
2073
  "input_ids",
2074
  "attention_mask"
2075
  ],
2076
+ "model_max_length": 2048,
2077
  "pad_token": "[PAD]",
2078
  "tokenizer_class": "PreTrainedTokenizerFast"
2079
+ }