mgoin commited on
Commit
96c90c4
1 Parent(s): 5c7b184

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -0
tokenizer_config.json CHANGED
@@ -29,6 +29,7 @@
29
  }
30
  },
31
  "bos_token": "<s>",
 
32
  "clean_up_tokenization_spaces": false,
33
  "eos_token": "</s>",
34
  "legacy": true,
 
29
  }
30
  },
31
  "bos_token": "<s>",
32
+ "chat_template": "{% set loop_messages = messages %}\n{% for message in loop_messages %}\n{% set content = message['role']+'\n'+ message['content'] + '\n' %}\n{% if message['role'] == 'System' %}\n{% set content = '<extra_id_0>' + content %}\n{% else %}\n{% set content = '<extra_id_1>' + content %}\n{% endif %}\n{{ content }}\n{% endfor %}\n{% if add_generation_prompt %}\n{{ '<extra_id_1>Assistant' }}\n\n{% endif %}",
33
  "clean_up_tokenization_spaces": false,
34
  "eos_token": "</s>",
35
  "legacy": true,