ZhankuiHe commited on
Commit
1f55f2d
1 Parent(s): d2b2893

Upload tokenizer

Browse files
item/tokenizer_config.json CHANGED
@@ -49065,7 +49065,7 @@
49065
  "special": false
49066
  }
49067
  },
49068
- "chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{ 'User: ' + message['content'] }}{% elif message['role'] == 'assistant' %}{{ 'System: ' + message['content'] }}{% endif %}{% if not loop.last %}{{'\n'}}{% endif %}{% endfor %}",
49069
  "clean_up_tokenization_spaces": true,
49070
  "init_inputs": [
49071
  [
 
49065
  "special": false
49066
  }
49067
  },
49068
+ "chat_template": "{% for message in messages %}{{ message.content + ' ' }}{% endfor %}",
49069
  "clean_up_tokenization_spaces": true,
49070
  "init_inputs": [
49071
  [
text/tokenizer_config.json CHANGED
@@ -43,6 +43,7 @@
43
  }
44
  },
45
  "bos_token": "<s>",
 
46
  "clean_up_tokenization_spaces": true,
47
  "cls_token": "<s>",
48
  "eos_token": "</s>",
 
43
  }
44
  },
45
  "bos_token": "<s>",
46
+ "chat_template": "{% for message in messages %}{{ message.content + ' ' }}{% endfor %}",
47
  "clean_up_tokenization_spaces": true,
48
  "cls_token": "<s>",
49
  "eos_token": "</s>",