Upload tokenizer
Browse files- item/tokenizer_config.json +1 -1
- text/tokenizer_config.json +1 -0
item/tokenizer_config.json
CHANGED
@@ -49065,7 +49065,7 @@
|
|
49065 |
"special": false
|
49066 |
}
|
49067 |
},
|
49068 |
-
"chat_template": "{% for message in messages %}{
|
49069 |
"clean_up_tokenization_spaces": true,
|
49070 |
"init_inputs": [
|
49071 |
[
|
|
|
49065 |
"special": false
|
49066 |
}
|
49067 |
},
|
49068 |
+
"chat_template": "{% for message in messages %}{{ message.content + ' ' }}{% endfor %}",
|
49069 |
"clean_up_tokenization_spaces": true,
|
49070 |
"init_inputs": [
|
49071 |
[
|
text/tokenizer_config.json
CHANGED
@@ -43,6 +43,7 @@
|
|
43 |
}
|
44 |
},
|
45 |
"bos_token": "<s>",
|
|
|
46 |
"clean_up_tokenization_spaces": true,
|
47 |
"cls_token": "<s>",
|
48 |
"eos_token": "</s>",
|
|
|
43 |
}
|
44 |
},
|
45 |
"bos_token": "<s>",
|
46 |
+
"chat_template": "{% for message in messages %}{{ message.content + ' ' }}{% endfor %}",
|
47 |
"clean_up_tokenization_spaces": true,
|
48 |
"cls_token": "<s>",
|
49 |
"eos_token": "</s>",
|