whizzzzkid
commited on
Commit
•
630dc22
1
Parent(s):
cf1ba25
Upload tokenizer
Browse files- tokenizer.json +2 -2
- tokenizer_config.json +2 -6
tokenizer.json
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:631fbd6e3f9415c226372013d876a6a5106009bb2637021ea2b73d524163dd5c
|
3 |
+
size 4239425
|
tokenizer_config.json
CHANGED
@@ -302,15 +302,11 @@
|
|
302 |
"<|extra0|>"
|
303 |
],
|
304 |
"bos_token": "<|endoftext|>",
|
305 |
-
"chat_template": "{%
|
306 |
"clean_up_tokenization_spaces": true,
|
307 |
"eos_token": "<|endoftext|>",
|
308 |
-
"
|
309 |
-
"model_max_length": 4096,
|
310 |
"pad_token": "<|endoftext|>",
|
311 |
-
"stride": 0,
|
312 |
"tokenizer_class": "GPT2Tokenizer",
|
313 |
-
"truncation_side": "right",
|
314 |
-
"truncation_strategy": "longest_first",
|
315 |
"unk_token": "<|endoftext|>"
|
316 |
}
|
|
|
302 |
"<|extra0|>"
|
303 |
],
|
304 |
"bos_token": "<|endoftext|>",
|
305 |
+
"chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ '<|system|>\\n' + system_message + '<|endoftext|>' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|user|>\\n' + content + '<|endoftext|>' + '\\n<|assistant|>\\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' + '\\n' }}{% endif %}{% endfor %}",
|
306 |
"clean_up_tokenization_spaces": true,
|
307 |
"eos_token": "<|endoftext|>",
|
308 |
+
"model_max_length": 2048,
|
|
|
309 |
"pad_token": "<|endoftext|>",
|
|
|
310 |
"tokenizer_class": "GPT2Tokenizer",
|
|
|
|
|
311 |
"unk_token": "<|endoftext|>"
|
312 |
}
|