Upload tokenizer
Browse files- README.md +3 -0
- tokenizer_config.json +1 -0
README.md
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
| 1 |
```
|
| 2 |
pre_text = "The following is an interaction between a user and an AI assistant that is related to the above text."
|
| 3 |
def ds_map_fn(row):
|
|
|
|
| 1 |
+
---
|
| 2 |
+
{}
|
| 3 |
+
---
|
| 4 |
```
|
| 5 |
pre_text = "The following is an interaction between a user and an AI assistant that is related to the above text."
|
| 6 |
def ds_map_fn(row):
|
tokenizer_config.json
CHANGED
|
@@ -37,6 +37,7 @@
|
|
| 37 |
},
|
| 38 |
"additional_special_tokens": [],
|
| 39 |
"bos_token": "<s>",
|
|
|
|
| 40 |
"clean_up_tokenization_spaces": false,
|
| 41 |
"eos_token": "</s>",
|
| 42 |
"legacy": true,
|
|
|
|
| 37 |
},
|
| 38 |
"additional_special_tokens": [],
|
| 39 |
"bos_token": "<s>",
|
| 40 |
+
"chat_template": "{% for message in messages %}{{ '[[Title]] ' + message['title'].strip() + '\n' }}{{ '[[Content]] ' + message['content'].strip() + '\n\n' + 'The following is an interaction between a user and an AI assistant that is related to the above text.\n\n[[User]] ' }}{% endfor %}",
|
| 41 |
"clean_up_tokenization_spaces": false,
|
| 42 |
"eos_token": "</s>",
|
| 43 |
"legacy": true,
|