Set `<start_of_turn` and `<end_of_turn>` to be special tokens (#2)
Browse files- tokenizer.json +2 -2
- tokenizer_config.json +2 -2
tokenizer.json
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:9cf8cddb82b68ab495546855952d707acfa17f23c97031e0cf346b207d73faae
|
3 |
+
size 17525357
|
tokenizer_config.json
CHANGED
@@ -856,7 +856,7 @@
|
|
856 |
"normalized": false,
|
857 |
"rstrip": false,
|
858 |
"single_word": false,
|
859 |
-
"special":
|
860 |
},
|
861 |
"107": {
|
862 |
"content": "<end_of_turn>",
|
@@ -864,7 +864,7 @@
|
|
864 |
"normalized": false,
|
865 |
"rstrip": false,
|
866 |
"single_word": false,
|
867 |
-
"special":
|
868 |
},
|
869 |
"108": {
|
870 |
"content": "\n",
|
|
|
856 |
"normalized": false,
|
857 |
"rstrip": false,
|
858 |
"single_word": false,
|
859 |
+
"special": true
|
860 |
},
|
861 |
"107": {
|
862 |
"content": "<end_of_turn>",
|
|
|
864 |
"normalized": false,
|
865 |
"rstrip": false,
|
866 |
"single_word": false,
|
867 |
+
"special": true
|
868 |
},
|
869 |
"108": {
|
870 |
"content": "\n",
|