Commit
·
a79f4c0
1
Parent(s):
84c6fb9
Upload tokenizer
Browse files- added_tokens.json +3 -2
- tokenizer_config.json +10 -2
added_tokens.json
CHANGED
@@ -99,6 +99,7 @@
|
|
99 |
"<extra_id_98>": 32001,
|
100 |
"<extra_id_99>": 32000,
|
101 |
"<extra_id_9>": 32090,
|
102 |
-
"<zk5>":
|
103 |
-
"zk5":
|
|
|
104 |
}
|
|
|
99 |
"<extra_id_98>": 32001,
|
100 |
"<extra_id_99>": 32000,
|
101 |
"<extra_id_9>": 32090,
|
102 |
+
"<https://zk5.digital>": 32102,
|
103 |
+
"<info@zk5.digital>": 32101,
|
104 |
+
"<zk5>": 32100
|
105 |
}
|
tokenizer_config.json
CHANGED
@@ -825,7 +825,7 @@
|
|
825 |
"special": true
|
826 |
},
|
827 |
"32100": {
|
828 |
-
"content": "zk5",
|
829 |
"lstrip": false,
|
830 |
"normalized": true,
|
831 |
"rstrip": false,
|
@@ -833,7 +833,15 @@
|
|
833 |
"special": false
|
834 |
},
|
835 |
"32101": {
|
836 |
-
"content": "<zk5>",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
837 |
"lstrip": false,
|
838 |
"normalized": true,
|
839 |
"rstrip": false,
|
|
|
825 |
"special": true
|
826 |
},
|
827 |
"32100": {
|
828 |
+
"content": "<zk5>",
|
829 |
"lstrip": false,
|
830 |
"normalized": true,
|
831 |
"rstrip": false,
|
|
|
833 |
"special": false
|
834 |
},
|
835 |
"32101": {
|
836 |
+
"content": "<info@zk5.digital>",
|
837 |
+
"lstrip": false,
|
838 |
+
"normalized": true,
|
839 |
+
"rstrip": false,
|
840 |
+
"single_word": false,
|
841 |
+
"special": false
|
842 |
+
},
|
843 |
+
"32102": {
|
844 |
+
"content": "<https://zk5.digital>",
|
845 |
"lstrip": false,
|
846 |
"normalized": true,
|
847 |
"rstrip": false,
|