c0scm0ch405 commited on
Commit
a79f4c0
·
1 Parent(s): 84c6fb9

Upload tokenizer

Browse files
Files changed (2) hide show
  1. added_tokens.json +3 -2
  2. tokenizer_config.json +10 -2
added_tokens.json CHANGED
@@ -99,6 +99,7 @@
99
  "<extra_id_98>": 32001,
100
  "<extra_id_99>": 32000,
101
  "<extra_id_9>": 32090,
102
- "<zk5>": 32101,
103
- "zk5": 32100
 
104
  }
 
99
  "<extra_id_98>": 32001,
100
  "<extra_id_99>": 32000,
101
  "<extra_id_9>": 32090,
102
+ "<https://zk5.digital>": 32102,
103
+ "<info@zk5.digital>": 32101,
104
+ "<zk5>": 32100
105
  }
tokenizer_config.json CHANGED
@@ -825,7 +825,7 @@
825
  "special": true
826
  },
827
  "32100": {
828
- "content": "zk5",
829
  "lstrip": false,
830
  "normalized": true,
831
  "rstrip": false,
@@ -833,7 +833,15 @@
833
  "special": false
834
  },
835
  "32101": {
836
- "content": "<zk5>",
 
 
 
 
 
 
 
 
837
  "lstrip": false,
838
  "normalized": true,
839
  "rstrip": false,
 
825
  "special": true
826
  },
827
  "32100": {
828
+ "content": "<zk5>",
829
  "lstrip": false,
830
  "normalized": true,
831
  "rstrip": false,
 
833
  "special": false
834
  },
835
  "32101": {
836
+ "content": "<info@zk5.digital>",
837
+ "lstrip": false,
838
+ "normalized": true,
839
+ "rstrip": false,
840
+ "single_word": false,
841
+ "special": false
842
+ },
843
+ "32102": {
844
+ "content": "<https://zk5.digital>",
845
  "lstrip": false,
846
  "normalized": true,
847
  "rstrip": false,