Hugging Face's logo

Lollitor
/
FullProteinTokenizer

Model card Files Community
Lollitor commited on
Commit
41d7d00
•
1 Parent(s): 2a13697

Upload tokenizer

Browse files
Files changed (2) hide show
  1. added_tokens.json +3 -2
  2. tokenizer_config.json +9 -1
added_tokens.json CHANGED
@@ -112,6 +112,7 @@
112
  "▁/": 139,
113
  "▁0": 159,
114
  "▁1": 146,
 
115
  "▁2": 157,
116
  "▁3": 163,
117
  "▁4": 143,
@@ -120,8 +121,8 @@
120
  "▁7": 144,
121
  "▁8": 149,
122
  "▁9": 132,
123
- "▁<PRO>": 170,
124
- "▁<SMI>": 171,
125
  "▁=": 155,
126
  "▁@": 165,
127
  "▁[": 145,
 
112
  "▁/": 139,
113
  "▁0": 159,
114
  "▁1": 146,
115
+ "▁15": 170,
116
  "▁2": 157,
117
  "▁3": 163,
118
  "▁4": 143,
 
121
  "▁7": 144,
122
  "▁8": 149,
123
  "▁9": 132,
124
+ "▁<PRO>": 171,
125
+ "▁<SMI>": 172,
126
  "▁=": 155,
127
  "▁@": 165,
128
  "▁[": 145,
tokenizer_config.json CHANGED
@@ -1161,7 +1161,7 @@
1161
  "special": false
1162
  },
1163
  "170": {
1164
- "content": "▁<PRO>",
1165
  "lstrip": true,
1166
  "normalized": true,
1167
  "rstrip": true,
@@ -1169,6 +1169,14 @@
1169
  "special": false
1170
  },
1171
  "171": {
 
 
 
 
 
 
 
 
1172
  "content": "▁<SMI>",
1173
  "lstrip": true,
1174
  "normalized": true,
 
1161
  "special": false
1162
  },
1163
  "170": {
1164
+ "content": "▁15",
1165
  "lstrip": true,
1166
  "normalized": true,
1167
  "rstrip": true,
 
1169
  "special": false
1170
  },
1171
  "171": {
1172
+ "content": "▁<PRO>",
1173
+ "lstrip": true,
1174
+ "normalized": true,
1175
+ "rstrip": true,
1176
+ "single_word": false,
1177
+ "special": false
1178
+ },
1179
+ "172": {
1180
  "content": "▁<SMI>",
1181
  "lstrip": true,
1182
  "normalized": true,