Hugging Face's logo

imgremlin
/
smiles_tokenizer_13_11

Model card Files Community
imgremlin commited on
Commit
df973a7
•
1 Parent(s): 1916e9a

Upload tokenizer

Browse files
Files changed (5) hide show
  1. merges.txt +1 -0
  2. special_tokens_map.json +15 -0
  3. tokenizer.json +363 -0
  4. tokenizer_config.json +64 -0
  5. vocab.json +1 -0
merges.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ #version: 0.2
special_tokens_map.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<s>",
3
+ "cls_token": "<s>",
4
+ "eos_token": "</s>",
5
+ "mask_token": {
6
+ "content": "<mask>",
7
+ "lstrip": true,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false
11
+ },
12
+ "pad_token": "<pad>",
13
+ "sep_token": "</s>",
14
+ "unk_token": "<unk>"
15
+ }
tokenizer.json ADDED
@@ -0,0 +1,363 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": {
4
+ "direction": "Right",
5
+ "max_length": 1024,
6
+ "strategy": "LongestFirst",
7
+ "stride": 0
8
+ },
9
+ "padding": {
10
+ "strategy": "BatchLongest",
11
+ "direction": "Right",
12
+ "pad_to_multiple_of": null,
13
+ "pad_id": 1,
14
+ "pad_type_id": 0,
15
+ "pad_token": "<pad>"
16
+ },
17
+ "added_tokens": [
18
+ {
19
+ "id": 0,
20
+ "content": "<s>",
21
+ "single_word": false,
22
+ "lstrip": false,
23
+ "rstrip": false,
24
+ "normalized": false,
25
+ "special": true
26
+ },
27
+ {
28
+ "id": 1,
29
+ "content": "<pad>",
30
+ "single_word": false,
31
+ "lstrip": false,
32
+ "rstrip": false,
33
+ "normalized": false,
34
+ "special": true
35
+ },
36
+ {
37
+ "id": 2,
38
+ "content": "</s>",
39
+ "single_word": false,
40
+ "lstrip": false,
41
+ "rstrip": false,
42
+ "normalized": false,
43
+ "special": true
44
+ },
45
+ {
46
+ "id": 3,
47
+ "content": "<unk>",
48
+ "single_word": false,
49
+ "lstrip": false,
50
+ "rstrip": false,
51
+ "normalized": false,
52
+ "special": true
53
+ },
54
+ {
55
+ "id": 4,
56
+ "content": "<mask>",
57
+ "single_word": false,
58
+ "lstrip": true,
59
+ "rstrip": false,
60
+ "normalized": false,
61
+ "special": true
62
+ }
63
+ ],
64
+ "normalizer": null,
65
+ "pre_tokenizer": {
66
+ "type": "ByteLevel",
67
+ "add_prefix_space": false,
68
+ "trim_offsets": true,
69
+ "use_regex": true
70
+ },
71
+ "post_processor": {
72
+ "type": "RobertaProcessing",
73
+ "sep": [
74
+ "</s>",
75
+ 2
76
+ ],
77
+ "cls": [
78
+ "<s>",
79
+ 0
80
+ ],
81
+ "trim_offsets": true,
82
+ "add_prefix_space": false
83
+ },
84
+ "decoder": {
85
+ "type": "ByteLevel",
86
+ "add_prefix_space": true,
87
+ "trim_offsets": true,
88
+ "use_regex": true
89
+ },
90
+ "model": {
91
+ "type": "BPE",
92
+ "dropout": null,
93
+ "unk_token": null,
94
+ "continuing_subword_prefix": "",
95
+ "end_of_word_suffix": "",
96
+ "fuse_unk": false,
97
+ "byte_fallback": false,
98
+ "vocab": {
99
+ "<s>": 0,
100
+ "<pad>": 1,
101
+ "</s>": 2,
102
+ "<unk>": 3,
103
+ "<mask>": 4,
104
+ "!": 5,
105
+ "\"": 6,
106
+ "#": 7,
107
+ "$": 8,
108
+ "%": 9,
109
+ "&": 10,
110
+ "'": 11,
111
+ "(": 12,
112
+ ")": 13,
113
+ "*": 14,
114
+ "+": 15,
115
+ ",": 16,
116
+ "-": 17,
117
+ ".": 18,
118
+ "/": 19,
119
+ "0": 20,
120
+ "1": 21,
121
+ "2": 22,
122
+ "3": 23,
123
+ "4": 24,
124
+ "5": 25,
125
+ "6": 26,
126
+ "7": 27,
127
+ "8": 28,
128
+ "9": 29,
129
+ ":": 30,
130
+ ";": 31,
131
+ "<": 32,
132
+ "=": 33,
133
+ ">": 34,
134
+ "?": 35,
135
+ "@": 36,
136
+ "A": 37,
137
+ "B": 38,
138
+ "C": 39,
139
+ "D": 40,
140
+ "E": 41,
141
+ "F": 42,
142
+ "G": 43,
143
+ "H": 44,
144
+ "I": 45,
145
+ "J": 46,
146
+ "K": 47,
147
+ "L": 48,
148
+ "M": 49,
149
+ "N": 50,
150
+ "O": 51,
151
+ "P": 52,
152
+ "Q": 53,
153
+ "R": 54,
154
+ "S": 55,
155
+ "T": 56,
156
+ "U": 57,
157
+ "V": 58,
158
+ "W": 59,
159
+ "X": 60,
160
+ "Y": 61,
161
+ "Z": 62,
162
+ "[": 63,
163
+ "\\": 64,
164
+ "]": 65,
165
+ "^": 66,
166
+ "_": 67,
167
+ "`": 68,
168
+ "a": 69,
169
+ "b": 70,
170
+ "c": 71,
171
+ "d": 72,
172
+ "e": 73,
173
+ "f": 74,
174
+ "g": 75,
175
+ "h": 76,
176
+ "i": 77,
177
+ "j": 78,
178
+ "k": 79,
179
+ "l": 80,
180
+ "m": 81,
181
+ "n": 82,
182
+ "o": 83,
183
+ "p": 84,
184
+ "q": 85,
185
+ "r": 86,
186
+ "s": 87,
187
+ "t": 88,
188
+ "u": 89,
189
+ "v": 90,
190
+ "w": 91,
191
+ "x": 92,
192
+ "y": 93,
193
+ "z": 94,
194
+ "{": 95,
195
+ "|": 96,
196
+ "}": 97,
197
+ "~": 98,
198
+ "¡": 99,
199
+ "¢": 100,
200
+ "£": 101,
201
+ "¤": 102,
202
+ "¥": 103,
203
+ "¦": 104,
204
+ "§": 105,
205
+ "¨": 106,
206
+ "©": 107,
207
+ "ª": 108,
208
+ "«": 109,
209
+ "¬": 110,
210
+ "®": 111,
211
+ "¯": 112,
212
+ "°": 113,
213
+ "±": 114,
214
+ "²": 115,
215
+ "³": 116,
216
+ "´": 117,
217
+ "µ": 118,
218
+ "¶": 119,
219
+ "·": 120,
220
+ "¸": 121,
221
+ "¹": 122,
222
+ "º": 123,
223
+ "»": 124,
224
+ "¼": 125,
225
+ "½": 126,
226
+ "¾": 127,
227
+ "¿": 128,
228
+ "À": 129,
229
+ "Á": 130,
230
+ "Â": 131,
231
+ "Ã": 132,
232
+ "Ä": 133,
233
+ "Å": 134,
234
+ "Æ": 135,
235
+ "Ç": 136,
236
+ "È": 137,
237
+ "É": 138,
238
+ "Ê": 139,
239
+ "Ë": 140,
240
+ "Ì": 141,
241
+ "Í": 142,
242
+ "Î": 143,
243
+ "Ï": 144,
244
+ "Ð": 145,
245
+ "Ñ": 146,
246
+ "Ò": 147,
247
+ "Ó": 148,
248
+ "Ô": 149,
249
+ "Õ": 150,
250
+ "Ö": 151,
251
+ "×": 152,
252
+ "Ø": 153,
253
+ "Ù": 154,
254
+ "Ú": 155,
255
+ "Û": 156,
256
+ "Ü": 157,
257
+ "Ý": 158,
258
+ "Þ": 159,
259
+ "ß": 160,
260
+ "à": 161,
261
+ "á": 162,
262
+ "â": 163,
263
+ "ã": 164,
264
+ "ä": 165,
265
+ "å": 166,
266
+ "æ": 167,
267
+ "ç": 168,
268
+ "è": 169,
269
+ "é": 170,
270
+ "ê": 171,
271
+ "ë": 172,
272
+ "ì": 173,
273
+ "í": 174,
274
+ "î": 175,
275
+ "ï": 176,
276
+ "ð": 177,
277
+ "ñ": 178,
278
+ "ò": 179,
279
+ "ó": 180,
280
+ "ô": 181,
281
+ "õ": 182,
282
+ "ö": 183,
283
+ "÷": 184,
284
+ "ø": 185,
285
+ "ù": 186,
286
+ "ú": 187,
287
+ "û": 188,
288
+ "ü": 189,
289
+ "ý": 190,
290
+ "þ": 191,
291
+ "ÿ": 192,
292
+ "Ā": 193,
293
+ "ā": 194,
294
+ "Ă": 195,
295
+ "ă": 196,
296
+ "Ą": 197,
297
+ "ą": 198,
298
+ "Ć": 199,
299
+ "ć": 200,
300
+ "Ĉ": 201,
301
+ "ĉ": 202,
302
+ "Ċ": 203,
303
+ "ċ": 204,
304
+ "Č": 205,
305
+ "č": 206,
306
+ "Ď": 207,
307
+ "ď": 208,
308
+ "Đ": 209,
309
+ "đ": 210,
310
+ "Ē": 211,
311
+ "ē": 212,
312
+ "Ĕ": 213,
313
+ "ĕ": 214,
314
+ "Ė": 215,
315
+ "ė": 216,
316
+ "Ę": 217,
317
+ "ę": 218,
318
+ "Ě": 219,
319
+ "ě": 220,
320
+ "Ĝ": 221,
321
+ "ĝ": 222,
322
+ "Ğ": 223,
323
+ "ğ": 224,
324
+ "Ġ": 225,
325
+ "ġ": 226,
326
+ "Ģ": 227,
327
+ "ģ": 228,
328
+ "Ĥ": 229,
329
+ "ĥ": 230,
330
+ "Ħ": 231,
331
+ "ħ": 232,
332
+ "Ĩ": 233,
333
+ "ĩ": 234,
334
+ "Ī": 235,
335
+ "ī": 236,
336
+ "Ĭ": 237,
337
+ "ĭ": 238,
338
+ "Į": 239,
339
+ "į": 240,
340
+ "İ": 241,
341
+ "ı": 242,
342
+ "IJ": 243,
343
+ "ij": 244,
344
+ "Ĵ": 245,
345
+ "ĵ": 246,
346
+ "Ķ": 247,
347
+ "ķ": 248,
348
+ "ĸ": 249,
349
+ "Ĺ": 250,
350
+ "ĺ": 251,
351
+ "Ļ": 252,
352
+ "ļ": 253,
353
+ "Ľ": 254,
354
+ "ľ": 255,
355
+ "Ŀ": 256,
356
+ "ŀ": 257,
357
+ "Ł": 258,
358
+ "ł": 259,
359
+ "Ń": 260
360
+ },
361
+ "merges": []
362
+ }
363
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "0": {
5
+ "content": "<s>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "1": {
13
+ "content": "<pad>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "2": {
21
+ "content": "</s>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "3": {
29
+ "content": "<unk>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ },
36
+ "4": {
37
+ "content": "<mask>",
38
+ "lstrip": true,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false,
42
+ "special": true
43
+ }
44
+ },
45
+ "bos_token": "<s>",
46
+ "clean_up_tokenization_spaces": true,
47
+ "cls_token": "<s>",
48
+ "eos_token": "</s>",
49
+ "errors": "replace",
50
+ "mask_token": "<mask>",
51
+ "max_length": 1024,
52
+ "model_max_length": 1024,
53
+ "pad_to_multiple_of": null,
54
+ "pad_token": "<pad>",
55
+ "pad_token_type_id": 0,
56
+ "padding_side": "right",
57
+ "sep_token": "</s>",
58
+ "stride": 0,
59
+ "tokenizer_class": "BartTokenizer",
60
+ "trim_offsets": true,
61
+ "truncation_side": "right",
62
+ "truncation_strategy": "longest_first",
63
+ "unk_token": "<unk>"
64
+ }
vocab.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"<s>":0,"<pad>":1,"</s>":2,"<unk>":3,"<mask>":4,"!":5,"\"":6,"#":7,"$":8,"%":9,"&":10,"'":11,"(":12,")":13,"*":14,"+":15,",":16,"-":17,".":18,"/":19,"0":20,"1":21,"2":22,"3":23,"4":24,"5":25,"6":26,"7":27,"8":28,"9":29,":":30,";":31,"<":32,"=":33,">":34,"?":35,"@":36,"A":37,"B":38,"C":39,"D":40,"E":41,"F":42,"G":43,"H":44,"I":45,"J":46,"K":47,"L":48,"M":49,"N":50,"O":51,"P":52,"Q":53,"R":54,"S":55,"T":56,"U":57,"V":58,"W":59,"X":60,"Y":61,"Z":62,"[":63,"\\":64,"]":65,"^":66,"_":67,"`":68,"a":69,"b":70,"c":71,"d":72,"e":73,"f":74,"g":75,"h":76,"i":77,"j":78,"k":79,"l":80,"m":81,"n":82,"o":83,"p":84,"q":85,"r":86,"s":87,"t":88,"u":89,"v":90,"w":91,"x":92,"y":93,"z":94,"{":95,"|":96,"}":97,"~":98,"¡":99,"¢":100,"£":101,"¤":102,"¥":103,"¦":104,"§":105,"¨":106,"©":107,"ª":108,"«":109,"¬":110,"®":111,"¯":112,"°":113,"±":114,"²":115,"³":116,"´":117,"µ":118,"¶":119,"·":120,"¸":121,"¹":122,"º":123,"»":124,"¼":125,"½":126,"¾":127,"¿":128,"À":129,"Á":130,"Â":131,"Ã":132,"Ä":133,"Å":134,"Æ":135,"Ç":136,"È":137,"É":138,"Ê":139,"Ë":140,"Ì":141,"Í":142,"Î":143,"Ï":144,"Ð":145,"Ñ":146,"Ò":147,"Ó":148,"Ô":149,"Õ":150,"Ö":151,"×":152,"Ø":153,"Ù":154,"Ú":155,"Û":156,"Ü":157,"Ý":158,"Þ":159,"ß":160,"à":161,"á":162,"â":163,"ã":164,"ä":165,"å":166,"æ":167,"ç":168,"è":169,"é":170,"ê":171,"ë":172,"ì":173,"í":174,"î":175,"ï":176,"ð":177,"ñ":178,"ò":179,"ó":180,"ô":181,"õ":182,"ö":183,"÷":184,"ø":185,"ù":186,"ú":187,"û":188,"ü":189,"ý":190,"þ":191,"ÿ":192,"Ā":193,"ā":194,"Ă":195,"ă":196,"Ą":197,"ą":198,"Ć":199,"ć":200,"Ĉ":201,"ĉ":202,"Ċ":203,"ċ":204,"Č":205,"č":206,"Ď":207,"ď":208,"Đ":209,"đ":210,"Ē":211,"ē":212,"Ĕ":213,"ĕ":214,"Ė":215,"ė":216,"Ę":217,"ę":218,"Ě":219,"ě":220,"Ĝ":221,"ĝ":222,"Ğ":223,"ğ":224,"Ġ":225,"ġ":226,"Ģ":227,"ģ":228,"Ĥ":229,"ĥ":230,"Ħ":231,"ħ":232,"Ĩ":233,"ĩ":234,"Ī":235,"ī":236,"Ĭ":237,"ĭ":238,"Į":239,"į":240,"İ":241,"ı":242,"IJ":243,"ij":244,"Ĵ":245,"ĵ":246,"Ķ":247,"ķ":248,"ĸ":249,"Ĺ":250,"ĺ":251,"Ļ":252,"ļ":253,"Ľ":254,"ľ":255,"Ŀ":256,"ŀ":257,"Ł":258,"ł":259,"Ń":260}