add tokenizer
Browse files- added_tokens.json +1 -1
- tokenizer.json +137 -138
- vocab.json +1 -1
added_tokens.json
CHANGED
@@ -1 +1 @@
|
|
1 |
-
{"<":
|
|
|
1 |
+
{"<": 254, ">": 253, "#": 255}
|
tokenizer.json
CHANGED
@@ -13,7 +13,7 @@
|
|
13 |
"special": true
|
14 |
},
|
15 |
{
|
16 |
-
"id":
|
17 |
"content": ">",
|
18 |
"single_word": false,
|
19 |
"lstrip": false,
|
@@ -22,7 +22,7 @@
|
|
22 |
"special": true
|
23 |
},
|
24 |
{
|
25 |
-
"id":
|
26 |
"content": "<",
|
27 |
"single_word": false,
|
28 |
"lstrip": false,
|
@@ -31,7 +31,7 @@
|
|
31 |
"special": true
|
32 |
},
|
33 |
{
|
34 |
-
"id":
|
35 |
"content": "#",
|
36 |
"single_word": false,
|
37 |
"lstrip": false,
|
@@ -183,141 +183,140 @@
|
|
183 |
"É": 116,
|
184 |
"Ê": 117,
|
185 |
"Ì": 118,
|
186 |
-
"
|
187 |
-
"
|
188 |
-
"
|
189 |
-
"
|
190 |
-
"
|
191 |
-
"
|
192 |
-
"
|
193 |
-
"
|
194 |
-
"
|
195 |
-
"
|
196 |
-
"
|
197 |
-
"
|
198 |
-
"
|
199 |
-
"
|
200 |
-
"
|
201 |
-
"
|
202 |
-
"
|
203 |
-
"
|
204 |
-
"
|
205 |
-
"
|
206 |
-
"
|
207 |
-
"
|
208 |
-
"
|
209 |
-
"
|
210 |
-
"
|
211 |
-
"
|
212 |
-
"
|
213 |
-
"
|
214 |
-
"
|
215 |
-
"
|
216 |
-
"
|
217 |
-
"
|
218 |
-
"
|
219 |
-
"
|
220 |
-
"
|
221 |
-
"
|
222 |
-
"
|
223 |
-
"
|
224 |
-
"
|
225 |
-
"
|
226 |
-
"
|
227 |
-
"
|
228 |
-
"
|
229 |
-
"
|
230 |
-
"
|
231 |
-
"
|
232 |
-
"
|
233 |
-
"
|
234 |
-
"
|
235 |
-
"
|
236 |
-
"
|
237 |
-
"
|
238 |
-
"
|
239 |
-
"
|
240 |
-
"
|
241 |
-
"
|
242 |
-
"
|
243 |
-
"
|
244 |
-
"
|
245 |
-
"
|
246 |
-
"
|
247 |
-
"
|
248 |
-
"
|
249 |
-
"
|
250 |
-
"
|
251 |
-
"
|
252 |
-
"
|
253 |
-
"
|
254 |
-
"
|
255 |
-
"
|
256 |
-
"
|
257 |
-
"
|
258 |
-
"
|
259 |
-
"
|
260 |
-
"
|
261 |
-
"
|
262 |
-
"
|
263 |
-
"
|
264 |
-
"
|
265 |
-
"
|
266 |
-
"
|
267 |
-
"
|
268 |
-
"
|
269 |
-
"
|
270 |
-
"
|
271 |
-
"
|
272 |
-
"
|
273 |
-
"
|
274 |
-
"
|
275 |
-
"
|
276 |
-
"
|
277 |
-
"
|
278 |
-
"
|
279 |
-
"
|
280 |
-
"
|
281 |
-
"
|
282 |
-
"
|
283 |
-
"
|
284 |
-
"
|
285 |
-
"
|
286 |
-
"
|
287 |
-
"
|
288 |
-
"
|
289 |
-
"
|
290 |
-
"
|
291 |
-
"
|
292 |
-
"
|
293 |
-
"
|
294 |
-
"
|
295 |
-
"
|
296 |
-
"
|
297 |
-
"
|
298 |
-
"
|
299 |
-
"
|
300 |
-
"
|
301 |
-
"
|
302 |
-
"
|
303 |
-
"
|
304 |
-
"
|
305 |
-
"
|
306 |
-
"
|
307 |
-
"
|
308 |
-
"
|
309 |
-
"
|
310 |
-
"
|
311 |
-
"
|
312 |
-
"
|
313 |
-
"
|
314 |
-
"
|
315 |
-
"
|
316 |
-
"
|
317 |
-
"
|
318 |
-
"
|
319 |
-
"
|
320 |
-
"∴": 253
|
321 |
},
|
322 |
"merges": []
|
323 |
}
|
|
|
13 |
"special": true
|
14 |
},
|
15 |
{
|
16 |
+
"id": 253,
|
17 |
"content": ">",
|
18 |
"single_word": false,
|
19 |
"lstrip": false,
|
|
|
22 |
"special": true
|
23 |
},
|
24 |
{
|
25 |
+
"id": 254,
|
26 |
"content": "<",
|
27 |
"single_word": false,
|
28 |
"lstrip": false,
|
|
|
31 |
"special": true
|
32 |
},
|
33 |
{
|
34 |
+
"id": 255,
|
35 |
"content": "#",
|
36 |
"single_word": false,
|
37 |
"lstrip": false,
|
|
|
183 |
"É": 116,
|
184 |
"Ê": 117,
|
185 |
"Ì": 118,
|
186 |
+
"Î": 119,
|
187 |
+
"Ï": 120,
|
188 |
+
"Ð": 121,
|
189 |
+
"Ô": 122,
|
190 |
+
"Ö": 123,
|
191 |
+
"×": 124,
|
192 |
+
"Ü": 125,
|
193 |
+
"ß": 126,
|
194 |
+
"à": 127,
|
195 |
+
"á": 128,
|
196 |
+
"â": 129,
|
197 |
+
"ã": 130,
|
198 |
+
"ä": 131,
|
199 |
+
"å": 132,
|
200 |
+
"æ": 133,
|
201 |
+
"ç": 134,
|
202 |
+
"è": 135,
|
203 |
+
"é": 136,
|
204 |
+
"ê": 137,
|
205 |
+
"ë": 138,
|
206 |
+
"ì": 139,
|
207 |
+
"í": 140,
|
208 |
+
"î": 141,
|
209 |
+
"ï": 142,
|
210 |
+
"ñ": 143,
|
211 |
+
"ò": 144,
|
212 |
+
"ó": 145,
|
213 |
+
"ô": 146,
|
214 |
+
"õ": 147,
|
215 |
+
"ö": 148,
|
216 |
+
"ø": 149,
|
217 |
+
"ù": 150,
|
218 |
+
"ú": 151,
|
219 |
+
"û": 152,
|
220 |
+
"ü": 153,
|
221 |
+
"ý": 154,
|
222 |
+
"ÿ": 155,
|
223 |
+
"ā": 156,
|
224 |
+
"ć": 157,
|
225 |
+
"č": 158,
|
226 |
+
"Ğ": 159,
|
227 |
+
"ğ": 160,
|
228 |
+
"Ġ": 161,
|
229 |
+
"Ģ": 162,
|
230 |
+
"ģ": 163,
|
231 |
+
"Ĥ": 164,
|
232 |
+
"ĥ": 165,
|
233 |
+
"Ĩ": 166,
|
234 |
+
"ĩ": 167,
|
235 |
+
"Ī": 168,
|
236 |
+
"ī": 169,
|
237 |
+
"Ĭ": 170,
|
238 |
+
"ĭ": 171,
|
239 |
+
"į": 172,
|
240 |
+
"İ": 173,
|
241 |
+
"ı": 174,
|
242 |
+
"IJ": 175,
|
243 |
+
"ij": 176,
|
244 |
+
"Ĵ": 177,
|
245 |
+
"ĵ": 178,
|
246 |
+
"Ķ": 179,
|
247 |
+
"ĸ": 180,
|
248 |
+
"Ĺ": 181,
|
249 |
+
"ĺ": 182,
|
250 |
+
"Ļ": 183,
|
251 |
+
"ļ": 184,
|
252 |
+
"Ľ": 185,
|
253 |
+
"ľ": 186,
|
254 |
+
"Ŀ": 187,
|
255 |
+
"ŀ": 188,
|
256 |
+
"Ł": 189,
|
257 |
+
"ł": 190,
|
258 |
+
"Ń": 191,
|
259 |
+
"ŋ": 192,
|
260 |
+
"ō": 193,
|
261 |
+
"Œ": 194,
|
262 |
+
"œ": 195,
|
263 |
+
"Ś": 196,
|
264 |
+
"ŝ": 197,
|
265 |
+
"Ş": 198,
|
266 |
+
"ş": 199,
|
267 |
+
"š": 200,
|
268 |
+
"ū": 201,
|
269 |
+
"ŵ": 202,
|
270 |
+
"ź": 203,
|
271 |
+
"Ž": 204,
|
272 |
+
"ž": 205,
|
273 |
+
"ǘ": 206,
|
274 |
+
"ȝ": 207,
|
275 |
+
"ɑ": 208,
|
276 |
+
"ɔ": 209,
|
277 |
+
"ə": 210,
|
278 |
+
"ɛ": 211,
|
279 |
+
"ɡ": 212,
|
280 |
+
"ɣ": 213,
|
281 |
+
"ɥ": 214,
|
282 |
+
"ɨ": 215,
|
283 |
+
"ɩ": 216,
|
284 |
+
"ɲ": 217,
|
285 |
+
"ʁ": 218,
|
286 |
+
"ʃ": 219,
|
287 |
+
"ʋ": 220,
|
288 |
+
"ʒ": 221,
|
289 |
+
"ʔ": 222,
|
290 |
+
"ʻ": 223,
|
291 |
+
"ʼ": 224,
|
292 |
+
"ʾ": 225,
|
293 |
+
"ʿ": 226,
|
294 |
+
"́": 227,
|
295 |
+
"̃": 228,
|
296 |
+
"̐": 229,
|
297 |
+
"̓": 230,
|
298 |
+
"͠": 231,
|
299 |
+
"Δ": 232,
|
300 |
+
"α": 233,
|
301 |
+
"β": 234,
|
302 |
+
"μ": 235,
|
303 |
+
"σ": 236,
|
304 |
+
"а": 237,
|
305 |
+
"е": 238,
|
306 |
+
"ḷ": 239,
|
307 |
+
"ṃ": 240,
|
308 |
+
"ṇ": 241,
|
309 |
+
"ṣ": 242,
|
310 |
+
"ṭ": 243,
|
311 |
+
"–": 244,
|
312 |
+
"‘": 245,
|
313 |
+
"’": 246,
|
314 |
+
"′": 247,
|
315 |
+
"‿": 248,
|
316 |
+
"₂": 249,
|
317 |
+
"€": 250,
|
318 |
+
"−": 251,
|
319 |
+
"∴": 252
|
|
|
320 |
},
|
321 |
"merges": []
|
322 |
}
|
vocab.json
CHANGED
@@ -1 +1 @@
|
|
1 |
-
{"<|endoftext|>":0,"\n":1," ":2,"!":3,"&":4,"'":5,"(":6,")":7,"*":8,"+":9,"-":10,".":11,"/":12,"0":13,"1":14,"2":15,"3":16,"4":17,"5":18,"6":19,"7":20,"8":21,"9":22,":":23,";":24,"@":25,"A":26,"B":27,"C":28,"D":29,"E":30,"F":31,"G":32,"H":33,"I":34,"J":35,"K":36,"L":37,"M":38,"N":39,"O":40,"P":41,"Q":42,"R":43,"S":44,"T":45,"U":46,"V":47,"W":48,"X":49,"Y":50,"Z":51,"a":52,"b":53,"c":54,"d":55,"e":56,"f":57,"g":58,"h":59,"i":60,"j":61,"k":62,"l":63,"m":64,"n":65,"o":66,"p":67,"q":68,"r":69,"s":70,"t":71,"u":72,"v":73,"w":74,"x":75,"y":76,"z":77,"¡":78,"¢":79,"£":80,"¤":81,"¥":82,"¦":83,"§":84,"¨":85,"©":86,"ª":87,"«":88,"¬":89,"®":90,"¯":91,"°":92,"±":93,"²":94,"³":95,"´":96,"µ":97,"¶":98,"·":99,"¸":100,"¹":101,"º":102,"»":103,"¼":104,"½":105,"¾":106,"¿":107,"Á":108,"Â":109,"Ã":110,"Ä":111,"Å":112,"Æ":113,"Ç":114,"È":115,"É":116,"Ê":117,"Ì":118,"
|
|
|
1 |
+
{"<|endoftext|>":0,"\n":1," ":2,"!":3,"&":4,"'":5,"(":6,")":7,"*":8,"+":9,"-":10,".":11,"/":12,"0":13,"1":14,"2":15,"3":16,"4":17,"5":18,"6":19,"7":20,"8":21,"9":22,":":23,";":24,"@":25,"A":26,"B":27,"C":28,"D":29,"E":30,"F":31,"G":32,"H":33,"I":34,"J":35,"K":36,"L":37,"M":38,"N":39,"O":40,"P":41,"Q":42,"R":43,"S":44,"T":45,"U":46,"V":47,"W":48,"X":49,"Y":50,"Z":51,"a":52,"b":53,"c":54,"d":55,"e":56,"f":57,"g":58,"h":59,"i":60,"j":61,"k":62,"l":63,"m":64,"n":65,"o":66,"p":67,"q":68,"r":69,"s":70,"t":71,"u":72,"v":73,"w":74,"x":75,"y":76,"z":77,"¡":78,"¢":79,"£":80,"¤":81,"¥":82,"¦":83,"§":84,"¨":85,"©":86,"ª":87,"«":88,"¬":89,"®":90,"¯":91,"°":92,"±":93,"²":94,"³":95,"´":96,"µ":97,"¶":98,"·":99,"¸":100,"¹":101,"º":102,"»":103,"¼":104,"½":105,"¾":106,"¿":107,"Á":108,"Â":109,"Ã":110,"Ä":111,"Å":112,"Æ":113,"Ç":114,"È":115,"É":116,"Ê":117,"Ì":118,"Î":119,"Ï":120,"Ð":121,"Ô":122,"Ö":123,"×":124,"Ü":125,"ß":126,"à":127,"á":128,"â":129,"ã":130,"ä":131,"å":132,"æ":133,"ç":134,"è":135,"é":136,"ê":137,"ë":138,"ì":139,"í":140,"î":141,"ï":142,"ñ":143,"ò":144,"ó":145,"ô":146,"õ":147,"ö":148,"ø":149,"ù":150,"ú":151,"û":152,"ü":153,"ý":154,"ÿ":155,"ā":156,"ć":157,"č":158,"Ğ":159,"ğ":160,"Ġ":161,"Ģ":162,"ģ":163,"Ĥ":164,"ĥ":165,"Ĩ":166,"ĩ":167,"Ī":168,"ī":169,"Ĭ":170,"ĭ":171,"į":172,"İ":173,"ı":174,"IJ":175,"ij":176,"Ĵ":177,"ĵ":178,"Ķ":179,"ĸ":180,"Ĺ":181,"ĺ":182,"Ļ":183,"ļ":184,"Ľ":185,"ľ":186,"Ŀ":187,"ŀ":188,"Ł":189,"ł":190,"Ń":191,"ŋ":192,"ō":193,"Œ":194,"œ":195,"Ś":196,"ŝ":197,"Ş":198,"ş":199,"š":200,"ū":201,"ŵ":202,"ź":203,"Ž":204,"ž":205,"ǘ":206,"ȝ":207,"ɑ":208,"ɔ":209,"ə":210,"ɛ":211,"ɡ":212,"ɣ":213,"ɥ":214,"ɨ":215,"ɩ":216,"ɲ":217,"ʁ":218,"ʃ":219,"ʋ":220,"ʒ":221,"ʔ":222,"ʻ":223,"ʼ":224,"ʾ":225,"ʿ":226,"́":227,"̃":228,"̐":229,"̓":230,"͠":231,"Δ":232,"α":233,"β":234,"μ":235,"σ":236,"а":237,"е":238,"ḷ":239,"ṃ":240,"ṇ":241,"ṣ":242,"ṭ":243,"–":244,"‘":245,"’":246,"′":247,"‿":248,"₂":249,"€":250,"−":251,"∴":252}
|