add tokenizer

added_tokens.json

@@ -1 +1 @@
-{"<unk>":128104, "<unk>":128105, "<unk>":128106, "<unk>":128107, "<unk>":128108, "<unk>":128109, "<unk>":128110, "<unk>":128111, "<triplet>": 128112, "<relation>": 128113, "el_EL": 128114, "ca_XX": 128115, "tp_XX": 128116, "<loc>": 128117, "<misc>": 128118, "<per>": 128119, "<num>": 128120, "<time>": 128121, "<org>": 128122, "<date>": 128123, "<eve>": 128124, "<cel>": 128125, "<media>": 128126, "<dis>": 128127, "<concept>": 128128}
+{"<triplet>": 128112, "<relation>": 128113, "el_EL": 128114, "ca_XX": 128115, "tp_XX": 128116, "<loc>": 128117, "<misc>": 128118, "<per>": 128119, "<num>": 128120, "<time>": 128121, "<org>": 128122, "<date>": 128123, "<eve>": 128124, "<cel>": 128125, "<media>": 128126, "<dis>": 128127, "<concept>": 128128}

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "</s>", "pad_token": "<pad>", "additional_special_tokens": ["<unk>", "<unk>", "<unk>", "<unk>", "<unk>", "<unk>", "<unk>", "<unk>","<triplet>", "<relation>", "el_EL", "ca_XX", "tp_XX", "<loc>", "<misc>", "<per>", "<num>", "<time>", "<org>", "<date>", "<eve>", "<cel>", "<media>", "<dis>", "<unk>", "<concept>", "<loc>", "<misc>", "<per>", "<num>", "<time>", "<org>", "<date>", "<eve>", "<cel>", "<media>", "<dis>", "<unk>", "<concept>", "__af__", "__am__", "__ar__", "__ast__", "__az__", "__ba__", "__be__", "__bg__", "__bn__", "__br__", "__bs__", "__ca__", "__ceb__", "__cs__", "__cy__", "__da__", "__de__", "__el__", "__en__", "__es__", "__et__", "__fa__", "__ff__", "__fi__", "__fr__", "__fy__", "__ga__", "__gd__", "__gl__", "__gu__", "__ha__", "__he__", "__hi__", "__hr__", "__ht__", "__hu__", "__hy__", "__id__", "__ig__", "__ilo__", "__is__", "__it__", "__ja__", "__jv__", "__ka__", "__kk__", "__km__", "__kn__", "__ko__", "__lb__", "__lg__", "__ln__", "__lo__", "__lt__", "__lv__", "__mg__", "__mk__", "__ml__", "__mn__", "__mr__", "__ms__", "__my__", "__ne__", "__nl__", "__no__", "__ns__", "__oc__", "__or__", "__pa__", "__pl__", "__ps__", "__pt__", "__ro__", "__ru__", "__sd__", "__si__", "__sk__", "__sl__", "__so__", "__sq__", "__sr__", "__ss__", "__su__", "__sv__", "__sw__", "__ta__", "__th__", "__tl__", "__tn__", "__tr__", "__uk__", "__ur__", "__uz__", "__vi__", "__wo__", "__xh__", "__yi__", "__yo__", "__zh__", "__zu__"]}
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "</s>", "pad_token": "<pad>", "additional_special_tokens": ["<triplet>", "<relation>", "el_EL", "ca_XX", "tp_XX", "<loc>", "<misc>", "<per>", "<num>", "<time>", "<org>", "<date>", "<eve>", "<cel>", "<media>", "<dis>", "<unk>", "<concept>", "<loc>", "<misc>", "<per>", "<num>", "<time>", "<org>", "<date>", "<eve>", "<cel>", "<media>", "<dis>", "<unk>", "<concept>", "__af__", "__am__", "__ar__", "__ast__", "__az__", "__ba__", "__be__", "__bg__", "__bn__", "__br__", "__bs__", "__ca__", "__ceb__", "__cs__", "__cy__", "__da__", "__de__", "__el__", "__en__", "__es__", "__et__", "__fa__", "__ff__", "__fi__", "__fr__", "__fy__", "__ga__", "__gd__", "__gl__", "__gu__", "__ha__", "__he__", "__hi__", "__hr__", "__ht__", "__hu__", "__hy__", "__id__", "__ig__", "__ilo__", "__is__", "__it__", "__ja__", "__jv__", "__ka__", "__kk__", "__km__", "__kn__", "__ko__", "__lb__", "__lg__", "__ln__", "__lo__", "__lt__", "__lv__", "__mg__", "__mk__", "__ml__", "__mn__", "__mr__", "__ms__", "__my__", "__ne__", "__nl__", "__no__", "__ns__", "__oc__", "__or__", "__pa__", "__pl__", "__ps__", "__pt__", "__ro__", "__ru__", "__sd__", "__si__", "__sk__", "__sl__", "__so__", "__sq__", "__sr__", "__ss__", "__su__", "__sv__", "__sw__", "__ta__", "__th__", "__tl__", "__tn__", "__tr__", "__uk__", "__ur__", "__uz__", "__vi__", "__wo__", "__xh__", "__yi__", "__yo__", "__zh__", "__zu__"]}