add tokenizer

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1 @@
+{"[START]": 250002, "[NIL]": 250005, "[TYPE]": 250006, "[TITLE]": 250004, "[END]": 250003}

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
+size 5069051

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "</s>", "pad_token": "<pad>", "cls_token": "<s>", "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false}, "additional_special_tokens": ["[START]", "[END]", "[TITLE]", "[NIL]", "[TYPE]"]}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b83a0787f6b7957e0734197117ba1f6bd8c83b516c26752670dbb064737986a
+size 17083828

tokenizer_config.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "sep_token": "</s>", "cls_token": "<s>", "unk_token": "<unk>", "pad_token": "<pad>", "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "clean_up_tokenization_spaces": true, "model_max_length": 512, "special_tokens_map_file": "../models/es_ce_medmentions/special_tokens_map.json", "name_or_path": "../models/es_ce_medmentions/", "tokenizer_class": "XLMRobertaTokenizer"}