add

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+spm.model filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,22 @@
+# Cohere `rerank-multilingual-v2.0` tokenizer 
+
+This is the tokenizer for the [Cohere Rerank Model](https://txt.cohere.com/rerank/).
+
+You can load it with the transformers library like this:
+```python
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("Cohere/rerank-multilingual-v2.0")
+text = "Hello World, this is my input string!"
+enc = tokenizer(text)
+print("Encoded input:")
+print(enc)
+
+inv_vocab = {v: k for k, v in tokenizer.vocab.items()}
+tokens = [inv_vocab[token_id] for token_id in enc['input_ids']]
+print("Tokens:")
+print(tokens)
+
+number_of_tokens = len(enc['input_ids'])
+print("Number of tokens:", number_of_tokens)
+```

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "[MASK]": 250101
+}

special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13c8d666d62a7bc4ac8f040aab68e942c861f93303156cc28f5c7e885d86d6e3
+size 4305025

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f06fc3bbcbbc8f07be861b9a6a69e177247bc549b2bcbf0483e4dd98c06b6f6
+size 16331301

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "bos_token": "[CLS]",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "split_by_punct": false,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "unk_token": "[UNK]",
+  "vocab_type": "spm"
+}