add files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+spm.model filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,22 @@
+# Cohere `rerank-english-v2.0` tokenizer 
+
+This is the tokenizer for the [Cohere Rerank Model](https://txt.cohere.com/rerank/).
+
+You can load it with the transformers library like this:
+```python
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("Cohere/rerank-english-v2.0")
+text = "Hello World, this is my input string!"
+enc = tokenizer(text)
+print("Encoded input:")
+print(enc)
+
+inv_vocab = {v: k for k, v in tokenizer.vocab.items()}
+tokens = [inv_vocab[token_id] for token_id in enc['input_ids']]
+print("Tokens:")
+print(tokens)
+
+number_of_tokens = len(enc['input_ids'])
+print("Number of tokens:", number_of_tokens)
+```

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "[MASK]": 128000
+}

special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
+size 2464616

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a86f883318afa11c8c10466f1bf4efaeb6ded28a52cbe57217a8fa0d0a2a87df
+size 8656551

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "bos_token": "[CLS]",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "split_by_punct": false,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "unk_token": "[UNK]",
+  "vocab_type": "spm"
+}