Repository: huggingface/transformers

Issue:
Loading MPNet from disc: ValueError: An instance of tokenizer class MPNetTokenizer cannot be converted in a Fast tokenizer instance.
## Environment info
- `transformers` version: 4.1.1 (pip version)
- Platform: Ubuntu 20.04
- Python version: 3.7
- PyTorch version (GPU?): Pytorch 1.7 GPU



## Information

Hi,
thanks for adding MPNet. I got quite promising results when using it for generating sentence embeddings.

However, there is an issue when saving and loading the MPNet model (when using version 4.1.1 of transformers, installed via pip):
```python
from transformers import AutoTokenizer, AutoModel

local_dir = 'mpnet-model/'

model = AutoModel.from_pretrained('microsoft/mpnet-base')
tokenizer = AutoTokenizer.from_pretrained('microsoft/mpnet-base')
model.save_pretrained(local_dir)
tokenizer.save_pretrained(local_dir)

#Load tokenizer and model from dir
model = AutoModel.from_pretrained(local_dir)

# The following command will throw an exception
tokenizer = AutoTokenizer.from_pretrained(local_dir)

```

This leads to the following error:
```
File "/home/reimers/miniconda3/envs/sbert/lib/python3.7/site-packages/transformers/convert_slow_tokenizer.py", line 636, in convert_slow_tokenizer
    f"An instance of tokenizer class {tokenizer_class_name} cannot be converted in a Fast tokenizer instance. "
ValueError: An instance of tokenizer class MPNetTokenizer cannot be converted in a Fast tokenizer instance. No converter was found. Currently available slow->fast convertors: ['AlbertTokenizer', 'BartTokenizer', 'BarthezTokenizer', 'BertTokenizer', 'CamembertTokenizer', 'DistilBertTokenizer', 'DPRReaderTokenizer', 'DPRQuestionEncoderTokenizer', 'DPRContextEncoderTokenizer', 'ElectraTokenizer', 'FunnelTokenizer', 'GPT2Tokenizer', 'HerbertTokenizer', 'LayoutLMTokenizer', 'LongformerTokenizer', 'LxmertTokenizer', 'MBartTokenizer', 'MobileBertTokenizer', 'OpenAIGPTTokenizer', 'PegasusTokenizer', 'ReformerTokenizer', 'RetriBertTokenizer', 'RobertaTokenizer', 'SqueezeBertTokenizer', 'T5Tokenizer', 'XLMRobertaTokenizer', 'XLNetTokenizer']
```




Discussion:
Looking into it!

An easy fix for now would be the following:

```python
from transformers import AutoTokenizer, AutoModel

local_dir = 'mpnet-model/'

model = AutoModel.from_pretrained('microsoft/mpnet-base')
tokenizer = AutoTokenizer.from_pretrained('microsoft/mpnet-base')
model.save_pretrained(local_dir)
tokenizer.save_pretrained(local_dir)

#Load tokenizer and model from dir
model = AutoModel.from_pretrained(local_dir)

# The following command will throw an exception
tokenizer = AutoTokenizer.from_pretrained(local_dir, use_fast=False)
```

but that's more of a hack than really solving the underlying problem as it just loads the slow tokenizer. Will check what's going on!


Resolving patch:
diff --git a/src/transformers/convert_slow_tokenizer.py b/src/transformers/convert_slow_tokenizer.py
--- a/src/transformers/convert_slow_tokenizer.py
+++ b/src/transformers/convert_slow_tokenizer.py
@@ -74,18 +74,6 @@ def converted(self) -> Tokenizer:
         vocab = self.original_tokenizer.vocab
         tokenizer = Tokenizer(WordPiece(vocab, unk_token=str(self.original_tokenizer.unk_token)))
 
-        # # Let the tokenizer know about special tokens if they are part of the vocab
-        # if tokenizer.token_to_id(str(self.original_tokenizer.unk_token)) is not None:
-        #     tokenizer.add_special_tokens([str(self.original_tokenizer.unk_token)])
-        # if tokenizer.token_to_id(str(self.original_tokenizer.sep_token)) is not None:
-        #     tokenizer.add_special_tokens([str(self.original_tokenizer.sep_token)])
-        # if tokenizer.token_to_id(str(self.original_tokenizer.cls_token)) is not None:
-        #     tokenizer.add_special_tokens([str(self.original_tokenizer.cls_token)])
-        # if tokenizer.token_to_id(str(self.original_tokenizer.pad_token)) is not None:
-        #     tokenizer.add_special_tokens([str(self.original_tokenizer.pad_token)])
-        # if tokenizer.token_to_id(str(self.original_tokenizer.mask_token)) is not None:
-        #     tokenizer.add_special_tokens([str(self.original_tokenizer.mask_token)])
-
         tokenize_chinese_chars = False
         strip_accents = False
         do_lower_case = False
@@ -125,18 +113,6 @@ def converted(self) -> Tokenizer:
         vocab = self.original_tokenizer.vocab
         tokenizer = Tokenizer(WordPiece(vocab, unk_token=str(self.original_tokenizer.unk_token)))
 
-        # # Let the tokenizer know about special tokens if they are part of the vocab
-        # if tokenizer.token_to_id(str(self.original_tokenizer.unk_token)) is not None:
-        #     tokenizer.add_special_tokens([str(self.original_tokenizer.unk_token)])
-        # if tokenizer.token_to_id(str(self.original_tokenizer.sep_token)) is not None:
-        #     tokenizer.add_special_tokens([str(self.original_tokenizer.sep_token)])
-        # if tokenizer.token_to_id(str(self.original_tokenizer.cls_token)) is not None:
-        #     tokenizer.add_special_tokens([str(self.original_tokenizer.cls_token)])
-        # if tokenizer.token_to_id(str(self.original_tokenizer.pad_token)) is not None:
-        #     tokenizer.add_special_tokens([str(self.original_tokenizer.pad_token)])
-        # if tokenizer.token_to_id(str(self.original_tokenizer.mask_token)) is not None:
-        #     tokenizer.add_special_tokens([str(self.original_tokenizer.mask_token)])
-
         tokenize_chinese_chars = False
         strip_accents = False
         do_lower_case = False
@@ -171,6 +147,45 @@ def converted(self) -> Tokenizer:
         return tokenizer
 
 
+class MPNetConverter(Converter):
+    def converted(self) -> Tokenizer:
+        vocab = self.original_tokenizer.vocab
+        tokenizer = Tokenizer(WordPiece(vocab, unk_token=str(self.original_tokenizer.unk_token)))
+
+        tokenize_chinese_chars = False
+        strip_accents = False
+        do_lower_case = False
+        if hasattr(self.original_tokenizer, "basic_tokenizer"):
+            tokenize_chinese_chars = self.original_tokenizer.basic_tokenizer.tokenize_chinese_chars
+            strip_accents = self.original_tokenizer.basic_tokenizer.strip_accents
+            do_lower_case = self.original_tokenizer.basic_tokenizer.do_lower_case
+
+        tokenizer.normalizer = normalizers.BertNormalizer(
+            clean_text=True,
+            handle_chinese_chars=tokenize_chinese_chars,
+            strip_accents=strip_accents,
+            lowercase=do_lower_case,
+        )
+        tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer()
+
+        cls = str(self.original_tokenizer.cls_token)
+        sep = str(self.original_tokenizer.sep_token)
+        cls_token_id = self.original_tokenizer.cls_token_id
+        sep_token_id = self.original_tokenizer.sep_token_id
+
+        tokenizer.post_processor = processors.TemplateProcessing(
+            single=f"{cls}:0 $A:0 {sep}:0",
+            pair=f"{cls}:0 $A:0 {sep}:0 {sep}:0 $B:1 {sep}:1",  # MPNet uses two [SEP] tokens
+            special_tokens=[
+                (cls, cls_token_id),
+                (sep, sep_token_id),
+            ],
+        )
+        tokenizer.decoder = decoders.WordPiece(prefix="##")
+
+        return tokenizer
+
+
 class OpenAIGPTConverter(Converter):
     def converted(self) -> Tokenizer:
         vocab = self.original_tokenizer.encoder
@@ -602,6 +617,7 @@ def post_processor(self):
     "LongformerTokenizer": RobertaConverter,
     "LxmertTokenizer": BertConverter,
     "MBartTokenizer": MBartConverter,
+    "MPNetTokenizer": MPNetConverter,
     "MobileBertTokenizer": BertConverter,
     "OpenAIGPTTokenizer": OpenAIGPTConverter,
     "PegasusTokenizer": PegasusConverter,


Test patch:
diff --git a/tests/test_tokenization_mpnet.py b/tests/test_tokenization_mpnet.py
--- a/tests/test_tokenization_mpnet.py
+++ b/tests/test_tokenization_mpnet.py
@@ -17,6 +17,7 @@
 import os
 import unittest
 
+from transformers import MPNetTokenizerFast
 from transformers.models.mpnet.tokenization_mpnet import VOCAB_FILES_NAMES, MPNetTokenizer
 from transformers.testing_utils import require_tokenizers, slow
 
@@ -27,7 +28,9 @@
 class MPNetTokenizerTest(TokenizerTesterMixin, unittest.TestCase):
 
     tokenizer_class = MPNetTokenizer
-    test_rust_tokenizer = False
+    rust_tokenizer_class = MPNetTokenizerFast
+    test_rust_tokenizer = True
+    space_between_special_tokens = True
 
     def setUp(self):
         super().setUp()