Repository: PaddlePaddle/PaddleNLP

Issue:
bart tokenizer对special token的处理和hf有diff
```python
from paddlenlp.transformers import BartTokenizer
from transformers import BartTokenizer as hf_BartTokenizer

model = 'bart-base'
tokenizer = BartTokenizer.from_pretrained(model)

hf_tokenizer = hf_BartTokenizer.from_pretrained('facebook/'+model)

sentence = "I love that girl, but <mask> does not <mask> me."

print(tokenizer.tokenize(sentence))
print(hf_tokenizer.tokenize(sentence))
```
```shell
# 对不齐的输出
paddlenlp
['I', 'Ġlove', 'Ġthat', 'Ġgirl', ',', 'Ġbut', '<mask>', 'does', 'Ġnot', '<mask>', 'me', '.']

hf
['I', 'Ġlove', 'Ġthat', 'Ġgirl', ',', 'Ġbut', '<mask>', 'Ġdoes', 'Ġnot', '<mask>', 'Ġme', '.']
```
paddlenlp==2.2.6可以对齐，paddlenlp>=2.3.0rc0对不齐



Discussion:
这个我先看下哈，roberta这些有AddedToken的好像都有问题，好像是的 `PretrainedTokenizer._wrap_init` 的问题

Resolving patch:
diff --git a/paddlenlp/transformers/tokenizer_utils.py b/paddlenlp/transformers/tokenizer_utils.py
--- a/paddlenlp/transformers/tokenizer_utils.py
+++ b/paddlenlp/transformers/tokenizer_utils.py
@@ -44,8 +44,8 @@
 from .tokenizer_utils_base import (AddedToken, BatchEncoding, EncodedInput,
                                    EncodedInputPair, PreTokenizedInput,
                                    PreTokenizedInputPair,
-                                   PretrainedTokenizerBase, TextInput,
-                                   TextInputPair, TruncationStrategy,
+                                   PretrainedTokenizerBase, SpecialTokensMixin,
+                                   TextInput, TextInputPair, TruncationStrategy,
                                    PaddingStrategy, TensorType)
 
 __all__ = [
@@ -559,24 +559,10 @@ def _wrap_init(self, original_init, *args, **kwargs):
         self._decode_use_source_tokenizer = False
 
     def _build_special_tokens_map_extended(self, **kwargs):
-        for key, value in kwargs.items():
-            if value is None:
-                continue
-            if key in self.SPECIAL_TOKENS_ATTRIBUTES:
-                if key == "additional_special_tokens":
-                    assert isinstance(
-                        value,
-                        (list, tuple)), f"Value {value} is not a list or tuple"
-                    assert all(
-                        isinstance(t, (str, AddedToken)) for t in value
-                    ), "One of the tokens is not a string or an AddedToken"
-                    setattr(self, key, value)
-                elif isinstance(value, (str, AddedToken)):
-                    setattr(self, key, value)
-                else:
-                    raise TypeError(
-                        f"special token {key} has to be either str or AddedToken but got: {type(value)}"
-                    )
+        if getattr(self, "_has_built_special_tokens", None):
+            return
+        SpecialTokensMixin.__init__(self, **kwargs)
+        self._has_built_special_tokens = True
 
     @property
     def vocab_size(self) -> int:
diff --git a/paddlenlp/transformers/tokenizer_utils_base.py b/paddlenlp/transformers/tokenizer_utils_base.py
--- a/paddlenlp/transformers/tokenizer_utils_base.py
+++ b/paddlenlp/transformers/tokenizer_utils_base.py
@@ -778,6 +778,14 @@ class SpecialTokensMixin:
     ]
 
     def __init__(self, verbose=True, **kwargs):
+        # TODO(guosheng): `SpecialTokensMixin.__init__` would be called by
+        # `PretrainedTokenizer._wrap_init` using args which might be str rather
+        # than instances of `AddedToken`, and this causes `AddedToken` settings
+        # made by `_build_special_tokens_map_extended` overwrited.
+        # Maybe `PretrainedTokenizer._wrap_init` should be called before init
+        # later.
+        if getattr(self, "_has_built_special_tokens", None):
+            return
         self._bos_token = None
         self._eos_token = None
         self._unk_token = None


Test patch: