Repository: huggingface/transformers
File: src/transformers/tokenization_python.py (Python)

Code before:
    """

    def __init__(self, **kwargs):
        # 1. Init the parent class

        self.tokens_trie = Trie()

        # Initialize total_vocab_size early to avoid issues if get_vocab() is called early (custom tokenizers)
        self.total_vocab_size = 0

        # 2. init `_added_tokens_decoder` if child class did not
        if not hasattr(self, "_added_tokens_decoder"):
            self._added_tokens_decoder: dict[int, AddedToken] = {}

        # 3. if a `added_tokens_decoder` is passed, we are loading from a saved tokenizer, we overwrite
        self._added_tokens_decoder.update(kwargs.pop("added_tokens_decoder", {}))
        self._added_tokens_encoder: dict[str, int] = {k.content: v for v, k in self._added_tokens_decoder.items()}

        # 4. Token type ID configuration for dynamic mask building
        # These can be overridden by subclasses to avoid overriding create_token_type_ids_from_sequences
        self.token_type_ids_pattern = kwargs.pop("token_type_ids_pattern", "bert_style")  # "all_zeros" or "bert_style"
        self.token_type_ids_include_special_tokens = kwargs.pop("token_type_ids_include_special_tokens", True)

        # 5. Special tokens mask configuration
        # Patterns: "none", "cls_sep", "eos", "bos", "bos_eos", "cls_double_sep", "prefix_suffix"
        self.special_tokens_pattern = kwargs.pop("special_tokens_pattern", "bos_eos")

        # 6. Set backend to "custom" if not already set (for direct PreTrainedTokenizer subclasses)
        if "backend" not in kwargs:
            kwargs["backend"] = "custom"

        # 7. init the parent class
        super().__init__(**kwargs)

        # 4. If some of the special tokens are not part of the vocab, we add them, at the end.
        # V5: the order of addition follows self.SPECIAL_TOKENS_ATTRIBUTES, then extra special tokens
        # Note: _add_tokens will automatically skip tokens that are already in the base vocab
        self._add_tokens(
            [token for token in self.all_special_tokens if token not in self._added_tokens_encoder],
            special_tokens=True,
        )

    @property
    def is_fast(self) -> bool:
        return False

    @property
    def added_tokens_encoder(self) -> dict[str, int]:
        """
        Returns the sorted mapping from string to index. The added tokens encoder is cached for performance
        optimisation in `self._added_tokens_encoder` for the slow tokenizers.

Review comment (suggestion):
cc @itazap @ArthurZucker , i want to clarify this part. Should we default to `None` because cls-sep ids arent always available for all tokenizers. Thus we are getting `[None, 1, 18001, 468, None]` as token ids for those models

Ignore the current change to `bos_eos`

Code after:
    """

    def __init__(self, **kwargs):
        # 1. Init the parent class

        self.tokens_trie = Trie()

        # Initialize total_vocab_size early to avoid issues if get_vocab() is called early (custom tokenizers)
        self.total_vocab_size = 0

        # 2. init `_added_tokens_decoder` if child class did not
        if not hasattr(self, "_added_tokens_decoder"):
            self._added_tokens_decoder: dict[int, AddedToken] = {}

        # 3. if a `added_tokens_decoder` is passed, we are loading from a saved tokenizer, we overwrite
        self._added_tokens_decoder.update(kwargs.pop("added_tokens_decoder", {}))
        self._added_tokens_encoder: dict[str, int] = {k.content: v for v, k in self._added_tokens_decoder.items()}

        # 4. Token type ID configuration for dynamic mask building
        # These can be overridden by subclasses to avoid overriding create_token_type_ids_from_sequences
        self.token_type_ids_pattern = kwargs.pop("token_type_ids_pattern", "bert_style")  # "all_zeros" or "bert_style"
        self.token_type_ids_include_special_tokens = kwargs.pop("token_type_ids_include_special_tokens", True)

        # 5. Special tokens mask configuration
        # Patterns: "none", "cls_sep", "eos", "bos", "bos_eos", "cls_double_sep", "prefix_suffix"
        self.special_tokens_pattern = kwargs.pop("special_tokens_pattern", None)

        # 6. Set backend to "custom" if not already set (for direct PreTrainedTokenizer subclasses)
        if "backend" not in kwargs:
            kwargs["backend"] = "custom"

        # 7. init the parent class
        super().__init__(**kwargs)

        # 4. If some of the special tokens are not part of the vocab, we add them, at the end.
        # V5: the order of addition follows self.SPECIAL_TOKENS_ATTRIBUTES, then extra special tokens
        # Note: _add_tokens will automatically skip tokens that are already in the base vocab
        self._add_tokens(
            [token for token in self.all_special_tokens if token not in self._added_tokens_encoder],
            special_tokens=True,
        )

    @property
    def is_fast(self) -> bool:
        return False

    @property
    def added_tokens_encoder(self) -> dict[str, int]:
        """
        Returns the sorted mapping from string to index. The added tokens encoder is cached for performance
        optimisation in `self._added_tokens_encoder` for the slow tokenizers.