Repository: huggingface/transformers
File: src/transformers/models/siglip2/tokenization_siglip2.py (Python)

Code before:
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tokenizer for SigLIP2.

SigLIP2 text encoder was trained on lowercased text and expects fixed padding/truncation
to a sequence length of 64 by default.
"""

from __future__ import annotations

from ..gemma.tokenization_gemma import GemmaTokenizer


def _lowercase_text(x):
    if x is None:
        return None
    if isinstance(x, str):
        return x.lower()
    if isinstance(x, (list, tuple)):
        return [_lowercase_text(i) for i in x]
    return x


class Siglip2Tokenizer(GemmaTokenizer):
    """
    SigLIP2 text encoder was trained on:
      - lowercased text
      - fixed padding/truncation to 64 tokens

    This tokenizer wrapper keeps Gemma's underlying vocab/tokenization,
    while applying SigLIP2-specific preprocessing defaults.
    """

    def __call__(self, text=None, text_pair=None, **kwargs):
        # SigLIP2 training: lowercase text
        text = _lowercase_text(text)
        text_pair = _lowercase_text(text_pair)

        # SigLIP2 training: fixed padding/truncation to 64
        kwargs.setdefault("padding", "max_length")
        kwargs.setdefault("truncation", True)
        kwargs.setdefault("max_length", 64)

        return super().__call__(text=text, text_pair=text_pair, **kwargs)


__all__ = ["Siglip2Tokenizer"]


Review comment (suggestion):
we never use inheritance in transformers like this. Lowercasing is a normalizer from `normalizers.Lowercase` that just needs to be added to the sequence of normalizers for siglip models. 

Code after:
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from tokenizers import Tokenizer, decoders, normalizers
from tokenizers.models import BPE

from ...tokenization_utils_tokenizers import TokenizersBackend


VOCAB_FILES_NAMES = {"tokenizer_file": "tokenizer.json"}


class Siglip2Tokenizer(TokenizersBackend):
    """
    Gemma tokenizer + SigLIP2 training default: lowercase normalization.
    """

    vocab_files_names = VOCAB_FILES_NAMES
    padding_side = "left"
    model_input_names = ["input_ids", "attention_mask"]
    model = BPE

    def __init__(
        self,
        vocab: str | dict[str, int] | None = None,
        merges: str | list[str] | None = None,
        unk_token: str = "<unk>",
        bos_token: str = "<bos>",
        eos_token: str = "<eos>",
        pad_token: str = "<pad>",
        mask_token: str = "<mask>",
        **kwargs,
    ):
        if vocab is None:
            vocab = {
                str(pad_token): 0,
                str(eos_token): 1,
                str(bos_token): 2,
                str(unk_token): 3,
                str(mask_token): 4,
            }
        self._vocab = vocab
        self._merges = merges or []