Repository: huggingface/transformers
File: src/transformers/models/siglip2/tokenization_siglip2.py (Python)

Code before:
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tokenizer for SigLIP2.

SigLIP2 text encoder was trained on lowercased text and expects fixed padding/truncation
to a sequence length of 64 by default.
"""

from __future__ import annotations

from ..gemma.tokenization_gemma import GemmaTokenizer


def _lowercase_text(x):
    if x is None:
        return None
    if isinstance(x, str):
        return x.lower()
    if isinstance(x, (list, tuple)):
        return [_lowercase_text(i) for i in x]
    return x


class Siglip2Tokenizer(GemmaTokenizer):
    """
    SigLIP2 text encoder was trained on:
      - lowercased text
      - fixed padding/truncation to 64 tokens

    This tokenizer wrapper keeps Gemma's underlying vocab/tokenization,
    while applying SigLIP2-specific preprocessing defaults.
    """

    def __call__(self, text=None, text_pair=None, **kwargs):
        # SigLIP2 training: lowercase text
        text = _lowercase_text(text)
        text_pair = _lowercase_text(text_pair)

        # SigLIP2 training: fixed padding/truncation to 64
        kwargs.setdefault("padding", "max_length")
        kwargs.setdefault("truncation", True)
        kwargs.setdefault("max_length", 64)

        return super().__call__(text=text, text_pair=text_pair, **kwargs)


__all__ = ["Siglip2Tokenizer"]


Review comment (suggestion):
if we are to inherit from existing tokenizer class, we better place it in modular. Modular will take care of copying the code and making sure that Siglip2 doesn't depend on Gemma at run-time

Code after:
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from tokenizers import Tokenizer, decoders, normalizers
from tokenizers.models import BPE

from ...tokenization_utils_tokenizers import TokenizersBackend


VOCAB_FILES_NAMES = {"tokenizer_file": "tokenizer.json"}


class Siglip2Tokenizer(TokenizersBackend):
    """
    Gemma tokenizer + SigLIP2 training default: lowercase normalization.
    """

    vocab_files_names = VOCAB_FILES_NAMES
    padding_side = "left"
    model_input_names = ["input_ids", "attention_mask"]
    model = BPE

    def __init__(
        self,
        vocab: str | dict[str, int] | None = None,
        merges: str | list[str] | None = None,
        unk_token: str = "<unk>",
        bos_token: str = "<bos>",
        eos_token: str = "<eos>",
        pad_token: str = "<pad>",
        mask_token: str = "<mask>",
        **kwargs,
    ):
        if vocab is None:
            vocab = {
                str(pad_token): 0,
                str(eos_token): 1,
                str(bos_token): 2,
                str(unk_token): 3,
                str(mask_token): 4,
            }
        self._vocab = vocab
        self._merges = merges or []