make config similar to transformers except for flash

configuration_siglip.py

@@ -1,5 +1,5 @@
 # coding=utf-8
-# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
+# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,17 +15,10 @@
 """ Siglip model configuration"""
 
 import os
-from collections import OrderedDict
-from typing import TYPE_CHECKING, Any, Mapping, Optional, Union
+from typing import Union
 
-
-if TYPE_CHECKING:
-    from transformers.processing_utils import ProcessorMixin
-    from transformers.utils import TensorType
-
-from transformers.configuration_utils import PretrainedConfig
-from transformers.onnx import OnnxConfig
-from transformers.utils import logging
+from ...configuration_utils import PretrainedConfig
+from ...utils import logging
 
 
 logger = logging.get_logger(__name__)
@@ -46,16 +39,16 @@ class SiglipTextConfig(PretrainedConfig):
     documentation from [`PretrainedConfig`] for more information.
 
     Args:
-        vocab_size (`int`, *optional*, defaults to 49408):
+        vocab_size (`int`, *optional*, defaults to 32000):
             Vocabulary size of the Siglip text model. Defines the number of different tokens that can be represented by
             the `inputs_ids` passed when calling [`SiglipModel`].
-        hidden_size (`int`, *optional*, defaults to 512):
+        hidden_size (`int`, *optional*, defaults to 768):
             Dimensionality of the encoder layers and the pooler layer.
-        intermediate_size (`int`, *optional*, defaults to 2048):
+        intermediate_size (`int`, *optional*, defaults to 3072):
             Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
         num_hidden_layers (`int`, *optional*, defaults to 12):
             Number of hidden layers in the Transformer encoder.
-        num_attention_heads (`int`, *optional*, defaults to 8):
+        num_attention_heads (`int`, *optional*, defaults to 12):
             Number of attention heads for each attention layer in the Transformer encoder.
         max_position_embeddings (`int`, *optional*, defaults to 64):
             The maximum sequence length that this model might ever be used with. Typically set this to something large
@@ -63,15 +56,16 @@ class SiglipTextConfig(PretrainedConfig):
         hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
             The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
             `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported.
-        layer_norm_eps (`float`, *optional*, defaults to 1e-6):
+        layer_norm_eps (`float`, *optional*, defaults to 1e-06):
             The epsilon used by the layer normalization layers.
         attention_dropout (`float`, *optional*, defaults to 0.0):
             The dropout ratio for the attention probabilities.
-        initializer_range (`float`, *optional*, defaults to 0.02):
-            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
-        initializer_factor (`float`, *optional*, defaults to 1):
-            A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
-            testing).
+        pad_token_id (`int`, *optional*, defaults to 1):
+            The id of the padding token in the vocabulary.
+        bos_token_id (`int`, *optional*, defaults to 49406):
+            The id of the beginning-of-sequence token in the vocabulary.
+        eos_token_id (`int`, *optional*, defaults to 49407):
+            The id of the end-of-sequence token in the vocabulary.
 
     Example:
 
@@ -87,22 +81,20 @@ class SiglipTextConfig(PretrainedConfig):
     >>> # Accessing the model configuration
     >>> configuration = model.config
     ```"""
+
     model_type = "siglip_text_model"
 
     def __init__(
         self,
-        vocab_size=49408,
-        hidden_size=512,
-        intermediate_size=2048,
-        projection_dim=512,
+        vocab_size=32000,
+        hidden_size=768,
+        intermediate_size=3072,
         num_hidden_layers=12,
-        num_attention_heads=8,
+        num_attention_heads=12,
         max_position_embeddings=64,
         hidden_act="gelu_pytorch_tanh",
         layer_norm_eps=1e-6,
         attention_dropout=0.0,
-        initializer_range=0.02,
-        initializer_factor=1.0,
         # This differs from `CLIPTokenizer`'s default and from openai/siglip
         # See https://github.com/huggingface/transformers/pull/24773#issuecomment-1632287538
         pad_token_id=1,
@@ -116,14 +108,11 @@ class SiglipTextConfig(PretrainedConfig):
         self.vocab_size = vocab_size
         self.hidden_size = hidden_size
         self.intermediate_size = intermediate_size
-        self.projection_dim = projection_dim
         self.num_hidden_layers = num_hidden_layers
         self.num_attention_heads = num_attention_heads
         self.max_position_embeddings = max_position_embeddings
         self.layer_norm_eps = layer_norm_eps
         self.hidden_act = hidden_act
-        self.initializer_range = initializer_range
-        self.initializer_factor = initializer_factor
         self.attention_dropout = attention_dropout
         self._flash_attn_2_enabled = _flash_attn_2_enabled
 
@@ -165,22 +154,19 @@ class SiglipVisionConfig(PretrainedConfig):
             Number of hidden layers in the Transformer encoder.
         num_attention_heads (`int`, *optional*, defaults to 12):
             Number of attention heads for each attention layer in the Transformer encoder.
+        num_channels (`int`, *optional*, defaults to 3):
+            Number of channels in the input images.
         image_size (`int`, *optional*, defaults to 224):
             The size (resolution) of each image.
-        patch_size (`int`, *optional*, defaults to 32):
+        patch_size (`int`, *optional*, defaults to 16):
             The size (resolution) of each patch.
         hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
             The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
             `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported.
-        layer_norm_eps (`float`, *optional*, defaults to 1e-6):
+        layer_norm_eps (`float`, *optional*, defaults to 1e-06):
             The epsilon used by the layer normalization layers.
         attention_dropout (`float`, *optional*, defaults to 0.0):
             The dropout ratio for the attention probabilities.
-        initializer_range (`float`, *optional*, defaults to 0.02):
-            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
-        initializer_factor (`float`, *optional*, defaults to 1):
-            A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
-            testing).
 
     Example:
 
@@ -203,17 +189,14 @@ class SiglipVisionConfig(PretrainedConfig):
         self,
         hidden_size=768,
         intermediate_size=3072,
-        projection_dim=512,
         num_hidden_layers=12,
         num_attention_heads=12,
         num_channels=3,
         image_size=224,
-        patch_size=32,
+        patch_size=16,
         hidden_act="gelu_pytorch_tanh",
         layer_norm_eps=1e-6,
         attention_dropout=0.0,
-        initializer_range=0.02,
-        initializer_factor=1.0,
         _flash_attn_2_enabled=True,
         **kwargs,
     ):
@@ -221,14 +204,11 @@ class SiglipVisionConfig(PretrainedConfig):
 
         self.hidden_size = hidden_size
         self.intermediate_size = intermediate_size
-        self.projection_dim = projection_dim
         self.num_hidden_layers = num_hidden_layers
         self.num_attention_heads = num_attention_heads
         self.num_channels = num_channels
         self.patch_size = patch_size
         self.image_size = image_size
-        self.initializer_range = initializer_range
-        self.initializer_factor = initializer_factor
         self.attention_dropout = attention_dropout
         self.layer_norm_eps = layer_norm_eps
         self.hidden_act = hidden_act
@@ -268,10 +248,6 @@ class SiglipConfig(PretrainedConfig):
             Dictionary of configuration options used to initialize [`SiglipTextConfig`].
         vision_config (`dict`, *optional*):
             Dictionary of configuration options used to initialize [`SiglipVisionConfig`].
-        projection_dim (`int`, *optional*, defaults to 512):
-            Dimentionality of text and vision projection layers.
-        logit_scale_init_value (`float`, *optional*, defaults to 2.6592):
-            The inital value of the *logit_scale* paramter. Default is used as per the original Siglip implementation.
         kwargs (*optional*):
             Dictionary of keyword arguments.
 
@@ -301,79 +277,9 @@ class SiglipConfig(PretrainedConfig):
 
     model_type = "siglip"
 
-    def __init__(
-        self, text_config=None, vision_config=None, projection_dim=512, logit_scale_init_value=2.6592, **kwargs
-    ):
-        # If `_config_dict` exist, we use them for the backward compatibility.
-        # We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot
-        # of confusion!).
-        text_config_dict = kwargs.pop("text_config_dict", None)
-        vision_config_dict = kwargs.pop("vision_config_dict", None)
-
+    def __init__(self, text_config=None, vision_config=None, **kwargs):
         super().__init__(**kwargs)
 
-        # Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in
-        # `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most
-        # cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`.
-        if text_config_dict is not None:
-            if text_config is None:
-                text_config = {}
-
-            # This is the complete result when using `text_config_dict`.
-            _text_config_dict = SiglipTextConfig(**text_config_dict).to_dict()
-
-            # Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different.
-            for key, value in _text_config_dict.items():
-                if key in text_config and value != text_config[key] and key not in ["transformers_version"]:
-                    # If specified in `text_config_dict`
-                    if key in text_config_dict:
-                        message = (
-                            f"`{key}` is found in both `text_config_dict` and `text_config` but with different values."
-                            f' The value `text_config_dict["{key}"]` will be used instead.'
-                        )
-                    # If inferred from default argument values (just to be super careful)
-                    else:
-                        message = (
-                            "`text_config_dict` is provided which will be used to initialize `SiglipTextConfig`. The "
-                            f'value `text_config["{key}"]` will be overriden.'
-                        )
-                    logger.warning(message)
-
-            # Update all values in `text_config` with the ones in `_text_config_dict`.
-            text_config.update(_text_config_dict)
-
-        if vision_config_dict is not None:
-            if vision_config is None:
-                vision_config = {}
-
-            # This is the complete result when using `vision_config_dict`.
-            _vision_config_dict = SiglipVisionConfig(**vision_config_dict).to_dict()
-            # convert keys to string instead of integer
-            if "id2label" in _vision_config_dict:
-                _vision_config_dict["id2label"] = {
-                    str(key): value for key, value in _vision_config_dict["id2label"].items()
-                }
-
-            # Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different.
-            for key, value in _vision_config_dict.items():
-                if key in vision_config and value != vision_config[key] and key not in ["transformers_version"]:
-                    # If specified in `vision_config_dict`
-                    if key in vision_config_dict:
-                        message = (
-                            f"`{key}` is found in both `vision_config_dict` and `vision_config` but with different "
-                            f'values. The value `vision_config_dict["{key}"]` will be used instead.'
-                        )
-                    # If inferred from default argument values (just to be super careful)
-                    else:
-                        message = (
-                            "`vision_config_dict` is provided which will be used to initialize `SiglipVisionConfig`. "
-                            f'The value `vision_config["{key}"]` will be overriden.'
-                        )
-                    logger.warning(message)
-
-            # Update all values in `vision_config` with the ones in `_vision_config_dict`.
-            vision_config.update(_vision_config_dict)
-
         if text_config is None:
             text_config = {}
             logger.info("`text_config` is `None`. Initializing the `SiglipTextConfig` with default values.")
@@ -385,8 +291,6 @@ class SiglipConfig(PretrainedConfig):
         self.text_config = SiglipTextConfig(**text_config)
         self.vision_config = SiglipVisionConfig(**vision_config)
 
-        self.projection_dim = projection_dim
-        self.logit_scale_init_value = logit_scale_init_value
         self.initializer_factor = 1.0
 
     @classmethod
@@ -400,49 +304,3 @@ class SiglipConfig(PretrainedConfig):
         """
 
         return cls(text_config=text_config.to_dict(), vision_config=vision_config.to_dict(), **kwargs)
-
-
-class SiglipOnnxConfig(OnnxConfig):
-    @property
-    def inputs(self) -> Mapping[str, Mapping[int, str]]:
-        return OrderedDict(
-            [
-                ("input_ids", {0: "batch", 1: "sequence"}),
-                ("pixel_values", {0: "batch", 1: "num_channels", 2: "height", 3: "width"}),
-                ("attention_mask", {0: "batch", 1: "sequence"}),
-            ]
-        )
-
-    @property
-    def outputs(self) -> Mapping[str, Mapping[int, str]]:
-        return OrderedDict(
-            [
-                ("logits_per_image", {0: "batch"}),
-                ("logits_per_text", {0: "batch"}),
-                ("text_embeds", {0: "batch"}),
-                ("image_embeds", {0: "batch"}),
-            ]
-        )
-
-    @property
-    def atol_for_validation(self) -> float:
-        return 1e-4
-
-    def generate_dummy_inputs(
-        self,
-        processor: "ProcessorMixin",
-        batch_size: int = -1,
-        seq_length: int = -1,
-        framework: Optional["TensorType"] = None,
-    ) -> Mapping[str, Any]:
-        text_input_dict = super().generate_dummy_inputs(
-            processor.tokenizer, batch_size=batch_size, seq_length=seq_length, framework=framework
-        )
-        image_input_dict = super().generate_dummy_inputs(
-            processor.image_processor, batch_size=batch_size, framework=framework
-        )
-        return {**text_input_dict, **image_input_dict}
-
-    @property
-    def default_onnx_opset(self) -> int:
-        return 14