Spaces:
Running
on
T4
Running
on
T4
Grounded-Segment-Anything
/
transformers_4_35_0
/models
/chinese_clip
/configuration_chinese_clip.py
# coding=utf-8 | |
# Copyright 2022 The OFA-Sys Team Authors and The HuggingFace Team. All rights reserved. | |
# | |
# Licensed under the Apache License, Version 2.0 (the "License"); | |
# you may not use this file except in compliance with the License. | |
# You may obtain a copy of the License at | |
# | |
# http://www.apache.org/licenses/LICENSE-2.0 | |
# | |
# Unless required by applicable law or agreed to in writing, software | |
# distributed under the License is distributed on an "AS IS" BASIS, | |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
# See the License for the specific language governing permissions and | |
# limitations under the License. | |
""" Chinese-CLIP model configuration""" | |
import os | |
from collections import OrderedDict | |
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union | |
if TYPE_CHECKING: | |
from ...processing_utils import ProcessorMixin | |
from ...utils import TensorType | |
from ...configuration_utils import PretrainedConfig | |
from ...onnx import OnnxConfig | |
from ...utils import logging | |
logger = logging.get_logger(__name__) | |
CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP = { | |
"OFA-Sys/chinese-clip-vit-base-patch16": ( | |
"https://huggingface.co/OFA-Sys/chinese-clip-vit-base-patch16/resolve/main/config.json" | |
), | |
} | |
class ChineseCLIPTextConfig(PretrainedConfig): | |
r""" | |
This is the configuration class to store the configuration of a [`ChineseCLIPModel`]. It is used to instantiate a | |
Chinese CLIP model according to the specified arguments, defining the model architecture. Instantiating a | |
configuration with the defaults will yield a similar configuration to that of the Chinese CLIP | |
[OFA-Sys/chinese-clip-vit-base-patch16](https: | |
//huggingface.co/OFA-Sys/chinese-clip-vit-base-patch16) architecture. | |
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the | |
documentation from [`PretrainedConfig`] for more information. | |
Args: | |
vocab_size (`int`, *optional*, defaults to 30522): | |
Vocabulary size of the CHINESE_CLIP model. Defines the number of different tokens that can be represented | |
by the `inputs_ids` passed when calling [`ChineseCLIPModel`]. | |
hidden_size (`int`, *optional*, defaults to 768): | |
Dimensionality of the encoder layers and the pooler layer. | |
num_hidden_layers (`int`, *optional*, defaults to 12): | |
Number of hidden layers in the Transformer encoder. | |
num_attention_heads (`int`, *optional*, defaults to 12): | |
Number of attention heads for each attention layer in the Transformer encoder. | |
intermediate_size (`int`, *optional*, defaults to 3072): | |
Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder. | |
hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`): | |
The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, | |
`"relu"`, `"silu"` and `"gelu_new"` are supported. | |
hidden_dropout_prob (`float`, *optional*, defaults to 0.1): | |
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. | |
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1): | |
The dropout ratio for the attention probabilities. | |
max_position_embeddings (`int`, *optional*, defaults to 512): | |
The maximum sequence length that this model might ever be used with. Typically set this to something large | |
just in case (e.g., 512 or 1024 or 2048). | |
type_vocab_size (`int`, *optional*, defaults to 2): | |
The vocabulary size of the `token_type_ids` passed when calling [`ChineseCLIPModel`]. | |
initializer_range (`float`, *optional*, defaults to 0.02): | |
The standard deviation of the truncated_normal_initializer for initializing all weight matrices. | |
layer_norm_eps (`float`, *optional*, defaults to 1e-12): | |
The epsilon used by the layer normalization layers. | |
position_embedding_type (`str`, *optional*, defaults to `"absolute"`): | |
Type of position embedding. Choose one of `"absolute"`, `"relative_key"`, `"relative_key_query"`. For | |
positional embeddings use `"absolute"`. For more information on `"relative_key"`, please refer to | |
[Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155). | |
For more information on `"relative_key_query"`, please refer to *Method 4* in [Improve Transformer Models | |
with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658). | |
use_cache (`bool`, *optional*, defaults to `True`): | |
Whether or not the model should return the last key/values attentions (not used by all models). Only | |
relevant if `config.is_decoder=True`. | |
Example: | |
```python | |
>>> from transformers import ChineseCLIPTextConfig, ChineseCLIPTextModel | |
>>> # Initializing a ChineseCLIPTextConfig with OFA-Sys/chinese-clip-vit-base-patch16 style configuration | |
>>> configuration = ChineseCLIPTextConfig() | |
>>> # Initializing a ChineseCLIPTextModel (with random weights) from the OFA-Sys/chinese-clip-vit-base-patch16 style configuration | |
>>> model = ChineseCLIPTextModel(configuration) | |
>>> # Accessing the model configuration | |
>>> configuration = model.config | |
```""" | |
model_type = "chinese_clip_text_model" | |
def __init__( | |
self, | |
vocab_size=30522, | |
hidden_size=768, | |
num_hidden_layers=12, | |
num_attention_heads=12, | |
intermediate_size=3072, | |
hidden_act="gelu", | |
hidden_dropout_prob=0.1, | |
attention_probs_dropout_prob=0.1, | |
max_position_embeddings=512, | |
type_vocab_size=2, | |
initializer_range=0.02, | |
initializer_factor=1.0, | |
layer_norm_eps=1e-12, | |
pad_token_id=0, | |
position_embedding_type="absolute", | |
use_cache=True, | |
**kwargs, | |
): | |
super().__init__(pad_token_id=pad_token_id, **kwargs) | |
self.vocab_size = vocab_size | |
self.hidden_size = hidden_size | |
self.num_hidden_layers = num_hidden_layers | |
self.num_attention_heads = num_attention_heads | |
self.hidden_act = hidden_act | |
self.intermediate_size = intermediate_size | |
self.hidden_dropout_prob = hidden_dropout_prob | |
self.attention_probs_dropout_prob = attention_probs_dropout_prob | |
self.max_position_embeddings = max_position_embeddings | |
self.type_vocab_size = type_vocab_size | |
self.initializer_range = initializer_range | |
self.initializer_factor = initializer_factor | |
self.layer_norm_eps = layer_norm_eps | |
self.position_embedding_type = position_embedding_type | |
self.use_cache = use_cache | |
def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": | |
cls._set_token_in_kwargs(kwargs) | |
config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) | |
# get the vision config dict if we are loading from ChineseCLIPConfig | |
if config_dict.get("model_type") == "chinese_clip": | |
config_dict = config_dict["text_config"] | |
if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type: | |
logger.warning( | |
f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " | |
f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." | |
) | |
return cls.from_dict(config_dict, **kwargs) | |
class ChineseCLIPVisionConfig(PretrainedConfig): | |
r""" | |
This is the configuration class to store the configuration of a [`ChineseCLIPModel`]. It is used to instantiate an | |
ChineseCLIP model according to the specified arguments, defining the model architecture. Instantiating a | |
configuration with the defaults will yield a similar configuration to that of the ChineseCLIP | |
[OFA-Sys/chinese-clip-vit-base-patch16](https: | |
//huggingface.co/OFA-Sys/chinese-clip-vit-base-patch16) architecture. | |
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the | |
documentation from [`PretrainedConfig`] for more information. | |
Args: | |
hidden_size (`int`, *optional*, defaults to 768): | |
Dimensionality of the encoder layers and the pooler layer. | |
intermediate_size (`int`, *optional*, defaults to 3072): | |
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. | |
num_hidden_layers (`int`, *optional*, defaults to 12): | |
Number of hidden layers in the Transformer encoder. | |
num_attention_heads (`int`, *optional*, defaults to 12): | |
Number of attention heads for each attention layer in the Transformer encoder. | |
image_size (`int`, *optional*, defaults to 224): | |
The size (resolution) of each image. | |
patch_size (`int`, *optional*, defaults to 32): | |
The size (resolution) of each patch. | |
hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`): | |
The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, | |
`"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported. | |
layer_norm_eps (`float`, *optional*, defaults to 1e-5): | |
The epsilon used by the layer normalization layers. | |
attention_dropout (`float`, *optional*, defaults to 0.0): | |
The dropout ratio for the attention probabilities. | |
initializer_range (`float`, *optional*, defaults to 0.02): | |
The standard deviation of the truncated_normal_initializer for initializing all weight matrices. | |
initializer_factor (`float``, *optional*, defaults to 1): | |
A factor for initializing all weight matrices (should be kept to 1, used internally for initialization | |
testing). | |
Example: | |
```python | |
>>> from transformers import ChineseCLIPVisionConfig, ChineseCLIPVisionModel | |
>>> # Initializing a ChineseCLIPVisionConfig with OFA-Sys/chinese-clip-vit-base-patch16 style configuration | |
>>> configuration = ChineseCLIPVisionConfig() | |
>>> # Initializing a ChineseCLIPVisionModel (with random weights) from the OFA-Sys/chinese-clip-vit-base-patch16 style configuration | |
>>> model = ChineseCLIPVisionModel(configuration) | |
>>> # Accessing the model configuration | |
>>> configuration = model.config | |
```""" | |
model_type = "chinese_clip_vision_model" | |
def __init__( | |
self, | |
hidden_size=768, | |
intermediate_size=3072, | |
projection_dim=512, | |
num_hidden_layers=12, | |
num_attention_heads=12, | |
num_channels=3, | |
image_size=224, | |
patch_size=32, | |
hidden_act="quick_gelu", | |
layer_norm_eps=1e-5, | |
attention_dropout=0.0, | |
initializer_range=0.02, | |
initializer_factor=1.0, | |
**kwargs, | |
): | |
super().__init__(**kwargs) | |
self.hidden_size = hidden_size | |
self.intermediate_size = intermediate_size | |
self.projection_dim = projection_dim | |
self.num_hidden_layers = num_hidden_layers | |
self.num_attention_heads = num_attention_heads | |
self.num_channels = num_channels | |
self.patch_size = patch_size | |
self.image_size = image_size | |
self.initializer_range = initializer_range | |
self.initializer_factor = initializer_factor | |
self.attention_dropout = attention_dropout | |
self.layer_norm_eps = layer_norm_eps | |
self.hidden_act = hidden_act | |
def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": | |
cls._set_token_in_kwargs(kwargs) | |
config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) | |
# get the vision config dict if we are loading from ChineseCLIPConfig | |
if config_dict.get("model_type") == "chinese_clip": | |
config_dict = config_dict["vision_config"] | |
if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type: | |
logger.warning( | |
f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " | |
f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." | |
) | |
return cls.from_dict(config_dict, **kwargs) | |
class ChineseCLIPConfig(PretrainedConfig): | |
r""" | |
[`ChineseCLIPConfig`] is the configuration class to store the configuration of a [`ChineseCLIPModel`]. It is used | |
to instantiate Chinese-CLIP model according to the specified arguments, defining the text model and vision model | |
configs. Instantiating a configuration with the defaults will yield a similar configuration to that of the | |
Chinese-CLIP [OFA-Sys/chinese-clip-vit-base-patch16](https://huggingface.co/OFA-Sys/chinese-clip-vit-base-patch16) | |
architecture. | |
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the | |
documentation from [`PretrainedConfig`] for more information. | |
Args: | |
text_config (`dict`, *optional*): | |
Dictionary of configuration options used to initialize [`ChineseCLIPTextConfig`]. | |
vision_config (`dict`, *optional*): | |
Dictionary of configuration options used to initialize [`ChineseCLIPVisionConfig`]. | |
projection_dim (`int`, *optional*, defaults to 512): | |
Dimentionality of text and vision projection layers. | |
logit_scale_init_value (`float`, *optional*, defaults to 2.6592): | |
The inital value of the *logit_scale* paramter. Default is used as per the original ChineseCLIP | |
implementation. | |
kwargs (*optional*): | |
Dictionary of keyword arguments. | |
Example: | |
```python | |
>>> from transformers import ChineseCLIPConfig, ChineseCLIPModel | |
>>> # Initializing a ChineseCLIPConfig with OFA-Sys/chinese-clip-vit-base-patch16 style configuration | |
>>> configuration = ChineseCLIPConfig() | |
>>> # Initializing a ChineseCLIPModel (with random weights) from the OFA-Sys/chinese-clip-vit-base-patch16 style configuration | |
>>> model = ChineseCLIPModel(configuration) | |
>>> # Accessing the model configuration | |
>>> configuration = model.config | |
>>> # We can also initialize a ChineseCLIPConfig from a ChineseCLIPTextConfig and a ChineseCLIPVisionConfig | |
>>> # Initializing a ChineseCLIPTextConfig and ChineseCLIPVisionConfig configuration | |
>>> config_text = ChineseCLIPTextConfig() | |
>>> config_vision = ChineseCLIPVisionConfig() | |
>>> config = ChineseCLIPConfig.from_text_vision_configs(config_text, config_vision) | |
```""" | |
model_type = "chinese_clip" | |
def __init__( | |
self, text_config=None, vision_config=None, projection_dim=512, logit_scale_init_value=2.6592, **kwargs | |
): | |
# If `_config_dict` exist, we use them for the backward compatibility. | |
# We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot | |
# of confusion!). | |
text_config_dict = kwargs.pop("text_config_dict", None) | |
vision_config_dict = kwargs.pop("vision_config_dict", None) | |
super().__init__(**kwargs) | |
# Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in | |
# `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most | |
# cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`. | |
if text_config_dict is not None: | |
if text_config is None: | |
text_config = {} | |
# This is the complete result when using `text_config_dict`. | |
_text_config_dict = ChineseCLIPTextConfig(**text_config_dict).to_dict() | |
# Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different. | |
for key, value in _text_config_dict.items(): | |
if key in text_config and value != text_config[key] and key not in ["transformers_version"]: | |
# If specified in `text_config_dict` | |
if key in text_config_dict: | |
message = ( | |
f"`{key}` is found in both `text_config_dict` and `text_config` but with different values. " | |
f'The value `text_config_dict["{key}"]` will be used instead.' | |
) | |
# If inferred from default argument values (just to be super careful) | |
else: | |
message = ( | |
f"`text_config_dict` is provided which will be used to initialize `ChineseCLIPTextConfig`. " | |
f'The value `text_config["{key}"]` will be overriden.' | |
) | |
logger.warning(message) | |
# Update all values in `text_config` with the ones in `_text_config_dict`. | |
text_config.update(_text_config_dict) | |
if vision_config_dict is not None: | |
if vision_config is None: | |
vision_config = {} | |
# This is the complete result when using `vision_config_dict`. | |
_vision_config_dict = ChineseCLIPVisionConfig(**vision_config_dict).to_dict() | |
# convert keys to string instead of integer | |
if "id2label" in _vision_config_dict: | |
_vision_config_dict["id2label"] = { | |
str(key): value for key, value in _vision_config_dict["id2label"].items() | |
} | |
# Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different. | |
for key, value in _vision_config_dict.items(): | |
if key in vision_config and value != vision_config[key] and key not in ["transformers_version"]: | |
# If specified in `vision_config_dict` | |
if key in vision_config_dict: | |
message = ( | |
f"`{key}` is found in both `vision_config_dict` and `vision_config` but with different " | |
f'values. The value `vision_config_dict["{key}"]` will be used instead.' | |
) | |
# If inferred from default argument values (just to be super careful) | |
else: | |
message = ( | |
f"`vision_config_dict` is provided which will be used to initialize " | |
f'`ChineseCLIPVisionConfig`. The value `vision_config["{key}"]` will be overriden.' | |
) | |
logger.warning(message) | |
# Update all values in `vision_config` with the ones in `_vision_config_dict`. | |
vision_config.update(_vision_config_dict) | |
if text_config is None: | |
text_config = {} | |
logger.info("`text_config` is `None`. Initializing the `ChineseCLIPTextConfig` with default values.") | |
if vision_config is None: | |
vision_config = {} | |
logger.info("`vision_config` is `None`. initializing the `ChineseCLIPVisionConfig` with default values.") | |
self.text_config = ChineseCLIPTextConfig(**text_config) | |
self.vision_config = ChineseCLIPVisionConfig(**vision_config) | |
self.projection_dim = projection_dim | |
self.logit_scale_init_value = logit_scale_init_value | |
self.initializer_factor = 1.0 | |
self.initializer_range = 0.02 | |
def from_text_vision_configs( | |
cls, text_config: ChineseCLIPTextConfig, vision_config: ChineseCLIPVisionConfig, **kwargs | |
): | |
r""" | |
Instantiate a [`ChineseCLIPConfig`] (or a derived class) from Chinese-CLIP text model configuration and | |
Chinese-CLIP vision model configuration. Returns: | |
[`ChineseCLIPConfig`]: An instance of a configuration object | |
""" | |
return cls(text_config=text_config.to_dict(), vision_config=vision_config.to_dict(), **kwargs) | |
class ChineseCLIPOnnxConfig(OnnxConfig): | |
def inputs(self) -> Mapping[str, Mapping[int, str]]: | |
return OrderedDict( | |
[ | |
("input_ids", {0: "batch", 1: "sequence"}), | |
("pixel_values", {0: "batch", 1: "num_channels", 2: "height", 3: "width"}), | |
("attention_mask", {0: "batch", 1: "sequence"}), | |
] | |
) | |
def outputs(self) -> Mapping[str, Mapping[int, str]]: | |
return OrderedDict( | |
[ | |
("logits_per_image", {0: "batch"}), | |
("logits_per_text", {0: "batch"}), | |
("text_embeds", {0: "batch"}), | |
("image_embeds", {0: "batch"}), | |
] | |
) | |
def atol_for_validation(self) -> float: | |
return 1e-4 | |
def generate_dummy_inputs( | |
self, | |
processor: "ProcessorMixin", | |
batch_size: int = -1, | |
seq_length: int = -1, | |
framework: Optional["TensorType"] = None, | |
) -> Mapping[str, Any]: | |
text_input_dict = super().generate_dummy_inputs( | |
processor.tokenizer, batch_size=batch_size, seq_length=seq_length, framework=framework | |
) | |
image_input_dict = super().generate_dummy_inputs( | |
processor.image_processor, batch_size=batch_size, framework=framework | |
) | |
return {**text_input_dict, **image_input_dict} | |
def default_onnx_opset(self) -> int: | |
return 14 | |