Leyo commited on
Commit
83cef5d
1 Parent(s): aa81487

transformers-like-implementation (#1)

Browse files

- make config similar to transformers except for flash (8d33426853dc8f1b3b5689aecda69055037d7598)
- make processing similar to transformers implementation (60a1dbcdc38421cee543a5bbb16517402f64945a)
- fix configuration imports (374a41982b51929632d5f52a2aba929a292aa50c)
- add processing/tokenization siglip (9d8f3cc6cc37ba5833a7073ba3363e99d19068eb)
- add convert siglip to hf (3b15dc97d30ffd7289428e4e8950c1c46d00986e)
- use transformers siglip modeling implementation except for flash attention (20f77128b113be3b61b34155eec760b1c38d9950)
- fix config (8254757eb3aab6dfd1bb9f076552260e0a172bf6)
- fix issues with erf and xavier init (b298ee1a7922b874ba8f313ab6ff2c953cf5dba6)
- add safetensors from google model (98130385412ccff0b249b6340d3f74a1615c2657)
- bilinear instead of bicubic (df7ef893928ca01fc0e6a62cf9265054846a526b)

.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ model.safetensors filter=lfs diff=lfs merge=lfs -text
config.json CHANGED
@@ -8,19 +8,16 @@
8
  "AutoModel": "HuggingFaceM4/siglip-so400m-14-384-flash-attn2--modeling_siglip.SiglipModel"
9
  },
10
  "initializer_factor": 1.0,
11
- "logit_scale_init_value": 2.6592,
12
  "model_type": "siglip",
13
- "projection_dim": 512,
14
  "text_config": {
15
  "hidden_size": 1152,
16
  "intermediate_size": 4304,
17
  "model_type": "siglip_text_model",
18
  "num_attention_heads": 16,
19
- "num_hidden_layers": 27,
20
- "vocab_size": 32000
21
  },
22
  "torch_dtype": "float32",
23
- "transformers_version": "4.35.2",
24
  "vision_config": {
25
  "hidden_size": 1152,
26
  "image_size": 384,
 
8
  "AutoModel": "HuggingFaceM4/siglip-so400m-14-384-flash-attn2--modeling_siglip.SiglipModel"
9
  },
10
  "initializer_factor": 1.0,
 
11
  "model_type": "siglip",
 
12
  "text_config": {
13
  "hidden_size": 1152,
14
  "intermediate_size": 4304,
15
  "model_type": "siglip_text_model",
16
  "num_attention_heads": 16,
17
+ "num_hidden_layers": 27
 
18
  },
19
  "torch_dtype": "float32",
20
+ "transformers_version": "4.37.0.dev0",
21
  "vision_config": {
22
  "hidden_size": 1152,
23
  "image_size": 384,
configuration_siglip.py CHANGED
@@ -1,5 +1,5 @@
1
  # coding=utf-8
2
- # Copyright 2023 The HuggingFace Inc. team. All rights reserved.
3
  #
4
  # Licensed under the Apache License, Version 2.0 (the "License");
5
  # you may not use this file except in compliance with the License.
@@ -15,16 +15,9 @@
15
  """ Siglip model configuration"""
16
 
17
  import os
18
- from collections import OrderedDict
19
- from typing import TYPE_CHECKING, Any, Mapping, Optional, Union
20
-
21
-
22
- if TYPE_CHECKING:
23
- from transformers.processing_utils import ProcessorMixin
24
- from transformers.utils import TensorType
25
 
26
  from transformers.configuration_utils import PretrainedConfig
27
- from transformers.onnx import OnnxConfig
28
  from transformers.utils import logging
29
 
30
 
@@ -46,16 +39,16 @@ class SiglipTextConfig(PretrainedConfig):
46
  documentation from [`PretrainedConfig`] for more information.
47
 
48
  Args:
49
- vocab_size (`int`, *optional*, defaults to 49408):
50
  Vocabulary size of the Siglip text model. Defines the number of different tokens that can be represented by
51
  the `inputs_ids` passed when calling [`SiglipModel`].
52
- hidden_size (`int`, *optional*, defaults to 512):
53
  Dimensionality of the encoder layers and the pooler layer.
54
- intermediate_size (`int`, *optional*, defaults to 2048):
55
  Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
56
  num_hidden_layers (`int`, *optional*, defaults to 12):
57
  Number of hidden layers in the Transformer encoder.
58
- num_attention_heads (`int`, *optional*, defaults to 8):
59
  Number of attention heads for each attention layer in the Transformer encoder.
60
  max_position_embeddings (`int`, *optional*, defaults to 64):
61
  The maximum sequence length that this model might ever be used with. Typically set this to something large
@@ -63,15 +56,16 @@ class SiglipTextConfig(PretrainedConfig):
63
  hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
64
  The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
65
  `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported.
66
- layer_norm_eps (`float`, *optional*, defaults to 1e-6):
67
  The epsilon used by the layer normalization layers.
68
  attention_dropout (`float`, *optional*, defaults to 0.0):
69
  The dropout ratio for the attention probabilities.
70
- initializer_range (`float`, *optional*, defaults to 0.02):
71
- The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
72
- initializer_factor (`float`, *optional*, defaults to 1):
73
- A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
74
- testing).
 
75
 
76
  Example:
77
 
@@ -87,22 +81,20 @@ class SiglipTextConfig(PretrainedConfig):
87
  >>> # Accessing the model configuration
88
  >>> configuration = model.config
89
  ```"""
 
90
  model_type = "siglip_text_model"
91
 
92
  def __init__(
93
  self,
94
- vocab_size=49408,
95
- hidden_size=512,
96
- intermediate_size=2048,
97
- projection_dim=512,
98
  num_hidden_layers=12,
99
- num_attention_heads=8,
100
  max_position_embeddings=64,
101
  hidden_act="gelu_pytorch_tanh",
102
  layer_norm_eps=1e-6,
103
  attention_dropout=0.0,
104
- initializer_range=0.02,
105
- initializer_factor=1.0,
106
  # This differs from `CLIPTokenizer`'s default and from openai/siglip
107
  # See https://github.com/huggingface/transformers/pull/24773#issuecomment-1632287538
108
  pad_token_id=1,
@@ -116,14 +108,11 @@ class SiglipTextConfig(PretrainedConfig):
116
  self.vocab_size = vocab_size
117
  self.hidden_size = hidden_size
118
  self.intermediate_size = intermediate_size
119
- self.projection_dim = projection_dim
120
  self.num_hidden_layers = num_hidden_layers
121
  self.num_attention_heads = num_attention_heads
122
  self.max_position_embeddings = max_position_embeddings
123
  self.layer_norm_eps = layer_norm_eps
124
  self.hidden_act = hidden_act
125
- self.initializer_range = initializer_range
126
- self.initializer_factor = initializer_factor
127
  self.attention_dropout = attention_dropout
128
  self._flash_attn_2_enabled = _flash_attn_2_enabled
129
 
@@ -165,22 +154,19 @@ class SiglipVisionConfig(PretrainedConfig):
165
  Number of hidden layers in the Transformer encoder.
166
  num_attention_heads (`int`, *optional*, defaults to 12):
167
  Number of attention heads for each attention layer in the Transformer encoder.
 
 
168
  image_size (`int`, *optional*, defaults to 224):
169
  The size (resolution) of each image.
170
- patch_size (`int`, *optional*, defaults to 32):
171
  The size (resolution) of each patch.
172
  hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
173
  The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
174
  `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported.
175
- layer_norm_eps (`float`, *optional*, defaults to 1e-6):
176
  The epsilon used by the layer normalization layers.
177
  attention_dropout (`float`, *optional*, defaults to 0.0):
178
  The dropout ratio for the attention probabilities.
179
- initializer_range (`float`, *optional*, defaults to 0.02):
180
- The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
181
- initializer_factor (`float`, *optional*, defaults to 1):
182
- A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
183
- testing).
184
 
185
  Example:
186
 
@@ -203,17 +189,14 @@ class SiglipVisionConfig(PretrainedConfig):
203
  self,
204
  hidden_size=768,
205
  intermediate_size=3072,
206
- projection_dim=512,
207
  num_hidden_layers=12,
208
  num_attention_heads=12,
209
  num_channels=3,
210
  image_size=224,
211
- patch_size=32,
212
  hidden_act="gelu_pytorch_tanh",
213
  layer_norm_eps=1e-6,
214
  attention_dropout=0.0,
215
- initializer_range=0.02,
216
- initializer_factor=1.0,
217
  _flash_attn_2_enabled=True,
218
  **kwargs,
219
  ):
@@ -221,14 +204,11 @@ class SiglipVisionConfig(PretrainedConfig):
221
 
222
  self.hidden_size = hidden_size
223
  self.intermediate_size = intermediate_size
224
- self.projection_dim = projection_dim
225
  self.num_hidden_layers = num_hidden_layers
226
  self.num_attention_heads = num_attention_heads
227
  self.num_channels = num_channels
228
  self.patch_size = patch_size
229
  self.image_size = image_size
230
- self.initializer_range = initializer_range
231
- self.initializer_factor = initializer_factor
232
  self.attention_dropout = attention_dropout
233
  self.layer_norm_eps = layer_norm_eps
234
  self.hidden_act = hidden_act
@@ -268,10 +248,6 @@ class SiglipConfig(PretrainedConfig):
268
  Dictionary of configuration options used to initialize [`SiglipTextConfig`].
269
  vision_config (`dict`, *optional*):
270
  Dictionary of configuration options used to initialize [`SiglipVisionConfig`].
271
- projection_dim (`int`, *optional*, defaults to 512):
272
- Dimentionality of text and vision projection layers.
273
- logit_scale_init_value (`float`, *optional*, defaults to 2.6592):
274
- The inital value of the *logit_scale* paramter. Default is used as per the original Siglip implementation.
275
  kwargs (*optional*):
276
  Dictionary of keyword arguments.
277
 
@@ -301,79 +277,9 @@ class SiglipConfig(PretrainedConfig):
301
 
302
  model_type = "siglip"
303
 
304
- def __init__(
305
- self, text_config=None, vision_config=None, projection_dim=512, logit_scale_init_value=2.6592, **kwargs
306
- ):
307
- # If `_config_dict` exist, we use them for the backward compatibility.
308
- # We pop out these 2 attributes before calling `super().__init__` to avoid them being saved (which causes a lot
309
- # of confusion!).
310
- text_config_dict = kwargs.pop("text_config_dict", None)
311
- vision_config_dict = kwargs.pop("vision_config_dict", None)
312
-
313
  super().__init__(**kwargs)
314
 
315
- # Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in
316
- # `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most
317
- # cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`.
318
- if text_config_dict is not None:
319
- if text_config is None:
320
- text_config = {}
321
-
322
- # This is the complete result when using `text_config_dict`.
323
- _text_config_dict = SiglipTextConfig(**text_config_dict).to_dict()
324
-
325
- # Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different.
326
- for key, value in _text_config_dict.items():
327
- if key in text_config and value != text_config[key] and key not in ["transformers_version"]:
328
- # If specified in `text_config_dict`
329
- if key in text_config_dict:
330
- message = (
331
- f"`{key}` is found in both `text_config_dict` and `text_config` but with different values."
332
- f' The value `text_config_dict["{key}"]` will be used instead.'
333
- )
334
- # If inferred from default argument values (just to be super careful)
335
- else:
336
- message = (
337
- "`text_config_dict` is provided which will be used to initialize `SiglipTextConfig`. The "
338
- f'value `text_config["{key}"]` will be overriden.'
339
- )
340
- logger.warning(message)
341
-
342
- # Update all values in `text_config` with the ones in `_text_config_dict`.
343
- text_config.update(_text_config_dict)
344
-
345
- if vision_config_dict is not None:
346
- if vision_config is None:
347
- vision_config = {}
348
-
349
- # This is the complete result when using `vision_config_dict`.
350
- _vision_config_dict = SiglipVisionConfig(**vision_config_dict).to_dict()
351
- # convert keys to string instead of integer
352
- if "id2label" in _vision_config_dict:
353
- _vision_config_dict["id2label"] = {
354
- str(key): value for key, value in _vision_config_dict["id2label"].items()
355
- }
356
-
357
- # Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different.
358
- for key, value in _vision_config_dict.items():
359
- if key in vision_config and value != vision_config[key] and key not in ["transformers_version"]:
360
- # If specified in `vision_config_dict`
361
- if key in vision_config_dict:
362
- message = (
363
- f"`{key}` is found in both `vision_config_dict` and `vision_config` but with different "
364
- f'values. The value `vision_config_dict["{key}"]` will be used instead.'
365
- )
366
- # If inferred from default argument values (just to be super careful)
367
- else:
368
- message = (
369
- "`vision_config_dict` is provided which will be used to initialize `SiglipVisionConfig`. "
370
- f'The value `vision_config["{key}"]` will be overriden.'
371
- )
372
- logger.warning(message)
373
-
374
- # Update all values in `vision_config` with the ones in `_vision_config_dict`.
375
- vision_config.update(_vision_config_dict)
376
-
377
  if text_config is None:
378
  text_config = {}
379
  logger.info("`text_config` is `None`. Initializing the `SiglipTextConfig` with default values.")
@@ -385,8 +291,6 @@ class SiglipConfig(PretrainedConfig):
385
  self.text_config = SiglipTextConfig(**text_config)
386
  self.vision_config = SiglipVisionConfig(**vision_config)
387
 
388
- self.projection_dim = projection_dim
389
- self.logit_scale_init_value = logit_scale_init_value
390
  self.initializer_factor = 1.0
391
 
392
  @classmethod
@@ -400,49 +304,3 @@ class SiglipConfig(PretrainedConfig):
400
  """
401
 
402
  return cls(text_config=text_config.to_dict(), vision_config=vision_config.to_dict(), **kwargs)
403
-
404
-
405
- class SiglipOnnxConfig(OnnxConfig):
406
- @property
407
- def inputs(self) -> Mapping[str, Mapping[int, str]]:
408
- return OrderedDict(
409
- [
410
- ("input_ids", {0: "batch", 1: "sequence"}),
411
- ("pixel_values", {0: "batch", 1: "num_channels", 2: "height", 3: "width"}),
412
- ("attention_mask", {0: "batch", 1: "sequence"}),
413
- ]
414
- )
415
-
416
- @property
417
- def outputs(self) -> Mapping[str, Mapping[int, str]]:
418
- return OrderedDict(
419
- [
420
- ("logits_per_image", {0: "batch"}),
421
- ("logits_per_text", {0: "batch"}),
422
- ("text_embeds", {0: "batch"}),
423
- ("image_embeds", {0: "batch"}),
424
- ]
425
- )
426
-
427
- @property
428
- def atol_for_validation(self) -> float:
429
- return 1e-4
430
-
431
- def generate_dummy_inputs(
432
- self,
433
- processor: "ProcessorMixin",
434
- batch_size: int = -1,
435
- seq_length: int = -1,
436
- framework: Optional["TensorType"] = None,
437
- ) -> Mapping[str, Any]:
438
- text_input_dict = super().generate_dummy_inputs(
439
- processor.tokenizer, batch_size=batch_size, seq_length=seq_length, framework=framework
440
- )
441
- image_input_dict = super().generate_dummy_inputs(
442
- processor.image_processor, batch_size=batch_size, framework=framework
443
- )
444
- return {**text_input_dict, **image_input_dict}
445
-
446
- @property
447
- def default_onnx_opset(self) -> int:
448
- return 14
 
1
  # coding=utf-8
2
+ # Copyright 2024 The HuggingFace Inc. team. All rights reserved.
3
  #
4
  # Licensed under the Apache License, Version 2.0 (the "License");
5
  # you may not use this file except in compliance with the License.
 
15
  """ Siglip model configuration"""
16
 
17
  import os
18
+ from typing import Union
 
 
 
 
 
 
19
 
20
  from transformers.configuration_utils import PretrainedConfig
 
21
  from transformers.utils import logging
22
 
23
 
 
39
  documentation from [`PretrainedConfig`] for more information.
40
 
41
  Args:
42
+ vocab_size (`int`, *optional*, defaults to 32000):
43
  Vocabulary size of the Siglip text model. Defines the number of different tokens that can be represented by
44
  the `inputs_ids` passed when calling [`SiglipModel`].
45
+ hidden_size (`int`, *optional*, defaults to 768):
46
  Dimensionality of the encoder layers and the pooler layer.
47
+ intermediate_size (`int`, *optional*, defaults to 3072):
48
  Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
49
  num_hidden_layers (`int`, *optional*, defaults to 12):
50
  Number of hidden layers in the Transformer encoder.
51
+ num_attention_heads (`int`, *optional*, defaults to 12):
52
  Number of attention heads for each attention layer in the Transformer encoder.
53
  max_position_embeddings (`int`, *optional*, defaults to 64):
54
  The maximum sequence length that this model might ever be used with. Typically set this to something large
 
56
  hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
57
  The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
58
  `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported.
59
+ layer_norm_eps (`float`, *optional*, defaults to 1e-06):
60
  The epsilon used by the layer normalization layers.
61
  attention_dropout (`float`, *optional*, defaults to 0.0):
62
  The dropout ratio for the attention probabilities.
63
+ pad_token_id (`int`, *optional*, defaults to 1):
64
+ The id of the padding token in the vocabulary.
65
+ bos_token_id (`int`, *optional*, defaults to 49406):
66
+ The id of the beginning-of-sequence token in the vocabulary.
67
+ eos_token_id (`int`, *optional*, defaults to 49407):
68
+ The id of the end-of-sequence token in the vocabulary.
69
 
70
  Example:
71
 
 
81
  >>> # Accessing the model configuration
82
  >>> configuration = model.config
83
  ```"""
84
+
85
  model_type = "siglip_text_model"
86
 
87
  def __init__(
88
  self,
89
+ vocab_size=32000,
90
+ hidden_size=768,
91
+ intermediate_size=3072,
 
92
  num_hidden_layers=12,
93
+ num_attention_heads=12,
94
  max_position_embeddings=64,
95
  hidden_act="gelu_pytorch_tanh",
96
  layer_norm_eps=1e-6,
97
  attention_dropout=0.0,
 
 
98
  # This differs from `CLIPTokenizer`'s default and from openai/siglip
99
  # See https://github.com/huggingface/transformers/pull/24773#issuecomment-1632287538
100
  pad_token_id=1,
 
108
  self.vocab_size = vocab_size
109
  self.hidden_size = hidden_size
110
  self.intermediate_size = intermediate_size
 
111
  self.num_hidden_layers = num_hidden_layers
112
  self.num_attention_heads = num_attention_heads
113
  self.max_position_embeddings = max_position_embeddings
114
  self.layer_norm_eps = layer_norm_eps
115
  self.hidden_act = hidden_act
 
 
116
  self.attention_dropout = attention_dropout
117
  self._flash_attn_2_enabled = _flash_attn_2_enabled
118
 
 
154
  Number of hidden layers in the Transformer encoder.
155
  num_attention_heads (`int`, *optional*, defaults to 12):
156
  Number of attention heads for each attention layer in the Transformer encoder.
157
+ num_channels (`int`, *optional*, defaults to 3):
158
+ Number of channels in the input images.
159
  image_size (`int`, *optional*, defaults to 224):
160
  The size (resolution) of each image.
161
+ patch_size (`int`, *optional*, defaults to 16):
162
  The size (resolution) of each patch.
163
  hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
164
  The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
165
  `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported.
166
+ layer_norm_eps (`float`, *optional*, defaults to 1e-06):
167
  The epsilon used by the layer normalization layers.
168
  attention_dropout (`float`, *optional*, defaults to 0.0):
169
  The dropout ratio for the attention probabilities.
 
 
 
 
 
170
 
171
  Example:
172
 
 
189
  self,
190
  hidden_size=768,
191
  intermediate_size=3072,
 
192
  num_hidden_layers=12,
193
  num_attention_heads=12,
194
  num_channels=3,
195
  image_size=224,
196
+ patch_size=16,
197
  hidden_act="gelu_pytorch_tanh",
198
  layer_norm_eps=1e-6,
199
  attention_dropout=0.0,
 
 
200
  _flash_attn_2_enabled=True,
201
  **kwargs,
202
  ):
 
204
 
205
  self.hidden_size = hidden_size
206
  self.intermediate_size = intermediate_size
 
207
  self.num_hidden_layers = num_hidden_layers
208
  self.num_attention_heads = num_attention_heads
209
  self.num_channels = num_channels
210
  self.patch_size = patch_size
211
  self.image_size = image_size
 
 
212
  self.attention_dropout = attention_dropout
213
  self.layer_norm_eps = layer_norm_eps
214
  self.hidden_act = hidden_act
 
248
  Dictionary of configuration options used to initialize [`SiglipTextConfig`].
249
  vision_config (`dict`, *optional*):
250
  Dictionary of configuration options used to initialize [`SiglipVisionConfig`].
 
 
 
 
251
  kwargs (*optional*):
252
  Dictionary of keyword arguments.
253
 
 
277
 
278
  model_type = "siglip"
279
 
280
+ def __init__(self, text_config=None, vision_config=None, **kwargs):
 
 
 
 
 
 
 
 
281
  super().__init__(**kwargs)
282
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
283
  if text_config is None:
284
  text_config = {}
285
  logger.info("`text_config` is `None`. Initializing the `SiglipTextConfig` with default values.")
 
291
  self.text_config = SiglipTextConfig(**text_config)
292
  self.vision_config = SiglipVisionConfig(**vision_config)
293
 
 
 
294
  self.initializer_factor = 1.0
295
 
296
  @classmethod
 
304
  """
305
 
306
  return cls(text_config=text_config.to_dict(), vision_config=vision_config.to_dict(), **kwargs)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
convert_siglip_to_hf.py ADDED
@@ -0,0 +1,413 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2024 The HuggingFace Inc. team.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """Convert SigLIP checkpoints from the original repository.
16
+
17
+ URL: https://github.com/google-research/big_vision/tree/main
18
+ """
19
+
20
+
21
+ import argparse
22
+ import collections
23
+ from pathlib import Path
24
+
25
+ import numpy as np
26
+ import requests
27
+ import torch
28
+ from huggingface_hub import hf_hub_download
29
+ from numpy import load
30
+ from PIL import Image
31
+
32
+ from transformers import SiglipConfig, SiglipImageProcessor, SiglipModel, SiglipProcessor, SiglipTokenizer
33
+ from transformers.utils import logging
34
+
35
+
36
+ logging.set_verbosity_info()
37
+ logger = logging.get_logger(__name__)
38
+
39
+
40
+ model_name_to_checkpoint = {
41
+ # base checkpoints
42
+ "siglip-base-patch16-224": "/Users/nielsrogge/Documents/SigLIP/webli_en_b16_224_63724782.npz",
43
+ "siglip-base-patch16-256": "/Users/nielsrogge/Documents/SigLIP/webli_en_b16_256_60500360.npz",
44
+ "siglip-base-patch16-384": "/Users/nielsrogge/Documents/SigLIP/webli_en_b16_384_68578854.npz",
45
+ "siglip-base-patch16-512": "/Users/nielsrogge/Documents/SigLIP/webli_en_b16_512_68580893.npz",
46
+ # large checkpoints
47
+ "siglip-large-patch16-256": "/Users/nielsrogge/Documents/SigLIP/webli_en_l16_256_60552751.npz",
48
+ "siglip-large-patch16-384": "/Users/nielsrogge/Documents/SigLIP/webli_en_l16_384_63634585.npz",
49
+ # multilingual checkpoint
50
+ "siglip-base-patch16-256-i18n": "/Users/nielsrogge/Documents/SigLIP/webli_i18n_b16_256_66117334.npz",
51
+ # so400m checkpoints
52
+ "siglip-so400m-patch14-384": "/Users/nielsrogge/Documents/SigLIP/webli_en_so400m_384_58765454.npz",
53
+ }
54
+
55
+ model_name_to_image_size = {
56
+ "siglip-base-patch16-224": 224,
57
+ "siglip-base-patch16-256": 256,
58
+ "siglip-base-patch16-384": 384,
59
+ "siglip-base-patch16-512": 512,
60
+ "siglip-large-patch16-256": 256,
61
+ "siglip-large-patch16-384": 384,
62
+ "siglip-base-patch16-256-i18n": 256,
63
+ "siglip-so400m-patch14-384": 384,
64
+ }
65
+
66
+
67
+ def get_siglip_config(model_name):
68
+ config = SiglipConfig()
69
+
70
+ vocab_size = 250000 if "i18n" in model_name else 32000
71
+ image_size = model_name_to_image_size[model_name]
72
+ patch_size = 16 if "patch16" in model_name else 14
73
+
74
+ # size of the architecture
75
+ config.vision_config.image_size = image_size
76
+ config.vision_config.patch_size = patch_size
77
+ config.text_config.vocab_size = vocab_size
78
+
79
+ if "base" in model_name:
80
+ pass
81
+ elif "large" in model_name:
82
+ config.text_config.hidden_size = 1024
83
+ config.text_config.intermediate_size = 4096
84
+ config.text_config.num_hidden_layers = 24
85
+ config.text_config.num_attention_heads = 16
86
+ config.vision_config.hidden_size = 1024
87
+ config.vision_config.intermediate_size = 4096
88
+ config.vision_config.num_hidden_layers = 24
89
+ config.vision_config.num_attention_heads = 16
90
+ elif "so400m" in model_name:
91
+ config.text_config.hidden_size = 1152
92
+ config.text_config.intermediate_size = 4304
93
+ config.text_config.num_hidden_layers = 27
94
+ config.text_config.num_attention_heads = 16
95
+ config.vision_config.hidden_size = 1152
96
+ config.vision_config.intermediate_size = 4304
97
+ config.vision_config.num_hidden_layers = 27
98
+ config.vision_config.num_attention_heads = 16
99
+ else:
100
+ raise ValueError("Model not supported")
101
+
102
+ return config
103
+
104
+
105
+ def create_rename_keys(config):
106
+ rename_keys = []
107
+ # fmt: off
108
+
109
+ # vision encoder
110
+
111
+ rename_keys.append(("params/img/embedding/kernel", "vision_model.embeddings.patch_embedding.weight"))
112
+ rename_keys.append(("params/img/embedding/bias", "vision_model.embeddings.patch_embedding.bias"))
113
+ rename_keys.append(("params/img/pos_embedding", "vision_model.embeddings.position_embedding.weight"))
114
+
115
+ for i in range(config.vision_config.num_hidden_layers):
116
+ rename_keys.append((f"params/img/Transformer/encoderblock_{i}/LayerNorm_0/scale", f"vision_model.encoder.layers.{i}.layer_norm1.weight"))
117
+ rename_keys.append((f"params/img/Transformer/encoderblock_{i}/LayerNorm_0/bias", f"vision_model.encoder.layers.{i}.layer_norm1.bias"))
118
+ rename_keys.append((f"params/img/Transformer/encoderblock_{i}/LayerNorm_1/scale", f"vision_model.encoder.layers.{i}.layer_norm2.weight"))
119
+ rename_keys.append((f"params/img/Transformer/encoderblock_{i}/LayerNorm_1/bias", f"vision_model.encoder.layers.{i}.layer_norm2.bias"))
120
+ rename_keys.append((f"params/img/Transformer/encoderblock_{i}/MlpBlock_0/Dense_0/kernel", f"vision_model.encoder.layers.{i}.mlp.fc1.weight"))
121
+ rename_keys.append((f"params/img/Transformer/encoderblock_{i}/MlpBlock_0/Dense_0/bias", f"vision_model.encoder.layers.{i}.mlp.fc1.bias"))
122
+ rename_keys.append((f"params/img/Transformer/encoderblock_{i}/MlpBlock_0/Dense_1/kernel", f"vision_model.encoder.layers.{i}.mlp.fc2.weight"))
123
+ rename_keys.append((f"params/img/Transformer/encoderblock_{i}/MlpBlock_0/Dense_1/bias", f"vision_model.encoder.layers.{i}.mlp.fc2.bias"))
124
+ rename_keys.append((f"params/img/Transformer/encoderblock_{i}/MultiHeadDotProductAttention_0/key/kernel", f"vision_model.encoder.layers.{i}.self_attn.k_proj.weight"))
125
+ rename_keys.append((f"params/img/Transformer/encoderblock_{i}/MultiHeadDotProductAttention_0/key/bias", f"vision_model.encoder.layers.{i}.self_attn.k_proj.bias"))
126
+ rename_keys.append((f"params/img/Transformer/encoderblock_{i}/MultiHeadDotProductAttention_0/value/kernel", f"vision_model.encoder.layers.{i}.self_attn.v_proj.weight"))
127
+ rename_keys.append((f"params/img/Transformer/encoderblock_{i}/MultiHeadDotProductAttention_0/value/bias", f"vision_model.encoder.layers.{i}.self_attn.v_proj.bias"))
128
+ rename_keys.append((f"params/img/Transformer/encoderblock_{i}/MultiHeadDotProductAttention_0/query/kernel", f"vision_model.encoder.layers.{i}.self_attn.q_proj.weight"))
129
+ rename_keys.append((f"params/img/Transformer/encoderblock_{i}/MultiHeadDotProductAttention_0/query/bias", f"vision_model.encoder.layers.{i}.self_attn.q_proj.bias"))
130
+ rename_keys.append((f"params/img/Transformer/encoderblock_{i}/MultiHeadDotProductAttention_0/out/kernel", f"vision_model.encoder.layers.{i}.self_attn.out_proj.weight"))
131
+ rename_keys.append((f"params/img/Transformer/encoderblock_{i}/MultiHeadDotProductAttention_0/out/bias", f"vision_model.encoder.layers.{i}.self_attn.out_proj.bias"))
132
+
133
+ rename_keys.append(("params/img/Transformer/encoder_norm/scale", "vision_model.post_layernorm.weight"))
134
+ rename_keys.append(("params/img/Transformer/encoder_norm/bias", "vision_model.post_layernorm.bias"))
135
+
136
+ rename_keys.append(("params/img/MAPHead_0/probe", "vision_model.head.probe"))
137
+ rename_keys.append(("params/img/MAPHead_0/LayerNorm_0/scale", "vision_model.head.layernorm.weight"))
138
+ rename_keys.append(("params/img/MAPHead_0/LayerNorm_0/bias", "vision_model.head.layernorm.bias"))
139
+ rename_keys.append(("params/img/MAPHead_0/MlpBlock_0/Dense_0/kernel", "vision_model.head.mlp.fc1.weight"))
140
+ rename_keys.append(("params/img/MAPHead_0/MlpBlock_0/Dense_0/bias", "vision_model.head.mlp.fc1.bias"))
141
+ rename_keys.append(("params/img/MAPHead_0/MlpBlock_0/Dense_1/kernel", "vision_model.head.mlp.fc2.weight"))
142
+ rename_keys.append(("params/img/MAPHead_0/MlpBlock_0/Dense_1/bias", "vision_model.head.mlp.fc2.bias"))
143
+ rename_keys.append(("params/img/MAPHead_0/MultiHeadDotProductAttention_0/out/kernel", "vision_model.head.attention.out_proj.weight"))
144
+ rename_keys.append(("params/img/MAPHead_0/MultiHeadDotProductAttention_0/out/bias", "vision_model.head.attention.out_proj.bias"))
145
+
146
+ # text encoder
147
+
148
+ rename_keys.append(("params/txt/Embed_0/embedding", "text_model.embeddings.token_embedding.weight"))
149
+ rename_keys.append(("params/txt/pos_embedding", "text_model.embeddings.position_embedding.weight"))
150
+
151
+ for i in range(config.text_config.num_hidden_layers):
152
+ rename_keys.append((f"params/txt/Encoder_0/encoderblock_{i}/LayerNorm_0/scale", f"text_model.encoder.layers.{i}.layer_norm1.weight"))
153
+ rename_keys.append((f"params/txt/Encoder_0/encoderblock_{i}/LayerNorm_0/bias", f"text_model.encoder.layers.{i}.layer_norm1.bias"))
154
+ rename_keys.append((f"params/txt/Encoder_0/encoderblock_{i}/LayerNorm_1/scale", f"text_model.encoder.layers.{i}.layer_norm2.weight"))
155
+ rename_keys.append((f"params/txt/Encoder_0/encoderblock_{i}/LayerNorm_1/bias", f"text_model.encoder.layers.{i}.layer_norm2.bias"))
156
+ rename_keys.append((f"params/txt/Encoder_0/encoderblock_{i}/MlpBlock_0/Dense_0/kernel", f"text_model.encoder.layers.{i}.mlp.fc1.weight"))
157
+ rename_keys.append((f"params/txt/Encoder_0/encoderblock_{i}/MlpBlock_0/Dense_0/bias", f"text_model.encoder.layers.{i}.mlp.fc1.bias"))
158
+ rename_keys.append((f"params/txt/Encoder_0/encoderblock_{i}/MlpBlock_0/Dense_1/kernel", f"text_model.encoder.layers.{i}.mlp.fc2.weight"))
159
+ rename_keys.append((f"params/txt/Encoder_0/encoderblock_{i}/MlpBlock_0/Dense_1/bias", f"text_model.encoder.layers.{i}.mlp.fc2.bias"))
160
+ rename_keys.append((f"params/txt/Encoder_0/encoderblock_{i}/MultiHeadDotProductAttention_0/key/kernel", f"text_model.encoder.layers.{i}.self_attn.k_proj.weight"))
161
+ rename_keys.append((f"params/txt/Encoder_0/encoderblock_{i}/MultiHeadDotProductAttention_0/key/bias", f"text_model.encoder.layers.{i}.self_attn.k_proj.bias"))
162
+ rename_keys.append((f"params/txt/Encoder_0/encoderblock_{i}/MultiHeadDotProductAttention_0/value/kernel", f"text_model.encoder.layers.{i}.self_attn.v_proj.weight"))
163
+ rename_keys.append((f"params/txt/Encoder_0/encoderblock_{i}/MultiHeadDotProductAttention_0/value/bias", f"text_model.encoder.layers.{i}.self_attn.v_proj.bias"))
164
+ rename_keys.append((f"params/txt/Encoder_0/encoderblock_{i}/MultiHeadDotProductAttention_0/query/kernel", f"text_model.encoder.layers.{i}.self_attn.q_proj.weight"))
165
+ rename_keys.append((f"params/txt/Encoder_0/encoderblock_{i}/MultiHeadDotProductAttention_0/query/bias", f"text_model.encoder.layers.{i}.self_attn.q_proj.bias"))
166
+ rename_keys.append((f"params/txt/Encoder_0/encoderblock_{i}/MultiHeadDotProductAttention_0/out/kernel", f"text_model.encoder.layers.{i}.self_attn.out_proj.weight"))
167
+ rename_keys.append((f"params/txt/Encoder_0/encoderblock_{i}/MultiHeadDotProductAttention_0/out/bias", f"text_model.encoder.layers.{i}.self_attn.out_proj.bias"))
168
+
169
+ rename_keys.append(("params/txt/Encoder_0/encoder_norm/scale", "text_model.final_layer_norm.weight"))
170
+ rename_keys.append(("params/txt/Encoder_0/encoder_norm/bias", "text_model.final_layer_norm.bias"))
171
+ rename_keys.append(("params/txt/head/kernel", "text_model.head.weight"))
172
+ rename_keys.append(("params/txt/head/bias", "text_model.head.bias"))
173
+
174
+ # learned temperature and bias
175
+ rename_keys.append(("params/t", "logit_scale"))
176
+ rename_keys.append(("params/b", "logit_bias"))
177
+
178
+ # fmt: on
179
+ return rename_keys
180
+
181
+
182
+ def rename_key(dct, old, new, config):
183
+ val = dct.pop(old)
184
+
185
+ if ("out_proj" in new or "v_proj" in new or "k_proj" in new or "q_proj" in new) and "vision" in new:
186
+ val = val.reshape(-1, config.vision_config.hidden_size)
187
+ if ("out_proj" in new or "v_proj" in new or "k_proj" in new or "q_proj" in new) and "text" in new:
188
+ val = val.reshape(-1, config.text_config.hidden_size)
189
+
190
+ if "patch_embedding.weight" in new:
191
+ val = val.transpose(3, 2, 0, 1)
192
+ elif new.endswith("weight") and "position_embedding" not in new and "token_embedding" not in new:
193
+ val = val.T
194
+
195
+ if "position_embedding" in new and "vision" in new:
196
+ val = val.reshape(-1, config.vision_config.hidden_size)
197
+ if "position_embedding" in new and "text" in new:
198
+ val = val.reshape(-1, config.text_config.hidden_size)
199
+
200
+ if new.endswith("bias"):
201
+ val = val.reshape(-1)
202
+
203
+ dct[new] = torch.from_numpy(val)
204
+
205
+
206
+ def read_in_q_k_v_head(state_dict, config):
207
+ # read in individual input projection layers
208
+ key_proj_weight = (
209
+ state_dict.pop("params/img/MAPHead_0/MultiHeadDotProductAttention_0/key/kernel")
210
+ .reshape(-1, config.vision_config.hidden_size)
211
+ .T
212
+ )
213
+ key_proj_bias = state_dict.pop("params/img/MAPHead_0/MultiHeadDotProductAttention_0/key/bias").reshape(-1)
214
+ value_proj_weight = (
215
+ state_dict.pop("params/img/MAPHead_0/MultiHeadDotProductAttention_0/value/kernel")
216
+ .reshape(-1, config.vision_config.hidden_size)
217
+ .T
218
+ )
219
+ value_proj_bias = state_dict.pop("params/img/MAPHead_0/MultiHeadDotProductAttention_0/value/bias").reshape(-1)
220
+ query_proj_weight = (
221
+ state_dict.pop("params/img/MAPHead_0/MultiHeadDotProductAttention_0/query/kernel")
222
+ .reshape(-1, config.vision_config.hidden_size)
223
+ .T
224
+ )
225
+ query_proj_bias = state_dict.pop("params/img/MAPHead_0/MultiHeadDotProductAttention_0/query/bias").reshape(-1)
226
+
227
+ # next, add them to the state dict as a single matrix + vector
228
+ state_dict["vision_model.head.attention.in_proj_weight"] = torch.from_numpy(
229
+ np.concatenate([query_proj_weight, key_proj_weight, value_proj_weight], axis=0)
230
+ )
231
+ state_dict["vision_model.head.attention.in_proj_bias"] = torch.from_numpy(
232
+ np.concatenate([query_proj_bias, key_proj_bias, value_proj_bias], axis=0)
233
+ )
234
+
235
+
236
+ # We will verify our results on an image of cute cats
237
+ def prepare_img():
238
+ url = "http://images.cocodataset.org/val2017/000000039769.jpg"
239
+ image = Image.open(requests.get(url, stream=True).raw)
240
+ return image
241
+
242
+
243
+ def flatten_nested_dict(params, parent_key="", sep="/"):
244
+ items = []
245
+
246
+ for k, v in params.items():
247
+ new_key = parent_key + sep + k if parent_key else k
248
+
249
+ if isinstance(v, collections.abc.MutableMapping):
250
+ items.extend(flatten_nested_dict(v, new_key, sep=sep).items())
251
+ else:
252
+ items.append((new_key, v))
253
+ return dict(items)
254
+
255
+
256
+ @torch.no_grad()
257
+ def convert_siglip_checkpoint(model_name, pytorch_dump_folder_path, verify_logits=True, push_to_hub=False):
258
+ """
259
+ Copy/paste/tweak model's weights to our SigLIP structure.
260
+ """
261
+
262
+ # define default SigLIP configuration
263
+ config = get_siglip_config(model_name)
264
+
265
+ # get checkpoint
266
+ checkpoint = model_name_to_checkpoint[model_name]
267
+
268
+ # get vocab file
269
+ if "i18n" in model_name:
270
+ vocab_file = "/Users/nielsrogge/Documents/SigLIP/multilingual_vocab/sentencepiece.model"
271
+ else:
272
+ vocab_file = "/Users/nielsrogge/Documents/SigLIP/english_vocab/sentencepiece.model"
273
+
274
+ # load original state dict
275
+ data = load(checkpoint)
276
+ state_dict = flatten_nested_dict(data)
277
+
278
+ # remove and rename some keys
279
+ rename_keys = create_rename_keys(config)
280
+ for src, dest in rename_keys:
281
+ rename_key(state_dict, src, dest, config)
282
+
283
+ # qkv matrices of attention pooling head need special treatment
284
+ read_in_q_k_v_head(state_dict, config)
285
+
286
+ # load HuggingFace model
287
+ model = SiglipModel(config).eval()
288
+ model.load_state_dict(state_dict)
289
+
290
+ # create processor
291
+ # important: make tokenizer not return attention_mask since original one doesn't require it
292
+ image_size = config.vision_config.image_size
293
+ size = {"height": image_size, "width": image_size}
294
+ image_processor = SiglipImageProcessor(size=size)
295
+ tokenizer = SiglipTokenizer(vocab_file=vocab_file, model_input_names=["input_ids"])
296
+ processor = SiglipProcessor(image_processor=image_processor, tokenizer=tokenizer)
297
+
298
+ # verify on dummy images and texts
299
+ url_1 = "https://cdn.openai.com/multimodal-neurons/assets/apple/apple-ipod.jpg"
300
+ image_1 = Image.open(requests.get(url_1, stream=True).raw).convert("RGB")
301
+ url_2 = "https://cdn.openai.com/multimodal-neurons/assets/apple/apple-blank.jpg"
302
+ image_2 = Image.open(requests.get(url_2, stream=True).raw).convert("RGB")
303
+ texts = ["an apple", "a picture of an apple"]
304
+
305
+ inputs = processor(images=[image_1, image_2], text=texts, return_tensors="pt", padding="max_length")
306
+
307
+ # verify input_ids against original ones
308
+ if image_size == 224:
309
+ filename = "siglip_pixel_values.pt"
310
+ elif image_size == 256:
311
+ filename = "siglip_pixel_values_256.pt"
312
+ elif image_size == 384:
313
+ filename = "siglip_pixel_values_384.pt"
314
+ elif image_size == 512:
315
+ filename = "siglip_pixel_values_512.pt"
316
+ else:
317
+ raise ValueError("Image size not supported")
318
+
319
+ filepath = hf_hub_download(repo_id="nielsr/test-image", filename=filename, repo_type="dataset")
320
+ original_pixel_values = torch.load(filepath)
321
+ filepath = hf_hub_download(repo_id="nielsr/test-image", filename="siglip_input_ids.pt", repo_type="dataset")
322
+ original_input_ids = torch.load(filepath)
323
+
324
+ if "i18n" not in model_name:
325
+ assert inputs.input_ids.tolist() == original_input_ids.tolist()
326
+
327
+ print("Mean of original pixel values:", original_pixel_values.mean())
328
+ print("Mean of new pixel values:", inputs.pixel_values.mean())
329
+
330
+ # note: we're testing with original pixel values here since we don't have exact pixel values
331
+ with torch.no_grad():
332
+ outputs = model(input_ids=inputs.input_ids, pixel_values=original_pixel_values)
333
+
334
+ # with torch.no_grad():
335
+ # outputs = model(input_ids=inputs.input_ids, pixel_values=inputs.pixel_values)
336
+
337
+ print(outputs.logits_per_image[:3, :3])
338
+
339
+ probs = torch.sigmoid(outputs.logits_per_image) # these are the probabilities
340
+ print(f"{probs[0][0]:.1%} that image 0 is '{texts[0]}'")
341
+ print(f"{probs[0][1]:.1%} that image 0 is '{texts[1]}'")
342
+
343
+ if verify_logits:
344
+ if model_name == "siglip-base-patch16-224":
345
+ expected_slice = torch.tensor(
346
+ [[-2.9621, -2.1672], [-0.2713, 0.2910]],
347
+ )
348
+ elif model_name == "siglip-base-patch16-256":
349
+ expected_slice = torch.tensor(
350
+ [[-3.1146, -1.9894], [-0.7312, 0.6387]],
351
+ )
352
+ elif model_name == "siglip-base-patch16-384":
353
+ expected_slice = torch.tensor(
354
+ [[-2.8098, -2.1891], [-0.4242, 0.4102]],
355
+ )
356
+ elif model_name == "siglip-base-patch16-512":
357
+ expected_slice = torch.tensor(
358
+ [[-2.7899, -2.2668], [-0.4295, -0.0735]],
359
+ )
360
+ elif model_name == "siglip-large-patch16-256":
361
+ expected_slice = torch.tensor(
362
+ [[-1.5827, -0.5801], [-0.9153, 0.1363]],
363
+ )
364
+ elif model_name == "siglip-large-patch16-384":
365
+ expected_slice = torch.tensor(
366
+ [[-2.1523, -0.2899], [-0.2959, 0.7884]],
367
+ )
368
+ elif model_name == "siglip-so400m-patch14-384":
369
+ expected_slice = torch.tensor([[-1.2441, -0.6649], [-0.7060, 0.7374]])
370
+ elif model_name == "siglip-base-patch16-256-i18n":
371
+ expected_slice = torch.tensor(
372
+ [[-0.9064, 0.1073], [-0.0299, 0.5304]],
373
+ )
374
+
375
+ assert torch.allclose(outputs.logits_per_image[:3, :3], expected_slice, atol=1e-4)
376
+ print("Looks ok!")
377
+
378
+ if pytorch_dump_folder_path is not None:
379
+ Path(pytorch_dump_folder_path).mkdir(exist_ok=True)
380
+ print(f"Saving model {model_name} to {pytorch_dump_folder_path}")
381
+ model.save_pretrained(pytorch_dump_folder_path)
382
+ print(f"Saving processor to {pytorch_dump_folder_path}")
383
+ processor.save_pretrained(pytorch_dump_folder_path)
384
+
385
+ if push_to_hub:
386
+ model.push_to_hub(f"nielsr/{model_name}")
387
+ processor.push_to_hub(f"nielsr/{model_name}")
388
+
389
+
390
+ if __name__ == "__main__":
391
+ parser = argparse.ArgumentParser()
392
+ # Required parameters
393
+ parser.add_argument(
394
+ "--model_name",
395
+ default="siglip-base-patch16-224",
396
+ type=str,
397
+ choices=model_name_to_checkpoint.keys(),
398
+ help="Name of the model you'd like to convert.",
399
+ )
400
+ parser.add_argument(
401
+ "--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model directory."
402
+ )
403
+ parser.add_argument(
404
+ "--verify_logits",
405
+ action="store_false",
406
+ help="Whether to verify logits against the original implementation.",
407
+ )
408
+ parser.add_argument(
409
+ "--push_to_hub", action="store_true", help="Whether or not to push the converted model to the 🤗 hub."
410
+ )
411
+
412
+ args = parser.parse_args()
413
+ convert_siglip_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.verify_logits, args.push_to_hub)
image_processing_siglip.py CHANGED
@@ -1,5 +1,5 @@
1
  # coding=utf-8
2
- # Copyright 2023 The HuggingFace Inc. team. All rights reserved.
3
  #
4
  # Licensed under the Apache License, Version 2.0 (the "License");
5
  # you may not use this file except in compliance with the License.
@@ -14,17 +14,16 @@
14
  # limitations under the License.
15
  """Image processor class for SigLIP."""
16
 
17
- from typing import Dict, Optional, Union
18
-
19
- import numpy as np
20
 
21
  from transformers.image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
22
  from transformers.image_transforms import (
23
- rescale,
24
  resize,
25
  to_channel_dimension_format,
26
  )
27
  from transformers.image_utils import (
 
 
28
  ChannelDimension,
29
  ImageInput,
30
  PILImageResampling,
@@ -54,7 +53,7 @@ class SiglipImageProcessor(BaseImageProcessor):
54
  `do_resize` in the `preprocess` method.
55
  size (`Dict[str, int]` *optional*, defaults to `{"height": 224, "width": 224}`):
56
  Size of the image after resizing. Can be overridden by `size` in the `preprocess` method.
57
- resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BILINEAR`):
58
  Resampling filter to use if resizing the image. Can be overridden by `resample` in the `preprocess` method.
59
  do_rescale (`bool`, *optional*, defaults to `True`):
60
  Whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by `do_rescale` in
@@ -62,6 +61,16 @@ class SiglipImageProcessor(BaseImageProcessor):
62
  rescale_factor (`int` or `float`, *optional*, defaults to `1/255`):
63
  Scale factor to use if rescaling the image. Can be overridden by `rescale_factor` in the `preprocess`
64
  method.
 
 
 
 
 
 
 
 
 
 
65
  """
66
 
67
  model_input_names = ["pixel_values"]
@@ -73,57 +82,24 @@ class SiglipImageProcessor(BaseImageProcessor):
73
  resample: PILImageResampling = PILImageResampling.BILINEAR,
74
  do_rescale: bool = True,
75
  rescale_factor: Union[int, float] = 1 / 255,
 
 
 
76
  **kwargs,
77
  ) -> None:
78
  super().__init__(**kwargs)
79
  size = size if size is not None else {"height": 224, "width": 224}
80
- size = get_size_dict(size, default_to_square=False)
 
81
 
82
  self.do_resize = do_resize
83
  self.size = size
84
  self.resample = resample
85
  self.do_rescale = do_rescale
86
  self.rescale_factor = rescale_factor
87
-
88
- def rescale(
89
- self,
90
- image: np.ndarray,
91
- rescale_factor: float,
92
- data_format: Optional[Union[str, ChannelDimension]] = None,
93
- input_data_format: Optional[Union[str, ChannelDimension]] = None,
94
- **kwargs,
95
- ) -> np.ndarray:
96
- """
97
- Rescale an image by a scale factor. image = image * scale, after which image = image * 2 - 1.
98
-
99
- Args:
100
- image (`np.ndarray`):
101
- Image to rescale.
102
- scale (`float`):
103
- The scaling factor to rescale pixel values by.
104
- data_format (`str` or `ChannelDimension`, *optional*):
105
- The channel dimension format for the output image. If unset, the channel dimension format of the input
106
- image is used. Can be one of:
107
- - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
108
- - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
109
- input_data_format (`ChannelDimension` or `str`, *optional*):
110
- The channel dimension format for the input image. If unset, the channel dimension format is inferred
111
- from the input image. Can be one of:
112
- - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
113
- - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
114
-
115
- Returns:
116
- `np.ndarray`: The rescaled image.
117
- """
118
- # first, rescale to 0->1
119
- rescaled_image = rescale(
120
- image, scale=rescale_factor, data_format=data_format, input_data_format=input_data_format, **kwargs
121
- )
122
-
123
- # next, rescale to -1->1
124
- rescaled_image = 2 * rescaled_image - 1
125
-
126
- return rescaled_image
127
 
128
  def preprocess(
129
  self,
@@ -133,6 +109,9 @@ class SiglipImageProcessor(BaseImageProcessor):
133
  resample: PILImageResampling = None,
134
  do_rescale: bool = None,
135
  rescale_factor: float = None,
 
 
 
136
  return_tensors: Optional[Union[str, TensorType]] = None,
137
  data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,
138
  input_data_format: Optional[Union[str, ChannelDimension]] = None,
@@ -156,6 +135,13 @@ class SiglipImageProcessor(BaseImageProcessor):
156
  Whether to rescale the image.
157
  rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
158
  Rescale factor to rescale the image by if `do_rescale` is set to `True`.
 
 
 
 
 
 
 
159
  return_tensors (`str` or `TensorType`, *optional*):
160
  The type of tensors to return. Can be one of:
161
  - Unset: Return a list of `np.ndarray`.
@@ -181,6 +167,9 @@ class SiglipImageProcessor(BaseImageProcessor):
181
  resample = resample if resample is not None else self.resample
182
  do_rescale = do_rescale if do_rescale is not None else self.do_rescale
183
  rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor
 
 
 
184
 
185
  images = make_list_of_images(images)
186
 
@@ -210,14 +199,21 @@ class SiglipImageProcessor(BaseImageProcessor):
210
  input_data_format = infer_channel_dimension_format(images[0])
211
 
212
  if do_resize:
 
213
  images = [
214
- resize(image=image, size=(size["width"], size["height"]), resample=resample, input_data_format=input_data_format)
215
  for image in images
216
  ]
217
 
218
  if do_rescale:
219
  images = [
220
- self.rescale(image=image, rescale_factor=rescale_factor, input_data_format=input_data_format)
 
 
 
 
 
 
221
  for image in images
222
  ]
223
 
 
1
  # coding=utf-8
2
+ # Copyright 2024 The HuggingFace Inc. team. All rights reserved.
3
  #
4
  # Licensed under the Apache License, Version 2.0 (the "License");
5
  # you may not use this file except in compliance with the License.
 
14
  # limitations under the License.
15
  """Image processor class for SigLIP."""
16
 
17
+ from typing import Dict, List, Optional, Union
 
 
18
 
19
  from transformers.image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
20
  from transformers.image_transforms import (
 
21
  resize,
22
  to_channel_dimension_format,
23
  )
24
  from transformers.image_utils import (
25
+ IMAGENET_STANDARD_MEAN,
26
+ IMAGENET_STANDARD_STD,
27
  ChannelDimension,
28
  ImageInput,
29
  PILImageResampling,
 
53
  `do_resize` in the `preprocess` method.
54
  size (`Dict[str, int]` *optional*, defaults to `{"height": 224, "width": 224}`):
55
  Size of the image after resizing. Can be overridden by `size` in the `preprocess` method.
56
+ resample (`PILImageResampling`, *optional*, defaults to `Resampling.BILINEAR`):
57
  Resampling filter to use if resizing the image. Can be overridden by `resample` in the `preprocess` method.
58
  do_rescale (`bool`, *optional*, defaults to `True`):
59
  Whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by `do_rescale` in
 
61
  rescale_factor (`int` or `float`, *optional*, defaults to `1/255`):
62
  Scale factor to use if rescaling the image. Can be overridden by `rescale_factor` in the `preprocess`
63
  method.
64
+ do_normalize (`bool`, *optional*, defaults to `True`):
65
+ Whether to normalize the image by the specified mean and standard deviation. Can be overridden by
66
+ `do_normalize` in the `preprocess` method.
67
+ image_mean (`float` or `List[float]`, *optional*, defaults to `[0.5, 0.5, 0.5]`):
68
+ Mean to use if normalizing the image. This is a float or list of floats the length of the number of
69
+ channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method.
70
+ image_std (`float` or `List[float]`, *optional*, defaults to `[0.5, 0.5, 0.5]`):
71
+ Standard deviation to use if normalizing the image. This is a float or list of floats the length of the
72
+ number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method.
73
+ Can be overridden by the `image_std` parameter in the `preprocess` method.
74
  """
75
 
76
  model_input_names = ["pixel_values"]
 
82
  resample: PILImageResampling = PILImageResampling.BILINEAR,
83
  do_rescale: bool = True,
84
  rescale_factor: Union[int, float] = 1 / 255,
85
+ do_normalize: bool = True,
86
+ image_mean: Optional[Union[float, List[float]]] = None,
87
+ image_std: Optional[Union[float, List[float]]] = None,
88
  **kwargs,
89
  ) -> None:
90
  super().__init__(**kwargs)
91
  size = size if size is not None else {"height": 224, "width": 224}
92
+ image_mean = image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
93
+ image_std = image_std if image_std is not None else IMAGENET_STANDARD_STD
94
 
95
  self.do_resize = do_resize
96
  self.size = size
97
  self.resample = resample
98
  self.do_rescale = do_rescale
99
  self.rescale_factor = rescale_factor
100
+ self.do_normalize = do_normalize
101
+ self.image_mean = image_mean
102
+ self.image_std = image_std
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
103
 
104
  def preprocess(
105
  self,
 
109
  resample: PILImageResampling = None,
110
  do_rescale: bool = None,
111
  rescale_factor: float = None,
112
+ do_normalize: bool = None,
113
+ image_mean: Optional[Union[float, List[float]]] = None,
114
+ image_std: Optional[Union[float, List[float]]] = None,
115
  return_tensors: Optional[Union[str, TensorType]] = None,
116
  data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,
117
  input_data_format: Optional[Union[str, ChannelDimension]] = None,
 
135
  Whether to rescale the image.
136
  rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
137
  Rescale factor to rescale the image by if `do_rescale` is set to `True`.
138
+ do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
139
+ Whether to normalize the image.
140
+ image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
141
+ Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.
142
+ image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
143
+ Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to
144
+ `True`.
145
  return_tensors (`str` or `TensorType`, *optional*):
146
  The type of tensors to return. Can be one of:
147
  - Unset: Return a list of `np.ndarray`.
 
167
  resample = resample if resample is not None else self.resample
168
  do_rescale = do_rescale if do_rescale is not None else self.do_rescale
169
  rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor
170
+ do_normalize = do_normalize if do_normalize is not None else self.do_normalize
171
+ image_mean = image_mean if image_mean is not None else self.image_mean
172
+ image_std = image_std if image_std is not None else self.image_std
173
 
174
  images = make_list_of_images(images)
175
 
 
199
  input_data_format = infer_channel_dimension_format(images[0])
200
 
201
  if do_resize:
202
+ height, width = size["height"], size["width"]
203
  images = [
204
+ resize(image=image, size=(height, width), resample=resample, input_data_format=input_data_format)
205
  for image in images
206
  ]
207
 
208
  if do_rescale:
209
  images = [
210
+ self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format)
211
+ for image in images
212
+ ]
213
+
214
+ if do_normalize:
215
+ images = [
216
+ self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format)
217
  for image in images
218
  ]
219
 
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:90e016fcc4615f3c5d998008727291dbe0c00c1fe8e1089cdb2c04565286422c
3
- size 3511961264
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ea2abad2b7f8a9c1aa5e49a244d5d57ffa71c56f720c94bc5d240ef4d6e1d94a
3
+ size 3511950624
modeling_siglip.py CHANGED
@@ -1,5 +1,5 @@
1
  # coding=utf-8
2
- # Copyright 2023 Google AI and The HuggingFace Team. All rights reserved.
3
  #
4
  # Licensed under the Apache License, Version 2.0 (the "License");
5
  # you may not use this file except in compliance with the License.
@@ -15,14 +15,20 @@
15
  """ PyTorch Siglip model."""
16
 
17
 
 
 
18
  from dataclasses import dataclass
19
  from typing import Any, Optional, Tuple, Union
20
 
 
21
  import torch
22
  import torch.nn.functional as F
23
  import torch.utils.checkpoint
24
  from torch import nn
 
 
25
  from transformers.activations import ACT2FN
 
26
  from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
27
  from transformers.modeling_utils import PreTrainedModel
28
  from transformers.utils import (
@@ -33,7 +39,6 @@ from transformers.utils import (
33
  logging,
34
  replace_return_docstrings,
35
  )
36
-
37
  from .configuration_siglip import SiglipConfig, SiglipTextConfig, SiglipVisionConfig
38
 
39
 
@@ -64,32 +69,104 @@ def _get_unpad_data(attention_mask):
64
  )
65
 
66
 
67
- # Copied from transformers.models.bart.modeling_bart._expand_mask
68
- def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None):
69
- """
70
- Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
71
  """
72
- bsz, src_len = mask.size()
73
- tgt_len = tgt_len if tgt_len is not None else src_len
 
 
74
 
75
- expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)
 
 
 
 
 
 
 
76
 
77
- inverted_mask = 1.0 - expanded_mask
78
 
79
- return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min)
 
 
 
 
 
 
 
 
 
 
 
80
 
81
 
82
- # contrastive loss function, adapted from
83
- # https://sachinruk.github.io/blog/2021-03-07-siglip.html
84
- def contrastive_loss(logits: torch.Tensor) -> torch.Tensor:
85
- return nn.functional.cross_entropy(logits, torch.arange(len(logits), device=logits.device))
86
 
87
 
88
- # Copied from transformers.models.clip.modeling_clip.clip_loss with clip->siglip
89
- def siglip_loss(similarity: torch.Tensor) -> torch.Tensor:
90
- caption_loss = contrastive_loss(similarity)
91
- image_loss = contrastive_loss(similarity.t())
92
- return (caption_loss + image_loss) / 2.0
93
 
94
 
95
  @dataclass
@@ -168,8 +245,7 @@ class SiglipOutput(ModelOutput):
168
  text_embeds(`torch.FloatTensor` of shape `(batch_size, output_dim`):
169
  The text embeddings obtained by applying the projection layer to the pooled output of [`SiglipTextModel`].
170
  image_embeds(`torch.FloatTensor` of shape `(batch_size, output_dim`):
171
- The image embeddings obtained by applying the projection layer to the pooled output of
172
- [`SiglipVisionModel`].
173
  text_model_output(`BaseModelOutputWithPooling`):
174
  The output of the [`SiglipTextModel`].
175
  vision_model_output(`BaseModelOutputWithPooling`):
@@ -254,10 +330,10 @@ class SiglipTextEmbeddings(nn.Module):
254
  return embeddings
255
 
256
 
257
- # Copied from transformers.models.clip.modeling_clip.CLIPAttention with CLIP->Siglip
258
  class SiglipAttention(nn.Module):
259
  """Multi-headed attention from 'Attention Is All You Need' paper"""
260
 
 
261
  def __init__(self, config):
262
  super().__init__()
263
  self.config = config
@@ -277,86 +353,57 @@ class SiglipAttention(nn.Module):
277
  self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
278
  self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
279
 
280
- def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
281
- return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
282
-
283
  def forward(
284
  self,
285
  hidden_states: torch.Tensor,
286
  attention_mask: Optional[torch.Tensor] = None,
287
- causal_attention_mask: Optional[torch.Tensor] = None,
288
  output_attentions: Optional[bool] = False,
289
  ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
290
  """Input shape: Batch x Time x Channel"""
291
 
292
- bsz, tgt_len, embed_dim = hidden_states.size()
293
 
294
- # get query proj
295
- query_states = self.q_proj(hidden_states) * self.scale
296
- key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
297
- value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
298
 
299
- proj_shape = (bsz * self.num_heads, -1, self.head_dim)
300
- query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape)
301
- key_states = key_states.view(*proj_shape)
302
- value_states = value_states.view(*proj_shape)
303
 
304
- src_len = key_states.size(1)
305
- attn_weights = torch.bmm(query_states, key_states.transpose(1, 2))
306
 
307
- if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len):
308
  raise ValueError(
309
- f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is"
310
  f" {attn_weights.size()}"
311
  )
312
 
313
- # apply the causal_attention_mask first
314
- if causal_attention_mask is not None:
315
- if causal_attention_mask.size() != (bsz, 1, tgt_len, src_len):
316
- raise ValueError(
317
- f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is"
318
- f" {causal_attention_mask.size()}"
319
- )
320
- attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + causal_attention_mask
321
- attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)
322
-
323
  if attention_mask is not None:
324
- if attention_mask.size() != (bsz, 1, tgt_len, src_len):
325
  raise ValueError(
326
- f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}"
327
  )
328
- attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask
329
- attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)
330
 
331
- attn_weights = nn.functional.softmax(attn_weights, dim=-1)
 
 
 
332
 
333
- if output_attentions:
334
- # this operation is a bit akward, but it's required to
335
- # make sure that attn_weights keeps its gradient.
336
- # In order to do so, attn_weights have to reshaped
337
- # twice and have to be reused in the following
338
- attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
339
- attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len)
340
- else:
341
- attn_weights_reshaped = None
342
-
343
- attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)
344
-
345
- attn_output = torch.bmm(attn_probs, value_states)
346
-
347
- if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim):
348
  raise ValueError(
349
- f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is"
350
  f" {attn_output.size()}"
351
  )
352
 
353
- attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim)
354
- attn_output = attn_output.transpose(1, 2)
355
- attn_output = attn_output.reshape(bsz, tgt_len, embed_dim)
356
 
357
  attn_output = self.out_proj(attn_output)
358
 
359
- return attn_output, attn_weights_reshaped
360
 
361
 
362
  class SiglipFlashAttention2(SiglipAttention):
@@ -581,16 +628,15 @@ class SiglipEncoderLayer(nn.Module):
581
  self,
582
  hidden_states: torch.Tensor,
583
  attention_mask: torch.Tensor,
584
- causal_attention_mask: torch.Tensor,
585
  output_attentions: Optional[bool] = False,
586
  ) -> Tuple[torch.FloatTensor]:
587
  """
588
  Args:
589
- hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
590
- attention_mask (`torch.FloatTensor`): attention mask of size
591
- `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
592
- `(config.encoder_attention_heads,)`.
593
- output_attentions (`bool`, *optional*):
594
  Whether or not to return the attentions tensors of all attention layers. See `attentions` under
595
  returned tensors for more detail.
596
  """
@@ -600,7 +646,6 @@ class SiglipEncoderLayer(nn.Module):
600
  hidden_states, attn_weights = self.self_attn(
601
  hidden_states=hidden_states,
602
  attention_mask=attention_mask,
603
- causal_attention_mask=causal_attention_mask,
604
  output_attentions=output_attentions,
605
  )
606
  hidden_states = residual + hidden_states
@@ -630,39 +675,45 @@ class SiglipPreTrainedModel(PreTrainedModel):
630
 
631
  def _init_weights(self, module):
632
  """Initialize the weights"""
633
- factor = self.config.initializer_factor
634
- if isinstance(module, SiglipTextEmbeddings):
635
- module.token_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02)
636
- module.position_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02)
637
- elif isinstance(module, SiglipVisionEmbeddings):
638
- factor = self.config.initializer_factor
639
- nn.init.normal_(module.patch_embedding.weight, std=module.config.initializer_range * factor)
640
- nn.init.normal_(module.position_embedding.weight, std=module.config.initializer_range * factor)
 
 
641
  elif isinstance(module, SiglipAttention):
642
- factor = self.config.initializer_factor
643
- in_proj_std = (module.embed_dim**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor
644
- out_proj_std = (module.embed_dim**-0.5) * factor
645
- nn.init.normal_(module.q_proj.weight, std=in_proj_std)
646
- nn.init.normal_(module.k_proj.weight, std=in_proj_std)
647
- nn.init.normal_(module.v_proj.weight, std=in_proj_std)
648
- nn.init.normal_(module.out_proj.weight, std=out_proj_std)
 
649
  elif isinstance(module, SiglipMLP):
650
- factor = self.config.initializer_factor
651
- in_proj_std = (
652
- (module.config.hidden_size**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor
653
- )
654
- fc_std = (2 * module.config.hidden_size) ** -0.5 * factor
655
- nn.init.normal_(module.fc1.weight, std=fc_std)
656
- nn.init.normal_(module.fc2.weight, std=in_proj_std)
657
- if isinstance(module, nn.LayerNorm):
 
 
 
 
 
 
 
 
 
658
  module.bias.data.zero_()
659
  module.weight.data.fill_(1.0)
660
- if isinstance(module, nn.Linear) and module.bias is not None:
661
- module.bias.data.zero_()
662
-
663
- def _set_gradient_checkpointing(self, module, value=False):
664
- if isinstance(module, SiglipEncoder):
665
- module.gradient_checkpointing = value
666
 
667
 
668
  SIGLIP_START_DOCSTRING = r"""
@@ -781,11 +832,11 @@ class SiglipEncoder(nn.Module):
781
  self.layers = nn.ModuleList([SiglipEncoderLayer(config) for _ in range(config.num_hidden_layers)])
782
  self.gradient_checkpointing = False
783
 
 
784
  def forward(
785
  self,
786
  inputs_embeds,
787
  attention_mask: Optional[torch.Tensor] = None,
788
- causal_attention_mask: Optional[torch.Tensor] = None,
789
  output_attentions: Optional[bool] = None,
790
  output_hidden_states: Optional[bool] = None,
791
  return_dict: Optional[bool] = None,
@@ -802,13 +853,6 @@ class SiglipEncoder(nn.Module):
802
  - 1 for tokens that are **not masked**,
803
  - 0 for tokens that are **masked**.
804
 
805
- [What are attention masks?](../glossary#attention-mask)
806
- causal_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
807
- Causal mask for the text model. Mask values selected in `[0, 1]`:
808
-
809
- - 1 for tokens that are **not masked**,
810
- - 0 for tokens that are **masked**.
811
-
812
  [What are attention masks?](../glossary#attention-mask)
813
  output_attentions (`bool`, *optional*):
814
  Whether or not to return the attentions tensors of all attention layers. See `attentions` under
@@ -829,28 +873,20 @@ class SiglipEncoder(nn.Module):
829
  all_attentions = () if output_attentions else None
830
 
831
  hidden_states = inputs_embeds
832
- for idx, encoder_layer in enumerate(self.layers):
833
  if output_hidden_states:
834
  encoder_states = encoder_states + (hidden_states,)
835
  if self.gradient_checkpointing and self.training:
836
-
837
- def create_custom_forward(module):
838
- def custom_forward(*inputs):
839
- return module(*inputs, output_attentions)
840
-
841
- return custom_forward
842
-
843
- layer_outputs = torch.utils.checkpoint.checkpoint(
844
- create_custom_forward(encoder_layer),
845
  hidden_states,
846
  attention_mask,
847
- causal_attention_mask,
848
  )
849
  else:
850
  layer_outputs = encoder_layer(
851
  hidden_states,
852
  attention_mask,
853
- causal_attention_mask,
854
  output_attentions=output_attentions,
855
  )
856
 
@@ -909,16 +945,15 @@ class SiglipTextTransformer(nn.Module):
909
 
910
  hidden_states = self.embeddings(input_ids=input_ids, position_ids=position_ids)
911
 
912
- # note: SigLIP's text model does not use q causal mask, unlike the original CLIP model.
913
  # expand attention_mask
914
  if attention_mask is not None:
915
- # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
916
- attention_mask = _expand_mask(attention_mask, hidden_states.dtype)
917
 
918
  encoder_outputs = self.encoder(
919
  inputs_embeds=hidden_states,
920
- attention_mask=None,
921
- causal_attention_mask=None,
922
  output_attentions=output_attentions,
923
  output_hidden_states=output_hidden_states,
924
  return_dict=return_dict,
@@ -985,7 +1020,8 @@ class SiglipTextModel(SiglipPreTrainedModel):
985
  >>> model = SiglipTextModel.from_pretrained("google/siglip-base-patch16-224")
986
  >>> tokenizer = AutoTokenizer.from_pretrained("google/siglip-base-patch16-224")
987
 
988
- >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
 
989
 
990
  >>> outputs = model(**inputs)
991
  >>> last_hidden_state = outputs.last_hidden_state
@@ -1130,7 +1166,7 @@ class SiglipVisionModel(SiglipPreTrainedModel):
1130
 
1131
  >>> outputs = model(**inputs)
1132
  >>> last_hidden_state = outputs.last_hidden_state
1133
- >>> pooled_output = outputs.pooler_output # pooled CLS states
1134
  ```"""
1135
  return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1136
 
@@ -1164,19 +1200,11 @@ class SiglipModel(SiglipPreTrainedModel):
1164
  text_config = config.text_config
1165
  vision_config = config.vision_config
1166
 
1167
- self.text_model = SiglipTextModel(text_config)
1168
- self.vision_model = SiglipVisionModel(vision_config)
1169
 
1170
- self.temperature = nn.Parameter(
1171
- torch.randn(
1172
- 1,
1173
- )
1174
- )
1175
- self.bias = nn.Parameter(
1176
- torch.randn(
1177
- 1,
1178
- )
1179
- )
1180
 
1181
  # Initialize weights and apply final processing
1182
  self.post_init()
@@ -1199,13 +1227,16 @@ class SiglipModel(SiglipPreTrainedModel):
1199
  Examples:
1200
 
1201
  ```python
1202
- >>> from transformers import AutoTokenizer, SiglipModel
 
1203
 
1204
- >>> model = SiglipModel.from_pretrained("google/siglip-base-patch16-224")
1205
  >>> tokenizer = AutoTokenizer.from_pretrained("google/siglip-base-patch16-224")
1206
 
1207
- >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
1208
- >>> text_features = model.get_text_features(**inputs)
 
 
1209
  ```"""
1210
  # Use SigLIP model's config for some fields (if specified) instead of those of vision & text components.
1211
  output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
@@ -1245,9 +1276,10 @@ class SiglipModel(SiglipPreTrainedModel):
1245
  ```python
1246
  >>> from PIL import Image
1247
  >>> import requests
1248
- >>> from transformers import AutoProcessor, SiglipModel
 
1249
 
1250
- >>> model = SiglipModel.from_pretrained("google/siglip-base-patch16-224")
1251
  >>> processor = AutoProcessor.from_pretrained("google/siglip-base-patch16-224")
1252
 
1253
  >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
@@ -1255,7 +1287,8 @@ class SiglipModel(SiglipPreTrainedModel):
1255
 
1256
  >>> inputs = processor(images=image, return_tensors="pt")
1257
 
1258
- >>> image_features = model.get_image_features(**inputs)
 
1259
  ```"""
1260
  # Use SiglipModel's config for some fields (if specified) instead of those of vision & text components.
1261
  output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
@@ -1296,21 +1329,26 @@ class SiglipModel(SiglipPreTrainedModel):
1296
  ```python
1297
  >>> from PIL import Image
1298
  >>> import requests
1299
- >>> from transformers import AutoProcessor, SiglipModel
 
1300
 
1301
- >>> model = SiglipModel.from_pretrained("google/siglip-base-patch16-224")
1302
  >>> processor = AutoProcessor.from_pretrained("google/siglip-base-patch16-224")
1303
 
1304
  >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
1305
  >>> image = Image.open(requests.get(url, stream=True).raw)
1306
 
1307
- >>> inputs = processor(
1308
- ... text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True
1309
- ... )
1310
 
1311
- >>> outputs = model(**inputs)
1312
- >>> logits_per_image = outputs.logits_per_image # this is the image-text similarity score
1313
- >>> probs = logits_per_image.softmax(dim=1) # we can take the softmax to get the label probabilities
 
 
 
 
1314
  ```"""
1315
  # Use SigLIP model's config for some fields (if specified) instead of those of vision & text components.
1316
  output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
@@ -1343,11 +1381,9 @@ class SiglipModel(SiglipPreTrainedModel):
1343
  text_embeds = text_embeds / text_embeds.norm(p=2, dim=-1, keepdim=True)
1344
 
1345
  # cosine similarity as logits
1346
- logits_per_text = torch.matmul(text_embeds, image_embeds.t()) * self.temperature.exp() + self.bias
1347
  logits_per_image = logits_per_text.t()
1348
 
1349
- z = torch.matmul(image_embeds, text_embeds.t()) * self.temperature.exp()
1350
-
1351
  loss = None
1352
  if return_loss:
1353
  raise NotImplementedError("SigLIP loss to be implemented")
 
1
  # coding=utf-8
2
+ # Copyright 2024 Google AI and The HuggingFace Team. All rights reserved.
3
  #
4
  # Licensed under the Apache License, Version 2.0 (the "License");
5
  # you may not use this file except in compliance with the License.
 
15
  """ PyTorch Siglip model."""
16
 
17
 
18
+ import math
19
+ import warnings
20
  from dataclasses import dataclass
21
  from typing import Any, Optional, Tuple, Union
22
 
23
+ import numpy as np
24
  import torch
25
  import torch.nn.functional as F
26
  import torch.utils.checkpoint
27
  from torch import nn
28
+ from torch.nn.init import _calculate_fan_in_and_fan_out
29
+
30
  from transformers.activations import ACT2FN
31
+ from transformers.modeling_attn_mask_utils import _prepare_4d_attention_mask
32
  from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
33
  from transformers.modeling_utils import PreTrainedModel
34
  from transformers.utils import (
 
39
  logging,
40
  replace_return_docstrings,
41
  )
 
42
  from .configuration_siglip import SiglipConfig, SiglipTextConfig, SiglipVisionConfig
43
 
44
 
 
69
  )
70
 
71
 
72
+ def _trunc_normal_(tensor, mean, std, a, b):
73
+ # Cut & paste from PyTorch official master until it's in a few official releases - RW
74
+ # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf
75
+ def norm_cdf(x):
76
+ # Computes standard normal cumulative distribution function
77
+ return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0
78
+
79
+ if (mean < a - 2 * std) or (mean > b + 2 * std):
80
+ warnings.warn(
81
+ "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
82
+ "The distribution of values may be incorrect.",
83
+ stacklevel=2,
84
+ )
85
+
86
+ # Values are generated by using a truncated uniform distribution and
87
+ # then using the inverse CDF for the normal distribution.
88
+ # Get upper and lower cdf values
89
+ l = norm_cdf((a - mean) / std)
90
+ u = norm_cdf((b - mean) / std)
91
+
92
+ # Uniformly fill tensor with values from [l, u], then translate to
93
+ # [2l-1, 2u-1].
94
+ tensor.uniform_(2 * l - 1, 2 * u - 1)
95
+
96
+ # Use inverse cdf transform for normal distribution to get truncated
97
+ # standard normal
98
+ if tensor.dtype == torch.bfloat16:
99
+ tensor = tensor.to(torch.float32)
100
+ tensor.erfinv_()
101
+ tensor = tensor.to(torch.bfloat16)
102
+ else:
103
+ tensor.erfinv_()
104
+
105
+ # Transform to proper mean, std
106
+ tensor.mul_(std * math.sqrt(2.0))
107
+ tensor.add_(mean)
108
+
109
+ # Clamp to ensure it's in the proper range
110
+ tensor.clamp_(min=a, max=b)
111
+
112
+
113
+ def trunc_normal_tf_(
114
+ tensor: torch.Tensor, mean: float = 0.0, std: float = 1.0, a: float = -2.0, b: float = 2.0
115
+ ) -> torch.Tensor:
116
+ """Fills the input Tensor with values drawn from a truncated
117
+ normal distribution. The values are effectively drawn from the
118
+ normal distribution :math:`\\mathcal{N}(\text{mean}, \text{std}^2)`
119
+ with values outside :math:`[a, b]` redrawn until they are within
120
+ the bounds. The method used for generating the random values works
121
+ best when :math:`a \\leq \text{mean} \\leq b`.
122
+
123
+ NOTE: this 'tf' variant behaves closer to Tensorflow / JAX impl where the
124
+ bounds [a, b] are applied when sampling the normal distribution with mean=0, std=1.0
125
+ and the result is subsquently scaled and shifted by the mean and std args.
126
+
127
+ Args:
128
+ tensor: an n-dimensional `torch.Tensor`
129
+ mean: the mean of the normal distribution
130
+ std: the standard deviation of the normal distribution
131
+ a: the minimum cutoff value
132
+ b: the maximum cutoff value
133
  """
134
+ with torch.no_grad():
135
+ _trunc_normal_(tensor, 0, 1.0, a, b)
136
+ tensor.mul_(std).add_(mean)
137
+
138
 
139
+ def variance_scaling_(tensor, scale=1.0, mode="fan_in", distribution="normal"):
140
+ fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
141
+ if mode == "fan_in":
142
+ denom = fan_in
143
+ elif mode == "fan_out":
144
+ denom = fan_out
145
+ elif mode == "fan_avg":
146
+ denom = (fan_in + fan_out) / 2
147
 
148
+ variance = scale / denom
149
 
150
+ if distribution == "truncated_normal":
151
+ # constant is stddev of standard normal truncated to (-2, 2)
152
+ trunc_normal_tf_(tensor, std=math.sqrt(variance) / 0.87962566103423978)
153
+ elif distribution == "normal":
154
+ with torch.no_grad():
155
+ tensor.normal_(std=math.sqrt(variance))
156
+ elif distribution == "uniform":
157
+ bound = math.sqrt(3 * variance)
158
+ with torch.no_grad():
159
+ tensor.uniform_(-bound, bound)
160
+ else:
161
+ raise ValueError(f"invalid distribution {distribution}")
162
 
163
 
164
+ def lecun_normal_(tensor):
165
+ variance_scaling_(tensor, mode="fan_in", distribution="truncated_normal")
 
 
166
 
167
 
168
+ def default_flax_embed_init(tensor):
169
+ variance_scaling_(tensor, mode="fan_in", distribution="normal")
 
 
 
170
 
171
 
172
  @dataclass
 
245
  text_embeds(`torch.FloatTensor` of shape `(batch_size, output_dim`):
246
  The text embeddings obtained by applying the projection layer to the pooled output of [`SiglipTextModel`].
247
  image_embeds(`torch.FloatTensor` of shape `(batch_size, output_dim`):
248
+ The image embeddings obtained by applying the projection layer to the pooled output of [`SiglipVisionModel`].
 
249
  text_model_output(`BaseModelOutputWithPooling`):
250
  The output of the [`SiglipTextModel`].
251
  vision_model_output(`BaseModelOutputWithPooling`):
 
330
  return embeddings
331
 
332
 
 
333
  class SiglipAttention(nn.Module):
334
  """Multi-headed attention from 'Attention Is All You Need' paper"""
335
 
336
+ # Copied from transformers.models.clip.modeling_clip.CLIPAttention.__init__
337
  def __init__(self, config):
338
  super().__init__()
339
  self.config = config
 
353
  self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
354
  self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
355
 
 
 
 
356
  def forward(
357
  self,
358
  hidden_states: torch.Tensor,
359
  attention_mask: Optional[torch.Tensor] = None,
 
360
  output_attentions: Optional[bool] = False,
361
  ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
362
  """Input shape: Batch x Time x Channel"""
363
 
364
+ batch_size, q_len, _ = hidden_states.size()
365
 
366
+ query_states = self.q_proj(hidden_states)
367
+ key_states = self.k_proj(hidden_states)
368
+ value_states = self.v_proj(hidden_states)
 
369
 
370
+ query_states = query_states.view(batch_size, q_len, self.num_heads, self.head_dim).transpose(1, 2)
371
+ key_states = key_states.view(batch_size, q_len, self.num_heads, self.head_dim).transpose(1, 2)
372
+ value_states = value_states.view(batch_size, q_len, self.num_heads, self.head_dim).transpose(1, 2)
 
373
 
374
+ k_v_seq_len = key_states.shape[-2]
375
+ attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) * self.scale
376
 
377
+ if attn_weights.size() != (batch_size, self.num_heads, q_len, k_v_seq_len):
378
  raise ValueError(
379
+ f"Attention weights should be of size {(batch_size, self.num_heads, q_len, k_v_seq_len)}, but is"
380
  f" {attn_weights.size()}"
381
  )
382
 
 
 
 
 
 
 
 
 
 
 
383
  if attention_mask is not None:
384
+ if attention_mask.size() != (batch_size, 1, q_len, k_v_seq_len):
385
  raise ValueError(
386
+ f"Attention mask should be of size {(batch_size, 1, q_len, k_v_seq_len)}, but is {attention_mask.size()}"
387
  )
388
+ attn_weights = attn_weights + attention_mask
 
389
 
390
+ # upcast attention to fp32
391
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
392
+ attn_weights = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)
393
+ attn_output = torch.matmul(attn_weights, value_states)
394
 
395
+ if attn_output.size() != (batch_size, self.num_heads, q_len, self.head_dim):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
396
  raise ValueError(
397
+ f"`attn_output` should be of size {(batch_size, self.num_heads, q_len, self.head_dim)}, but is"
398
  f" {attn_output.size()}"
399
  )
400
 
401
+ attn_output = attn_output.transpose(1, 2).contiguous()
402
+ attn_output = attn_output.reshape(batch_size, q_len, self.embed_dim)
 
403
 
404
  attn_output = self.out_proj(attn_output)
405
 
406
+ return attn_output, attn_weights
407
 
408
 
409
  class SiglipFlashAttention2(SiglipAttention):
 
628
  self,
629
  hidden_states: torch.Tensor,
630
  attention_mask: torch.Tensor,
 
631
  output_attentions: Optional[bool] = False,
632
  ) -> Tuple[torch.FloatTensor]:
633
  """
634
  Args:
635
+ hidden_states (`torch.FloatTensor`):
636
+ Input to the layer of shape `(batch, seq_len, embed_dim)`.
637
+ attention_mask (`torch.FloatTensor`):
638
+ Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values.
639
+ output_attentions (`bool`, *optional*, defaults to `False`):
640
  Whether or not to return the attentions tensors of all attention layers. See `attentions` under
641
  returned tensors for more detail.
642
  """
 
646
  hidden_states, attn_weights = self.self_attn(
647
  hidden_states=hidden_states,
648
  attention_mask=attention_mask,
 
649
  output_attentions=output_attentions,
650
  )
651
  hidden_states = residual + hidden_states
 
675
 
676
  def _init_weights(self, module):
677
  """Initialize the weights"""
678
+
679
+ if isinstance(module, SiglipVisionEmbeddings):
680
+ width = (
681
+ self.config.vision_config.hidden_size
682
+ if isinstance(self.config, SiglipConfig)
683
+ else self.config.hidden_size
684
+ )
685
+ nn.init.normal_(module.position_embedding.weight, std=1 / np.sqrt(width))
686
+ elif isinstance(module, nn.Embedding):
687
+ default_flax_embed_init(module.weight)
688
  elif isinstance(module, SiglipAttention):
689
+ nn.init.normal_(module.q_proj.weight)
690
+ nn.init.normal_(module.k_proj.weight)
691
+ nn.init.normal_(module.v_proj.weight)
692
+ nn.init.normal_(module.out_proj.weight)
693
+ nn.init.zeros_(module.q_proj.bias)
694
+ nn.init.zeros_(module.k_proj.bias)
695
+ nn.init.zeros_(module.v_proj.bias)
696
+ nn.init.zeros_(module.out_proj.bias)
697
  elif isinstance(module, SiglipMLP):
698
+ nn.init.normal_(module.fc1.weight)
699
+ nn.init.normal_(module.fc2.weight)
700
+ nn.init.normal_(module.fc1.bias, std=1e-6)
701
+ nn.init.normal_(module.fc2.bias, std=1e-6)
702
+ elif isinstance(module, SiglipMultiheadAttentionPoolingHead):
703
+ nn.init.normal_(module.probe.data)
704
+ nn.init.normal_(module.attention.in_proj_weight.data)
705
+ nn.init.zeros_(module.attention.in_proj_bias.data)
706
+ elif isinstance(module, SiglipModel):
707
+ logit_scale_init = torch.log(torch.tensor(1.0))
708
+ module.logit_scale.data.fill_(logit_scale_init)
709
+ module.logit_bias.data.zero_()
710
+ elif isinstance(module, (nn.Linear, nn.Conv2d)):
711
+ lecun_normal_(module.weight)
712
+ if module.bias is not None:
713
+ nn.init.zeros_(module.bias)
714
+ elif isinstance(module, nn.LayerNorm):
715
  module.bias.data.zero_()
716
  module.weight.data.fill_(1.0)
 
 
 
 
 
 
717
 
718
 
719
  SIGLIP_START_DOCSTRING = r"""
 
832
  self.layers = nn.ModuleList([SiglipEncoderLayer(config) for _ in range(config.num_hidden_layers)])
833
  self.gradient_checkpointing = False
834
 
835
+ # Ignore copy
836
  def forward(
837
  self,
838
  inputs_embeds,
839
  attention_mask: Optional[torch.Tensor] = None,
 
840
  output_attentions: Optional[bool] = None,
841
  output_hidden_states: Optional[bool] = None,
842
  return_dict: Optional[bool] = None,
 
853
  - 1 for tokens that are **not masked**,
854
  - 0 for tokens that are **masked**.
855
 
 
 
 
 
 
 
 
856
  [What are attention masks?](../glossary#attention-mask)
857
  output_attentions (`bool`, *optional*):
858
  Whether or not to return the attentions tensors of all attention layers. See `attentions` under
 
873
  all_attentions = () if output_attentions else None
874
 
875
  hidden_states = inputs_embeds
876
+ for encoder_layer in self.layers:
877
  if output_hidden_states:
878
  encoder_states = encoder_states + (hidden_states,)
879
  if self.gradient_checkpointing and self.training:
880
+ layer_outputs = self._gradient_checkpointing_func(
881
+ encoder_layer.__call__,
 
 
 
 
 
 
 
882
  hidden_states,
883
  attention_mask,
884
+ output_attentions,
885
  )
886
  else:
887
  layer_outputs = encoder_layer(
888
  hidden_states,
889
  attention_mask,
 
890
  output_attentions=output_attentions,
891
  )
892
 
 
945
 
946
  hidden_states = self.embeddings(input_ids=input_ids, position_ids=position_ids)
947
 
948
+ # note: SigLIP's text model does not use a causal mask, unlike the original CLIP model.
949
  # expand attention_mask
950
  if attention_mask is not None:
951
+ # [batch_size, seq_len] -> [batch_size, 1, tgt_seq_len, src_seq_len]
952
+ attention_mask = _prepare_4d_attention_mask(attention_mask, hidden_states.dtype)
953
 
954
  encoder_outputs = self.encoder(
955
  inputs_embeds=hidden_states,
956
+ attention_mask=attention_mask,
 
957
  output_attentions=output_attentions,
958
  output_hidden_states=output_hidden_states,
959
  return_dict=return_dict,
 
1020
  >>> model = SiglipTextModel.from_pretrained("google/siglip-base-patch16-224")
1021
  >>> tokenizer = AutoTokenizer.from_pretrained("google/siglip-base-patch16-224")
1022
 
1023
+ >>> # important: make sure to set padding="max_length" as that's how the model was trained
1024
+ >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding="max_length", return_tensors="pt")
1025
 
1026
  >>> outputs = model(**inputs)
1027
  >>> last_hidden_state = outputs.last_hidden_state
 
1166
 
1167
  >>> outputs = model(**inputs)
1168
  >>> last_hidden_state = outputs.last_hidden_state
1169
+ >>> pooled_output = outputs.pooler_output # pooled features
1170
  ```"""
1171
  return_dict = return_dict if return_dict is not None else self.config.use_return_dict
1172
 
 
1200
  text_config = config.text_config
1201
  vision_config = config.vision_config
1202
 
1203
+ self.text_model = SiglipTextTransformer(text_config)
1204
+ self.vision_model = SiglipVisionTransformer(vision_config)
1205
 
1206
+ self.logit_scale = nn.Parameter(torch.randn(1))
1207
+ self.logit_bias = nn.Parameter(torch.randn(1))
 
 
 
 
 
 
 
 
1208
 
1209
  # Initialize weights and apply final processing
1210
  self.post_init()
 
1227
  Examples:
1228
 
1229
  ```python
1230
+ >>> from transformers import AutoTokenizer, AutoModel
1231
+ >>> import torch
1232
 
1233
+ >>> model = AutoModel.from_pretrained("google/siglip-base-patch16-224")
1234
  >>> tokenizer = AutoTokenizer.from_pretrained("google/siglip-base-patch16-224")
1235
 
1236
+ >>> # important: make sure to set padding="max_length" as that's how the model was trained
1237
+ >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding="max_length", return_tensors="pt")
1238
+ >>> with torch.no_grad():
1239
+ ... text_features = model.get_text_features(**inputs)
1240
  ```"""
1241
  # Use SigLIP model's config for some fields (if specified) instead of those of vision & text components.
1242
  output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
 
1276
  ```python
1277
  >>> from PIL import Image
1278
  >>> import requests
1279
+ >>> from transformers import AutoProcessor, AutoModel
1280
+ >>> import torch
1281
 
1282
+ >>> model = AutoModel.from_pretrained("google/siglip-base-patch16-224")
1283
  >>> processor = AutoProcessor.from_pretrained("google/siglip-base-patch16-224")
1284
 
1285
  >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
 
1287
 
1288
  >>> inputs = processor(images=image, return_tensors="pt")
1289
 
1290
+ >>> with torch.no_grad():
1291
+ ... image_features = model.get_image_features(**inputs)
1292
  ```"""
1293
  # Use SiglipModel's config for some fields (if specified) instead of those of vision & text components.
1294
  output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
 
1329
  ```python
1330
  >>> from PIL import Image
1331
  >>> import requests
1332
+ >>> from transformers import AutoProcessor, AutoModel
1333
+ >>> import torch
1334
 
1335
+ >>> model = AutoModel.from_pretrained("google/siglip-base-patch16-224")
1336
  >>> processor = AutoProcessor.from_pretrained("google/siglip-base-patch16-224")
1337
 
1338
  >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
1339
  >>> image = Image.open(requests.get(url, stream=True).raw)
1340
 
1341
+ >>> texts = ["a photo of 2 cats", "a photo of 2 dogs"]
1342
+ >>> # important: we pass `padding=max_length` since the model was trained with this
1343
+ >>> inputs = processor(text=texts, images=image, padding="max_length", return_tensors="pt")
1344
 
1345
+ >>> with torch.no_grad():
1346
+ ... outputs = model(**inputs)
1347
+
1348
+ >>> logits_per_image = outputs.logits_per_image
1349
+ >>> probs = torch.sigmoid(logits_per_image) # these are the probabilities
1350
+ >>> print(f"{probs[0][0]:.1%} that image 0 is '{texts[0]}'")
1351
+ 31.9% that image 0 is 'a photo of 2 cats'
1352
  ```"""
1353
  # Use SigLIP model's config for some fields (if specified) instead of those of vision & text components.
1354
  output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
 
1381
  text_embeds = text_embeds / text_embeds.norm(p=2, dim=-1, keepdim=True)
1382
 
1383
  # cosine similarity as logits
1384
+ logits_per_text = torch.matmul(text_embeds, image_embeds.t()) * self.logit_scale.exp() + self.logit_bias
1385
  logits_per_image = logits_per_text.t()
1386
 
 
 
1387
  loss = None
1388
  if return_loss:
1389
  raise NotImplementedError("SigLIP loss to be implemented")
processing_siglip.py ADDED
@@ -0,0 +1,143 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2024 The HuggingFace Inc. team.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """
16
+ Image/Text processor class for SigLIP.
17
+ """
18
+
19
+ from typing import List, Optional, Union
20
+
21
+ from transformers.feature_extraction_utils import BatchFeature
22
+ from transformers.image_utils import ImageInput
23
+ from transformers.processing_utils import ProcessorMixin
24
+ from transformers.tokenization_utils_base import PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
25
+ from transformers.utils import TensorType
26
+
27
+
28
+ class SiglipProcessor(ProcessorMixin):
29
+ r"""
30
+ Constructs a Siglip processor which wraps a Siglip image processor and a Siglip tokenizer into a single processor.
31
+
32
+ [`SiglipProcessor`] offers all the functionalities of [`SiglipImageProcessor`] and [`SiglipTokenizer`]. See the
33
+ [`~SiglipProcessor.__call__`] and [`~SiglipProcessor.decode`] for more information.
34
+
35
+ Args:
36
+ image_processor ([`SiglipImageProcessor`]):
37
+ The image processor is a required input.
38
+ tokenizer ([`SiglipTokenizer`]):
39
+ The tokenizer is a required input.
40
+ """
41
+
42
+ attributes = ["image_processor", "tokenizer"]
43
+ image_processor_class = "SiglipImageProcessor"
44
+ tokenizer_class = "SiglipTokenizer"
45
+
46
+ def __init__(self, image_processor, tokenizer):
47
+ super().__init__(image_processor, tokenizer)
48
+
49
+ def __call__(
50
+ self,
51
+ text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
52
+ images: ImageInput = None,
53
+ padding: Union[bool, str, PaddingStrategy] = False,
54
+ truncation: Union[bool, str, TruncationStrategy] = None,
55
+ max_length: int = None,
56
+ return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH,
57
+ ) -> BatchFeature:
58
+ """
59
+ Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
60
+ and `kwargs` arguments to SiglipTokenizer's [`~SiglipTokenizer.__call__`] if `text` is not `None` to encode
61
+ the text. To prepare the image(s), this method forwards the `images` argument to
62
+ SiglipImageProcessor's [`~SiglipImageProcessor.__call__`] if `images` is not `None`. Please refer to the doctsring
63
+ of the above two methods for more information.
64
+
65
+ Args:
66
+ text (`str`, `List[str]`, `List[List[str]]`):
67
+ The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
68
+ (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
69
+ `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
70
+ images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`):
71
+ The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
72
+ tensor. In case of a NumPy array/PyTorch tensor, each image should be of shape (C, H, W), where C is a
73
+ number of channels, H and W are image height and width.
74
+ padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):
75
+ Select a strategy to pad the returned sequences (according to the model's padding side and padding
76
+ index) among:
77
+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
78
+ sequence if provided).
79
+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
80
+ acceptable input length for the model if that argument is not provided.
81
+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
82
+ lengths).
83
+ max_length (`int`, *optional*):
84
+ Maximum length of the returned list and optionally padding length (see above).
85
+ truncation (`bool`, *optional*):
86
+ Activates truncation to cut input sequences longer than `max_length` to `max_length`.
87
+ return_tensors (`str` or [`~utils.TensorType`], *optional*):
88
+ If set, will return tensors of a particular framework. Acceptable values are:
89
+
90
+ - `'tf'`: Return TensorFlow `tf.constant` objects.
91
+ - `'pt'`: Return PyTorch `torch.Tensor` objects.
92
+ - `'np'`: Return NumPy `np.ndarray` objects.
93
+ - `'jax'`: Return JAX `jnp.ndarray` objects.
94
+
95
+ Returns:
96
+ [`BatchFeature`]: A [`BatchFeature`] with the following fields:
97
+
98
+ - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
99
+ - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
100
+ `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
101
+ `None`).
102
+ - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
103
+ """
104
+
105
+ if text is None and images is None:
106
+ raise ValueError("You have to specify either text or images. Both cannot be none.")
107
+
108
+ if text is not None:
109
+ encoding = self.tokenizer(
110
+ text, return_tensors=return_tensors, padding=padding, truncation=truncation, max_length=max_length
111
+ )
112
+
113
+ if images is not None:
114
+ image_features = self.image_processor(images, return_tensors=return_tensors)
115
+
116
+ if text is not None and images is not None:
117
+ encoding["pixel_values"] = image_features.pixel_values
118
+ return encoding
119
+ elif text is not None:
120
+ return encoding
121
+ else:
122
+ return BatchFeature(data=dict(**image_features), tensor_type=return_tensors)
123
+
124
+ def decode(self, *args, **kwargs):
125
+ """
126
+ This method forwards all its arguments to SiglipTokenizer's [`~PreTrainedTokenizer.decode`]. Please refer to
127
+ the docstring of this method for more information.
128
+ """
129
+ return self.tokenizer.decode(*args, **kwargs)
130
+
131
+ def batch_decode(self, *args, **kwargs):
132
+ """
133
+ This method forwards all its arguments to SiglipTokenizer's [`~PreTrainedTokenizer.batch_decode`]. Please
134
+ refer to the docstring of this method for more information.
135
+ """
136
+ return self.tokenizer.batch_decode(*args, **kwargs)
137
+
138
+ @property
139
+ # Copied from transformers.models.clip.processing_clip.CLIPProcessor.model_input_names with CLIP->Siglip, T5->Siglip
140
+ def model_input_names(self):
141
+ tokenizer_input_names = self.tokenizer.model_input_names
142
+ image_processor_input_names = self.image_processor.model_input_names
143
+ return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
tokenization_siglip.py ADDED
@@ -0,0 +1,389 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2024 The HuggingFace Inc. team.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """ Tokenization class for SigLIP model."""
16
+
17
+ import os
18
+ import re
19
+ import string
20
+ import warnings
21
+ from shutil import copyfile
22
+ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
23
+
24
+ import sentencepiece as spm
25
+
26
+ from transformers.convert_slow_tokenizer import import_protobuf
27
+ from transformers.tokenization_utils import PreTrainedTokenizer
28
+ from transformers.tokenization_utils_base import AddedToken
29
+
30
+
31
+ if TYPE_CHECKING:
32
+ from transformers.tokenization_utils_base import TextInput
33
+ from transformers.utils import logging, requires_backends
34
+
35
+
36
+ logger = logging.get_logger(__name__)
37
+
38
+ VOCAB_FILES_NAMES = {"vocab_file": "spiece.model"}
39
+
40
+ PRETRAINED_VOCAB_FILES_MAP = {
41
+ "vocab_file": {
42
+ "google/siglip-base-patch16-224": "https://huggingface.co/google/siglip-base-patch16-224/resolve/main/spiece.model",
43
+ }
44
+ }
45
+
46
+ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
47
+ "google/siglip-base-patch16-224": 256,
48
+ }
49
+
50
+ SPIECE_UNDERLINE = "▁"
51
+
52
+
53
+ class SiglipTokenizer(PreTrainedTokenizer):
54
+ """
55
+ Construct a Siglip tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
56
+
57
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
58
+ this superclass for more information regarding those methods.
59
+
60
+ Args:
61
+ vocab_file (`str`):
62
+ [SentencePiece](https://github.com/google/sentencepiece) file (generally has a *.spm* extension) that
63
+ contains the vocabulary necessary to instantiate a tokenizer.
64
+ eos_token (`str`, *optional*, defaults to `"</s>"`):
65
+ The end of sequence token.
66
+ unk_token (`str`, *optional*, defaults to `"<unk>"`):
67
+ The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
68
+ token instead.
69
+ pad_token (`str`, *optional*, defaults to `"</s>"`):
70
+ The token used for padding, for example when batching sequences of different lengths.
71
+ additional_special_tokens (`List[str]`, *optional*):
72
+ Additional special tokens used by the tokenizer.
73
+ sp_model_kwargs (`dict`, *optional*):
74
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
75
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
76
+ to set:
77
+
78
+ - `enable_sampling`: Enable subword regularization.
79
+ - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
80
+
81
+ - `nbest_size = {0,1}`: No sampling is performed.
82
+ - `nbest_size > 1`: samples from the nbest_size results.
83
+ - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
84
+ using forward-filtering-and-backward-sampling algorithm.
85
+
86
+ - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
87
+ BPE-dropout.
88
+ model_max_length (`int`, *optional*, defaults to 64):
89
+ The maximum length (in number of tokens) for model inputs.
90
+ do_lower_case (`bool`, *optional*, defaults to `True`):
91
+ Whether or not to lowercase the input when tokenizing.
92
+ """
93
+
94
+ vocab_files_names = VOCAB_FILES_NAMES
95
+ pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
96
+ max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
97
+ model_input_names = ["input_ids", "attention_mask"]
98
+
99
+ def __init__(
100
+ self,
101
+ vocab_file,
102
+ eos_token="</s>",
103
+ unk_token="<unk>",
104
+ pad_token="</s>",
105
+ additional_special_tokens=None,
106
+ sp_model_kwargs: Optional[Dict[str, Any]] = None,
107
+ model_max_length=64,
108
+ do_lower_case=True,
109
+ **kwargs,
110
+ ) -> None:
111
+ requires_backends(self, "protobuf")
112
+
113
+ pad_token = (
114
+ AddedToken(pad_token, rstrip=True, lstrip=True, normalized=False, special=True)
115
+ if isinstance(pad_token, str)
116
+ else pad_token
117
+ )
118
+ unk_token = (
119
+ AddedToken(unk_token, rstrip=True, lstrip=True, normalized=False, special=True)
120
+ if isinstance(unk_token, str)
121
+ else unk_token
122
+ )
123
+ eos_token = (
124
+ AddedToken(eos_token, rstrip=True, lstrip=True, normalized=False, special=True)
125
+ if isinstance(eos_token, str)
126
+ else eos_token
127
+ )
128
+
129
+ self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
130
+
131
+ self.do_lower_case = do_lower_case
132
+ self.vocab_file = vocab_file
133
+
134
+ self.sp_model = self.get_spm_processor()
135
+ self.vocab_file = vocab_file
136
+
137
+ super().__init__(
138
+ eos_token=eos_token,
139
+ unk_token=unk_token,
140
+ pad_token=pad_token,
141
+ additional_special_tokens=additional_special_tokens,
142
+ sp_model_kwargs=self.sp_model_kwargs,
143
+ model_max_length=model_max_length,
144
+ do_lower_case=do_lower_case,
145
+ **kwargs,
146
+ )
147
+
148
+ def get_spm_processor(self):
149
+ tokenizer = spm.SentencePieceProcessor(**self.sp_model_kwargs)
150
+ with open(self.vocab_file, "rb") as f:
151
+ sp_model = f.read()
152
+ model_pb2 = import_protobuf()
153
+ model = model_pb2.ModelProto.FromString(sp_model)
154
+ normalizer_spec = model_pb2.NormalizerSpec()
155
+ normalizer_spec.add_dummy_prefix = False
156
+ model.normalizer_spec.MergeFrom(normalizer_spec)
157
+ sp_model = model.SerializeToString()
158
+ tokenizer.LoadFromSerializedProto(sp_model)
159
+ return tokenizer
160
+
161
+ @property
162
+ # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.vocab_size
163
+ def vocab_size(self):
164
+ return self.sp_model.get_piece_size()
165
+
166
+ # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.get_vocab
167
+ def get_vocab(self):
168
+ vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
169
+ vocab.update(self.added_tokens_encoder)
170
+ return vocab
171
+
172
+ # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.get_special_tokens_mask
173
+ def get_special_tokens_mask(
174
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
175
+ ) -> List[int]:
176
+ """
177
+ Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
178
+ special tokens using the tokenizer `prepare_for_model` method.
179
+
180
+ Args:
181
+ token_ids_0 (`List[int]`):
182
+ List of IDs.
183
+ token_ids_1 (`List[int]`, *optional*):
184
+ Optional second list of IDs for sequence pairs.
185
+ already_has_special_tokens (`bool`, *optional*, defaults to `False`):
186
+ Whether or not the token list is already formatted with special tokens for the model.
187
+
188
+ Returns:
189
+ `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
190
+ """
191
+ if already_has_special_tokens:
192
+ return super().get_special_tokens_mask(
193
+ token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
194
+ )
195
+
196
+ # normal case: some special tokens
197
+ if token_ids_1 is None:
198
+ return ([0] * len(token_ids_0)) + [1]
199
+ return ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]
200
+
201
+ # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer._add_eos_if_not_present
202
+ def _add_eos_if_not_present(self, token_ids: List[int]) -> List[int]:
203
+ """Do not add eos again if user already added it."""
204
+ if len(token_ids) > 0 and token_ids[-1] == self.eos_token_id:
205
+ warnings.warn(
206
+ f"This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated"
207
+ " eos tokens being added."
208
+ )
209
+ return token_ids
210
+ else:
211
+ return token_ids + [self.eos_token_id]
212
+
213
+ # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.create_token_type_ids_from_sequences
214
+ def create_token_type_ids_from_sequences(
215
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
216
+ ) -> List[int]:
217
+ """
218
+ Create a mask from the two sequences passed to be used in a sequence-pair classification task. T5 does not make
219
+ use of token type ids, therefore a list of zeros is returned.
220
+
221
+ Args:
222
+ token_ids_0 (`List[int]`):
223
+ List of IDs.
224
+ token_ids_1 (`List[int]`, *optional*):
225
+ Optional second list of IDs for sequence pairs.
226
+
227
+ Returns:
228
+ `List[int]`: List of zeros.
229
+ """
230
+ eos = [self.eos_token_id]
231
+
232
+ if token_ids_1 is None:
233
+ return len(token_ids_0 + eos) * [0]
234
+ return len(token_ids_0 + eos + token_ids_1 + eos) * [0]
235
+
236
+ # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.build_inputs_with_special_tokens
237
+ def build_inputs_with_special_tokens(
238
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
239
+ ) -> List[int]:
240
+ """
241
+ Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
242
+ adding special tokens. A sequence has the following format:
243
+
244
+ - single sequence: `X </s>`
245
+ - pair of sequences: `A </s> B </s>`
246
+
247
+ Args:
248
+ token_ids_0 (`List[int]`):
249
+ List of IDs to which the special tokens will be added.
250
+ token_ids_1 (`List[int]`, *optional*):
251
+ Optional second list of IDs for sequence pairs.
252
+
253
+ Returns:
254
+ `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
255
+ """
256
+ token_ids_0 = self._add_eos_if_not_present(token_ids_0)
257
+ if token_ids_1 is None:
258
+ return token_ids_0
259
+ else:
260
+ token_ids_1 = self._add_eos_if_not_present(token_ids_1)
261
+ return token_ids_0 + token_ids_1
262
+
263
+ # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.__getstate__
264
+ def __getstate__(self):
265
+ state = self.__dict__.copy()
266
+ state["sp_model"] = None
267
+ return state
268
+
269
+ # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.__setstate__
270
+ def __setstate__(self, d):
271
+ self.__dict__ = d
272
+
273
+ # for backward compatibility
274
+ if not hasattr(self, "sp_model_kwargs"):
275
+ self.sp_model_kwargs = {}
276
+
277
+ self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
278
+ self.sp_model.Load(self.vocab_file)
279
+
280
+ def remove_punctuation(self, text: str) -> str:
281
+ return text.translate(str.maketrans("", "", string.punctuation))
282
+
283
+ # source: https://github.com/google-research/big_vision/blob/3b8e5ab6ad4f96e32b32826f9e1b8fd277914f9c/big_vision/evaluators/proj/image_text/prompt_engineering.py#L94
284
+ def canonicalize_text(self, text, *, keep_punctuation_exact_string=None):
285
+ """Returns canonicalized `text` (puncuation removed).
286
+
287
+ Args:
288
+ text (`str`):
289
+ String to be canonicalized.
290
+ keep_punctuation_exact_string (`str`, *optional*):
291
+ If provided, then this exact string is kept. For example providing '{}' will keep any occurrences of '{}'
292
+ (but will still remove '{' and '}' that appear separately).
293
+ """
294
+ if keep_punctuation_exact_string:
295
+ text = keep_punctuation_exact_string.join(
296
+ self.remove_punctuation(part) for part in text.split(keep_punctuation_exact_string)
297
+ )
298
+ else:
299
+ text = self.remove_punctuation(text)
300
+ text = re.sub(r"\s+", " ", text)
301
+ text = text.strip()
302
+
303
+ return text
304
+
305
+ def tokenize(self, text: "TextInput", add_special_tokens=False, **kwargs) -> List[str]:
306
+ """
307
+ Converts a string to a list of tokens.
308
+ """
309
+ tokens = super().tokenize(SPIECE_UNDERLINE + text.replace(SPIECE_UNDERLINE, " "), **kwargs)
310
+
311
+ if len(tokens) > 1 and tokens[0] == SPIECE_UNDERLINE and tokens[1] in self.all_special_tokens:
312
+ tokens = tokens[1:]
313
+ return tokens
314
+
315
+ @property
316
+ # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.unk_token_length
317
+ def unk_token_length(self):
318
+ return len(self.sp_model.encode(str(self.unk_token)))
319
+
320
+ def _tokenize(self, text, **kwargs):
321
+ """
322
+ Returns a tokenized string.
323
+
324
+ We de-activated the `add_dummy_prefix` option, thus the sentencepiece internals will always strip any
325
+ SPIECE_UNDERLINE.
326
+
327
+ For example: `self.sp_model.encode(f"{SPIECE_UNDERLINE}Hey", out_type = str)` will give `['H', 'e', 'y']` instead of `['▁He', 'y']`.
328
+
329
+ Thus we always encode `f"{unk_token}text"` and strip the `unk_token`. Here is an example with `unk_token = "<unk>"` and `unk_token_length = 4`.
330
+ `self.tokenizer.sp_model.encode("<unk> Hey", out_type = str)[4:]`.
331
+ """
332
+ text = self.canonicalize_text(text, keep_punctuation_exact_string=None)
333
+ tokens = self.sp_model.encode(text, out_type=str)
334
+
335
+ # 1. Encode string + prefix ex: "<unk> Hey"
336
+ tokens = self.sp_model.encode(self.unk_token + text, out_type=str)
337
+ # 2. Remove self.unk_token from ['<','unk','>', '▁Hey']
338
+ return tokens[self.unk_token_length :] if len(tokens) >= self.unk_token_length else tokens
339
+
340
+ # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer._convert_token_to_id
341
+ def _convert_token_to_id(self, token):
342
+ """Converts a token (str) in an id using the vocab."""
343
+ return self.sp_model.piece_to_id(token)
344
+
345
+ # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer._convert_id_to_token
346
+ def _convert_id_to_token(self, index):
347
+ """Converts an index (integer) in a token (str) using the vocab."""
348
+ token = self.sp_model.IdToPiece(index)
349
+ return token
350
+
351
+ # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.convert_tokens_to_string
352
+ def convert_tokens_to_string(self, tokens):
353
+ """Converts a sequence of tokens (string) in a single string."""
354
+ current_sub_tokens = []
355
+ # since we manually add the prefix space, we have to remove it
356
+ tokens[0] = tokens[0].lstrip(SPIECE_UNDERLINE)
357
+ out_string = ""
358
+ prev_is_special = False
359
+ for token in tokens:
360
+ # make sure that special tokens are not decoded using sentencepiece model
361
+ if token in self.all_special_tokens:
362
+ if not prev_is_special:
363
+ out_string += " "
364
+ out_string += self.sp_model.decode(current_sub_tokens) + token
365
+ prev_is_special = True
366
+ current_sub_tokens = []
367
+ else:
368
+ current_sub_tokens.append(token)
369
+ prev_is_special = False
370
+ out_string += self.sp_model.decode(current_sub_tokens)
371
+ return out_string.strip()
372
+
373
+ # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.save_vocabulary
374
+ def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
375
+ if not os.path.isdir(save_directory):
376
+ logger.error(f"Vocabulary path ({save_directory}) should be a directory")
377
+ return
378
+ out_vocab_file = os.path.join(
379
+ save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
380
+ )
381
+
382
+ if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
383
+ copyfile(self.vocab_file, out_vocab_file)
384
+ elif not os.path.isfile(self.vocab_file):
385
+ with open(out_vocab_file, "wb") as fi:
386
+ content_spiece_model = self.sp_model.serialized_model_proto()
387
+ fi.write(content_spiece_model)
388
+
389
+ return (out_vocab_file,)