Repository: huggingface/transformers
Pull request: Make projection LayerNorm configurable in LFM2-VL

Make projection LayerNorm configurable in LFM2-VL models.

Changed files:
filename: src/transformers/models/lfm2_vl/configuration_lfm2_vl.py
patch: @@ -46,6 +46,8 @@ class Lfm2VlConfig(PreTrainedConfig):
             The hidden size of the multimodal projector.
         projector_bias (`bool`, *optional*, defaults to `True`):
             Whether to use bias in the multimodal projector.
+        projector_use_layernorm (`bool`, *optional*, defaults to `True`):
+            Whether to use layernorm in the multimodal projector.
         downsample_factor (`int`, *optional*, defaults to 2):
             The downsample_factor factor of the vision backbone.
     """
@@ -61,13 +63,15 @@ def __init__(
         projector_hidden_act="gelu",
         projector_hidden_size=2560,
         projector_bias=True,
+        projector_use_layernorm=True,
         downsample_factor=2,
         **kwargs,
     ):
         self.image_token_id = image_token_id
         self.projector_hidden_act = projector_hidden_act
         self.projector_hidden_size = projector_hidden_size
         self.projector_bias = projector_bias
+        self.projector_use_layernorm = projector_use_layernorm
         self.downsample_factor = downsample_factor
 
         if isinstance(vision_config, dict):

filename: src/transformers/models/lfm2_vl/modeling_lfm2_vl.py
patch: @@ -41,7 +41,8 @@ def __init__(self, config: Lfm2VlConfig):
         super().__init__()
         in_channels = config.vision_config.hidden_size * (config.downsample_factor**2)
         self.factor = config.downsample_factor
-        self.layer_norm = nn.LayerNorm(in_channels)
+        self.use_layer_norm = config.projector_use_layernorm
+        self.layer_norm = nn.LayerNorm(in_channels) if config.projector_use_layernorm else None
         self.linear_1 = nn.Linear(
             in_channels,
             config.projector_hidden_size,
@@ -56,7 +57,8 @@ def __init__(self, config: Lfm2VlConfig):
 
     def forward(self, image_features: torch.Tensor):
         image_features = self.pixel_unshuffle(image_features)
-        image_features = self.layer_norm(image_features)
+        if self.use_layer_norm:
+            image_features = self.layer_norm(image_features)
         hidden_states = self.linear_1(image_features)
         hidden_states = self.act(hidden_states)
         hidden_states = self.linear_2(hidden_states)

filename: src/transformers/models/lfm2_vl/modular_lfm2_vl.py
patch: @@ -41,7 +41,8 @@ def __init__(self, config: Lfm2VlConfig):
         super().__init__()
         in_channels = config.vision_config.hidden_size * (config.downsample_factor**2)
         self.factor = config.downsample_factor
-        self.layer_norm = nn.LayerNorm(in_channels)
+        self.use_layer_norm = config.projector_use_layernorm
+        self.layer_norm = nn.LayerNorm(in_channels) if config.projector_use_layernorm else None
         self.linear_1 = nn.Linear(
             in_channels,
             config.projector_hidden_size,
@@ -56,7 +57,8 @@ def __init__(self, config: Lfm2VlConfig):
 
     def forward(self, image_features: torch.Tensor):
         image_features = self.pixel_unshuffle(image_features)
-        image_features = self.layer_norm(image_features)
+        if self.use_layer_norm:
+            image_features = self.layer_norm(image_features)
         hidden_states = self.linear_1(image_features)
         hidden_states = self.act(hidden_states)
         hidden_states = self.linear_2(hidden_states)