Repository: huggingface/transformers
Pull request: Fix tiles mixing for batched input, add tie_word_embeddings to LFM2VL config

Fixes: For batched input with multiple images tiled input was mixed between batches.Now, using a util function.
Adds: `tie_word_embeddings` to Lfm2VlConfig. Tests for above bug + updated integration test for the newest lfm2vl model.

cc: @zucchini-nlp


Changed files:
filename: src/transformers/models/lfm2_vl/configuration_lfm2_vl.py
patch: @@ -49,6 +49,8 @@ class Lfm2VlConfig(PreTrainedConfig):
             Whether to use layernorm in the multimodal projector.
         downsample_factor (`int`, *optional*, defaults to 2):
             The downsample_factor factor of the vision backbone.
+        tie_word_embeddings (`bool`, *optional*, defaults to `True`):
+            Whether to tie the word embeddings of the text backbone.
     """
 
     model_type = "lfm2_vl"
@@ -64,6 +66,7 @@ def __init__(
         projector_bias=True,
         projector_use_layernorm=True,
         downsample_factor=2,
+        tie_word_embeddings=True,
         **kwargs,
     ):
         self.image_token_id = image_token_id
@@ -87,6 +90,7 @@ def __init__(
 
         self.vision_config = vision_config
         self.text_config = text_config
+        self.tie_word_embeddings = getattr(text_config, "tie_embedding", tie_word_embeddings)
 
         super().__init__(**kwargs)
 

filename: src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.py
patch: @@ -23,6 +23,7 @@
     group_images_by_shape,
     reorder_images,
 )
+from ...image_transforms import split_to_tiles
 from ...image_utils import (
     IMAGENET_STANDARD_MEAN,
     IMAGENET_STANDARD_STD,
@@ -310,14 +311,7 @@ def crop_image_to_patches(
         )
 
         # split the image into patches
-        processed_images = (
-            resized_image.unfold(2, size=tile_size, step=tile_size)
-            .unfold(3, size=tile_size, step=tile_size)
-            .contiguous()
-            .view(batch_size, num_channels, -1, tile_size, tile_size)
-            .permute(2, 0, 1, 3, 4)
-            .reshape(batch_size, -1, num_channels, tile_size, tile_size)
-        )
+        processed_images = split_to_tiles(resized_image, num_tiles_height=grid_height, num_tiles_width=grid_width)
 
         # Re-order processed images to a nested image structure, so it can be reordered back correctly
         # Note that the images can't be stacked because the thumbnail image is of bigger size than patches

filename: tests/models/lfm2_vl/test_image_processing_lfm2_vl.py
patch: @@ -831,3 +831,78 @@ def test_batch_with_tiling(self):
         self.assertEqual(result.pixel_values.shape[0], expected_total)
         self.assertEqual(result.spatial_shapes.shape[0], expected_total)
         self.assertEqual(result.pixel_attention_mask.shape[0], expected_total)
+
+    def test_batch_tiling(self):
+        """Test that patches from different images don't get mixed when batch processing with tiling.
+
+        This test verifies that when processing a batch of images with tiling enabled,
+        patches from image 0 don't end up in image 1's output and vice versa.
+        This was a bug caused by incorrect permute/reshape operations in crop_image_to_patches.
+        """
+        for use_thumbnail in [False, True]:
+            with self.subTest(use_thumbnail=use_thumbnail):
+                image_processing = self.fast_image_processing_class(
+                    do_image_splitting=True,
+                    use_thumbnail=use_thumbnail,
+                    min_tiles=2,
+                    max_tiles=4,
+                    tile_size=512,
+                )
+
+                # Create two large images with completely different solid colors
+                # Red image: RGB = (255, 0, 0)
+                # Blue image: RGB = (0, 0, 255)
+                red_image = Image.new("RGB", (1024, 1024), color=(255, 0, 0))
+                blue_image = Image.new("RGB", (1024, 1024), color=(0, 0, 255))
+
+                result = image_processing(
+                    [[red_image], [blue_image]],
+                    return_tensors="pt",
+                    return_row_col_info=True,
+                    do_rescale=False,  # Keep original pixel values for easier verification
+                    do_normalize=False,
+                )
+
+                # Each 1024x1024 image should be split into 2x2 = 4 tiles
+                red_tiles = result.image_rows[0].item() * result.image_cols[0].item()
+                blue_tiles = result.image_rows[1].item() * result.image_cols[1].item()
+                self.assertEqual(red_tiles, 4)
+                self.assertEqual(blue_tiles, 4)
+
+                # Calculate expected total patches
+                # Without thumbnail: 4 + 4 = 8
+                # With thumbnail: (4 + 1) + (4 + 1) = 10
+                thumb_count = 1 if use_thumbnail else 0
+                expected_total = (red_tiles + thumb_count) + (blue_tiles + thumb_count)
+                self.assertEqual(result.pixel_values.shape[0], expected_total)
+
+                pixel_values = result.pixel_values
+                patch_size = image_processing.encoder_patch_size
+                patches_per_image = red_tiles + thumb_count
+
+                # Check red image patches (and thumbnail if enabled)
+                # All should have high red, zero blue
+                for i in range(patches_per_image):
+                    first_patch = pixel_values[i][0].view(3, patch_size, patch_size)
+                    red_mean = first_patch[0].float().mean().item()
+                    blue_mean = first_patch[2].float().mean().item()
+                    patch_type = "thumbnail" if use_thumbnail and i == red_tiles else f"patch {i}"
+                    self.assertGreater(
+                        red_mean,
+                        blue_mean,
+                        f"Red image {patch_type} has more blue than red - patches may be interleaved",
+                    )
+
+                # Check blue image patches (and thumbnail if enabled)
+                # All should have high blue, zero red
+                for i in range(patches_per_image, 2 * patches_per_image):
+                    first_patch = pixel_values[i][0].view(3, patch_size, patch_size)
+                    red_mean = first_patch[0].float().mean().item()
+                    blue_mean = first_patch[2].float().mean().item()
+                    local_idx = i - patches_per_image
+                    patch_type = "thumbnail" if use_thumbnail and local_idx == blue_tiles else f"patch {local_idx}"
+                    self.assertGreater(
+                        blue_mean,
+                        red_mean,
+                        f"Blue image {patch_type} has more red than blue - patches may be interleaved",
+                    )

filename: tests/models/lfm2_vl/test_modeling_lfm2_vl.py
patch: @@ -311,3 +311,86 @@ def test_integration_test_batched(self):
             "In this image, we see a cat that is lying on its side on a cat bed.",
         ]
         self.assertListEqual(generated_texts, expected_generated_text)
+
+
+@require_torch_accelerator
+@slow
+class Lfm2_5VlForConditionalGenerationIntegrationTest(unittest.TestCase):
+    def setUp(self):
+        self.processor = AutoProcessor.from_pretrained("LiquidAI/LFM2.5-VL-1.6B")
+        self.processor.tokenizer.padding_side = "left"
+        self.image = Image.open(
+            requests.get("http://images.cocodataset.org/val2017/000000039769.jpg", stream=True).raw
+        )
+        self.image2 = Image.open(
+            BytesIO(
+                requests.get(
+                    "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"
+                ).content
+            )
+        )
+
+    def tearDown(self):
+        cleanup(torch_device, gc_collect=True)
+
+    def test_integration_test(self):
+        model = Lfm2VlForConditionalGeneration.from_pretrained(
+            "LiquidAI/LFM2.5-VL-1.6B",
+            dtype=torch.bfloat16,
+            device_map="auto",
+        )
+
+        # Create inputs
+        text = "<image>In this image, we see"
+        images = self.image
+        inputs = self.processor(text=text, images=images, return_tensors="pt")
+        inputs.to(device=torch_device, dtype=torch.bfloat16)
+
+        generated_ids = model.generate(**inputs, max_new_tokens=20, do_sample=False)
+        generated_texts = self.processor.batch_decode(generated_ids, skip_special_tokens=True)
+
+        expected_generated_text = (
+            "In this image, we see two cats lying on a pink blanket. One cat is a tabby, and the other is a"
+        )
+        self.assertEqual(generated_texts[0], expected_generated_text)
+
+    def test_integration_test_high_resolution(self):
+        model = Lfm2VlForConditionalGeneration.from_pretrained(
+            "LiquidAI/LFM2.5-VL-1.6B",
+            dtype=torch.bfloat16,
+            device_map="auto",
+        )
+
+        # Create inputs
+        text = "<image>In this image, we see"
+        images = self.image2
+        inputs = self.processor(text=text, images=images, return_tensors="pt")
+        inputs.to(device=torch_device, dtype=torch.bfloat16)
+
+        generated_ids = model.generate(**inputs, max_new_tokens=20, do_sample=False)
+        generated_texts = self.processor.batch_decode(generated_ids, skip_special_tokens=True)
+
+        expected_generated_text = "In this image, we see the Statue of Liberty, an iconic symbol of freedom and democracy. It stands on Liberty Island in"
+        self.assertEqual(generated_texts[0], expected_generated_text)
+
+    def test_integration_test_batched(self):
+        model = Lfm2VlForConditionalGeneration.from_pretrained(
+            "LiquidAI/LFM2.5-VL-1.6B",
+            dtype=torch.bfloat16,
+            device_map="auto",
+        )
+
+        # Create inputs
+        text = ["<image>In this image, we see", "<image>In this image, we see"]
+        images = [[self.image2], [self.image]]
+        inputs = self.processor(text=text, images=images, return_tensors="pt", padding=True)
+        inputs.to(device=torch_device, dtype=torch.bfloat16)
+
+        generated_ids = model.generate(**inputs, max_new_tokens=20, do_sample=False)
+        generated_texts = self.processor.batch_decode(generated_ids, skip_special_tokens=True)
+
+        expected_generated_text = [
+            "In this image, we see the Statue of Liberty, an iconic symbol of freedom and democracy. It stands on Liberty Island in",
+            "In this image, we see two cats lying on a pink blanket. One cat is a tabby, and the other is a",
+        ]
+        self.assertListEqual(generated_texts, expected_generated_text)