Repository: huggingface/transformers
Pull request: Mllama: raise better error

# What does this PR do?

Passing images as a flat list do not give same logits in mllama, as passing the images in nested batch format. To get the model working correctly, one should pass as many images per batch, as there are image tokens.

See below reproducer

```python
import requests
import torch
from PIL import Image
from transformers import MllamaForConditionalGeneration, AutoProcessor

url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/0052a70beed5bf71b92610a43a52df6d286cd5f3/diffusers/rabbit.jpg"
image = Image.open(requests.get(url, stream=True).raw)
texts = ["

What do you see here?", "

What do you see here but longer?"]

repo_id = "mv11/11"
processor = AutoProcessor.from_pretrained(repo_id)
model = MllamaForConditionalGeneration.from_pretrained(repo_id, device_map='auto')

batch = processor(text=texts, images=[image, image], return_tensors="pt", padding=True) # .to(model.device)
with torch.no_grad():
    model_output = model(
        input_ids = batch['input_ids'],
        attention_mask = batch['attention_mask'], 
        pixel_values = batch['pixel_values'], 
        aspect_ratio_ids = batch['aspect_ratio_ids'], 
        aspect_ratio_mask = batch['aspect_ratio_mask'], 
        cross_attention_mask = batch['cross_attention_mask'],
        )


batch = processor(text=texts, images=[[image], [image]], return_tensors="pt", padding=True) # .to(model.device)
with torch.no_grad():
    model_output_2 = model(
        input_ids = batch['input_ids'],
        attention_mask = batch['attention_mask'], 
        pixel_values = batch['pixel_values'], 
        aspect_ratio_ids = batch['aspect_ratio_ids'], 
        aspect_ratio_mask = batch['aspect_ratio_mask'], 
        cross_attention_mask = batch['cross_attention_mask'],
        )

print(torch.allclose(model_output_2.logits, model_output.logits))
>>> False
```

Changed files:
filename: src/transformers/models/mllama/processing_mllama.py
patch: @@ -301,12 +301,16 @@ def __call__(
                 raise ValueError(
                     "If a batch of text is provided, there should be either no images or at least one image per sample"
                 )
-            if sum(n_images_in_images) != sum(n_images_in_text):
+            if sum(n_images_in_text) > 0 and n_images_in_images != n_images_in_text:
                 if images is None:
                     raise ValueError("No image were provided, but there are image tokens in the prompt")
                 else:
+                    add_message = ""
+                    if sum(n_images_in_images) == sum(n_images_in_text):
+                        add_message = "Make sure to pass your images as a nested list, where each sub-list holds images per batch"
                     raise ValueError(
-                        f"The number of image token ({sum(n_images_in_text)}) should be the same as in the number of provided images ({sum(n_images_in_images)})"
+                        f"The number of image tokens in each text ({n_images_in_text}) should be the same as the "
+                        f"number of provided images per batch ({n_images_in_images}). {add_message}"
                     )
 
         if images is not None:

filename: tests/models/mllama/test_processor_mllama.py
patch: @@ -327,6 +327,11 @@ def test_process_interleaved_images_prompts_image_error(self):
         with self.assertRaises(ValueError):
             processor(text=text, images=None, padding=True)
 
+        # see https://github.com/huggingface/transformers/pull/35934
+        images = [self.image1, self.image2]
+        with self.assertRaises(ValueError):
+            processor(text=text, images=None, padding=True)
+
     # Override as MllamaProcessor needs image tokens in prompts
     def prepare_text_inputs(self, batch_size: Optional[int] = None):
         if batch_size is None:
@@ -340,3 +345,32 @@ def prepare_text_inputs(self, batch_size: Optional[int] = None):
         return ["lower newer 
", "
 upper older longer string"] + ["
 lower newer"] * (
             batch_size - 2
         )
+
+    def test_unstructured_kwargs_batched(self):
+        # Overriden because Mllama expects images in nested format. For 2 images it can't infer
+        # the correct nesting, so we better throw an error
+        if "image_processor" not in self.processor_class.attributes:
+            self.skipTest(f"image_processor attribute not present in {self.processor_class}")
+        processor_components = self.prepare_components()
+        processor_kwargs = self.prepare_processor_dict()
+        processor = self.processor_class(**processor_components, **processor_kwargs)
+        self.skip_processor_without_typed_kwargs(processor)
+
+        input_str = self.prepare_text_inputs(batch_size=2)
+        image_input = self.prepare_image_inputs(batch_size=2)
+        image_input = [[image_input[0]], [image_input[1]]]
+        inputs = processor(
+            text=input_str,
+            images=image_input,
+            return_tensors="pt",
+            do_rescale=True,
+            rescale_factor=-1,
+            padding="longest",
+            max_length=76,
+        )
+
+        self.assertLessEqual(inputs[self.images_input_name][0][0].mean(), 0)
+        self.assertTrue(
+            len(inputs[self.text_input_name][0]) == len(inputs[self.text_input_name][1])
+            and len(inputs[self.text_input_name][1]) < 76
+        )