Repository: huggingface/transformers
Pull request: Idefics3 without cache fix

# What does this PR do?

This PR fixes generation for models using the `Idefics3ForConditionalGeneration` architecture with `use_cache=False`.

## Testing

<details>
<summary>docling_repro.py</summary>

```py
import os
import torch
import time
from datetime import timedelta
from transformers import AutoProcessor, AutoModelForImageTextToText
from transformers.image_utils import load_image


use_cache = os.getenv("USE_CACHE", "true").lower() in ["true", "1", "yes"]
max_new_tokens = int(os.getenv("MAX_NEW_TOKENS", "50"))
include_images = os.getenv("INCLUDE_IMAGES", "true").lower() in ["true", "1", "yes"]

# Load model and processor
DEVICE = "cpu"
if torch.cuda.is_available():
    DEVICE = "cuda"
elif torch.backends.mps.is_available():
    DEVICE = "mps"
print(f"USING DEVICE: {DEVICE}")
start_time = time.time()
model_path = "ibm-granite/granite-docling-258M"
processor = AutoProcessor.from_pretrained(model_path)

# Prepare inputs
image = load_image("https://huggingface.co/ibm-granite/granite-docling-258M/resolve/main/assets/new_arxiv.png")
messages = [
    {
        "role": "user",
        "content": [
            {"type": "text", "text": "Convert this page to docling."}
        ]
    },
]
images = None
if include_images:
    images = [image]
    messages[0]["content"] = [{"type": "image"}] + messages[0]["content"]

prompt = processor.apply_chat_template(messages, add_generation_prompt=True)
print(prompt)
inputs = processor(text=prompt, images=images, return_tensors="pt")
inputs = inputs.to(DEVICE)
print("==> Done preparing inputs: {}s".format(timedelta(seconds=time.time() - start_time).total_seconds()))
for k, v in inputs.items():
    if isinstance(v, torch.Tensor):
        print(f"===> [Input tensor {k}] shape: {v.shape}, sum: {v.sum()}")

# Generate outputs
model = AutoModelForImageTextToText.from_pretrained(
    pretrained_model_name_or_path=model_path,
    torch_dtype=torch.bfloat16,
).to(DEVICE)
print("==> Done loading model: {}s".format(timedelta(seconds=time.time() - start_time).total_seconds()))
generated_ids = model.generate(**inputs, use_cache=use_cache, max_new_tokens=max_new_tokens, do_sample=False)
print("==> Done generating: {}s".format(timedelta(seconds=time.time() - start_time).total_seconds()))
prompt_length = inputs.input_ids.shape[1]
trimmed_generated_ids = generated_ids[:, prompt_length:]
print(f"==> New tokens shape: {trimmed_generated_ids.shape}")
doctags = processor.batch_decode(
    trimmed_generated_ids,
    skip_special_tokens=False,
)[0].lstrip()

print(f"DocTags: \n{doctags}\n")
```

</details>

```sh
# These should produce identical outputs
USE_CACHE=0 python docling_repro.py
USE_CACHE=1 python docling_repro.py
```

## Before submitting
- [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case).
- [x] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#create-a-pull-request),
      Pull Request section?
- [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link
      to it if that's the case.
- [ ] Did you make sure to update the documentation with your changes? Here are the
      [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and
      [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation).
- [ ] Did you write any new necessary tests?


## Who can review?

- multimodal models: @zucchini-nlp


Changed files:
filename: src/transformers/models/idefics2/modeling_idefics2.py
patch: @@ -1117,6 +1117,7 @@ def prepare_inputs_for_generation(
         image_hidden_states=None,
         logits_to_keep=None,
         is_first_iteration=False,
+        use_cache=False,
         **kwargs,
     ):
         # Overwritten -- there are mutually exclusive inputs (if the logic to make `image_hidden_states` take
@@ -1132,10 +1133,11 @@ def prepare_inputs_for_generation(
             image_hidden_states=image_hidden_states,
             logits_to_keep=logits_to_keep,
             is_first_iteration=is_first_iteration,
+            use_cache=use_cache,
             **kwargs,
         )
 
-        if image_hidden_states is not None or not is_first_iteration:
+        if image_hidden_states is not None or (use_cache and not is_first_iteration):
             model_inputs["pixel_values"] = None
             model_inputs["pixel_attention_mask"] = None
 

filename: src/transformers/models/idefics3/modeling_idefics3.py
patch: @@ -882,6 +882,7 @@ def prepare_inputs_for_generation(
         image_hidden_states=None,
         logits_to_keep=None,
         is_first_iteration=False,
+        use_cache=False,
         **kwargs,
     ):
         # Overwritten -- there are mutually exclusive inputs (if the logic to make `image_hidden_states` take
@@ -897,10 +898,11 @@ def prepare_inputs_for_generation(
             image_hidden_states=image_hidden_states,
             logits_to_keep=logits_to_keep,
             is_first_iteration=is_first_iteration,
+            use_cache=use_cache,
             **kwargs,
         )
 
-        if image_hidden_states is not None or not is_first_iteration:
+        if image_hidden_states is not None or (use_cache and not is_first_iteration):
             model_inputs["pixel_values"] = None
             model_inputs["pixel_attention_mask"] = None
 

filename: src/transformers/models/smolvlm/modeling_smolvlm.py
patch: @@ -851,6 +851,7 @@ def prepare_inputs_for_generation(
         image_hidden_states=None,
         logits_to_keep=None,
         is_first_iteration=False,
+        use_cache=False,
         **kwargs,
     ):
         # Overwritten -- there are mutually exclusive inputs (if the logic to make `image_hidden_states` take
@@ -866,10 +867,11 @@ def prepare_inputs_for_generation(
             image_hidden_states=image_hidden_states,
             logits_to_keep=logits_to_keep,
             is_first_iteration=is_first_iteration,
+            use_cache=use_cache,
             **kwargs,
         )
 
-        if image_hidden_states is not None or not is_first_iteration:
+        if image_hidden_states is not None or (use_cache and not is_first_iteration):
             model_inputs["pixel_values"] = None
             model_inputs["pixel_attention_mask"] = None