Repository: huggingface/transformers
Pull request: SmolVLM test fixes

# What does this PR do?

This PR makes the CI for SmolVLM again: fixes the export test + adjusts the expected values for AMD.

Fixes # (issue)

## Before submitting
- [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case).
- [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#create-a-pull-request),
      Pull Request section?
- [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link
      to it if that's the case.
- [ ] Did you make sure to update the documentation with your changes? Here are the
      [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and
      [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation).
- [ ] Did you write any new necessary tests?


## Who can review?

Anyone in the community is free to review the PR once the tests have passed. Feel free to tag
members/contributors who may be interested in your PR.

<!-- Your PR will be replied to more quickly if you can figure out the right person to tag with @

 If you know how to use git blame, that is the easiest way, otherwise, here is a rough guide of **who to tag**.
 Please tag fewer than 3 people.

Models:

- text models: @ArthurZucker
- vision models: @amyeroberts, @qubvel
- speech models: @eustlb
- graph models: @clefourrier

Library:

- flax: @gante and @Rocketknight1
- generate: @zucchini-nlp (visual-language models) or @gante (all others)
- pipelines: @Rocketknight1
- tensorflow: @gante and @Rocketknight1
- tokenizers: @ArthurZucker
- trainer: @zach-huggingface, @SunMarc and @qgallouedec
- chat templates: @Rocketknight1

Integrations:

- deepspeed: HF Trainer/Accelerate: @SunMarc @zach-huggingface
- ray/raytune: @richardliaw, @amogkam
- Big Model Inference: @SunMarc
- quantization (bitsandbytes, autogpt): @SunMarc @MekkCyber

Documentation: @stevhliu

HF projects:

- accelerate: [different repo](https://github.com/huggingface/accelerate)
- datasets: [different repo](https://github.com/huggingface/datasets)
- diffusers: [different repo](https://github.com/huggingface/diffusers)
- rust tokenizers: [different repo](https://github.com/huggingface/tokenizers)

Maintained examples (not research project or legacy):

- Flax: @Rocketknight1
- PyTorch: See Models above and tag the person corresponding to the modality of the example.
- TensorFlow: @Rocketknight1

 -->


Changed files:
filename: src/transformers/integrations/executorch.py
patch: @@ -128,11 +128,7 @@ def export_text_decoder(self):
         """Export the text decoder component."""
 
         # Create text decoder exportable wrapper
-        self.exportable_text_decoder = TorchExportableModuleForDecoderOnlyLM(
-            model=self.text_decoder,
-            max_batch_size=self.max_batch_size,
-            max_cache_len=self.max_cache_len,
-        )
+        self.exportable_text_decoder = TorchExportableModuleForDecoderOnlyLM(model=self.text_decoder)
 
         # Use the existing text decoder exportable wrapper
         seq_length = 3

filename: src/transformers/models/smolvlm/modeling_smolvlm.py
patch: @@ -27,7 +27,7 @@
 
 from ...activations import ACT2FN
 from ...cache_utils import Cache, DynamicCache
-from ...generation import GenerationMixin
+from ...generation import GenerationConfig, GenerationMixin
 from ...modeling_attn_mask_utils import _prepare_4d_attention_mask
 from ...modeling_flash_attention_utils import FlashAttentionKwargs
 from ...modeling_layers import GradientCheckpointingLayer
@@ -844,6 +844,7 @@ def __init__(self, config):
         self.image_token_id = self.config.image_token_id
         self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False)
         self.vocab_size = config.text_config.vocab_size
+        self.model.text_model.generation_config = GenerationConfig.from_model_config(config)
 
         # Initialize weights and apply final processing
         self.post_init()

filename: src/transformers/models/smolvlm/modular_smolvlm.py
patch: @@ -20,6 +20,7 @@
 from torch import nn
 
 from ...cache_utils import Cache, DynamicCache
+from ...generation import GenerationConfig
 from ...modeling_flash_attention_utils import FlashAttentionKwargs
 from ...processing_utils import Unpack
 from ...utils import auto_docstring, can_return_tuple, logging
@@ -341,6 +342,7 @@ class SmolVLMForConditionalGeneration(Idefics3ForConditionalGeneration):
     def __init__(self, config):
         super().__init__(config)
         self.model = SmolVLMModel(config)
+        self.model.text_model.generation_config = GenerationConfig.from_model_config(config)
         self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False)
         self.post_init()
 

filename: tests/models/smolvlm/test_modeling_smolvlm.py
patch: @@ -27,6 +27,7 @@
     is_vision_available,
 )
 from transformers.testing_utils import (
+    Expectations,
     cleanup,
     is_flaky,
     require_torch,
@@ -44,6 +45,7 @@
     import torch
 
     from transformers import (
+        GenerationConfig,
         SmolVLMConfig,
         SmolVLMForConditionalGeneration,
         SmolVLMModel,
@@ -595,7 +597,19 @@ def test_integration_test_video(self):
         generated_ids = model.generate(**inputs, max_new_tokens=20)
         generated_texts = self.processor.batch_decode(generated_ids, skip_special_tokens=True)
 
-        expected_generated_text = 'User: You are provided the following series of nine frames from a 0:00:09 [H:MM:SS] video.\n\nFrame from 00:00:\nFrame from 00:01:\nFrame from 00:02:\nFrame from 00:03:\nFrame from 00:04:\nFrame from 00:05:\nFrame from 00:06:\nFrame from 00:08:\nFrame from 00:09:\n\nDescribe this video in detail\nAssistant: The video depicts a large language model architecture, specifically a language model with a "quick brown" feature'  # fmt: skip
+        expected_generated_strings = Expectations(
+            {
+                (None, None): 'User: You are provided the following series of nine frames from a 0:00:09 [H:MM:SS] video.\n\nFrame from 00:00:\nFrame from 00:01:\nFrame from 00:02:\nFrame from 00:03:\nFrame from 00:04:\nFrame from 00:05:\nFrame from 00:06:\nFrame from 00:08:\nFrame from 00:09:\n\nDescribe this video in detail\nAssistant: The video depicts a large language model architecture, specifically a language model with a "quick brown" feature',  # fmt: skip
+                ("cuda", (8, 0)): 'User: You are provided the following series of nine frames from a 0:00:09 [H:MM:SS] video.\n\nFrame from 00:00:\nFrame from 00:01:\nFrame from 00:02:\nFrame from 00:03:\nFrame from 00:04:\nFrame from 00:05:\nFrame from 00:06:\nFrame from 00:08:\nFrame from 00:09:\n\nDescribe this video in detail\nAssistant: The video showcases a large language model architecture, specifically a "Quick Brown" model, which is designed',  # fmt: skip
+                ("cuda", (8, 6)): 'User: You are provided the following series of nine frames from a 0:00:09 [H:MM:SS] video.\n\nFrame from 00:00:\nFrame from 00:01:\nFrame from 00:02:\nFrame from 00:03:\nFrame from 00:04:\nFrame from 00:05:\nFrame from 00:06:\nFrame from 00:08:\nFrame from 00:09:\n\nDescribe this video in detail\nAssistant: The video showcases a large language model, specifically a neural network model, which is designed to learn and',  # fmt: skip
+                ("rocm", None): 'User: You are provided the following series of nine frames from a 0:00:09 [H:MM:SS] video.\n\nFrame from 00:00:\nFrame from 00:01:\nFrame from 00:02:\nFrame from 00:03:\nFrame from 00:04:\nFrame from 00:05:\nFrame from 00:06:\nFrame from 00:08:\nFrame from 00:09:\n\nDescribe this video in detail\nAssistant: The video showcases a large language model architecture, specifically a "Quick Brown" model, which is designed',  # fmt: skip
+            }
+        )  # fmt: skip
+
+        expected_generated_text = expected_generated_strings.get_expectation()
+
+        print(f"Generated text: {generated_texts[0]}")
+
         self.assertEqual(generated_texts[0], expected_generated_text)
 
     @slow
@@ -661,13 +675,25 @@ def test_export_smolvlm_text_decoder(self):
         config.text_config.use_cache = True
         config.text_config.attn_implementation = "sdpa"
 
+        generation_config = GenerationConfig(
+            use_cache=True,
+            cache_implementation="static",
+            max_length=1234,
+            cache_config={
+                "batch_size": 1,
+                "max_cache_len": 1234,
+            },
+        )
+
         # Load the model and extract the text decoder
         model = SmolVLMForConditionalGeneration.from_pretrained(
             model_id,
             torch_dtype=torch.float32,
             config=config,
         )
 
+        model.model.text_model.generation_config = generation_config
+
         text_decoder = model.model.text_model
         text_decoder.eval()