Repository: huggingface/transformers
Pull request: Load generation config from nested configs

# What does this PR do?

Fixes https://github.com/huggingface/transformers/issues/42794. The issue was that model's generation params are saved in text config, but we couldn't call `config.get_text_config()` on a dict object. We can delete the second `load_pretrained` because each model gets a default generation config from model config in `__init__`

We could technically get the text config from dict, but I don't see a reason for adding more code if we can delete redundant blocks 


Changed files:
filename: src/transformers/generation/configuration_utils.py
patch: @@ -1150,20 +1150,25 @@ def from_model_config(cls, model_config: Union["PreTrainedConfig", dict]) -> "Ge
 
         # Removes all `None` from the model config dict -- this lets the generation config defaults to take hold
         config_dict = {key: value for key, value in config_dict.items() if value is not None}
-
         generation_config = cls.from_dict(config_dict, return_unused_kwargs=False, _from_model_config=True)
 
         # Special case: some models have generation attributes set in the decoder. Use them if still unset in the
         # generation config (which in turn is defined from the outer attributes of model config).
-        if not isinstance(model_config, dict):
-            decoder_config = model_config.get_text_config(decoder=True)
-            if decoder_config is not model_config:
-                default_generation_config = GenerationConfig()
-                decoder_config_dict = decoder_config.to_dict()
-                for attr in generation_config.to_dict():
-                    is_unset = getattr(generation_config, attr) == getattr(default_generation_config, attr)
-                    if attr in decoder_config_dict and is_unset:
-                        setattr(generation_config, attr, decoder_config_dict[attr])
+        if isinstance(model_config, dict):
+            decoder_possible_text_config_names = ("decoder", "generator", "text_config")
+            for text_config_name in decoder_possible_text_config_names:
+                if text_config := model_config.get(text_config_name):
+                    model_config = text_config
+                    break
+        else:
+            model_config = model_config.get_text_config(decoder=True)
+            model_config = model_config.to_dict()
+
+        default_generation_config = GenerationConfig()
+        for attr in generation_config.to_dict():
+            is_unset = getattr(generation_config, attr) == getattr(default_generation_config, attr)
+            if attr in model_config and is_unset:
+                setattr(generation_config, attr, model_config[attr])
 
         # If any `output_...` flag is set to `True`, we ensure `return_dict_in_generate` is set to `True`.
         if not generation_config.return_dict_in_generate:

filename: src/transformers/generation/utils.py
patch: @@ -406,6 +406,9 @@ def adjust_generation_fn(
                     **repo_loading_kwargs,
                 )
             except OSError:
+                # `self` already has a generation config created from model config, but model config will
+                # not contain any generation-specific params. These are popped at config's `__init__`.
+                # Thus we have to load from `config.json` and create a generation config from it (for BART)
                 logger.info(
                     "Generation config file not found, using a generation config created from the model config."
                 )
@@ -417,6 +420,7 @@ def adjust_generation_fn(
                     _from_model_config=True,
                     **repo_loading_kwargs,
                 )
+
             # Load custom generate function if `pretrained_model_name_or_path` defines it (and override `generate`)
             if hasattr(self, "load_custom_generate") and trust_remote_code:
                 try:

filename: tests/generation/test_utils.py
patch: @@ -67,7 +67,6 @@
         AutoModelForImageTextToText,
         AutoModelForSeq2SeqLM,
         AutoModelForSpeechSeq2Seq,
-        AutoModelForVision2Seq,
         BartForConditionalGeneration,
         BartTokenizer,
         DataCollatorWithFlattening,
@@ -4427,7 +4426,7 @@ def test_generate_vision2text_conditioning(self):
         """Test that `decoder_input_ids` can be used to condition the generation in vision-to-text models"""
         pixel_values = floats_tensor((2, 3, 30, 30))
         conditioning_input = torch.tensor([[10], [10]])  # this should be the 2nd output token, after the BOS token
-        model = AutoModelForVision2Seq.from_pretrained(
+        model = AutoModelForImageTextToText.from_pretrained(
             "hf-internal-testing/tiny-random-VisionEncoderDecoderModel-vit-gpt2"
         )
         pixel_values = pixel_values.to(torch_device)
@@ -4447,6 +4446,24 @@ def test_generate_vision2text_conditioning(self):
         self.assertTrue(np.array_equal(output_sequences_decoder_input_ids, output_sequences_input_ids))
         self.assertTrue(np.array_equal(output_sequences_decoder_input_ids[:, 1:2], conditioning_input))
 
+    @pytest.mark.generate
+    def test_load_generation_config_from_text_subconfig(self):
+        """
+        Tests that generation config is be loaded from model's `text_config` when not present
+        in the model repo. We should infer the text config correctly and re-use special tokens
+        for generation. See https://github.com/huggingface/transformers/issues/42794
+        """
+        model = AutoModelForImageTextToText.from_pretrained(
+            "hf-internal-testing/tiny-random-LlavaForConditionalGeneration-no-generation-config",
+            device_map=torch_device,
+        )
+        self.assertTrue(model.generation_config.eos_token_id is not None)
+        self.assertTrue(model.generation_config.bos_token_id is not None)
+        self.assertTrue(model.generation_config.pad_token_id is not None)
+
+        # test that we can generate without inputs, i.e. from BOS
+        _ = model.generate()
+
     @require_read_token
     @slow
     @require_torch_accelerator