Repository: huggingface/transformers
Pull request: Fix T5 v1.1 detection

PR #41541 refactored `tie_word_embeddings` handling (among other things) which subtly broke detection of T5 v1.1 vs. original detection. As a consequence, decoder output scaling was always applied, regardless of T5 version.

This is resolved by using the correct value for `tie_word_embeddings`.

**Testing:**

This was not covered by the tests since the tests instantiate the config once and modify attributes on the config. This is problematic since all the decision logic is happening in `T5Config.__init__`. This was addressed by having a specific `get_config_v1_1` method that initializes the config as if it were coming from a v1.1 model (e.g., flan-t5).



Changed files:
filename: src/transformers/models/t5/configuration_t5.py
patch: @@ -140,7 +140,7 @@ def __init__(
         # The model code was relying on saved configs where `tie_word_embeddings` is
         # set to `False` in 1.1v and using it as indicator of whether to scale or not
         # But in fact we tie weights always and force it to be `True`
-        self.scale_decoder_outputs = kwargs.get("tie_word_embeddings") is not False
+        self.scale_decoder_outputs = tie_word_embeddings is True
         self.tie_word_embeddings = True
 
         super().__init__(is_encoder_decoder=is_encoder_decoder, **kwargs)

filename: tests/models/mt5/test_modeling_mt5.py
patch: @@ -46,7 +46,6 @@
     )
 
 
-# Copied from tests.models.t5.test_modeling_t5.T5ModelTester with T5->MT5
 class MT5ModelTester:
     def __init__(
         self,
@@ -421,20 +420,6 @@ def create_and_check_model_fp16_forward(
         output = model(input_ids, decoder_input_ids=input_ids, attention_mask=attention_mask)["last_hidden_state"]
         self.parent.assertFalse(torch.isnan(output).any().item())
 
-    def check_resize_embeddings_t5_v1_1(
-        self,
-        config,
-    ):
-        prev_vocab_size = config.vocab_size
-
-        config.tie_word_embeddings = False
-        model = MT5ForConditionalGeneration(config=config).to(torch_device).eval()
-        model.resize_token_embeddings(prev_vocab_size - 10)
-
-        self.parent.assertEqual(model.get_input_embeddings().weight.shape[0], prev_vocab_size - 10)
-        self.parent.assertEqual(model.get_output_embeddings().weight.shape[0], prev_vocab_size - 10)
-        self.parent.assertEqual(model.config.vocab_size, prev_vocab_size - 10)
-
     def prepare_config_and_inputs_for_common(self):
         config_and_inputs = self.prepare_config_and_inputs()
         (
@@ -456,7 +441,6 @@ def prepare_config_and_inputs_for_common(self):
 
 
 @require_torch
-# Copied from tests.models.t5.test_modeling_t5.T5ModelTest with T5->MT5, google-t5/t5-small->google/mt5-small
 class MT5ModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase):
     all_model_classes = (
         (MT5Model, MT5ForConditionalGeneration, MT5ForSequenceClassification, MT5ForQuestionAnswering)
@@ -549,16 +533,10 @@ def test_shift_right(self):
 
     def test_model(self):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        if config_and_inputs[0].__class__.__name__ == "T" + "5Config":
+            self.assertTrue(config_and_inputs[0].scale_decoder_outputs)
         self.model_tester.create_and_check_model(*config_and_inputs)
 
-    def test_model_v1_1(self):
-        config_and_inputs = self.model_tester.prepare_config_and_inputs()
-        # check that gated gelu feed forward and different word embeddings work
-        config = config_and_inputs[0]
-        config.tie_word_embeddings = False
-        config.feed_forward_proj = "gated-gelu"
-        self.model_tester.create_and_check_model(config, *config_and_inputs[1:])
-
     # MT5ForSequenceClassification does not support inputs_embeds
     def test_inputs_embeds(self):
         config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
@@ -652,10 +630,6 @@ def test_model_fp16_forward(self):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model_fp16_forward(*config_and_inputs)
 
-    def test_v1_1_resize_embeddings(self):
-        config = self.model_tester.prepare_config_and_inputs()[0]
-        self.model_tester.check_resize_embeddings_t5_v1_1(config)
-
     @slow
     def test_model_from_pretrained(self):
         model_name = "google/mt5-small"

filename: tests/models/t5/test_modeling_t5.py
patch: @@ -166,6 +166,28 @@ def get_config(self):
             decoder_start_token_id=self.decoder_start_token_id,
         )
 
+    def get_config_v1_1(self):
+        return T5Config(
+            vocab_size=self.vocab_size,
+            d_model=self.hidden_size,
+            d_ff=self.d_ff,
+            d_kv=self.hidden_size // self.num_attention_heads,
+            num_layers=self.num_hidden_layers,
+            num_decoder_layers=self.decoder_layers,
+            num_heads=self.num_attention_heads,
+            relative_attention_num_buckets=self.relative_attention_num_buckets,
+            dropout_rate=self.dropout_rate,
+            initializer_factor=self.initializer_factor,
+            eos_token_id=self.eos_token_id,
+            bos_token_id=self.pad_token_id,
+            pad_token_id=self.pad_token_id,
+            decoder_start_token_id=self.decoder_start_token_id,
+            # V1.1 related params: uses gated-gelu and `tie_word_embeddings=False` as an
+            # indicator to not scale decoder outputs
+            feed_forward_proj="gated-gelu",
+            tie_word_embeddings=False,
+        )
+
     def check_prepare_lm_labels_via_shift_left(
         self,
         config,
@@ -436,6 +458,8 @@ def check_resize_embeddings_t5_v1_1(
     ):
         prev_vocab_size = config.vocab_size
 
+        # V1.1 related params: uses gated-gelu and `tie_word_embeddings=False`
+        config.feed_forward_proj = "gated-gelu"
         config.tie_word_embeddings = False
         model = T5ForConditionalGeneration(config=config).to(torch_device).eval()
         model.resize_token_embeddings(prev_vocab_size - 10)
@@ -557,15 +581,17 @@ def test_shift_right(self):
 
     def test_model(self):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
+        self.assertTrue(config_and_inputs[0].scale_decoder_outputs)
         self.model_tester.create_and_check_model(*config_and_inputs)
 
     def test_model_v1_1(self):
         config_and_inputs = self.model_tester.prepare_config_and_inputs()
-        # check that gated gelu feed forward and different word embeddings work
-        config = config_and_inputs[0]
-        config.tie_word_embeddings = False
-        config.feed_forward_proj = "gated-gelu"
-        self.model_tester.create_and_check_model(config, *config_and_inputs[1:])
+        config_v1 = self.model_tester.get_config_v1_1()
+        config_and_inputs = list(config_and_inputs)
+        config_and_inputs[0] = config_v1
+
+        self.assertFalse(config_and_inputs[0].scale_decoder_outputs)
+        self.model_tester.create_and_check_model(*config_and_inputs)
 
     # T5ForSequenceClassification does not support inputs_embeds
     def test_inputs_embeds(self):