Repository: huggingface/transformers
Pull request: Fix non FA2 tests after FA2 installed in CI docker image

# What does this PR do?



Changed files:
filename: tests/models/glm4v/test_modeling_glm4v.py
patch: @@ -14,7 +14,6 @@
 """Testing suite for the PyTorch GLM-4.1V model."""
 
 import copy
-import gc
 import unittest
 
 from transformers import (
@@ -25,6 +24,7 @@
     is_torch_available,
 )
 from transformers.testing_utils import (
+    cleanup,
     require_flash_attn,
     require_torch,
     require_torch_gpu,
@@ -309,8 +309,7 @@ def setUp(self):
         ]
 
     def tearDown(self):
-        gc.collect()
-        torch.cuda.empty_cache()
+        cleanup(torch_device, gc_collect=True)
 
     @slow
     def test_small_model_integration_test(self):
@@ -376,10 +375,8 @@ def test_small_model_integration_test_with_video(self):
         model = Glm4vForConditionalGeneration.from_pretrained(
             "THUDM/GLM-4.1V-9B-Thinking", dtype=torch.float16, device_map="auto"
         )
-        questions = ["Describe this video."] * 2
-        video_urls = [
-            "https://huggingface.co/datasets/hf-internal-testing/fixtures_videos/resolve/main/tennis.mp4"
-        ] * 2
+        questions = ["Describe this video."]
+        video_urls = ["https://huggingface.co/datasets/hf-internal-testing/fixtures_videos/resolve/main/tennis.mp4"]
         messages = [
             [
                 {
@@ -400,8 +397,7 @@ def test_small_model_integration_test_with_video(self):
         ).to(torch_device)
         output = model.generate(**inputs, max_new_tokens=30)
         EXPECTED_DECODED_TEXT = [
-            "\n012345Describe this video.\n<think>Got it, let's analyze the video. First, the scene is a room with a wooden floor, maybe a traditional Japanese room with tatami",
-            "\n012345Describe this video.\n<think>Got it, let's analyze the video. First, the scene is a room with a wooden floor, maybe a traditional Japanese room with tatami"
+            "\n012345Describe this video.\n<think>Got it, let's analyze the video. First, the scene is an indoor tennis court. There are two players: one in the foreground wearing"
         ]  # fmt: skip
         self.assertEqual(
             processor.batch_decode(output, skip_special_tokens=True),
@@ -509,8 +505,8 @@ def test_small_model_integration_test_batch_flashatt2(self):
         output = model.generate(**inputs, max_new_tokens=30)
 
         EXPECTED_DECODED_TEXT = [
-            "\nWhat kind of dog is this?\n<think>Got it, let's look at the image. The animal in the picture has a stocky build, thick fur, and a face that's",
-            "\nWhat kind of dog is this?\n<think>Got it, let's look at the image. Wait, the animals here are cats, not dogs. The question is about a dog, but"
+            "\nWhat kind of dog is this?\n<think>Got it, let's look at the image. The animal in the picture is not a dog; it's a cat. Specifically, it looks",
+            "\nWhat kind of dog is this?\n<think>Got it, let's look at the image. Wait, the animals here are cats, not dogs. The question is about a dog, but",
         ]  # fmt: skip
         self.assertEqual(
             self.processor.batch_decode(output, skip_special_tokens=True),

filename: tests/models/mistral/test_modeling_mistral.py
patch: @@ -345,16 +345,26 @@ def test_generation_beyond_sliding_window_dynamic(self, attn_implementation: str
         """Test that we can correctly generate beyond the sliding window. This is non-trivial as Mistral will use
         a DynamicCache with only sliding layers."""
 
+        # Impossible to test it with this model (even with < 100 tokens), probably due to the compilation of a large model.
+        if attn_implementation == "flex_attention":
+            self.skipTest(
+                reason="`flex_attention` gives `torch._inductor.exc.InductorError: RuntimeError: No valid triton configs. OutOfMemoryError: out of resource: triton_tem_fused_0 Required: 147456 Hardware limit:101376 Reducing block sizes or `num_stages` may help.`"
+            )
+
         model_id = "mistralai/Mistral-7B-v0.1"
         EXPECTED_COMPLETIONS = [
-            "This is a nice place. This is a nice place. This is a nice place. This is",
+            "scenery, scenery, scenery, scenery, scenery,",
             ", green, yellow, orange, purple, pink, brown, black, white, gray, silver",
         ]
 
         input_text = [
-            "This is a nice place. " * 800 + "I really enjoy the scenery,",  # This is larger than 4096 tokens
+            "This is a nice place. " * 682 + "I really enjoy the scenery,",  # This has 4101 tokens, 15 more than 4096
             "A list of colors: red, blue",  # This will almost all be padding tokens
         ]
+
+        if attn_implementation == "eager":
+            input_text = input_text[:1]
+
         tokenizer = AutoTokenizer.from_pretrained(model_id, padding="left")
         tokenizer.pad_token_id = tokenizer.eos_token_id
         inputs = tokenizer(input_text, padding=True, return_tensors="pt").to(torch_device)
@@ -364,14 +374,14 @@ def test_generation_beyond_sliding_window_dynamic(self, attn_implementation: str
         )
 
         # Make sure prefill is larger than sliding window
-        input_size = inputs.input_ids.shape[-1]
+        batch_size, input_size = inputs.input_ids.shape
         self.assertTrue(input_size > model.config.sliding_window)
 
         # Should already be Dynamic by default, but let's make sure!
         out = model.generate(**inputs, max_new_tokens=20, cache_implementation="dynamic", return_dict_in_generate=True)
-        output_text = tokenizer.batch_decode(out.sequences[:, input_size:])
+        output_text = tokenizer.batch_decode(out.sequences[:batch_size, input_size:])
 
-        self.assertEqual(output_text, EXPECTED_COMPLETIONS)
+        self.assertEqual(output_text, EXPECTED_COMPLETIONS[:batch_size])
 
         # Let's check that the dynamic cache has hybrid layers!
         dynamic_cache = out.past_key_values

filename: tests/models/qwen2/test_modeling_qwen2.py
patch: @@ -317,7 +317,7 @@ def test_3b_generation(self):
         model_id = "Qwen/Qwen2.5-3B"
         tokenizer = AutoTokenizer.from_pretrained(model_id)
         model = Qwen2ForCausalLM.from_pretrained(
-            model_id, use_sliding_window=True, max_window_layers=28, sliding_window=2048
+            model_id, use_sliding_window=True, max_window_layers=28, sliding_window=2048, dtype=torch.float16
         ).to(torch_device)
         # we need a long text to test sliding window
         # fmt: off
@@ -710,24 +710,27 @@ def test_3b_generation(self):
         # fmt: on
 
         input_ids = tokenizer(LONG_TEXT, return_tensors="pt").input_ids.to(torch_device)
-        generated_ids = model.generate(input_ids, max_new_tokens=50)[:, input_ids.shape[1] :]
+        generated_ids = model.generate(input_ids, max_new_tokens=20)[:, input_ids.shape[1] :]
 
-        torch.testing.assert_close(generated_ids.cpu(), torch.tensor([[  576,  4570, 71818,   374,  6509,   825,   315,   279,  1429, 88228, 21984,   315,   279, 11220,  4948,  8584,   304,   279,   467, 19859, 4180,  4168,    13,  1084, 14230, 16170,   315,  3349, 19256,   304, 279,  2266,   315, 13444, 14550,   448, 50867,   429,   525,   330, 4145,     1,   476,   330, 88845,     1,   323,  1246,  3425,   264]], dtype=torch.long))  # fmt: skip
+        torch.testing.assert_close(generated_ids.cpu(), torch.tensor([[279, 467, 19859, 4180, 4168, 572, 264, 882, 315, 2244, 2297, 304, 5616, 13, 576, 66827, 66846, 572, 304, 17704]], dtype=torch.long))  # fmt: skip
         self.assertEqual(
             tokenizer.decode(generated_ids[0]),
-            """ The Guanzi is considered one of the most foundational texts of the developing political economy in the Warring States period. It addresses principles of price regulation in the context of effectively dealing with commodities that are "light" or "heavy" and how whether a""",
+            " the Warring States period was a time of great change in China. The Zhou dynasty was in decline",
         )
         model.config._attn_implementation = "eager"
-        new_generated_ids = model.generate(input_ids, max_new_tokens=50)[:, input_ids.shape[1] :]
+        new_generated_ids = model.generate(input_ids, max_new_tokens=20)[:, input_ids.shape[1] :]
         with self.subTest("Eager matches sdpa"):
             torch.testing.assert_close(generated_ids, new_generated_ids, rtol=1e-4, atol=1e-4)
 
-        model.config._attn_implementation = "flex_attention"
-        new_generated_ids = model.generate(input_ids, max_new_tokens=50)[:, input_ids.shape[1] :]
-        with self.subTest("Eager matches Flex attention"):
-            torch.testing.assert_close(generated_ids, new_generated_ids, rtol=1e-4, atol=1e-4)
+        # `flex_attention` gives `torch._inductor.exc.InductorError: RuntimeError: No valid triton configs. OutOfMemoryError: out of resource: triton_tem_fused_0 Required: 147456 Hardware limit:101376 Reducing block sizes or `num_stages` may help.`
+        # Impossible to test it with this model (even with < 100 tokens), probably due to the compilation of a large model.
+
+        # model.config._attn_implementation = "flex_attention"
+        # new_generated_ids = model.generate(input_ids, max_new_tokens=20)[:, input_ids.shape[1] :]
+        # with self.subTest("Eager matches Flex attention"):
+        #     torch.testing.assert_close(generated_ids, new_generated_ids, rtol=1e-4, atol=1e-4)
 
         model.config._attn_implementation = "flash_attention_2"
-        new_generated_ids = model.generate(input_ids, max_new_tokens=50)[:, input_ids.shape[1] :]
+        new_generated_ids = model.generate(input_ids, max_new_tokens=20)[:, input_ids.shape[1] :]
         with self.subTest("Eager matches flash attention"):
             torch.testing.assert_close(generated_ids, new_generated_ids, rtol=1e-4, atol=1e-4)

filename: tests/models/qwen3/test_modeling_qwen3.py
patch: @@ -146,7 +146,7 @@ def test_model_600m_generation(self):
     @require_flash_attn
     @pytest.mark.flash_attn_test
     def test_model_600m_long_prompt(self):
-        EXPECTED_OUTPUT_TOKEN_IDS = [306, 338]
+        EXPECTED_OUTPUT_TOKEN_IDS = [198, 198]
         # An input with 4097 tokens that is above the size of the sliding window
         input_ids = [1] + [306, 338] * 2048
         model = Qwen3ForCausalLM.from_pretrained(
@@ -309,7 +309,7 @@ def test_600m_generation(self):
         model_id = "Qwen/Qwen3-0.6B-Base"
         tokenizer = AutoTokenizer.from_pretrained(model_id)
         model = Qwen3ForCausalLM.from_pretrained(
-            model_id, use_sliding_window=True, max_window_layers=28, sliding_window=2048
+            model_id, use_sliding_window=True, max_window_layers=28, sliding_window=2048, dtype=torch.float16
         ).to(torch_device)
         # we need a long text to test sliding window
         # fmt: off
@@ -714,10 +714,13 @@ def test_600m_generation(self):
         with self.subTest("Eager matches sdpa"):
             torch.testing.assert_close(generated_ids, new_generated_ids, rtol=1e-4, atol=1e-4)
 
-        model.config._attn_implementation = "flex_attention"
-        new_generated_ids = model.generate(input_ids, max_new_tokens=50)[:, input_ids.shape[1] :]
-        with self.subTest("Eager matches Flex attention"):
-            torch.testing.assert_close(generated_ids, new_generated_ids, rtol=1e-4, atol=1e-4)
+        # `flex_attention` gives `torch._inductor.exc.InductorError: RuntimeError: No valid triton configs. OutOfMemoryError: out of resource: triton_tem_fused_0 Required: 147456 Hardware limit:101376 Reducing block sizes or `num_stages` may help.`
+        # Impossible to test it with this model (even with < 100 tokens), probably due to the compilation of a large model.
+
+        # model.config._attn_implementation = "flex_attention"
+        # new_generated_ids = model.generate(input_ids, max_new_tokens=50)[:, input_ids.shape[1] :]
+        # with self.subTest("Eager matches Flex attention"):
+        #     torch.testing.assert_close(generated_ids, new_generated_ids, rtol=1e-4, atol=1e-4)
 
         model.config._attn_implementation = "flash_attention_2"
         new_generated_ids = model.generate(input_ids, max_new_tokens=50)[:, input_ids.shape[1] :]