Repository: huggingface/transformers
Pull request: Fix `InternVL` integration test

# What does this PR do?

They never pass on T4 runners ...

Changed files:
filename: tests/models/internvl/test_modeling_internvl.py
patch: @@ -284,6 +284,7 @@ class InternVLQwen2IntegrationTest(unittest.TestCase):
     def setUp(self):
         self.small_model_checkpoint = "OpenGVLab/InternVL3-1B-hf"
         self.medium_model_checkpoint = "OpenGVLab/InternVL3-2B-hf"
+        cleanup(torch_device, gc_collect=True)
 
     def tearDown(self):
         cleanup(torch_device, gc_collect=True)
@@ -329,7 +330,8 @@ def test_qwen2_small_model_integration_forward(self):
         expected_logits_all = Expectations(
             {
                 ("xpu", 3): torch.tensor([11.7500, 14.7500, 14.1250, 10.5625, 6.7812], dtype=torch.bfloat16),
-                ("cuda", 7): torch.tensor([11.9375, 14.8750, 14.0625, 10.7500, 6.9062], dtype=torch.bfloat16),
+                ("cuda", 7): torch.tensor([11.9375, 14.7500, 14.4375, 10.8125,  7.0938], dtype=torch.bfloat16),
+                ("cuda", 8): torch.tensor([11.8750, 14.8125, 14.3125, 10.8125,  6.9375], dtype=torch.bfloat16),
             }
         )  # fmt: skip
         expected_logits = expected_logits_all.get_expectation()
@@ -358,7 +360,8 @@ def test_qwen2_small_model_integration_generate_text_only(self):
         expected_outputs = Expectations(
             {
                 ("xpu", 3): "Whispers of dawn,\nSilent whispers of the night,\nNew day's light.",
-                ("cuda", 7): "Whispers of dawn,\nSilent whispers of the night,\nNew day's light begins.",
+                ("cuda", 7): "Whispers of dawn,\nSilent whispers of the night,\nNew day's light.",
+                ("cuda", 8): "Whispers of dawn,\nSilent whispers of the night,\nNew day's light begins.",
             }
         )  # fmt: skip
         expected_output = expected_outputs.get_expectation()
@@ -425,7 +428,7 @@ def test_qwen2_small_model_integration_batched_generate(self):
         expected_outputs = Expectations(
             {
                 ("xpu", 3): 'user\n\nDescribe this image\nassistant\nThe image shows a street scene with a traditional Chinese archway, known as a "Chinese Gate" or "Chinese Gate"',
-                ("cuda", 7): 'user\n\nDescribe this image\nassistant\nThe image shows a street scene with a traditional Chinese archway, known as a "Chinese Gate" or "Chinese Gate of',
+                ("cuda", 7): 'user\n\nDescribe this image\nassistant\nThe image shows a street scene with a traditional Chinese archway, known as a "Chinese Gate" or "Chinese Arch,"',
             }
         )  # fmt: skip
         expected_output = expected_outputs.get_expectation()
@@ -483,7 +486,7 @@ def test_qwen2_small_model_integration_batched_generate_multi_image(self):
         expected_outputs = Expectations(
             {
                 ("xpu", 3): "user\n\nWhat are the differences between these two images?\nassistant\nThe images show the Statue of Liberty and the Golden Gate Bridge from different angles. Here are the differences:\n\n1. **Foreground",
-                ("cuda", 7): "user\n\nWhat are the differences between these two images?\nassistant\nThe images show the Statue of Liberty and the Golden Gate Bridge from different angles. Here are the differences:\n\n1. **Angle",
+                ("cuda", 7): "user\n\nWhat are the differences between these two images?\nassistant\nThe images show the Statue of Liberty and the Golden Gate Bridge from different angles. Here are the differences:\n\n1. **Foreground",
             }
         )  # fmt: skip
         expected_output = expected_outputs.get_expectation()
@@ -606,7 +609,7 @@ def test_qwen2_small_model_integration_interleaved_images_videos(self):
         expected_outputs = Expectations(
             {
                 ("xpu", 3): "user\n\n\nWhat are the differences between these two images?\nassistant\nThe images depict two distinct scenes:\n\n1. **Left Image:**\n   - The Statue of Liberty is prominently featured on an",
-                ("cuda", 7): "user\n\n\nWhat are the differences between these two images?\nassistant\nThe images depict two distinct scenes:\n\n1. **Left Image**: This shows the Statue of Liberty on Liberty Island, with the",
+                ("cuda", 7): "user\n\n\nWhat are the differences between these two images?\nassistant\nThe images depict two distinct scenes:\n\n1. **Left Image:**\n   - The Statue of Liberty is prominently featured on an",
             }
         )  # fmt: skip
         expected_output = expected_outputs.get_expectation()
@@ -620,7 +623,7 @@ def test_qwen2_small_model_integration_interleaved_images_videos(self):
         expected_outputs = Expectations(
             {
                 ("xpu", 3): "user\nFrame1: \nFrame2: \nFrame3: \nFrame4: \nFrame5: \nFrame6: \nFrame7: \nFrame8: \nWhat type of shot is the man performing?\nassistant\nThe man is performing a forehand shot.",
-                ("cuda", 7): "user\nFrame1: \nFrame2: \nFrame3: \nFrame4: \nFrame5: \nFrame6: \nFrame7: \nFrame8: \nWhat type of shot is the man performing?\nassistant\nA forehand shot",
+                ("cuda", 7): "user\nFrame1: \nFrame2: \nFrame3: \nFrame4: \nFrame5: \nFrame6: \nFrame7: \nFrame8: \nWhat type of shot is the man performing?\nassistant\nThe man is performing a forehand shot.",
             }
         )  # fmt: skip
         expected_output = expected_outputs.get_expectation()
@@ -646,6 +649,7 @@ class InternVLLlamaIntegrationTest(unittest.TestCase):
     def setUp(self):
         self.small_model_checkpoint = "OpenGVLab/InternVL2_5-2B-MPO-hf"
         self.medium_model_checkpoint = "OpenGVLab/InternVL2_5-8B-MPO-hf"
+        cleanup(torch_device, gc_collect=True)
 
     def tearDown(self):
         cleanup(torch_device, gc_collect=True)
@@ -688,13 +692,22 @@ def test_llama_small_model_integration_forward(self):
             output = model(**inputs)
 
         actual_logits = output.logits[0, -1, :5].cpu()
-        expected_logits = torch.tensor([-9.8750, -0.4258, 1.4844, -10.3125, -10.3125], dtype=torch.bfloat16)
+
+        expected_logits_all = Expectations(
+            {
+                ("xpu", 3): torch.tensor([-9.8750, -0.5703, 1.4297, -10.3125, -10.3125], dtype=torch.bfloat16),
+                ("cuda", 7): torch.tensor([-9.8750, -0.5703, 1.4297, -10.3125, -10.3125], dtype=torch.bfloat16),
+                ("cuda", 8): torch.tensor([-9.8750,  -0.5117,   1.4297, -10.3750, -10.3750], dtype=torch.bfloat16),
+            }
+        )  # fmt: skip
+        expected_logits = torch.tensor(expected_logits_all.get_expectation(), dtype=torch.bfloat16)
+
         # The original implementation and the transformers implementation do not match exactly, hence the higher tolerance.
         # The difference is likely due to the different implementations of the attention mechanism (different order of operations)
         # between the transformers Llama model and the original InternLM model.
         # The difference has almost no effect on the output tokens, but it does affect the logits a lot more.
         self.assertTrue(
-            torch.allclose(actual_logits, expected_logits, atol=1),
+            torch.allclose(actual_logits, expected_logits, atol=1e-3),
             f"Actual logits: {actual_logits}"
             f"\nExpected logits: {expected_logits}"
             f"\nDifference: {torch.abs(actual_logits - expected_logits)}",
@@ -765,7 +778,8 @@ def test_llama_small_model_integration_batched_generate(self):
         expected_outputs = Expectations(
             {
                 ("xpu", 3): "user\n\nWrite a haiku for this image\nassistant\nMajestic snow-capped peaks,\nWooden path leads to calm lake,\nNature's peaceful grace.",
-                ("cuda", 7): "user\n\nWrite a haiku for this image\nassistant\nMajestic snow-capped peaks,\nWooden dock stretches to the sea,\nSilent water mirrors.",
+                ("cuda", 7): "user\n\nWrite a haiku for this image\nassistant\nMajestic snow-capped peaks,\nWooden path leads to calm lake,\nNature's peaceful grace.",
+                ("cuda", 8): "user\n\nWrite a haiku for this image\nassistant\nMajestic snow-capped peaks,\nA wooden path leads to the sea,\nPeaceful, still waters.",
             }
         )  # fmt: skip
         expected_output = expected_outputs.get_expectation()
@@ -819,7 +833,7 @@ def test_llama_small_model_integration_batched_generate_multi_image(self):
         # Check first output
         decoded_output = processor.decode(output[0], skip_special_tokens=True)
         # Batching seems to alter the output slightly, but it is also the case in the original implementation. This seems to be expected: https://github.com/huggingface/transformers/issues/23017#issuecomment-1649630232
-        expected_output = 'user\n\nWrite a haiku for this image\nassistant\nMajestic snow-capped peaks,\nA wooden path leads to the sea,\nPeaceful, still waters.'  # fmt: skip
+        expected_output = "user\n\nWrite a haiku for this image\nassistant\nMajestic snow-capped peaks,\nWooden path leads to calm lake,\nNature's peaceful grace."  # fmt: skip
         self.assertEqual(
             decoded_output,
             expected_output,
@@ -940,7 +954,8 @@ def test_llama_small_model_integration_interleaved_images_videos(self):
         expected_outputs = Expectations(
             {
                 ("xpu", 3): "user\n\n\nWhat are the difference between these two images?\nassistant\nI apologize for the confusion in my previous response. After re-examining the images, I can see that they are actually",
-                ("cuda", 7): "user\n\n\nWhat are the difference between these two images?\nassistant\nI apologize for the confusion in my previous response. Upon closer inspection, the differences between the two images are:\n\n1. **",
+                ("cuda", 7): "user\n\n\nWhat are the difference between these two images?\nassistant\nI apologize for the confusion in my previous response. After re-examining the images, I can see that they are actually",
+                ("cuda", 8): "user\n\n\nWhat are the difference between these two images?\nassistant\nI apologize for the confusion in my previous response. After closely examining the images again, I can see that there are several differences",
             }
         )  # fmt: skip
         expected_output = expected_outputs.get_expectation()
@@ -956,6 +971,7 @@ def test_llama_small_model_integration_interleaved_images_videos(self):
             {
                 ("xpu", 3): "user\nFrame1: \nFrame2: \nFrame3: \nFrame4: \nFrame5: \nFrame6: \nFrame7: \nFrame8: \nWhat type of shot is the man performing?\nassistant\nThe man is performing a forehand shot. This is a common shot in tennis where the player swings the racket across their",
                 ("cuda", 7): "user\nFrame1: \nFrame2: \nFrame3: \nFrame4: \nFrame5: \nFrame6: \nFrame7: \nFrame8: \nWhat type of shot is the man performing?\nassistant\nThe man is performing a forehand shot. This is a common shot in tennis where the player swings the racket across their",
+                ("cuda", 8): "user\nFrame1: \nFrame2: \nFrame3: \nFrame4: \nFrame5: \nFrame6: \nFrame7: \nFrame8: \nWhat type of shot is the man performing?\nassistant\nThe man is performing a forehand shot. This is a common shot in tennis where the player swings the racket across their",
             }
         )  # fmt: skip
         expected_output = expected_outputs.get_expectation()
@@ -970,7 +986,8 @@ def test_llama_small_model_integration_interleaved_images_videos(self):
         expected_outputs = Expectations(
             {
                 ("xpu", 3): "user\n\nWrite a haiku for this image\nassistant\nMajestic snow-capped peaks,\nWooden dock stretches to the sea,\nSilent water mirrors.",
-                ("cuda", 7): "user\n\nWrite a haiku for this image\nassistant\nMajestic snow-capped peaks,\nA wooden path leads to the sea,\nPeaceful, untouched dreams.",
+                ("cuda", 7): "user\n\nWrite a haiku for this image\nassistant\nMajestic snow-capped peaks,\nWooden dock stretches to the sea,\nSilent water mirrors.",
+                ("cuda", 8): "user\n\nWrite a haiku for this image\nassistant\nMajestic snow-capped peaks,\nWooden dock stretches to the sea,\nSilent water mirrors.",
             }
         )  # fmt: skip
         expected_output = expected_outputs.get_expectation()