Repository: huggingface/transformers
Pull request: [MistralCommonBackend] Upgrade mistral-common to v1.10.0

# What does this PR do?

This PR add support to mistral-common v1.10.0. This involves:
- reasoning effort feature
- Python 3.14

Also add `image_sizes` in `apply_chat_template` output to match what is expected by standard processor.


Changed files:
filename: setup.py
patch: @@ -93,7 +93,7 @@
     "kenlm",
     "kernels>=0.10.2,<0.11",
     "librosa",
-    "mistral-common[image]>=1.8.8",
+    "mistral-common[image]>=1.10.0",
     "nltk<=3.8.1",
     "num2words",
     "numpy>=1.17",
@@ -185,8 +185,7 @@ def deps_list(*pkgs):
 extras["kernels"] = deps_list("kernels")
 extras["sentencepiece"] = deps_list("sentencepiece", "protobuf")
 extras["tiktoken"] = deps_list("tiktoken", "blobfile")
-if PYTHON_MINOR_VERSION < 14:
-    extras["mistral-common"] = deps_list("mistral-common[image]")
+extras["mistral-common"] = deps_list("mistral-common[image]")
 extras["chat_template"] = deps_list("jinja2", "jmespath")
 extras["sklearn"] = deps_list("scikit-learn")
 extras["accelerate"] = deps_list("accelerate")
@@ -238,8 +237,7 @@ def deps_list(*pkgs):
     + extras["sentencepiece"]
     + extras["serving"]
 )
-if PYTHON_MINOR_VERSION < 14:
-    extras["testing"] += extras["mistral-common"]
+extras["testing"] += extras["mistral-common"]
 
 extras["deepspeed-testing"] = extras["deepspeed"] + extras["testing"] + extras["optuna"] + extras["sentencepiece"]
 extras["all"] = (
@@ -254,8 +252,7 @@ def deps_list(*pkgs):
     + extras["chat_template"]
     + extras["num2words"]
 )
-if PYTHON_MINOR_VERSION < 14:
-    extras["all"] += extras["mistral-common"]
+extras["all"] += extras["mistral-common"]
 
 extras["dev"] = extras["all"] + extras["testing"] + extras["ja"] + extras["sklearn"]
 

filename: src/transformers/dependency_versions_table.py
patch: @@ -25,7 +25,7 @@
     "kenlm": "kenlm",
     "kernels": "kernels>=0.10.2,<0.11",
     "librosa": "librosa",
-    "mistral-common[image]": "mistral-common[image]>=1.8.8",
+    "mistral-common[image]": "mistral-common[image]>=1.10.0",
     "nltk": "nltk<=3.8.1",
     "num2words": "num2words",
     "numpy": "numpy>=1.17",

filename: src/transformers/tokenization_mistral_common.py
patch: @@ -23,6 +23,7 @@
 from huggingface_hub import create_repo
 
 from transformers.audio_utils import load_audio_as
+from transformers.image_utils import get_image_size
 from transformers.tokenization_utils_base import (
     VERY_LARGE_INTEGER,
     AddedToken,
@@ -38,7 +39,7 @@
 
 
 if is_mistral_common_available():
-    from mistral_common.protocol.instruct.request import ChatCompletionRequest
+    from mistral_common.protocol.instruct.request import ChatCompletionRequest, ReasoningEffort
     from mistral_common.protocol.instruct.validator import ValidationMode
     from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy, SpecialTokens
     from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
@@ -1034,6 +1035,7 @@ def apply_chat_template(  # type: ignore[override]
         max_length: int | None = None,
         return_tensors: str | TensorType | None = None,
         return_dict: bool = True,
+        reasoning_effort: ReasoningEffort | None = None,
         **kwargs,
     ) -> str | list[int] | list[str] | list[list[int]] | BatchEncoding:
         """
@@ -1080,7 +1082,13 @@ def apply_chat_template(  # type: ignore[override]
                 - `'pt'`: Return PyTorch `torch.Tensor` objects.
             return_dict (`bool`, defaults to `False`):
                 Whether to return a dictionary with named outputs. Has no effect if tokenize is `False`.
-                If at least one conversation contains an image, its pixel values will be returned in the `pixel_values` key.
+                If at least one conversation contains an image, its pixel values will be returned in the `pixel_values` key and image sizes in the `image_sizes` key.
+            reasoning_effort (`ReasoningEffort`, *optional*):
+                The reasoning effort to use for the chat completion for models that support it. Possible values are:
+                - `ReasoningEffort.none`: The model will not reason.
+                - `ReasoningEffort.high`: The model will use a reasoning approach.
+                If not specified, the default reasoning effort will be used.
+
             kwargs (additional keyword arguments, *optional*):
                 Not supported by `MistralCommonBackend.apply_chat_template`.
                 Will raise an error if used.
@@ -1176,6 +1184,7 @@ def _maybe_adapt_message(message: dict[str, Any]) -> None:
                 messages=messages,
                 tools=tools,
                 continue_final_message=continue_final_message,
+                reasoning_effort=reasoning_effort,
             )
 
             tokenized_request = self.tokenizer.encode_chat_completion(chat_request)
@@ -1215,6 +1224,8 @@ def _maybe_adapt_message(message: dict[str, Any]) -> None:
                     else:
                         raise ValueError(f"Unsupported return_tensors type: {return_tensors}")
                     out.data["pixel_values"] = pixel_values
+                if images:
+                    out.data["image_sizes"] = self._get_image_sizes_for_tensor(images, return_tensors)
                 if audios:
                     if return_tensors is not None:
                         raise NotImplementedError(
@@ -1233,6 +1244,31 @@ def _maybe_adapt_message(message: dict[str, Any]) -> None:
             )
             return outputs
 
+    def _get_image_sizes_for_tensor(
+        self, images: list[np.ndarray], return_tensors: str | TensorType | None
+    ) -> list[list[int]] | np.ndarray | torch.Tensor:
+        """
+        Convert image sizes to the appropriate format based on return_tensors.
+
+        Args:
+            images: List of image arrays
+            return_tensors: The tensor type to return
+
+        Returns:
+            Image sizes in the appropriate format
+        """
+        image_sizes = []
+        for image in images:
+            height, width = get_image_size(image)
+            image_sizes.append([height, width])
+
+        if return_tensors == "pt":
+            return torch.tensor(image_sizes, dtype=torch.long)
+        elif return_tensors == "np":
+            return np.array(image_sizes, dtype=np.int64)
+        else:
+            return image_sizes
+
     def build_inputs_with_special_tokens(self, token_ids_0: list[int], token_ids_1: None = None) -> list[int]:
         """
         Build model inputs from a sequence by adding special tokens.

filename: tests/test_tokenization_mistral_common.py
patch: @@ -891,14 +891,15 @@ def test_apply_chat_template_basic(self):
         # Test 1:
         # with tokenize
         self.assertEqual(
-            self.tokenizer.apply_chat_template(conversation, tokenize=False),
+            self.tokenizer.apply_chat_template(conversation, tokenize=False, reasoning_effort=None),
             expected_tokenized.text,
         )
 
         # Test 2:
         # without tokenize
         self.assertEqual(
-            self.tokenizer.apply_chat_template(conversation, tokenize=True).input_ids, expected_tokenized.tokens
+            self.tokenizer.apply_chat_template(conversation, tokenize=True, reasoning_effort=None).input_ids,
+            expected_tokenized.tokens,
         )
 
         with self.assertRaises(
@@ -1067,10 +1068,41 @@ def test_apply_chat_template_with_image(self):
         for o, e in zip(output_dict["pixel_values"], expected_tokenized.images):
             self.assertTrue(np.allclose(o, e))
 
-        output_dict = self.tokenizer.apply_chat_template(conversation, tokenize=True, return_tensors="pt")
-        self.assertEqual(output_dict["input_ids"].tolist()[0], expected_tokenized.tokens)
+        # Test with return_tensors="pt"
+        output_dict_pt = self.tokenizer.apply_chat_template(conversation, tokenize=True, return_tensors="pt")
+        self.assertEqual(output_dict_pt["input_ids"].tolist()[0], expected_tokenized.tokens)
         expected_images_pt_tensor = torch.from_numpy(np.stack(expected_tokenized.images))
-        self.assertTrue(torch.allclose(output_dict["pixel_values"], expected_images_pt_tensor))
+        self.assertTrue(torch.allclose(output_dict_pt["pixel_values"], expected_images_pt_tensor))
+        self.assertIn("image_sizes", output_dict_pt)
+        self.assertIsInstance(output_dict_pt["image_sizes"], torch.Tensor)
+        self.assertEqual(output_dict_pt["image_sizes"].shape, torch.Size([1, 2]))
+
+        # Test with return_tensors="np"
+        output_dict_np = self.tokenizer.apply_chat_template(conversation, tokenize=True, return_tensors="np")
+        self.assertEqual(output_dict_np["input_ids"].tolist()[0], expected_tokenized.tokens)
+        expected_images_np_array = np.stack(expected_tokenized.images)
+        self.assertTrue(np.allclose(output_dict_np["pixel_values"], expected_images_np_array))
+        self.assertIn("image_sizes", output_dict_np)
+        self.assertIsInstance(output_dict_np["image_sizes"], np.ndarray)
+        self.assertEqual(output_dict_np["image_sizes"].shape, (1, 2))
+
+        # Test with return_tensors=None (default - Python lists)
+        output_dict_default = self.tokenizer.apply_chat_template(conversation, tokenize=True)
+        self.assertEqual(output_dict_default["input_ids"], expected_tokenized.tokens)
+        self.assertEqual(len(output_dict_default["pixel_values"]), len(expected_tokenized.images))
+        for o, e in zip(output_dict_default["pixel_values"], expected_tokenized.images):
+            self.assertTrue(np.allclose(o, e))
+        self.assertIn("image_sizes", output_dict_default)
+        self.assertIsInstance(output_dict_default["image_sizes"], list)
+        self.assertEqual(len(output_dict_default["image_sizes"]), 1)
+
+        # Check pt, np, list are equals
+        self.assertEqual(output_dict_pt["image_sizes"].tolist(), output_dict_np["image_sizes"].tolist())
+        self.assertEqual(output_dict_pt["image_sizes"].tolist(), output_dict_default["image_sizes"])
+
+        actual_height, actual_width = output_dict_pt["image_sizes"].tolist()[0]
+        self.assertEqual(actual_height, 308, f"Expected height 308, got {actual_height}")
+        self.assertEqual(actual_width, 224, f"Expected width 224, got {actual_width}")
 
     def test_apply_chat_template_with_audio(self):
         ref_conversation = conversation = [