Repository: huggingface/transformers
File: src/transformers/models/voxtral/processing_voxtral.py (Python)

Code before:
        unsupported_keys = {"sampling_rate", "load_audio_from_video"} & kwargs.keys()
        if unsupported_keys:
            for key in unsupported_keys:
                kwargs.pop(key)
            logger.warning(
                f"{', '.join(sorted(unsupported_keys))} {'is' if len(unsupported_keys) == 1 else 'are'} not supported for VoxtralProcessor's apply_chat_template and will be ignored."
            )

        output_kwargs = self._merge_kwargs(
            VoxtralProcessorKwargs,
            **kwargs,
        )
        text_kwargs = output_kwargs["text_kwargs"]
        audio_kwargs = output_kwargs["audio_kwargs"]
        return_tensors = text_kwargs.get("return_tensors", None)

        if return_tensors != "pt":
            raise ValueError(f"{self.__class__.__name__} only supports `return_tensors='pt'`.")

        tokenizer_kwargs = output_kwargs["text_kwargs"]
        tokenizer_kwargs["return_tensors"] = None  # let's not return tensors here
        encoded_instruct_inputs = self.tokenizer.apply_chat_template(conversations, **tokenizer_kwargs)

        if text_kwargs.get("tokenize", False):
            if text_kwargs.get("return_dict", False):
                audio = encoded_instruct_inputs.pop("audio", None)
                data = dict(encoded_instruct_inputs)
                if audio is not None:
                    max_source_positions = audio_kwargs.pop("max_source_positions")
                    data["input_features"] = self._retrieve_input_features(audio, max_source_positions, **audio_kwargs)

                return BatchFeature(data=data, tensor_type=return_tensors)

        if not is_batched:
            return encoded_instruct_inputs[0]

        return encoded_instruct_inputs

    @auto_docstring(
        custom_intro=r"""
    Method to prepare text to be fed as input to the model. This method forwards the `text`
    arguments to MistralCommonBackend's [`~MistralCommonBackend.__call__`] to encode
    the text. Please refer to the docstring of the above methods for more information.
    This method does not support audio. To prepare the audio, please use:
    1. `apply_chat_template` [`~VoxtralProcessor.apply_chat_template`] method.
    2. `apply_transcription_request` [`~VoxtralProcessor.apply_transcription_request`] method.
    """
    )
    def __call__(
        self,
        text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None,

Review comment (suggestion):
use `text_kwargs` instead of creating new `tokenizer_kwargs`?

```python
text_kwargs = output_kwargs["text_kwargs"]
...

text_kwargs["return_tensors"] = None  # let's not return tensors here
encoded_instruct_inputs = self.tokenizer.apply_chat_template(conversations, **text_kwargs)
```

Code after:
        # - `sampling_rate` is already fixed in `VoxtralProcessorKwargs._defaults` and audio loading is
        #   delegated to `mistral_common`'s tokenizer which handles it internally.
        # - `load_audio_from_video` is irrelevant as Voxtral is a speech-only model with no video support.
        # We strip them here to avoid passing unrecognized kwargs to `_merge_kwargs`.
        unsupported_keys = {"sampling_rate", "load_audio_from_video"} & kwargs.keys()
        if unsupported_keys:
            for key in unsupported_keys:
                kwargs.pop(key)
            logger.warning(
                f"{', '.join(sorted(unsupported_keys))} {'is' if len(unsupported_keys) == 1 else 'are'} not supported for VoxtralProcessor's apply_chat_template and will be ignored."
            )

        output_kwargs = self._merge_kwargs(
            VoxtralProcessorKwargs,
            **kwargs,
        )
        text_kwargs = output_kwargs["text_kwargs"]
        audio_kwargs = output_kwargs["audio_kwargs"]
        return_tensors = text_kwargs.get("return_tensors", None)

        if return_tensors != "pt":
            raise ValueError(f"{self.__class__.__name__} only supports `return_tensors='pt'`.")

        tokenizer_kwargs = output_kwargs["text_kwargs"]
        tokenizer_kwargs["return_tensors"] = None  # let's not return tensors here
        encoded_instruct_inputs = self.tokenizer.apply_chat_template(conversations, **tokenizer_kwargs)

        if text_kwargs.get("tokenize", False):
            if text_kwargs.get("return_dict", False):
                audio = encoded_instruct_inputs.pop("audio", None)
                data = dict(encoded_instruct_inputs)
                if audio is not None:
                    max_source_positions = audio_kwargs.pop("max_source_positions")
                    data["input_features"] = self._retrieve_input_features(audio, max_source_positions, **audio_kwargs)

                return BatchFeature(data=data, tensor_type=return_tensors)

        if not is_batched:
            return encoded_instruct_inputs[0]

        return encoded_instruct_inputs

    @auto_docstring(
        custom_intro=r"""
    Method to prepare text to be fed as input to the model. This method forwards the `text`
    arguments to MistralCommonBackend's [`~MistralCommonBackend.__call__`] to encode
    the text. Please refer to the docstring of the above methods for more information.
    This method does not support audio. To prepare the audio, please use:
    1. `apply_chat_template` [`~VoxtralProcessor.apply_chat_template`] method.
    2. `apply_transcription_request` [`~VoxtralProcessor.apply_transcription_request`] method.
    """