Repository: vllm-project/vllm
File: vllm/model_executor/models/nano_nemotron_vl.py (Python)

Code before:

    def apply(
        self,
        processor_inputs: ProcessorInputs,
        timing_ctx: TimingContext | None = None,
    ) -> MultiModalInputs:
        if (hf_processor_mm_kwargs := processor_inputs.hf_processor_mm_kwargs) is None:
            hf_processor_mm_kwargs = {}

        use_audio_in_video = bool(
            hf_processor_mm_kwargs.get("use_audio_in_video", False)
        )

        hf_processor_mm_kwargs = {
            k: v for k, v in hf_processor_mm_kwargs.items() if k != "use_audio_in_video"
        }

        processor_inputs.hf_processor_mm_kwargs = hf_processor_mm_kwargs

        if not (
            use_audio_in_video
            and "video" in processor_inputs.mm_data_items
            and "audio" not in processor_inputs.mm_data_items
        ):
            return super().apply(
                processor_inputs,
                timing_ctx,
            )

        mm_items, audio_items = self._extract_audio_from_videos(
            processor_inputs.mm_data_items
        )
        processor_inputs.mm_data_items = mm_items

        prompt = processor_inputs.prompt
        if not isinstance(prompt, str):
            tokenizer = self.info.get_tokenizer()
            prompt = tokenizer.decode(prompt, skip_special_tokens=False)

        for _ in audio_items:
            prompt = prompt.replace("<video>", "<video>" + AUDIO_CONTEXT, 1)

        processor_inputs.prompt = prompt

        if processor_inputs.tokenization_kwargs is None:
            processor_inputs.tokenization_kwargs = {}

        # Bypass the cached path: the HF processor must receive the
        # prompt (with injected <so_embedding>) and the audio data
        # together so it can perform audio-token replacement natively.
        (

Review comment (suggestion):
We will not support string inputs in a future release, so I suggest apply the prompt modification in terms of tokens here if possible

Code after:
        use_audio_in_video = bool(
            hf_processor_mm_kwargs.get("use_audio_in_video", False)
        )

        hf_processor_mm_kwargs = {
            k: v for k, v in hf_processor_mm_kwargs.items() if k != "use_audio_in_video"
        }

        processor_inputs.hf_processor_mm_kwargs = hf_processor_mm_kwargs

        if not (
            use_audio_in_video
            and "video" in processor_inputs.mm_data_items
            and "audio" not in processor_inputs.mm_data_items
        ):
            return super().apply(
                processor_inputs,
                timing_ctx,
            )

        mm_items, audio_items = self._extract_audio_from_videos(
            processor_inputs.mm_data_items
        )
        processor_inputs.mm_data_items = mm_items

        prompt = processor_inputs.prompt
        tokenizer = self.info.get_tokenizer()
        if not isinstance(prompt, str):
            prompt = tokenizer.decode(prompt, skip_special_tokens=False)

        for _ in audio_items:
            prompt = prompt.replace("<video>", "<video>" + AUDIO_CONTEXT, 1)

        processor_inputs.prompt = tokenizer.encode(prompt, add_special_tokens=False)

        if processor_inputs.tokenization_kwargs is None:
            processor_inputs.tokenization_kwargs = {}

        # Bypass the cached path: the HF processor must receive the
        # prompt (with injected <so_embedding>) and the audio data
        # together so it can perform audio-token replacement natively.
        (
            prompt_ids,
            mm_info,
            is_update_applied,
        ) = self._apply_hf_processor(
            processor_inputs,
            timing_ctx=timing_ctx,
        )

        prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(