ronantakizawa/codereview-bench huggingface dataset:
Repository: vllm-project/vllm
File: tests/lora/test_moe_lora_align_sum.py (Python)

Code before:
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import random

import pytest
import torch

from vllm import _custom_ops as ops
from vllm.platforms import current_platform

DEVICE_TYPE = current_platform.device_type


def round_up(x, base):
    return ((x + base - 1) // base) * base


def CEILDIV(x, y):
    return (x + y - 1) // y


def sample_data(num_experts, max_loras, num_tokens, topk_num):
    topk_ids = torch.zeros((num_tokens, topk_num), dtype=torch.int32)
    token_lora_mapping = torch.zeros((num_tokens,), dtype=torch.int32)

    for i in range(num_tokens):
        pool = list(range(num_experts))
        random.shuffle(pool)
        for j in range(topk_num):
            topk_ids[i, j] = pool[j]
        token_lora_mapping[i] = random.randint(0, max_loras - 1)

    return topk_ids.to(DEVICE_TYPE), token_lora_mapping.to(DEVICE_TYPE)


Review comment (suggestion):
I am thinking we can revert such UTs now that we didn't enable these in xpu CI. We will have a unified change for device in the future.

Code after:
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import random

import pytest
import torch

from vllm import _custom_ops as ops


def round_up(x, base):
    return ((x + base - 1) // base) * base


def CEILDIV(x, y):
    return (x + y - 1) // y


def sample_data(num_experts, max_loras, num_tokens, topk_num):
    topk_ids = torch.zeros((num_tokens, topk_num), dtype=torch.int32)
    token_lora_mapping = torch.zeros((num_tokens,), dtype=torch.int32)

    for i in range(num_tokens):
        pool = list(range(num_experts))
        random.shuffle(pool)
        for j in range(topk_num):
            topk_ids[i, j] = pool[j]
        token_lora_mapping[i] = random.randint(0, max_loras - 1)

    return topk_ids.to("cuda"), token_lora_mapping.to("cuda")


@pytest.mark.parametrize("num_tokens", [100, 200, 1024, 4096])  # 81920
@pytest.mark.parametrize("topk_num", [6])
Repository: vllm-project/vllm
File: vllm/plugins/io_processors/__init__.py (Python)

Code before:
            logger.warning("Failed to load plugin %s.", name, exc_info=True)

    num_available_plugins = len(loadable_plugins.keys())
    if num_available_plugins == 0:
        raise ValueError(
            f"No IOProcessor plugins installed but one is required ({model_plugin})."
        )

    if model_plugin not in loadable_plugins:
        raise ValueError(
            f"The model requires the '{model_plugin}' IO Processor plugin "
            "but it is not installed. "
            f"Available plugins: {list(loadable_plugins.keys())}"
        )

    activated_plugin_cls = loadable_plugins[model_plugin]

    # for backward compatibility, the plugin does not have a renderer argument
    if "renderer" not in inspect.signature(activated_plugin_cls.__init__).parameters:
        logger.warning(
            "The renderer argument will be required in v0.18, "
            "please update your IOProcessor plugin: %s",
            activated_plugin_cls,
        )
        return resolve_obj_by_qualname(activated_plugin_cls)(vllm_config)
    return resolve_obj_by_qualname(activated_plugin_cls)(vllm_config, renderer)


Review comment (suggestion):
```suggestion
    activated_plugin_typ = resolve_obj_by_qualname(activated_plugin_cls)

    # for backward compatibility, the plugin does not have a renderer argument
    if "renderer" not in inspect.signature(activated_plugin_typ.__init__).parameters:
        logger.warning(
            "The renderer argument will be required in v0.18, "
            "please update your IOProcessor plugin: %s",
            activated_plugin_cls,
        )
        return activated_plugin_typ(vllm_config)
    return activated_plugin_typ(vllm_config, renderer)
```

Code after:
            logger.warning("Failed to load plugin %s.", name, exc_info=True)

    num_available_plugins = len(loadable_plugins.keys())
    if num_available_plugins == 0:
        raise ValueError(
            f"No IOProcessor plugins installed but one is required ({model_plugin})."
        )

    if model_plugin not in loadable_plugins:
        raise ValueError(
            f"The model requires the '{model_plugin}' IO Processor plugin "
            "but it is not installed. "
            f"Available plugins: {list(loadable_plugins.keys())}"
        )

    activated_plugin_cls = loadable_plugins[model_plugin]

    activated_plugin_typ = resolve_obj_by_qualname(activated_plugin_cls)

    # for backward compatibility, the plugin does not have a renderer argument
    if "renderer" not in inspect.signature(activated_plugin_typ.__init__).parameters:
        logger.warning(
            "The renderer argument will be required in v0.18, "
            "please update your IOProcessor plugin: %s",
            activated_plugin_cls,
        )
        return activated_plugin_typ(vllm_config)
    return activated_plugin_typ(vllm_config, renderer)
Repository: vllm-project/vllm
File: vllm/v1/worker/gpu/sample/states.py (Python)

Code before:
    def apply_min_p(
        self,
        logits: torch.Tensor,
        idx_mapping: torch.Tensor,
        idx_mapping_np: np.ndarray,
    ) -> None:
        if np.all(self.min_p.np[idx_mapping_np] == 0.0):
            # No request uses min_p. Skip the kernel launch.
            return
        apply_min_p(logits, idx_mapping, self.min_p.gpu)

    def apply_top_k_top_p(
        self,
        logits: torch.Tensor,
        idx_mapping: torch.Tensor,
        idx_mapping_np: np.ndarray,
    ) -> torch.Tensor:
        do_top_k = np.any(self.top_k.np[idx_mapping_np] != self.vocab_size)
        top_k = self.top_k.gpu[idx_mapping] if do_top_k else None

        do_top_p = np.any(self.top_p.np[idx_mapping_np] != 1.0)
        top_p = self.top_p.gpu[idx_mapping] if do_top_p else None

        if do_top_k or do_top_p:
            logits = apply_top_k_top_p(logits, top_k, top_p)
        return logits

    def max_num_logprobs(self, idx_mapping_np: np.ndarray) -> int:
        return int(np.max(self.num_logprobs[idx_mapping_np]))


Review comment (suggestion):
suggestion
```suggestion
        do_top_k = np.any(self.top_k.np[idx_mapping_np] != self.vocab_size)
        do_top_p = np.any(self.top_p.np[idx_mapping_np] != 1.0)
        if not do_top_k and not do_top_p:
            return logits

        top_k = self.top_k.gpu[idx_mapping] if do_top_k else None
        top_p = self.top_p.gpu[idx_mapping] if do_top_p else None
        return apply_top_k_top_p(logits, top_k, top_p)
```

Code after:
    def apply_min_p(
        self,
        logits: torch.Tensor,
        idx_mapping: torch.Tensor,
        idx_mapping_np: np.ndarray,
    ) -> None:
        if np.all(self.min_p.np[idx_mapping_np] == 0.0):
            # No request uses min_p. Skip the kernel launch.
            return
        apply_min_p(logits, idx_mapping, self.min_p.gpu)

    def apply_top_k_top_p(
        self,
        logits: torch.Tensor,
        idx_mapping: torch.Tensor,
        idx_mapping_np: np.ndarray,
    ) -> torch.Tensor:
        do_top_k = np.any(self.top_k.np[idx_mapping_np] != self.vocab_size)
        do_top_p = np.any(self.top_p.np[idx_mapping_np] != 1.0)
        if not (do_top_k or do_top_p):
            return logits

        top_k = self.top_k.gpu[idx_mapping] if do_top_k else None
        top_p = self.top_p.gpu[idx_mapping] if do_top_p else None
        return apply_top_k_top_p(logits, top_k, top_p)

    def max_num_logprobs(self, idx_mapping_np: np.ndarray) -> int:
        return int(np.max(self.num_logprobs[idx_mapping_np]))
Repository: vllm-project/vllm
File: vllm/model_executor/models/interfaces.py (Python)

Code before:

        return instance

    @staticmethod
    def _find_quant_config(*args, **kwargs) -> QuantizationConfig | None:
        """Find quant config passed through model constructor args"""
        from vllm.config import VllmConfig  # avoid circular import

        args_values = list(args) + list(kwargs.values())
        for arg in args_values:
            if isinstance(arg, VllmConfig):
                return arg.quant_config

            if isinstance(arg, QuantizationConfig):
                return arg

        return None


@runtime_checkable
class SupportsRealtime(Protocol):
    """The interface required for all models that support transcription."""

    supports_realtime: ClassVar[Literal[True]] = True

    realtime_max_tokens: ClassVar[int] = 1

    @classmethod
    async def buffer_realtime_audio(
        cls,
        audio_stream: AsyncGenerator[np.ndarray, None],
        input_stream: asyncio.Queue[list[int]],
        model_config: ModelConfig,
    ) -> AsyncGenerator[PromptType, None]: ...


@overload
def supports_realtime(
    model: type[object],
) -> TypeIs[type[SupportsRealtime]]: ...


@overload
def supports_realtime(model: object) -> TypeIs[SupportsRealtime]: ...


def supports_realtime(
    model: type[object] | object,
) -> TypeIs[type[SupportsRealtime]] | TypeIs[SupportsRealtime]:
    return getattr(model, "supports_realtime", False)


Review comment (suggestion):
Maybe add a docstring here? If the model isn't trained on streaming input I guess `realtime_max_tokens` has to be 1.

Code after:

        return instance

    @staticmethod
    def _find_quant_config(*args, **kwargs) -> QuantizationConfig | None:
        """Find quant config passed through model constructor args"""
        from vllm.config import VllmConfig  # avoid circular import

        args_values = list(args) + list(kwargs.values())
        for arg in args_values:
            if isinstance(arg, VllmConfig):
                return arg.quant_config

            if isinstance(arg, QuantizationConfig):
                return arg

        return None


@runtime_checkable
class SupportsRealtime(Protocol):
    """The interface required for all models that support transcription."""

    supports_realtime: ClassVar[Literal[True]] = True

    realtime_max_tokens: ClassVar[int] = 1
    """Maximum tokens to generate per streaming audio segment.
    Override in subclasses based on the model's expected output length."""

    @classmethod
    async def buffer_realtime_audio(
        cls,
        audio_stream: AsyncGenerator[np.ndarray, None],
        input_stream: asyncio.Queue[list[int]],
        model_config: ModelConfig,
    ) -> AsyncGenerator[PromptType, None]: ...


@overload
def supports_realtime(
    model: type[object],
) -> TypeIs[type[SupportsRealtime]]: ...


@overload
def supports_realtime(model: object) -> TypeIs[SupportsRealtime]: ...


def supports_realtime(
    model: type[object] | object,
) -> TypeIs[type[SupportsRealtime]] | TypeIs[SupportsRealtime]:
Repository: vllm-project/vllm
File: vllm/model_executor/models/qwen3_asr_realtime.py (Python)

Code before:
        audio_placeholder = cls.get_placeholder_str("audio", 0)
        prompt_template = (
            f"
user\n{audio_placeholder}
\n"
            f"
assistant\n"
        )

        is_first_yield = True

        async for audio_chunk in audio_stream:
            buffer.write_audio(audio_chunk)

            while (segment := buffer.read_audio()) is not None:
                if is_first_yield:
                    prompt_token_ids = tokenizer.encode(prompt_template)
                    is_first_yield = False
                else:
                    prompt_token_ids = tokenizer.encode(prompt_template)

                yield TokensPrompt(
                    prompt_token_ids=prompt_token_ids,
                    multi_modal_data={"audio": segment},
                )

        remaining = buffer.flush()
        if remaining is not None and len(remaining) > 0:
            prompt_token_ids = tokenizer.encode(prompt_template)
            yield TokensPrompt(
                prompt_token_ids=prompt_token_ids,
                multi_modal_data={"audio": remaining},
            )

    @classmethod
    def get_speech_to_text_config(
        cls, model_config: ModelConfig, task_type: str
    ) -> SpeechToTextConfig:
        processor = cached_processor_from_config(model_config)
        feature_extractor = processor.feature_extractor
        return SpeechToTextConfig(
            max_audio_clip_s=None,
            sample_rate=feature_extractor.sampling_rate,
            min_energy_split_window_size=None,
        )


Review comment (suggestion):
We should only call `prompt_token_ids = tokenizer.encode(prompt_template)` once

Code after:

        audio_placeholder = cls.get_placeholder_str("audio", 0)
        prompt_template = (
            f"
user\n{audio_placeholder}
\n
assistant\n"
        )

        prompt_token_ids = tokenizer.encode(prompt_template)

        async for audio_chunk in audio_stream:
            buffer.write_audio(audio_chunk)

            while (segment := buffer.read_audio()) is not None:
                yield TokensPrompt(
                    prompt_token_ids=prompt_token_ids,
                    multi_modal_data={"audio": segment},
                )

        remaining = buffer.flush()
        if remaining is not None and len(remaining) > 0:
            yield TokensPrompt(
                prompt_token_ids=prompt_token_ids,
                multi_modal_data={"audio": remaining},
            )

    @classmethod
    def get_speech_to_text_config(
        cls, model_config: ModelConfig, task_type: str
    ) -> SpeechToTextConfig:
        processor = cached_processor_from_config(model_config)
        feature_extractor = processor.feature_extractor
        return SpeechToTextConfig(
            max_audio_clip_s=None,
            sample_rate=feature_extractor.sampling_rate,
            min_energy_split_window_size=None,
        )
Repository: vllm-project/vllm
File: vllm/model_executor/layers/quantization/base_config.py (Python)

Code before:

    def maybe_update_config(self, model_name: str):  # noqa: B027
        """
        Interface to update values after config initialization.
        """
        pass

    def is_mxfp4_quant(self, prefix: str, layer: torch.nn.Module) -> bool:
        """
        Determine if mxfp4 quantization will be used for this config.

        This allows hidden_size rounding to happen before moe_config creation
        without needing to instantiate quant_method first.

        Args:
            prefix: The layer prefix/name in the model
            layer: The layer module

        Returns:
            True if this config uses MXFP4 quantization, False otherwise
        """
        return False

    @classmethod
    def uses_meta_device_weights(cls) -> bool:
        """Whether this method creates weights on meta device for online quant."""
        return False


Review comment (suggestion):
nit: specify for the reader the reason why online quant methods would want to do this - to save peak memory by post-processing weights layerwise instead of after all weights are loaded

Code after:
        :param hf_to_vllm_mapper: maps from hf model structure (the assumed
            structure of the qconfig) to vllm model structure
        """
        # TODO (@kylesayrs): add implementations for all subclasses
        pass

    def maybe_update_config(self, model_name: str):  # noqa: B027
        """
        Interface to update values after config initialization.
        """
        pass

    def is_mxfp4_quant(self, prefix: str, layer: torch.nn.Module) -> bool:
        """
        Determine if mxfp4 quantization will be used for this config.

        This allows hidden_size rounding to happen before moe_config creation
        without needing to instantiate quant_method first.

        Args:
            prefix: The layer prefix/name in the model
            layer: The layer module

        Returns:
            True if this config uses MXFP4 quantization, False otherwise
        """
        return False
Repository: vllm-project/vllm
File: vllm/utils/mistral.py (Python)

Code before:
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Provides lazy import of the vllm.tokenizers.mistral module."""

from typing import TYPE_CHECKING, Any, TypeGuard

from vllm.utils.import_utils import LazyLoader

if TYPE_CHECKING:
    # if type checking, eagerly import the module
    import vllm.tokenizers.mistral as mt
else:
    mt = LazyLoader("mt", globals(), "vllm.tokenizers.mistral")


def is_mistral_tokenizer(obj: Any) -> TypeGuard[mt.MistralTokenizer]:
    """Return true if the object is a MistralTokenizer instance."""
    cls = type(obj)
    # Check for special class attribute, this avoids importing the class to
    # do an isinstance() check.
    return bool(getattr(cls, "IS_MISTRAL_TOKENIZER", False))


Review comment (suggestion):
```suggestion
def is_mistral_tokenizer(obj: TokenizerLike) -> TypeGuard[mt.MistralTokenizer]:
```

Be a bit stricter to avoid misuse


Code after:
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Provides lazy import of the vllm.tokenizers.mistral module."""

from __future__ import annotations

from typing import TYPE_CHECKING, TypeGuard

from vllm.tokenizers import TokenizerLike
from vllm.utils.import_utils import LazyLoader

if TYPE_CHECKING:
    # if type checking, eagerly import the module
    import vllm.tokenizers.mistral as mt
else:
    mt = LazyLoader("mt", globals(), "vllm.tokenizers.mistral")


def is_mistral_tokenizer(obj: TokenizerLike | None) -> TypeGuard[mt.MistralTokenizer]:
    """Return true if the tokenizer is a MistralTokenizer instance."""
    cls = type(obj)
    # Check for special class attribute, this avoids importing the class to
    # do an isinstance() check.  If the attribute is True, do an isinstance
    # check to be sure we have the correct type.
    return bool(
        getattr(cls, "IS_MISTRAL_TOKENIZER", False)
        and isinstance(obj, mt.MistralTokenizer)
    )
Repository: vllm-project/vllm
File: vllm/v1/worker/gpu/sample/penalties.py (Python)

Code before:
    if (block_idx + 1) * BLOCK_SIZE >= prompt_len:
        mask = block < prefill_len
        mask &= block >= prompt_len
        output_tokens = tl.load(
            all_token_ids_ptr + req_state_idx * all_token_ids_stride + block, mask=mask
        )
        tl.atomic_add(
            output_bin_counts_ptr
            + req_state_idx * output_bin_counts_stride
            + output_tokens,
            1,
            mask=mask,
        )


def bincount(
    idx_mapping: torch.Tensor,
    all_token_ids: torch.Tensor,
    prompt_len: torch.Tensor,
    prefill_len: torch.Tensor,
    prompt_bin_mask: torch.Tensor,
    output_bin_counts: torch.Tensor,
    max_model_len: int,
) -> None:
    prompt_bin_mask[idx_mapping].zero_()
    output_bin_counts[idx_mapping].zero_()
    num_reqs = idx_mapping.shape[0]
    BLOCK_SIZE = 1024
    num_blocks = triton.cdiv(max_model_len, BLOCK_SIZE)
    _bincount_kernel[(num_reqs, num_blocks)](
        idx_mapping,
        all_token_ids,
        all_token_ids.stride(0),
        prompt_len,
        prefill_len,
        prompt_bin_mask,
        prompt_bin_mask.stride(0),
        output_bin_counts,
        output_bin_counts.stride(0),
        BLOCK_SIZE=BLOCK_SIZE,
    )


def use_penalty(sampling_params: SamplingParams) -> bool:
    return (
        sampling_params.repetition_penalty != 1.0
        or sampling_params.frequency_penalty != 0.0
        or sampling_params.presence_penalty != 0.0
    )


Review comment (suggestion):
Indexing a tensor like this will create a new tensor unless it's an assignment (so this won't actually modify the original tensors)

```suggestion
    prompt_bin_mask[idx_mapping] = 0
    output_bin_counts[idx_mapping] = 0
```

Code after:
        )
        tl.atomic_add(
            output_bin_counts_ptr
            + req_state_idx * output_bin_counts_stride
            + output_tokens,
            1,
            mask=mask,
        )


def bincount(
    idx_mapping: torch.Tensor,
    all_token_ids: torch.Tensor,
    prompt_len: torch.Tensor,
    prefill_len: torch.Tensor,
    prompt_bin_mask: torch.Tensor,
    output_bin_counts: torch.Tensor,
    max_prefill_len: int,
) -> None:
    prompt_bin_mask[idx_mapping] = 0
    output_bin_counts[idx_mapping] = 0
    num_reqs = idx_mapping.shape[0]
    BLOCK_SIZE = 1024
    num_blocks = triton.cdiv(max_prefill_len, BLOCK_SIZE)
    _bincount_kernel[(num_reqs, num_blocks)](
        idx_mapping,
        all_token_ids,
        all_token_ids.stride(0),
        prompt_len,
        prefill_len,
        prompt_bin_mask,
        prompt_bin_mask.stride(0),
        output_bin_counts,
        output_bin_counts.stride(0),
        BLOCK_SIZE=BLOCK_SIZE,
    )


def use_penalty(sampling_params: SamplingParams) -> bool:
    return (
        sampling_params.repetition_penalty != 1.0
        or sampling_params.frequency_penalty != 0.0
        or sampling_params.presence_penalty != 0.0
    )
Repository: vllm-project/vllm
File: vllm/model_executor/layers/quantization/bitsandbytes.py (Python)

Code before:
from packaging import version

from vllm.model_executor.layers.fused_moe.config import (
    FusedMoEConfig,
    FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.layer import (
    FusedMoE,
    FusedMoEMethodBase,
)
from vllm.model_executor.layers.linear import (
    LinearBase,
    LinearMethodBase,
    UnquantizedLinearMethod,
    set_weight_attrs,
)
from vllm.model_executor.layers.quantization import (
    QuantizationConfig,
    QuantizationMethods,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op


def _check_bitsandbytes_version():
    min_version = "0.49.2" if current_platform.is_rocm() else "0.46.1"
    try:
        import bitsandbytes

        if version.parse(bitsandbytes.__version__) < version.parse(min_version):
            raise ImportError(
                "bitsandbytes version is wrong. Please "
                f"install bitsandbytes>={min_version}."
            )
    except ImportError as err:
        raise ImportError(
            f"Please install bitsandbytes>={min_version} via "
            f"`pip install bitsandbytes>={min_version}` to use "
            "bitsandbytes quantizer."
        ) from err


class BitsAndBytesConfig(QuantizationConfig):
    """Config class for BitsAndBytes Quantization.

    Reference: https://arxiv.org/abs/2305.14314
    """

    def __init__(
        self,
        load_in_8bit: bool = False,

Review comment (suggestion):
For Transformers v5 compatibility

```suggestion
    min_version = "0.49.2" if current_platform.is_rocm() else "0.48.1"
```

Code after:
from packaging import version

from vllm.model_executor.layers.fused_moe.config import (
    FusedMoEConfig,
    FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.layer import (
    FusedMoE,
    FusedMoEMethodBase,
)
from vllm.model_executor.layers.linear import (
    LinearBase,
    LinearMethodBase,
    UnquantizedLinearMethod,
    set_weight_attrs,
)
from vllm.model_executor.layers.quantization import (
    QuantizationConfig,
    QuantizationMethods,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op


def _check_bitsandbytes_version():
    min_version = "0.49.2" if current_platform.is_rocm() else "0.48.1"
    try:
        import bitsandbytes

        if version.parse(bitsandbytes.__version__) < version.parse(min_version):
            raise ImportError(
                "bitsandbytes version is wrong. Please "
                f"install bitsandbytes>={min_version}."
            )
    except ImportError as err:
        raise ImportError(
            f"Please install bitsandbytes>={min_version} via "
            f"`pip install bitsandbytes>={min_version}` to use "
            "bitsandbytes quantizer."
        ) from err


class BitsAndBytesConfig(QuantizationConfig):
    """Config class for BitsAndBytes Quantization.

    Reference: https://arxiv.org/abs/2305.14314
    """

    def __init__(
        self,
        load_in_8bit: bool = False,