internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: vllm-project/vllm

Issue:
the syntax in serving_completion.py is not compatible in python3.8
in #2529 @simon-mo introduce some python typing syntax, which is not compatible in python3.8

like  TypeTokenIDs = list[int] in https://github.com/vllm-project/vllm/blob/93b38bea5dd03e1b140ca997dfaadef86f8f1855/vllm/entrypoints/openai/serving_completion.py#L22C1-L22C25

which should be TypeTokenIDs=List[int] in python3.8.

could you please fix it?


Discussion:

Resolving patch:
diff --git a/vllm/entrypoints/openai/serving_completion.py b/vllm/entrypoints/openai/serving_completion.py
--- a/vllm/entrypoints/openai/serving_completion.py
+++ b/vllm/entrypoints/openai/serving_completion.py
@@ -1,7 +1,7 @@
 import asyncio
 import time
 from fastapi import Request
-from typing import AsyncGenerator, AsyncIterator, Callable, List, Optional
+from typing import AsyncGenerator, AsyncIterator, Callable, List, Optional, Dict, Tuple
 from vllm.logger import init_logger
 from vllm.utils import random_uuid
 from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -19,8 +19,8 @@
 
 logger = init_logger(__name__)
 
-TypeTokenIDs = list[int]
-TypeTopLogProbs = List[Optional[dict[int, float]]]
+TypeTokenIDs = List[int]
+TypeTopLogProbs = List[Optional[Dict[int, float]]]
 TypeCreateLogProbsFn = Callable[
     [TypeTokenIDs, TypeTopLogProbs, Optional[int], int], LogProbs]
 
@@ -29,7 +29,7 @@ async def completion_stream_generator(
     request: CompletionRequest,
     raw_request: Request,
     on_abort,
-    result_generator: AsyncIterator[tuple[int, RequestOutput]],
+    result_generator: AsyncIterator[Tuple[int, RequestOutput]],
     create_logprobs_fn: TypeCreateLogProbsFn,
     request_id: str,
     created_time: int,
@@ -126,7 +126,7 @@ async def completion_stream_generator(
     yield "data: [DONE]\n\n"
 
 
-def parse_prompt_format(prompt) -> tuple[bool, list]:
+def parse_prompt_format(prompt) -> Tuple[bool, list]:
     # get the prompt, openai supports the following
     # "a string, array of strings, array of tokens, or array of token arrays."
     prompt_is_tokens = False
@@ -151,7 +151,7 @@ def parse_prompt_format(prompt) -> tuple[bool, list]:
 
 
 def request_output_to_completion_response(
-    final_res_batch: list[RequestOutput],
+    final_res_batch: List[RequestOutput],
     request: CompletionRequest,
     create_logprobs_fn: TypeCreateLogProbsFn,
     request_id: str,
@@ -302,7 +302,7 @@ async def create_completion(self, request: CompletionRequest,
         except ValueError as e:
             return self.create_error_response(str(e))
 
-        result_generator: AsyncIterator[tuple[
+        result_generator: AsyncIterator[Tuple[
             int, RequestOutput]] = merge_async_iterators(*generators)
 
         # Similar to the OpenAI API, when n != best_of, we do not stream the


Test patch:
Repository: vllm-project/vllm

Issue:
[Minor] Fix test_cache.py CI test failure
While working on https://github.com/vllm-project/vllm/pull/2677, I noticed there was a test failure that came from a race condition from merging https://github.com/vllm-project/vllm/pull/2279 and https://github.com/vllm-project/vllm/pull/2616 "at the same time"


Discussion:
There seems to be a deeper problem with these tests -- maybe @LiuXiaoxuanPKU can have a look?

Resolving patch:
diff --git a/vllm/utils.py b/vllm/utils.py
--- a/vllm/utils.py
+++ b/vllm/utils.py
@@ -258,10 +258,13 @@ def create_kv_caches_with_random(
         key_cache = torch.empty(size=key_cache_shape,
                                 dtype=torch_dtype,
                                 device=device)
-        if cache_dtype in ["auto", "half", "bfloat16", "float"]:
-            key_cache.uniform_(-scale, scale)
-        elif cache_dtype == 'fp8_e5m2':
+        if cache_dtype == 'fp8_e5m2':
             _generate_random_fp8_e5m2(key_cache, -scale, scale)
+        elif torch_dtype in [torch.half, torch.bfloat16, torch.float]:
+            key_cache.uniform_(-scale, scale)
+        else:
+            raise ValueError(
+                f"Does not support key cache of type {cache_dtype}")
         key_caches.append(key_cache)
 
     value_cache_shape = (num_blocks, num_heads, head_size, block_size)
@@ -270,9 +273,12 @@ def create_kv_caches_with_random(
         value_cache = torch.empty(size=value_cache_shape,
                                   dtype=torch_dtype,
                                   device=device)
-        if cache_dtype in ["auto", "half", "bfloat16", "float"]:
-            value_cache.uniform_(-scale, scale)
-        elif cache_dtype == 'fp8_e5m2':
+        if cache_dtype == 'fp8_e5m2':
             _generate_random_fp8_e5m2(value_cache, -scale, scale)
+        elif torch_dtype in [torch.half, torch.bfloat16, torch.float]:
+            value_cache.uniform_(-scale, scale)
+        else:
+            raise ValueError(
+                f"Does not support value cache of type {cache_dtype}")
         value_caches.append(value_cache)
     return key_caches, value_caches


Test patch:
diff --git a/tests/kernels/test_cache.py b/tests/kernels/test_cache.py
--- a/tests/kernels/test_cache.py
+++ b/tests/kernels/test_cache.py
@@ -177,16 +177,15 @@ def test_swap_blocks(
     num_blocks: int,
     dtype: torch.dtype,
     seed: int,
-    device: int,
+    device: str,
 ) -> None:
     random.seed(seed)
     torch.random.manual_seed(seed)
     if torch.cuda.is_available():
         torch.cuda.manual_seed(seed)
-    src_device = f"{direction[0]}:{device}" if direction[
-        0] == "cuda" else direction[0]
-    dst_device = f"{direction[1]}:{device}" if direction[
-        1] == "cuda" else direction[1]
+
+    src_device = device if direction[0] == "cuda" else 'cpu'
+    dst_device = device if direction[1] == "cuda" else 'cpu'
 
     src_blocks = random.sample(range(num_blocks), num_mappings)
     # For the same device, mapping must not overlap
Repository: vllm-project/vllm

Issue:
Fix AttributeError: MixtralModel object has no attribute org_vocab_size.
Mixtral cannot start because of this error:
```
  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1695, in __getattr__
    raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")
AttributeError: 'MixtralModel' object has no attribute 'org_vocab_size'. Did you mean: 'vocab_size'?
```

Related PR: #2831


Discussion:

Resolving patch:
diff --git a/vllm/model_executor/models/mixtral.py b/vllm/model_executor/models/mixtral.py
--- a/vllm/model_executor/models/mixtral.py
+++ b/vllm/model_executor/models/mixtral.py
@@ -285,15 +285,19 @@ def __init__(
         self,
         config: MixtralConfig,
         linear_method: Optional[LinearMethodBase] = None,
+        lora_config: Optional[LoRAConfig] = None,
     ) -> None:
         super().__init__()
         self.padding_idx = config.pad_token_id
-        self.vocab_size = config.vocab_size
+        lora_vocab = (lora_config.lora_extra_vocab_size *
+                      (lora_config.max_loras or 1)) if lora_config else 0
+        self.vocab_size = config.vocab_size + lora_vocab
+        self.org_vocab_size = config.vocab_size
 
         self.embed_tokens = VocabParallelEmbedding(
-            config.vocab_size,
+            self.vocab_size,
             config.hidden_size,
-            org_num_embeddings=self.org_vocab_size,
+            org_num_embeddings=config.vocab_size,
         )
         self.layers = nn.ModuleList([
             MixtralDecoderLayer(config, linear_method=linear_method)
@@ -350,7 +354,9 @@ def __init__(
         super().__init__()
         self.config = config
         self.linear_method = linear_method
-        self.model = MixtralModel(config, linear_method)
+        self.model = MixtralModel(config,
+                                  linear_method,
+                                  lora_config=lora_config)
         self.unpadded_vocab_size = config.vocab_size
         if lora_config:
             self.unpadded_vocab_size += lora_config.lora_extra_vocab_size


Test patch:
Repository: vllm-project/vllm

Issue:
[v0.3.1] Release Tracker
**ETA**: Feb 14-16 th

## Major changes

TBD

## PRs to be merged before the release

- [x] #2855 
- [x] #2845 
- [x] ~~#2514~~
- [x] Ensure memory release when `LLM` class is deleted. #2882 
- [x] #2875 #2880


Discussion:
@simon-mo Please feel free to add more!
I would really like #2804 but it seems to be blocked by either FlashInfer or other libraries 
@simon-mo I think the main concern here is AMD because the ROCm xformers patch uses `xformers==0.0.23` while we're upgrading to `xformers==0.0.24` for CUDA.
I see. We might be able to distribute different version with varying version pins...
need supporting miqu-1-70b-sf-gptq.  thanks a lot！
I would like to see a fix to #2795. I and two other users have been unable to use the latest version of vLLM with Ray, however, it works perfectly well after downgrading to the previous version.
#2761 brings back support for quantized MoE models like Mixtral/Deepseek. Also brings a great speedup (2-3x). 

Possible to include it in the next release so that quantized models are not broken?
@umarbutler In this release, we will disable to the custom all reduce, which should address #2795. 
@casper-hansen We should definitely get this PR in in next two weeks, but we need to cut a release today to address critical issues. 
We will need to include https://github.com/vllm-project/vllm/pull/2875 in the release as well
@pcmoritz Added. Thanks!
@caoshiyi Would it be possible to merge #2517 before the release?
@WoosukKwon Sorry for the delay. Will address the comments tonight.

Resolving patch:
diff --git a/vllm/__init__.py b/vllm/__init__.py
--- a/vllm/__init__.py
+++ b/vllm/__init__.py
@@ -8,7 +8,7 @@
 from vllm.outputs import CompletionOutput, RequestOutput
 from vllm.sampling_params import SamplingParams
 
-__version__ = "0.3.0"
+__version__ = "0.3.1"
 
 __all__ = [
     "LLM",


Test patch:
Repository: vllm-project/vllm

Issue:
[v0.3.3] Release Tracker
**ETA**: Feb 29th - Mar 1st

## Major changes

* StarCoder2 support
* Performance optimization and LoRA support for Gemma
* Performance optimization for MoE kernel
* 2/3/8-bit GPTQ support
* [Experimental] AWS Inferentia2 support

## PRs to be merged before the release

- [x] #2330 #2223
- [ ] ~~#2761~~
- [x] #2819 
- [x] #3087 #3099
- [x] #3089 


Discussion:
#2819
#3087
Starcoder2
#3099 is a fix for the #3087 regression
#2760 Fixes for custom all reduce on some platforms

#2497 Adds support for Marlin INT4 kernels ~3x faster than current GPTQ kernels

Resolving patch:
diff --git a/vllm/__init__.py b/vllm/__init__.py
--- a/vllm/__init__.py
+++ b/vllm/__init__.py
@@ -8,7 +8,7 @@
 from vllm.outputs import CompletionOutput, RequestOutput
 from vllm.sampling_params import SamplingParams
 
-__version__ = "0.3.2"
+__version__ = "0.3.3"
 
 __all__ = [
     "LLM",


Test patch:
Repository: vllm-project/vllm

Issue:
why vllm==0.3.3 need to access google
![微信图片_20240304112403](https://github.com/vllm-project/vllm/assets/38678334/f21e1ec0-bd6f-4b26-aeee-d6e4e5822fc2)



Discussion:
+1 - https://github.com/vllm-project/vllm/pull/2931 breaks vllm for offline.



Resolving patch:
diff --git a/vllm/utils.py b/vllm/utils.py
--- a/vllm/utils.py
+++ b/vllm/utils.py
@@ -173,7 +173,7 @@ def get_ip() -> str:
     # try ipv4
     s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
     try:
-        s.connect(("dns.google", 80))  # Doesn't need to be reachable
+        s.connect(("8.8.8.8", 80))  # Doesn't need to be reachable
         return s.getsockname()[0]
     except OSError:
         # try ipv6


Test patch:
Repository: vllm-project/vllm

Issue:
Support Lora for qwen2



Discussion:

Resolving patch:
diff --git a/vllm/model_executor/models/qwen2.py b/vllm/model_executor/models/qwen2.py
--- a/vllm/model_executor/models/qwen2.py
+++ b/vllm/model_executor/models/qwen2.py
@@ -46,6 +46,7 @@
 from vllm.model_executor.weight_utils import (default_weight_loader,
                                               hf_model_weights_iterator)
 from vllm.sequence import SamplerOutput
+from vllm.config import LoRAConfig
 
 KVCache = Tuple[torch.Tensor, torch.Tensor]
 
@@ -264,12 +265,35 @@ def forward(
 
 
 class Qwen2ForCausalLM(nn.Module):
+    packed_modules_mapping = {
+        "qkv_proj": [
+            "q_proj",
+            "k_proj",
+            "v_proj",
+        ],
+        "gate_up_proj": [
+            "gate_proj",
+            "up_proj",
+        ],
+    }
+
+    # LoRA specific attributes
+    supported_lora_modules = [
+        "qkv_proj",
+        "o_proj",
+        "gate_up_proj",
+        "down_proj",
+    ]
+    embedding_modules = {}
+    embedding_padding_modules = []
 
     def __init__(
         self,
         config: Qwen2Config,
         linear_method: Optional[LinearMethodBase] = None,
+        lora_config: Optional[LoRAConfig] = None,
     ) -> None:
+        del lora_config
         super().__init__()
         self.config = config
         self.linear_method = linear_method


Test patch:
Repository: vllm-project/vllm

Issue:
The same model cannot be loaded by two different users
As pointed out here, the way lockfiles are created prevents the second user from loading any models that a previous user has loaded at any point: https://github.com/vllm-project/vllm/issues/2179

This is still an issue with the only workaround being to force-delete the lockfile created by another user.


Discussion:

Resolving patch:
diff --git a/vllm/model_executor/weight_utils.py b/vllm/model_executor/weight_utils.py
--- a/vllm/model_executor/weight_utils.py
+++ b/vllm/model_executor/weight_utils.py
@@ -34,7 +34,7 @@ def get_lock(model_name_or_path: str, cache_dir: Optional[str] = None):
     lock_dir = cache_dir if cache_dir is not None else _vllm_filelocks_path
     os.makedirs(os.path.dirname(lock_dir), exist_ok=True)
     lock_file_name = model_name_or_path.replace("/", "-") + ".lock"
-    lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name))
+    lock = filelock.SoftFileLock(os.path.join(lock_dir, lock_file_name))
     return lock
 
 


Test patch:
Repository: vllm-project/vllm

Issue:
[CI] Test examples in CI
### Anything you want to discuss about vllm.

Current scripts in `examples/` directory are not tested in CI. We should run them to ensure passing 


Discussion:

Resolving patch:
diff --git a/examples/llava_example.py b/examples/llava_example.py
--- a/examples/llava_example.py
+++ b/examples/llava_example.py
@@ -78,7 +78,13 @@ def main(args):
     # Make sure the local directory exists or create it
     os.makedirs(local_directory, exist_ok=True)
 
-    # Use AWS CLI to sync the directory
-    subprocess.check_call(
-        ["aws", "s3", "sync", s3_bucket_path, local_directory])
+    # Use AWS CLI to sync the directory, assume anonymous access
+    subprocess.check_call([
+        "aws",
+        "s3",
+        "sync",
+        s3_bucket_path,
+        local_directory,
+        "--no-sign-request",
+    ])
     main(args)


Test patch:
Repository: vllm-project/vllm

Issue:
[Feature]: Cannot use FlashAttention backend for Volta and Turing GPUs. (but FlashAttention v1.0.9 supports Turing GPU.)
### 🚀 The feature, motivation and pitch

Turing GPU can use FlashAttention v1.0.9 which can reduce use of vram significantly.

FlashAttention has no plan to support Turing GPU in FlashAttention v2 actually.
so please support FlashAttention v1.0.9. thanks a lot!

many friends having 8*2080ti need this help.

### Alternatives

_No response_

### Additional context

_No response_


Discussion:
+1 - it would be great to have flash attention support for volta GPUs
+1, if it can support GPUs based on Turing architecture, that would be great.
+1

Resolving patch:
diff --git a/vllm/attention/selector.py b/vllm/attention/selector.py
--- a/vllm/attention/selector.py
+++ b/vllm/attention/selector.py
@@ -25,7 +25,7 @@ class _Backend(enum.Enum):
 def get_attn_backend(dtype: torch.dtype) -> Type[AttentionBackend]:
     backend = _which_attn_to_use(dtype)
     if backend == _Backend.FLASH_ATTN:
-        logger.info("Using FlashAttention backend.")
+        logger.info("Using FlashAttention-2 backend.")
         from vllm.attention.backends.flash_attn import (  # noqa: F401
             FlashAttentionBackend)
         return FlashAttentionBackend
@@ -62,12 +62,12 @@ def _which_attn_to_use(dtype: torch.dtype) -> _Backend:
     # NVIDIA GPUs.
     if torch.cuda.get_device_capability()[0] < 8:
         # Volta and Turing NVIDIA GPUs.
-        logger.info("Cannot use FlashAttention backend for Volta and Turing "
+        logger.info("Cannot use FlashAttention-2 backend for Volta and Turing "
                     "GPUs.")
         return _Backend.XFORMERS
 
     if dtype not in (torch.float16, torch.bfloat16):
-        logger.info("Cannot use FlashAttention backend for dtype other than "
+        logger.info("Cannot use FlashAttention-2 backend for dtype other than "
                     "torch.float16 or torch.bfloat16.")
         return _Backend.XFORMERS
 
@@ -75,8 +75,8 @@ def _which_attn_to_use(dtype: torch.dtype) -> _Backend:
         import flash_attn  # noqa: F401
     except ImportError:
         logger.info(
-            "Cannot use FlashAttention backend because the flash_attn package "
-            "is not found. Please install it for better performance.")
+            "Cannot use FlashAttention-2 backend because the flash_attn "
+            "package is not found. Please install it for better performance.")
         return _Backend.XFORMERS
 
     backend_by_env_var = os.getenv(VLLM_ATTENTION_BACKEND)


Test patch:
Repository: vllm-project/vllm

Issue:
[Bug]: `v0.4.2` python3.8 `TypeError: 'type' object is not subscriptable` (python3.9 syntax)
### Your current environment

```text
The output of `python collect_env.py`
```


### 🐛 Describe the bug

`python -m vllm.entrypoints.openai.api_server --model microsoft/Phi-3-mini-4k-instruct --dtype auto --max-model-len 2000`

Error
```
File ".../python3.8/site-packages/vllm/entrypoints/openai/api_server.py", line 37, in <module>
    _running_tasks: Set[asyncio.Task[Any]] = set()
TypeError: 'type' object is not subscriptable
```

Used python3.10 syntax


Discussion:
cc @rkooo567 is this file checked by mypy? It is strange that it passes mypy checking.
> cc @rkooo567 is this file checked by mypy? It is strange that it passes mypy checking.

I guess that type checking is disabled for imported items due to setting `follow_imports = "skip"`.

Resolving patch:
diff --git a/vllm/engine/async_llm_engine.py b/vllm/engine/async_llm_engine.py
--- a/vllm/engine/async_llm_engine.py
+++ b/vllm/engine/async_llm_engine.py
@@ -1,8 +1,8 @@
 import asyncio
 import time
 from functools import partial
-from typing import (Any, AsyncIterator, Callable, Dict, Iterable, List,
-                    Optional, Set, Tuple, Type, Union)
+from typing import (AsyncIterator, Callable, Dict, Iterable, List, Optional,
+                    Set, Tuple, Type, Union)
 
 from transformers import PreTrainedTokenizer
 
@@ -327,7 +327,7 @@ def __init__(self,
         # We need to keep a reference to unshielded
         # task as well to prevent it from being garbage
         # collected
-        self._background_loop_unshielded: Optional[asyncio.Task[Any]] = None
+        self._background_loop_unshielded: Optional[asyncio.Task] = None
         self.start_engine_loop = start_engine_loop
         self._errored_with: Optional[BaseException] = None
 
diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py
--- a/vllm/entrypoints/openai/api_server.py
+++ b/vllm/entrypoints/openai/api_server.py
@@ -4,7 +4,7 @@
 import re
 from contextlib import asynccontextmanager
 from http import HTTPStatus
-from typing import Any, Set
+from typing import Set
 
 import fastapi
 import uvicorn
@@ -34,7 +34,7 @@
 openai_serving_completion: OpenAIServingCompletion
 logger = init_logger(__name__)
 
-_running_tasks: Set[asyncio.Task[Any]] = set()
+_running_tasks: Set[asyncio.Task] = set()
 
 
 @asynccontextmanager


Test patch:
Repository: vllm-project/vllm

Issue:
[Feature]: rope_scaling for qwen2 
### 🚀 The feature, motivation and pitch

We found that qwen2 such as Qwen2Attention does not accept rope_scaling. However, we need to use yarn/ntk feature. Could you support that?

### Alternatives

_No response_

### Additional context

_No response_


Discussion:

Resolving patch:
diff --git a/vllm/model_executor/models/qwen2.py b/vllm/model_executor/models/qwen2.py
--- a/vllm/model_executor/models/qwen2.py
+++ b/vllm/model_executor/models/qwen2.py
@@ -89,7 +89,8 @@ def __init__(self,
                  use_sliding_window: bool = False,
                  cache_config: Optional[CacheConfig] = None,
                  quant_config: Optional[QuantizationConfig] = None,
-                 sliding_window: Optional[int] = None) -> None:
+                 sliding_window: Optional[int] = None,
+                 rope_scaling: Optional[Tuple] = None) -> None:
         super().__init__()
         self.hidden_size = hidden_size
         tp_size = get_tensor_model_parallel_world_size()
@@ -133,6 +134,7 @@ def __init__(self,
             rotary_dim=self.head_dim,
             max_position=max_position,
             base=self.rope_theta,
+            rope_scaling=rope_scaling,
         )
         self.attn = Attention(self.num_heads,
                               self.head_dim,
@@ -169,6 +171,7 @@ def __init__(
         self.hidden_size = config.hidden_size
         # Requires transformers > 4.32.0
         rope_theta = getattr(config, "rope_theta", 1000000)
+        rope_scaling = getattr(config, "rope_scaling", None)
         use_sliding_window = (config.use_sliding_window
                               and layer_idx < config.max_window_layers)
         self.self_attn = Qwen2Attention(
@@ -180,7 +183,8 @@ def __init__(
             use_sliding_window=use_sliding_window,
             cache_config=cache_config,
             quant_config=quant_config,
-            sliding_window=config.sliding_window)
+            sliding_window=config.sliding_window,
+            rope_scaling=rope_scaling)
         self.mlp = Qwen2MLP(
             hidden_size=self.hidden_size,
             intermediate_size=config.intermediate_size,


Test patch: