Repository: vllm-project/vllm
Issue category: Performance Issue

Issue:
[Performance]: vllm 0.5.4 with enable_chunked_prefill =True, throughput is slightly lower than 0.5.3~0.5.0.
### Your current environment

<details>
<summary>environment</summary>

# Hardware & Nvidia driver & OS
GPU models and configuration: GPU 0: NVIDIA GeForce RTX 4090       

Nvidia driver version: 550.107.02 

OS: Ubuntu 22.04.4 LTS (x86_64) 

# conda env create -f environment_linux.yml
```
name: vllm_xxx
channels:
  - conda-forge
  - pytorch
  - nvidia
  - defaults
dependencies:
  - python=3.11
  - anaconda
  - pip
  - pip:
      - easydict
  
pip install vllm==v0.5.4 / v0.5.3.post1 / v0.5.2 / v0.5.1
```

</details>


### 🐛 Describe the bug

Throughput and latency vary with max_num_seqs。

vllm 0.5.4 with enable_chunked_prefill =True, throughput is slightly lower than 0.5.3~0.5.1. very strange.

![chunked_prefill](https://github.com/user-attachments/assets/0250456a-60b8-45e9-82ad-316204f6e778)

| max_num_seqs | requests/s | disable 0.5.4 | requests/s | disable 0.5.3 | requests/s | disable 0.5.2 | requests/s | disable 0.5.1 | requests/s | enable 0.5.4 | requests/s | enable 0.5.3 | requests/s | enable 0.5.2 | requests/s | enable 0.5.1 |
|--------------|------------|---------------|------------|---------------|------------|---------------|------------|---------------|------------|--------------|------------|--------------|------------|--------------|------------|--------------|
| 1024         | 2.5115     | 36.1          | 2.4747     | 36.69         | 2.4878     | 36.82         | 2.4972     | 36.33         | 3.1797     | 39.72        | 3.1341     | 40.15        | 3.1696     | 39.67        | 3.1745     | 39.61        |
| 768          | 3.2838     | 42.6          | 3.2242     | 43.48         | 3.2608     | 42.93         | 3.2648     | 42.88         | 4.1047     | 43.4         | 3.9708     | 44.42        | 4.0413     | 43.6         | 4.0439     | 43.57        |
| 512          | 4.1063     | 51.93         | 4.0102     | 53.22         | 4.0966     | 52.07         | 4.0998     | 51.97         | 4.6486     | 46.79        | 4.6377     | 46.25        | 4.7419     | 45.15        | 4.747      | 45.13        |
| 384          | 4.1705     | 54.83         | 4.0749     | 56.35         | 4.1538     | 55.07         | 4.1587     | 55            | 4.798      | 49.66        | 4.8605     | 45.93        | 4.9834     | 44.72        | 5.0019     | 44.61        |
| 256          | 4.3613     | 59.37         | 4.2659     | 60.94         | 4.3586     | 59.44         | 4.3632     | 59.36         | 4.9096     | 48.27        | 5.4424     | 42.23        | 5.5876     | 41.1         | 5.5949     | 41.04        |
| 128          | 4.7441     | 42.75         | 4.6511     | 43.76         | 4.7564     | 42.62         | 4.7583     | 42.58         | 4.2047     | 32.77        | 4.0718     | 29.7         | 4.1528     | 29.11        | 4.1544     | 29.1         |
| 64           | 3.7161     | 29.56         | 3.6446     | 30.2          | 3.6972     | 29.71         | 3.7044     | 29.66         | 2.435      | 26.88        | 2.3879     | 25.74        | 2.4141     | 25.45        | 2.4175     | 25.41        |
| 32           | 2.6923     | 20.76         | 2.6465     | 21.16         | 2.6702     | 20.94         | 2.6735     | 20.92         | 1.6103     | 19.83        | 1.5846     | 19.55        | 1.593      | 19.44        | 1.5942     | 19.43        |

```
import os
import random
import numpy as np
import time


def benchmark(args):
    random.seed(args.seed)

    os.environ["VLLM_LOGGING_LEVEL"] = "ERROR"
    os.environ["VLLM_NO_USAGE_STATS"] = "True"

    import vllm
    from vllm import LLMEngine, EngineArgs, SamplingParams, TextPrompt

    print(vllm.__version__)

    engine_args = EngineArgs(
        model=args.model,
        tokenizer=args.tokenizer,
        quantization=args.quantization,
        tensor_parallel_size=args.tensor_parallel_size,
        seed=args.seed,
        trust_remote_code=args.trust_remote_code,
        dtype=args.dtype,
        max_model_len=args.max_model_len,
        gpu_memory_utilization=args.gpu_memory_utilization,
        enforce_eager=args.enforce_eager,
        kv_cache_dtype=args.kv_cache_dtype,
        quantization_param_path=args.quantization_param_path,
        device=args.device,
        enable_prefix_caching=args.enable_prefix_caching,
        download_dir=args.download_dir,
        enable_chunked_prefill=args.enable_chunked_prefill,
        max_num_batched_tokens=args.max_num_batched_tokens,
        max_num_seqs=args.max_num_seqs,
        distributed_executor_backend=args.distributed_executor_backend,
        disable_log_stats=True
    )
    engine = LLMEngine.from_engine_args(engine_args)

    prompt = "hi" * (args.input_len - 1)
    requests = [(prompt, args.input_len, args.output_len)
                for _ in range(args.num_prompts)]

    start = time.perf_counter()
    for request_id, (prompt, _, output_len) in enumerate(requests):
        inputs = TextPrompt(prompt=prompt)
        sampling_params = SamplingParams(
            n=args.n,
            temperature=0.0 if args.use_beam_search else 1.0,
            top_p=1.0,
            use_beam_search=args.use_beam_search,
            ignore_eos=True,
            max_tokens=output_len,
        )
        engine.add_request(str(request_id), inputs, sampling_params)

    out = []
    while engine.has_unfinished_requests():
        request_outputs = engine.step()
        out.append((time.perf_counter(), request_outputs))
    end = time.perf_counter()

    timestamp = {}
    for t, rs in out:
        for r in rs:
            request_id = r.request_id
            if request_id not in timestamp:
                timestamp[request_id] = []
            timestamp[request_id].append(t)

    tpot = []
    for v in timestamp.values():
        dd = [v[i]-v[i-1] for i in range(1, len(v))]
        tpot.extend(dd)

    tpot = np.mean(tpot)
    elapsed_time = end - start

    total_num_tokens = sum(prompt_len + output_len
                           for _, prompt_len, output_len in requests)

    print(f"Throughput: {len(requests) / elapsed_time:.4f} requests/s, "
          f"{total_num_tokens / elapsed_time:.4f} tokens/s, "
          f"Delay {tpot*1000:0.2f} ms")


if __name__ == '__main__':
    from easydict import EasyDict as edict
    args = edict()

    args.dataset = None
    args.input_len = 512
    args.output_len = 512

    args.model = "Qwen/Qwen2-7B-Instruct"
    args.trust_remote_code = False
    args.tokenizer = args.model
    args.quantization = None
    args.quantization_param_path = None
    args.tensor_parallel_size = 1
    args.seed = 0
    args.n = 1
    args.use_beam_search = False
    args.num_prompts = 1000
    args.dtype = 'auto'
    args.max_model_len = 10000
    args.enforce_eager = True
    args.kv_cache_dtype = "auto"
    args.device = "cuda"
    args.enable_prefix_caching = False
    args.gpu_memory_utilization = 0.9
    args.output_json = None
    args.distributed_executor_backend = None
    args.download_dir = None

    import sys
    from concurrent.futures import ProcessPoolExecutor

    def run(args):
        with ProcessPoolExecutor(1) as executor:
            f = executor.submit(benchmark, args)
            f.result()

    max_num_seqs_list = [1024, 768, 512, 384, 256, 128, 64, 32]

    print()
    print("enable_chunked_prefill = False")
    for max_num_seqs in max_num_seqs_list:
        print("max_num_seqs", max_num_seqs)
        args.enable_chunked_prefill = False
        args.max_num_batched_tokens = None
        args.max_num_seqs = max_num_seqs
        run(args)

    print()
    print("enable_chunked_prefill = True")
    for max_num_seqs in max_num_seqs_list:
        print("max_num_seqs", max_num_seqs)
        args.enable_chunked_prefill = True
        args.max_num_seqs = max_num_seqs
        args.max_num_batched_tokens = args.max_num_seqs
        run(args)
```



[Performance]: vllm 0.5.4 with enable_chunked_prefill =True, throughput is slightly lower than 0.5.3~0.5.0.
### Your current environment

<details>
<summary>environment</summary>

# Hardware & Nvidia driver & OS
GPU models and configuration: GPU 0: NVIDIA GeForce RTX 4090       

Nvidia driver version: 550.107.02 

OS: Ubuntu 22.04.4 LTS (x86_64) 

# conda env create -f environment_linux.yml
```
name: vllm_xxx
channels:
  - conda-forge
  - pytorch
  - nvidia
  - defaults
dependencies:
  - python=3.11
  - anaconda
  - pip
  - pip:
      - easydict
  
pip install vllm==v0.5.4 / v0.5.3.post1 / v0.5.2 / v0.5.1
```

</details>


### 🐛 Describe the bug

Throughput and latency vary with max_num_seqs。

vllm 0.5.4 with enable_chunked_prefill =True, throughput is slightly lower than 0.5.3~0.5.1. very strange.

![chunked_prefill](https://github.com/user-attachments/assets/0250456a-60b8-45e9-82ad-316204f6e778)

| max_num_seqs | requests/s | disable 0.5.4 | requests/s | disable 0.5.3 | requests/s | disable 0.5.2 | requests/s | disable 0.5.1 | requests/s | enable 0.5.4 | requests/s | enable 0.5.3 | requests/s | enable 0.5.2 | requests/s | enable 0.5.1 |
|--------------|------------|---------------|------------|---------------|------------|---------------|------------|---------------|------------|--------------|------------|--------------|------------|--------------|------------|--------------|
| 1024         | 2.5115     | 36.1          | 2.4747     | 36.69         | 2.4878     | 36.82         | 2.4972     | 36.33         | 3.1797     | 39.72        | 3.1341     | 40.15        | 3.1696     | 39.67        | 3.1745     | 39.61        |
| 768          | 3.2838     | 42.6          | 3.2242     | 43.48         | 3.2608     | 42.93         | 3.2648     | 42.88         | 4.1047     | 43.4         | 3.9708     | 44.42        | 4.0413     | 43.6         | 4.0439     | 43.57        |
| 512          | 4.1063     | 51.93         | 4.0102     | 53.22         | 4.0966     | 52.07         | 4.0998     | 51.97         | 4.6486     | 46.79        | 4.6377     | 46.25        | 4.7419     | 45.15        | 4.747      | 45.13        |
| 384          | 4.1705     | 54.83         | 4.0749     | 56.35         | 4.1538     | 55.07         | 4.1587     | 55            | 4.798      | 49.66        | 4.8605     | 45.93        | 4.9834     | 44.72        | 5.0019     | 44.61        |
| 256          | 4.3613     | 59.37         | 4.2659     | 60.94         | 4.3586     | 59.44         | 4.3632     | 59.36         | 4.9096     | 48.27        | 5.4424     | 42.23        | 5.5876     | 41.1         | 5.5949     | 41.04        |
| 128          | 4.7441     | 42.75         | 4.6511     | 43.76         | 4.7564     | 42.62         | 4.7583     | 42.58         | 4.2047     | 32.77        | 4.0718     | 29.7         | 4.1528     | 29.11        | 4.1544     | 29.1         |
| 64           | 3.7161     | 29.56         | 3.6446     | 30.2          | 3.6972     | 29.71         | 3.7044     | 29.66         | 2.435      | 26.88        | 2.3879     | 25.74        | 2.4141     | 25.45        | 2.4175     | 25.41        |
| 32           | 2.6923     | 20.76         | 2.6465     | 21.16         | 2.6702     | 20.94         | 2.6735     | 20.92         | 1.6103     | 19.83        | 1.5846     | 19.55        | 1.593      | 19.44        | 1.5942     | 19.43        |

```
import os
import random
import numpy as np
import time


def benchmark(args):
    random.seed(args.seed)

    os.environ["VLLM_LOGGING_LEVEL"] = "ERROR"
    os.environ["VLLM_NO_USAGE_STATS"] = "True"

    import vllm
    from vllm import LLMEngine, EngineArgs, SamplingParams, TextPrompt

    print(vllm.__version__)

    engine_args = EngineArgs(
        model=args.model,
        tokenizer=args.tokenizer,
        quantization=args.quantization,
        tensor_parallel_size=args.tensor_parallel_size,
        seed=args.seed,
        trust_remote_code=args.trust_remote_code,
        dtype=args.dtype,
        max_model_len=args.max_model_len,
        gpu_memory_utilization=args.gpu_memory_utilization,
        enforce_eager=args.enforce_eager,
        kv_cache_dtype=args.kv_cache_dtype,
        quantization_param_path=args.quantization_param_path,
        device=args.device,
        enable_prefix_caching=args.enable_prefix_caching,
        download_dir=args.download_dir,
        enable_chunked_prefill=args.enable_chunked_prefill,
        max_num_batched_tokens=args.max_num_batched_tokens,
        max_num_seqs=args.max_num_seqs,
        distributed_executor_backend=args.distributed_executor_backend,
        disable_log_stats=True
    )
    engine = LLMEngine.from_engine_args(engine_args)

    prompt = "hi" * (args.input_len - 1)
    requests = [(prompt, args.input_len, args.output_len)
                for _ in range(args.num_prompts)]

    start = time.perf_counter()
    for request_id, (prompt, _, output_len) in enumerate(requests):
        inputs = TextPrompt(prompt=prompt)
        sampling_params = SamplingParams(
            n=args.n,
            temperature=0.0 if args.use_beam_search else 1.0,
            top_p=1.0,
            use_beam_search=args.use_beam_search,
            ignore_eos=True,
            max_tokens=output_len,
        )
        engine.add_request(str(request_id), inputs, sampling_params)

    out = []
    while engine.has_unfinished_requests():
        request_outputs = engine.step()
        out.append((time.perf_counter(), request_outputs))
    end = time.perf_counter()

    timestamp = {}
    for t, rs in out:
        for r in rs:
            request_id = r.request_id
            if request_id not in timestamp:
                timestamp[request_id] = []
            timestamp[request_id].append(t)

    tpot = []
    for v in timestamp.values():
        dd = [v[i]-v[i-1] for i in range(1, len(v))]
        tpot.extend(dd)

    tpot = np.mean(tpot)
    elapsed_time = end - start

    total_num_tokens = sum(prompt_len + output_len
                           for _, prompt_len, output_len in requests)

    print(f"Throughput: {len(requests) / elapsed_time:.4f} requests/s, "
          f"{total_num_tokens / elapsed_time:.4f} tokens/s, "
          f"Delay {tpot*1000:0.2f} ms")


if __name__ == '__main__':
    from easydict import EasyDict as edict
    args = edict()

    args.dataset = None
    args.input_len = 512
    args.output_len = 512

    args.model = "Qwen/Qwen2-7B-Instruct"
    args.trust_remote_code = False
    args.tokenizer = args.model
    args.quantization = None
    args.quantization_param_path = None
    args.tensor_parallel_size = 1
    args.seed = 0
    args.n = 1
    args.use_beam_search = False
    args.num_prompts = 1000
    args.dtype = 'auto'
    args.max_model_len = 10000
    args.enforce_eager = True
    args.kv_cache_dtype = "auto"
    args.device = "cuda"
    args.enable_prefix_caching = False
    args.gpu_memory_utilization = 0.9
    args.output_json = None
    args.distributed_executor_backend = None
    args.download_dir = None

    import sys
    from concurrent.futures import ProcessPoolExecutor

    def run(args):
        with ProcessPoolExecutor(1) as executor:
            f = executor.submit(benchmark, args)
            f.result()

    max_num_seqs_list = [1024, 768, 512, 384, 256, 128, 64, 32]

    print()
    print("enable_chunked_prefill = False")
    for max_num_seqs in max_num_seqs_list:
        print("max_num_seqs", max_num_seqs)
        args.enable_chunked_prefill = False
        args.max_num_batched_tokens = None
        args.max_num_seqs = max_num_seqs
        run(args)

    print()
    print("enable_chunked_prefill = True")
    for max_num_seqs in max_num_seqs_list:
        print("max_num_seqs", max_num_seqs)
        args.enable_chunked_prefill = True
        args.max_num_seqs = max_num_seqs
        args.max_num_batched_tokens = args.max_num_seqs
        run(args)
```





Hints:
update

I managed to compile vllm 0.5.4 with torch == 2.3.1 xformers == 0.0.27 vllm-flash-attn == 2.5.9.post1 (from vllm 0.5.3 requirements )， the bug is still there.

![chunked_prefill2](https://github.com/user-attachments/assets/fbfca114-ae8b-4e13-ae3c-3afe9d3730d7)


The bug only affects chunked_prefill, everything works fine when chunked_prefill is turned off.

update

I managed to merge vllm 0.5.3 python part and vllm 0.5.4 c part，the bug gone.  labeled as vllm 0.5.3+.

![chunked_prefill3](https://github.com/user-attachments/assets/c02f1f5c-a0bf-4f0e-b502-b9cb040316f2)

so, bug in vllm 0.5.4 python part, and not in attention folder.



Caused by [core][scheduler] simplify and improve scheduler #6867

https://github.com/vllm-project/vllm/pull/6867

Revert the commit, the bug gone. 
@youkaichao ^^
first, this is not a bug. it's better to appear with a "performance" tag.

second, there are various workloads you can test, and it is difficult to have any code change that benefits all workloads.

that pr is very important for optimizing the case in large offline batch inference setting. If you can achieve the same effect while keeping chunked prefill performance the untouched, we'd welcome that contribution.
That PR caused chunked prefill peak throughput to drop by 15%. 
I thought it would be interesting to figure out why.
update

I managed to compile vllm 0.5.4 with torch == 2.3.1 xformers == 0.0.27 vllm-flash-attn == 2.5.9.post1 (from vllm 0.5.3 requirements )， the bug is still there.

![chunked_prefill2](https://github.com/user-attachments/assets/fbfca114-ae8b-4e13-ae3c-3afe9d3730d7)


The bug only affects chunked_prefill, everything works fine when chunked_prefill is turned off.

update

I managed to merge vllm 0.5.3 python part and vllm 0.5.4 c part，the bug gone.  labeled as vllm 0.5.3+.

![chunked_prefill3](https://github.com/user-attachments/assets/c02f1f5c-a0bf-4f0e-b502-b9cb040316f2)

so, bug in vllm 0.5.4 python part, and not in attention folder.



Caused by [core][scheduler] simplify and improve scheduler #6867

https://github.com/vllm-project/vllm/pull/6867

Revert the commit, the bug gone. 
@youkaichao ^^
first, this is not a bug. it's better to appear with a "performance" tag.

second, there are various workloads you can test, and it is difficult to have any code change that benefits all workloads.

that pr is very important for optimizing the case in large offline batch inference setting. If you can achieve the same effect while keeping chunked prefill performance the untouched, we'd welcome that contribution.
That PR caused chunked prefill peak throughput to drop by 15%. 
I thought it would be interesting to figure out why.

Patch:
diff --git a/vllm/core/scheduler.py b/vllm/core/scheduler.py
index 4c2f715820317..81c78bda3b505 100644
--- a/vllm/core/scheduler.py
+++ b/vllm/core/scheduler.py
@@ -1027,16 +1027,21 @@ def _schedule_chunked_prefill(self) -> SchedulerOutputs:
 
         # Update waiting requests.
         self.waiting.extendleft(running_scheduled.preempted)
+
         # Update new running requests.
-        self.running.extend([s.seq_group for s in prefills.seq_groups])
-        self.running.extend(
-            [s.seq_group for s in running_scheduled.decode_seq_groups])
-        self.running.extend(
-            [s.seq_group for s in running_scheduled.prefill_seq_groups])
+        # By default, vLLM scheduler prioritizes prefills.
+        # Once chunked prefill is enabled,
+        # the policy is changed to prioritize decode requests.
         self.running.extend(
             [s.seq_group for s in swapped_in.decode_seq_groups])
         self.running.extend(
             [s.seq_group for s in swapped_in.prefill_seq_groups])
+        self.running.extend(
+            [s.seq_group for s in running_scheduled.decode_seq_groups])
+        self.running.extend(
+            [s.seq_group for s in running_scheduled.prefill_seq_groups])
+        self.running.extend([s.seq_group for s in prefills.seq_groups])
+
         # Update swapped requests.
         self.swapped.extend(running_scheduled.swapped_out)
         return SchedulerOutputs(


Test patch:
diff --git a/tests/basic_correctness/test_chunked_prefill.py b/tests/basic_correctness/test_chunked_prefill.py
index fc6f829c37b06..a63ac380e8598 100644
--- a/tests/basic_correctness/test_chunked_prefill.py
+++ b/tests/basic_correctness/test_chunked_prefill.py
@@ -116,6 +116,9 @@ def test_models_with_fp8_kv_cache(
         pytest.skip(
             "#7378: CUDA illegal memory access (undiagnosed) facebook/opt-125m"
         )
+    if ((model, kv_cache_dtype, chunked_prefill_token_size) == (
+            "nm-testing/Qwen2-1.5B-Instruct-FP8-K-V", "fp8_e4m3", 4)):
+        pytest.skip("flakey test, see: #7874 #8051")
 
     max_num_seqs = chunked_prefill_token_size
     max_num_batched_tokens = chunked_prefill_token_size