Repository: vllm-project/vllm
File: tests/v1/core/test_scheduler.py (Python)

Code before:

    block_size = 16
    scheduler = _create_encoder_decoder_scheduler(block_size=block_size)

    # Simulate a mix of short and long audio inputs.
    test_data = [
        ("short_audio", 50),
        ("medium_audio", 750),
        ("long_audio", 1500),
        ("tiny_audio", 16),
        ("odd_audio", 333),
    ]

    requests = []
    for i, (name, enc_len) in enumerate(test_data):
        req = create_requests(
            num_requests=1,
            num_tokens=100,
            mm_hashes_list=[[f"hash_{name}"]],
            mm_positions=[[PlaceholderRange(offset=0, length=enc_len)]],
            req_ids=[f"req_{name}"],
        )[0]
        requests.append(req)

    for req in requests:
        scheduler.add_request(req)

    output = scheduler.schedule()
    assert len(output.scheduled_new_reqs) == len(test_data)

    # Verify each request gets exactly the right number of blocks.
    total_blocks_variable = 0
    total_blocks_fixed_max = 0
    max_enc_len = max(enc_len for _, enc_len in test_data)

    for req, (name, enc_len) in zip(requests, test_data):
        expected_blocks = ceil(enc_len / block_size)
        actual_blocks = _get_num_cross_attn_blocks(scheduler, req.request_id)

        assert actual_blocks == expected_blocks, (
            f"Request '{name}' ({enc_len} encoder tokens): "
            f"expected {expected_blocks} blocks, got {actual_blocks}"
        )

        total_blocks_variable += actual_blocks
        total_blocks_fixed_max += ceil(max_enc_len / block_size)

    # The variable-length approach should use fewer total blocks than
    # a fixed-max approach (unless all inputs are the same length).
    assert total_blocks_variable < total_blocks_fixed_max, (
        f"Variable allocation ({total_blocks_variable} blocks) should use fewer "

Review comment (suggestion):
this is actually misleading and had me go back to your change in scheduler.
Whisper only takes in fixed-input len spectograms (1500).
This test is suggesting that different audios will end up allocating different num blocks, if that was the case whisper would break.

Code after:
    shift_computed_tokens in the boundary calculation.

    Without the fix, the scheduler would fail to schedule encoder inputs
    at the boundary, causing "Encoder cache miss" errors.
    """
    scheduler = create_scheduler(
        model="llava-hf/llava-1.5-7b-hf",
        max_num_batched_tokens=1024,
        disable_chunked_mm_input=True,
        max_model_len=2048,
        num_speculative_tokens=4,  # This enables EAGLE with shift=1
    )

    mm_start_pos = 100
    mm_length = 576

    mm_positions = [
        [PlaceholderRange(offset=mm_start_pos, length=mm_length)],
    ]

    requests = create_requests(
        num_requests=1,
        num_tokens=mm_start_pos + mm_length + 100,
        mm_positions=mm_positions,
    )

    # Start with some tokens already computed to simulate decoding
    request = requests[0]
    request.num_computed_tokens = 0

    scheduler.add_request(request)
    output = scheduler.schedule()

    assert output is not None
    shift_computed_tokens = 1
    req_id = request.request_id

    assert req_id in output.num_scheduled_tokens
    num_scheduled = output.num_scheduled_tokens[req_id]

    mm_feature = request.mm_features[0]
    start_pos = mm_feature.mm_position.offset
    tokens_end = request.num_computed_tokens + num_scheduled
    scheduled_end_with_shift = tokens_end + shift_computed_tokens

    # Assert that we scheduled into the MM range (test setup verification)
    assert scheduled_end_with_shift > start_pos, (
        f"Test setup error: expected to schedule into MM range. "
        f"scheduled_end_with_shift={scheduled_end_with_shift}, "
        f"start_pos={start_pos}"
    )