Repository: vllm-project/vllm

Issue:
InternLM 20B Support
Traceback (most recent call last):
  File "/root/miniconda3/envs/vllm/lib/python3.8/runpy.py", line 194, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/root/miniconda3/envs/vllm/lib/python3.8/runpy.py", line 87, in _run_code
    exec(code, run_globals)
  File "/home/ubuntu/vllm/vllm/entrypoints/openai/api_server.py", line 613, in <module>
    engine = AsyncLLMEngine.from_engine_args(engine_args)
  File "/home/ubuntu/vllm/vllm/engine/async_llm_engine.py", line 487, in from_engine_args
    engine = cls(engine_args.worker_use_ray,
  File "/home/ubuntu/vllm/vllm/engine/async_llm_engine.py", line 270, in __init__
    self.engine = self._init_engine(*args, **kwargs)
  File "/home/ubuntu/vllm/vllm/engine/async_llm_engine.py", line 306, in _init_engine
    return engine_class(*args, **kwargs)
  File "/home/ubuntu/vllm/vllm/engine/llm_engine.py", line 113, in __init__
    self._init_cache()
  File "/home/ubuntu/vllm/vllm/engine/llm_engine.py", line 193, in _init_cache
    num_blocks = self._run_workers(
  File "/home/ubuntu/vllm/vllm/engine/llm_engine.py", line 704, in _run_workers
    all_outputs = ray.get(all_outputs)
  File "/root/miniconda3/envs/vllm/lib/python3.8/site-packages/ray/_private/auto_init_hook.py", line 24, in auto_init_wrapper
    return fn(*args, **kwargs)
  File "/root/miniconda3/envs/vllm/lib/python3.8/site-packages/ray/_private/client_mode_hook.py", line 103, in wrapper
    return func(*args, **kwargs)
  File "/root/miniconda3/envs/vllm/lib/python3.8/site-packages/ray/_private/worker.py", line 2547, in get
    raise value.as_instanceof_cause()
ray.exceptions.RayTaskError(RuntimeError): ray::RayWorker.execute_method() (pid=337309, ip=10.19.136.100, actor_id=436e9ee99d472a7e607b250401000000, repr=<vllm.engine.ray_utils.RayWorker object at 0x7fb71822abe0>)
  File "/home/ubuntu/vllm/vllm/engine/ray_utils.py", line 32, in execute_method
    return executor(*args, **kwargs)
  File "/root/miniconda3/envs/vllm/lib/python3.8/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
    return func(*args, **kwargs)
  File "/home/ubuntu/vllm/vllm/worker/worker.py", line 111, in profile_num_available_blocks
    self.model(
  File "/root/miniconda3/envs/vllm/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/ubuntu/vllm/vllm/model_executor/models/internlm.py", line 240, in forward
    next_tokens = self.sampler(self.lm_head.weight, hidden_states,
  File "/root/miniconda3/envs/vllm/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/ubuntu/vllm/vllm/model_executor/layers/sampler.py", line 86, in forward
    sample_results = _sample(probs, logprobs, input_metadata)
  File "/home/ubuntu/vllm/vllm/model_executor/layers/sampler.py", line 441, in _sample
    sample_results = _random_sample(seq_groups, is_prompts,
  File "/home/ubuntu/vllm/vllm/model_executor/layers/sampler.py", line 326, in _random_sample
    random_samples = torch.multinomial(probs,
RuntimeError: probability tensor contains either `inf`, `nan` or element < 0

When I run InternLM 20B Chat, here is the error.



Discussion:
I got the same error! T_T
may be vllm-0.1.7  can solve this error
#1496   shall fix this issue. I stated detailedly in the PR, maybe not the right way, so still need help.

Resolving patch:
diff --git a/vllm/model_executor/models/internlm.py b/vllm/model_executor/models/internlm.py
--- a/vllm/model_executor/models/internlm.py
+++ b/vllm/model_executor/models/internlm.py
@@ -62,6 +62,7 @@ def __init__(
         self,
         hidden_size: int,
         num_heads: int,
+        bias: bool,
         rope_theta: float = 10000,
         max_position_embeddings: int = 8192,
     ):
@@ -81,13 +82,13 @@ def __init__(
         self.qkv_proj = ColumnParallelLinear(
             hidden_size,
             3 * self.total_num_heads * self.head_dim,
-            bias=True,
+            bias=bias,
             gather_output=False,
         )
         self.o_proj = RowParallelLinear(
             self.total_num_heads * self.head_dim,
             hidden_size,
-            bias=True,
+            bias=bias,
             input_is_parallel=True,
         )
         self.attn = PagedAttentionWithRoPE(
@@ -126,6 +127,7 @@ def __init__(self, config: LlamaConfig):
         self.self_attn = InternLMAttention(
             hidden_size=self.hidden_size,
             num_heads=config.num_attention_heads,
+            bias=config.bias,
             rope_theta=rope_theta,
             max_position_embeddings=max_position_embeddings,
         )


Test patch: