Repository: open-compass/VLMEvalKit
Issue: qwen2.5vl无法使用多个GPU：Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cuda:1! (when checking argument for argument mat2 in method wrapper_CUDA_bmm)
Root cause: Resource Mishandling

使用命令行运行：torchrun --nproc-per-node=2 */VLMEvalKit/run.py --data MathVista_MINI --model Qwen2.5-VL-7B --work-dir */VLMEvalKit/results --verbose

遇到错误：
[2025-02-20 11:03:36] ERROR - run.py: main - 433: Model Qwen2.5-VL-7B x Dataset MathVista_MINI combination failed: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cuda:1! (when checking argument for argument mat2 in method wrapper_CUDA_bmm), skipping this combination.
Traceback (most recent call last):
  File "*/VLMEvalKit/run.py", line 316, in main
    model = infer_data_job(
  File "*/VLMEvalKit/vlmeval/inference.py", line 165, in infer_data_job
    model = infer_data(
  File "*/VLMEvalKit/vlmeval/inference.py", line 130, in infer_data
    response = model.generate(message=struct, dataset=dataset_name)
  File "*/VLMEvalKit/vlmeval/vlm/base.py", line 115, in generate
    return self.generate_inner(message, dataset)
  File "*/VLMEvalKit/vlmeval/vlm/qwen2_vl/model.py", line 197, in generate_inner
    generated_ids = self.model.generate(
  File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 115, in decorate_context
    return func(*args, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py", line 2227, in generate
    result = self._sample(
  File "/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py", line 3215, in _sample
    outputs = self(**model_inputs, return_dict=True)
  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/accelerate/hooks.py", line 170, in new_forward
    output = module._old_forward(*args, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py", line 1863, in forward
    outputs = self.model(
  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py", line 1184, in forward
    position_embeddings = self.rotary_emb(hidden_states, position_ids)
  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/accelerate/hooks.py", line 170, in new_forward
    output = module._old_forward(*args, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 115, in decorate_context
    return func(*args, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py", line 616, in forward
    freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(2, 3)
RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cuda:1! (when checking argument for argument mat2 in method wrapper_CUDA_bmm)

运行的代码是vlmeval/vlm/qwen2_vl/model.py，以下这个版本：
class Qwen2VLChat(Qwen2VLPromptMixin, BaseModel):
    INSTALL_REQ = False
    INTERLEAVE = True
    VIDEO_LLM = True

    def __init__(
        self,
        model_path: str,
        min_pixels: int | None = None,
        max_pixels: int | None = None,
        max_new_tokens=2048,
        top_p=0.001,
        top_k=1,
        temperature=0.01,
        repetition_penalty=1.0,
        use_custom_prompt: bool = True,
        system_prompt: str | None = None,
        post_process: bool = False,  # if True, will try to only extract stuff in the last \boxed{}.
        verbose: bool = False,
    ):
        super().__init__(use_custom_prompt=use_custom_prompt)
        self.min_pixels = min_pixels
        self.max_pixels = max_pixels
        self.generate_kwargs = dict(
            max_new_tokens=max_new_tokens,
            top_p=top_p,
            top_k=top_k,
            temperature=temperature,
            repetition_penalty=repetition_penalty,
        )
        self.system_prompt = system_prompt
        self.verbose = verbose
        self.post_process = post_process
        self.fps = 2.0
        self.nframe = 64
        self.FRAME_FACTOR = 2
        rank, world_size = get_rank_and_world_size()
        assert model_path is not None
        self.model_path = model_path
        MODEL_CLS = None  

        if '2.5' in model_path:
            from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessor
            MODEL_CLS = Qwen2_5_VLForConditionalGeneration
            self.processor = AutoProcessor.from_pretrained(model_path)
        else:
            from transformers import Qwen2VLForConditionalGeneration, Qwen2VLProcessor
            MODEL_CLS = Qwen2VLForConditionalGeneration
            self.processor = Qwen2VLProcessor.from_pretrained(model_path)

        gpu_mems = get_gpu_memory()
        print(gpu_mems)
        max_gpu_mem = max(gpu_mems) if gpu_mems != [] else -1
        assert max_gpu_mem > 0

        # If only one process and GPU memory is less than 40GB
        if '72b' in self.model_path.lower():
            self.model = MODEL_CLS.from_pretrained(
                # model_path, torch_dtype='auto', device_map=split_model(), attn_implementation='flash_attention_2'
                model_path, torch_dtype='auto', device_map=split_model()
            )
            self.model.eval()
        elif auto_split_flag():
            assert world_size == 1, 'Only support world_size == 1 when AUTO_SPLIT is set for non-72B Qwen2-VL'
            # Will Use All GPUs to run one model
            self.model = MODEL_CLS.from_pretrained(
                # model_path, torch_dtype='auto', device_map='auto', attn_implementation='flash_attention_2'
                model_path, torch_dtype='auto', device_map='auto'
            )
        else:
            self.model = MODEL_CLS.from_pretrained(
                # model_path, torch_dtype='auto', device_map='cpu', attn_implementation='flash_attention_2'
                model_path, torch_dtype='auto', device_map='auto'
            )
            self.model.cuda().eval()

咋解决的哥


> 咋解决的哥

--nproc-per-node=1就行，会分布在多个gpu上计算

> > 咋解决的哥
> 
> --nproc-per-node=1就行，会分布在多个gpu上计算

I tried but still couldn't

auto_split_flag在哪定义的呀，我运行报错ImportError: cannot import name 'auto_split_flag' from 'vlmeval.smp'