Repository: open-compass/VLMEvalKit
Issue: RuntimeError: cutlassF: no kernel found to launch!
Root cause: Resource Mishandling

```
import os
import torch

os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"
# torch.backends.cuda.enable_mem_efficient_sdp(False)
# torch.backends.cuda.enable_flash_sdp(False)

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)
from vlmeval.config import supported_VLM
model = supported_VLM['idefics_9b_instruct']()

ret = model.generate('apple.jpg', 'What is in this image?')
```

输出错误信息
```

Some parameters are on the meta device device because they were offloaded to the cpu.
Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
[/mnt/workspace/VLMEvalKit/vlmeval/vlm/idefics.py:27](https://eais-notebook.data.aliyun.com/eais-bjzhygnopx1lreshkapv/lab/workspaces/auto-E/tree/VLMEvalKit/VLMEvalKit/vlmeval/vlm/idefics.py#line=26): UserWarning: Following kwargs received: {'max_new_tokens': 512}, will use as generation config. 
  warnings.warn(
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Cell In[2], line 13
     10 from vlmeval.config import supported_VLM
     11 model = supported_VLM['idefics_9b_instruct']()
---> 13 ret = model.generate('apple.jpg', 'What is in this image?')
     14 # ret: "The image features a red apple with a leaf on it."
     15 #ret = model.interleave_generate(
     16 #    ['apple.jpg', 'apple.jpg', 'How many apples are there in the provided images? ']
     17 #)
     18 # ret: "There are two apples in the provided images."

File [/mnt/workspace/VLMEvalKit/vlmeval/vlm/base.py:115](https://eais-notebook.data.aliyun.com/eais-bjzhygnopx1lreshkapv/lab/workspaces/auto-E/tree/VLMEvalKit/VLMEvalKit/vlmeval/vlm/base.py#line=114), in BaseModel.generate(self, message, dataset)
    113 for item in message:
    114     assert item['type'] in self.allowed_types, f'Invalid input type: {item["type"]}'
--> 115 return self.generate_inner(message, dataset)

File [/mnt/workspace/VLMEvalKit/vlmeval/vlm/idefics.py:47](https://eais-notebook.data.aliyun.com/eais-bjzhygnopx1lreshkapv/lab/workspaces/auto-E/tree/VLMEvalKit/VLMEvalKit/vlmeval/vlm/idefics.py#line=46), in IDEFICS.generate_inner(self, message, dataset)
     40 exit_condition = self.processor.tokenizer(
     41     '<end_of_utterance>', add_special_tokens=False
     42 ).input_ids
     43 bad_words_ids = self.processor.tokenizer(
     44     ['<image>', '<fake_token_around_image>'], add_special_tokens=False
     45 ).input_ids
---> 47 generated_ids = self.model.generate(
     48     **inputs,
     49     eos_token_id=exit_condition,
     50     bad_words_ids=bad_words_ids,
     51     **self.kwargs,
     52 )
     53 generated_text = self.processor.batch_decode(
     54     generated_ids, skip_special_tokens=True
     55 )
     56 text = generated_text[0].split('\nAssistant: ')[-1]

File [/usr/local/lib/python3.10/site-packages/torch/utils/_contextlib.py:115](https://eais-notebook.data.aliyun.com/usr/local/lib/python3.10/site-packages/torch/utils/_contextlib.py#line=114), in context_decorator.<locals>.decorate_context(*args, **kwargs)
    112 @functools.wraps(func)
    113 def decorate_context(*args, **kwargs):
    114     with ctx_factory():
--> 115         return func(*args, **kwargs)

File [/usr/local/lib/python3.10/site-packages/transformers/generation/utils.py:1914](https://eais-notebook.data.aliyun.com/usr/local/lib/python3.10/site-packages/transformers/generation/utils.py#line=1913), in GenerationMixin.generate(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, assistant_model, streamer, negative_prompt_ids, negative_prompt_attention_mask, **kwargs)
   1906     input_ids, model_kwargs = self._expand_inputs_for_generation(
   1907         input_ids=input_ids,
   1908         expand_size=generation_config.num_return_sequences,
   1909         is_encoder_decoder=self.config.is_encoder_decoder,
   1910         **model_kwargs,
   1911     )
   1913     # 13. run sample (it degenerates to greedy search when `generation_config.do_sample=False`)
-> 1914     result = self._sample(
   1915         input_ids,
   1916         logits_processor=prepared_logits_processor,
   1917         logits_warper=prepared_logits_warper,
   1918         stopping_criteria=prepared_stopping_criteria,
   1919         generation_config=generation_config,
   1920         synced_gpus=synced_gpus,
   1921         streamer=streamer,
   1922         **model_kwargs,
   1923     )
   1925 elif generation_mode in (GenerationMode.BEAM_SAMPLE, GenerationMode.BEAM_SEARCH):
   1926     # 11. prepare logits warper
   1927     prepared_logits_warper = (
   1928         self._get_logits_warper(generation_config, device=input_ids.device)
   1929         if generation_config.do_sample
   1930         else None
   1931     )

File [/usr/local/lib/python3.10/site-packages/transformers/generation/utils.py:2651](https://eais-notebook.data.aliyun.com/usr/local/lib/python3.10/site-packages/transformers/generation/utils.py#line=2650), in GenerationMixin._sample(self, input_ids, logits_processor, stopping_criteria, generation_config, synced_gpus, streamer, logits_warper, **model_kwargs)
   2648 model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs)
   2650 # forward pass to get next token
-> 2651 outputs = self(
   2652     **model_inputs,
   2653     return_dict=True,
   2654     output_attentions=output_attentions,
   2655     output_hidden_states=output_hidden_states,
   2656 )
   2658 if synced_gpus and this_peer_finished:
   2659     continue  # don't waste resources running the code we don't need

File [/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py:1518](https://eais-notebook.data.aliyun.com/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py#line=1517), in Module._wrapped_call_impl(self, *args, **kwargs)
   1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517 else:
-> 1518     return self._call_impl(*args, **kwargs)

File [/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py:1527](https://eais-notebook.data.aliyun.com/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py#line=1526), in Module._call_impl(self, *args, **kwargs)
   1522 # If we don't have any hooks, we want to skip the rest of the logic in
   1523 # this function, and just call forward.
   1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1525         or _global_backward_pre_hooks or _global_backward_hooks
   1526         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527     return forward_call(*args, **kwargs)
   1529 try:
   1530     result = None

File [/usr/local/lib/python3.10/site-packages/accelerate/hooks.py:169](https://eais-notebook.data.aliyun.com/usr/local/lib/python3.10/site-packages/accelerate/hooks.py#line=168), in add_hook_to_module.<locals>.new_forward(module, *args, **kwargs)
    167         output = module._old_forward(*args, **kwargs)
    168 else:
--> 169     output = module._old_forward(*args, **kwargs)
    170 return module._hf_hook.post_forward(module, output)

File [/usr/local/lib/python3.10/site-packages/transformers/models/idefics/modeling_idefics.py:1493](https://eais-notebook.data.aliyun.com/usr/local/lib/python3.10/site-packages/transformers/models/idefics/modeling_idefics.py#line=1492), in IdeficsForVisionText2Text.forward(self, input_ids, attention_mask, position_ids, past_key_values, inputs_embeds, pixel_values, image_encoder_embeddings, perceiver_embeddings, image_attention_mask, labels, use_cache, output_attentions, output_hidden_states, interpolate_pos_encoding, return_dict)
   1490 return_dict = return_dict if return_dict is not None else self.config.use_return_dict
   1492 # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
-> 1493 outputs = self.model(
   1494     input_ids=input_ids,
   1495     attention_mask=attention_mask,
   1496     position_ids=position_ids,
   1497     past_key_values=past_key_values,
   1498     inputs_embeds=inputs_embeds,
   1499     pixel_values=pixel_values,
   1500     image_encoder_embeddings=image_encoder_embeddings,
   1501     perceiver_embeddings=perceiver_embeddings,
   1502     image_attention_mask=image_attention_mask,
   1503     use_cache=use_cache,
   1504     output_attentions=output_attentions,
   1505     output_hidden_states=output_hidden_states,
   1506     interpolate_pos_encoding=interpolate_pos_encoding,
   1507     return_dict=return_dict,
   1508 )
   1510 hidden_states = outputs[0]
   1511 logits = self.lm_head(hidden_states)

File [/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py:1518](https://eais-notebook.data.aliyun.com/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py#line=1517), in Module._wrapped_call_impl(self, *args, **kwargs)
   1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517 else:
-> 1518     return self._call_impl(*args, **kwargs)

File [/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py:1527](https://eais-notebook.data.aliyun.com/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py#line=1526), in Module._call_impl(self, *args, **kwargs)
   1522 # If we don't have any hooks, we want to skip the rest of the logic in
   1523 # this function, and just call forward.
   1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1525         or _global_backward_pre_hooks or _global_backward_hooks
   1526         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527     return forward_call(*args, **kwargs)
   1529 try:
   1530     result = None

File [/usr/local/lib/python3.10/site-packages/transformers/models/idefics/modeling_idefics.py:1326](https://eais-notebook.data.aliyun.com/usr/local/lib/python3.10/site-packages/transformers/models/idefics/modeling_idefics.py#line=1325), in IdeficsModel.forward(self, input_ids, attention_mask, position_ids, past_key_values, inputs_embeds, pixel_values, image_encoder_embeddings, perceiver_embeddings, image_attention_mask, use_cache, output_attentions, output_hidden_states, interpolate_pos_encoding, return_dict)
   1309     layer_outputs = self._gradient_checkpointing_func(
   1310         vblock,
   1311         decoder_layer,
   (...)
   1323         self.gated_cross_attn_layers,
   1324     )
   1325 else:
-> 1326     layer_outputs = vblock(
   1327         decoder_layer,
   1328         hidden_states,
   1329         attention_mask=attention_mask,
   1330         position_ids=position_ids,
   1331         past_key_value=past_key_value,
   1332         image_hidden_states=image_hidden_states,
   1333         image_attention_mask=image_attention_mask,
   1334         cross_attention_gate=cross_attention_gate,
   1335         output_attentions=output_attentions,
   1336         use_cache=use_cache,
   1337         layer_idx=idx,
   1338         cross_layer_interval=self.cross_layer_interval,
   1339         gated_cross_attn_layers=self.gated_cross_attn_layers,
   1340     )
   1342 hidden_states = layer_outputs[0]
   1344 if use_cache:

File [/usr/local/lib/python3.10/site-packages/transformers/models/idefics/modeling_idefics.py:1278](https://eais-notebook.data.aliyun.com/usr/local/lib/python3.10/site-packages/transformers/models/idefics/modeling_idefics.py#line=1277), in IdeficsModel.forward.<locals>.vblock(main_block, hidden_states, attention_mask, position_ids, past_key_value, image_hidden_states, image_attention_mask, cross_attention_gate, output_attentions, use_cache, layer_idx, cross_layer_interval, gated_cross_attn_layers)
   1276 if layer_idx % cross_layer_interval == 0:
   1277     xblock = gated_cross_attn_layers[layer_idx // cross_layer_interval]
-> 1278     outputs = xblock(
   1279         hidden_states,
   1280         attention_mask=attention_mask,
   1281         image_hidden_states=image_hidden_states,
   1282         image_attention_mask=image_attention_mask,
   1283         cross_attention_gate=cross_attention_gate,
   1284         output_attentions=output_attentions,
   1285         use_cache=use_cache,
   1286         past_key_value=None,  # not implemented
   1287     )
   1288     hidden_states = outputs[0]
   1290 layer_outputs = main_block(
   1291     hidden_states,
   1292     attention_mask=attention_mask,
   (...)
   1296     use_cache=use_cache,
   1297 )

File /usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
   1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517 else:
-> 1518     return self._call_impl(*args, **kwargs)

File [/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py:1527](https://eais-notebook.data.aliyun.com/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py#line=1526), in Module._call_impl(self, *args, **kwargs)
   1522 # If we don't have any hooks, we want to skip the rest of the logic in
   1523 # this function, and just call forward.
   1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1525         or _global_backward_pre_hooks or _global_backward_hooks
   1526         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527     return forward_call(*args, **kwargs)
   1529 try:
   1530     result = None

File [/usr/local/lib/python3.10/site-packages/accelerate/hooks.py:169](https://eais-notebook.data.aliyun.com/usr/local/lib/python3.10/site-packages/accelerate/hooks.py#line=168), in add_hook_to_module.<locals>.new_forward(module, *args, **kwargs)
    167         output = module._old_forward(*args, **kwargs)
    168 else:
--> 169     output = module._old_forward(*args, **kwargs)
    170 return module._hf_hook.post_forward(module, output)

File [/usr/local/lib/python3.10/site-packages/transformers/models/idefics/modeling_idefics.py:888](https://eais-notebook.data.aliyun.com/usr/local/lib/python3.10/site-packages/transformers/models/idefics/modeling_idefics.py#line=887), in IdeficsGatedCrossAttentionLayer.forward(self, hidden_states, attention_mask, image_hidden_states, image_attention_mask, cross_attention_gate, output_attentions, use_cache, past_key_value)
    885 hidden_states = self.input_layernorm(hidden_states)
    887 # Self Attention
--> 888 hidden_states, self_attn_weights, present_key_value = self.cross_attn(
    889     hidden_states=hidden_states,
    890     key_value_states=image_hidden_states,
    891     attention_mask=image_attention_mask,
    892     output_attentions=output_attentions,
    893 )
    894 hidden_states = nn.functional.dropout(hidden_states, p=self.config, training=self.training)
    895 # Fill in zeros for cross_attention hidden_states of tokens attending to no images

File [/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py:1518](https://eais-notebook.data.aliyun.com/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py#line=1517), in Module._wrapped_call_impl(self, *args, **kwargs)
   1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517 else:
-> 1518     return self._call_impl(*args, **kwargs)

File [/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py:1527](https://eais-notebook.data.aliyun.com/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py#line=1526), in Module._call_impl(self, *args, **kwargs)
   1522 # If we don't have any hooks, we want to skip the rest of the logic in
   1523 # this function, and just call forward.
   1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1525         or _global_backward_pre_hooks or _global_backward_hooks
   1526         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527     return forward_call(*args, **kwargs)
   1529 try:
   1530     result = None

File [/usr/local/lib/python3.10/site-packages/accelerate/hooks.py:169](https://eais-notebook.data.aliyun.com/usr/local/lib/python3.10/site-packages/accelerate/hooks.py#line=168), in add_hook_to_module.<locals>.new_forward(module, *args, **kwargs)
    167         output = module._old_forward(*args, **kwargs)
    168 else:
--> 169     output = module._old_forward(*args, **kwargs)
    170 return module._hf_hook.post_forward(module, output)

File [/usr/local/lib/python3.10/site-packages/transformers/models/idefics/modeling_idefics.py:669](https://eais-notebook.data.aliyun.com/usr/local/lib/python3.10/site-packages/transformers/models/idefics/modeling_idefics.py#line=668), in IdeficsAttention.forward(self, hidden_states, key_value_states, attention_mask, position_ids, past_key_value, output_attentions, use_cache)
    664 # We dispatch to SDPA's Flash Attention or Efficient kernels via this `is_causal` if statement instead of an inline conditional assignment
    665 # in SDPA to support both torch.compile's dynamic shapes and full graph options. An inline conditional prevents dynamic shapes from compiling.
    666 # The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d that does not create a causal mask in case q_len == 1.
    667 is_causal = True if self.is_causal and attention_mask is None and q_len > 1 else False
--> 669 attn_output = torch.nn.functional.scaled_dot_product_attention(
    670     query_states,
    671     key_states,
    672     value_states,
    673     attn_mask=attention_mask,
    674     dropout_p=self.dropout if self.training else 0.0,
    675     is_causal=is_causal,
    676 )
    678 if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
    679     raise ValueError(
    680         f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
    681         f" {attn_output.size()}"
    682     )

RuntimeError: cutlassF: no kernel found to launch!
```

Hi, @liuchunming033. 请检查下是否有足够的显存运行模型

Close due to no response in a long time. Feel free to reopen it if needed.