---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Cell In[4], line 7
      4 from accelerate import dispatch_model
      6 device_map = auto_configure_device_map(2)
----> 7 model = dispatch_model(model, device_map=device_map)
      9 tokenizer = AutoTokenizer.from_pretrained('internlm/internlm-xcomposer2-vl-7b', trust_remote_code=True)
     11 text = '<ImageHere>Please describe this image in detail.'

File /opt/conda/lib/python3.10/site-packages/accelerate/big_modeling.py:419, in dispatch_model(model, device_map, main_device, state_dict, offload_dir, offload_index, offload_buffers, skip_keys, preload_module_classes, force_hooks)
    414         tied_params_map[data_ptr] = {}
    416         # Note: To handle the disk offloading case, we can not simply use weights_map[param_name].data_ptr() as the reference pointer,
    417         # as we have no guarantee that safetensors' `file.get_tensor()` will always give the same pointer.
--> 419 attach_align_device_hook_on_blocks(
    420     model,
    421     execution_device=execution_device,
    422     offload=offload,
    423     offload_buffers=offload_buffers,
    424     weights_map=weights_map,
    425     skip_keys=skip_keys,
    426     preload_module_classes=preload_module_classes,
    427     tied_params_map=tied_params_map,
    428 )
    430 # warn if there is any params on the meta device
    431 offloaded_devices_str = " and ".join(
    432     [device for device in set(device_map.values()) if device in ("cpu", "disk")]
    433 )

File /opt/conda/lib/python3.10/site-packages/accelerate/hooks.py:648, in attach_align_device_hook_on_blocks(module, execution_device, offload, weights_map, offload_buffers, module_name, skip_keys, preload_module_classes, tied_params_map)
    646 for child_name, child in module.named_children():
    647     child_name = f"{module_name}.{child_name}" if len(module_name) > 0 else child_name
--> 648     attach_align_device_hook_on_blocks(
    649         child,
    650         execution_device=execution_device,
    651         offload=offload,
    652         weights_map=weights_map,
    653         offload_buffers=offload_buffers,
    654         module_name=child_name,
    655         preload_module_classes=preload_module_classes,
    656         skip_keys=skip_keys,
    657         tied_params_map=tied_params_map,
    658     )

File /opt/conda/lib/python3.10/site-packages/accelerate/hooks.py:648, in attach_align_device_hook_on_blocks(module, execution_device, offload, weights_map, offload_buffers, module_name, skip_keys, preload_module_classes, tied_params_map)
    646 for child_name, child in module.named_children():
    647     child_name = f"{module_name}.{child_name}" if len(module_name) > 0 else child_name
--> 648     attach_align_device_hook_on_blocks(
    649         child,
    650         execution_device=execution_device,
    651         offload=offload,
    652         weights_map=weights_map,
    653         offload_buffers=offload_buffers,
    654         module_name=child_name,
    655         preload_module_classes=preload_module_classes,
    656         skip_keys=skip_keys,
    657         tied_params_map=tied_params_map,
    658     )

File /opt/conda/lib/python3.10/site-packages/accelerate/hooks.py:648, in attach_align_device_hook_on_blocks(module, execution_device, offload, weights_map, offload_buffers, module_name, skip_keys, preload_module_classes, tied_params_map)
    646 for child_name, child in module.named_children():
    647     child_name = f"{module_name}.{child_name}" if len(module_name) > 0 else child_name
--> 648     attach_align_device_hook_on_blocks(
    649         child,
    650         execution_device=execution_device,
    651         offload=offload,
    652         weights_map=weights_map,
    653         offload_buffers=offload_buffers,
    654         module_name=child_name,
    655         preload_module_classes=preload_module_classes,
    656         skip_keys=skip_keys,
    657         tied_params_map=tied_params_map,
    658     )

File /opt/conda/lib/python3.10/site-packages/accelerate/hooks.py:608, in attach_align_device_hook_on_blocks(module, execution_device, offload, weights_map, offload_buffers, module_name, skip_keys, preload_module_classes, tied_params_map)
    599 if module_name in execution_device and module_name in offload and not offload[module_name]:
    600     hook = AlignDevicesHook(
    601         execution_device=execution_device[module_name],
    602         offload_buffers=offload_buffers,
   (...)
    606         tied_params_map=tied_params_map,
    607     )
--> 608     add_hook_to_module(module, hook)
    609     attach_execution_device_hook(module, execution_device[module_name], tied_params_map=tied_params_map)
    610 elif module_name in execution_device and module_name in offload:

File /opt/conda/lib/python3.10/site-packages/accelerate/hooks.py:157, in add_hook_to_module(module, hook, append)
    154     old_forward = module.forward
    155     module._old_forward = old_forward
--> 157 module = hook.init_hook(module)
    158 module._hf_hook = hook
    160 def new_forward(module, *args, **kwargs):

File /opt/conda/lib/python3.10/site-packages/accelerate/hooks.py:275, in AlignDevicesHook.init_hook(self, module)
    273 if not self.offload and self.execution_device is not None:
    274     for name, _ in named_module_tensors(module, recurse=self.place_submodules):
--> 275         set_module_tensor_to_device(module, name, self.execution_device, tied_params_map=self.tied_params_map)
    276 elif self.offload:
    277     self.original_devices = {
    278         name: param.device for name, param in named_module_tensors(module, recurse=self.place_submodules)
    279     }

File /opt/conda/lib/python3.10/site-packages/accelerate/utils/modeling.py:457, in set_module_tensor_to_device(module, tensor_name, device, value, dtype, fp16_statistics, tied_params_map)
    455     torch.xpu.empty_cache()
    456 else:
--> 457     torch.cuda.empty_cache()
    459 # When handling tied weights, we update tied_params_map to keep track of the tied weights that have already been allocated on the device in
    460 # order to avoid duplicating memory, see above.
    461 if (
    462     tied_params_map is not None
    463     and old_value.data_ptr() in tied_params_map
    464     and device not in tied_params_map[old_value.data_ptr()]
    465 ):

File /opt/conda/lib/python3.10/site-packages/torch/cuda/memory.py:159, in empty_cache()
    148 r"""Releases all unoccupied cached memory currently held by the caching
    149 allocator so that those can be used in other GPU application and visible in
    150 `nvidia-smi`.
   (...)
    156     more details about GPU memory management.
    157 """
    158 if is_initialized():
--> 159     torch._C._cuda_emptyCache()

RuntimeError: CUDA error: unspecified launch failure
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions