4bit Blip2 compatibility
Author: betterftr
Pull request: no

### System Info

I am getting an error after loading Blip2 in 4bit, cant inference, cant train. Can anyone help?


### Who can help?

_No response_

### Information

- [ ] The official example scripts
- [ ] My own modified scripts

### Tasks

- [ ] An officially supported task in the `examples` folder (such as GLUE/SQuAD, ...)
- [ ] My own task or dataset (give details below)

### Reproduction

`import torch
from transformers import Blip2ForConditionalGeneration, AutoProcessor, Blip2Processor, AutoModelForCausalLM, BitsAndBytesConfig
from peft import prepare_model_for_kbit_training

#processor = AutoProcessor.from_pretrained("Salesforce/blip2-opt-6.7b-coco")
#model = Blip2ForConditionalGeneration.from_pretrained("Salesforce/blip2-opt-6.7b-coco", device_map='auto', load_in_8bit=True)

nf4_config = BitsAndBytesConfig(
   load_in_4bit=True,
   bnb_4bit_quant_type="nf4",
   bnb_4bit_use_double_quant=True,
   bnb_4bit_compute_dtype=torch.bfloat16
)

processor = AutoProcessor.from_pretrained("Salesforce/blip2-opt-6.7b-coco")
model = Blip2ForConditionalGeneration.from_pretrained("Salesforce/blip2-opt-6.7b-coco", device_map='auto', quantization_config=nf4_config)`

Then when I want to train with PEFT or just do a single image captioning with the loaded model I get:

`FP4 quantization state not initialized. Please call .cuda() or .to(device) on the LinearFP4 layer first.
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
Cell In[10], line 6
      3 pixel_values = inputs.pixel_values
      5 #generated_ids = model.generate(pixel_values=pixel_values, min_length=50, max_new_tokens=50, length_penalty=1.4, top_k=150, top_p=0.95, repetition_penalty=2.1, num_beams=5, temperature=0.75)
----> 6 generated_ids = model.generate(pixel_values=pixel_values, max_length=50)
      7 generated_caption = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
      8 print(generated_caption)

File H:\CONDA\envs\blip\lib\site-packages\torch\utils\_contextlib.py:115, in context_decorator.<locals>.decorate_context(*args, **kwargs)
    112 @functools.wraps(func)
    113 def decorate_context(*args, **kwargs):
    114     with ctx_factory():
--> 115         return func(*args, **kwargs)

File H:\CONDA\envs\blip\lib\site-packages\transformers\models\blip_2\modeling_blip_2.py:1854, in Blip2ForConditionalGeneration.generate(self, pixel_values, input_ids, attention_mask, **generate_kwargs)
   1851 inputs_embeds = self.get_input_embeddings()(input_ids)
   1852 inputs_embeds = torch.cat([language_model_inputs, inputs_embeds.to(language_model_inputs.device)], dim=1)
-> 1854 outputs = self.language_model.generate(
   1855     inputs_embeds=inputs_embeds,
   1856     attention_mask=attention_mask,
   1857     **generate_kwargs,
   1858 )
   1860 return outputs

File H:\CONDA\envs\blip\lib\site-packages\torch\utils\_contextlib.py:115, in context_decorator.<locals>.decorate_context(*args, **kwargs)
    112 @functools.wraps(func)
    113 def decorate_context(*args, **kwargs):
    114     with ctx_factory():
--> 115         return func(*args, **kwargs)

File H:\CONDA\envs\blip\lib\site-packages\transformers\generation\utils.py:1518, in GenerationMixin.generate(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, assistant_model, streamer, **kwargs)
   1512         raise ValueError(
   1513             "num_return_sequences has to be 1 when doing greedy search, "
   1514             f"but is {generation_config.num_return_sequences}."
   1515         )
   1517     # 11. run greedy search
-> 1518     return self.greedy_search(
   1519         input_ids,
   1520         logits_processor=logits_processor,
   1521         stopping_criteria=stopping_criteria,
   1522         pad_token_id=generation_config.pad_token_id,
   1523         eos_token_id=generation_config.eos_token_id,
   1524         output_scores=generation_config.output_scores,
   1525         return_dict_in_generate=generation_config.return_dict_in_generate,
   1526         synced_gpus=synced_gpus,
   1527         streamer=streamer,
   1528         **model_kwargs,
   1529     )
   1531 elif is_contrastive_search_gen_mode:
   1532     if generation_config.num_return_sequences > 1:

File H:\CONDA\envs\blip\lib\site-packages\transformers\generation\utils.py:2335, in GenerationMixin.greedy_search(self, input_ids, logits_processor, stopping_criteria, max_length, pad_token_id, eos_token_id, output_attentions, output_hidden_states, output_scores, return_dict_in_generate, synced_gpus, streamer, **model_kwargs)
   2332 model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs)
   2334 # forward pass to get next token
-> 2335 outputs = self(
   2336     **model_inputs,
   2337     return_dict=True,
   2338     output_attentions=output_attentions,
   2339     output_hidden_states=output_hidden_states,
   2340 )
   2342 if synced_gpus and this_peer_finished:
   2343     continue  # don't waste resources running the code we don't need

File H:\CONDA\envs\blip\lib\site-packages\torch\nn\modules\module.py:1501, in Module._call_impl(self, *args, **kwargs)
   1496 # If we don't have any hooks, we want to skip the rest of the logic in
   1497 # this function, and just call forward.
   1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1499         or _global_backward_pre_hooks or _global_backward_hooks
   1500         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501     return forward_call(*args, **kwargs)
   1502 # Do not call functions when jit is used
   1503 full_backward_hooks, non_full_backward_hooks = [], []

File H:\CONDA\envs\blip\lib\site-packages\accelerate\hooks.py:165, in add_hook_to_module.<locals>.new_forward(*args, **kwargs)
    163         output = old_forward(*args, **kwargs)
    164 else:
--> 165     output = old_forward(*args, **kwargs)
    166 return module._hf_hook.post_forward(module, output)

File H:\CONDA\envs\blip\lib\site-packages\transformers\models\opt\modeling_opt.py:957, in OPTForCausalLM.forward(self, input_ids, attention_mask, head_mask, past_key_values, inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict)
    944 # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
    945 outputs = self.model.decoder(
    946     input_ids=input_ids,
    947     attention_mask=attention_mask,
   (...)
    954     return_dict=return_dict,
    955 )
--> 957 logits = self.lm_head(outputs[0]).contiguous()
    959 loss = None
    960 if labels is not None:
    961     # move labels to correct device to enable model parallelism

File H:\CONDA\envs\blip\lib\site-packages\torch\nn\modules\module.py:1501, in Module._call_impl(self, *args, **kwargs)
   1496 # If we don't have any hooks, we want to skip the rest of the logic in
   1497 # this function, and just call forward.
   1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1499         or _global_backward_pre_hooks or _global_backward_hooks
   1500         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501     return forward_call(*args, **kwargs)
   1502 # Do not call functions when jit is used
   1503 full_backward_hooks, non_full_backward_hooks = [], []

File H:\CONDA\envs\blip\lib\site-packages\accelerate\hooks.py:165, in add_hook_to_module.<locals>.new_forward(*args, **kwargs)
    163         output = old_forward(*args, **kwargs)
    164 else:
--> 165     output = old_forward(*args, **kwargs)
    166 return module._hf_hook.post_forward(module, output)

File H:\CONDA\envs\blip\lib\site-packages\bitsandbytes\nn\modules.py:219, in Linear4bit.forward(self, x)
    216     x = x.to(self.compute_dtype)
    218 bias = None if self.bias is None else self.bias.to(self.compute_dtype)
--> 219 out = bnb.matmul_4bit(x, self.weight.t(), bias=bias, quant_state=self.weight.quant_state)
    221 out = out.to(inp_dtype)
    223 return out

AttributeError: 'Parameter' object has no attribute 'quant_state'`



### Expected behavior

8 bit works fine

Comments:
cc @younesbelkada 

Hi @betterftr 
Thanks for the issue, indeed there seems to be a bug, that should be fixed in https://github.com/huggingface/transformers/pull/23895

cc @SunMarc if you have time to look into this ? 

Hey @betterftr , i'm able to run the following script with this env. Let me know if this works on your side. 
- `transformers` version: 4.34.0.dev0 (main branch)
- `accelerate` version: 0.23
- `bitsandbytes` version: 0.41.1
``` py 
import torch
from transformers import Blip2ForConditionalGeneration, Blip2Processor, BitsAndBytesConfig
from PIL import Image
import requests

nf4_config = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_quant_type="nf4",
bnb_4bit_use_double_quant=True,
bnb_4bit_compute_dtype=torch.float16
)

processor = Blip2Processor.from_pretrained("Salesforce/blip2-opt-6.7b-coco")
model = Blip2ForConditionalGeneration.from_pretrained("Salesforce/blip2-opt-6.7b-coco", device_map='auto', quantization_config=nf4_config)

def prepare_img():
    url = "https://huggingface.co/hf-internal-testing/blip-test-image/resolve/main/demo.jpg"
    image = Image.open(requests.get(url, stream=True).raw)
    return image

image = prepare_img()
inputs = processor(images=[image, image], return_tensors="pt").to(dtype=torch.float16)

predictions = model.generate(**inputs, num_beams=2)
print(processor.batch_decode(predictions, skip_special_tokens=True)[0].strip())
# print -> a woman sitting on the beach with her dog
```

This issue has been automatically marked as stale because it has not had recent activity. If you think this still needs to be addressed please comment on this thread.

Please note that issues that do not follow the [contributing guidelines](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md) are likely to be ignored.

@SunMarc  I am getting similar error while using llma2 7b model, and I am using the latest version of transformers


here is the code
```
from transformers import AutoTokenizer, set_seed, BitsAndBytesConfig, AutoTokenizer, AutoModelForCausalLM
bnb_config = BitsAndBytesConfig(
        load_in_4bit=True,
        bnb_4bit_use_double_quant=True,
        bnb_4bit_quant_type="nf4",
        bnb_4bit_compute_dtype=torch.float16,
    )
model_name = 'llm-models/Llama-2-7b-hf'
model = AutoModelForCausalLM.from_pretrained(
    model_name,  
    quantization_config=bnb_config,
    device_map="cuda",
    trust_remote_code=True,
)
tokenizer = AutoTokenizer.from_pretrained(model_name, device_map='cuda')
def generate_text(prompt):
    # Tokenize the prompt
    inputs = tokenizer.encode(prompt, return_tensors='pt')
    
    print(f'inputs is {inputs} on {inputs.device}')
    
    inputs = inputs.to('cuda:0')
    
    print(f'inputs is {inputs} on {inputs.device}')
    
    # Generate a response
    outputs = model.generate(inputs)
    
    # Decode the response
    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
    
    return response

prompt = 'User1: Hey, I need a new laptop. Which one should I buy?'
response = generate_text(prompt)
print(response)
```
**package info**
transformers==4.38.1
accelerate==0.21.0
bitsandbytes==0.42.0

I also tried 4.34, it doesn't work either. Besides that, I check this [PR](https://github.com/huggingface/transformers/pull/23895), it doesn't look like that it is in any of the release branch nor the maser branch

here is the error I get

FP4 quantization state not initialized. Please call .cuda() or .to(device) on the LinearFP4 layer first.
---------------------------------------------------------------------------
AssertionError                            Traceback (most recent call last)
Cell In[18], line 2
      1 prompt = 'User1: Hey, I need a new laptop. Which one should I buy?'
----> 2 response = generate_text(prompt)
      3 print(response)

Cell In[17], line 13, in generate_text(prompt)
     10 print(f'inputs is {inputs} on {inputs.device}')
     12 # Generate a response
---> 13 outputs = model.generate(inputs)
     15 # Decode the response
     16 response = tokenizer.decode(outputs[0], skip_special_tokens=True)

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/torch/utils/_contextlib.py:115, in context_decorator.<locals>.decorate_context(*args, **kwargs)
    112 @functools.wraps(func)
    113 def decorate_context(*args, **kwargs):
    114     with ctx_factory():
--> 115         return func(*args, **kwargs)

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/transformers/generation/utils.py:1345, in GenerationMixin.generate(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, assistant_model, streamer, **kwargs)
   1337         logger.warning(
   1338             "A decoder-only architecture is being used, but right-padding was detected! For correct "
   1339             "generation results, please set `padding_side='left'` when initializing the tokenizer."
   1340         )
   1342 if self.config.is_encoder_decoder and "encoder_outputs" not in model_kwargs:
   1343     # if model is encoder decoder encoder_outputs are created
   1344     # and added to `model_kwargs`
-> 1345     model_kwargs = self._prepare_encoder_decoder_kwargs_for_generation(
   1346         inputs_tensor, model_kwargs, model_input_name
   1347     )
   1349 # 5. Prepare `input_ids` which will be used for auto-regressive generation
   1350 if self.config.is_encoder_decoder:

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/transformers/generation/utils.py:644, in GenerationMixin._prepare_encoder_decoder_kwargs_for_generation(self, inputs_tensor, model_kwargs, model_input_name)
    642 encoder_kwargs["return_dict"] = True
    643 encoder_kwargs[model_input_name] = inputs_tensor
--> 644 model_kwargs["encoder_outputs"]: ModelOutput = encoder(**encoder_kwargs)
    646 return model_kwargs

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/torch/nn/modules/module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
   1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517 else:
-> 1518     return self._call_impl(*args, **kwargs)

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/torch/nn/modules/module.py:1527, in Module._call_impl(self, *args, **kwargs)
   1522 # If we don't have any hooks, we want to skip the rest of the logic in
   1523 # this function, and just call forward.
   1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1525         or _global_backward_pre_hooks or _global_backward_hooks
   1526         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527     return forward_call(*args, **kwargs)
   1529 try:
   1530     result = None

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/accelerate/hooks.py:165, in add_hook_to_module.<locals>.new_forward(*args, **kwargs)
    163         output = old_forward(*args, **kwargs)
    164 else:
--> 165     output = old_forward(*args, **kwargs)
    166 return module._hf_hook.post_forward(module, output)

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py:1094, in T5Stack.forward(self, input_ids, attention_mask, encoder_hidden_states, encoder_attention_mask, inputs_embeds, head_mask, cross_attn_head_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)
   1081     layer_outputs = checkpoint(
   1082         create_custom_forward(layer_module),
   1083         hidden_states,
   (...)
   1091         None,  # past_key_value is always None with gradient checkpointing
   1092     )
   1093 else:
-> 1094     layer_outputs = layer_module(
   1095         hidden_states,
   1096         attention_mask=extended_attention_mask,
   1097         position_bias=position_bias,
   1098         encoder_hidden_states=encoder_hidden_states,
   1099         encoder_attention_mask=encoder_extended_attention_mask,
   1100         encoder_decoder_position_bias=encoder_decoder_position_bias,
   1101         layer_head_mask=layer_head_mask,
   1102         cross_attn_layer_head_mask=cross_attn_layer_head_mask,
   1103         past_key_value=past_key_value,
   1104         use_cache=use_cache,
   1105         output_attentions=output_attentions,
   1106     )
   1108 # layer_outputs is a tuple with:
   1109 # hidden-states, key-value-states, (self-attention position bias), (self-attention weights), (cross-attention position bias), (cross-attention weights)
   1110 if use_cache is False:

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/torch/nn/modules/module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
   1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517 else:
-> 1518     return self._call_impl(*args, **kwargs)

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/torch/nn/modules/module.py:1527, in Module._call_impl(self, *args, **kwargs)
   1522 # If we don't have any hooks, we want to skip the rest of the logic in
   1523 # this function, and just call forward.
   1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1525         or _global_backward_pre_hooks or _global_backward_hooks
   1526         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527     return forward_call(*args, **kwargs)
   1529 try:
   1530     result = None

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/accelerate/hooks.py:165, in add_hook_to_module.<locals>.new_forward(*args, **kwargs)
    163         output = old_forward(*args, **kwargs)
    164 else:
--> 165     output = old_forward(*args, **kwargs)
    166 return module._hf_hook.post_forward(module, output)

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py:694, in T5Block.forward(self, hidden_states, attention_mask, position_bias, encoder_hidden_states, encoder_attention_mask, encoder_decoder_position_bias, layer_head_mask, cross_attn_layer_head_mask, past_key_value, use_cache, output_attentions, return_dict)
    691 else:
    692     self_attn_past_key_value, cross_attn_past_key_value = None, None
--> 694 self_attention_outputs = self.layer[0](
    695     hidden_states,
    696     attention_mask=attention_mask,
    697     position_bias=position_bias,
    698     layer_head_mask=layer_head_mask,
    699     past_key_value=self_attn_past_key_value,
    700     use_cache=use_cache,
    701     output_attentions=output_attentions,
    702 )
    703 hidden_states, present_key_value_state = self_attention_outputs[:2]
    704 attention_outputs = self_attention_outputs[2:]  # Keep self-attention outputs and relative position weights

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/torch/nn/modules/module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
   1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517 else:
-> 1518     return self._call_impl(*args, **kwargs)

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/torch/nn/modules/module.py:1527, in Module._call_impl(self, *args, **kwargs)
   1522 # If we don't have any hooks, we want to skip the rest of the logic in
   1523 # this function, and just call forward.
   1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1525         or _global_backward_pre_hooks or _global_backward_hooks
   1526         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527     return forward_call(*args, **kwargs)
   1529 try:
   1530     result = None

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/accelerate/hooks.py:165, in add_hook_to_module.<locals>.new_forward(*args, **kwargs)
    163         output = old_forward(*args, **kwargs)
    164 else:
--> 165     output = old_forward(*args, **kwargs)
    166 return module._hf_hook.post_forward(module, output)

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py:601, in T5LayerSelfAttention.forward(self, hidden_states, attention_mask, position_bias, layer_head_mask, past_key_value, use_cache, output_attentions)
    590 def forward(
    591     self,
    592     hidden_states,
   (...)
    598     output_attentions=False,
    599 ):
    600     normed_hidden_states = self.layer_norm(hidden_states)
--> 601     attention_output = self.SelfAttention(
    602         normed_hidden_states,
    603         mask=attention_mask,
    604         position_bias=position_bias,
    605         layer_head_mask=layer_head_mask,
    606         past_key_value=past_key_value,
    607         use_cache=use_cache,
    608         output_attentions=output_attentions,
    609     )
    610     hidden_states = hidden_states + self.dropout(attention_output[0])
    611     outputs = (hidden_states,) + attention_output[1:]  # add attentions if we output them

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/torch/nn/modules/module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
   1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517 else:
-> 1518     return self._call_impl(*args, **kwargs)

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/torch/nn/modules/module.py:1527, in Module._call_impl(self, *args, **kwargs)
   1522 # If we don't have any hooks, we want to skip the rest of the logic in
   1523 # this function, and just call forward.
   1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1525         or _global_backward_pre_hooks or _global_backward_hooks
   1526         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527     return forward_call(*args, **kwargs)
   1529 try:
   1530     result = None

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/accelerate/hooks.py:165, in add_hook_to_module.<locals>.new_forward(*args, **kwargs)
    163         output = old_forward(*args, **kwargs)
    164 else:
--> 165     output = old_forward(*args, **kwargs)
    166 return module._hf_hook.post_forward(module, output)

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/transformers/models/t5/modeling_t5.py:520, in T5Attention.forward(self, hidden_states, mask, key_value_states, position_bias, past_key_value, layer_head_mask, query_length, use_cache, output_attentions)
    517     return hidden_states
    519 # get query states
--> 520 query_states = shape(self.q(hidden_states))  # (batch_size, n_heads, seq_length, dim_per_head)
    522 # get key/value states
    523 key_states = project(
    524     hidden_states, self.k, key_value_states, past_key_value[0] if past_key_value is not None else None
    525 )

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/torch/nn/modules/module.py:1518, in Module._wrapped_call_impl(self, *args, **kwargs)
   1516     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1517 else:
-> 1518     return self._call_impl(*args, **kwargs)

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/torch/nn/modules/module.py:1527, in Module._call_impl(self, *args, **kwargs)
   1522 # If we don't have any hooks, we want to skip the rest of the logic in
   1523 # this function, and just call forward.
   1524 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1525         or _global_backward_pre_hooks or _global_backward_hooks
   1526         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1527     return forward_call(*args, **kwargs)
   1529 try:
   1530     result = None

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/accelerate/hooks.py:165, in add_hook_to_module.<locals>.new_forward(*args, **kwargs)
    163         output = old_forward(*args, **kwargs)
    164 else:
--> 165     output = old_forward(*args, **kwargs)
    166 return module._hf_hook.post_forward(module, output)

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/bitsandbytes/nn/modules.py:256, in Linear4bit.forward(self, x)
    253     x = x.to(self.compute_dtype)
    255 bias = None if self.bias is None else self.bias.to(self.compute_dtype)
--> 256 out = bnb.matmul_4bit(x, self.weight.t(), bias=bias, quant_state=self.weight.quant_state)
    258 out = out.to(inp_dtype)
    260 return out

File /opt/conda/envs/domino-ray/lib/python3.10/site-packages/bitsandbytes/autograd/_functions.py:566, in matmul_4bit(A, B, quant_state, out, bias)
    565 def matmul_4bit(A: tensor, B: tensor, quant_state: F.QuantState, out: tensor = None, bias=None):
--> 566     assert quant_state is not None
    567     if A.numel() == A.shape[-1] and A.requires_grad == False:
    568         if A.shape[-1] % quant_state.blocksize != 0:

AssertionError: 


Found the issue, changing device_map to 'auto' fix the issue. Anyone can explain why?
Instead of

```
model = AutoModelForCausalLM.from_pretrained(
    model_name,  
    quantization_config=bnb_config,
    device_map="cuda",
    trust_remote_code=True,
)
```
it should be

```
model = AutoModelForCausalLM.from_pretrained(
    model_name,  
    quantization_config=bnb_config,
    device_map="auto",
    trust_remote_code=True,
)
```
transformers and accelerate versions are as belpow
transformers==4.31.0
accelerate==0.21.0

Hi @robinsonmhj, the issue was on the accelerate library. You need to update it. The issue was that  device_map="cuda" was not handled properly. If you pass device_map={"":0}, it should work. 

in my case, i solved `FP4 quantization state not initialized` error by:

```diff
model = AutoModelForCausalLM.from_pretrained(
    ...
-   low_cpu_mem_usage=False,
)
```