Repository: huggingface/peft

Issue:
error while loading lora
### System Info

A800, multiple loras

### Who can help?

_No response_

### Information

- [X] The official example scripts
- [ ] My own modified scripts

### Tasks

- [ ] An officially supported task in the `examples` folder
- [ ] My own task or dataset (give details below)

### Reproduction

```python
model = AutoModelForCausalLM.from_pretrained(
    "Qwen-7B-Chat-Int4",
    device_map="auto",
    trust_remote_code=True
).eval()
from peft import PeftModel
model = PeftModel.from_pretrained(model,'lora1', adapter_name="lora1",device_map="cuda")
model.load_adapter('lora2',adapter_name = 'lora2')
```

#########
it happens error

```
File /home/anaconda3/envs/lib/python3.10/site-packages/peft/peft_model.py:686, in PeftModel.load_adapter(self, model_id, adapter_name, is_trainable, **kwargs)
    684     else:
    685         peft_config.inference_mode = not is_trainable
--> 686     self.add_adapter(adapter_name, peft_config)
    688 adapters_weights = load_peft_weights(model_id, device=torch_device, **hf_hub_download_kwargs)
    690 # load the weights into the model

File /home/anaconda3/envs/lib/python3.10/site-packages/peft/peft_model.py:616, in PeftModel.add_adapter(self, adapter_name, peft_config)
    614     else:
    615         self.peft_config[adapter_name] = peft_config
--> 616         self.base_model.inject_adapter(self.base_model.model, adapter_name)
    617 except Exception:  # somthing went wrong, roll back
    618     if adapter_name in self.peft_config:

File /home/anaconda3/envslib/python3.10/site-packages/peft/tuners/tuners_utils.py:252, in BaseTuner.inject_adapter(self, model, adapter_name)
    245     parent, target, target_name = _get_submodules(model, key)
    247     optional_kwargs = {
    248         "loaded_in_8bit": getattr(model, "is_loaded_in_8bit", False),
    249         "loaded_in_4bit": getattr(model, "is_loaded_in_4bit", False),
    250         "current_key": key,
    251     }
--> 252     self._create_and_replace(peft_config, adapter_name, target, target_name, parent, **optional_kwargs)
    254 if not is_target_modules_in_base_model:
    255     raise ValueError(
    256         f"Target modules {peft_config.target_modules} not found in the base model. "
    257         f"Please check the target modules and try again."
    258     )

File /home/anaconda3/envs/lib/python3.10/site-packages/peft/tuners/lora/model.py:200, in LoraModel._create_and_replace(self, lora_config, adapter_name, target, target_name, parent, current_key, **optional_kwargs)
    192     target.update_layer(
    193         adapter_name,
    194         r,
   (...)
    197         lora_config.init_lora_weights,
    198     )
    199 else:
--> 200     new_module = self._create_new_module(lora_config, adapter_name, target, **kwargs)
    201     if adapter_name != self.active_adapter:
    202         # adding an additional adapter: it is not automatically trainable
    203         new_module.requires_grad_(False)

File /homeanaconda3/envs/lib/python3.10/site-packages/peft/tuners/lora/model.py:296, in LoraModel._create_new_module(lora_config, adapter_name, target, **kwargs)
    294 elif AutoGPTQQuantLinear is not None and isinstance(target_base_layer, AutoGPTQQuantLinear):
    295     new_module = QuantLinear(target, adapter_name, **kwargs)
--> 296     print(target.qweight)
    297     #target.weight = target.qweight
    298 elif isinstance(target_base_layer, torch.nn.Embedding):

File /home/anaconda3/envs/lib/python3.10/site-packages/torch/nn/modules/module.py:1614, in Module.__getattr__(self, name)
   1612     if name in modules:
   1613         return modules[name]
-> 1614 raise AttributeError("'{}' object has no attribute '{}'".format(
   1615     type(self).__name__, name))


### Expected behavior

load multiple loras to one model by using model.load_adapter


Discussion:

Resolving patch:
diff --git a/src/peft/tuners/adalora/model.py b/src/peft/tuners/adalora/model.py
--- a/src/peft/tuners/adalora/model.py
+++ b/src/peft/tuners/adalora/model.py
@@ -173,10 +173,10 @@ def _create_new_module(lora_config, adapter_name, target, **kwargs):
         if loaded_in_8bit and isinstance(target_base_layer, bnb.nn.Linear8bitLt):
             kwargs.update(
                 {
-                    "has_fp16_weights": target.state.has_fp16_weights,
-                    "memory_efficient_backward": target.state.memory_efficient_backward,
-                    "threshold": target.state.threshold,
-                    "index": target.index,
+                    "has_fp16_weights": target_base_layer.state.has_fp16_weights,
+                    "memory_efficient_backward": target_base_layer.state.memory_efficient_backward,
+                    "threshold": target_base_layer.state.threshold,
+                    "index": target_base_layer.index,
                 }
             )
             new_module = SVDLinear8bitLt(target, adapter_name, **kwargs)
@@ -184,9 +184,9 @@ def _create_new_module(lora_config, adapter_name, target, **kwargs):
             fourbit_kwargs = kwargs.copy()
             fourbit_kwargs.update(
                 {
-                    "compute_dtype": target.compute_dtype,
-                    "compress_statistics": target.weight.compress_statistics,
-                    "quant_type": target.weight.quant_type,
+                    "compute_dtype": target_base_layer.compute_dtype,
+                    "compress_statistics": target_base_layer.weight.compress_statistics,
+                    "quant_type": target_base_layer.weight.quant_type,
                 }
             )
             new_module = SVDLinear4bit(target, adapter_name, **fourbit_kwargs)
diff --git a/src/peft/tuners/ia3/model.py b/src/peft/tuners/ia3/model.py
--- a/src/peft/tuners/ia3/model.py
+++ b/src/peft/tuners/ia3/model.py
@@ -100,10 +100,10 @@ def _create_new_module(ia3_config, adapter_name, target, **kwargs):
             eightbit_kwargs = kwargs.copy()
             eightbit_kwargs.update(
                 {
-                    "has_fp16_weights": target.state.has_fp16_weights,
-                    "memory_efficient_backward": target.state.memory_efficient_backward,
-                    "threshold": target.state.threshold,
-                    "index": target.index,
+                    "has_fp16_weights": target_base_layer.state.has_fp16_weights,
+                    "memory_efficient_backward": target_base_layer.state.memory_efficient_backward,
+                    "threshold": target_base_layer.state.threshold,
+                    "index": target_base_layer.index,
                 }
             )
             new_module = Linear8bitLt(target, adapter_name, is_feedforward=is_feedforward, **eightbit_kwargs)
@@ -111,9 +111,9 @@ def _create_new_module(ia3_config, adapter_name, target, **kwargs):
             fourbit_kwargs = kwargs.copy()
             fourbit_kwargs.update(
                 {
-                    "compute_dtype": target.compute_dtype,
-                    "compress_statistics": target.weight.compress_statistics,
-                    "quant_type": target.weight.quant_type,
+                    "compute_dtype": target_base_layer.compute_dtype,
+                    "compress_statistics": target_base_layer.weight.compress_statistics,
+                    "quant_type": target_base_layer.weight.quant_type,
                 }
             )
             new_module = Linear4bit(target, adapter_name, is_feedforward=is_feedforward, **fourbit_kwargs)
diff --git a/src/peft/tuners/lora/model.py b/src/peft/tuners/lora/model.py
--- a/src/peft/tuners/lora/model.py
+++ b/src/peft/tuners/lora/model.py
@@ -163,6 +163,16 @@ def _create_and_replace(
         if quantization_config is not None:
             kwargs["gptq_quantization_config"] = quantization_config
 
+        linear_types = (Linear,)
+        if is_bnb_available():
+            from .bnb import Linear8bitLt
+
+            linear_types += (Linear8bitLt,)
+        if is_bnb_4bit_available():
+            from .bnb import Linear4bit
+
+            linear_types += (Linear4bit,)
+
         # TODO: better deal with that
         if isinstance(target, Conv2d):
             target.update_layer_conv2d(
@@ -180,7 +190,7 @@ def _create_and_replace(
                 lora_config.lora_dropout,
                 lora_config.init_lora_weights,
             )
-        elif isinstance(target, Linear):
+        elif isinstance(target, linear_types):
             target.update_layer(
                 adapter_name,
                 r,
@@ -284,15 +294,15 @@ def _create_new_module(lora_config, adapter_name, target, **kwargs):
             fourbit_kwargs = kwargs.copy()
             fourbit_kwargs.update(
                 {
-                    "compute_dtype": target.compute_dtype,
-                    "compress_statistics": target.weight.compress_statistics,
-                    "quant_type": target.weight.quant_type,
+                    "compute_dtype": target_base_layer.compute_dtype,
+                    "compress_statistics": target_base_layer.weight.compress_statistics,
+                    "quant_type": target_base_layer.weight.quant_type,
                 }
             )
             new_module = Linear4bit(target, adapter_name, **fourbit_kwargs)
         elif AutoGPTQQuantLinear is not None and isinstance(target_base_layer, AutoGPTQQuantLinear):
             new_module = QuantLinear(target, adapter_name, **kwargs)
-            target.weight = target.qweight
+            target.qweight = target_base_layer.qweight
         elif isinstance(target_base_layer, torch.nn.Embedding):
             embedding_kwargs = kwargs.copy()
             embedding_kwargs.pop("fan_in_fan_out", None)


Test patch:
diff --git a/tests/test_common_gpu.py b/tests/test_common_gpu.py
--- a/tests/test_common_gpu.py
+++ b/tests/test_common_gpu.py
@@ -19,6 +19,7 @@
 import pytest
 import torch
 import torch.nn.functional as F
+from parameterized import parameterized
 from transformers import (
     AutoModelForCausalLM,
     AutoModelForSeq2SeqLM,
@@ -31,6 +32,7 @@
 )
 
 from peft import (
+    AdaLoraConfig,
     AdaptionPromptConfig,
     IA3Config,
     LoraConfig,
@@ -186,17 +188,128 @@ def test_ia3_bnb_8bit_quantization(self):
     @require_bitsandbytes
     @pytest.mark.multi_gpu_tests
     @pytest.mark.single_gpu_tests
-    def test_lora_bnb_4bit_quantization_from_pretrained_safetensors(self):
+    @parameterized.expand(["4bit", "8bit"])
+    def test_lora_bnb_quantization_from_pretrained_safetensors(self, quantization):
         r"""
-        Test that tests if the 4bit quantization using LoRA works as expected with safetensors weights.
+        Tests that the bnb quantization using LoRA works as expected with safetensors weights.
         """
         model_id = "facebook/opt-350m"
         peft_model_id = "ybelkada/test-st-lora"
+        kwargs = {"device_map": "auto"}
+        if quantization == "4bit":
+            kwargs["load_in_4bit"] = True
+        else:
+            kwargs["load_in_8bit"] = True
 
-        model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", load_in_4bit=True)
+        model = AutoModelForCausalLM.from_pretrained(model_id, **kwargs)
         model = PeftModel.from_pretrained(model, peft_model_id)
 
-        _ = model.generate(input_ids=torch.LongTensor([[0, 2, 3, 1]]).to(0))
+        model.generate(input_ids=torch.LongTensor([[0, 2, 3, 1]]).to(0))
+
+        # loading a 2nd adapter works, #1239
+        model.load_adapter(peft_model_id, "adapter2")
+        model.set_adapter("adapter2")
+        model.generate(input_ids=torch.LongTensor([[0, 2, 3, 1]]).to(0))
+
+    @require_bitsandbytes
+    @pytest.mark.multi_gpu_tests
+    @pytest.mark.single_gpu_tests
+    @parameterized.expand(["4bit", "8bit"])
+    def test_adalora_bnb_quantization_from_pretrained_safetensors(self, quantization):
+        r"""
+        Tests that the bnb quantization using AdaLora works as expected with safetensors weights.
+        """
+        model_id = "facebook/opt-350m"
+        kwargs = {"device_map": "auto"}
+        if quantization == "4bit":
+            kwargs["load_in_4bit"] = True
+        else:
+            kwargs["load_in_8bit"] = True
+
+        model = AutoModelForCausalLM.from_pretrained(model_id, **kwargs)
+        config = AdaLoraConfig(task_type=TaskType.CAUSAL_LM)
+        peft_model = get_peft_model(model, config)
+        peft_model = prepare_model_for_kbit_training(peft_model)
+        peft_model.generate(input_ids=torch.LongTensor([[0, 2, 3, 1]]).to(0))
+
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            peft_model.save_pretrained(tmp_dir)
+            model = AutoModelForCausalLM.from_pretrained(model_id, **kwargs)
+            model = PeftModel.from_pretrained(model, tmp_dir)
+            model = prepare_model_for_kbit_training(peft_model)
+            model.generate(input_ids=torch.LongTensor([[0, 2, 3, 1]]).to(0))
+
+            # loading a 2nd adapter works, #1239
+            model.load_adapter(tmp_dir, "adapter2")
+            model.set_adapter("adapter2")
+            model.generate(input_ids=torch.LongTensor([[0, 2, 3, 1]]).to(0))
+
+    @require_bitsandbytes
+    @pytest.mark.multi_gpu_tests
+    @pytest.mark.single_gpu_tests
+    @parameterized.expand(["4bit", "8bit"])
+    def test_ia3_bnb_quantization_from_pretrained_safetensors(self, quantization):
+        r"""
+        Tests that the bnb quantization using IA³ works as expected with safetensors weights.
+        """
+        model_id = "facebook/opt-350m"
+        kwargs = {"device_map": "auto"}
+        if quantization == "4bit":
+            kwargs["load_in_4bit"] = True
+        else:
+            kwargs["load_in_8bit"] = True
+
+        model = AutoModelForCausalLM.from_pretrained(model_id, **kwargs)
+        config = IA3Config(task_type=TaskType.CAUSAL_LM)
+        peft_model = get_peft_model(model, config)
+        peft_model = prepare_model_for_kbit_training(peft_model)
+        peft_model.generate(input_ids=torch.LongTensor([[0, 2, 3, 1]]).to(0))
+
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            peft_model.save_pretrained(tmp_dir)
+            model = AutoModelForCausalLM.from_pretrained(model_id, **kwargs)
+            model = PeftModel.from_pretrained(model, tmp_dir)
+            model = prepare_model_for_kbit_training(model)
+            model.generate(input_ids=torch.LongTensor([[0, 2, 3, 1]]).to(0))
+
+            # loading a 2nd adapter works, #1239
+            model.load_adapter(tmp_dir, "adapter2")
+            model.set_adapter("adapter2")
+            model.generate(input_ids=torch.LongTensor([[0, 2, 3, 1]]).to(0))
+
+    @pytest.mark.single_gpu_tests
+    def test_lora_gptq_quantization_from_pretrained_safetensors(self):
+        r"""
+        Tests that the autogptq quantization using LoRA works as expected with safetensors weights.
+        """
+        from transformers import GPTQConfig
+
+        model_id = "marcsun13/opt-350m-gptq-4bit"
+        quantization_config = GPTQConfig(bits=4, use_exllama=False)
+        kwargs = {
+            "pretrained_model_name_or_path": model_id,
+            "torch_dtype": torch.float16,
+            "device_map": "auto",
+            "quantization_config": quantization_config,
+        }
+        model = AutoModelForCausalLM.from_pretrained(**kwargs)
+        model = prepare_model_for_kbit_training(model)
+
+        config = LoraConfig(task_type="CAUSAL_LM")
+        peft_model = get_peft_model(model, config)
+        peft_model.generate(input_ids=torch.LongTensor([[0, 2, 3, 1]]).to(0))
+
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            peft_model.save_pretrained(tmp_dir)
+            model = AutoModelForCausalLM.from_pretrained(**kwargs)
+            model = PeftModel.from_pretrained(model, tmp_dir)
+            model = prepare_model_for_kbit_training(model)
+            model.generate(input_ids=torch.LongTensor([[0, 2, 3, 1]]).to(0))
+
+            # loading a 2nd adapter works, #1239
+            model.load_adapter(tmp_dir, "adapter2")
+            model.set_adapter("adapter2")
+            model.generate(input_ids=torch.LongTensor([[0, 2, 3, 1]]).to(0))
 
     @require_bitsandbytes
     @pytest.mark.multi_gpu_tests