Repository: huggingface/peft

Issue:
PeftModel.eval()
This issue happens when I reload the lora fine-tuned model for inference.
my code is:
```python
peft_config = PeftConfig.from_pretrained(peft_model_id)

orig_model = BloomForCausalLM.from_pretrained(peft_config.base_model_name_or_path, local_files_only=True)
finetune_model = PeftModel.from_pretrained(orig_model, peft_model_id)
tokenizer = AutoTokenizer.from_pretrained(peft_config.base_model_name_or_path, local_files_only=True)
finetune_model.eval() 
```
The issue is:
``` python
 ./peft/src/peft/tuners/lora.py:414 in train           

│   411    delta_w = F.conv1d(                                                        
│   412                            self.lora_A.weight.data.unsqueeze(0),                                
│   413                            self.lora_B.weight.data.unsqueeze(-1),                               
│ ❱ 414                           groups=sum(self.enable_lora),                                          
│   415                      ).squeeze(0)  
│   416    self.weight.data += self.zero_pad(transpose(delta_w * self.scaling, self  
│   417    self.merged = True   
RuntimeError: Expected 4-dimensional input for 4-dimensional weight [8192, 8, 1, 1], but got 3-dimensional input of size [1, 16, 4096] instead
```
My `adaptor_config.json` is:
```json
{
  "base_model_name_or_path": "/mnt/mt-hdd/model/bloom-7b1",
  "bias": "none",
  "enable_lora": [
    true,
    false,
    true
  ],
  "fan_in_fan_out": true,
  "inference_mode": true,
  "lora_alpha": 32,
  "lora_dropout": 0.1,
  "merge_weights": false,
  "modules_to_save": null,
  "peft_type": "LORA",
  "r": 8,
  "target_modules": [
    "query_key_value"
  ],
  "task_type": "CAUSAL_LM"
}
```
It seems the size of `self.lora_A` and `self.lora_B` do not match. 



Discussion:
I have the same problem, could some one help us to troubleshoot this issue?
Me too... 😢 
I also have the same problem, could some one help us ?
Install old version `peft==0.1.0` fix the eval issue

Resolving patch:
diff --git a/src/peft/tuners/lora.py b/src/peft/tuners/lora.py
--- a/src/peft/tuners/lora.py
+++ b/src/peft/tuners/lora.py
@@ -127,12 +127,14 @@ def _find_and_replace(self):
                 "You can install it with `pip install bitsandbytes`."
             )
         is_target_modules_in_base_model = False
+        is_hf_device_map_available = hasattr(self.model, "hf_device_map")
         kwargs = {
             "r": self.peft_config.r,
             "lora_alpha": self.peft_config.lora_alpha,
             "lora_dropout": self.peft_config.lora_dropout,
             "fan_in_fan_out": self.peft_config.fan_in_fan_out,
-            "merge_weights": self.peft_config.merge_weights or self.peft_config.inference_mode,
+            "merge_weights": (self.peft_config.merge_weights or self.peft_config.inference_mode)
+            and not is_hf_device_map_available,
         }
         key_list = [key for key, _ in self.model.named_modules()]
         for key in key_list:
@@ -174,7 +176,7 @@ def _find_and_replace(self):
                                 "fan_in_fan_out is set to True but the target module is not a Conv1D. "
                                 "Setting fan_in_fan_out to False."
                             )
-                            kwargs["fan_in_fan_out"] = False
+                            kwargs["fan_in_fan_out"] = self.peft_config.fan_in_fan_out = False
                     new_module = MergedLinear(in_features, out_features, bias=bias, **kwargs)
                 self._replace_module(parent, target_name, new_module, target)
         if not is_target_modules_in_base_model:
@@ -414,22 +416,30 @@ def train(self, mode: bool = True):
         if not mode and self.merge_weights and not self.merged:
             # Merge the weights and mark it
             if self.r > 0 and any(self.enable_lora):
-                delta_w = F.conv1d(
-                    self.lora_A.weight.data.unsqueeze(0),
-                    self.lora_B.weight.data.unsqueeze(-1),
-                    groups=sum(self.enable_lora),
-                ).squeeze(0)
-                self.weight.data += self.zero_pad(transpose(delta_w * self.scaling, self.fan_in_fan_out))
+                delta_w = (
+                    F.conv1d(
+                        self.lora_A.weight.data.unsqueeze(0),
+                        self.lora_B.weight.data,
+                        groups=sum(self.enable_lora),
+                    )
+                    .squeeze(0)
+                    .transpose(-2, -1)
+                )
+                self.weight.data += transpose(self.zero_pad(delta_w * self.scaling), not self.fan_in_fan_out)
             self.merged = True
         elif self.merge_weights and self.merged:
             # Make sure that the weights are not merged
             if self.r > 0 and any(self.enable_lora):
-                delta_w = F.conv1d(
-                    self.lora_A.weight.data.unsqueeze(0),
-                    self.lora_B.weight.data.unsqueeze(-1),
-                    groups=sum(self.enable_lora),
-                ).squeeze(0)
-                self.weight.data -= self.zero_pad(transpose(delta_w * self.scaling, self.fan_in_fan_out))
+                delta_w = (
+                    F.conv1d(
+                        self.lora_A.weight.data.unsqueeze(0),
+                        self.lora_B.weight.data,
+                        groups=sum(self.enable_lora),
+                    )
+                    .squeeze(0)
+                    .transpose(-2, -1)
+                )
+                self.weight.data -= transpose(self.zero_pad(delta_w * self.scaling), not self.fan_in_fan_out)
             self.merged = False
 
     def eval(self):
@@ -440,12 +450,16 @@ def eval(self):
     def forward(self, x: torch.Tensor):
         if self.disable_adapters:
             if self.r > 0 and self.merged and any(self.enable_lora):
-                delta_w = F.conv1d(
-                    self.lora_A.weight.data.unsqueeze(0),
-                    self.lora_B.weight.data.unsqueeze(-1),
-                    groups=sum(self.enable_lora),
-                ).squeeze(0)
-                self.weight.data -= self.zero_pad(transpose(delta_w * self.scaling, self.fan_in_fan_out))
+                delta_w = (
+                    F.conv1d(
+                        self.lora_A.weight.data.unsqueeze(0),
+                        self.lora_B.weight.data,
+                        groups=sum(self.enable_lora),
+                    )
+                    .squeeze(0)
+                    .transpose(-2, -1)
+                )
+                self.weight.data -= transpose(self.zero_pad(delta_w * self.scaling), not self.fan_in_fan_out)
                 self.merged = False
             return F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias)
         elif self.merged:


Test patch: