Repository: huggingface/transformers
Pull request: FIX Multiple PEFT errors after v5 transition

# What does this PR do?

Multiple PEFT tests are failing due to recent changes in transformers.

- hf_device_map attribute may not exist in some cases
- respect inference_mode in load_adapter
- new model loading requires changes for bnb (SCB attribute etc.)

These tests have been failing for quite some time now but nobody noticed.


## Before submitting
- [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case).
- [x] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#create-a-pull-request),
      Pull Request section?
- [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link
      to it if that's the case.
- [ ] Did you make sure to update the documentation with your changes? Here are the
      [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and
      [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation).
- [ ] Did you write any new necessary tests?

Changed files:
filename: src/transformers/integrations/peft.py
patch: @@ -566,19 +566,32 @@ def load_adapter(
         else:
             checkpoint_files, sharded_metadata = [], {}
 
+        device_map = getattr(self, "hf_device_map", {"": self.device})
         load_config = replace(
             load_config,
             pretrained_model_name_or_path=peft_model_id,
             sharded_metadata=sharded_metadata,
             weight_mapping=peft_weight_conversions,
+            device_map=device_map,
         )
         loading_info, _ = self._load_pretrained_model(
             model=self,
             state_dict=adapter_state_dict,
             checkpoint_files=checkpoint_files,
             load_config=load_config,
+            # pass expected keys explicitly, otherwise they are determined from the state_dict, which can contain
+            # unexpected entries, like "layer.SCB" from a bnb layer.
+            expected_keys=[n for n, _ in self.named_parameters()],
         )
 
+        if peft_config.inference_mode:
+            from peft.tuners.tuners_utils import BaseTunerLayer
+
+            self.eval()
+            for module in self.modules():
+                if isinstance(module, BaseTunerLayer):
+                    module.requires_grad_(False)
+
         adapter_key_markers = {adapter_name}
         if peft_config is not None and getattr(peft_config, "peft_type", None) is not None:
             adapter_key_markers.add(peft_config.peft_type.value.lower())

filename: src/transformers/modeling_utils.py
patch: @@ -4185,6 +4185,7 @@ def _load_pretrained_model(
         state_dict: dict | None,
         checkpoint_files: list[str] | None,
         load_config: LoadStateDictConfig,
+        expected_keys: list[str] | None = None,
     ) -> tuple[LoadStateDictInfo, dict]:
         """Perform the actual loading of some checkpoints into a `model`, by reading them from disk and dispatching them accordingly."""
         is_quantized = load_config.is_quantized
@@ -4194,7 +4195,7 @@ def _load_pretrained_model(
         }
 
         # Model's definition arriving here is final (TP hooks added, quantized layers replaces)
-        expected_keys = list(model.state_dict().keys())
+        expected_keys = list(model.state_dict().keys()) if expected_keys is None else expected_keys
 
         if logger.level >= logging.WARNING:
             verify_tp_plan(expected_keys, getattr(model, "_tp_plan", None))

filename: tests/peft_integration/test_peft_integration.py
patch: @@ -463,7 +463,6 @@ def test_peft_from_pretrained_kwargs(self):
 
                 module = peft_model.model.decoder.layers[0].self_attn.v_proj
                 self.assertTrue(module.__class__.__name__ == "Linear8bitLt")
-                self.assertTrue(peft_model.hf_device_map is not None)
 
                 # dummy generation
                 _ = peft_model.generate(input_ids=torch.LongTensor([[0, 1, 2, 3, 4, 5, 6, 7]]).to(torch_device))
@@ -484,7 +483,6 @@ def test_peft_save_quantized(self):
 
                 module = peft_model.model.decoder.layers[0].self_attn.v_proj
                 self.assertTrue(module.__class__.__name__ == "Linear4bit")
-                self.assertTrue(peft_model.hf_device_map is not None)
 
                 with tempfile.TemporaryDirectory() as tmpdirname:
                     peft_model.save_pretrained(tmpdirname)
@@ -503,7 +501,6 @@ def test_peft_save_quantized(self):
 
                 module = peft_model.model.decoder.layers[0].self_attn.v_proj
                 self.assertTrue(module.__class__.__name__ == "Linear8bitLt")
-                self.assertTrue(peft_model.hf_device_map is not None)
 
                 with tempfile.TemporaryDirectory() as tmpdirname:
                     peft_model.save_pretrained(tmpdirname)
@@ -529,7 +526,6 @@ def test_peft_save_quantized_regression(self):
 
                 module = peft_model.model.decoder.layers[0].self_attn.v_proj
                 self.assertTrue(module.__class__.__name__ == "Linear4bit")
-                self.assertTrue(peft_model.hf_device_map is not None)
 
                 with tempfile.TemporaryDirectory() as tmpdirname:
                     peft_model.save_pretrained(tmpdirname)
@@ -547,7 +543,6 @@ def test_peft_save_quantized_regression(self):
 
                 module = peft_model.model.decoder.layers[0].self_attn.v_proj
                 self.assertTrue(module.__class__.__name__ == "Linear8bitLt")
-                self.assertTrue(peft_model.hf_device_map is not None)
 
                 with tempfile.TemporaryDirectory() as tmpdirname:
                     peft_model.save_pretrained(tmpdirname)