|
|
import copy |
|
|
import os |
|
|
import shutil |
|
|
import tempfile |
|
|
import unittest |
|
|
|
|
|
import peft |
|
|
import torch |
|
|
from modelscope import Preprocessor |
|
|
from modelscope.models.nlp.structbert import SbertConfig, SbertForSequenceClassification |
|
|
from peft import PeftModel, inject_adapter_in_model |
|
|
from peft.config import PeftConfigMixin |
|
|
from peft.tuners.lora import Linear |
|
|
from peft.utils import WEIGHTS_NAME |
|
|
from torch import nn |
|
|
|
|
|
from swift import AdaLoraConfig, LoraConfig, LoRAConfig, Swift, get_peft_model |
|
|
|
|
|
|
|
|
class TestPeft(unittest.TestCase): |
|
|
|
|
|
def setUp(self): |
|
|
print(('Testing %s.%s' % (type(self).__name__, self._testMethodName))) |
|
|
self.tmp_dir = tempfile.TemporaryDirectory().name |
|
|
if not os.path.exists(self.tmp_dir): |
|
|
os.makedirs(self.tmp_dir) |
|
|
|
|
|
def tearDown(self): |
|
|
shutil.rmtree(self.tmp_dir) |
|
|
super().tearDown() |
|
|
|
|
|
def test_peft_lora_injection(self): |
|
|
model = SbertForSequenceClassification(SbertConfig()) |
|
|
model2 = copy.deepcopy(model) |
|
|
lora_config = LoraConfig(target_modules=['query', 'key', 'value']) |
|
|
model = Swift.prepare_model(model, lora_config) |
|
|
model.save_pretrained(self.tmp_dir, safe_serialization=False) |
|
|
with open(os.path.join(self.tmp_dir, 'configuration.json'), 'w') as f: |
|
|
f.write('{}') |
|
|
self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, WEIGHTS_NAME))) |
|
|
model2 = Swift.from_pretrained(model2, self.tmp_dir) |
|
|
state_dict = model.state_dict() |
|
|
state_dict2 = model2.state_dict() |
|
|
for key in state_dict: |
|
|
self.assertTrue(key in state_dict2) |
|
|
self.assertTrue(all(torch.isclose(state_dict[key], state_dict2[key]).flatten().detach().cpu())) |
|
|
|
|
|
@unittest.skip |
|
|
def test_lora_merge(self): |
|
|
|
|
|
def reset_lora_parameters(self, adapter_name, init_lora_weights): |
|
|
if init_lora_weights is False: |
|
|
return |
|
|
|
|
|
if adapter_name == 'default': |
|
|
ratio = 1.0 |
|
|
elif adapter_name == 'second': |
|
|
ratio = 2.0 |
|
|
else: |
|
|
ratio = 3.0 |
|
|
|
|
|
if adapter_name in self.lora_A.keys(): |
|
|
nn.init.ones_(self.lora_A[adapter_name].weight) |
|
|
self.lora_A[adapter_name].weight.data = self.lora_A[adapter_name].weight.data * ratio |
|
|
nn.init.ones_(self.lora_B[adapter_name].weight) |
|
|
|
|
|
Linear.reset_lora_parameters = reset_lora_parameters |
|
|
|
|
|
model = SbertForSequenceClassification(SbertConfig()) |
|
|
lora_config = LoRAConfig(target_modules=['query', 'key', 'value']) |
|
|
model = Swift.prepare_model(model, lora_config) |
|
|
lora_config2 = LoRAConfig(target_modules=['query', 'key', 'value']) |
|
|
model = Swift.prepare_model(model, {'second': lora_config2}) |
|
|
model.add_weighted_adapter(['default', 'second'], |
|
|
weights=[0.7, 0.3], |
|
|
adapter_name='test', |
|
|
combination_type='cat') |
|
|
self.assertTrue(model.base_model.bert.encoder.layer[0].attention.self.key.active_adapter == ['test']) |
|
|
|
|
|
model2 = SbertForSequenceClassification(SbertConfig()) |
|
|
lora_config = LoraConfig(target_modules=['query', 'key', 'value']) |
|
|
model2 = get_peft_model(model2, lora_config) |
|
|
lora_config2 = LoraConfig(target_modules=['query', 'key', 'value']) |
|
|
inject_adapter_in_model(lora_config2, model2, adapter_name='second') |
|
|
model2.add_weighted_adapter(['default', 'second'], |
|
|
weights=[0.7, 0.3], |
|
|
adapter_name='test', |
|
|
combination_type='cat') |
|
|
state_dict = model.state_dict() |
|
|
state_dict2 = model2.state_dict() |
|
|
state_dict2 = {key[len('base_model.model.'):]: value for key, value in state_dict2.items() if 'lora' in key} |
|
|
for key in state_dict: |
|
|
self.assertTrue(key in state_dict2) |
|
|
self.assertTrue(all(torch.isclose(state_dict[key], state_dict2[key]).flatten().detach().cpu())) |
|
|
|
|
|
preprocessor = Preprocessor.from_pretrained('damo/nlp_structbert_sentence-similarity_chinese-base') |
|
|
inputs = preprocessor('how are you') |
|
|
print(model(**inputs)) |
|
|
model.save_pretrained(self.tmp_dir) |
|
|
model3 = SbertForSequenceClassification(SbertConfig()) |
|
|
model3 = Swift.from_pretrained(model3, self.tmp_dir) |
|
|
state_dict3 = model3.state_dict() |
|
|
for key in state_dict: |
|
|
self.assertTrue(key in state_dict3) |
|
|
self.assertTrue(all(torch.isclose(state_dict[key], state_dict3[key]).flatten().detach().cpu())) |
|
|
|
|
|
def test_lora_reload_by_peft(self): |
|
|
lora_config = LoRAConfig(target_modules=['query', 'key', 'value']) |
|
|
model = SbertForSequenceClassification(SbertConfig()) |
|
|
model2 = copy.deepcopy(model) |
|
|
model = Swift.prepare_model(model, lora_config) |
|
|
model.save_pretrained(self.tmp_dir, peft_format=True) |
|
|
model2 = PeftModel.from_pretrained(model2, self.tmp_dir) |
|
|
state_dict = model.state_dict() |
|
|
state_dict2 = model2.state_dict() |
|
|
state_dict2 = {key[len('base_model.model.'):]: value for key, value in state_dict2.items() if 'lora' in key} |
|
|
for key in state_dict: |
|
|
self.assertTrue(key in state_dict2) |
|
|
self.assertTrue(all(torch.isclose(state_dict[key], state_dict2[key]).flatten().detach().cpu())) |
|
|
|
|
|
def test_peft_adalora_injection(self): |
|
|
model = SbertForSequenceClassification(SbertConfig()) |
|
|
model2 = copy.deepcopy(model) |
|
|
adalora_config = AdaLoraConfig(target_modules=['query', 'key', 'value'], total_step=1) |
|
|
model = Swift.prepare_model(model, adalora_config) |
|
|
model.save_pretrained(self.tmp_dir, safe_serialization=False) |
|
|
with open(os.path.join(self.tmp_dir, 'configuration.json'), 'w') as f: |
|
|
f.write('{}') |
|
|
self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, WEIGHTS_NAME))) |
|
|
model2 = Swift.from_pretrained(model2, self.tmp_dir) |
|
|
state_dict = model.state_dict() |
|
|
state_dict2 = model2.state_dict() |
|
|
for key in state_dict: |
|
|
self.assertTrue(key in state_dict2) |
|
|
self.assertTrue(all(torch.isclose(state_dict[key], state_dict2[key]).flatten().detach().cpu())) |
|
|
|
|
|
@unittest.skip |
|
|
def test_peft_lora_dtype(self): |
|
|
model = SbertForSequenceClassification(SbertConfig()) |
|
|
model2 = copy.deepcopy(model) |
|
|
model3 = copy.deepcopy(model) |
|
|
lora_config = LoraConfig(target_modules=['query', 'key', 'value'], lora_dtype='float16') |
|
|
model = Swift.prepare_model(model, lora_config) |
|
|
model.save_pretrained(self.tmp_dir, safe_serialization=False) |
|
|
self.assertTrue(os.path.exists(os.path.join(self.tmp_dir, 'additional_config.json'))) |
|
|
model2 = Swift.from_pretrained(model2, self.tmp_dir) |
|
|
self.assertTrue(model2.base_model.model.bert.encoder.layer[0].attention.self.key.lora_A.default.weight.dtype == |
|
|
torch.float16) |
|
|
self.assertTrue(model2.peft_config['default'].lora_dtype == 'float16') |
|
|
state_dict = model.state_dict() |
|
|
state_dict2 = model2.state_dict() |
|
|
for key in state_dict: |
|
|
self.assertTrue(key in state_dict2) |
|
|
self.assertTrue(all(torch.isclose(state_dict[key], state_dict2[key]).flatten().detach().cpu())) |
|
|
|
|
|
PeftConfigMixin.from_pretrained = PeftConfigMixin.from_pretrained_origin |
|
|
model3 = Swift.from_pretrained(model3, self.tmp_dir) |
|
|
self.assertTrue(model3.base_model.model.bert.encoder.layer[0].attention.self.key.lora_A.default.weight.dtype == |
|
|
torch.float32) |
|
|
self.assertTrue(isinstance(model3.peft_config['default'], peft.LoraConfig)) |
|
|
|
