Spaces:
Runtime error
Runtime error
File size: 1,287 Bytes
88f55d9 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 |
import torch
from peft import PeftModel
from transformers import LlamaTokenizer, LlamaForCausalLM
def load_model(
base,
finetuned,
mode_cpu,
mode_mps,
mode_full_gpu,
mode_8bit,
mode_4bit,
force_download_ckpt
):
tokenizer = LlamaTokenizer.from_pretrained(base)
tokenizer.pad_token_id = 0
tokenizer.padding_side = "left"
if not multi_gpu:
model = LlamaForCausalLM.from_pretrained(
base,
load_in_8bit=mode_8bit,
load_in_4bit=mode_4bit,
device_map="auto",
)
model = PeftModel.from_pretrained(
model,
finetuned,
# force_download=force_download_ckpt,
device_map={'': 0}
)
return model, tokenizer
else:
model = LlamaForCausalLM.from_pretrained(
base,
load_in_8bit=mode_8bit,
load_in_4bit=mode_4bit,
torch_dtype=torch.float16,
device_map="auto",
)
model = PeftModel.from_pretrained(
model,
finetuned,
# force_download=force_download_ckpt,
torch_dtype=torch.float16
)
model.half()
return model, tokenizer
|