test

app.py

@@ -7,8 +7,13 @@ from typing import Tuple
 
 import gradio as gr
 import torch
+from fastchat.serve.inference import compress_module
+from fastchat.serve.inference import raise_warning_for_old_weights
 from huggingface_hub import Repository
+from huggingface_hub import hf_hub_download
+from peft import LoraConfig
 from peft import PeftModel
+from peft import set_peft_model_state_dict
 from transformers import AutoModelForCausalLM
 from transformers import GenerationConfig
 from transformers import LlamaTokenizer
@@ -58,35 +63,32 @@ try:
 except Exception:
     pass
 
+checkpoint_name = hf_hub_download(repo_id=LORA_WEIGHTS, filename="adapter_model.bin", use_auth_token=HF_TOKEN)
 if device == "cuda":
     model = AutoModelForCausalLM.from_pretrained(
-        BASE_MODEL,
-        load_in_8bit=True,
-        device_map="auto",
+        BASE_MODEL, load_in_8bit=True, device_map="auto", torch_dtype=torch.float16
     )
-    model = PeftModel.from_pretrained(model, LORA_WEIGHTS, load_in_8bit=True,)
 elif device == "mps":
     model = AutoModelForCausalLM.from_pretrained(
         BASE_MODEL,
         device_map={"": device},
         load_in_8bit=True,
-    )
-    model = PeftModel.from_pretrained(
-        model,
-        LORA_WEIGHTS,
-        device_map={"": device},
-        load_in_8bit=True,
+        torch_dtype=torch.float16,
     )
 else:
     model = AutoModelForCausalLM.from_pretrained(
-        BASE_MODEL, device_map={"": device},load_in_8bit=True, low_cpu_mem_usage=True
-    )
-    model = PeftModel.from_pretrained(
-        model,
-        LORA_WEIGHTS,
+        BASE_MODEL,
         device_map={"": device},
         load_in_8bit=True,
+        low_cpu_mem_usage=True,
+        torch_dtype=torch.float16,
     )
+adapters_weights = torch.load(checkpoint_name)
+set_peft_model_state_dict(model, adapters_weights)
+raise_warning_for_old_weights(BASE_MODEL, model)
+compress_module(model, device)
+if device == "cuda" or device == "mps":
+    model = model.to(device)
 
 
 def generate_prompt(instruction: str, input: Optional[str] = None):

pyproject.toml

@@ -9,13 +9,14 @@ readme = "README.md"
 [tool.poetry.dependencies]
 python = "^3.9"
 peft = "^0.3.0"
-transformers = {git = "https://github.com/huggingface/transformers.git", branch = "main"}
-gradio = "^3.32.0"
+gradio = "^3.23.0"
 torch = "^2.0.1"
 huggingface-hub = "^0.14.1"
 sentencepiece = "^0.1.99"
 bitsandbytes = "^0.38.1"
 accelerate = "^0.19.0"
+fschat = "^0.2.3"
+transformers = "^4.29.2"
 
 
 [tool.poetry.group.dev.dependencies]