test

app.py

@@ -24,12 +24,30 @@ from peft import LoraConfig
 from peft import PeftModel
 from peft import get_peft_model
 from peft import set_peft_model_state_dict
-import transformers
+from transformers import LlamaForCausalLM
+from transformers import LlamaTokenizer
 from transformers import PreTrainedModel
 from transformers import PreTrainedTokenizerBase
 
-transformers.AutoTokenizer = transformers.LlamaTokenizer
-transformers.AutoModelForCausalLM = transformers.LlamaForCausalLM
+
+class LLaMAdapter(BaseAdapter):
+    "Model adapater for vicuna-v1.1"
+
+    def match(self, model_path: str):
+        return "llama" in model_path
+
+    def load_model(self, model_path: str, from_pretrained_kwargs: dict):
+        tokenizer = LlamaTokenizer.from_pretrained(model_path, use_fast=False)
+        model = LlamaForCausalLM.from_pretrained(
+            model_path,
+            low_cpu_mem_usage=True,
+            **from_pretrained_kwargs,
+        )
+        return model, tokenizer
+
+
+model_adapters.insert(-1, LLaMAdapter())
+
 
 def load_lora_model(
     model_path: str,
@@ -48,7 +66,7 @@ def load_lora_model(
         device=device,
         num_gpus=num_gpus,
         max_gpu_memory=max_gpu_memory,
-        load_8bit=load_8bit,
+        load_8bit=False,
         cpu_offloading=cpu_offloading,
         debug=debug,
     )