update

Dockerfile

@@ -33,4 +33,4 @@ RUN --mount=type=secret,id=HF_TOKEN,mode=0444,required=true \
     git config --global credential.helper store && \
     huggingface-cli login --token $(cat /run/secrets/HF_TOKEN) --add-to-git-credential
 RUN poetry run python model_pull.py
-ENTRYPOINT ["poetry", "run", "python", "app.py", "--host", "0.0.0.0", "--port", "7860"]
+ENTRYPOINT ["/root/.local/bin/poetry", "run", "python", "app.py", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -3,15 +3,19 @@ from typing import Optional
 import gradio as gr
 import torch
 from peft import PeftModel
-from transformers import AutoModelForCausalLM
-from transformers import AutoTokenizer
 from transformers import GenerationConfig
 
 print("starting server ...")
+
+assert (
+    "LlamaTokenizer" in transformers._import_structure["models.llama"]
+), "LLaMA is now in HuggingFace's main branch.\nPlease reinstall it: pip uninstall transformers && pip install git+https://github.com/huggingface/transformers.git"
+from transformers import LlamaTokenizer, LlamaForCausalLM
+
 BASE_MODEL = "decapoda-research/llama-13b-hf"
 LORA_WEIGHTS = "izumi-lab/llama-13b-japanese-lora-v0-1ep"
 
-tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
+tokenizer = LlamaTokenizer.from_pretrained(BASE_MODEL)
 
 if torch.cuda.is_available():
     device = "cuda"
@@ -25,7 +29,7 @@ except Exception:
     pass
 
 if device == "cuda":
-    model = AutoModelForCausalLM.from_pretrained(
+    model = LlamaForCausalLM.from_pretrained(
         BASE_MODEL,
         load_in_8bit=False,
         torch_dtype=torch.float16,
@@ -35,7 +39,7 @@ if device == "cuda":
         model, LORA_WEIGHTS, torch_dtype=torch.float16
     )
 elif device == "mps":
-    model = AutoModelForCausalLM.from_pretrained(
+    model = LlamaForCausalLM.from_pretrained(
         BASE_MODEL,
         device_map={"": device},
         torch_dtype=torch.float16,
@@ -47,7 +51,7 @@ elif device == "mps":
         torch_dtype=torch.float16,
     )
 else:
-    model = AutoModelForCausalLM.from_pretrained(
+    model = LlamaForCausalLM.from_pretrained(
         BASE_MODEL, device_map={"": device}, low_cpu_mem_usage=True
     )
     model = PeftModel.from_pretrained(

model_pull.py

@@ -1,13 +1,16 @@
 import torch
 from peft import PeftModel
-from transformers import AutoModelForCausalLM
-from transformers import AutoTokenizer
+
+assert (
+    "LlamaTokenizer" in transformers._import_structure["models.llama"]
+), "LLaMA is now in HuggingFace's main branch.\nPlease reinstall it: pip uninstall transformers && pip install git+https://github.com/huggingface/transformers.git"
+from transformers import LlamaTokenizer, LlamaForCausalLM
 
 BASE_MODEL = "decapoda-research/llama-13b-hf"
 LORA_WEIGHTS = "izumi-lab/llama-13b-japanese-lora-v0-1ep"
 
-tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
-model = AutoModelForCausalLM.from_pretrained(
+tokenizer = LlamaTokenizer.from_pretrained(BASE_MODEL)
+model = LlamaForCausalLM.from_pretrained(
     BASE_MODEL,
     load_in_8bit=False,
     torch_dtype=torch.float16,