update

Dockerfile

@@ -15,9 +15,7 @@ RUN sed -i 's http://deb.debian.org http://cdn-aws.deb.debian.org g' /etc/apt/so
     rm -rf /var/lib/apt/lists/* && \
     git lfs install
 
-COPY --link --chown=1000 ./ /home/user/app
 RUN useradd -m -u 1000 user
-WORKDIR /home/user/app
 USER user
 
 RUN curl https://pyenv.run | bash
@@ -33,10 +31,17 @@ RUN eval "$(pyenv init -)" && \
     curl -sSL https://install.python-poetry.org | python -
 ENV PATH /home/user/.local/bin:${PATH}
 
+COPY --link --chown=1000 ./pyproject.toml /home/user/app/pyproject.toml
+COPY --link --chown=1000 ./model_pull.py /home/user/app/model_pull.py
+WORKDIR /home/user/app
+
 RUN poetry install
 RUN --mount=type=secret,id=HF_TOKEN,mode=0444,required=true \
     git config --global credential.helper store && \
     huggingface-cli login --token $(cat /run/secrets/HF_TOKEN) --add-to-git-credential
 RUN poetry run python model_pull.py
+
+COPY --link --chown=1000 ./app.py /home/user/app/app.py
+
 EXPOSE 7860
 ENTRYPOINT ["/home/user/.local/bin/poetry", "run", "python", "app.py", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -29,31 +29,31 @@ except Exception:
 if device == "cuda":
     model = LlamaForCausalLM.from_pretrained(
         BASE_MODEL,
-        load_in_8bit=False,
-        torch_dtype=torch.float16,
+        load_in_8bit=True,
         device_map="auto",
     )
-    model = PeftModel.from_pretrained(model, LORA_WEIGHTS, torch_dtype=torch.float16)
+    model = PeftModel.from_pretrained(model, LORA_WEIGHTS, load_in_8bit=True)
 elif device == "mps":
     model = LlamaForCausalLM.from_pretrained(
         BASE_MODEL,
         device_map={"": device},
-        torch_dtype=torch.float16,
+        load_in_8bit=True
     )
     model = PeftModel.from_pretrained(
         model,
         LORA_WEIGHTS,
         device_map={"": device},
-        torch_dtype=torch.float16,
+        load_in_8bit=True
     )
 else:
     model = LlamaForCausalLM.from_pretrained(
-        BASE_MODEL, device_map={"": device}, low_cpu_mem_usage=True
+        BASE_MODEL, device_map={"": device}, load_in_8bit=True, low_cpu_mem_usage=True
     )
     model = PeftModel.from_pretrained(
         model,
         LORA_WEIGHTS,
         device_map={"": device},
+        load_in_8bit=True
     )
 
 
@@ -136,4 +136,4 @@ g = gr.Interface(
 )
 g.queue(concurrency_count=1)
 print("loading completed")
-g.launch()
+g.launch(server_name="0.0.0.0", server_port=7860)