test

Dockerfile

@@ -0,0 +1,47 @@
+ARG CUDA_IMAGE="12.1.1-devel-ubuntu22.04"
+FROM nvidia/cuda:${CUDA_IMAGE}
+
+# We need to set the host to 0.0.0.0 to allow outside access
+ENV HOST 0.0.0.0
+
+RUN apt-get update && apt-get upgrade -y \
+    && apt-get install -y git build-essential \
+    python3 python3-pip gcc wget \
+    ocl-icd-opencl-dev opencl-headers clinfo \
+    libclblast-dev libopenblas-dev \
+    && mkdir -p /etc/OpenCL/vendors && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
+
+COPY . .
+
+# setting build related env vars
+ENV CUDA_DOCKER_ARCH=all
+ENV LLAMA_CUBLAS=1
+
+# Install depencencies
+RUN python3 -m pip install --upgrade pip pytest cmake \
+    scikit-build setuptools fastapi uvicorn sse-starlette \
+    pydantic-settings huggingface_hub hf_transfer
+
+# Install llama-cpp-python (build with cuda)
+RUN CMAKE_ARGS="-DLLAMA_CUBLAS=on" pip install llama-cpp-python
+
+RUN useradd -m -u 1000 user
+# Switch to the "user" user
+USER user
+# Set home to the user's home directory
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH \
+    PYTHONPATH=$HOME/app \
+	PYTHONUNBUFFERED=1 \
+	GRADIO_ALLOW_FLAGGING=never \
+	GRADIO_NUM_PORTS=1 \
+	GRADIO_SERVER_NAME=0.0.0.0 \
+	GRADIO_THEME=huggingface \
+	SYSTEM=spaces
+
+WORKDIR $HOME/app
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

main.py

@@ -0,0 +1,76 @@
+import os
+import copy
+import time
+import llama_cpp
+from llama_cpp import Llama
+from huggingface_hub import hf_hub_download  
+
+import uvicorn
+from fastapi import FastAPI, Request
+
+
+llm = Llama(
+    model_path=hf_hub_download(
+        repo_id=os.environ.get("REPO_ID", "TheBloke/Llama-2-7b-Chat-GGUF"),
+        filename=os.environ.get("MODEL_FILE", "llama-2-7b-chat.Q5_0.gguf"),
+    ),
+    n_ctx=2048,
+    n_gpu_layers=50, # change n_gpu_layers if you have more or less VRAM 
+) 
+
+history = []
+
+system_message = """
+You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
+If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.
+"""
+
+
+def generate_text(message, history):
+    temp = ""
+    input_prompt = f"[INST] <<SYS>>\n{system_message}\n<</SYS>>\n\n "
+    for interaction in history:
+        input_prompt = input_prompt + str(interaction[0]) + " [/INST] " + str(interaction[1]) + " </s><s> [INST] "
+
+    input_prompt = input_prompt + str(message) + " [/INST] "
+
+    output = llm(
+        input_prompt,
+        temperature=0.15,
+        top_p=0.1,
+        top_k=40, 
+        repeat_penalty=1.1,
+        max_tokens=1024,
+        stop=[
+            "<|prompter|>",
+            "<|endoftext|>",
+            "<|endoftext|> \n",
+            "ASSISTANT:",
+            "USER:",
+            "SYSTEM:",
+        ],
+    )
+    # for out in output:
+    #     stream = copy.deepcopy(out)
+    #     temp += stream["choices"][0]["text"]
+    #     yield temp
+
+    history = ["init", input_prompt]
+
+    print(history)
+    print(output)
+    return output
+
+app = FastAPI()
+
+@app.post("/api/generate")
+async def generate(request: Request):
+    # Receive the request as JSON
+    data = await request.json()
+    # Check if the event is a completed order
+    if data['message']:
+        response = generate_text(data['message'], history)
+        return {"status": "success", "data":response}
+    else:
+        # If the event is not what we're looking for, ignore it
+        return {"status": "ignored"}