chenhaodev commited on
Commit
4eec78b
1 Parent(s): d40b4e3

Upload 3 files

Browse files
Files changed (3) hide show
  1. Dockerfile +24 -0
  2. app.py +53 -0
  3. requirements.txt +2 -0
Dockerfile ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Use an alias for the base image for easier updates
2
+ FROM python:3.10 as base
3
+
4
+ # Set model
5
+ ENV MODEL=mlabonne/UltraMerge-7B-GGUF
6
+ ENV QUANT=Q6_K
7
+ ENV CHAT_TEMPLATE=llama-2
8
+
9
+ # Set the working directory
10
+ WORKDIR /app
11
+
12
+ # Install Python requirements
13
+ COPY ./requirements.txt /app/
14
+ RUN pip install --no-cache-dir --upgrade -r requirements.txt
15
+
16
+ # Download model
17
+ RUN MODEL_NAME_FILE=$(echo ${MODEL#*/} | tr '[:upper:]' '[:lower:]' | sed 's/-gguf$//') && \
18
+ wget https://huggingface.co/mradermacher/UltraMerge-7B-GGUF/resolve/main/UltraMerge-7B.Q6_K.gguf -O model.gguf
19
+
20
+ # Copy the rest of your application
21
+ COPY . .
22
+
23
+ # Command to run the application
24
+ CMD ["python", "app.py"]
app.py ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import json
3
+ import gradio as gr
4
+ from llama_cpp import Llama
5
+
6
+ # Get environment variables
7
+ model_id = os.getenv('MODEL')
8
+ quant = os.getenv('QUANT')
9
+ chat_template = os.getenv('CHAT_TEMPLATE')
10
+
11
+ # Interface variables
12
+ model_name = model_id.split('/')[1].split('-GGUF')[0]
13
+ title = f"llm-2x34b-gguf-chat-v1: {model_name}"
14
+ description = f"Chat with <a href=\"https://huggingface.co/{model_id}\">{model_name}</a> in GGUF format ({quant})!"
15
+
16
+ # Initialize the LLM
17
+ llm = Llama(model_path="model.gguf",
18
+ n_ctx=32768,
19
+ n_threads=2,
20
+ chat_format=chat_template)
21
+
22
+ # Function for streaming chat completions
23
+ def chat_stream_completion(message, history, system_prompt):
24
+ messages_prompts = [{"role": "system", "content": system_prompt}]
25
+ for human, assistant in history:
26
+ messages_prompts.append({"role": "user", "content": human})
27
+ messages_prompts.append({"role": "assistant", "content": assistant})
28
+ messages_prompts.append({"role": "user", "content": message})
29
+
30
+ response = llm.create_chat_completion(
31
+ messages=messages_prompts,
32
+ stream=True
33
+ )
34
+ message_repl = ""
35
+ for chunk in response:
36
+ if len(chunk['choices'][0]["delta"]) != 0 and "content" in chunk['choices'][0]["delta"]:
37
+ message_repl = message_repl + chunk['choices'][0]["delta"]["content"]
38
+ yield message_repl
39
+
40
+ # Gradio chat interface
41
+ gr.ChatInterface(
42
+ fn=chat_stream_completion,
43
+ title=title,
44
+ description=description,
45
+ additional_inputs=[gr.Textbox("You are helpful medical assistant.")],
46
+ additional_inputs_accordion="System prompt",
47
+ examples=[
48
+ ["How to diagnose CHF?"],
49
+ ["please write an extensive cancer patient story and case study for healthcare providers"],
50
+ ["Also, please write journey how the cancer was discovered and what symptoms led to patient realization that something might be wrong."],
51
+ ["Additionally, please write up the intake scenario for her to be seen by a cancer specialist."]
52
+ ]
53
+ ).queue().launch(server_name="0.0.0.0")
requirements.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ llama-cpp-python
2
+ gradio