Spaces:
Sleeping
Sleeping
init
Browse files- .gitignore +3 -0
- Dockerfile +32 -0
- app.py +200 -0
- llm_backend.py +152 -0
.gitignore
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
.env/
|
2 |
+
data/
|
3 |
+
.vscode/
|
Dockerfile
ADDED
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
ARG CUDA_IMAGE="12.1.1-devel-ubuntu22.04"
|
2 |
+
FROM nvidia/cuda:${CUDA_IMAGE}
|
3 |
+
|
4 |
+
# We need to set the host to 0.0.0.0 to allow outside access
|
5 |
+
ENV HOST 0.0.0.0
|
6 |
+
RUN useradd -m -u 1000 user
|
7 |
+
WORKDIR /home/user/app
|
8 |
+
COPY --link --chown=1000 ./ /home/user/app
|
9 |
+
|
10 |
+
RUN apt-get update && apt-get upgrade -y \
|
11 |
+
&& apt-get install -y git git-lfs build-essential \
|
12 |
+
python3 python3-pip gcc wget \
|
13 |
+
ocl-icd-opencl-dev opencl-headers clinfo \
|
14 |
+
libclblast-dev libopenblas-dev \
|
15 |
+
&& mkdir -p /etc/OpenCL/vendors && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
|
16 |
+
|
17 |
+
# setting build related env vars
|
18 |
+
ENV CUDA_DOCKER_ARCH=all
|
19 |
+
ENV LLAMA_CUBLAS=1
|
20 |
+
|
21 |
+
# Install depencencies
|
22 |
+
RUN python3 -m pip install --no-cache-dir --upgrade pip pytest cmake scikit-build setuptools fastapi uvicorn sse-starlette pydantic-settings starlette-context huggingface-hub==0.14.1 flask apscheduler
|
23 |
+
|
24 |
+
# Install llama-cpp-python (build with cuda)
|
25 |
+
RUN CMAKE_ARGS="-DLLAMA_CUBLAS=on" pip install --no-cache-dir llama-cpp-python
|
26 |
+
|
27 |
+
RUN git config --global user.email "amatenkov@ntr.ai"
|
28 |
+
RUN git config --global user.name "Andrew Matenkov"
|
29 |
+
|
30 |
+
EXPOSE 7860
|
31 |
+
# Run the server
|
32 |
+
CMD python3 -m app
|
app.py
ADDED
@@ -0,0 +1,200 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from flask import Flask, request, Response
|
2 |
+
import logging
|
3 |
+
from llama_cpp import Llama
|
4 |
+
import threading
|
5 |
+
from huggingface_hub import snapshot_download#, Repository
|
6 |
+
import huggingface_hub
|
7 |
+
import gc
|
8 |
+
import os.path
|
9 |
+
import xml.etree.ElementTree as ET
|
10 |
+
from apscheduler.schedulers.background import BackgroundScheduler
|
11 |
+
from datetime import datetime, timedelta
|
12 |
+
from llm_backend import LlmBackend
|
13 |
+
import json
|
14 |
+
|
15 |
+
llm = LlmBackend()
|
16 |
+
_lock = threading.Lock()
|
17 |
+
|
18 |
+
SYSTEM_PROMPT = os.environ.get('SYSTEM_PROMPT') or "Ты — русскоязычный автоматический ассистент. Ты максимально точно и отвечаешь на запросы пользователя, используя русский язык."
|
19 |
+
|
20 |
+
CONTEXT_SIZE = os.environ.get('CONTEXT_SIZE') or 500
|
21 |
+
ENABLE_GPU = os.environ.get('ENABLE_GPU') or False
|
22 |
+
GPU_LAYERS = os.environ.get('GPU_LAYERS') or 0
|
23 |
+
N_GQA = os.environ.get('N_GQA') or None #must be set to 8 for 70b models
|
24 |
+
CHAT_FORMAT = os.environ.get('CHAT_FORMAT') or 'llama-2'
|
25 |
+
|
26 |
+
# Create a lock object
|
27 |
+
lock = threading.Lock()
|
28 |
+
|
29 |
+
app = Flask(__name__)
|
30 |
+
# Configure Flask logging
|
31 |
+
app.logger.setLevel(logging.DEBUG)
|
32 |
+
|
33 |
+
# Variable to store the last request time
|
34 |
+
last_request_time = datetime.now()
|
35 |
+
|
36 |
+
# Initialize the model when the application starts
|
37 |
+
#model_path = "../models/model-q4_K.gguf" # Replace with the actual model path
|
38 |
+
#model_name = "model/ggml-model-q4_K.gguf"
|
39 |
+
|
40 |
+
#repo_name = "IlyaGusev/saiga2_13b_gguf"
|
41 |
+
#model_name = "model-q4_K.gguf"
|
42 |
+
|
43 |
+
#epo_name = "IlyaGusev/saiga2_70b_gguf"
|
44 |
+
#model_name = "ggml-model-q4_1.gguf"
|
45 |
+
|
46 |
+
repo_name = "IlyaGusev/saiga2_7b_gguf"
|
47 |
+
model_name = "model-q4_K.gguf"
|
48 |
+
local_dir = '.'
|
49 |
+
|
50 |
+
if os.path.isdir('/data'):
|
51 |
+
app.logger.info('Persistent storage enabled')
|
52 |
+
|
53 |
+
model = None
|
54 |
+
|
55 |
+
MODEL_PATH = snapshot_download(repo_id=repo_name, allow_patterns=model_name) + '/' + model_name
|
56 |
+
app.logger.info('Model path: ' + MODEL_PATH)
|
57 |
+
|
58 |
+
DATASET_REPO_URL = "https://huggingface.co/datasets/muryshev/saiga-chat"
|
59 |
+
DATA_FILENAME = "data-saiga-cuda-release.xml"
|
60 |
+
DATA_FILE = os.path.join("dataset", DATA_FILENAME)
|
61 |
+
|
62 |
+
HF_TOKEN = os.environ.get("HF_TOKEN")
|
63 |
+
app.logger.info("hfh: "+huggingface_hub.__version__)
|
64 |
+
|
65 |
+
# repo = Repository(
|
66 |
+
# local_dir="dataset", clone_from=DATASET_REPO_URL, use_auth_token=HF_TOKEN
|
67 |
+
# )
|
68 |
+
|
69 |
+
|
70 |
+
|
71 |
+
# def log(req: str = '', resp: str = ''):
|
72 |
+
# if req or resp:
|
73 |
+
# element = ET.Element("row", {"time": str(datetime.now()) })
|
74 |
+
# req_element = ET.SubElement(element, "request")
|
75 |
+
# req_element.text = req
|
76 |
+
# resp_element = ET.SubElement(element, "response")
|
77 |
+
# resp_element.text = resp
|
78 |
+
|
79 |
+
# with open(DATA_FILE, "ab+") as xml_file:
|
80 |
+
# xml_file.write(ET.tostring(element, encoding="utf-8"))
|
81 |
+
|
82 |
+
# commit_url = repo.push_to_hub()
|
83 |
+
# app.logger.info(commit_url)
|
84 |
+
|
85 |
+
def generate_tokens(model, generator):
|
86 |
+
global stop_generation
|
87 |
+
app.logger.info('generate_tokens started')
|
88 |
+
with lock:
|
89 |
+
try:
|
90 |
+
for token in generator:
|
91 |
+
if token == model.token_eos() or stop_generation:
|
92 |
+
stop_generation = False
|
93 |
+
app.logger.info('End generating')
|
94 |
+
yield b'' # End of chunk
|
95 |
+
break
|
96 |
+
|
97 |
+
token_str = model.detokenize([token])#.decode("utf-8", errors="ignore")
|
98 |
+
yield token_str
|
99 |
+
except Exception as e:
|
100 |
+
app.logger.info('generator exception')
|
101 |
+
app.logger.info(e)
|
102 |
+
yield b'' # End of chunk
|
103 |
+
|
104 |
+
@app.route('/change_context_size', methods=['GET'])
|
105 |
+
def handler_change_context_size():
|
106 |
+
global stop_generation, model
|
107 |
+
stop_generation = True
|
108 |
+
|
109 |
+
new_size = int(request.args.get('size', CONTEXT_SIZE))
|
110 |
+
init_model(new_size, ENABLE_GPU, GPU_LAYERS)
|
111 |
+
|
112 |
+
return Response('Size changed', content_type='text/plain')
|
113 |
+
|
114 |
+
@app.route('/stop_generation', methods=['GET'])
|
115 |
+
def handler_stop_generation():
|
116 |
+
global stop_generation
|
117 |
+
stop_generation = True
|
118 |
+
return Response('Stopped', content_type='text/plain')
|
119 |
+
|
120 |
+
@app.route('/', methods=['GET', 'PUT', 'DELETE', 'PATCH'])
|
121 |
+
def generate_unknown_response():
|
122 |
+
app.logger.info('unknown method: '+request.method)
|
123 |
+
try:
|
124 |
+
request_payload = request.get_json()
|
125 |
+
app.logger.info('payload: '+request.get_json())
|
126 |
+
except Exception as e:
|
127 |
+
app.logger.info('payload empty')
|
128 |
+
|
129 |
+
return Response('What do you want?', content_type='text/plain')
|
130 |
+
|
131 |
+
response_tokens = bytearray()
|
132 |
+
def generate_and_log_tokens(user_request, generator):
|
133 |
+
global response_tokens, last_request_time
|
134 |
+
for token in llm.generate_tokens(generator):
|
135 |
+
if token == b'': # or (max_new_tokens is not None and i >= max_new_tokens):
|
136 |
+
last_request_time = datetime.now()
|
137 |
+
# log(json.dumps(user_request), response_tokens.decode("utf-8", errors="ignore"))
|
138 |
+
response_tokens = bytearray()
|
139 |
+
break
|
140 |
+
response_tokens.extend(token)
|
141 |
+
yield token
|
142 |
+
|
143 |
+
@app.route('/', methods=['POST'])
|
144 |
+
def generate_response():
|
145 |
+
|
146 |
+
app.logger.info('generate_response')
|
147 |
+
with _lock:
|
148 |
+
if not llm.is_model_loaded():
|
149 |
+
app.logger.info('model loading')
|
150 |
+
init_model()
|
151 |
+
|
152 |
+
data = request.get_json()
|
153 |
+
app.logger.info(data)
|
154 |
+
messages = data.get("messages", [])
|
155 |
+
preprompt = data.get("preprompt", "")
|
156 |
+
parameters = data.get("parameters", {})
|
157 |
+
|
158 |
+
# Extract parameters from the request
|
159 |
+
p = {
|
160 |
+
'temperature': parameters.get("temperature", 0.01),
|
161 |
+
'truncate': parameters.get("truncate", 1000),
|
162 |
+
'max_new_tokens': parameters.get("max_new_tokens", 1024),
|
163 |
+
'top_p': parameters.get("top_p", 0.85),
|
164 |
+
'repetition_penalty': parameters.get("repetition_penalty", 1.2),
|
165 |
+
'top_k': parameters.get("top_k", 30),
|
166 |
+
'return_full_text': parameters.get("return_full_text", False)
|
167 |
+
}
|
168 |
+
|
169 |
+
generator = llm.create_chat_generator_for_saiga(messages=messages, parameters=p)
|
170 |
+
app.logger.info('Generator created')
|
171 |
+
|
172 |
+
|
173 |
+
|
174 |
+
|
175 |
+
# Use Response to stream tokens
|
176 |
+
return Response(generate_and_log_tokens(user_request='1', generator=generator), content_type='text/plain', status=200, direct_passthrough=True)
|
177 |
+
|
178 |
+
def init_model():
|
179 |
+
llm.load_model(model_path=MODEL_PATH, context_size=CONTEXT_SIZE, enable_gpu=ENABLE_GPU, gpu_layer_number=GPU_LAYERS, n_gqa=N_GQA)
|
180 |
+
|
181 |
+
# Function to check if no requests were made in the last 5 minutes
|
182 |
+
def check_last_request_time():
|
183 |
+
global last_request_time
|
184 |
+
current_time = datetime.now()
|
185 |
+
if (current_time - last_request_time).total_seconds() > 300: # 5 minutes in seconds
|
186 |
+
# Perform the action (e.g., set a variable)
|
187 |
+
llm.unload_model()
|
188 |
+
app.logger.info(f"Model unloaded at {current_time}")
|
189 |
+
else:
|
190 |
+
app.logger.info(f"No action needed at {current_time}")
|
191 |
+
|
192 |
+
|
193 |
+
if __name__ == "__main__":
|
194 |
+
scheduler = BackgroundScheduler()
|
195 |
+
scheduler.add_job(check_last_request_time, trigger='interval', minutes=1)
|
196 |
+
scheduler.start()
|
197 |
+
|
198 |
+
init_model()
|
199 |
+
|
200 |
+
app.run(host="0.0.0.0", port=7860, debug=True, threaded=True)
|
llm_backend.py
ADDED
@@ -0,0 +1,152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from llama_cpp import Llama
|
2 |
+
import gc
|
3 |
+
import threading
|
4 |
+
|
5 |
+
class LlmBackend:
|
6 |
+
|
7 |
+
SYSTEM_PROMPT = "Ты — русскоязычный автоматический ассистент. Ты максимально точно и отвечаешь на запросы пользователя, используя русский язык."
|
8 |
+
SYSTEM_TOKEN = 1788
|
9 |
+
USER_TOKEN = 1404
|
10 |
+
BOT_TOKEN = 9225
|
11 |
+
LINEBREAK_TOKEN = 13
|
12 |
+
|
13 |
+
ROLE_TOKENS = {
|
14 |
+
"user": USER_TOKEN,
|
15 |
+
"bot": BOT_TOKEN,
|
16 |
+
"system": SYSTEM_TOKEN
|
17 |
+
}
|
18 |
+
|
19 |
+
_instance = None
|
20 |
+
_model = None
|
21 |
+
_lock = threading.Lock()
|
22 |
+
|
23 |
+
def __new__(cls):
|
24 |
+
if cls._instance is None:
|
25 |
+
cls._instance = super(LlmBackend, cls).__new__(cls)
|
26 |
+
return cls._instance
|
27 |
+
|
28 |
+
|
29 |
+
def is_model_loaded(self):
|
30 |
+
return self._model is not None
|
31 |
+
|
32 |
+
def load_model(self, model_path, context_size=2000, enable_gpu=True, gpu_layer_number=35, n_gqa=8, chat_format='llama-2'):
|
33 |
+
|
34 |
+
if self._model is not None:
|
35 |
+
self.unload_model()
|
36 |
+
|
37 |
+
with self._lock:
|
38 |
+
if enable_gpu:
|
39 |
+
self._model = Llama(
|
40 |
+
model_path=model_path,
|
41 |
+
chat_format=chat_format,
|
42 |
+
n_ctx=context_size,
|
43 |
+
n_parts=1,
|
44 |
+
#n_batch=100,
|
45 |
+
logits_all=True,
|
46 |
+
#n_threads=12,
|
47 |
+
verbose=True,
|
48 |
+
n_gpu_layers=gpu_layer_number,
|
49 |
+
n_gqa=n_gqa #must be set for 70b models
|
50 |
+
)
|
51 |
+
return self._model
|
52 |
+
else:
|
53 |
+
self._model = Llama(
|
54 |
+
model_path=model_path,
|
55 |
+
chat_format=chat_format,
|
56 |
+
n_ctx=context_size,
|
57 |
+
n_parts=1,
|
58 |
+
#n_batch=100,
|
59 |
+
logits_all=True,
|
60 |
+
#n_threads=12,
|
61 |
+
verbose=True,
|
62 |
+
n_gqa=n_gqa #must be set for 70b models
|
63 |
+
)
|
64 |
+
return self._model
|
65 |
+
|
66 |
+
def set_system_prompt(self, prompt):
|
67 |
+
with self._lock:
|
68 |
+
self.SYSTEM_PROMPT = prompt
|
69 |
+
|
70 |
+
def unload_model(self):
|
71 |
+
with self._lock:
|
72 |
+
if self._model is not None:
|
73 |
+
self._model.llama_free_model()
|
74 |
+
del self._model
|
75 |
+
|
76 |
+
def generate_tokens(self, generator):
|
77 |
+
print('generate_tokens called')
|
78 |
+
with self._lock:
|
79 |
+
print('generate_tokens started')
|
80 |
+
try:
|
81 |
+
for token in generator:
|
82 |
+
if token == self._model.token_eos():
|
83 |
+
print('End generating')
|
84 |
+
yield b'' # End of chunk
|
85 |
+
break
|
86 |
+
|
87 |
+
token_str = self._model.detokenize([token])#.decode("utf-8", errors="ignore")
|
88 |
+
yield token_str
|
89 |
+
except Exception as e:
|
90 |
+
print('generator exception')
|
91 |
+
print(e)
|
92 |
+
yield b'' # End of chunk
|
93 |
+
|
94 |
+
def create_chat_completion(self, messages, stream=True):
|
95 |
+
print('create_chat_completion called')
|
96 |
+
with self._lock:
|
97 |
+
print('create_chat_completion started')
|
98 |
+
try:
|
99 |
+
return self._model.create_chat_completion(messages=messages, stream=stream)
|
100 |
+
except Exception as e:
|
101 |
+
print('create_chat_completion exception')
|
102 |
+
print(e)
|
103 |
+
return None
|
104 |
+
|
105 |
+
|
106 |
+
def get_message_tokens(self, role, content):
|
107 |
+
message_tokens = self._model.tokenize(content.encode("utf-8"))
|
108 |
+
message_tokens.insert(1, self.ROLE_TOKENS[role])
|
109 |
+
message_tokens.insert(2, self.LINEBREAK_TOKEN)
|
110 |
+
message_tokens.append(self._model.token_eos())
|
111 |
+
return message_tokens
|
112 |
+
|
113 |
+
def get_system_tokens(self):
|
114 |
+
system_message = {
|
115 |
+
"role": "system",
|
116 |
+
"content": self.SYSTEM_PROMPT
|
117 |
+
}
|
118 |
+
return self.get_message_tokens(self._model, **system_message)
|
119 |
+
|
120 |
+
def create_chat_generator_for_saiga(self, messages, parameters):
|
121 |
+
print('create_chat_completion called')
|
122 |
+
with self._lock:
|
123 |
+
tokens = self.get_system_tokens()
|
124 |
+
for message in messages:
|
125 |
+
message_tokens = self.get_message_tokens(role=message.get("from"), content=message.get("content", ""))
|
126 |
+
tokens.extend(message_tokens)
|
127 |
+
|
128 |
+
tokens.extend([self._model.token_bos(), self.BOT_TOKEN, self.LINEBREAK_TOKEN])
|
129 |
+
generator = self._model.generate(
|
130 |
+
tokens,
|
131 |
+
top_k=parameters['top_k'],
|
132 |
+
top_p=parameters['top_p'],
|
133 |
+
temp=parameters['temperature'],
|
134 |
+
repeat_penalty=parameters['repetition_penalty']
|
135 |
+
)
|
136 |
+
return generator
|
137 |
+
|
138 |
+
def generate_tokens(self, generator):
|
139 |
+
with self._lock:
|
140 |
+
try:
|
141 |
+
for token in generator:
|
142 |
+
if token == self._model.token_eos():
|
143 |
+
yield b'' # End of chunk
|
144 |
+
break
|
145 |
+
|
146 |
+
token_str = self._model.detokenize([token])#.decode("utf-8", errors="ignore")
|
147 |
+
yield token_str
|
148 |
+
except Exception as e:
|
149 |
+
yield b'' # End of chunk
|
150 |
+
|
151 |
+
|
152 |
+
|