Update app.py

app.py

@@ -12,25 +12,12 @@ from llama_cpp import Llama
 
 
 SYSTEM_PROMPT = "Ты — Сайга, русскоязычный автоматический ассистент. Ты разговариваешь с людьми и помогаешь им."
-SYSTEM_TOKEN = 1788
-USER_TOKEN = 1404
-BOT_TOKEN = 9225
-LINEBREAK_TOKEN = 13
-
-
-ROLE_TOKENS = {
-    "user": USER_TOKEN,
-    "bot": BOT_TOKEN,
-    "system": SYSTEM_TOKEN
-}
 
 
 def get_message_tokens(model, role, content):
-    message_tokens = model.tokenize(content.encode("utf-8"))
-    message_tokens.insert(1, ROLE_TOKENS[role])
-    message_tokens.insert(2, LINEBREAK_TOKEN)
-    message_tokens.append(model.token_eos())
-    return message_tokens
+    content = f"{role}\n{content}\n</s>"
+    content = content.encode("utf-8")
+    return model.tokenize(content, special=True)
 
 
 def get_system_tokens(model):
@@ -38,18 +25,31 @@ def get_system_tokens(model):
     return get_message_tokens(model, **system_message)
 
 
-repo_name = "IlyaGusev/saiga2_13b_gguf"
-model_name = "model-q4_K.gguf"
+def load_model(
+    directory: str = ".",
+    model_name: str = "model-q4_K.gguf",
+    model_url: str = "https://huggingface.co/IlyaGusev/saiga2_13b_gguf/resolve/main/model-q4_K.gguf"
+):
+    final_model_path = os.path.join(directory, model_name)
+    
+    print("Downloading all files...")
+    if not os.path.exists(final_model_path):
+        with open(final_model_path, "wb") as f:
+            http_get(model_url, f)
+    os.chmod(final_model_path, 0o777)
+    print("Files downloaded!")
+    
+    model = Llama(
+        model_path=final_model_path,
+        n_ctx=2048
+    )
+    
+    print("Model loaded!")
+    return model
 
-snapshot_download(repo_id=repo_name, local_dir=".", allow_patterns=model_name)
 
-model = Llama(
-    model_path=model_name,
-    n_ctx=2000,
-    n_parts=1,
-)
+MODEL = load_model()
 
-max_new_tokens = 1500
 
 def user(message, history):
     new_history = history + [[message, None]]
@@ -63,6 +63,7 @@ def bot(
     top_k,
     temp
 ):
+    model = MODEL
     tokens = get_system_tokens(model)[:]
     tokens.append(LINEBREAK_TOKEN)
 
@@ -88,7 +89,7 @@ def bot(
 
     partial_text = ""
     for i, token in enumerate(generator):
-        if token == model.token_eos() or (max_new_tokens is not None and i >= max_new_tokens):
+        if token == model.token_eos():
             break
         partial_text += model.detokenize([token]).decode("utf-8", "ignore")
         history[-1][1] = partial_text