new gguf format

model.py

@@ -56,26 +56,21 @@ def run(message: str,
         top_k: int = 49,
         repeat_penalty: float = 1.0) -> Iterator[str]:
     global llm
+    llm.reset()
 
     prompt = get_prompt(message, chat_history, system_prompt)
 
-    stop=["</s>"]
-
+    #stop=["</s>"]
+    stop = ["USER:", "ASSISTANT:"]
     outputs = []
-    try:
-        for text in llm(prompt,
-            max_tokens=max_new_tokens,
-            stop=stop,
-            temperature=temperature,
-            top_p=top_p,
-            top_k=top_k,
-            repeat_penalty=repeat_penalty,
-            stream=True):
-            outputs.append(text['choices'][0]['text'])
-            yield ''.join(outputs) 
-    except Exception as e:
-        print(f"Error: {e}")
-        yield "Error in llm, reinitialising llm..."
-        del llm
-        llm = load_llm()
-        yield "llm reinitialised." 
+    for text in llm(prompt,
+        max_tokens=max_new_tokens,
+        stop=stop,
+        temperature=temperature,
+        top_p=top_p,
+        top_k=0,
+        repeat_penalty=repeat_penalty,
+        mirostat_mode=2, mirostat_tau=8.0, mirostat_eta=0.2,
+        stream=True):
+        outputs.append(text['choices'][0]['text'])
+        yield ''.join(outputs)

settings.py

@@ -18,7 +18,7 @@ LLAMA_VERBOSE=False
 
 MAX_MAX_NEW_TOKENS = 2048
 DEFAULT_MAX_NEW_TOKENS = 1024
-MAX_INPUT_TOKEN_LENGTH = 4000
+MAX_INPUT_TOKEN_LENGTH = 3072
 
 if IS_LOCAL:
     from settings_local import *