First model version

main.py

@@ -1,17 +1,26 @@
+import os
+
 from langchain.llms import CTransformers
 from fastapi import FastAPI
 from pydantic import BaseModel
 
 file_name = "zephyr-7b-beta.Q4_K_S.gguf"
 config = {
-    "max_new_tokens": 1024,
-    "model_type": "mistral",
-    # "stream": True,
+    "max_new_token": 1024,
+    "repetition_penalty": 1.1,
+    "temperature": 0.5,
+    "top_k": 50,
+    "top_p": 0.9,
+    "stream": True,
+    "threads": int(os.cpu_count() / 2),
 }
 llm = CTransformers(
     model=file_name,
+    model_type="mistral",
+    lib="avx2",
     **config
 )
+print(llm)
 
 
 class validation(BaseModel):

requirements.txt

@@ -5,4 +5,6 @@ uvicorn
 requests
 python-dotenv
 ctransformers
-langchain
+torch
+langchain
+ctransformers