Update main.py

main.py

@@ -4,7 +4,7 @@ from pydantic import BaseModel
 
 #Model loading
 llm = AutoModelForCausalLM.from_pretrained("Meta-Llama-3-8B-Instruct-Q4_K_M.gguf",
-model_type='llama',
+# model_type='llama',
 max_new_tokens = 1096,
 threads = 3,
 )
@@ -21,7 +21,11 @@ app = FastAPI()
 @app.post("/llm_on_cpu")
 async def stream(item: validation):
     system_prompt = 'Below is an instruction that describes a task. Write a response that appropriately completes the request.'
-    E_INST = "</s>"
-    user, assistant = "<|user|>", "<|assistant|>"
-    prompt = f"{system_prompt}{E_INST}\n{user}\n{item.prompt.strip()}{E_INST}\n{assistant}\n"
+    prompt = f'''
+    <|begin_of_text|><|start_header_id|>system<|end_header_id|>
+
+    {system_prompt}<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+    {item.prompt.strip()}<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+    '''
     return llm(prompt)