test

main.py

@@ -2,11 +2,12 @@ from ctransformers import AutoModelForCausalLM
 from fastapi import FastAPI
 from pydantic import BaseModel
 
-file_name = "zephyr-7b-beta.Q4_K_S.gguf"
+file_name = "gemma-2b.Q2_K.gguf"
 llm = AutoModelForCausalLM.from_pretrained(file_name,
     model_type='mistral',
-    max_new_tokens = 1096,
-    threads = 3,
+    max_new_tokens=2_096,
+    context_length=8_000,
+    threads=3,
 )
 
 #Pydantic object