reduce n_ctx

app.py

@@ -168,14 +168,14 @@ def split_docs(documents,chunk_size=1000):
 def load_llama2_llamaCpp():
     core_model_name = "llama-2-7b-chat.Q4_0.gguf"
     #n_gpu_layers = 32
-    n_batch = 8
+    n_batch = 32
     callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
     llm = LlamaCpp(
         model_path=core_model_name,
         #n_gpu_layers=n_gpu_layers,
         n_batch=n_batch,
         callback_manager=callback_manager,
-        verbose=True,n_ctx = 4096, temperature = 0.1, max_tokens = 256
+        verbose=True,n_ctx = 1024, temperature = 0.1, max_tokens = 256
     )
     return llm