change batch size

app.py

@@ -159,8 +159,8 @@ class UploadDoc:
     
         return documents
     
-def split_docs(documents,chunk_size=500):
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=100)
+def split_docs(documents,chunk_size=1000):
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=200)
     sp_docs = text_splitter.split_documents(documents)
     return sp_docs
 
@@ -168,7 +168,7 @@ def split_docs(documents,chunk_size=500):
 def load_llama2_llamaCpp():
     core_model_name = "llama-2-7b-chat.Q4_0.gguf"
     #n_gpu_layers = 32
-    n_batch = 512
+    n_batch = 8
     callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
     llm = LlamaCpp(
         model_path=core_model_name,
@@ -243,7 +243,7 @@ def main():
             text += page.extract_text()
         db = FAISS.from_texts(text, embeddings)
         qa_chain = ConversationalRetrievalChain(
-            retriever =db.as_retriever(search_type="similarity_score_threshold", search_kwargs={'k':3,  "score_threshold": 0.7}), 
+            retriever =db.as_retriever(search_type="similarity_score_threshold", search_kwargs={'k':2,  "score_threshold": 0.7}), 
             question_generator=question_generator,
             #condense_question_prompt=CONDENSE_QUESTION_PROMPT,
             combine_docs_chain=doc_chain,