Update app.py

app.py

@@ -7,6 +7,15 @@ import langchain
 import chromadb
 import glob
 
+import pickle
+
+from PyPDF2 import PdfReader
+from PyPDF2 import PdfWriter
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.vectorstores import ElasticVectorSearch, Pinecone, Weaviate, FAISS
+from langchain.chains.question_answering import load_qa_chain
+from langchain.llms import OpenAI
 
 from langchain.embeddings.openai import OpenAIEmbeddings
 from langchain.vectorstores import Chroma
@@ -28,6 +37,9 @@ from langchain.chains.question_answering import load_qa_chain
 
 
 
+
+
+
 def get_empty_state():
     return {"total_tokens": 0, "messages": []}
 
@@ -69,22 +81,51 @@ def submit_message(prompt, prompt_template, temperature, max_tokens, context_len
     os.environ["OPENAI_API_KEY"] = os.environ['openai_key']
     
     # load in all the files
-    path = './files'
+    #path = './files'
+    #pdf_files = glob.glob(os.path.join(path, "*.pdf"))
     #pdf_files = glob.glob(os.path.join(path, "*.pdf"))
-    pdf_files = glob.glob(os.path.join(path, "*.pdf"))
 
-    for file in pdf_files:
-      loader = PyPDFLoader(file)
-      pages = loader.load_and_split()
-      text_splitter = TokenTextSplitter(chunk_size=1000, chunk_overlap=0)
-      split_pages = text_splitter.split_documents(pages)
+    #for file in pdf_files:
+     # loader = PyPDFLoader(file)
+     # pages = loader.load_and_split()
+     # text_splitter = TokenTextSplitter(chunk_size=1000, chunk_overlap=0)
+     # split_pages = text_splitter.split_documents(pages)
      
-    persist_directory = "./embeddings"
-    embeddings = OpenAIEmbeddings()
-    vectordb = Chroma.from_documents(split_pages, embeddings, persist_directory=persist_directory)
-    vectordb.persist()
-
+    #persist_directory = "./embeddings"
+    #embeddings = OpenAIEmbeddings()
+    #vectordb = Chroma.from_documents(split_pages, embeddings, persist_directory=persist_directory)
+    #vectordb.persist()
 
+    path = './files'
+    pdf_files = glob.glob(os.path.join(path, "*.pdf"))
+    
+    merger = PdfWriter()
+    
+    # add all file in the list to the merger object
+    for pdf in pdf_files:
+      merger.append(pdf)
+    merger.write("merged-pdf.pdf")
+    merger.close()
+    
+    reader = PdfReader("merged-pdf.pdf")
+    raw_text = ''
+    for i, page in enumerate(reader.pages):
+      text = page.extract_text()
+      if text:
+          raw_text += text
+    text_splitter = CharacterTextSplitter(        
+      separator = "\n",
+      chunk_size = 1000,
+      chunk_overlap  = 200,
+      length_function = len,
+    )
+    texts = text_splitter.split_text(raw_text)
+    len(texts)
+    embeddings = OpenAIEmbeddings()
+        
+        
+    with open("foo.pkl", 'wb') as f:
+        pickle.dump(embeddings, f)
 
     
     history = state['messages']
@@ -111,16 +152,27 @@ def submit_message(prompt, prompt_template, temperature, max_tokens, context_len
         #query = str(system_prompt + history[-context_length*2:] +  [prompt_msg])
         #completion = completion.run(query)
         # from https://blog.devgenius.io/chat-with-document-s-using-openai-chatgpt-api-and-text-embedding-6a0ce3dc8bc8
-        completion_chain = load_qa_chain(ChatOpenAI(temperature=temperature, max_tokens=max_tokens, model_name="gpt-3.5-turbo"), chain_type="stuff" )
-        completion = RetrievalQA(combine_documents_chain=completion_chain, retriever=vectordb.as_retriever(), return_source_documents=False)
+        #completion_chain = load_qa_chain(ChatOpenAI(temperature=temperature, max_tokens=max_tokens, model_name="gpt-3.5-turbo"), chain_type="stuff" )
+        #completion = RetrievalQA(combine_documents_chain=completion_chain, retriever=vectordb.as_retriever(), return_source_documents=False)
         #completion = RetrievalQA.from_chain_type(llm=ChatOpenAI(temperature=temperature, max_tokens=max_tokens, model_name="gpt-3.5-turbo"), chain_type="stuff", retriever=vectordb.as_retriever(), return_source_documents=True)
-        query = str(system_prompt + history[-context_length*2:] +  [prompt_msg])
+        #query = str(system_prompt + history[-context_length*2:] +  [prompt_msg])
         #completion = completion({"query": query})
-        completion = completion.run(query)
+        #completion = completion.run(query)
 
 #        completion = completion({"question": query, "chat_history": history[-context_length*2:]})
 
+        with open("foo.pkl", 'rb') as f:
+            new_docsearch = pickle.load(f)
+        
+        docsearch = FAISS.from_texts(texts, new_docsearch)
+        query = str(system_prompt + history[-context_length*2:] +  [prompt_msg])
+        docs = docsearch.similarity_search(query)
+        #print(docs[0].page_content)
 
+        chain = load_qa_chain(ChatOpenAI(temperature=temperature, max_tokens=max_tokens, model_name="gpt-3.5-turbo"), chain_type="stuff")
+        completion = chain.run(input_documents=docs, question=query)
+        
+        
 # VectorDBQA.from_chain_type(llm=OpenAI(), chain_type="stuff", vectorstore=docsearch, return_source_documents=True)
 # https://colab.research.google.com/drive/1dzdNDZyofRB0f2KIB4gHXmIza7ehMX30?usp=sharing#scrollTo=b-ejDn_JfpWW