Reverted

pdf2vectorstore.py

@@ -5,7 +5,6 @@ from bs4 import BeautifulSoup
 from pdf2image import convert_from_path
 import pytesseract
 import pickle
-import concurrent.futures
 
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.document_loaders import UnstructuredFileLoader
@@ -24,13 +23,9 @@ def extract_pdf_text(filename):
     pytesseract.pytesseract.tesseract_cmd = 'tesseract'  
     images = convert_from_path(filename)
     text = ""
+    for image in images:
+        text += pytesseract.image_to_string(image)
     
-    with concurrent.futures.ThreadPoolExecutor() as executor:
-        extracted_texts = executor.map(pytesseract.image_to_string, images)
-    
-    for extracted_text in extracted_texts:
-        text += extracted_text
-
     return text
 
 def get_arxiv_pdf_url(paper_link):
@@ -48,14 +43,8 @@ def read_paper(paper_link):
     print("Reading paper...")
     pdf_filename = 'paper.pdf'
     pdf_url = get_arxiv_pdf_url(paper_link)
-
-    with concurrent.futures.ThreadPoolExecutor() as executor:
-        pdf_future = executor.submit(download_pdf, pdf_url, pdf_filename)
-        pdf_future.result()
-
-        text_future = executor.submit(extract_pdf_text, pdf_filename)
-        text = text_future.result()
-
+    download_pdf(pdf_url, pdf_filename)
+    text = extract_pdf_text(pdf_filename)
     os.remove(pdf_filename)
 
     return text
@@ -77,7 +66,7 @@ def convert_to_vectorstore(arxiv_url, api_key):
     documents = text_splitter.split_documents(raw_documents)
     os.environ["OPENAI_API_KEY"] = api_key
     embeddings = OpenAIEmbeddings()
-    vectorstore = FAISS.from_documents(documents, embeddings)
     os.environ["OPENAI_API_KEY"] = ""
+    vectorstore = FAISS.from_documents(documents, embeddings)
 
     return vectorstore