Optimised parsing

app.py

@@ -137,7 +137,7 @@ with block:
                 <p>Powered by <a href='https://openai.com/'>OpenAI</a>, <a href='https://arxiv.org/'>arXiv</a> and <a href='https://github.com/hwchase17/langchain'>LangChain 🦜️🔗</a></p>
                 <p>ArxivGPT is a chatbot that answers questions about research papers. It uses a pretrained GPT-3.5 model to generate answers.</p>
                 <p>Currently, it can answer questions about the paper you just linked and can also answer questions about the paper's contents.</p>
-                <p>It's still in development, so please report any bugs you find. It can take up to a minute to start a conversation for every new paper as there is a parsing delay.</p>
+                <p>It's still in development, so please report any bugs you find.</p>
                 <p>The answers can be quite limited as there is a 4097 token limit for GPT-3.5, hence waiting for GPT-4 access to upgrade.</p>
                 <p>Possible upgrades coming up: GPT-4, faster parsing, status messages, other research paper hubs.</p>
             </div>

pdf2vectorstore.py

@@ -5,6 +5,7 @@ from bs4 import BeautifulSoup
 from pdf2image import convert_from_path
 import pytesseract
 import pickle
+import concurrent.futures
 
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.document_loaders import UnstructuredFileLoader
@@ -23,9 +24,13 @@ def extract_pdf_text(filename):
     pytesseract.pytesseract.tesseract_cmd = 'tesseract'  
     images = convert_from_path(filename)
     text = ""
-    for image in images:
-        text += pytesseract.image_to_string(image)
     
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        extracted_texts = executor.map(pytesseract.image_to_string, images)
+    
+    for extracted_text in extracted_texts:
+        text += extracted_text
+
     return text
 
 def get_arxiv_pdf_url(paper_link):
@@ -43,8 +48,14 @@ def read_paper(paper_link):
     print("Reading paper...")
     pdf_filename = 'paper.pdf'
     pdf_url = get_arxiv_pdf_url(paper_link)
-    download_pdf(pdf_url, pdf_filename)
-    text = extract_pdf_text(pdf_filename)
+
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        pdf_future = executor.submit(download_pdf, pdf_url, pdf_filename)
+        pdf_future.result()
+
+        text_future = executor.submit(extract_pdf_text, pdf_filename)
+        text = text_future.result()
+
     os.remove(pdf_filename)
 
     return text
@@ -66,7 +77,7 @@ def convert_to_vectorstore(arxiv_url, api_key):
     documents = text_splitter.split_documents(raw_documents)
     os.environ["OPENAI_API_KEY"] = api_key
     embeddings = OpenAIEmbeddings()
-    os.environ["OPENAI_API_KEY"] = ""
     vectorstore = FAISS.from_documents(documents, embeddings)
+    os.environ["OPENAI_API_KEY"] = ""
 
     return vectorstore