Upload process_documents.py

process_documents.py

@@ -10,12 +10,15 @@ deep_strip = lambda text: re.sub(r"\s+", " ", text or "").strip()
 
 def process_documents(urls):
     snippets = []
+    documents = []
     for source_id, url in enumerate(urls):
         if url.endswith(".pdf"):
             snippets.extend(process_pdf(url, source_id))
+            documents.append("\n".join([snip.page_content for snip in snippets]))
         else:
             snippets.extend(process_web(url, source_id))
-    return snippets
+            documents.append("\n".join([snip.page_content for snip in snippets]))
+    return snippets, documents
 
 
 def process_web(url, source_id):