Added to intro, temp disabled web page load, address pdf load issue

.gitignore

@@ -1,2 +1,3 @@
 *.pyc
-*.ipynb
+*.ipynb
+*.pdf

app.py

@@ -63,6 +63,8 @@ def docs_to_faiss_save(docs_out:PandasDataFrame, embeddings=embeddings):
 
     print(f"> Total split documents: {len(docs_out)}")
 
+    print(docs_out)
+
     vectorstore_func = FAISS.from_documents(documents=docs_out, embedding=embeddings)
         
     '''  
@@ -99,7 +101,7 @@ with block:
     #with gr.Row():
     gr.Markdown("<h1><center>Lightweight PDF / web page QA bot</center></h1>")        
     
-    gr.Markdown("By default the Lambeth Borough Plan '[Lambeth 2030 : Our Future, Our Lambeth](https://www.lambeth.gov.uk/better-fairer-lambeth/projects/lambeth-2030-our-future-our-lambeth)' is loaded. If you want to talk about another document or web page, please select below. The chatbot will not answer questions where answered can't be found on the website.\n\nIf switching topic, please click the 'New topic' button as the bot will assume follow up questions are linked to the first. Sources are shown underneath the chat area.")
+    gr.Markdown("Chat with a document (alpha). By default the Lambeth Borough Plan '[Lambeth 2030 : Our Future, Our Lambeth](https://www.lambeth.gov.uk/better-fairer-lambeth/projects/lambeth-2030-our-future-our-lambeth)' is loaded. If you want to talk about another document or web page (feature temporarily disabled), please select below. The chatbot will not answer questions where answered can't be found on the website.\n\nIf switching topic, please click the 'New topic' button as the bot will assume follow up questions are linked to the first. Sources are shown underneath the chat area.\n\nPlease note that LLM chatbots may give incomplete or incorrect information, so please use with care.")
 
     with gr.Tab("Chatbot"):
 
@@ -135,11 +137,11 @@ with block:
             in_pdf = gr.File(label="Upload pdf", file_count="multiple", file_types=['.pdf'])
             load_pdf = gr.Button(value="Load in file", variant="secondary", scale=0)
         
-        with gr.Accordion("Web page", open = False):
+        with gr.Accordion("Web page - Temporarily disabled", open = False):
             with gr.Row():
                 in_web = gr.Textbox(label="Enter webpage url")
                 in_div = gr.Textbox(label="(Advanced) Webpage div for text extraction", value="p", placeholder="p")
-            load_web = gr.Button(value="Load in webpage", variant="secondary", scale=0) 
+            load_web = gr.Button(value="Load in webpage", variant="secondary", scale=0, visible=False) 
         
         ingest_embed_out = gr.Textbox(label="File/webpage preparation progress")
 
@@ -151,7 +153,7 @@ with block:
     ingest_metadata = gr.State()
     ingest_docs = gr.State()
 
-    #embeddings_state = gr.State()
+    embeddings_state = gr.State()
     vectorstore_state = gr.State()
 
     chat_history_state = gr.State()
@@ -163,7 +165,7 @@ with block:
     # Load in a pdf
     load_pdf_click = load_pdf.click(ing.parse_file, inputs=[in_pdf], outputs=[ingest_text]).\
              then(ing.text_to_docs, inputs=[ingest_text], outputs=[ingest_docs]).\
-             then(docs_to_faiss_save, inputs=[ingest_docs], outputs=ingest_embed_out)
+             then(docs_to_faiss_save, inputs=[ingest_docs], outputs=ingest_embed_out) # #then(load_embeddings, outputs=[embeddings_state]).\
              #then(hide_examples)
 
     # Load in a webpage

chatfuncs/ingest.py

@@ -272,6 +272,8 @@ def parse_html(page_url, div_filter="p"):
     texts.append(clean_text)
     metadatas.append({"source": page_url, "date":str(date)})
 
+    print(metadatas)
+
     return texts, metadatas
 
 # +
@@ -300,9 +302,10 @@ def text_to_docs(text_dict: dict, chunk_size: int = chunk_size) -> List[Document
             print(f"Unsupported file type {ext} for {file_path}. Skipping.")
             continue
 
-
-        match = re.search(r'.*[\/\\](.+)$', file_path)
-        filename_end = match.group(1)
+        #match = re.search(r'.*[\/\\](.+)$', file_path)
+        match = re.search(r'(.*[\/\\])?(.+)$', file_path)
+        
+        filename_end = match.group(2) if match else ''
 
         # Add filename as metadata
         for doc in docs: doc.metadata["source"] = filename_end
@@ -311,7 +314,7 @@ def text_to_docs(text_dict: dict, chunk_size: int = chunk_size) -> List[Document
         doc_sections.extend(docs)
         #parent_doc_sections.extend(parent_docs)
 
-    return doc_sections, page_docs
+    return doc_sections#, page_docs
 
 def pdf_text_to_docs(text, chunk_size: int = chunk_size) -> List[Document]:
     """Converts a string or list of strings to a list of Documents
@@ -378,7 +381,9 @@ def html_text_to_docs(texts, metadatas, chunk_size:int = chunk_size):
     documents = text_splitter.create_documents(texts, metadatas=metadatas)
 
     for i, section in enumerate(documents):
-        section.metadata["section"] = i + 1
+        section.metadata["page_section"] = i + 1
+
+    
 
     return documents
 
@@ -456,7 +461,7 @@ def load_embeddings(model_name = "thenlper/gte-base"):
 
     embeddings = embeddings_func
 
-    #return embeddings_func
+    return embeddings_func
 
 def embed_faiss_save_to_zip(docs_out, save_to="faiss_lambeth_census_embedding", model_name = "thenlper/gte-base"):
 

chatfuncs/ingest_borough_plan.py

@@ -1,7 +1,8 @@
 import ingest as ing
 import pandas as pd
 
-borough_plan_text = ing.parse_file([open("faiss_embedding/Lambeth_2030-Our_Future_Our_Lambeth.pdf")])
+
+borough_plan_text = ing.parse_file([open("Lambeth_2030-Our_Future_Our_Lambeth.pdf")])
 print("Borough plan text created")
 
 #print(borough_plan_text)
@@ -11,6 +12,5 @@ print("Borough plan docs created")
 
 embedding_model = "thenlper/gte-base"
 
-ing.load_embeddings(model_name = embedding_model)
-ing.embed_faiss_save_to_zip(borough_plan_docs, save_to="faiss_embedding", model_name = embedding_model)
-#borough_plan_parent_docs.to_csv("borough_plan_parent_docs.csv", index=False)
+embeddings = ing.load_embeddings(model_name = embedding_model)
+ing.embed_faiss_save_to_zip(borough_plan_docs, save_to="faiss_embedding", model_name = embedding_model)