Update space

app.py

@@ -23,70 +23,6 @@ from bs4 import BeautifulSoup
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
 """ 
 
-# List of files
-pdf_files = [
-    #"RCP0032 Intake 10 Student Internship Summary Reports.pdf",
-    #"Research Computing Platform Student Internship Handbook.pdf",
-    #"Student Projects Outline - Summer 2425.pdf"
-]
-
-urls = [
-    'https://wehi-researchcomputing.github.io/email_acknowledgement', 
-    'https://wehi-researchcomputing.github.io/email-week-two', 
-    'https://wehi-researchcomputing.github.io/code-of-conduct', 
-    'https://wehi-researchcomputing.github.io/faq#i-need-to-use-storage-and-compute-what-are-my-options', 
-    'https://wehi-researchcomputing.github.io/student-schex', 
-    'https://wehi-researchcomputing.github.io/student-cryoem', 
-    'https://wehi-researchcomputing.github.io/email-week-three', 
-    'https://wehi-researchcomputing.github.io/students#key-documents-to-review-and-faq', 
-    'https://wehi-researchcomputing.github.io/intake_dates', 
-    'https://wehi-researchcomputing.github.io/assets/rcp_logo.png', 
-    'https://wehi-researchcomputing.github.io/project-wikis',
-    'https://wehi-researchcomputing.github.io/student-flux', 
-    'https://wehi-researchcomputing.github.io/explanation_about_ohs', 
-    'https://wehi-researchcomputing.github.io/top-5-mistakes', 
-    'https://wehi-researchcomputing.github.io/student-quantum', 
-    'https://wehi-researchcomputing.github.io/student-immunology-web-application', 
-    'https://wehi-researchcomputing.github.io/how-to-apply', 
-    'https://wehi-researchcomputing.github.io/email-week-one', 
-    'https://wehi-researchcomputing.github.io/student-genomics-metadata.html', 
-    'https://wehi-researchcomputing.github.io/students', 
-    'https://wehi-researchcomputing.github.io/student-haemosphere', 
-    'https://wehi-researchcomputing.github.io/email-week-four', 
-    'https://wehi-researchcomputing.github.io/faq#you-ask-us-to-be-as-independent-as-possible-how-can-we-do-that', 
-    'https://wehi-researchcomputing.github.io/student-imaging', 
-    'https://wehi-researchcomputing.github.io/faq#how-should-i-ask-for-help-to-solve-a-problem', 
-    'https://wehi-researchcomputing.github.io/faq#how-do-i-get-access-to-the-wehi-wide-student-intern-group-using-my-wehi-email-address', 
-    'https://wehi-researchcomputing.github.io/student-data-commons', 
-    'https://wehi-researchcomputing.github.io/faq#what-do-i-need-to-do-for-my-final-presentation-and-summary-report', 
-    'https://wehi-researchcomputing.github.io/complex-projects', 
-    'https://wehi-researchcomputing.github.io/faq#onboarding-onto-the-internship-program-workday', 
-    'https://wehi-researchcomputing.github.io/RDM-0220-RCP-Student-Internship-Handbook.pdf', 
-    'https://wehi-researchcomputing.github.io/email-final-week', 
-    'https://wehi-researchcomputing.github.io/faq#i-know-that-the-internships-are-usually-100-offsite-but-what-if-i-want-to-go-into-the-office-sometimes', 
-    'https://wehi-researchcomputing.github.io/student-bionix', 
-    'https://wehi-researchcomputing.github.io/student-duplex-sequencing', 
-    'https://wehi-researchcomputing.github.io/social_media_policy', 
-    'https://wehi-researchcomputing.github.io/email-week-ten', 
-    'https://wehi-researchcomputing.github.io/student-aive', 
-    'https://wehi-researchcomputing.github.io/software_maturity_model', 
-    'https://wehi-researchcomputing.github.io/student-organiser', 
-    'https://wehi-researchcomputing.github.io/expectations_open_source_contributors', 
-    'https://wehi-researchcomputing.github.io/student-genomics-qc', 
-    'https://wehi-researchcomputing.github.io/student-immunology-modelling', 
-    'https://wehi-researchcomputing.github.io/faq', 
-    'https://wehi-researchcomputing.github.io/student-genomics-invoicing', 
-    'https://wehi-researchcomputing.github.io/emaiL-one-week-before', 
-    'https://wehi-researchcomputing.github.io/student-capacity-planning.html', 
-    'https://wehi-researchcomputing.github.io/email-week-five', 
-    'https://wehi-researchcomputing.github.io/emails-and-key-milestones', 
-    'https://wehi-researchcomputing.github.io/student-clinical-dashboards', 
-    'https://wehi-researchcomputing.github.io/', 
-    'https://wehi-researchcomputing.github.io/student-loxcoder', 
-    'https://wehi-researchcomputing.github.io/student-mixOmics.html',
-    "https://wehi-researchcomputing.github.io/faq#what-are-the-key-things-to-do-before-the-weekly-meetings"
-]
-
 # LLM Model#
 llm = HuggingFacePipeline.from_model_id(
     model_id="meta-llama/Llama-3.2-3B",
@@ -97,35 +33,8 @@ llm = HuggingFacePipeline.from_model_id(
 # Initialize embedding model "all-MiniLM-L6-v2"
 embedding_model = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
 
-if os.path.exists("./chroma_db") and os.listdir("./chroma_db"):
-    # Load the existing ChromaDB database
-    vector_store = Chroma(persist_directory="./chroma_db", embedding_function=embedding_model)
-else:
-    # Initialize an empty list to store all documents
-    all_docs = []
-
-    # Load PDFs with metadata
-    for pdf_file in pdf_files:
-        pdf_loader = PyPDFLoader(pdf_file)
-        pdf_docs = pdf_loader.load()
-        for doc in pdf_docs:
-            doc.metadata["source"] = pdf_file  # Add source metadata
-        all_docs.extend(pdf_docs)
-
-    # Load URLs with metadata
-    for url in urls:
-        url_loader = WebBaseLoader(url)
-        web_docs = url_loader.load()
-        for doc in web_docs:
-            doc.metadata["source"] = url  # Add source metadata
-        all_docs.extend(web_docs)
-
-    # Split documents into chunks
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=256, chunk_overlap=50)
-    chunks = text_splitter.split_documents(all_docs)
-
-    vector_store = Chroma.from_documents(chunks, embedding_model, persist_directory="./chroma_db")
-
+# Load the existing ChromaDB database
+vector_store = Chroma(persist_directory="./chroma_db", embedding_function=embedding_model)
 
 # See full prompt at https://smith.langchain.com/hub/rlm/rag-prompt
 # Basically a solid prompt for RAG