UPDATE: Web Crawler

app.py

@@ -1,8 +1,11 @@
 import io
+import re
 from functions import *
 from PyPDF2 import PdfReader
+from bs4 import BeautifulSoup
 from fastapi import FastAPI, File, UploadFile
 from fastapi.middleware.cors import CORSMiddleware
+from langchain_community.document_loaders import RecursiveUrlLoader
 
 
 app = FastAPI(title = "ConversAI", root_path = "/api/v1")
@@ -47,6 +50,17 @@ async def addText(vectorstore: str, text: str):
     return addDocuments(text = text, vectorstore = vectorstore)
 
 
+@app.post("/addWebsite")
+async def addWebsite(vectorstore: str, websiteUrl: str):
+    def bs4_extractor(html: str) -> str:
+        soup = BeautifulSoup(html, "lxml")
+        return re.sub(r"\n\n+", "\n\n", soup.text).strip()
+    loader = RecursiveUrlLoader(websiteUrl, max_depth=2, timeout = 60, extractor=bs4_extractor)
+    docs = loader.load()
+    text = "\n\n".join([docs[doc].page_content for doc in docs])
+    return addDocuments(text = text, vectorstore = vectorstore)
+
+
 @app.post("/answerQuery")
 async def answerQuestion(query: str, vectorstore: str, llmModel: str = "llama3-70b-8192"):
     return answerQuery(query=query, vectorstore=vectorstore, llmModel=llmModel)

functions.py

@@ -32,7 +32,7 @@ vectorEmbeddings = HuggingFaceEmbeddings(
     model_kwargs = model_kwargs,
     encode_kwargs = encode_kwargs
 )
-sparseEmbeddings = FastEmbedSparse(model = "Qdrant/BM25", threads = 100, parallel = 0)
+sparseEmbeddings = FastEmbedSparse(model = "Qdrant/BM25", parallel = 2)
 prompt = """
 ### Role
     - **Primary Function**: You are an AI chatbot dedicated to assisting users with their inquiries, issues, and requests. Your goal is to deliver excellent, friendly, and efficient responses at all times. Listen attentively, understand user needs, and provide the best assistance possible or direct them to appropriate resources. If a question is unclear, ask for clarification. Always conclude your replies on a positive note.

requirements.txt

@@ -1,3 +1,4 @@
+bs4
 huggingface-hub
 fastapi
 fastembed-gpu