Update app.py

app.py

@@ -72,12 +72,13 @@ from langchain.chains import RetrievalQA
 
 from langchain.memory import ConversationBufferMemory
 
+from langchain_community.document_transformers import BeautifulSoupTransformer
 
 
 # Function to index URLs in RAG
 def index_urls_in_rag(urls=[]):
       # Load the RAG model
-    rag_model = "BAAI/bge-base-en-v1.5"
+    rag_model = "jinaai/jina-embeddings-v2-base-de"
     encode_kwargs = {
         "normalize_embeddings": True
     }  # set True to compute cosine similarity
@@ -96,9 +97,15 @@ def index_urls_in_rag(urls=[]):
         loader = WebBaseLoader(url)
         document = loader.load()
 
+        # Transform
+        bs_transformer = BeautifulSoupTransformer()
+        docs_transformed = bs_transformer.transform_documents(
+            document, class_to_extract=["p", "li", "div", "a"]
+        )
+        
         # Split the document into chunks
         text_splitter = RecursiveCharacterTextSplitter()
-        document_chunks = text_splitter.split_documents(document)
+        document_chunks = text_splitter.split_documents(docs_transformed)
         print(document_chunks)
         # Index document chunks into the vector store
         vector_store.add_documents(document_chunks)