Update custom_llm.py

custom_llm.py

@@ -16,21 +16,48 @@ from langchain_core.messages import AIMessage, HumanMessage
 from langchain_community.document_loaders import DirectoryLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.document_loaders import PyMuPDFLoader
-import os
+import os, requests, bs4
 from langchain.embeddings import HuggingFaceEmbeddings
 from langchain.vectorstores import FAISS
 
 
+def load_web(web_url):
+    
+    r = requests.get(web_url)
+    soup=bs4.BeautifulSoup(r.content,"html.parser")
+    # input_list=
+    input_list = [div.text.strip() for div in soup.find_all("div") if div.text.strip() !='']
+
+    unique_strings = {}
+
+    for item in input_list:
+        # Remove '\n' and leading/trailing whitespaces
+        # cleaned_item = item.strip('\n').strip()
+        cleaned_item = item.strip()
+
+        # Check if the cleaned_item is not in the dictionary or if it's shorter
+        if cleaned_item not in unique_strings or len(item) > len(unique_strings[cleaned_item]):
+            # Add the cleaned_item to the dictionary with the original item as value
+            unique_strings[cleaned_item] = item
+
+    # Create a new list with the unique strings
+    result_list = list(unique_strings.values())
+    
+    return result_list
+
+
 def create_vectorstore():
     loader = os.getenv('knowledge_base')
+    web_loader = load_web("https://lintasmediadanawa.com")
     
     splitter = RecursiveCharacterTextSplitter(chunk_size=512, chunk_overlap=20)
     
-    docs = splitter.create_documents([loader])
+    docs = splitter.create_documents([loader]+web_loader)
     
     emb_model = HuggingFaceEmbeddings(model_name='sentence-transformers/paraphrase-multilingual-mpnet-base-v2', encode_kwargs={'normalize_embeddings': True})
     db = FAISS.from_documents(docs, emb_model)
     return db
+    
 
 def custom_chain_with_history(llm, memory):