optimize code

README.md

@@ -43,4 +43,7 @@ After that, you can starting chat with your custom bot about the topic in your U
 
 The vector index storage by this structure: 
 
-![Screenshot 2023-11-13 at 20.03.04.png](..%2F..%2F..%2F..%2Fvar%2Ffolders%2Fzc%2Fcsmhsgrd0bz3bbkycljwdk2c0000gn%2FT%2FTemporaryItems%2FNSIRD_screencaptureui_ZTP7r9%2FScreenshot%202023-11-13%20at%2020.03.04.png)
+![Screenshot 2023-11-13 at 20.03.04.png](..%2F..%2F..%2F..%2Fvar%2Ffolders%2Fzc%2Fcsmhsgrd0bz3bbkycljwdk2c0000gn%2FT%2FTemporaryItems%2FNSIRD_screencaptureui_ZTP7r9%2FScreenshot%202023-11-13%20at%2020.03.04.png)
+
+I've already adding the data from this page: https://www.presight.io/privacy-policy.html
+You can check and chat with bot for Information in that or You can training more knowledge for it

app.py

@@ -1,3 +1,4 @@
+import gradio as gr
 from typing import Optional, Tuple
 from queue import Empty, Queue
 from threading import Thread
@@ -9,15 +10,15 @@ from bot.web_scrapping.default import *
 from langchain.chat_models import ChatOpenAI
 from langchain.prompts import HumanMessagePromptTemplate
 from langchain.schema import AIMessage, BaseMessage, HumanMessage, SystemMessage
-import gradio as gr
 
-set_api_key()
+set_api_key(api_key='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')
 human_message_prompt_template = HumanMessagePromptTemplate.from_template("{text}")
 
 
 def bot_learning(urls, file_formats, llm, prompt, chat_mode=False):
+    index = content_crawler_and_index(url=str(urls), llm=llm, prompt=prompt, file_format=file_formats)
     if chat_mode:
-        return content_crawler_and_index(url=str(urls), llm=llm, prompt=prompt, file_format=file_formats)
+        return index
     else:
         return 'Training Completed'
 
@@ -63,7 +64,6 @@ def chat_start(
         except Empty:
             continue
     messages.append(AIMessage(content=content))
-    logger.info(f"Done!")
     return chat, "", chatbot_messages, messages
 
 

bot/utils/constanst.py

@@ -5,4 +5,4 @@ API_KEY = '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'
 
 def set_api_key(api_key=API_KEY):
     os.environ['OPENAI_API_KEY'] = api_key
-    return 'API KEY SUCCESSFULLY'
+    return True

bot/web_scrapping/crawler_and_indexer.py

@@ -2,22 +2,16 @@ from bs4 import BeautifulSoup
 from urllib import request
 from bot.web_scrapping.searchable_index import SearchableIndex
 from bot.utils.show_log import logger
-from bot.utils.constanst import set_api_key
-import pandas as pd
 import requests
 import os
 
-set_api_key(api_key='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')
-
 
 def save_content_to_file(url=None, text=None, output_folder=None, file_format=None):
     file_path = os.path.join(output_folder, f"combined_content.{file_format}")
 
     write_functions = {
         'txt': lambda: write_text(file_path, text),
-        'pdf': lambda: write_pdf(url, file_path),
-        'csv': lambda: write_csv(file_path, text),
-        'xml': lambda: write_xml(file_path, text)
+        'pdf': lambda: write_pdf(url, file_path)
     }
 
     write_function = write_functions.get(file_format)
@@ -40,17 +34,6 @@ def write_pdf(url, file_path):
     request.urlretrieve(url, file_path)
 
 
-def write_csv(file_path, text):
-    df = pd.DataFrame({'Content': [t.text for t in text]})
-    df.to_csv(file_path, mode='a', index=False, header=False)
-
-
-def write_xml(file_path, text):
-    xml_content = ''.join([f'<item>{t.text}</item>' for t in text])
-    with open(file_path, "a", encoding="utf-8") as file:
-        file.write(xml_content)
-
-
 def content_crawler_and_index(url, llm, prompt, file_format='txt', output_folder='learning_documents'):
     if url == 'NO_URL':
         file_path = output_folder
@@ -74,15 +57,3 @@ def content_crawler_and_index(url, llm, prompt, file_format='txt', output_folder
 
 if __name__ == '__main__':
     pass
-    # Example usage:
-    # First URL
-    # idx = content_crawler_and_index("https://www.presight.io/terms-of-use.html", file_format='txt')
-    #
-    # Second URL (appends content to existing files)
-    # idx = content_crawler_and_index(url='https://arxiv.org/pdf/2309.11235v1.pdf', file_format='pdf')
-    # # example get response chatbot
-    # prompt = 'explain the paper'
-    # llm = ChatOpenAI(model_name='gpt-3.5-turbo', temperature=0)
-    # response = SearchableIndex.query(prompt, llm, idx)
-    # print(response)
-    # logger.info(response)

bot/web_scrapping/searchable_index.py

@@ -1,19 +1,13 @@
 from langchain.vectorstores import FAISS
 from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.document_loaders import (
-    PyPDFLoader,
-    DataFrameLoader,
-)
-from langchain.document_loaders.csv_loader import CSVLoader
+from langchain.document_loaders import PyPDFLoader
 from langchain.embeddings.openai import OpenAIEmbeddings
 from langchain.chains.retrieval_qa.base import RetrievalQA
 from langchain.chat_models import ChatOpenAI
 from bot.utils.show_log import logger
-import pandas as pd
 import threading
 import glob
 import os
-import asyncio
 import queue
 
 
@@ -24,7 +18,6 @@ class Query:
         self.index = index
 
     def query(self):
-        """Query the vectorstore."""
         llm = self.llm or ChatOpenAI(model_name='gpt-3.5-turbo', temperature=0)
         chain = RetrievalQA.from_chain_type(
             llm, retriever=self.index.as_retriever()
@@ -37,7 +30,7 @@ class SearchableIndex:
         self.path = path
 
     @classmethod
-    def get_splits(cls, path, target_col=None, sheet_name=None):
+    def get_splits(cls, path):
         extension = os.path.splitext(path)[1].lower()
         doc_list = None
         if extension == ".txt":
@@ -57,19 +50,12 @@ class SearchableIndex:
             for pg in pages:
                 pg_splits = text_split.split_text(pg.page_content)
                 doc_list.extend(pg_splits)
-        elif extension == ".xml":
-            df = pd.read_excel(io=path, engine='openpyxl', sheet_name=sheet_name)
-            df_loader = DataFrameLoader(df, page_content_column=target_col)
-            doc_list = df_loader.load()
-        elif extension == ".csv":
-            csv_loader = CSVLoader(path)
-            doc_list = csv_loader.load()
         if doc_list is None:
             raise ValueError("Unsupported file format")
         return doc_list
 
     @classmethod
-    def merge_or_create_index(cls, index_store, faiss_db, embeddings, logger):
+    def merge_or_create_index(cls, index_store, faiss_db, embeddings, loggers):
         if os.path.exists(index_store):
             local_db = FAISS.load_local(index_store, embeddings)
             local_db.merge_from(faiss_db)
@@ -79,39 +65,35 @@ class SearchableIndex:
             operation_info = "New store creation"
 
         local_db.save_local(index_store)
-        logger.info(f"{operation_info} index completed")
+        loggers.info(f"{operation_info} index completed")
         return local_db
 
     @classmethod
-    def load_index(cls, index_files, embeddings, logger):
+    def load_or_check_index(cls, index_files, embeddings, loggers, result_queue):
         if index_files:
-            return FAISS.load_local(index_files[0], embeddings)
-        logger.warning("Index store does not exist")
+            local_db = FAISS.load_local(index_files[0], embeddings)
+            result_queue.put(local_db)
+            return local_db
+        loggers.warning("Index store does not exist")
         return None
 
     @classmethod
-    def check_and_load_index(cls, index_files, embeddings, logger, result_queue):
-        local_db = cls.load_index(index_files, embeddings, logger)
-        result_queue.put(local_db)
-
-    @classmethod
-    def load_index_asynchronously(cls, index_files, embeddings, logger):
+    def load_index_asynchronously(cls, index_files, embeddings, loggers):
         result_queue = queue.Queue()
         thread = threading.Thread(
-            target=cls.check_and_load_index,
-            args=(index_files, embeddings, logger, result_queue)
+            target=cls.load_or_check_index,
+            args=(index_files, embeddings, loggers, result_queue)
         )
         thread.start()
-        thread.join()  # Wait for the thread to finish
         return result_queue.get()
 
     @classmethod
-    def embed_index(cls, url, path, llm, prompt, target_col=None, sheet_name=None):
+    def embed_index(cls, url, path, llm, prompt):
         embeddings = OpenAIEmbeddings()
 
         if path:
             if url != 'NO_URL':
-                doc_list = cls.get_splits(path, target_col, sheet_name)
+                doc_list = cls.get_splits(path)
                 faiss_db = FAISS.from_texts(doc_list, embeddings)
                 index_store = os.path.splitext(path)[0] + "_index"
                 local_db = cls.merge_or_create_index(index_store, faiss_db, embeddings, logger)
@@ -124,10 +106,3 @@ class SearchableIndex:
 
 if __name__ == '__main__':
     pass
-    # Examples for search query
-    # index = SearchableIndex.embed_index(
-    #     path="/Users/macbook/Downloads/AI_test_exam/ChatBot/learning_documents/combined_content.txt")
-    # prompt = 'show more detail about types of data collected'
-    # llm = ChatOpenAI(model_name='gpt-3.5-turbo', temperature=0)
-    # result = SearchableIndex.query(prompt, llm=llm, index=index)
-    # print(result)

learning_documents/combined_content_index/index.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b0d19bc7afb8227f67225b52afd8c746bc67aceca43fb5e5c84a19e94cda0e9d
-size 3959
+oid sha256:5895a023300d06204d031fa44543d35fa977de8f8808b2e6691775a95ae1ae84
+size 6059