own knowledge gpt

.gitignore

@@ -0,0 +1,3 @@
+.idea
+.DS_Store
+__pycache__

README.md

@@ -1,12 +1,13 @@
 ---
-title: Own Knowledge GPT
-emoji: 💻
-colorFrom: red
-colorTo: yellow
+title: Presight GPT
+emoji: 🚀
+colorFrom: indigo
+colorTo: red
 sdk: gradio
 sdk_version: 4.2.0
 app_file: app.py
 pinned: false
+license: mit
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,194 @@
+from typing import List, Optional, Tuple
+from queue import Empty, Queue
+from threading import Thread
+from bot.web_scrapping.crawler_and_indexer import content_crawler_and_index
+from bot.web_scrapping.searchable_index import SearchableIndex
+from bot.utils.callbacks import QueueCallback
+from bot.utils.constanst import set_api_key
+from bot.utils.show_log import logger
+from langchain.chat_models import ChatOpenAI
+from langchain.prompts import HumanMessagePromptTemplate
+from langchain.schema import AIMessage, BaseMessage, HumanMessage, SystemMessage
+import gradio as gr
+
+set_api_key()
+MODELS_NAMES = ["gpt-3.5-turbo"]
+DEFAULT_TEMPERATURE = 0.7
+
+ChatHistory = List[str]
+
+default_system_prompt = 'Put your prompt here'
+default_system_format = 'txt'
+human_message_prompt_template = HumanMessagePromptTemplate.from_template("{text}")
+
+
+def learning_feedbacks():
+    return 'Training Completed'
+
+
+def bot_learning(urls, file_formats, chat_mode=False):
+    index = content_crawler_and_index(url=str(urls), file_format=file_formats)
+    if chat_mode:
+        return index
+    else:
+        fb = learning_feedbacks()
+        return fb
+
+
+def chat_start(
+        chat: Optional[ChatOpenAI],
+        message: str,
+        chatbot_messages: ChatHistory,
+        messages: List[BaseMessage], ) -> Tuple[str, str, ChatOpenAI, ChatHistory, List[BaseMessage]]:
+    if not chat:
+        queue = Queue()
+        chat = ChatOpenAI(
+            model_name=MODELS_NAMES[0],
+            temperature=DEFAULT_TEMPERATURE,
+            streaming=True,
+            callbacks=([QueueCallback(queue)])
+        )
+    else:
+        queue = chat.callbacks[0].queue
+
+    job_done = object()
+    messages.append(HumanMessage(content=f':{message}'))
+    chatbot_messages.append((message, ""))
+    index = bot_learning(urls='NO_URL', file_formats='txt', chat_mode=True)
+
+    def query_retrieval():
+        response = SearchableIndex.query(message, chat, index)
+        chatbot_message = AIMessage(content=response)
+        messages.append(chatbot_message)
+        queue.put(job_done)
+
+    t = Thread(target=query_retrieval)
+    t.start()
+    content = ""
+    while True:
+        try:
+            next_token = queue.get(True, timeout=1)
+            if next_token is job_done:
+                break
+            content += next_token
+            chatbot_messages[-1] = (message, content)
+            yield chat, "", chatbot_messages, messages
+        except Empty:
+            continue
+    messages.append(AIMessage(content=content))
+    logger.info(f"Done!")
+    return chat, "", chatbot_messages, messages
+
+
+def system_prompt_handler(value: str) -> str:
+    return value
+
+
+def on_clear_button_click(system_prompt: str) -> Tuple[str, List, List]:
+    return "", [], [SystemMessage(content=system_prompt)]
+
+
+def on_apply_settings_button_click(
+        system_prompt: str, model_name: str, temperature: float
+):
+    logger.info(
+        f"Applying settings: model_name={model_name}, temperature={temperature}"
+    )
+    chat = ChatOpenAI(
+        model_name=model_name,
+        temperature=temperature,
+        streaming=True,
+        callbacks=[QueueCallback(Queue())],
+        max_tokens=1000,
+    )
+    chat.callbacks[0].queue.empty()
+    return chat, *on_clear_button_click(system_prompt)
+
+
+with gr.Blocks() as demo:
+    system_prompt = gr.State(default_system_prompt)
+    messages = gr.State([SystemMessage(content=default_system_prompt)])
+    chat = gr.State(None)
+
+    with gr.Column(elem_id="col_container"):
+        gr.Markdown("# Welcome to OWN-GPT! 🤖")
+        gr.Markdown(
+            "Demo Chat Bot Platform"
+        )
+
+        chatbot = gr.Chatbot()
+        with gr.Column():
+            message = gr.Textbox(label="Type some message")
+            message.submit(
+                chat_start,
+                [chat, message, chatbot, messages],
+                [chat, message, chatbot, messages],
+                queue=True,
+            )
+            message_button = gr.Button("Submit", variant="primary")
+            message_button.click(
+                chat_start,
+                [chat, message, chatbot, messages],
+                [chat, message, chatbot, messages],
+            )
+        with gr.Column():
+            learning_status = gr.Textbox(label='Training Status')
+            url = gr.Textbox(label="URL to Documents")
+            file_format = gr.Textbox(label="Set your file format:", placeholder='Example: pdf, txt')
+            url.submit(
+                bot_learning,
+                [url, file_format],
+                [learning_status]
+            )
+            training_button = gr.Button("Training", variant="primary")
+            training_button.click(
+                bot_learning,
+                [url, file_format],
+                [learning_status]
+            )
+        with gr.Row():
+            with gr.Column():
+                clear_button = gr.Button("Clear")
+                clear_button.click(
+                    on_clear_button_click,
+                    [system_prompt],
+                    [message, chatbot, messages],
+                    queue=False,
+                )
+            with gr.Accordion("Settings", open=False):
+                model_name = gr.Dropdown(
+                    choices=MODELS_NAMES, value=MODELS_NAMES[0], label="model"
+                )
+                temperature = gr.Slider(
+                    minimum=0.0,
+                    maximum=1.0,
+                    value=0.7,
+                    step=0.1,
+                    label="temperature",
+                    interactive=True,
+                )
+                apply_settings_button = gr.Button("Apply")
+                apply_settings_button.click(
+                    on_apply_settings_button_click,
+                    [system_prompt, model_name, temperature],
+                    [chat, message, chatbot, messages],
+                )
+
+        with gr.Column():
+            system_prompt_area = gr.TextArea(
+                default_system_prompt, lines=4, label="prompt", interactive=True
+            )
+            system_prompt_area.input(
+                system_prompt_handler,
+                inputs=[system_prompt_area],
+                outputs=[system_prompt],
+            )
+            system_prompt_button = gr.Button("Set")
+        system_prompt_button.click(
+            on_apply_settings_button_click,
+            [system_prompt, model_name, temperature],
+            [chat, message, chatbot, messages],
+        )
+
+demo.queue()
+demo.launch()

bot/utils/callbacks.py

@@ -0,0 +1,17 @@
+from queue import Queue
+from typing import Any
+
+from langchain.callbacks.base import BaseCallbackHandler
+
+
+class QueueCallback(BaseCallbackHandler):
+    """Callback handler for streaming LLM responses to a queue."""
+
+    def __init__(self, queue: Queue):
+        self.queue = queue
+
+    def on_llm_new_token(self, token: str, **kwargs: Any) -> None:
+        self.queue.put(token)
+
+    def on_llm_end(self, *args, **kwargs: Any) -> None:
+        return self.queue.empty()

bot/utils/constanst.py

@@ -0,0 +1,8 @@
+import os
+
+API_KEY = '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'
+
+
+def set_api_key(api_key=API_KEY):
+    os.environ['OPENAI_API_KEY'] = api_key
+    return 'API KEY SUCCESSFULLY'

bot/utils/show_log.py

@@ -0,0 +1,12 @@
+import logging
+
+# Configure logging to display in terminal only
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+# Create a stream handler to output to the terminal
+stream_handler = logging.StreamHandler()
+stream_handler.setLevel(logging.INFO)
+
+# Get the root logger and add the stream handler
+logger = logging.getLogger()
+logger.addHandler(stream_handler)

bot/web_scrapping/crawler_and_indexer.py

@@ -0,0 +1,85 @@
+from bs4 import BeautifulSoup
+from urllib import request
+from bot.web_scrapping.searchable_index import SearchableIndex
+from bot.utils.show_log import logger
+from bot.utils.constanst import set_api_key
+import pandas as pd
+import requests
+import os
+
+set_api_key(api_key='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')
+
+
+def save_content_to_file(url=None, text=None, output_folder=None, file_format=None):
+    file_path = os.path.join(output_folder, f"combined_content.{file_format}")
+    if file_format == 'txt':
+        with open(f"{file_path}", "a", encoding="utf-8") as file:
+            for t in text:
+                file.write(f'{t.text}\n')
+        logger.info(f"Content appended to {file_path}")
+    elif file_format == 'pdf':
+        request.urlretrieve(url, file_path)
+        logger.info(f"Content appended to {file_path}")
+    elif file_format == 'csv':
+        df = pd.DataFrame({'Content': [t.text for t in text]})
+        df.to_csv(f"{file_path}", mode='a', index=False, header=False)
+        logger.info(f"Content appended to {file_path}")
+    elif file_format == 'xml':
+        xml_content = ''.join([f'<item>{t.text}</item>' for t in text])
+        with open(f"{file_path}", "a", encoding="utf-8") as file:
+            file.write(xml_content)
+        logger.info(f"Content appended to {file_path}")
+    else:
+        logger.warning("Invalid file format. Supported formats: txt, pdf, csv, xml")
+    return file_path
+
+
+def content_crawler_and_index(url, file_format='txt', output_folder='learning_documents'):
+    if url != 'NO_URL':
+        # Send an HTTP GET request to the URL
+        responses = requests.get(url)
+        # Check if the request was successful
+        if responses.status_code == 200:
+            # Create output folder if it doesn't exist
+            if not os.path.exists(output_folder):
+                os.makedirs(output_folder)
+            # Parse the HTML content using BeautifulSoup
+            soup = BeautifulSoup(responses.text, "html.parser")
+            text = soup.find_all(['h2', 'p', 'i', 'ul'])
+            if text:
+                # Save content based on the specified file format
+                file_path = save_content_to_file(text=text, output_folder=output_folder, file_format=file_format)
+
+                # Create or update the index
+                index = SearchableIndex.embed_index(url, file_path)
+                if os.path.isfile(file_path):
+                    os.remove(file_path)
+                return index
+            else:
+                file_path = save_content_to_file(url=url, output_folder=output_folder, file_format=file_format)
+                index = SearchableIndex.embed_index(url, file_path)
+                if os.path.isfile(file_path):
+                    os.remove(file_path)
+                return index
+
+        else:
+            logger.warning("Failed to retrieve content from the URL.")
+    else:
+        index = SearchableIndex.embed_index(url=url, path=output_folder)
+        return index
+
+
+if __name__ == '__main__':
+    pass
+    # Example usage:
+    # First URL
+    # idx = content_crawler_and_index("https://www.presight.io/terms-of-use.html", file_format='txt')
+    #
+    # Second URL (appends content to existing files)
+    # idx = content_crawler_and_index(url='https://arxiv.org/pdf/2309.11235v1.pdf', file_format='pdf')
+    # # example get response chatbot
+    # prompt = 'explain the paper'
+    # llm = ChatOpenAI(model_name='gpt-3.5-turbo', temperature=0)
+    # response = SearchableIndex.query(prompt, llm, idx)
+    # print(response)
+    # logger.info(response)

bot/web_scrapping/searchable_index.py

@@ -0,0 +1,148 @@
+from langchain.vectorstores import FAISS
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.document_loaders import (
+    PyPDFLoader,
+    DataFrameLoader,
+)
+from langchain.document_loaders.csv_loader import CSVLoader
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.chains.retrieval_qa.base import RetrievalQA
+from langchain.chat_models import ChatOpenAI
+from bot.utils.show_log import logger
+import pandas as pd
+import threading
+import glob
+import os
+import queue
+
+
+class SearchableIndex:
+    def __init__(self, path):
+        self.path = path
+
+    def get_text_splits(self):
+        with open(self.path, 'r') as txt:
+            data = txt.read()
+
+        text_split = RecursiveCharacterTextSplitter(chunk_size=1000,
+                                                    chunk_overlap=0,
+                                                    length_function=len)
+        doc_list = text_split.split_text(data)
+        return doc_list
+
+    def get_pdf_splits(self):
+        loader = PyPDFLoader(self.path)
+        pages = loader.load_and_split()
+        text_split = RecursiveCharacterTextSplitter(chunk_size=1000,
+                                                    chunk_overlap=0,
+                                                    length_function=len)
+        doc_list = []
+        for pg in pages:
+            pg_splits = text_split.split_text(pg.page_content)
+            doc_list.extend(pg_splits)
+        return doc_list
+
+    def get_xml_splits(self, target_col, sheet_name):
+        df = pd.read_excel(io=self.path,
+                           engine='openpyxl',
+                           sheet_name=sheet_name)
+
+        df_loader = DataFrameLoader(df,
+                                    page_content_column=target_col)
+
+        excel_docs = df_loader.load()
+
+        return excel_docs
+
+    def get_csv_splits(self):
+        csv_loader = CSVLoader(self.path)
+        csv_docs = csv_loader.load()
+        return csv_docs
+
+    @classmethod
+    def merge_or_create_index(cls, index_store, faiss_db, embeddings, logger):
+        if os.path.exists(index_store):
+            local_db = FAISS.load_local(index_store, embeddings)
+            local_db.merge_from(faiss_db)
+            logger.info("Merge index completed")
+            local_db.save_local(index_store)
+            return local_db
+        else:
+            faiss_db.save_local(folder_path=index_store)
+            logger.info("New store created and loaded...")
+            local_db = FAISS.load_local(index_store, embeddings)
+            return local_db
+
+    @classmethod
+    def check_and_load_index(cls, index_files, embeddings, logger, path, result_queue):
+        if index_files:
+            local_db = FAISS.load_local(index_files[0], embeddings)
+            file_to_remove = os.path.join(path, 'combined_content.txt')
+            if os.path.exists(file_to_remove):
+                os.remove(file_to_remove)
+        else:
+            raise logger.warning("Index store does not exist")
+        result_queue.put(local_db)  # Put the result in the queue
+
+    @classmethod
+    def embed_index(cls, url, path, target_col=None, sheet_name=None):
+        embeddings = OpenAIEmbeddings()
+
+        def process_docs(queues, extension):
+            nonlocal doc_list
+            instance = cls(path)
+            if extension == ".txt":
+                doc_list = instance.get_text_splits()
+            elif extension == ".pdf":
+                doc_list = instance.get_pdf_splits()
+            elif extension == ".xml":
+                doc_list = instance.get_xml_splits(target_col, sheet_name)
+            elif extension == ".csv":
+                doc_list = instance.get_csv_splits()
+            else:
+                doc_list = None
+            queues.put(doc_list)
+
+        if url != 'NO_URL' and path:
+            file_extension = os.path.splitext(path)[1].lower()
+            data_queue = queue.Queue()
+            thread = threading.Thread(target=process_docs, args=(data_queue, file_extension))
+            thread.start()
+            doc_list = data_queue.get()
+            if not doc_list:
+                raise ValueError("Unsupported file format")
+
+            faiss_db = FAISS.from_texts(doc_list, embeddings)
+            index_store = os.path.splitext(path)[0] + "_index"
+            local_db = cls.merge_or_create_index(index_store, faiss_db, embeddings, logger)
+            return local_db, index_store
+        elif url == 'NO_URL' and path:
+            index_files = glob.glob(os.path.join(path, '*_index'))
+
+            result_queue = queue.Queue()  # Create a queue to store the result
+
+            thread = threading.Thread(target=cls.check_and_load_index,
+                                      args=(index_files, embeddings, logger, path, result_queue))
+            thread.start()
+            local_db = result_queue.get()  # Retrieve the result from the queue
+            return local_db
+
+    @classmethod
+    def query(cls, question: str, llm, index):
+        """Query the vectorstore."""
+        llm = llm or ChatOpenAI(model_name='gpt-3.5-turbo', temperature=0)
+        chain = RetrievalQA.from_chain_type(
+            llm, retriever=index.as_retriever()
+        )
+        return chain.run(question)
+
+
+if __name__ == '__main__':
+    pass
+    # Examples for search query
+    # index = SearchableIndex.embed_index(
+    #     path="/Users/macbook/Downloads/AI_test_exam/ChatBot/learning_documents/combined_content.txt")
+    # prompt = 'show more detail about types of data collected'
+    # llm = ChatOpenAI(model_name='gpt-3.5-turbo', temperature=0)
+    # result = SearchableIndex.query(prompt, llm=llm, index=index)
+    # print(result)

bot/web_scrapping/single_crawler.py

@@ -0,0 +1,57 @@
+import os
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+from fpdf import FPDF
+
+
+def content_crawler(url, file_format='txt', output_file='privacy_policy'):
+    # Send an HTTP GET request to the URL
+    response = requests.get(url)
+
+    # Check if the request was successful
+    if response.status_code == 200:
+        # Parse the HTML content using BeautifulSoup
+        soup = BeautifulSoup(response.text, "html.parser")
+        text = soup.find_all(['h2', 'p', 'i', 'ul'])
+
+        # Create output folder if it doesn't exist
+        if not os.path.exists('../learning_documents'):
+            os.makedirs('../learning_documents')
+
+        # Save content based on the specified file format
+        output_path = os.path.join('../learning_documents', output_file)
+
+        if file_format == 'txt':
+            with open(f"{output_path}.txt", "w", encoding="utf-8") as file:
+                for t in text:
+                    file.write(f'{t.text}\n')
+            print(f"Content saved to {output_path}.txt")
+        elif file_format == 'pdf':
+            pdf = FPDF()
+            pdf.set_auto_page_break(auto=True, margin=15)
+            pdf.add_page()
+            pdf.set_font("Arial", "B", 8)
+            for t in text:
+                pdf.cell(0, 10, t.text, ln=True)
+            pdf.output(f"{output_path}.pdf")
+            print(f"Content saved to {output_path}.pdf")
+        elif file_format == 'csv':
+            df = pd.DataFrame({'Content': [t.text for t in text]})
+            df.to_csv(f"{output_path}.csv", index=False)
+            print(f"Content saved to {output_path}.csv")
+        elif file_format == 'xml':
+            xml_content = ''.join([f'<item>{t.text}</item>' for t in text])
+            with open(f"{output_path}.xml", "w", encoding="utf-8") as file:
+                file.write(f'<root>{xml_content}</root>')
+            print(f"Content saved to {output_path}.xml")
+        else:
+            print("Invalid file format. Supported formats: txt, pdf, csv, xml")
+    else:
+        print("Failed to retrieve content from the URL.")
+
+
+if __name__ == '__main__':
+    pass
+    # Example usage:
+    # content_crawler("https://www.presight.io/privacy-policy.html", file_format='pdf', output_file='privacy_policy')

requirements.txt

@@ -0,0 +1,166 @@
+aiofiles==23.2.1
+aiohttp==3.8.6
+aiosignal==1.3.1
+altair==5.1.2
+annotated-types==0.6.0
+anyio==3.7.1
+async-timeout==4.0.3
+attrs==23.1.0
+backoff==2.2.1
+bcrypt==4.0.1
+beautifulsoup4==4.12.2
+cachetools==5.3.2
+certifi==2023.7.22
+chardet==5.2.0
+charset-normalizer==3.3.2
+chroma-hnswlib==0.7.3
+chromadb==0.4.16
+ci-info==0.3.0
+click==8.1.7
+colorama==0.4.6
+coloredlogs==15.0.1
+configobj==5.0.8
+configparser==6.0.0
+contourpy==1.2.0
+cycler==0.12.1
+dataclasses-json==0.6.2
+Deprecated==1.2.14
+emoji==2.8.0
+etelemetry==0.3.1
+exceptiongroup==1.1.3
+faiss-cpu==1.7.4
+fastapi==0.104.1
+ffmpy==0.3.1
+filelock==3.13.1
+filetype==1.2.0
+flatbuffers==23.5.26
+fonttools==4.44.0
+fpdf==1.7.2
+frozenlist==1.4.0
+fsspec==2023.10.0
+future==0.18.3
+google-auth==2.23.4
+googleapis-common-protos==1.61.0
+gradio==3.45.2
+gradio_client==0.5.3
+grpcio==1.59.2
+h11==0.14.0
+httpcore==1.0.2
+httplib2==0.22.0
+httptools==0.6.1
+httpx==0.25.1
+huggingface-hub==0.17.3
+humanfriendly==10.0
+idna==3.4
+importlib-metadata==6.8.0
+importlib-resources==6.1.1
+install==1.3.5
+isodate==0.6.1
+Jinja2==3.1.2
+joblib==1.3.2
+jsonpatch==1.33
+jsonpointer==2.4
+jsonschema==4.19.2
+jsonschema-specifications==2023.7.1
+kiwisolver==1.4.5
+kubernetes==28.1.0
+langchain==0.0.334
+langdetect==1.0.9
+langsmith==0.0.63
+looseversion==1.3.0
+lxml==4.9.3
+markdown-it-py==3.0.0
+MarkupSafe==2.1.3
+marshmallow==3.20.1
+matplotlib==3.8.1
+mdurl==0.1.2
+monotonic==1.6
+mpmath==1.3.0
+multidict==6.0.4
+mypy-extensions==1.0.0
+networkx==3.2.1
+nibabel==5.1.0
+nipype==1.8.6
+nltk==3.8.1
+numpy==1.26.1
+oauthlib==3.2.2
+onnxruntime==1.16.2
+openai==0.27.3
+opentelemetry-api==1.21.0
+opentelemetry-exporter-otlp-proto-common==1.21.0
+opentelemetry-exporter-otlp-proto-grpc==1.21.0
+opentelemetry-proto==1.21.0
+opentelemetry-sdk==1.21.0
+opentelemetry-semantic-conventions==0.42b0
+orjson==3.9.10
+overrides==7.4.0
+packaging==23.2
+pandas==2.1.2
+pathlib==1.0.1
+pdfminer==20191125
+Pillow==10.1.0
+posthog==3.0.2
+protobuf==4.25.0
+prov==2.0.0
+pulsar-client==3.3.0
+pyasn1==0.5.0
+pyasn1-modules==0.3.0
+pycryptodome==3.19.0
+pydantic==2.4.2
+pydantic_core==2.10.1
+pydot==1.4.2
+pydub==0.25.1
+Pygments==2.16.1
+pyparsing==3.1.1
+pypdf==3.17.0
+PyPDF2==3.0.1
+PyPika==0.48.9
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-iso639==2023.6.15
+python-magic==0.4.27
+python-multipart==0.0.6
+pytz==2023.3.post1
+pyxnat==1.6
+PyYAML==6.0.1
+rapidfuzz==3.5.2
+rdflib==7.0.0
+referencing==0.30.2
+regex==2023.10.3
+requests==2.31.0
+requests-oauthlib==1.3.1
+rich==13.6.0
+rpds-py==0.12.0
+rsa==4.9
+scipy==1.11.3
+semantic-version==2.10.0
+shellingham==1.5.4
+simplejson==3.19.2
+six==1.16.0
+sniffio==1.3.0
+soupsieve==2.5
+SQLAlchemy==2.0.23
+starlette==0.27.0
+sympy==1.12
+tabulate==0.9.0
+tenacity==8.2.3
+tiktoken==0.5.1
+tokenizers==0.14.1
+tomlkit==0.12.0
+toolz==0.12.0
+tqdm==4.66.1
+traits==6.3.2
+typer==0.9.0
+typing-inspect==0.9.0
+typing_extensions==4.8.0
+tzdata==2023.3
+unstructured==0.10.29
+urllib3==1.26.18
+uvicorn==0.24.0.post1
+uvloop==0.19.0
+watchfiles==0.21.0
+websocket-client==1.6.4
+websockets==11.0.3
+wrapt==1.16.0
+yarl==1.9.2
+zipp==3.17.0