fix files

.gitignore

@@ -0,0 +1,4 @@
+*.pyc
+workspace/*
+*.bak
+.DS_Store

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

app-content.json

@@ -0,0 +1,17 @@
+[
+    {
+        "name": "Digital Identity Guidelines",
+        "url": "https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-63-3.pdf",
+        "content_type": "Standards"
+    },
+    {
+        "name": "OWASP Top 10 for LLM Applications",
+        "url": "https://owasp.org/www-project-top-10-for-large-language-model-applications/assets/PDF/OWASP-Top-10-for-LLMs-2023-v1_0_1.pdf",
+        "content_type": "References"
+    },
+    {
+        "name": "UAE NESA Regulation",
+        "url": "https://u.ae/-/media/guidelines/Guidelines-2020/UAE-IA-Regulation-v11-1.pdf",
+        "content_type": "Standards"
+    }
+]

app.py

@@ -0,0 +1,79 @@
+import streamlit as st
+from streamlit_option_menu import option_menu
+# Import your page modules
+from modules import nav_about, nav_query_docs, nav_researcher, nav_summarizer, nav_file_manager
+from modules import app_constants, app_logger, common_utils
+from modules.message_store import MessageStore
+
+app_logger = app_logger.app_logger
+WORKSPACE_DIRECTORY = app_constants.WORKSPACE_DIRECTORY
+
+# Page configuration
+st.set_page_config(page_title="ZySec AI", page_icon=":sparkles:", layout="wide")
+
+# Initialize MessageStore in the session state
+if 'message_store' not in st.session_state:
+    st.session_state['message_store'] = MessageStore()
+def request_username():
+    st.title("Welcome to ZySec AI")
+    app_logger.info("App started")
+    username = st.text_input("How do you want me to call you?",value="Security Ninja" ,placeholder="Enter your name")
+    submit_button = st.button('Submit')
+
+    if submit_button and username:
+        st.session_state['username'] = username
+        return True  # Indicates that a username was submitted
+    return False
+
+def main():
+    common_utils.setup_initial_folders()
+    if 'messages' not in st.session_state:
+        st.session_state['messages'] = []
+
+    if 'username' not in st.session_state or not st.session_state['username']:
+        if request_username():
+            st.rerun()
+        return
+
+    # Sidebar navigation
+    with st.sidebar:
+        selected = option_menu(
+            "ZySec AI", 
+            ["Private AI", "Playbooks", "Standards", "Policies", "Researcher", "Summarizer","Files", "About"], 
+            icons=["shield-lock", "book-half", "file-earmark-ruled", "journal-bookmark", "search", "file-text","files", "info-circle"], 
+            default_index=0, 
+            menu_icon="cast", 
+            styles={}
+        )
+    st.markdown("---")
+
+
+    try:
+        message_store = st.session_state['message_store']
+
+
+        if selected == "Private AI":
+            #nav_private_ai.app(message_store)
+            nav_query_docs.app(message_store,current_page="nav_private_ai")
+        elif selected == "Playbooks":
+            nav_query_docs.app(message_store,current_page="nav_playbooks",use_retrieval_chain=True)
+        elif selected == "Standards":
+            nav_query_docs.app(message_store,current_page="nav_standards",use_retrieval_chain=True)
+        elif selected == "Policies":
+            nav_query_docs.app(message_store,current_page="nav_policies",use_retrieval_chain=True)
+        elif selected == "Researcher":
+            nav_researcher.app(message_store)
+        elif selected == "Summarizer":
+            nav_summarizer.app()
+        elif selected == "Files":
+            nav_file_manager.app()
+        elif selected == "About":
+            nav_about.app()
+        else:
+            pass
+
+    except Exception as e:
+        st.error(f"Looks like there's a gap here; maybe we forgot to add some data!: {e}")
+
+if __name__ == "__main__":
+    main()

modules/app_constants.py

@@ -0,0 +1,56 @@
+import os
+from langchain_community.document_loaders import (CSVLoader, TextLoader, UnstructuredExcelLoader, Docx2txtLoader,
+                                                   UnstructuredFileLoader, UnstructuredMarkdownLoader, UnstructuredHTMLLoader, JSONLoader)
+from chromadb.config import Settings
+
+from modules import app_logger
+
+app_logger = app_logger.app_logger
+# Use shared_variable in this module
+openai_api_key = os.environ.get("OPENAI_API_KEY", "NONE")
+
+# Set default values if environment variables are not found
+#mongodb_uri = os.environ.get("MONGODB_URI", "mongodb://localhost:27017")
+local_model_uri = os.environ.get("LOCAL_OPENAI_URI", "http://localhost:8000/v1")
+#local_model_uri = os.environ.get("LOCAL_OPENAI_URI", None)
+DOCUMENT_MAP = {
+    ".html": UnstructuredHTMLLoader,
+    ".txt": TextLoader,
+    ".md": UnstructuredMarkdownLoader,
+    ".py": TextLoader,
+    ".json": JSONLoader,
+    ".jsonl": JSONLoader,
+    ".pdf": UnstructuredFileLoader,
+    ".csv": CSVLoader,
+    ".xls": UnstructuredExcelLoader,
+    ".xlsx": UnstructuredExcelLoader,
+    ".docx": Docx2txtLoader,
+    ".doc": Docx2txtLoader,
+}
+MODELS_PATH = "./models"
+EMBEDDING_MODEL_NAME = 'sentence-transformers/all-MiniLM-L6-v2'
+MODEL_NAME = 'gpt-3.5-turbo'
+# Constants
+WORKSPACE_DIRECTORY = './workspace/'
+
+
+CHROMA_SETTINGS = Settings(
+    anonymized_telemetry=False,
+    is_persistent=True,
+)
+INGEST_THREADS = os.cpu_count() or 8
+
+CHUNK_SIZE = 880
+CHUNK_OVERLAP = 200
+PROCESSED_DOCS = 'index_processed.log'
+SEARCH_COUNT = 5
+MESSAGE_HISTORY = 4
+RAG_K = 3
+RAG_TECHNIQUE = 'refine'
+SUMMARIZER_BATCH = 3
+MAX_FILE_SIZE = 10 #not implement
+LOCAL_PERSISTANT_DB = WORKSPACE_DIRECTORY + "db/"
+CONTENT_TYPE = ["Policies", "Playbooks", "Standards", "Reference Docs"]
+SYSTEM_CONTENT_DATA = "app-content.json"
+SYSTEM_DEPLOYMENT_MODE = 0 #private-0, openai-1, demo-2
+ZYSEC_DEMO = "http://zysec.is-a-geek.com:8000/v1" #not enabled yet

modules/app_logger.py

@@ -0,0 +1,31 @@
+
+import logging
+
+
+def setup_logger():
+    # Format for our loglines
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s', 
+                                  datefmt='%Y-%m-%d %H:%M:%S')
+
+    # Setup basic configuration for logging
+    logging.basicConfig(filename='app.log', 
+                        level=logging.ERROR, 
+                        format='%(asctime)s - %(levelname)s - %(message)s')
+
+    # Create a logger
+    logger = logging.getLogger('AppLogger')
+
+    # Create handlers (if you want to log to file and console)
+    file_handler = logging.FileHandler('app.log')
+    file_handler.setFormatter(formatter)
+
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(formatter)
+
+    # Add handlers to the logger
+    logger.addHandler(file_handler)
+    logger.addHandler(console_handler)
+
+    return logger
+
+app_logger = setup_logger()

modules/app_page_definitions.py

@@ -0,0 +1,56 @@
+#app_page_definitions.py
+PAGE_CONFIG = {
+    "nav_private_ai": {
+        "title": "Private AI",
+        "caption": "🔒 Delve into AI-driven cybersecurity insights and strategies.",
+        "greeting": "Greetings! I'm ZySec, your dedicated AI assistant in Cyber Security.",
+        "system_role": "You are ZySec, an AI Assistant specialisted in CyberSecurity. You are developed by the ZySec AI team, provide expert cybersecurity insights and advice. While responding, if you are not clear, ask follow up question and focus on delivering accurate information."
+    },
+    "nav_standards": {
+        "title": "Standards Discovery",
+        "caption": "📚 Navigate through industry standards and gain valuable insights.",
+        "greeting": "Welcome to Standards Discovery! Let's analyze documents for streamlined insights.",
+        "system_role": "As ZySec in Standards Assistance, developed by the ZySec AI team, guide users through complex standards, focusing on cybersecurity nuances. At any point while responding, if you are not clear ask follow up question and focus on delivering accurate information.",
+        "content": ["standards","standard","framework","regulatory"]
+    },
+    "nav_playbooks": {
+        "title": "Playbooks Deep-Dive",
+        "caption": "📖 Uncover insights within playbooks for informed decision-making.",
+        "greeting": "Prepared to explore your playbook? Upload your document to begin.",
+        "system_role": "As ZySec, developed by the ZySec AI team, specialize in dissecting playbooks and documents for precise answers and insights. Structure responses clearly, utilizing bullet points to highlight key insights.",
+        "content": ["procedures", "playbooks","others","breaches","reference docs"]
+    },
+    "nav_researcher": {
+        "title": "Research Expert",
+        "caption": "🌐 Conduct in-depth research and gather information from the web.",
+        "greeting": "Hi, I'm ZySec, ready to assist with thorough Internet-based research.",
+        "system_role": "As ZySec, embodying the role of a Research Assistant and developed by the ZySec AI team, provide in-depth research support and insights."
+    },
+    "nav_summarize": {
+        "title": "Summarization",
+        "caption": "✍️ Transform extensive content into concise summaries effortlessly.",
+        "greeting": "Hey there! ZySec here, your expert in content summarization.",
+        "system_role": "As ZySec, developed by the ZySec AI team, focus on distilling content into clear, succinct summaries. Organize information in a structured manner."
+    },
+    "nav_policies": {
+        "title": "Policy Expert",
+        "caption": "📃 Explore and understand complex policies with AI assistance.",
+        "greeting": "Welcome to Policy Analysis! I'm ZySec, here to guide you through policy intricacies.",
+        "system_role": "As ZySec, specializing in policy analysis and advice, and developed by the ZySec AI team, use AI to clarify and explain policies.",
+        "content": ["policies","policy","guidelines"]
+    },
+    "nav_file_manager": {
+        "title": "File Manager",
+        "caption": "📃 Explore the content in your application, enable to maintain focus of ZySec."
+    },
+    "nav_about": {
+        "title": "System Controls",
+        "caption": "⚙️ Manage and control system settings including AI model configurations."
+    },
+    "default": {
+        "title": "Welcome to ZySec AI",
+        "caption": "🌟 Navigate the world of Cyber Security with AI-driven insights.",
+        "greeting": "How can I assist you with Cyber Security today?",
+        "system_role": "As ZySec, developed by the ZySec AI team, offer specific insights and guidance on security-related queries."
+    }
+}

modules/app_prompt.py

@@ -0,0 +1,64 @@
+# app_combined_prompt.py
+import modules.app_constants as app_constants  # Ensure this is correctly referenced
+from langchain_openai import ChatOpenAI
+from langchain.chains import RetrievalQAWithSourcesChain
+from openai import OpenAI
+from modules import app_logger, common_utils, app_st_session_utils
+
+# Use the logger from app_config
+app_logger = app_logger.app_logger
+
+# Define a function to query the language model
+def query_llm(prompt, page="nav_private_ai", retriever=None, message_store=None, use_retrieval_chain=False, last_page=None, username=""):
+    try:
+        # Choose the language model client based on the use_retrieval_chain flag
+        if use_retrieval_chain:
+            app_logger.info("Using ChatOpenAI with RetrievalQAWithSourcesChain")
+            llm = ChatOpenAI(
+                model_name=app_constants.MODEL_NAME,
+                openai_api_key=app_constants.openai_api_key,
+                base_url=app_constants.local_model_uri,
+                streaming=True
+            )
+            qa = RetrievalQAWithSourcesChain.from_chain_type(
+                llm=llm,
+                chain_type=app_constants.RAG_TECHNIQUE,
+                retriever=retriever,
+                return_source_documents=False
+            )
+        else:
+            app_logger.info("Using direct OpenAI API call")
+            llm = OpenAI(
+                base_url=app_constants.local_model_uri,
+                api_key=app_constants.openai_api_key
+            )
+
+        # Update page messages if there's a change in the page
+        if last_page != page:
+            app_logger.info(f"Updating messages for new page: {page}")
+            common_utils.get_system_role(page, message_store)
+
+        # Construct messages to send to the LLM, excluding timestamps
+        messages_to_send = common_utils.construct_messages_to_send(page, message_store, prompt)
+        app_logger.debug(messages_to_send)
+        # Sending the messages to the LLM and retrieving the response
+        response = None
+        if use_retrieval_chain:
+            response = qa.invoke(prompt)
+        else:
+            response = llm.chat.completions.create(
+                model=app_constants.MODEL_NAME,
+                messages=messages_to_send
+            )
+
+        # Process the response
+        raw_msg = response.get('answer') if use_retrieval_chain else response.choices[0].message.content
+        source_info = response.get('sources', '').strip() if use_retrieval_chain else ''
+        formatted_msg = app_st_session_utils.format_response(raw_msg + "Source: " + source_info if source_info else raw_msg)
+
+        return formatted_msg
+
+    except Exception as e:
+        error_message = f"An error occurred while querying the language model: {e}"
+        app_logger.error(error_message)
+        return error_message

modules/app_researcher.py

@@ -0,0 +1,159 @@
+import requests
+from bs4 import BeautifulSoup
+import html2text
+import re
+import os
+from modules import app_constants, file_utils, app_logger
+import json
+from langchain_openai import ChatOpenAI
+from langchain.schema import HumanMessage, SystemMessage
+import spacy
+from duckduckgo_search import DDGS
+nlp = spacy.load("en_core_web_sm")
+
+# Use the logger from app_config
+app_logger = app_logger.app_logger
+
+TMP_DIRECTORY = app_constants.WORKSPACE_DIRECTORY + 'tmp'
+DEFAULT_SEARCH_COUNT = app_constants.SEARCH_COUNT
+
+def download_and_clean(url):
+    try:
+        headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0"}
+        response = requests.get(url, headers=headers)
+        response.raise_for_status()
+
+        soup = BeautifulSoup(response.content, 'html.parser')
+        for script in soup(["script", "style", "img", "a"]):
+            script.extract()
+
+        body_text = soup.get_text()
+        h = html2text.HTML2Text()
+        h.ignore_links = True
+        h.ignore_images = True
+        h.ignore_emphasis = True
+        h.ignore_tables = True
+        clean_text = h.handle(body_text)
+        clean_text = re.sub(r'[^\w\s\n<>/\.]+', '', clean_text)  # Include '.' in the allowed characters
+        clean_text = re.sub(r'\s+', ' ', clean_text).strip()
+        return clean_text
+
+    except requests.exceptions.RequestException as e:
+        app_logger.error(f"Error while downloading and cleaning URL {url}: {str(e)}")
+        return None
+
+def save_notes_to_file(topic, note, source_url):
+    # Process the text
+    doc = nlp(note)
+
+    # Ensure the temp directory exists
+    if not os.path.exists(TMP_DIRECTORY):
+        os.makedirs(TMP_DIRECTORY)
+
+    # Sanitize the filename and create the full path
+    sanitized_filename = file_utils.sanitize_filename(topic)+'.jsonl'
+    full_path = os.path.join(TMP_DIRECTORY, sanitized_filename)
+
+    # Initialize variables for accumulating sentences
+    text_block = ""
+    word_count = 0
+
+    # Append each sentence to form a text block and write it to the file
+    with open(full_path, 'a') as file:
+        for sent in doc.sents:
+            sentence_word_count = len(sent.text.split())
+            if word_count + sentence_word_count > 240:  # If adding the sentence exceeds the max limit
+                # Write the current text block to the file
+                if word_count >= 120:  # Ensure the text block meets the minimum word count
+                    data = {
+                        "note": text_block,
+                        "source_url": source_url
+                    }
+                    file.write(json.dumps(data) + '\n')
+                # Reset text block and word count
+                text_block = sent.text
+                word_count = sentence_word_count
+            else:
+                # Add the sentence to the text block
+                text_block += ' ' + sent.text if text_block else sent.text
+                word_count += sentence_word_count
+
+        # Write any remaining text block to the file if it meets the minimum word count
+        if word_count >= 300:
+            data = {
+                "note": text_block,
+                "source_url": source_url
+            }
+            file.write(json.dumps(data) + '\n')
+
+    app_logger.info(f"Notes saved to file {full_path}")
+    return full_path
+
+
+def url_list_downloader(url_list, topic):
+    notes_file = None
+    for url in url_list:
+        try:
+            text = download_and_clean(url)
+            if text:
+                notes_file = save_notes_to_file(topic, text, url)
+        except Exception as e:
+            app_logger.error(f"Error during processing for URL {url}: {e}")
+    return notes_file
+
+def search_term_ddg(topic,count=DEFAULT_SEARCH_COUNT):
+    try:
+        llm = ChatOpenAI(
+            model_name=app_constants.MODEL_NAME,
+            openai_api_key=app_constants.openai_api_key,
+            base_url=app_constants.local_model_uri,
+            streaming=True
+        )
+        prompt = [
+            SystemMessage(content="Generate 5 plain keywords in comma separated based on user input. For example ['cat','bat','monkey','donkey','eagel']"),
+            HumanMessage(content=topic),
+        ]
+        response = llm(prompt)
+        # Extract string content from the response object
+        if hasattr(response, 'content'):
+            search_keywords = response.content
+        else:
+            raise ValueError("Invalid response format")
+        
+        # Splitting and trimming the keywords
+        search_keywords = [keyword.strip() for keyword in search_keywords.split(',')]
+        #print(search_keywords)
+        # Limiting keywords to a maximum of 8
+        search_keywords = search_keywords[:8]
+
+        urls = []
+        # Initialize DDGS with a timeout
+        with DDGS(timeout=3) as ddgs:
+            for term in search_keywords:
+                # Fetch results for each search term
+                results = ddgs.text(f"{topic} {term}", max_results=count)
+                for result in results:
+                    url = result['href']
+                    if not url.endswith(('.pdf', '.ppt', '.pptx', '.doc', '.docx')):
+                        urls.append(url)
+        return sorted(set(urls))
+
+    except Exception as e:
+        app_logger.error(f"An error occurred while searching for topic {topic}: {e}")
+        return []
+
+def explore_url_on_internet(topic, count=DEFAULT_SEARCH_COUNT):
+    app_logger.info(f"Starting research on topic {topic}")
+    # Sanitize the filename and create the full path
+    sanitized_filename = file_utils.sanitize_filename(topic)+'.jsonl'
+    full_path = os.path.join(TMP_DIRECTORY, sanitized_filename)
+
+    # Check if the file already exists
+    if os.path.exists(full_path):
+        app_logger.info(f"File already exists skipping download: ",full_path)
+        note_file = full_path
+    else:
+        url_list = search_term_ddg(topic,count)
+        note_file = url_list_downloader(url_list, topic)
+    app_logger.info(f"Research on Internet completed for {topic}, file: {note_file}")
+    return note_file

modules/app_st_session_utils.py

@@ -0,0 +1,109 @@
+import streamlit as st
+from modules import app_logger, app_prompt
+import streamlit.components.v1 as components
+from modules import database_utils,common_utils,app_page_definitions
+import datetime
+
+# Use the logger from app_config
+app_logger = app_logger.app_logger
+def initialize_session_state(key, default_value):
+    """
+    Initialize a session state variable with a default value if it doesn't exist.
+
+    Args:
+    key (str): The key of the session state variable.
+    default_value (Any): The default value to initialize the session state variable with.
+    """
+    if key not in st.session_state:
+        st.session_state[key] = default_value
+
+def update_session_state(key, value):
+    """Update a session state variable."""
+    st.session_state[key] = value
+
+def setup_page_session_state(current_page):
+    initialize_session_state('current_page', current_page)
+    initialize_session_state('page_loaded', False)
+    initialize_session_state('message_store', app_prompt.MessageStore())
+
+def log_session_info(message):
+    """Log session-related information."""
+    try:
+        app_logger.info(message)
+    except Exception as e:
+        app_logger.error(f"Logging error: {e}")
+
+def manage_message_history(current_page):
+    """Manage the history of messages for the current page."""
+    try:
+        message_store = st.session_state['message_store']
+        if st.session_state['current_page'] != current_page:
+            message_store.set_history(st.session_state['current_page'], st.session_state["messages"])
+            st.session_state["messages"] = message_store.get_history(current_page)
+            st.session_state['current_page'] = current_page
+            log_session_info(f"Updated message history for page: {current_page}")
+    except Exception as e:
+        log_session_info(f"Error managing message history: {e}")
+
+def display_chat_message(role, content):
+    """Display a chat message based on the role."""
+    if role in ['user', 'assistant']:
+        st.chat_message(role).write(content)
+    else:
+        log_session_info(f"Invalid role '{role}' in display_chat_message")
+
+def reset_session_state():
+    """Reset the session state to its initial values."""
+    for key in list(st.session_state.keys()):
+        del st.session_state[key]
+    # Reinitialize variables if necessary here
+
+def get_session_data(key, default=None):
+    """Retrieve data stored in the session state."""
+    return st.session_state.get(key, default)
+
+def reload_page():
+    js = "window.location.reload();"
+    components.html(f"<script>{js}</script>", height=0, width=0)
+
+def initialize_or_retrieve_db(db_path):
+    """
+    Initialize the database if not already initialized or if the database path has changed.
+    Retrieve the database from the session state if already initialized.
+
+    Args:
+    db_path (str): The file path to the database.
+
+    Returns:
+    The initialized or retrieved database object.
+    """
+    app_logger.info("initializing db", db_path)
+    if 'db_retriever' not in st.session_state or st.session_state['db_path'] != db_path:
+        # Database not initialized or path has changed
+        db_retriever = database_utils.initialize_chroma_db(db_path)
+        if db_retriever is not None:
+            st.session_state['db_retriever'] = db_retriever
+            st.session_state['db_path'] = db_path
+            app_logger.info(f"Database initialized at {db_path}")
+        else:
+            app_logger.error(f"Failed to initialize database at {db_path}")
+            return None
+    return st.session_state['db_retriever']
+
+# Function to format the response
+def format_response(response):
+    return response.replace('\r\n', '\n').replace('\r', '\n').strip()
+
+# Add a message to the session state
+def add_message_to_session(role, content, add_to_history=True):
+    timestamp = datetime.datetime.now()
+    message = {"role": role, "content": content, "timestamp": timestamp}
+    if "messages" not in st.session_state:
+        st.session_state["messages"] = []
+    if add_to_history and role in ["user", "assistant"]:
+        st.session_state["messages"].append(message)
+        # Update message_store with the new message
+        if 'message_store' in st.session_state:
+            current_page = st.session_state.get('current_page', 'default_page')
+            st.session_state['message_store'].update_message(current_page, 'history', message)
+

modules/app_to_vectorstore.py

@@ -0,0 +1,102 @@
+
+#app_to_vectorstore.py
+
+import os
+from langchain_community.vectorstores import Chroma
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain.text_splitter import CharacterTextSplitter
+from modules import common_utils,file_utils
+from modules import app_logger
+# Assuming all necessary loader classes are imported
+
+from modules import app_constants
+
+app_logger = app_logger.app_logger
+
+TEMP_DIR = app_constants.WORKSPACE_DIRECTORY + "tmp"
+DB_DIR = app_constants.WORKSPACE_DIRECTORY + "db"
+
+processed_files_record = os.path.join(app_constants.WORKSPACE_DIRECTORY, app_constants.PROCESSED_DOCS)
+
+def load_documents_from_jsonl(file_path, loader_class):
+    try:
+        loader = loader_class(file_path, json_lines=True, text_content=False, jq_schema='.')
+        return loader.load()
+    except Exception as e:
+        app_logger.error(f"Error loading documents from JSONL file {file_path}: {e}")
+        return None
+
+def update_processed_files_record(file_md5,module, file_path):
+    try:
+        with open(processed_files_record, 'a') as file:  # 'a' mode will create the file if it doesn't exist
+            file.write(f"{file_md5},{module},{file_path}\n")
+    except Exception as e:
+        app_logger.error(f"Error updating processed files record: {e}")
+
+def is_file_processed(file_md5):
+    if os.path.exists(processed_files_record):
+        with open(processed_files_record, 'r') as file:
+            for line in file:
+                md5, _ = line.strip().split(',', 1)
+                if md5 == file_md5:
+                    return True
+    return False
+
+def get_chroma_index(file_path, current_page="nav_playbooks", is_persistent=True):
+    app_logger.info(f"Starting get_chroma_index for {file_path}")
+    file_md5 = file_utils.compute_md5(file_path)
+    if is_file_processed(file_md5):
+        app_logger.info(f"File {file_path} has already been processed. Skipping.")
+        db = None
+        return False
+    
+    _, file_extension = os.path.splitext(file_path)
+    loader_class = app_constants.DOCUMENT_MAP.get(file_extension.lower(), None)
+
+    if not loader_class:
+        app_logger.error(f"No suitable loader found for file type {file_extension}")
+        return None, False
+
+    embedding_model = app_constants.EMBEDDING_MODEL_NAME
+    chunk_size = app_constants.CHUNK_SIZE
+    chunk_overlap = app_constants.CHUNK_OVERLAP
+
+    storage_dir = DB_DIR if is_persistent else TEMP_DIR
+
+    base_filename = f"{current_page}_chroma_db" if is_persistent else f"{os.path.splitext(os.path.basename(file_path))[0]}_chroma_db"
+    sanitized_base_filename = file_utils.sanitize_filename(base_filename)
+    chroma_persist_directory = os.path.join(storage_dir, sanitized_base_filename)
+
+
+    embeddings = HuggingFaceEmbeddings(model_name=embedding_model)
+    try:
+        if file_extension.lower() == '.jsonl':
+            documents = load_documents_from_jsonl(file_path, loader_class)
+        else:
+            loader = loader_class(file_path)
+            documents = loader.load()
+
+        if not documents:
+            app_logger.error(f"No documents loaded from {file_path}.")
+            db = None
+            return False
+
+        text_splitter = CharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        docs = text_splitter.split_documents(documents)
+
+        if not docs:
+            app_logger.error(f"No documents to process after splitting from {file_path}.")
+            db = None
+            return False
+
+        db = Chroma.from_documents(docs, embeddings, persist_directory=chroma_persist_directory, client_settings=app_constants.CHROMA_SETTINGS)
+        update_processed_files_record(file_md5,current_page, file_path)
+        app_logger.info("Created index and saved to disk")
+        db.persist()
+    except Exception as e:
+        app_logger.error(f"Error in get_chroma_index for {file_path}: {e}")
+        db = None
+        return False
+    app_logger.info("Completed get_chroma_index operation")
+    db = None
+    return True

modules/common_utils.py

@@ -0,0 +1,118 @@
+import os
+from modules import app_constants,app_page_definitions
+from modules import app_logger
+# Use the logger from app_config
+app_logger = app_logger.app_logger
+work_dir = app_constants.WORKSPACE_DIRECTORY
+
+def get_system_role(page, message_store):
+    system_role = app_page_definitions.PAGE_CONFIG.get(page, {}).get("system_role", "Default system role message")
+    message_store.update_message(page, "system", system_role)
+
+def get_page_greeting(page_key, username="", files_indexed=[]):
+    """Return a greeting message for a specific page, including a list of indexed files."""
+    try:
+        # Define the default greeting
+        default_greeting = "Hello! How can I assist you today?"
+        # Fetch the greeting from page configuration or use the default
+        greeting = app_page_definitions.PAGE_CONFIG.get(page_key, {}).get("greeting", default_greeting)
+
+        # Personalize greeting if username is provided
+        if username:
+            greeting = greeting.replace("Hello", f"Hello {username}")
+
+        # Format the indexed files into a list
+        if files_indexed:
+            files_list = "\n".join([f"{i+1}. {file}" for i, file in enumerate(files_indexed)])
+            additional_message = f"I'm familiar with the following documents:\n{files_list}"
+            # Append the file list to the greeting message
+            greeting = f"{greeting}\n\n{additional_message}"
+
+        return greeting
+    except Exception as e:
+        # Handle any exceptions and return a default error message
+        return f"Error generating greeting message: {e}"
+
+def setup_initial_folders():
+    docs_path = os.path.join(work_dir, "docs")
+    db_path = os.path.join(work_dir, "db")
+    tmp_path = os.path.join(work_dir, "tmp")
+    os.makedirs(docs_path, exist_ok=True)
+    os.makedirs(db_path, exist_ok=True)
+    os.makedirs(tmp_path, exist_ok=True)
+    processed_docs_path = work_dir+app_constants.PROCESSED_DOCS
+    if not os.path.exists(processed_docs_path):
+        open(processed_docs_path, 'a').close()
+
+def construct_messages_to_send(page, message_store, prompt):
+    """
+    Construct a list of messages to send to the language model.
+
+    Args:
+    page (str): The current page identifier.
+    message_store (MessageStore): The message store instance containing message histories.
+    prompt (str): The current user prompt.
+
+    Returns:
+    List[Dict[str, str]]: A list of messages structured for the language model.
+    """
+    messages_to_send = []
+
+    # Retrieve the system and greeting messages if available
+    system_message_content = message_store.get_message(page, "system")
+    greeting_message_content = message_store.get_message(page, "greeting")
+    if system_message_content:
+        messages_to_send.append({"role": "system", "content": system_message_content})
+    if greeting_message_content:
+        messages_to_send.append({"role": "assistant", "content": greeting_message_content})
+
+    # Include recent user and assistant messages from the message history
+    history_messages = message_store.get_history(page)
+
+    # Check if there are enough messages in the history, if not, adjust the slicing
+    num_messages_to_include = 4  # Include last two pairs (user and assistant)
+    if len(history_messages) < num_messages_to_include:
+        num_messages_to_include = len(history_messages)
+
+    recent_history = history_messages[-num_messages_to_include:]
+    for msg in recent_history:
+        messages_to_send.append({"role": msg["role"], "content": msg["content"]})
+
+    # Append the current user prompt
+    messages_to_send.append({"role": "user", "content": prompt})
+
+    return messages_to_send
+
+
+def get_content_mapping_to_module(content_type):
+    content_type_lower = content_type.lower()
+    # Iterate through each page in PAGE_CONFIG
+    for page, config in app_page_definitions.PAGE_CONFIG.items():
+        # Check if 'content' key exists
+        if 'content' in config:
+            # Convert all content types in the list to lowercase for comparison
+            content_list_lower = [ct.lower() for ct in config['content']]
+            # Check if content_type_lower is in the list
+            if content_type_lower in content_list_lower:
+                return page
+    # Default return if no match is found
+    return "nav_playbooks"
+
+def read_processed_log():
+    processed_paths = set()
+    log_file_path = os.path.join(work_dir, 'index_processed.log')
+    
+    try:
+        with open(log_file_path, 'r') as log_file:
+            for line in log_file:
+                parts = line.strip().split(',')
+                if len(parts) > 1:
+                    # Extract the file path (assuming it's the last part)
+                    file_path = parts[-1]
+                    processed_paths.add(file_path)
+        return processed_paths
+    except FileNotFoundError:
+        app_logger.error(f"File not found: {log_file_path}")
+    except Exception as e:
+        app_logger.error(f"An error occurred while reading the log file: {e}")
+    return processed_paths

modules/database_utils.py

@@ -0,0 +1,75 @@
+# database_utils.py
+import os
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import Chroma
+from modules import app_constants, app_logger
+
+app_logger = app_logger.app_logger
+
+def initialize_chroma_db(file_path):
+    """
+    Initializes or creates a new Chroma database.
+
+    :param file_path: Path to the Chroma database file
+    :return: A retriever object if initialization is successful, None otherwise
+    """
+    # Initialize embeddings
+    embeddings = HuggingFaceEmbeddings(model_name=app_constants.EMBEDDING_MODEL_NAME)
+
+    # Initialize Chroma database
+    try:
+        if os.path.exists(file_path):
+            app_logger.info(f"Using existing Chroma database at {file_path}.")
+        else:
+            app_logger.info(f"Chroma database not found at {file_path}. Creating a new one.")
+            os.makedirs(os.path.dirname(file_path), exist_ok=True)
+
+        db = Chroma(persist_directory=file_path, embedding_function=embeddings, client_settings=app_constants.CHROMA_SETTINGS)
+    except Exception as e:
+        app_logger.error(f"Failed to initialize Chroma database at {file_path}. Reason: {e}")
+        return None
+
+    # Create a retriever from the Chroma database
+    #retriever = db.as_retriever()
+    return db
+
+def get_chroma_db_files(directory):
+    """Retrieve files ending with 'chroma_db' from the given directory."""
+    return [f for f in os.listdir(directory) if f.endswith('chroma_db')]
+
+def format_db_name(db_name):
+    """Format the database name to a more readable form."""
+    return db_name.replace('_', ' ').replace('chroma db', '').title().strip()
+
+def delete_doc_from_chroma_db(db_path, source_doc):
+    """
+    Deletes all items related to a given source document in a Chroma database located at a specific path.
+
+    :param db_path: Path to the Chroma database file
+    :param source_doc: The source document identifier to match
+    """
+    # Initialize embeddings (assuming this step is necessary for your Chroma setup)
+    embeddings = HuggingFaceEmbeddings(model_name=app_constants.EMBEDDING_MODEL_NAME)
+
+    # Initialize Chroma database
+    if not os.path.exists(db_path):
+        app_logger.error(f"No Chroma database found at {db_path}.")
+        return
+
+    db = Chroma(persist_directory=db_path, embedding_function=embeddings, client_settings=app_constants.CHROMA_SETTINGS)
+
+    ids_to_delete = []
+
+    # Iterate over documents in the database
+    for doc in db:
+        # Check if the document is related to the source document
+        if doc.metadata.get('source') == source_doc:
+            # Add the document's ID to the list of IDs to delete
+            ids_to_delete.append(doc.id)
+    # Delete documents with matching IDs
+    if ids_to_delete:
+        db.delete(ids=ids_to_delete)
+        db.persist()
+        app_logger.error(f"Deleted {len(ids_to_delete)} items related to '{source_doc}'.")
+    else:
+        app_logger.error(f"No items found related to '{source_doc}'.")

modules/file_utils.py

@@ -0,0 +1,178 @@
+#file utils.py
+import os
+from modules import app_constants, app_to_vectorstore,app_page_definitions,common_utils
+from modules import app_logger
+import json
+import requests
+import hashlib
+import re, csv
+
+# Use the logger from app_config
+app_logger = app_logger.app_logger
+work_dir = app_constants.WORKSPACE_DIRECTORY
+system_content_file = metadata_path=app_constants.SYSTEM_CONTENT_DATA
+
+def download_file(url):
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        sanitized_filename = sanitize_filename(url.split('/')[-1])
+        sanitized_local_path = os.path.join(app_constants.WORKSPACE_DIRECTORY+"/docs/", sanitized_filename)
+        with open(sanitized_local_path, 'wb') as f:
+            f.write(response.content)
+        app_logger.info(f"File downloaded successfully: {sanitized_local_path}")
+        return True
+    except Exception as e:
+        app_logger.error(f"Failed to download file from {url}. Error: {e}")
+        return False
+
+def index_file(local_path, module):
+    try:
+        status = app_to_vectorstore.get_chroma_index(local_path,module,True)
+        app_logger.info(f"File indexed successfully: {local_path}")
+    except Exception as e:
+        app_logger.error(f"Failed to index file. Error: {e}")
+        db.persist()
+        db = None
+    return status
+    
+def compute_md5(file_path):
+    hash_md5 = hashlib.md5()
+    try:
+        with open(file_path, "rb") as f:
+            for chunk in iter(lambda: f.read(4096), b""):
+                hash_md5.update(chunk)
+        return hash_md5.hexdigest()
+    except Exception as e:
+        app_logger.error(f"Error computing MD5 for {file_path}: {e}")
+        return None
+
+def sanitize_filename(filename):
+    """Sanitize the filename by removing or replacing invalid characters and handling URLs."""
+
+    # Extract filename from URL or file path
+    filename = os.path.basename(filename)
+
+    # Make the filename lowercase and replace spaces with underscores
+    sanitized = filename.lower().replace(' ', '_')
+
+    # Replace invalid characters with underscores
+    sanitized = re.sub(r'[^\w\-_\.]', '_', sanitized)
+
+    # Shorten the filename if it's too long
+    max_length = 255  # Max length can be adjusted
+    if len(sanitized) > max_length:
+        # Keep the file extension if present
+        file_parts = os.path.splitext(sanitized)
+        ext = file_parts[1]
+        sanitized = sanitized[:max_length - len(ext)] + ext
+    return sanitized
+
+def delete_files(work_dir=work_dir):
+    for root, dirs, files in os.walk(work_dir, topdown=False):
+        for name in files:
+            file_path = os.path.join(root, name)
+            try:
+                os.unlink(file_path)
+                app_logger.info(f"Deleted file: {file_path}")
+            except Exception as e:
+                app_logger.error(f"Failed to delete {file_path}. Reason: {e}")
+
+        for name in dirs:
+            dir_path = os.path.join(root, name)
+            try:
+                os.rmdir(dir_path)
+                app_logger.info(f"Deleted directory: {dir_path}")
+            except Exception as e:
+                app_logger.error(f"Failed to delete {dir_path}. Reason: {e}")
+    remove_local_uploads()
+
+def save_uploaded_file(uploaded_file, uploads_path, sanitized_filename=None):
+    if sanitized_filename is None:
+        sanitized_filename = sanitize_filename(uploaded_file.name)
+    file_path = os.path.join(uploads_path, sanitized_filename)
+
+    with open(file_path, "wb") as f:
+        f.write(uploaded_file.getbuffer())
+    app_logger.info(f"File '{sanitized_filename}' uploaded to {uploads_path}")
+    return file_path
+
+def perform_file_operation(resource, operation):
+    url = resource.get("url", "")
+    content_type = resource.get("content_type", "")
+    file_name = work_dir+"docs/" +sanitize_filename(url)
+    if operation == "download":
+        #print(file_name)
+        if url:
+            download_success = download_file(url)
+            if download_success:
+                app_logger.info(f"File {resource['name']} downloaded successfully.")
+            else:
+                app_logger.error(f"Failed to download file {resource['name']}.")
+    elif operation == "learn":
+        module = common_utils.get_content_mapping_to_module(content_type)
+        # Handle 'learn' operation here if needed
+        index_file(file_name, module)
+    else:
+        app_logger.error(f"Unknown operation: {operation}")
+
+
+def get_indexed_files_for_page(page_id):
+    try:
+        filtered_files = []
+
+        # Open and read the CSV file
+        with open(os.path.join(work_dir, app_constants.PROCESSED_DOCS), mode='r', newline='', encoding='utf-8') as file:
+            csv_reader = csv.reader(file)
+            for row in csv_reader:
+                # Check if the second item in the row matches the page_id
+                if len(row) > 2 and row[1].lower() == page_id.lower():
+                    # Extract just the file name from the full path (third item in the row)
+                    file_name = os.path.basename(row[2])
+                    filtered_files.append(file_name)
+
+        return filtered_files
+    except Exception as e:
+        return []
+
+def update_json_file(data, file_path):
+    with open(file_path, "w") as file:
+        json.dump(data, file, indent=4)
+
+def load_json_data(file_path):
+    with open(file_path, "r") as file:
+        return json.load(file)
+    
+def handle_content_update(uploaded_file=None, manual_name="", manual_url="", selected_content_type=""):
+    system_content_file = app_constants.SYSTEM_CONTENT_DATA  # Define before use
+    uploads_directory = os.path.join(work_dir, "docs")  # Define before use
+    file_data = load_json_data(system_content_file)
+
+    if uploaded_file:
+        filename = sanitize_filename(uploaded_file.name if uploaded_file else manual_name) 
+        file_path = save_file(uploaded_file, filename, uploads_directory)
+    else:
+        filename = sanitize_filename(manual_url)
+        file_path = save_file(uploaded_file, filename, uploads_directory) if uploaded_file else manual_url
+
+    new_entry = {"name": filename, "url": file_path, "content_type": selected_content_type}
+    file_data.append(new_entry)
+    update_json_file(file_data, system_content_file)
+
+def save_file(uploaded_file, filename, directory):
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+    file_path = os.path.join(directory, filename)
+    with open(file_path, "wb") as file:
+        file.write(uploaded_file.getbuffer())
+    return file_path
+
+def remove_local_uploads(file_path=app_constants.SYSTEM_CONTENT_DATA):
+    # Read the JSON data from the file
+    with open(file_path, 'r') as file:
+        data = json.load(file)
+    # Filter out entries where the 'url' points to a local file
+    filtered_data = [entry for entry in data if not entry['url'].startswith('./')]
+    # Write the filtered data back to the file
+    with open(file_path, 'w') as file:
+        json.dump(filtered_data, file, indent=4)

modules/message_store.py

@@ -0,0 +1,23 @@
+# Define the MessageStore class
+class MessageStore:
+    def __init__(self):
+        self.messages = {}
+
+    def update_message(self, page, message_type, message):
+        if page not in self.messages:
+            self.messages[page] = {"system": None, "greeting": None, "history": []}
+        if message_type in ["system", "greeting"]:
+            self.messages[page][message_type] = message
+        elif message_type == "history":
+            self.messages[page]["history"].append(message)
+
+    def get_message(self, page, message_type):
+        return self.messages.get(page, {}).get(message_type, "")
+
+    def get_history(self, page):
+        return self.messages.get(page, {}).get("history", [])
+
+    def set_history(self, page, history):
+        if page not in self.messages:
+            self.messages[page] = {"system": None, "greeting": None, "history": []}
+        self.messages[page]["history"] = history

modules/nav_about.py

@@ -0,0 +1,121 @@
+from . import app_constants
+import streamlit as st
+from modules import app_logger,app_st_session_utils, app_page_definitions
+
+# Use the logger from app_config
+app_logger = app_logger.app_logger
+
+
+def app():
+    app_logger.info("Starting Streamlit app - Configuration Tool")
+    current_page = "nav_about"
+    # Fetch page configuration from app_page_definitions
+    page_config = app_page_definitions.PAGE_CONFIG.get(current_page, app_page_definitions.PAGE_CONFIG["default"])
+
+    # Use configurations for title, caption, and greeting from page_config
+    st.title(page_config["title"])
+    st.caption(page_config["caption"])
+
+
+    # Subheader for Server Mode Selection
+    st.subheader("Server Mode Selection")
+    mode_to_index = {"private": 0, "demo": 1, "openai": 2}
+    default_index = mode_to_index.get(app_constants.SYSTEM_DEPLOYMENT_MODE, 0)  # Default to "Local" if not found
+    # Radio buttons for selecting the server mode
+    server_mode = st.radio("Select Server Mode", ["Private", "ZySec Demo", "OpenAI"],index=default_index)
+
+    # Initialize variables for settings
+    local_model_uri, remote_model_uri, openai_api_key = None, None, None
+
+    # Conditional rendering of settings and their descriptions based on the selected server mode
+    if server_mode == "Private":
+        st.markdown("### Local Settings")
+        st.markdown("""
+        **Private Mode** is for running the model directly on your machine or on a local server. 
+        This mode is ideal if you have the necessary resources and want to keep data processing in-house. 
+        You can also use a local instance deployed with a URL endpoint.
+        """)
+        local_model_uri = st.text_input("Private Model Base URL Endpoint (OpenAI Compatible). Example http://localhost:8000/v1", key="local_model_uri",value=app_constants.local_model_uri)
+        st.info("Use update configuration for changes to be affected")
+
+
+    elif server_mode == "ZySec Demo":
+        st.markdown("### ZySec Demo Settings")
+        st.markdown("""
+        **ZySec Demo Mode** is designed for users who prefer to use ZySec's resources. 
+        This mode provides free access to a deployed model managed by the ZySec team, 
+        subject to availability. It's a great choice for trying out ZySec without any setup.
+        """)
+        remote_model_uri = st.text_input("Remote Model Base URL Endpoint",value=app_constants.ZYSEC_DEMO, key="remote_model_uri",disabled=True)
+        st.info("Use update configuration for changes to be affected")
+
+    elif server_mode == "OpenAI":
+        st.markdown("### OpenAI Settings")
+        st.markdown("""
+        **OpenAI Mode** leverages the OpenAI's Large Language Models (LLM) for processing. 
+        This mode allows you to integrate OpenAI's powerful AI capabilities while keeping 
+        the rest of the functionalities security-centric. An OpenAI API key is required.
+        """)
+        openai_api_key = st.text_input("OpenAI API Key", type="password", key="openai_api_key",value=app_constants.openai_api_key)
+        st.markdown(
+            "Need an OpenAI API key? [Get it here](https://platform.openai.com/api-keys).", 
+            unsafe_allow_html=True
+        )
+        st.info("Use update configuration for changes to be affected")
+
+    # Update app_constants based on user input
+    if st.button("Update Configuration"):
+        if server_mode == "Private":
+            app_constants.SYSTEM_DEPLOYMENT_MODE = "private"
+            app_constants.local_model_uri = local_model_uri
+            # Reset other modes' settings
+            app_constants.openai_api_key = "NO-API-KEY-NEEDED"
+            st.info("Use update configuration for changes to be affected")
+        elif server_mode == "ZySec Demo":
+            app_constants.SYSTEM_DEPLOYMENT_MODE = "demo"
+            app_constants.local_model_uri = remote_model_uri
+            # Reset other modes' setting
+            app_constants.openai_api_key = "NO-API-KEY-NEEDED"
+            st.info("Use update configuration for changes to be affected")
+        elif server_mode == "OpenAI":
+            app_constants.SYSTEM_DEPLOYMENT_MODE = "openai"
+            app_constants.openai_api_key = openai_api_key
+            # Reset other modes' settings
+            app_constants.local_model_uri = None
+            st.info("Use update configuration for changes to be affected")
+        st.success("Configuration updated for " + server_mode + " mode.")
+
+    with st.expander("About ZySec and the Author"):
+        st.markdown("""
+            ### About ZySec
+            ZySec is at the forefront of integrating **Cyber Security with Artificial Intelligence**, aiming to revolutionize how security professionals engage with technology. This project is driven by the aspiration to blend AI's innovative capabilities with the intricacies of cybersecurity, all while upholding the utmost standards of privacy.
+            
+            ZySec is not just a tool; it's a vision to elevate enterprise security functions, harnessing AI to propel these capabilities to new heights. We encourage you to explore our roadmap and see how ZySec is poised to transform the cybersecurity landscape.
+            
+            [🔗 View Our Road Map](https://github.com/ZySec-AI/ZySec/blob/main/roadmap.md)
+            
+            [🔗 Explore the Project on GitHub](https://github.com/ZySec-AI/ZySec.git)
+
+            [🔗 Contact Us](https://docs.google.com/forms/d/e/1FAIpQLSdkqIjQUoUOorsWXVzgQhJ-vbp1OpN1ZI0u3u8fK_o-UxII2w/viewform)
+
+            ### ZySec 7B Model
+            **ZySec-v1-7B** stands as a pivotal innovation for security professionals, leveraging the advanced capabilities of HuggingFace's Zephyr language model series. This AI model is crafted to be an omnipresent cybersecurity ally, offering on-demand, expert guidance in cybersecurity issues. Picture ZySec-7B as an ever-present digital teammate, adept at navigating the complexities of security challenges. ZySec-7B's training spans over 30 unique domains, each enriched with thousands of data points, delivering unparalleled expertise.
+            
+            [🔗 Checkout Model on HuggingFace](https://huggingface.co/aihub-app/ZySec-7B-v1)
+
+            ### About the Author - Venkatesh Siddi
+            **Venkatesh Siddi** is a seasoned expert in the cybersecurity domain, beyond traditional cybersecurity, Venkatesh is deeply invested in leveraging **Artificial Intelligence and Machine Learning** to tackle complex cybersecurity challenges. He has led multiple projects involving big data, cloud security, and technology design.
+                    
+            [🔗 Connect with Venkatesh on LinkedIn](https://www.linkedin.com/in/venkycs/)
+
+        """, unsafe_allow_html=True)
+
+        # Session and Data Reset
+    st.subheader("Clear Session")
+    if st.button("Reset Session"):
+        # Clear all items from the session state
+        for key in list(st.session_state.keys()):
+            del st.session_state[key]
+        # Reload the page to reflect the session reset
+        app_st_session_utils.reload_page()
+        st.rerun()

modules/nav_file_manager.py

@@ -0,0 +1,136 @@
+#nav_file_manager.py
+
+from . import app_constants
+import streamlit as st
+from modules import app_logger, app_page_definitions,file_utils,common_utils
+import os, json
+
+# Use the logger from app_config
+app_logger = app_logger.app_logger
+system_content_file = app_constants.SYSTEM_CONTENT_DATA
+work_dir = os.path.join(app_constants.WORKSPACE_DIRECTORY, "docs")
+
+def app():
+    app_logger.info("Starting Streamlit app - File Manager")
+    current_page = "nav_file_manager"
+
+    # Fetch page configuration
+    page_config = app_page_definitions.PAGE_CONFIG.get(current_page, app_page_definitions.PAGE_CONFIG["default"])
+
+    # Page setup
+    st.title(page_config["title"])
+    st.caption(page_config["caption"])
+    with open(system_content_file, "r") as file:
+        content_data = json.load(file)
+    processed_paths = common_utils.read_processed_log()
+    if 'download_states' not in st.session_state:
+        st.session_state['download_states'] = {}
+    if 'learn_states' not in st.session_state:
+        st.session_state['learn_states'] = {}
+    for index, item in enumerate(content_data):
+        name = item["name"]
+        url = item["url"]
+        content_type = item["content_type"]
+        unique_identifier = f"{index}_{name}_{url.replace('http://', '').replace('https://', '')}"  # Ensure the identifier is unique
+
+        # Create a row of columns for each item
+        col1, col2, col3, col4 = st.columns([3, 2, 1, 1])
+
+        # Display the name as a hyperlink in the first column
+        col1.markdown(f"[{name}]({url})")
+
+        # Display the content type in the second column
+        col2.write(content_type)
+
+        file_path = os.path.join(work_dir, file_utils.sanitize_filename(url))
+        file_exists = os.path.exists(file_path)
+        file_processed = file_path in processed_paths
+
+        # Checkboxes in the third and fourth columns
+        download_key = f"download_{unique_identifier}"
+        learn_key = f"learn_{unique_identifier}"
+
+        # Initialize session state for checkboxes if not already set
+        if download_key not in st.session_state['download_states']:
+            st.session_state['download_states'][download_key] = file_exists
+        if learn_key not in st.session_state['learn_states']:
+            st.session_state['learn_states'][learn_key] = file_processed
+
+        # Logic for enabling/disabling checkboxes
+        download_disabled = file_exists
+        learn_disabled = not file_exists or file_processed
+
+        # Display checkboxes
+        download_checked = col3.checkbox("Download", value=st.session_state['download_states'][download_key], key=download_key, disabled=download_disabled)
+        learn_checked = col4.checkbox("Learn It", value=st.session_state['learn_states'][learn_key], key=learn_key, disabled=learn_disabled)
+
+        # Check if the state of checkboxes changed
+        if download_checked != st.session_state['download_states'][download_key]:
+            if download_checked:
+                file_utils.perform_file_operation(item, "download")
+            st.session_state['download_states'][download_key] = download_checked
+
+        if learn_checked != st.session_state['learn_states'][learn_key]:
+            if learn_checked:
+                file_utils.perform_file_operation(item, "learn")
+            st.session_state['learn_states'][learn_key] = learn_checked
+
+
+    with st.expander("Manage Content in the System", expanded=True):
+        st.markdown("""
+            **Content Types:** 
+            - **Policies:** Guidelines for operations, organizational or industry-wide. 
+            - **Playbooks:** How-to guides and procedures for operational guidance. 
+            - **Standards:** Compliance with regulatory or industry best practices. 
+            - **Reference Docs:** In-depth information like technical manuals and research, however, they go into playbooks.
+        """)
+
+        selected_content_type = st.selectbox("Select Content Type", ["Select Type"] + app_constants.CONTENT_TYPE)
+        
+        if selected_content_type and selected_content_type != 'Select Type':
+            upload_choice = st.radio("Choose an option", ("Upload File", "Enter File Details Manually"))
+
+            if upload_choice == "Upload File":
+                uploaded_file = st.file_uploader("Choose a file", type=['pdf', 'docx', 'txt', 'html'])
+
+                if uploaded_file is not None:
+                    # Check if the file has already been processed in this session
+                    if 'processed_files' not in st.session_state:
+                        st.session_state['processed_files'] = set()
+
+                    file_details = (uploaded_file.name, uploaded_file.size)
+
+                    if file_details not in st.session_state['processed_files']:
+                        with st.spinner("Processing your file..."):
+                            file_utils.handle_content_update(uploaded_file=uploaded_file, selected_content_type=selected_content_type)
+                        st.session_state['processed_files'].add(file_details)
+                        st.success("File processed.")
+                    else:
+                        st.info("This file has already been processed in this session.")
+
+            elif upload_choice == "Enter File Details Manually":
+                with st.form("file_details_form"):
+                    manual_name = st.text_input("Document Name")
+                    manual_url = st.text_input("Download URL")
+                    submit_button = st.form_submit_button("Submit")
+
+                    if submit_button and manual_url and manual_name:
+                        # Use session state to check if the form details have already been submitted
+                        form_details = (manual_name, manual_url)
+
+                        if 'submitted_forms' not in st.session_state:
+                            st.session_state['submitted_forms'] = set()
+
+                        if form_details not in st.session_state['submitted_forms']:
+                            file_utils.handle_content_update(manual_name=manual_name, manual_url=manual_url, selected_content_type=selected_content_type)
+                            st.session_state['submitted_forms'].add(form_details)
+                            st.success("Form details processed.")
+                        else:
+                            st.info("These details have already been submitted in this session.")
+
+
+    st.write("Using below clear option, you can clear all data in the system and start fresh to index and upload information!")
+    if st.button("Clear Data"):
+        file_utils.delete_files()
+        st.write("All data cleared.")
+        st.rerun()

modules/nav_query_docs.py

@@ -0,0 +1,62 @@
+import streamlit as st
+from modules import app_logger, app_page_definitions, app_prompt, app_constants, app_st_session_utils,common_utils,file_utils
+import time
+# Use the logger from app_config
+app_logger = app_logger.app_logger
+
+def app(message_store, current_page="nav_private_ai", use_retrieval_chain=False):
+    app_logger.info(f"Starting Streamlit app - {current_page}")
+
+    # Fetch page configuration from app_page_definitions
+    page_config = app_page_definitions.PAGE_CONFIG.get(current_page, app_page_definitions.PAGE_CONFIG["default"])
+    files_indexed = file_utils.get_indexed_files_for_page(current_page)
+    #print(files_indexed)
+    # Use configurations for title, caption, and greeting from page_config
+    st.title(page_config["title"])
+    st.caption(page_config["caption"])
+
+    # Initialize or update session state variables
+    app_st_session_utils.initialize_session_state('current_page', current_page)
+    app_st_session_utils.initialize_session_state('page_loaded', False)
+    app_st_session_utils.initialize_session_state('message_store', message_store)
+    db_retriever_playbooks = False
+    if use_retrieval_chain:
+        db_retriever_playbooks = True
+        # Initialize or retrieve the database
+        persistent_db = app_constants.LOCAL_PERSISTANT_DB + current_page + '_chroma_db'
+        db_retriever_playbooks = app_st_session_utils.initialize_or_retrieve_db(persistent_db)
+
+    message_store = st.session_state['message_store']
+
+    # Manage message history
+    app_st_session_utils.manage_message_history(current_page)
+    greeting_message = common_utils.get_page_greeting(st.session_state['current_page'], st.session_state.get('username', ''),files_indexed)
+    st.chat_message("assistant").markdown(greeting_message, unsafe_allow_html=True)
+
+    # Display chat messages
+    for message in st.session_state.get("messages", []):
+        app_st_session_utils.display_chat_message(message["role"], message["content"])
+
+    # Handle user prompt
+    prompt = st.chat_input("Let's talk! Enter your query below.")
+    if prompt:
+        st.chat_message("user").write(prompt)
+        app_logger.info(f"Processed user prompt: {prompt}")
+        start_time = time.time()
+        with st.spinner("Processing request..."):
+            if use_retrieval_chain:
+                if db_retriever_playbooks:
+                    formatted_response = app_prompt.query_llm(prompt,page=current_page, retriever=db_retriever_playbooks.as_retriever(search_type="similarity", search_kwargs={"k": app_constants.RAG_K}), message_store=message_store, use_retrieval_chain=use_retrieval_chain)
+                    app_st_session_utils.display_chat_message("assistant", formatted_response)  # Updated line
+                    app_st_session_utils.add_message_to_session("user", prompt)
+                    app_st_session_utils.add_message_to_session("assistant", formatted_response)           
+                else:
+                    st.error("Unable to initialize the database. Please try again later.")
+            else:
+                formatted_response = app_prompt.query_llm(prompt,page=current_page, message_store=message_store, retriever=False)
+                app_st_session_utils.display_chat_message("assistant", formatted_response)  # Updated line
+                app_st_session_utils.add_message_to_session("user", prompt)
+                app_st_session_utils.add_message_to_session("assistant", formatted_response)
+        end_time = time.time()  # End timing
+        processing_time = end_time - start_time  # Calculate processing time
+        st.info(f"Processing time: {processing_time:.2f} seconds")  # Log processing time

modules/nav_researcher.py

@@ -0,0 +1,77 @@
+import streamlit as st
+from modules import app_prompt, app_researcher, app_logger, database_utils, app_to_vectorstore, app_page_definitions
+from modules import app_st_session_utils,app_constants,common_utils  # Importing the session utilities module
+
+# Use the logger from app_config
+app_logger = app_logger.app_logger
+
+def app(message_store):
+    app_logger.info("Navigating to nav_researcher page")
+
+    # Fetch page configuration from app_page_definitions
+    current_page = "nav_researcher"
+    page_config = app_page_definitions.PAGE_CONFIG.get(current_page)
+
+    st.title(page_config["title"])
+
+    # Initialize or update session state variables using session utilities
+    app_st_session_utils.initialize_session_state('current_page', current_page)
+    app_st_session_utils.initialize_session_state('page_loaded', False)
+    app_st_session_utils.initialize_session_state('message_store', message_store)
+
+    st.caption(page_config["caption"])
+
+    topic = st.text_input("Enter Topic for Research", "Threat Management")
+    research_button = st.button("Go Research on Internet")
+
+    if research_button and topic:
+        with st.spinner('Searching...'):
+            try:
+                research_notes = app_researcher.explore_url_on_internet(topic, count=app_constants.SEARCH_COUNT)
+                status = app_to_vectorstore.get_chroma_index(research_notes, is_persistent=False)
+                app_logger.info("Internet research completed successfully")
+                st.success("Internet research completed")
+                st.session_state['research_done'] = True
+            except Exception as e:
+                app_logger.error(f"Error during internet research: {e}")
+                st.error(f"Error during internet research: {e}")
+
+    TEMP_DIR = app_constants.WORKSPACE_DIRECTORY + "tmp"
+    db_files = database_utils.get_chroma_db_files(TEMP_DIR)
+    
+    # Create a mapping of formatted names to actual file names
+    formatted_db_names = [database_utils.format_db_name(db) for db in db_files]
+    name_to_file_map = dict(zip(formatted_db_names, db_files))
+
+    # Display formatted names in the dropdown and use the selection to get the actual file name
+    selected_db_formatted = st.selectbox("Pick Researched topic from drop-down and start chatting!", formatted_db_names)
+    selected_db_actual = name_to_file_map[selected_db_formatted]
+    research_notes = TEMP_DIR + '/' + selected_db_actual
+
+    # Initialize or retrieve the database using the new function
+    db_retriever = app_st_session_utils.initialize_or_retrieve_db(research_notes)
+
+    app_st_session_utils.manage_message_history(current_page)
+    
+    greeting_message = common_utils.get_page_greeting(st.session_state['current_page'], st.session_state.get('username', ''))
+    st.chat_message("assistant").markdown(greeting_message, unsafe_allow_html=True)
+    app_st_session_utils.update_session_state('page_loaded', True)
+
+
+    # Displaying chat messages
+    for message in st.session_state.get("messages", []):
+        app_st_session_utils.display_chat_message(message["role"], message["content"])
+
+    # Handling user prompt
+    prompt = st.chat_input("Let's Talk! Conversation secure and private!")
+    if prompt:
+        st.chat_message("user").write(prompt)
+        with st.spinner("Processing your request..."):
+            if db_retriever:
+                formatted_response = app_prompt.query_llm(prompt,page=current_page, retriever=db_retriever.as_retriever(search_type="similarity", search_kwargs={"k": app_constants.RAG_K}), message_store=st.session_state['message_store'],use_retrieval_chain=True)
+                st.chat_message("assistant").markdown(formatted_response, unsafe_allow_html=True)
+                app_st_session_utils.add_message_to_session("user", prompt)
+                app_st_session_utils.add_message_to_session("assistant", formatted_response)
+                app_logger.info(f"Processed user prompt: {prompt}")
+            else:
+                st.error("Unable to initialize the database. Please try again later.")

modules/nav_summarizer.py

@@ -0,0 +1,113 @@
+
+import os
+import streamlit as st
+from langchain_openai import ChatOpenAI
+from langchain.chains import load_summarize_chain
+from langchain_community.document_loaders import PyPDFLoader, TextLoader, UnstructuredWordDocumentLoader
+from langchain.prompts import PromptTemplate
+from modules import app_page_definitions, app_logger,app_constants,file_utils
+
+# Use the logger from app_config
+app_logger = app_logger.app_logger
+
+# Configurable batch size (4 pages per batch)
+batch_size = app_constants.SUMMARIZER_BATCH
+WORKSPACE_DIRECTORY = app_constants.WORKSPACE_DIRECTORY
+
+
+def process_file(file_path, file_type):
+    if file_type == "text/plain":
+        loader = TextLoader(file_path)
+    elif file_type == "application/pdf":
+        loader = PyPDFLoader(file_path)
+    elif file_type == "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+        loader = UnstructuredWordDocumentLoader(file_path)
+    else:
+        raise ValueError(f"Unsupported file type: {file_type}")
+    app_logger.info(f"Processing file {file_path} of type {file_type}")
+    return loader.load_and_split()
+
+def app():
+    app_logger.info("Starting Streamlit app - Summarizer Tool page")
+
+    # Fetch page configuration from app_page_definitions
+    page_config = app_page_definitions.PAGE_CONFIG.get("nav_summarize")
+
+    st.title(page_config["title"])
+    st.caption(page_config["caption"])
+    st.session_state.current_page = "nav_summarize"
+
+    uploaded_file = st.file_uploader("Upload your document here:", type=['txt', 'pdf', 'docx'], key="file_uploader")
+
+    if uploaded_file is not None:
+        file_path = file_utils.save_uploaded_file(uploaded_file,uploads_path=WORKSPACE_DIRECTORY + "/tmp")
+        docs = process_file(file_path, uploaded_file.type)
+
+        total_docs = len(docs)
+        app_logger.info(f"Total documents processed: {total_docs}")
+
+        if total_docs > 1:
+            doc_range = st.slider("Select document range for summarization", 1, total_docs, (1, total_docs))
+        else:
+            doc_range = (1, 1)
+
+        progress_bar = st.progress(0)
+
+        if st.button("Summarize"):
+            with st.spinner('Processing... Please wait'):
+                llm = ChatOpenAI(
+                    model_name=app_constants.MODEL_NAME,
+                    openai_api_key=app_constants.openai_api_key,
+                    base_url=app_constants.local_model_uri,
+                    streaming=True
+                )
+
+                prompt_template = """Write a concise summary of the following:
+                {text}
+                CONCISE SUMMARY:"""
+                prompt = PromptTemplate.from_template(prompt_template)
+
+                refine_template = (
+                    "You are a content writer and your job is to produce a summary of input\n"
+                    "We have provided an existing summary up to a certain point: {existing_answer}\n"
+                    "Start and end properly and refine the existing summary "
+                    "with some more context below.\n"
+                    "------------\n"
+                    "{text}\n"
+                    "------------\n"
+                    "Given the new context, refine the original summary. "
+                    "If the context isn't useful, return the original summary."
+                )
+                refine_prompt = PromptTemplate.from_template(refine_template)
+
+                chain = load_summarize_chain(
+                    llm=llm,
+                    chain_type="refine",
+                    question_prompt=prompt,
+                    refine_prompt=refine_prompt,
+                    return_intermediate_steps=True,
+                    input_key="input_documents",
+                    output_key="output_text",
+                )
+
+                start_doc, end_doc = doc_range
+                for i in range(start_doc - 1, min(end_doc, total_docs), batch_size):
+                    batch_docs = docs[i:min(i + batch_size, total_docs)]
+
+                    progress_value = (i + len(batch_docs)) / total_docs
+                    progress_bar.progress(progress_value)
+
+                    with st.expander(f"Processing Documents {i + 1} - {i + len(batch_docs)}", expanded=False):
+                        intermediate_summary = chain.invoke({"input_documents": batch_docs}, return_only_outputs=True)
+                        st.write(intermediate_summary)
+
+                selected_docs = docs[start_doc - 1:end_doc]
+                final_summary_response = chain.invoke({"input_documents": selected_docs}, return_only_outputs=True)
+                final_summary = final_summary_response['output_text'] if 'output_text' in final_summary_response else "No summary generated."
+                st.text_area("Final Summary", final_summary, height=300)
+
+                st.success("Summarization Completed!")
+            progress_bar.empty()
+    else:
+        st.warning("Please upload a document to summarize.")
+        app_logger.warning("No document uploaded for summarization")

pyvenv.cfg

@@ -0,0 +1,5 @@
+home = /Users/mars/miniconda3/bin
+include-system-site-packages = false
+version = 3.11.5
+executable = /Users/mars/miniconda3/bin/python3.11
+command = /Users/mars/miniconda3/bin/python -m venv /Volumes/localdisk/projects/zysec.ai

requirements.txt

@@ -0,0 +1,240 @@
+aiohttp==3.9.3
+aiosignal==1.3.1
+altair==5.2.0
+annotated-types==0.6.0
+antlr4-python3-runtime==4.9.3
+anyio==4.2.0
+asgiref==3.7.2
+attrs==23.2.0
+backoff==2.2.1
+bcrypt==4.1.2
+beautifulsoup4==4.12.3
+blinker==1.7.0
+blis==0.7.11
+bs4==0.0.2
+build==1.0.3
+cachetools==5.3.2
+catalogue==2.0.10
+certifi==2024.2.2
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+chroma-hnswlib==0.7.3
+chromadb==0.4.22
+click==8.1.7
+cloudpathlib==0.16.0
+coloredlogs==15.0.1
+confection==0.1.4
+contourpy==1.2.0
+cryptography==42.0.2
+curl_cffi==0.6.0b9
+cycler==0.12.1
+cymem==2.0.8
+dataclasses-json==0.6.4
+dataclasses-json-speakeasy==0.5.11
+Deprecated==1.2.14
+diskcache==5.6.3
+distro==1.9.0
+docx2txt==0.8
+duckduckgo_search==4.4.3
+effdet==0.4.1
+emoji==2.10.1
+en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl#sha256=86cc141f63942d4b2c5fcee06630fd6f904788d2f0ab005cce45aadb8fb73889
+entrypoints==0.4
+Faker==23.2.1
+fastapi==0.109.2
+favicon==0.7.0
+filelock==3.13.1
+filetype==1.2.0
+flatbuffers==23.5.26
+fonttools==4.48.1
+frozenlist==1.4.1
+fsspec==2024.2.0
+gitdb==4.0.11
+GitPython==3.1.41
+google-auth==2.27.0
+googleapis-common-protos==1.62.0
+grpcio==1.60.1
+h11==0.14.0
+htbuilder==0.6.2
+html2text==2020.1.16
+httpcore==1.0.2
+httptools==0.6.1
+httpx==0.26.0
+huggingface-hub==0.20.3
+humanfriendly==10.0
+humanize==4.9.0
+idna==3.6
+importlib-metadata==6.11.0
+importlib-resources==6.1.1
+iopath==0.1.10
+Jinja2==3.1.3
+joblib==1.3.2
+jq==1.6.0
+jsonpatch==1.33
+jsonpath-python==1.0.6
+jsonpointer==2.4
+jsonschema==4.21.1
+jsonschema-specifications==2023.12.1
+kiwisolver==1.4.5
+kubernetes==29.0.0
+langchain==0.1.6
+langchain-community==0.0.19
+langchain-core==0.1.22
+langchain-openai==0.0.5
+langcodes==3.3.0
+langdetect==1.0.9
+langsmith==0.0.87
+layoutparser==0.3.4
+llama_cpp_python==0.2.53
+lxml==5.1.0
+Markdown==3.5.2
+markdown-it-py==3.0.0
+markdownlit==0.0.7
+MarkupSafe==2.1.5
+marshmallow==3.20.2
+matplotlib==3.8.2
+mdurl==0.1.2
+mmh3==4.1.0
+monotonic==1.6
+more-itertools==10.2.0
+mpmath==1.3.0
+multidict==6.0.5
+murmurhash==1.0.10
+mypy-extensions==1.0.0
+nest-asyncio==1.6.0
+networkx==3.2.1
+nltk==3.8.1
+numpy==1.26.4
+oauthlib==3.2.2
+omegaconf==2.3.0
+onnx==1.15.0
+onnxruntime==1.15.1
+openai==1.12.0
+opencv-python==4.9.0.80
+opentelemetry-api==1.22.0
+opentelemetry-exporter-otlp-proto-common==1.22.0
+opentelemetry-exporter-otlp-proto-grpc==1.22.0
+opentelemetry-instrumentation==0.43b0
+opentelemetry-instrumentation-asgi==0.43b0
+opentelemetry-instrumentation-fastapi==0.43b0
+opentelemetry-proto==1.22.0
+opentelemetry-sdk==1.22.0
+opentelemetry-semantic-conventions==0.43b0
+opentelemetry-util-http==0.43b0
+overrides==7.7.0
+packaging==23.2
+pandas==2.2.0
+pdf2image==1.17.0
+pdfminer.six==20221105
+pdfplumber==0.10.4
+pikepdf==8.12.0
+pillow==10.2.0
+pillow_heif==0.15.0
+pip-review==1.3.0
+portalocker==2.8.2
+posthog==3.4.0
+preshed==3.0.9
+prometheus_client==0.20.0
+protobuf==4.25.2
+pulsar-client==3.4.0
+pyarrow==15.0.0
+pyasn1==0.5.1
+pyasn1-modules==0.3.0
+pycocotools==2.0.7
+pycparser==2.21
+pydantic==2.6.1
+pydantic-settings==2.1.0
+pydantic_core==2.16.2
+pydeck==0.8.1b0
+Pygments==2.17.2
+pymdown-extensions==10.7
+pyparsing==3.1.1
+pypdf==4.0.1
+pypdfium2==4.27.0
+PyPika==0.48.9
+pyproject_hooks==1.0.0
+pytesseract==0.3.10
+python-dateutil==2.8.2
+python-dotenv==1.0.1
+python-iso639==2024.2.7
+python-magic==0.4.27
+python-multipart==0.0.9
+pytz==2024.1
+PyYAML==6.0.1
+rapidfuzz==3.6.1
+referencing==0.33.0
+regex==2023.12.25
+requests==2.31.0
+requests-oauthlib==1.3.1
+rich==13.7.0
+rpds-py==0.17.1
+rsa==4.9
+safetensors==0.4.2
+scikit-learn==1.4.0
+scipy==1.12.0
+sentence-transformers==2.3.1
+sentencepiece==0.1.99
+six==1.16.0
+smart-open==6.4.0
+smmap==5.0.1
+sniffio==1.3.0
+soupsieve==2.5
+spacy==3.7.2
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+SQLAlchemy==2.0.26
+srsly==2.4.8
+sse-starlette==2.0.0
+st-annotated-text==4.0.1
+starlette==0.36.3
+starlette-context==0.3.6
+streamlit==1.31.1
+streamlit-camera-input-live==0.2.0
+streamlit-card==1.0.0
+streamlit-embedcode==0.1.2
+streamlit-extras==0.4.0
+streamlit-faker==0.0.3
+streamlit-image-coordinates==0.1.6
+streamlit-keyup==0.2.3
+streamlit-option-menu==0.3.12
+streamlit-toggle-switch==1.0.2
+streamlit-vertical-slider==2.5.5
+sympy==1.12
+tabulate==0.9.0
+tenacity==8.2.3
+thinc==8.2.3
+threadpoolctl==3.2.0
+tiktoken==0.5.2
+timm==0.9.12
+tokenizers==0.15.2
+toml==0.10.2
+toolz==0.12.1
+torch==2.2.0
+torchvision==0.17.0
+tornado==6.4
+tqdm==4.66.2
+transformers==4.37.2
+typer==0.9.0
+typing-inspect==0.9.0
+typing_extensions==4.9.0
+tzdata==2024.1
+tzlocal==5.2
+unstructured==0.12.4
+unstructured-client==0.18.0
+unstructured-inference==0.7.23
+unstructured.pytesseract==0.3.12
+urllib3==2.2.0
+uvicorn==0.27.1
+uvloop==0.19.0
+validators==0.22.0
+vulture==2.11
+wasabi==1.1.2
+watchdog==4.0.0
+watchfiles==0.21.0
+weasel==0.3.4
+websocket-client==1.7.0
+websockets==12.0
+wrapt==1.16.0
+yarl==1.9.4
+zipp==3.17.0

start_model_server.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Define model directory and file
+
+#ZySec-7B-v1.Q2_K.gguf - MISTRAL
+#ZySec-7B-v1.Q4_K_M.gguf - MISTRAL
+#ZySec-7B-v1.Q8_0.gguf - MISTRAL
+
+#ZySec-7B-v2.Q2_K.gguf - GEMMA
+#ZySec-7B-v2.Q4_K_M.gguf - GEMMA
+#ZySec-7B-v2.Q8_0.gguf - GEMMA
+
+model_dir="models/ZySec-AI"
+model_file="ZySec-7B-v1.Q2_K.gguf"
+model_path="$model_dir/$model_file"
+
+# Function to handle SIGINT (Ctrl+C)
+handle_sigint() {
+  echo "SIGINT received, stopping the server and exiting..."
+  kill $server_pid
+  exit
+}
+
+# Trap SIGINT (Ctrl+C) and execute the handle_sigint function
+trap handle_sigint SIGINT
+
+# Step 2: Check for curl and download model file if it doesn't exist
+if ! command -v curl &> /dev/null
+then
+    echo "curl could not be found, please install it."
+    exit 1
+fi
+
+if [ ! -d "$model_dir" ]; then
+    echo "Directory $model_dir does not exist. Creating now."
+    mkdir -p "$model_dir" && echo "Directory created." || { echo "Failed to create directory."; exit 1; }
+fi
+
+if [ ! -f "$model_path" ]; then
+    echo "Model file $model_file does not exist. Downloading now."
+    cd "$model_dir" || { echo "Failed to navigate to $model_dir"; exit 1; }
+    curl -L -o "$model_file" "https://huggingface.co/ZySec-AI/ZySec-7B-v2-GGUF/resolve/main/$model_file?download=true" && echo "Download completed." || { echo "Failed to download model."; exit 1; }
+else
+    echo "Model file $model_file already exists. Skipping download."
+fi
+
+# Function to start or restart the model server
+start_model_server() {
+    # Check if port 8000 is already in use
+    if lsof -i:8000 -sTCP:LISTEN -t >/dev/null ; then
+        echo "Port 8000 is already in use. Assuming the model server is running."
+        return
+    fi
+
+    echo "Starting model server..."
+    python3 -m llama_cpp.server --model "./$model_path" --n_batch 4 --n_ctx 8196 --n_batch 200 --verbose true --n_gpu_layers 50 --chat_format zephyr &
+    server_pid=$!
+    wait $server_pid
+
+    echo "Model server stopped. Exiting."
+    exit 1
+}
+
+# Step 4: Start model server in the background
+start_model_server &
+wait

start_web_ui.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Step 1: Git pull
+echo "Pulling latest changes from Git repository..."
+git pull
+
+# Step 3: Setup and activate virtual environment
+venv_path="zysec"
+if [ ! -d "$venv_path" ]; then
+    echo "Creating virtual environment 'ZySec'..."
+    python3 -m venv $venv_path
+fi
+
+echo "Activating virtual environment 'ZySec'..."
+source $venv_path/bin/activate
+
+# Check if we are in the right virtual environment
+if [[ "$VIRTUAL_ENV" != "" && "$VIRTUAL_ENV" == *"$venv_path" ]]; then
+    echo "Now in the 'ZySec' virtual environment."
+    # Install requirements
+    pip3 install -r requirements.txt -q
+else
+    echo "Failed to activate 'ZySec' virtual environment. Exiting."
+    exit 1
+fi
+
+# Step 5: Start the Streamlit app
+echo "Assuming model instance is running.. you can start it or review settings in about section to connect to remote instance."
+echo "Starting Streamlit app..."
+streamlit run app.py
+