Synced repo using 'sync_with_huggingface' Github Action

.chainlit/config.toml

@@ -0,0 +1,121 @@
+[project]
+# Whether to enable telemetry (default: true). No personal data is collected.
+enable_telemetry = true
+
+
+# List of environment variables to be provided by each user to use the app.
+user_env = []
+
+# Duration (in seconds) during which the session is saved when the connection is lost
+session_timeout = 3600
+
+# Enable third parties caching (e.g LangChain cache)
+cache = false
+
+# Authorized origins
+allow_origins = ["*"]
+
+# Follow symlink for asset mount (see https://github.com/Chainlit/chainlit/issues/317)
+# follow_symlink = false
+
+[features]
+# Show the prompt playground
+prompt_playground = true
+
+# Process and display HTML in messages. This can be a security risk (see https://stackoverflow.com/questions/19603097/why-is-it-dangerous-to-render-user-generated-html-or-javascript)
+unsafe_allow_html = false
+
+# Process and display mathematical expressions. This can clash with "$" characters in messages.
+latex = false
+
+# Automatically tag threads with the current chat profile (if a chat profile is used)
+auto_tag_thread = true
+
+# Authorize users to spontaneously upload files with messages
+[features.spontaneous_file_upload]
+    enabled = true
+    accept = ["*/*"]
+    max_files = 20
+    max_size_mb = 500
+
+[features.audio]
+    # Threshold for audio recording
+    min_decibels = -45
+    # Delay for the user to start speaking in MS
+    initial_silence_timeout = 3000
+    # Delay for the user to continue speaking in MS. If the user stops speaking for this duration, the recording will stop.
+    silence_timeout = 1500
+    # Above this duration (MS), the recording will forcefully stop.
+    max_duration = 15000
+    # Duration of the audio chunks in MS
+    chunk_duration = 1000
+    # Sample rate of the audio
+    sample_rate = 44100
+
+[UI]
+# Name of the app and chatbot.
+name = "SCC Sherpa"
+
+# Show the readme while the thread is empty.
+show_readme_as_default = true
+
+# Description of the app and chatbot. This is used for HTML tags.
+# description = "SCC Guide"
+
+# Large size content are by default collapsed for a cleaner ui
+default_collapse_content = true
+
+# The default value for the expand messages settings.
+default_expand_messages = false
+
+# Hide the chain of thought details from the user in the UI.
+hide_cot = false
+
+# Link to your github repo. This will add a github button in the UI's header.
+# github = ""
+
+# Specify a CSS file that can be used to customize the user interface.
+# The CSS file can be served from the public directory or via an external link.
+custom_css = "/public/test.css"
+
+# Specify a Javascript file that can be used to customize the user interface.
+# The Javascript file can be served from the public directory.
+# custom_js = "/public/test.js"
+
+# Specify a custom font url.
+# custom_font = "https://fonts.googleapis.com/css2?family=Inter:wght@400;500;700&display=swap"
+
+# Specify a custom meta image url.
+# custom_meta_image_url = "https://chainlit-cloud.s3.eu-west-3.amazonaws.com/logo/chainlit_banner.png"
+
+# Specify a custom build directory for the frontend.
+# This can be used to customize the frontend code.
+# Be careful: If this is a relative path, it should not start with a slash.
+# custom_build = "./public/build"
+
+[UI.theme]
+    #layout = "wide"
+    #font_family = "Inter, sans-serif"
+# Override default MUI light theme. (Check theme.ts)
+[UI.theme.light]
+    #background = "#FAFAFA"
+    #paper = "#FFFFFF"
+
+    [UI.theme.light.primary]
+        #main = "#F80061"
+        #dark = "#980039"
+        #light = "#FFE7EB"
+
+# Override default MUI dark theme. (Check theme.ts)
+[UI.theme.dark]
+    background = "#1C1C1C" # Slightly lighter dark background color
+    paper = "#2A2A2A"      # Slightly lighter dark paper color
+
+    [UI.theme.dark.primary]
+        main = "#89CFF0"    # Primary color
+        dark = "#3700B3"    # Dark variant of primary color
+        light = "#CFBCFF"   # Lighter variant of primary color
+
+
+[meta]
+generated_by = "1.1.202"

.chainlit/translations/en-US.json

@@ -0,0 +1,231 @@
+{
+    "components": {
+        "atoms": {
+            "buttons": {
+                "userButton": {
+                    "menu": {
+                        "settings": "Settings",
+                        "settingsKey": "S",
+                        "APIKeys": "API Keys",
+                        "logout": "Logout"
+                    }
+                }
+            }
+        },
+        "molecules": {
+            "newChatButton": {
+                "newChat": "New Chat"
+            },
+            "tasklist": {
+                "TaskList": {
+                    "title": "\ud83d\uddd2\ufe0f Task List",
+                    "loading": "Loading...",
+                    "error": "An error occured"
+                }
+            },
+            "attachments": {
+                "cancelUpload": "Cancel upload",
+                "removeAttachment": "Remove attachment"
+            },
+            "newChatDialog": {
+                "createNewChat": "Create new chat?",
+                "clearChat": "This will clear the current messages and start a new chat.",
+                "cancel": "Cancel",
+                "confirm": "Confirm"
+            },
+            "settingsModal": {
+                "settings": "Settings",
+                "expandMessages": "Expand Messages",
+                "hideChainOfThought": "Hide Chain of Thought",
+                "darkMode": "Dark Mode"
+            },
+            "detailsButton": {
+                "using": "Using",
+                "running": "Running",
+                "took_one": "Took {{count}} step",
+                "took_other": "Took {{count}} steps"
+            },
+            "auth": {
+                "authLogin": {
+                    "title": "Login to access the app.",
+                    "form": {
+                        "email": "Email address",
+                        "password": "Password",
+                        "noAccount": "Don't have an account?",
+                        "alreadyHaveAccount": "Already have an account?",
+                        "signup": "Sign Up",
+                        "signin": "Sign In",
+                        "or": "OR",
+                        "continue": "Continue",
+                        "forgotPassword": "Forgot password?",
+                        "passwordMustContain": "Your password must contain:",
+                        "emailRequired": "email is a required field",
+                        "passwordRequired": "password is a required field"
+                    },
+                    "error": {
+                        "default": "Unable to sign in.",
+                        "signin": "Try signing in with a different account.",
+                        "oauthsignin": "Try signing in with a different account.",
+                        "redirect_uri_mismatch": "The redirect URI is not matching the oauth app configuration.",
+                        "oauthcallbackerror": "Try signing in with a different account.",
+                        "oauthcreateaccount": "Try signing in with a different account.",
+                        "emailcreateaccount": "Try signing in with a different account.",
+                        "callback": "Try signing in with a different account.",
+                        "oauthaccountnotlinked": "To confirm your identity, sign in with the same account you used originally.",
+                        "emailsignin": "The e-mail could not be sent.",
+                        "emailverify": "Please verify your email, a new email has been sent.",
+                        "credentialssignin": "Sign in failed. Check the details you provided are correct.",
+                        "sessionrequired": "Please sign in to access this page."
+                    }
+                },
+                "authVerifyEmail": {
+                    "almostThere": "You're almost there! We've sent an email to ",
+                    "verifyEmailLink": "Please click on the link in that email to complete your signup.",
+                    "didNotReceive": "Can't find the email?",
+                    "resendEmail": "Resend email",
+                    "goBack": "Go Back",
+                    "emailSent": "Email sent successfully.",
+                    "verifyEmail": "Verify your email address"
+                },
+                "providerButton": {
+                    "continue": "Continue with {{provider}}",
+                    "signup": "Sign up with {{provider}}"
+                },
+                "authResetPassword": {
+                    "newPasswordRequired": "New password is a required field",
+                    "passwordsMustMatch": "Passwords must match",
+                    "confirmPasswordRequired": "Confirm password is a required field",
+                    "newPassword": "New password",
+                    "confirmPassword": "Confirm password",
+                    "resetPassword": "Reset Password"
+                },
+                "authForgotPassword": {
+                    "email": "Email address",
+                    "emailRequired": "email is a required field",
+                    "emailSent": "Please check the email address {{email}} for instructions to reset your password.",
+                    "enterEmail": "Enter your email address and we will send you instructions to reset your password.",
+                    "resendEmail": "Resend email",
+                    "continue": "Continue",
+                    "goBack": "Go Back"
+                }
+            }
+        },
+        "organisms": {
+            "chat": {
+                "history": {
+                    "index": {
+                        "showHistory": "Show history",
+                        "lastInputs": "Last Inputs",
+                        "noInputs": "Such empty...",
+                        "loading": "Loading..."
+                    }
+                },
+                "inputBox": {
+                    "input": {
+                        "placeholder": "Type your message here..."
+                    },
+                    "speechButton": {
+                        "start": "Start recording",
+                        "stop": "Stop recording"
+                    },
+                    "SubmitButton": {
+                        "sendMessage": "Send message",
+                        "stopTask": "Stop Task"
+                    },
+                    "UploadButton": {
+                        "attachFiles": "Attach files"
+                    },
+                    "waterMark": {
+                        "text": "Built with"
+                    }
+                },
+                "Messages": {
+                    "index": {
+                        "running": "Running",
+                        "executedSuccessfully": "executed successfully",
+                        "failed": "failed",
+                        "feedbackUpdated": "Feedback updated",
+                        "updating": "Updating"
+                    }
+                },
+                "dropScreen": {
+                    "dropYourFilesHere": "Drop your files here"
+                },
+                "index": {
+                    "failedToUpload": "Failed to upload",
+                    "cancelledUploadOf": "Cancelled upload of",
+                    "couldNotReachServer": "Could not reach the server",
+                    "continuingChat": "Continuing previous chat"
+                },
+                "settings": {
+                    "settingsPanel": "Settings panel",
+                    "reset": "Reset",
+                    "cancel": "Cancel",
+                    "confirm": "Confirm"
+                }
+            },
+            "threadHistory": {
+                "sidebar": {
+                    "filters": {
+                        "FeedbackSelect": {
+                            "feedbackAll": "Feedback: All",
+                            "feedbackPositive": "Feedback: Positive",
+                            "feedbackNegative": "Feedback: Negative"
+                        },
+                        "SearchBar": {
+                            "search": "Search"
+                        }
+                    },
+                    "DeleteThreadButton": {
+                        "confirmMessage": "This will delete the thread as well as it's messages and elements.",
+                        "cancel": "Cancel",
+                        "confirm": "Confirm",
+                        "deletingChat": "Deleting chat",
+                        "chatDeleted": "Chat deleted"
+                    },
+                    "index": {
+                        "pastChats": "Past Chats"
+                    },
+                    "ThreadList": {
+                        "empty": "Empty...",
+                        "today": "Today",
+                        "yesterday": "Yesterday",
+                        "previous7days": "Previous 7 days",
+                        "previous30days": "Previous 30 days"
+                    },
+                    "TriggerButton": {
+                        "closeSidebar": "Close sidebar",
+                        "openSidebar": "Open sidebar"
+                    }
+                },
+                "Thread": {
+                    "backToChat": "Go back to chat",
+                    "chatCreatedOn": "This chat was created on"
+                }
+            },
+            "header": {
+                "chat": "Chat",
+                "readme": "Readme"
+            }
+        }
+    },
+    "hooks": {
+        "useLLMProviders": {
+            "failedToFetchProviders": "Failed to fetch providers:"
+        }
+    },
+    "pages": {
+        "Design": {},
+        "Env": {
+            "savedSuccessfully": "Saved successfully",
+            "requiredApiKeys": "Required API Keys",
+            "requiredApiKeysInfo": "To use this app, the following API keys are required. The keys are stored on your device's local storage."
+        },
+        "Page": {
+            "notPartOfProject": "You are not part of this project."
+        },
+        "ResumeButton": {
+            "resumeChat": "Resume Chat"
+        }
+    }
+}

Dockerfile

@@ -0,0 +1,29 @@
+FROM python:3.9
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+RUN pip install --no-cache-dir transformers==4.36.2 torch==2.1.2
+
+RUN pip install --upgrade --force-reinstall --no-cache-dir llama-cpp-python==0.2.32
+
+COPY . /code
+
+RUN ls -R
+
+# Change permissions to allow writing to the directory
+RUN chmod -R 777 /code
+
+# Create a logs directory and set permissions
+RUN mkdir /code/logs && chmod 777 /code/logs
+
+# Create a cache directory within the application's working directory
+RUN mkdir /.cache && chmod -R 777 /.cache
+
+RUN --mount=type=secret,id=HUGGINGFACEHUB_API_TOKEN,mode=0444,required=true 
+RUN --mount=type=secret,id=OPENAI_API_KEY,mode=0444,required=true 
+
+CMD python code/modules/vector_db.py && chainlit run code/main.py --host 0.0.0.0 --port 7860

Dockerfile.dev

@@ -0,0 +1,27 @@
+FROM python:3.11
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --upgrade pip
+
+RUN pip install --no-cache-dir -r /code/requirements.txt
+
+COPY . /code
+
+RUN ls -R
+
+# Change permissions to allow writing to the directory
+RUN chmod -R 777 /code
+
+# Create a logs directory and set permissions
+RUN mkdir /code/logs && chmod 777 /code/logs
+
+# Create a cache directory within the application's working directory
+RUN mkdir /.cache && chmod -R 777 /.cache
+
+# Expose the port the app runs on
+EXPOSE 8051
+
+CMD chainlit run code/main.py --port 8051

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 DL4DS
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

chainlit.md

@@ -0,0 +1,10 @@
+# Welcome to DL4DS Tutor! 🚀🤖
+
+Hi there, this is an LLM chatbot designed to help answer your questions.
+This is still very much a Work in Progress.
+
+### --- Please wait while the Tutor loads... ---
+
+## Useful Links 🔗
+
+- **Documentation:**  [Chainlit Documentation](https://docs.chainlit.io) 📚

code/config.yml

@@ -0,0 +1,32 @@
+embedding_options:
+  embedd_files: False # bool
+  data_path: 'storage/data' # str
+  url_file_path: 'storage/data/urls.txt' # str
+  expand_urls: True # bool
+  db_option : 'FAISS' # str [FAISS, Chroma, RAGatouille]
+  db_path : 'vectorstores' # str
+  model : 'sentence-transformers/all-MiniLM-L6-v2' # str [sentence-transformers/all-MiniLM-L6-v2, text-embedding-ada-002']
+  search_top_k : 3 # int
+  score_threshold : 0.0 # float
+  lambda_mult: 0.5 # float - Determines Diversity of the retrieved results
+llm_params: 
+  use_history: True # bool
+  memory_window: 3 # int
+  llm_loader: 'local_llm' # str [local_llm, openai]
+  openai_params:
+    model: 'gpt-4' # str [gpt-3.5-turbo-1106, gpt-4]
+  local_llm_params:
+    model: "storage/models/tinyllama-1.1b-chat-v0.3.Q5_K_M.gguf"
+    model_type: "llama"
+    temperature: 0.2
+splitter_options:
+  use_splitter: True # bool
+  split_by_token : True # bool
+  remove_leftover_delimiters: True # bool
+  remove_chunks: False # bool
+  chunk_size : 300 # int
+  chunk_overlap : 30 # int
+  chunk_separators : ["\n\n", "\n", " ", ""] # list of strings
+  front_chunks_to_remove : null # int or None
+  last_chunks_to_remove : null # int or None
+  delimiters_to_remove : ['\t', '\n', '   ', '  '] # list of strings

code/main.py

@@ -0,0 +1,124 @@
+from langchain.document_loaders import PyPDFLoader, DirectoryLoader
+from langchain import PromptTemplate
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.chains import RetrievalQA
+from langchain.llms import CTransformers
+import chainlit as cl
+from langchain_community.chat_models import ChatOpenAI
+from langchain_community.embeddings import OpenAIEmbeddings
+import yaml
+import logging
+from dotenv import load_dotenv
+
+from modules.llm_tutor import LLMTutor
+from modules.constants import *
+from modules.helpers import get_sources
+
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+# Console Handler
+console_handler = logging.StreamHandler()
+console_handler.setLevel(logging.INFO)
+formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
+console_handler.setFormatter(formatter)
+logger.addHandler(console_handler)
+
+# File Handler
+log_file_path = "log_file.log"  # Change this to your desired log file path
+file_handler = logging.FileHandler(log_file_path)
+file_handler.setLevel(logging.INFO)
+file_handler.setFormatter(formatter)
+logger.addHandler(file_handler)
+
+
+# Adding option to select the chat profile
+@cl.set_chat_profiles
+async def chat_profile():
+    return [
+        cl.ChatProfile(
+            name="Llama",
+            markdown_description="Use the local LLM: **Tiny Llama**.",
+        ),
+        # cl.ChatProfile(
+        #     name="Mistral",
+        #     markdown_description="Use the local LLM: **Mistral**.",
+        # ),
+        cl.ChatProfile(
+            name="gpt-3.5-turbo-1106",
+            markdown_description="Use OpenAI API for **gpt-3.5-turbo-1106**.",
+        ),
+        cl.ChatProfile(
+            name="gpt-4",
+            markdown_description="Use OpenAI API for **gpt-4**.",
+        ),
+    ]
+
+
+@cl.author_rename
+def rename(orig_author: str):
+    rename_dict = {"Chatbot": "AI Tutor"}
+    return rename_dict.get(orig_author, orig_author)
+
+
+# chainlit code
+@cl.on_chat_start
+async def start():
+    with open("code/config.yml", "r") as f:
+        config = yaml.safe_load(f)
+        print(config)
+        logger.info("Config file loaded")
+        logger.info(f"Config: {config}")
+        logger.info("Creating llm_tutor instance")
+
+    chat_profile = cl.user_session.get("chat_profile")
+    if chat_profile is not None:
+        if chat_profile.lower() in ["gpt-3.5-turbo-1106", "gpt-4"]:
+            config["llm_params"]["llm_loader"] = "openai"
+            config["llm_params"]["openai_params"]["model"] = chat_profile.lower()
+        elif chat_profile.lower() == "llama":
+            config["llm_params"]["llm_loader"] = "local_llm"
+            config["llm_params"]["local_llm_params"]["model"] = LLAMA_PATH
+            config["llm_params"]["local_llm_params"]["model_type"] = "llama"
+        elif chat_profile.lower() == "mistral":
+            config["llm_params"]["llm_loader"] = "local_llm"
+            config["llm_params"]["local_llm_params"]["model"] = MISTRAL_PATH
+            config["llm_params"]["local_llm_params"]["model_type"] = "mistral"
+
+        else:
+            pass
+
+    llm_tutor = LLMTutor(config, logger=logger)
+
+    chain = llm_tutor.qa_bot()
+    model = config["llm_params"]["local_llm_params"]["model"]
+    msg = cl.Message(content=f"Starting the bot {model}...")
+    await msg.send()
+    msg.content = f"{opening_message}"
+    await msg.update()
+
+    cl.user_session.set("chain", chain)
+
+
+@cl.on_message
+async def main(message):
+    user = cl.user_session.get("user")
+    chain = cl.user_session.get("chain")
+    # cb = cl.AsyncLangchainCallbackHandler(
+    #     stream_final_answer=True, answer_prefix_tokens=["FINAL", "ANSWER"]
+    # )
+    # cb.answer_reached = True
+    # res=await chain.acall(message, callbacks=[cb])
+    res = await chain.acall(message.content)
+    print(f"response: {res}")
+    try:
+        answer = res["answer"]
+    except:
+        answer = res["result"]
+    print(f"answer: {answer}")
+
+    answer_with_sources, source_elements = get_sources(res, answer)
+
+    await cl.Message(content=answer_with_sources, elements=source_elements).send()

code/modules/chat_model_loader.py

@@ -0,0 +1,39 @@
+from langchain_community.chat_models import ChatOpenAI
+from langchain.llms import CTransformers
+from langchain.llms.huggingface_pipeline import HuggingFacePipeline
+from transformers import AutoTokenizer, TextStreamer
+from langchain.llms import LlamaCpp
+import torch
+import transformers
+import os
+from langchain.callbacks.manager import CallbackManager
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+
+
+class ChatModelLoader:
+    def __init__(self, config):
+        self.config = config
+        self.huggingface_token = os.getenv("HUGGINGFACEHUB_API_TOKEN")
+
+    def load_chat_model(self):
+        if self.config["llm_params"]["llm_loader"] == "openai":
+            llm = ChatOpenAI(
+                model_name=self.config["llm_params"]["openai_params"]["model"]
+            )
+        elif self.config["llm_params"]["llm_loader"] == "local_llm":
+            n_batch = 512  # Should be between 1 and n_ctx, consider the amount of VRAM in your GPU.
+            model_path = self.config["llm_params"]["local_llm_params"]["model"]
+            llm = LlamaCpp(
+                model_path=model_path,
+                n_batch=n_batch,
+                n_ctx=2048,
+                f16_kv=True,
+                verbose=True,
+                n_threads=2,
+                temperature=self.config["llm_params"]["local_llm_params"][
+                    "temperature"
+                ],
+            )
+        else:
+            raise ValueError("Invalid LLM Loader")
+        return llm

code/modules/constants.py

@@ -0,0 +1,81 @@
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+
+# API Keys - Loaded from the .env file
+
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+HUGGINGFACE_TOKEN = os.getenv("HUGGINGFACE_TOKEN")
+
+opening_message = "Hello! You can ask me questions about the MSDS program at Boston University."
+
+# Prompt Templates
+
+openai_prompt_template = """Use the following pieces of information to answer the user's question.
+If you don't know the answer, just say that you don't know.
+
+Context: {context}
+Question: {question}
+
+Only return the helpful answer below and nothing else.
+Helpful answer:
+"""
+
+openai_prompt_template_with_history = """Use the following pieces of information to answer the user's question.
+If you don't know the answer, just say that you don't know, don't try to make up an answer.
+Use the history to answer the question if you can.
+Chat History:
+{chat_history}
+Context: {context}
+Question: {question}
+
+Only return the helpful answer below and nothing else.
+Helpful answer:
+"""
+
+tinyllama_prompt_template = """
+<|im_start|>system
+Assistant is an intelligent chatbot designed to help students with questions regarding the course. Only answer questions using the context below and if you're not sure of an answer, you can say "I don't know". Always give a breif and concise answer to the question. Use the history to answer the question if you can.
+
+Context:
+{context}
+<|im_end|>
+<|im_start|>user
+Question: Who is the instructor for this course?
+<|im_end|>
+<|im_start|>assistant
+The instructor for this course is Prof. Thomas Gardos.
+<|im_end|>
+<|im_start|>user
+Question: {question}
+<|im_end|>
+<|im_start|>assistant
+"""
+
+tinyllama_prompt_template_with_history = """
+<|im_start|>system
+Assistant is an intelligent chatbot designed to help students with questions regarding the course. Only answer questions using the context below and if you're not sure of an answer, you can say "I don't know". Always give a breif and concise answer to the question.
+
+Chat History:
+{chat_history}
+Context:
+{context}
+<|im_end|>
+<|im_start|>user
+Question: Who is the instructor for this course?
+<|im_end|>
+<|im_start|>assistant
+The instructor for this course is Prof. Thomas Gardos.
+<|im_end|>
+<|im_start|>user
+Question: {question}
+<|im_end|>
+<|im_start|>assistant
+"""
+
+
+# Model Paths
+
+LLAMA_PATH = "storage/models/tinyllama-1.1b-chat-v0.3.Q5_K_M.gguf"
+MISTRAL_PATH = "storage/models/mistral-7b-v0.1.Q4_K_M.gguf"

code/modules/data_loader.py

@@ -0,0 +1,290 @@
+import os
+import re
+import requests
+import pysrt
+from langchain_community.document_loaders import (
+    PyMuPDFLoader,
+    Docx2txtLoader,
+    YoutubeLoader,
+    WebBaseLoader,
+    TextLoader,
+)
+from langchain_community.document_loaders import UnstructuredMarkdownLoader
+from llama_parse import LlamaParse
+from langchain.schema import Document
+import logging
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from ragatouille import RAGPretrainedModel
+from langchain.chains import LLMChain
+from langchain.llms import OpenAI
+from langchain import PromptTemplate
+
+try:
+    from modules.helpers import get_metadata
+except:
+    from helpers import get_metadata
+
+logger = logging.getLogger(__name__)
+
+
+class PDFReader:
+    def __init__(self):
+        pass
+
+    def get_loader(self, pdf_path):
+        loader = PyMuPDFLoader(pdf_path)
+        return loader
+
+    def get_documents(self, loader):
+        return loader.load()
+
+
+class FileReader:
+    def __init__(self):
+        self.pdf_reader = PDFReader()
+
+    def extract_text_from_pdf(self, pdf_path):
+        text = ""
+        with open(pdf_path, "rb") as file:
+            reader = PyPDF2.PdfReader(file)
+            num_pages = len(reader.pages)
+            for page_num in range(num_pages):
+                page = reader.pages[page_num]
+                text += page.extract_text()
+        return text
+
+    def download_pdf_from_url(self, pdf_url):
+        response = requests.get(pdf_url)
+        if response.status_code == 200:
+            with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as temp_file:
+                temp_file.write(response.content)
+                temp_file_path = temp_file.name
+            return temp_file_path
+        else:
+            print("Failed to download PDF from URL:", pdf_url)
+            return None
+
+    def read_pdf(self, temp_file_path: str):
+        loader = self.pdf_reader.get_loader(temp_file_path)
+        documents = self.pdf_reader.get_documents(loader)
+        return documents
+
+    def read_txt(self, temp_file_path: str):
+        loader = TextLoader(temp_file_path, autodetect_encoding=True)
+        return loader.load()
+
+    def read_docx(self, temp_file_path: str):
+        loader = Docx2txtLoader(temp_file_path)
+        return loader.load()
+
+    def read_srt(self, temp_file_path: str):
+        subs = pysrt.open(temp_file_path)
+        text = ""
+        for sub in subs:
+            text += sub.text
+        return [Document(page_content=text)]
+
+    def read_youtube_transcript(self, url: str):
+        loader = YoutubeLoader.from_youtube_url(
+            url, add_video_info=True, language=["en"], translation="en"
+        )
+        return loader.load()
+
+    def read_html(self, url: str):
+        loader = WebBaseLoader(url)
+        return loader.load()
+
+    def read_tex_from_url(self, tex_url):
+        response = requests.get(tex_url)
+        if response.status_code == 200:
+            return [Document(page_content=response.text)]
+        else:
+            print("Failed to fetch .tex file from URL:", tex_url)
+            return None
+
+
+class ChunkProcessor:
+    def __init__(self, config):
+        self.config = config
+
+        if config["splitter_options"]["use_splitter"]:
+            if config["splitter_options"]["split_by_token"]:
+                self.splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
+                    chunk_size=config["splitter_options"]["chunk_size"],
+                    chunk_overlap=config["splitter_options"]["chunk_overlap"],
+                    separators=config["splitter_options"]["chunk_separators"],
+                    disallowed_special=(),
+                )
+            else:
+                self.splitter = RecursiveCharacterTextSplitter(
+                    chunk_size=config["splitter_options"]["chunk_size"],
+                    chunk_overlap=config["splitter_options"]["chunk_overlap"],
+                    separators=config["splitter_options"]["chunk_separators"],
+                    disallowed_special=(),
+                )
+        else:
+            self.splitter = None
+        logger.info("ChunkProcessor instance created")
+
+    def remove_delimiters(self, document_chunks: list):
+        for chunk in document_chunks:
+            for delimiter in self.config["splitter_options"]["delimiters_to_remove"]:
+                chunk.page_content = re.sub(delimiter, " ", chunk.page_content)
+        return document_chunks
+
+    def remove_chunks(self, document_chunks: list):
+        front = self.config["splitter_options"]["front_chunk_to_remove"]
+        end = self.config["splitter_options"]["last_chunks_to_remove"]
+        for _ in range(front):
+            del document_chunks[0]
+        for _ in range(end):
+            document_chunks.pop()
+        logger.info(f"\tNumber of pages after skipping: {len(document_chunks)}")
+        return document_chunks
+
+    def process_chunks(
+        self, documents, file_type="txt", source="", page=0, metadata={}
+    ):
+        documents = [Document(page_content=documents, source=source, page=page)]
+        if (
+            file_type == "txt"
+            or file_type == "docx"
+            or file_type == "srt"
+            or file_type == "tex"
+        ):
+            document_chunks = self.splitter.split_documents(documents)
+        elif file_type == "pdf":
+            document_chunks = documents  # Full page for now
+
+        # add the source and page number back to the metadata
+        for chunk in document_chunks:
+            chunk.metadata["source"] = source
+            chunk.metadata["page"] = page
+
+            # add the metadata extracted from the document
+            for key, value in metadata.items():
+                chunk.metadata[key] = value
+
+        if self.config["splitter_options"]["remove_leftover_delimiters"]:
+            document_chunks = self.remove_delimiters(document_chunks)
+        if self.config["splitter_options"]["remove_chunks"]:
+            document_chunks = self.remove_chunks(document_chunks)
+
+        return document_chunks
+
+    def get_chunks(self, file_reader, uploaded_files, weblinks):
+        self.document_chunks_full = []
+        self.parent_document_names = []
+        self.child_document_names = []
+        self.documents = []
+        self.document_metadata = []
+
+        addl_metadata = get_metadata(uploaded_files)  # For any additional metadata
+
+        for file_index, file_path in enumerate(uploaded_files):
+            file_name = os.path.basename(file_path)
+            if file_name not in self.parent_document_names:
+                file_type = file_name.split(".")[-1].lower()
+
+                # try:
+                if file_type == "pdf":
+                    documents = file_reader.read_pdf(file_path)
+                elif file_type == "txt":
+                    documents = file_reader.read_txt(file_path)
+                elif file_type == "docx":
+                    documents = file_reader.read_docx(file_path)
+                elif file_type == "srt":
+                    documents = file_reader.read_srt(file_path)
+                elif file_type == "tex":
+                    documents = file_reader.read_tex_from_url(file_path)
+                else:
+                    logger.warning(f"Unsupported file type: {file_type}")
+                    continue
+
+                for doc in documents:
+                    page_num = doc.metadata.get("page", 0)
+                    self.documents.append(doc.page_content)
+                    self.document_metadata.append(
+                        {"source": file_path, "page": page_num}
+                    )
+                    metadata = addl_metadata.get(file_path, {})
+                    self.document_metadata[-1].update(metadata)
+
+                    self.child_document_names.append(f"{file_name}_{page_num}")
+
+                    self.parent_document_names.append(file_name)
+                    if self.config["embedding_options"]["db_option"] not in [
+                        "RAGatouille"
+                    ]:
+                        document_chunks = self.process_chunks(
+                            self.documents[-1],
+                            file_type,
+                            source=file_path,
+                            page=page_num,
+                            metadata=metadata,
+                        )
+                        self.document_chunks_full.extend(document_chunks)
+
+            # except Exception as e:
+            #     logger.error(f"Error processing file {file_name}: {str(e)}")
+
+        self.process_weblinks(file_reader, weblinks)
+
+        logger.info(
+            f"Total document chunks extracted: {len(self.document_chunks_full)}"
+        )
+        return (
+            self.document_chunks_full,
+            self.child_document_names,
+            self.documents,
+            self.document_metadata,
+        )
+
+    def process_weblinks(self, file_reader, weblinks):
+        if weblinks[0] != "":
+            logger.info(f"Splitting weblinks: total of {len(weblinks)}")
+
+            for link_index, link in enumerate(weblinks):
+                if link not in self.parent_document_names:
+                    try:
+                        logger.info(f"\tSplitting link {link_index+1} : {link}")
+                        if "youtube" in link:
+                            documents = file_reader.read_youtube_transcript(link)
+                        else:
+                            documents = file_reader.read_html(link)
+
+                        for doc in documents:
+                            page_num = doc.metadata.get("page", 0)
+                            self.documents.append(doc.page_content)
+                            self.document_metadata.append(
+                                {"source": link, "page": page_num}
+                            )
+                            self.child_document_names.append(f"{link}")
+
+                        self.parent_document_names.append(link)
+                        if self.config["embedding_options"]["db_option"] not in [
+                            "RAGatouille"
+                        ]:
+                            document_chunks = self.process_chunks(
+                                self.documents[-1],
+                                "txt",
+                                source=link,
+                                page=0,
+                                metadata={"source_type": "webpage"},
+                            )
+                            self.document_chunks_full.extend(document_chunks)
+                    except Exception as e:
+                        logger.error(
+                            f"Error splitting link {link_index+1} : {link}: {str(e)}"
+                        )
+
+
+class DataLoader:
+    def __init__(self, config):
+        self.file_reader = FileReader()
+        self.chunk_processor = ChunkProcessor(config)
+
+    def get_chunks(self, uploaded_files, weblinks):
+        return self.chunk_processor.get_chunks(
+            self.file_reader, uploaded_files, weblinks
+        )

code/modules/embedding_model_loader.py

@@ -0,0 +1,38 @@
+from langchain_community.embeddings import OpenAIEmbeddings
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.embeddings import LlamaCppEmbeddings
+
+try:
+    from modules.constants import *
+except:
+    from constants import *
+import os
+
+
+class EmbeddingModelLoader:
+    def __init__(self, config):
+        self.config = config
+
+    def load_embedding_model(self):
+        if self.config["embedding_options"]["model"] in ["text-embedding-ada-002"]:
+            embedding_model = OpenAIEmbeddings(
+                deployment="SL-document_embedder",
+                model=self.config["embedding_options"]["model"],
+                show_progress_bar=True,
+                openai_api_key=OPENAI_API_KEY,
+                disallowed_special=(),
+            )
+        else:
+            embedding_model = HuggingFaceEmbeddings(
+                model_name=self.config["embedding_options"]["model"],
+                model_kwargs={
+                    "device": "cpu",
+                    "token": f"{HUGGINGFACE_TOKEN}",
+                    "trust_remote_code": True,
+                },
+            )
+            # embedding_model = LlamaCppEmbeddings(
+            #     model_path=os.path.abspath("storage/llama-7b.ggmlv3.q4_0.bin")
+            # )
+
+        return embedding_model

code/modules/helpers.py

@@ -0,0 +1,244 @@
+import requests
+from bs4 import BeautifulSoup
+from tqdm import tqdm
+import chainlit as cl
+from langchain import PromptTemplate
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urlparse, urljoin, urldefrag
+import asyncio
+import aiohttp
+from aiohttp import ClientSession
+
+try:
+    from modules.constants import *
+except:
+    from constants import *
+
+"""
+Ref: https://python.plainenglish.io/scraping-the-subpages-on-a-website-ea2d4e3db113
+"""
+
+
+class WebpageCrawler:
+    def __init__(self):
+        self.dict_href_links = {}
+
+    async def fetch(self, session: ClientSession, url: str) -> str:
+        async with session.get(url) as response:
+            try:
+                return await response.text()
+            except UnicodeDecodeError:
+                return await response.text(encoding="latin1")
+
+    def url_exists(self, url: str) -> bool:
+        try:
+            response = requests.head(url)
+            return response.status_code == 200
+        except requests.ConnectionError:
+            return False
+
+    async def get_links(self, session: ClientSession, website_link: str, base_url: str):
+        html_data = await self.fetch(session, website_link)
+        soup = BeautifulSoup(html_data, "html.parser")
+        list_links = []
+        for link in soup.find_all("a", href=True):
+            href = link["href"].strip()
+            full_url = urljoin(base_url, href)
+            normalized_url = self.normalize_url(full_url)  # sections removed
+            if (
+                normalized_url not in self.dict_href_links
+                and self.is_child_url(normalized_url, base_url)
+                and self.url_exists(normalized_url)
+            ):
+                self.dict_href_links[normalized_url] = None
+                list_links.append(normalized_url)
+
+        return list_links
+
+    async def get_subpage_links(
+        self, session: ClientSession, urls: list, base_url: str
+    ):
+        tasks = [self.get_links(session, url, base_url) for url in urls]
+        results = await asyncio.gather(*tasks)
+        all_links = [link for sublist in results for link in sublist]
+        return all_links
+
+    async def get_all_pages(self, url: str, base_url: str):
+        async with aiohttp.ClientSession() as session:
+            dict_links = {url: "Not-checked"}
+            counter = None
+            while counter != 0:
+                unchecked_links = [
+                    link
+                    for link, status in dict_links.items()
+                    if status == "Not-checked"
+                ]
+                if not unchecked_links:
+                    break
+                new_links = await self.get_subpage_links(
+                    session, unchecked_links, base_url
+                )
+                for link in unchecked_links:
+                    dict_links[link] = "Checked"
+                    print(f"Checked: {link}")
+                dict_links.update(
+                    {
+                        link: "Not-checked"
+                        for link in new_links
+                        if link not in dict_links
+                    }
+                )
+                counter = len(
+                    [
+                        status
+                        for status in dict_links.values()
+                        if status == "Not-checked"
+                    ]
+                )
+
+            checked_urls = [
+                url for url, status in dict_links.items() if status == "Checked"
+            ]
+            return checked_urls
+
+    def is_webpage(self, url: str) -> bool:
+        try:
+            response = requests.head(url, allow_redirects=True)
+            content_type = response.headers.get("Content-Type", "").lower()
+            return "text/html" in content_type
+        except requests.RequestException:
+            return False
+
+    def clean_url_list(self, urls):
+        files, webpages = [], []
+
+        for url in urls:
+            if self.is_webpage(url):
+                webpages.append(url)
+            else:
+                files.append(url)
+
+        return files, webpages
+
+    def is_child_url(self, url, base_url):
+        return url.startswith(base_url)
+
+    def normalize_url(self, url: str):
+        # Strip the fragment identifier
+        defragged_url, _ = urldefrag(url)
+        return defragged_url
+
+
+def get_urls_from_file(file_path: str):
+    """
+    Function to get urls from a file
+    """
+    with open(file_path, "r") as f:
+        urls = f.readlines()
+    urls = [url.strip() for url in urls]
+    return urls
+
+
+def get_base_url(url):
+    parsed_url = urlparse(url)
+    base_url = f"{parsed_url.scheme}://{parsed_url.netloc}/"
+    return base_url
+
+
+def get_prompt(config):
+    if config["llm_params"]["use_history"]:
+        if config["llm_params"]["llm_loader"] == "local_llm":
+            custom_prompt_template = tinyllama_prompt_template_with_history
+        elif config["llm_params"]["llm_loader"] == "openai":
+            custom_prompt_template = openai_prompt_template_with_history
+        # else:
+        #     custom_prompt_template = tinyllama_prompt_template_with_history # default
+        prompt = PromptTemplate(
+            template=custom_prompt_template,
+            input_variables=["context", "chat_history", "question"],
+        )
+    else:
+        if config["llm_params"]["llm_loader"] == "local_llm":
+            custom_prompt_template = tinyllama_prompt_template
+        elif config["llm_params"]["llm_loader"] == "openai":
+            custom_prompt_template = openai_prompt_template
+        # else:
+        #     custom_prompt_template = tinyllama_prompt_template
+        prompt = PromptTemplate(
+            template=custom_prompt_template,
+            input_variables=["context", "question"],
+        )
+    return prompt
+
+
+def get_sources(res, answer):
+    source_elements = []
+    source_dict = {}  # Dictionary to store URL elements
+
+    for idx, source in enumerate(res["source_documents"]):
+        source_metadata = source.metadata
+        url = source_metadata["source"]
+        score = source_metadata.get("score", "N/A")
+        page = source_metadata.get("page", 1)
+        date = source_metadata.get("date", "N/A")
+
+        url_name = f"{url}_{page}"
+        if url_name not in source_dict:
+            source_dict[url_name] = {
+                "text": source.page_content,
+                "url": url,
+                "score": score,
+                "page": page,
+                "date": date,
+            }
+        else:
+            source_dict[url_name]["text"] += f"\n\n{source.page_content}"
+
+    # First, display the answer
+    full_answer = "**Answer:**\n"
+    full_answer += answer
+
+    # Then, display the sources
+    full_answer += "\n\n**Sources:**\n"
+    for idx, (url_name, source_data) in enumerate(source_dict.items()):
+        full_answer += f"\nSource {idx + 1} (Score: {source_data['score']}): {source_data['url']}\n"
+
+        name = f"Source {idx + 1} Text\n"
+        full_answer += name
+        source_elements.append(
+            cl.Text(name=name, content=source_data["text"], display="side")
+        )
+
+        # Add a PDF element if the source is a PDF file
+        if source_data["url"].lower().endswith(".pdf"):
+            name = f"Source {idx + 1} PDF\n"
+            full_answer += name
+            pdf_url = f"{source_data['url']}#page={source_data['page']+1}"
+            source_elements.append(cl.Pdf(name=name, url=pdf_url, display="side"))
+
+    full_answer += "\n**Metadata:**\n"
+    for idx, (url_name, source_data) in enumerate(source_dict.items()):
+        full_answer += f"Source {idx+1} Metadata\n"
+        source_elements.append(
+            cl.Text(
+                name=f"Source {idx+1} Metadata",
+                content=f"Page: {source_data['page']}\nDate: {source_data['date']}\n",
+                display="side",
+            )
+        )
+
+    return full_answer, source_elements
+
+
+def get_metadata(file_names):
+    """
+    Function to get any additional metadata from the files
+    Returns a dict with the file_name: {metadata: value}
+    """
+    metadata_dict = {}
+    for file in file_names:
+        metadata_dict[file] = {
+            "source_type": "N/A",
+        }
+    return metadata_dict

code/modules/llm_tutor.py

@@ -0,0 +1,92 @@
+from langchain import PromptTemplate
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain_community.chat_models import ChatOpenAI
+from langchain_community.embeddings import OpenAIEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.chains import RetrievalQA, ConversationalRetrievalChain
+from langchain.llms import CTransformers
+from langchain.memory import ConversationBufferWindowMemory
+from langchain.chains.conversational_retrieval.prompts import QA_PROMPT
+import os
+from modules.constants import *
+from modules.helpers import get_prompt
+from modules.chat_model_loader import ChatModelLoader
+from modules.vector_db import VectorDB, VectorDBScore
+
+
+class LLMTutor:
+    def __init__(self, config, logger=None):
+        self.config = config
+        self.vector_db = VectorDB(config, logger=logger)
+        if self.config["embedding_options"]["embedd_files"]:
+            self.vector_db.create_database()
+            self.vector_db.save_database()
+
+    def set_custom_prompt(self):
+        """
+        Prompt template for QA retrieval for each vectorstore
+        """
+        prompt = get_prompt(self.config)
+        # prompt = QA_PROMPT
+
+        return prompt
+
+    # Retrieval QA Chain
+    def retrieval_qa_chain(self, llm, prompt, db):
+        if self.config["embedding_options"]["db_option"] in ["FAISS", "Chroma"]:
+            retriever = VectorDBScore(
+                vectorstore=db,
+                # search_kwargs={
+                #     "k": self.config["embedding_options"]["search_top_k"],
+                #     "lambda_mult": self.config["embedding_options"]["lambda_mult"],
+                # },
+            )
+        elif self.config["embedding_options"]["db_option"] == "RAGatouille":
+            retriever = db.as_langchain_retriever(
+                k=self.config["embedding_options"]["search_top_k"]
+            )
+        if self.config["llm_params"]["use_history"]:
+            memory = ConversationBufferWindowMemory(
+                k=self.config["llm_params"]["memory_window"],
+                memory_key="chat_history",
+                return_messages=True,
+                output_key="answer",
+            )
+            qa_chain = ConversationalRetrievalChain.from_llm(
+                llm=llm,
+                chain_type="stuff",
+                retriever=retriever,
+                return_source_documents=True,
+                memory=memory,
+                combine_docs_chain_kwargs={"prompt": prompt},
+            )
+        else:
+            qa_chain = RetrievalQA.from_chain_type(
+                llm=llm,
+                chain_type="stuff",
+                retriever=retriever,
+                return_source_documents=True,
+                chain_type_kwargs={"prompt": prompt},
+            )
+        return qa_chain
+
+    # Loading the model
+    def load_llm(self):
+        chat_model_loader = ChatModelLoader(self.config)
+        llm = chat_model_loader.load_chat_model()
+        return llm
+
+    # QA Model Function
+    def qa_bot(self):
+        db = self.vector_db.load_database()
+        self.llm = self.load_llm()
+        qa_prompt = self.set_custom_prompt()
+        qa = self.retrieval_qa_chain(self.llm, qa_prompt, db)
+
+        return qa
+
+    # output function
+    def final_result(query):
+        qa_result = qa_bot()
+        response = qa_result({"query": query})
+        return response

code/modules/vector_db.py

@@ -0,0 +1,226 @@
+import logging
+import os
+import yaml
+from langchain_community.vectorstores import FAISS, Chroma
+from langchain.schema.vectorstore import VectorStoreRetriever
+from langchain.callbacks.manager import CallbackManagerForRetrieverRun
+from langchain.schema.document import Document
+from langchain_core.callbacks import AsyncCallbackManagerForRetrieverRun
+from ragatouille import RAGPretrainedModel
+
+try:
+    from modules.embedding_model_loader import EmbeddingModelLoader
+    from modules.data_loader import DataLoader
+    from modules.constants import *
+    from modules.helpers import *
+except:
+    from embedding_model_loader import EmbeddingModelLoader
+    from data_loader import DataLoader
+    from constants import *
+    from helpers import *
+
+from typing import List
+
+
+class VectorDBScore(VectorStoreRetriever):
+
+    # See https://github.com/langchain-ai/langchain/blob/61dd92f8215daef3d9cf1734b0d1f8c70c1571c3/libs/langchain/langchain/vectorstores/base.py#L500
+    def _get_relevant_documents(
+        self, query: str, *, run_manager: CallbackManagerForRetrieverRun
+    ) -> List[Document]:
+        docs_and_similarities = (
+            self.vectorstore.similarity_search_with_relevance_scores(
+                query, **self.search_kwargs
+            )
+        )
+        # Make the score part of the document metadata
+        for doc, similarity in docs_and_similarities:
+            doc.metadata["score"] = similarity
+
+        docs = [doc for doc, _ in docs_and_similarities]
+        return docs
+
+    async def _aget_relevant_documents(
+        self, query: str, *, run_manager: AsyncCallbackManagerForRetrieverRun
+    ) -> List[Document]:
+        docs_and_similarities = (
+            self.vectorstore.similarity_search_with_relevance_scores(
+                query, **self.search_kwargs
+            )
+        )
+        # Make the score part of the document metadata
+        for doc, similarity in docs_and_similarities:
+            doc.metadata["score"] = similarity
+
+        docs = [doc for doc, _ in docs_and_similarities]
+        return docs
+
+
+class VectorDB:
+    def __init__(self, config, logger=None):
+        self.config = config
+        self.db_option = config["embedding_options"]["db_option"]
+        self.document_names = None
+        self.webpage_crawler = WebpageCrawler()
+
+        # Set up logging to both console and a file
+        if logger is None:
+            self.logger = logging.getLogger(__name__)
+            self.logger.setLevel(logging.INFO)
+
+            # Console Handler
+            console_handler = logging.StreamHandler()
+            console_handler.setLevel(logging.INFO)
+            formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
+            console_handler.setFormatter(formatter)
+            self.logger.addHandler(console_handler)
+
+            # File Handler
+            log_file_path = "vector_db.log"  # Change this to your desired log file path
+            file_handler = logging.FileHandler(log_file_path, mode="w")
+            file_handler.setLevel(logging.INFO)
+            file_handler.setFormatter(formatter)
+            self.logger.addHandler(file_handler)
+        else:
+            self.logger = logger
+
+        self.logger.info("VectorDB instance instantiated")
+
+    def load_files(self):
+        files = os.listdir(self.config["embedding_options"]["data_path"])
+        files = [
+            os.path.join(self.config["embedding_options"]["data_path"], file)
+            for file in files
+        ]
+        urls = get_urls_from_file(self.config["embedding_options"]["url_file_path"])
+        if self.config["embedding_options"]["expand_urls"]:
+            all_urls = []
+            for url in urls:
+                loop = asyncio.get_event_loop()
+                all_urls.extend(
+                    loop.run_until_complete(
+                        self.webpage_crawler.get_all_pages(
+                            url, url
+                        )  # only get child urls, if you want to get all urls, replace the second argument with the base url
+                    )
+                )
+            urls = all_urls
+        return files, urls
+
+    def create_embedding_model(self):
+        self.logger.info("Creating embedding function")
+        self.embedding_model_loader = EmbeddingModelLoader(self.config)
+        self.embedding_model = self.embedding_model_loader.load_embedding_model()
+
+    def initialize_database(
+        self,
+        document_chunks: list,
+        document_names: list,
+        documents: list,
+        document_metadata: list,
+    ):
+        if self.db_option in ["FAISS", "Chroma"]:
+            self.create_embedding_model()
+        # Track token usage
+        self.logger.info("Initializing vector_db")
+        self.logger.info("\tUsing {} as db_option".format(self.db_option))
+        if self.db_option == "FAISS":
+            self.vector_db = FAISS.from_documents(
+                documents=document_chunks, embedding=self.embedding_model
+            )
+        elif self.db_option == "Chroma":
+            self.vector_db = Chroma.from_documents(
+                documents=document_chunks,
+                embedding=self.embedding_model,
+                persist_directory=os.path.join(
+                    self.config["embedding_options"]["db_path"],
+                    "db_"
+                    + self.config["embedding_options"]["db_option"]
+                    + "_"
+                    + self.config["embedding_options"]["model"],
+                ),
+            )
+        elif self.db_option == "RAGatouille":
+            self.RAG = RAGPretrainedModel.from_pretrained("colbert-ir/colbertv2.0")
+            index_path = self.RAG.index(
+                index_name="new_idx",
+                collection=documents,
+                document_ids=document_names,
+                document_metadatas=document_metadata,
+            )
+        self.logger.info("Completed initializing vector_db")
+
+    def create_database(self):
+        data_loader = DataLoader(self.config)
+        self.logger.info("Loading data")
+        files, urls = self.load_files()
+        files, webpages = self.webpage_crawler.clean_url_list(urls)
+        if "storage/data/urls.txt" in files:
+            files.remove("storage/data/urls.txt")
+        document_chunks, document_names, documents, document_metadata = (
+            data_loader.get_chunks(files, webpages)
+        )
+        self.logger.info("Completed loading data")
+        self.initialize_database(
+            document_chunks, document_names, documents, document_metadata
+        )
+
+    def save_database(self):
+        if self.db_option == "FAISS":
+            self.vector_db.save_local(
+                os.path.join(
+                    self.config["embedding_options"]["db_path"],
+                    "db_"
+                    + self.config["embedding_options"]["db_option"]
+                    + "_"
+                    + self.config["embedding_options"]["model"],
+                )
+            )
+        elif self.db_option == "Chroma":
+            # db is saved in the persist directory during initialization
+            pass
+        elif self.db_option == "RAGatouille":
+            # index is saved during initialization
+            pass
+        self.logger.info("Saved database")
+
+    def load_database(self):
+        self.create_embedding_model()
+        if self.db_option == "FAISS":
+            self.vector_db = FAISS.load_local(
+                os.path.join(
+                    self.config["embedding_options"]["db_path"],
+                    "db_"
+                    + self.config["embedding_options"]["db_option"]
+                    + "_"
+                    + self.config["embedding_options"]["model"],
+                ),
+                self.embedding_model,
+                allow_dangerous_deserialization=True,
+            )
+        elif self.db_option == "Chroma":
+            self.vector_db = Chroma(
+                persist_directory=os.path.join(
+                    self.config["embedding_options"]["db_path"],
+                    "db_"
+                    + self.config["embedding_options"]["db_option"]
+                    + "_"
+                    + self.config["embedding_options"]["model"],
+                ),
+                embedding_function=self.embedding_model,
+            )
+        elif self.db_option == "RAGatouille":
+            self.vector_db = RAGPretrainedModel.from_index(
+                ".ragatouille/colbert/indexes/new_idx"
+            )
+        self.logger.info("Loaded database")
+        return self.vector_db
+
+
+if __name__ == "__main__":
+    with open("code/config.yml", "r") as f:
+        config = yaml.safe_load(f)
+    print(config)
+    vector_db = VectorDB(config)
+    vector_db.create_database()
+    vector_db.save_database()

public/test.css

@@ -0,0 +1,16 @@
+a[href*='https://github.com/Chainlit/chainlit'] {
+    visibility: hidden;
+}
+
+.message-avatar .MuiAvatar-root {
+    background-color: transparent; /* Remove the background color */
+    color: #FFFFFF; /* Change this to your desired text color */
+    border: 0.25px solid #FFFFFF; /* Add a white border for the circle */
+    border-radius: 50%; /* Ensure the avatar remains circular */
+    background-image: url('http://localhost:8051/logo?theme=dark'); /* Path to your logo */
+    background-size: cover; /* Ensure the logo covers the entire avatar */
+    background-position: center; /* Center the logo */
+    background-repeat: no-repeat; /* Prevent the logo from repeating */
+    width: 38px; /* Adjust the width as needed */
+    height: 38px; /* Adjust the height as needed */
+}

requirements.txt

@@ -0,0 +1,19 @@
+# Automatically generated by https://github.com/damnever/pigar.
+
+beautifulsoup4==4.12.3
+chainlit==1.1.202
+langchain==0.1.20
+langchain-community==0.0.38
+langchain-core==0.1.52
+llama-parse==0.4.4
+pysrt==1.1.2
+python-dotenv==1.0.1
+PyYAML==6.0.1
+RAGatouille==0.0.8.post2
+requests==2.32.3
+torch==2.3.1
+tqdm==4.66.4
+transformers==4.41.2
+llama-cpp-python==0.2.77
+fake_useragent==1.5.1
+chromadb==0.5.0

storage/data/urls.txt

@@ -0,0 +1 @@
+https://www.bu.edu/cds-faculty/programs-admissions/ms-data-science/