Duplicate from rohan13/makerlab-bot

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+env/
+.idea

README.md

@@ -0,0 +1,13 @@
+---
+title: Makerlab Bot
+emoji: 📉
+colorFrom: blue
+colorTo: blue
+sdk: gradio
+sdk_version: 3.23.0
+app_file: app.py
+pinned: false
+duplicated_from: rohan13/makerlab-bot
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,42 @@
+from flask import Flask, render_template
+from flask_executor import Executor
+from flask_socketio import SocketIO, emit
+from flask_cors import cross_origin, CORS
+from main import run
+from gevent import monkey
+
+monkey.patch_all(ssl=False)
+app = Flask(__name__)
+app.config['SECRET_KEY'] = 'secret!'
+socketio = SocketIO(app, cors_allowed_origins="*", async_mode='gevent', logger=True)
+cors = CORS(app)
+
+executor = Executor(app)
+
+executor.init_app(app)
+app.config['EXECUTOR_MAX_WORKERS'] = 10
+
+@app.route('/')
+def index():
+    return render_template('index.html')
+
+
+@socketio.on('message')
+def handle_message(data):
+    question = data['question']
+    print("question: " + question)
+
+    if executor.futures:
+        emit('response', {'response': 'Server is busy, please try again later'})
+        return
+
+    try:
+        future = executor.submit(run, question)
+        response = future.result()
+        emit('response', {'response': response})
+    except Exception as e:
+        traceback.print_exc()
+        emit('response', {'response': 'Server is busy. Please try again later.'})
+
+if __name__ == '__main__':
+    socketio.run(app, host="0.0.0.0", port=7860)

main.py

@@ -0,0 +1,27 @@
+from utils import create_index, get_agent_chain, get_prompt_and_tools, get_search_index
+from utils import get_custom_agent, get_prompt_and_tools_for_custom_agent
+question_starters = ['who', 'why', 'what', 'how', 'where', 'when', 'which', 'whom', 'whose']
+
+def run(question):
+
+    index = get_search_index()
+
+    # prompt, tools = get_prompt_and_tools()
+
+    # agent_chain = get_agent_chain(prompt, tools)
+
+    prompt, tools = get_prompt_and_tools_for_custom_agent()
+
+    agent_chain = get_custom_agent(prompt, tools)
+
+    result = None
+
+    try:
+        result = agent_chain.run(question)
+        print(result)
+    except ValueError as ve:
+        if "Could not parse LLM output:" in ve.args[0] and question.lower().startswith(tuple(question_starters)) and not question.lower().endswith('?'):
+            question = question + '?'
+            result = agent_chain.run(question)
+
+    return result

open_ai.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5a87eb7dbbfd3245fc8025fb2467723bdcba8cdf308127050de9f8bbdeb21bc
+size 2838573

open_ai.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c399ec43dab5c11fabcc598d507b88db77a59b019c1b2533f2792199c6a1fcc
+size 3171039

requirements.txt

@@ -0,0 +1,11 @@
+faiss-cpu==1.7.3
+langchain==0.0.131
+beautifulsoup4==4.12.0
+PyPDF2==3.0.1
+openai==0.27.4
+flask==2.2.3
+flask-socketio==5.3.3
+flask-cors==3.0.10
+flask-executor==1.0.0
+gevent==22.10.2
+gevent-websocket==0.10.1

static/chatbot.js

@@ -0,0 +1,160 @@
+$(document).ready(function() {
+    // Initialize variables
+    var $chatContainer = $('.chat-container');
+    var $chatHeader = $('.chat-header');
+    var $chatBody = $('.chat-body');
+    var $chatInput = $('.chat-input');
+    var $input = $('.chat-input input');
+    var $submit = $('.chat_submit');
+    var session_id = '';
+    $chatBody.children().each(function() {
+        $(this).addClass('chat-message');
+    });
+    const buttonLabels = ["What is Makerlab?", "What is 3D printing?",
+        "Who are the founders of Makerlab?", "What are the 3D printing prices at Makerlab?",
+        "How can I host a birthday at Makerlab?", "Can I book an appointment at Makerlab?",
+        "Tell me about softwares used to create 3D printing designs", "Hi, I am bob. Tell me when Makerlab was founded.",
+        "Can I get my custom designs 3D printed at Makerlab?", "Can I host a private event at Makerlab?",
+        "Does Makerlab host any workshop?", "When is Makerlab open?", "How can I contact the Makerlab Team?"];
+
+
+    // Initialize SocketIO connection
+    var socket = io.connect('https://' + document.domain + ':' + location.port);
+    const container = document.getElementById("button-container");
+
+
+    for (let i = 0; i < buttonLabels.length; i++) {
+
+        const button = document.createElement("button");
+        button.innerHTML = buttonLabels[i];
+        button.setAttribute("class", "queries");
+        button.setAttribute("id", `button-${i}`);
+        button.style.margin = "5px";
+        container.appendChild(button);
+    }
+    scrollButtons();
+
+
+    // Function to send message to Flask-SocketIO app
+    function sendMessage(message) {
+        console.log("message: " + message )
+        socket.emit('message', {'question': message});
+    }
+
+    // Function to display message
+    function displayMessage(message, isUser, hasHtml) {
+        var $message = $('<div>').addClass('chat-message round');
+        if (hasHtml) {
+            $messageText = $('<p>').html(message);
+        } else {
+            $messageText = $('<p>').html(message.replace(/(https?:\/\/[^\s,]+)/g, '<a href="$1" target="_blank">$1</a>').replace(/(SOURCES:)/, '<br>$1'));
+        }
+        // var $messageText = $('<p>').html(message.replace(/(https?:\/\/[^\s,]+)/g, '<a href="$1">$1</a>'));
+
+        $message.append($messageText);
+        if (isUser) {
+            $message.addClass('user');
+        } else {
+            $message.addClass('bot')
+        }
+        if ($chatBody) {
+            $chatBody.append($message);
+            if ($chatBody[0]) {
+                $chatBody.animate({scrollTop: $chatBody[0].scrollHeight}, 300);
+            }
+        } else {
+            $('.chat-container').append($message);
+            $('.chat-container').animate({scrollTop: 0}, 300);
+        }
+    }
+
+
+    socket.on('response', function(data) {
+        console.log("Received response: " + data.response)
+        var response = data.response;
+        displayMessage(response, false);
+    });
+
+
+    // Send message on submit
+    $submit.click(function(event) {
+        event.preventDefault();
+        var message = $input.val().trim();
+        console.log("Submit clicked: " + message)
+        if (message !== '') {
+            displayMessage(message, true);
+            sendMessage(message);
+            $input.val('');
+        }
+    });
+
+    // Send message on enter key press
+    $input.keydown(function(event) {
+        if (event.keyCode === 13) {
+            event.preventDefault();
+            $submit.click();
+        }
+    });
+
+    // Initial message
+    displayMessage('Learn about <a href="https://makerlab.illinois.edu/" target="_blank">Makerlab</a>', false, true);
+    displayMessage('This bot is powered by Gpt 3.5 and thus may get things wrong. You can click on any of the questions on the left and they will get copied to this window. You can also type in a question here.  If you find the bot useful or have feedback on improving it, drop us a line at <a href="https://makerlab.illinois.edu/contact" target="_blank">Makerlab - Contact</a>', false, true);
+
+    // Function to minimize the widget
+    function minimizeWidget() {
+        $chatContainer.addClass('minimized');
+        $chatHeader.hide();
+        $chatBody.hide()
+        $chatInput.hide();
+        $chatContainer.append('<div class="chat-bot-icon"><i class="fa fa-android"></i></div>');
+    }
+
+    // Function to maximize the widget
+    function maximizeWidget() {
+        $chatContainer.removeClass('minimized');
+        $chatBody.show()
+        $chatHeader.show();
+        $chatInput.show();
+        $('.chat-bot-icon').remove();
+    }
+
+    // Minimize the widget on click of close button
+    $chatHeader.find('.chat-close').click(function() {
+        minimizeWidget();
+    });
+
+    // Maximize the widget on click of chat-bot-icon
+    $chatContainer.on('click', '.chat-bot-icon', function() {
+        maximizeWidget();
+    });
+
+    // Add event listener to each button
+    $('.queries').click(function() {
+        // Set the value of the input field to the text content of the clicked button
+        $('input[type="text"]').val($(this).text());
+    });
+
+    function scrollButtons() {
+        var container = document.getElementById("button-container");
+        var buttons = container.querySelectorAll(".queries");
+        var current = 0;
+
+        var scrollInterval = setInterval(function() {
+            buttons[current].scrollIntoView({ behavior: "smooth", block: "nearest", inline: "center" });
+            current = (current + 1) % buttons.length;
+        }, 1000);
+
+        container.addEventListener("mouseenter", function() {
+            clearInterval(scrollInterval);
+        });
+
+        container.addEventListener("mouseleave", function() {
+            scrollInterval = setInterval(function() {
+                buttons[current].scrollIntoView({ behavior: "smooth", block: "nearest", inline: "center" });
+                current = (current + 1) % buttons.length;
+            }, 1000);
+        });
+    }
+
+
+});

static/style.css

@@ -0,0 +1,296 @@
+.chat-container {
+    position: fixed;
+    bottom: 30px;
+    right: 30px;
+    z-index: 999;
+    background-color: #fff;
+    border-radius: 10px;
+    box-shadow: 0px 0px 20px rgba(0, 0, 0, 0.2);
+    max-width: 400px;
+    min-width: 400px;
+}
+
+.round {
+    border-radius: 10px;
+    -webkit-border-radius: 10px;
+    -moz-border-radius: 30px;
+
+}
+
+.chat-header {
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    padding: 10px;
+    background-color: darkblue;
+    color: #fff;
+    border-top-left-radius: 10px;
+    border-top-right-radius: 10px;
+}
+
+.chat-header h4 {
+    margin: 0;
+}
+
+.chat-close {
+    cursor: pointer;
+}
+
+.chat-body {
+    height: 400px;
+    overflow-y: scroll;
+    padding: 10px;
+    word-wrap: break-word;
+    display:flex;
+    flex-direction: column;
+}
+
+.chat-message {
+    margin: 10px;
+}
+
+.chat-message p {
+    margin: 0;
+    padding: 10px;
+    font-size: 16px;
+    line-height: 1.4;
+    position: relative;
+    word-wrap: break-word;
+    border-radius: 10px;
+    color: #fff;
+}
+
+.chat-message.user {
+    display: flex;
+    align-self: flex-end;
+    justify-content: flex-end;
+    text-align: right;
+    align-items: center;
+    background-color: rgba(0, 0, 139, 0.75);
+    border-top-right-radius: 0px;
+    border-bottom-right-radius: 0px;
+    border-bottom-left-radius: 10px;
+    word-wrap: break-word;
+}
+
+
+.chat-message.bot {
+    display: flex;
+    align-self: flex-start;
+    justify-content: flex-start;
+    text-align: left;
+    align-items: center;
+    background-color: rgba(0, 0, 139, 0.75);
+    border-top-left-radius: 0px;
+    border-bottom-right-radius: 10px;
+    border-bottom-left-radius: 0px;
+    word-wrap: break-word;
+}
+
+.chat-message.bot p {
+    margin: 0;
+    padding: 10px;
+    font-size: 16px;
+    line-height: 1.4;
+    position: relative;
+    word-wrap: break-word;
+    border-radius: 10px;
+    overflow-wrap: anywhere;
+}
+
+.chat-message.user:after {
+    content: "";
+    position: relative;
+    top: 0;
+    right: -15px;
+    width: 0;
+    height: 0;
+    border-top: 15px solid transparent;
+    border-bottom: 15px solid transparent;
+    border-left: 16px solid #00008BBF;
+    border-top-right-radius: 10px;
+}
+
+.chat-message.bot:before {
+    content: "";
+    position: relative;
+    top: 0;
+    left: -15px;
+    width: 0;
+    height: 0;
+    border-top: 15px solid transparent;
+    border-bottom: 15px solid transparent;
+    border-right: 15px solid #00008BBF;
+    border-top-left-radius: 10px;
+}
+
+
+.chat-input {
+    display: flex;
+    margin-top: 10px;
+}
+
+.chat-input input {
+    flex-grow: 1;
+    border: none;
+    border-radius: 5px;
+    padding: 8px 10px;
+    font-size: 16px;
+    margin-right: 10px;
+    box-shadow: 0px 0px 5px rgba(0, 0, 0, 0.1);
+}
+
+.chat-input button {
+    background-color: #00008BBF;
+    color: #fff;
+    border: none;
+    border-radius: 5px;
+    padding: 8px 10px;
+    font-size: 16px;
+    cursor: pointer;
+    box-shadow: 0px 0px 5px rgba(0, 0, 0, 0.1);
+}
+
+/* CSS for chat-container when minimized */
+.chat-container.minimized {
+    min-width: 70px;
+    height: 70px;
+    border-radius: 50%;
+    position: fixed;
+    bottom: 10px;
+    right: 10px;
+    z-index: 9999;
+    background-color: #fff;
+    box-shadow: 0px 2px 10px rgba(0, 0, 0, 0.3);
+    cursor: pointer;
+}
+
+/* CSS for chat-bot-icon */
+.chat-bot-icon {
+    font-size: 30px;
+    color: #00008BBF;
+    position: absolute;
+    top: 50%;
+    left: 50%;
+    transform: translate(-50%, -50%);
+}
+
+/* CSS for chat-header when not minimized */
+.chat-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    background-color: #6c7ae0;
+    color: #fff;
+    padding: 10px;
+    border-top-left-radius: 5px;
+    border-top-right-radius: 5px;
+}
+
+/* CSS for chat-container when not minimized */
+.chat-container:not(.minimized) {
+    border-radius: 5px;
+    position: fixed;
+    bottom: 10px;
+    right: 10px;
+    z-index: 9999;
+    background-color: #fff;
+    box-shadow: 0px 2px 10px rgba(0, 0, 0, 0.3);
+}
+
+/* CSS for chat-bot-icon when chat-container is minimized */
+.chat-container.minimized .chat-bot-icon {
+    display: block;
+}
+
+/* CSS for chat-bot-icon when chat-container is not minimized */
+.chat-container:not(.minimized) .chat-bot-icon {
+    display: none;
+}
+
+.queries {
+
+    padding: 8px 12px;
+    font-size: 16px;
+    font-weight: bold;
+    text-align: center;
+    text-decoration: none;
+    border: 0.5px solid #a5a0a0;
+    border-radius: 20px;
+    color: #000;
+    background-color: #343a404a;
+    cursor: pointer;
+    margin: 5px;
+}
+
+.queries:hover {
+    background-color: #343a40ad;
+}
+
+.queries:active {
+    background-color: #0053a4;
+}
+
+#button-container {
+    display: flex;
+    position: relative;
+    left: 2%;
+    top: 40%;
+    flex-direction: column;
+    justify-content: inherit;
+    align-items: center;
+    width: auto;
+    overflow-y: scroll;
+    max-height: 350px;
+    padding-top: 110%;
+    margin-top: 2%;
+
+}
+
+#button-container button {
+    margin-bottom: 10px;
+}
+
+.query-heading {
+    display: flex;
+    position: relative;
+    width: auto%;
+    background-color: #fff;
+    padding: 10px;
+    z-index: 1;
+    justify-content: inherit;
+    width: 100%;
+    border-bottom: 1px solid #2f4f4f5e;
+}
+
+.sample-query {
+    display: flex;
+    position: absolute;
+    left: 30%;
+    top: 10%;
+    flex-direction: column;
+    justify-content: flex-start;
+    align-items: center;
+    width: auto;
+    padding: 10px;
+    border: 1px solid #2f4f4f5e;
+    justify-content: center;
+    border-radius: 10px;
+    max-width: 30%;
+}
+
+::-webkit-scrollbar {
+    width: 8px;
+}
+
+::-webkit-scrollbar-track {
+    background-color: #f4f4f4;
+}
+
+::-webkit-scrollbar-thumb {
+    background-color: #a3bfe9a6;
+    border-radius: 20px;
+}
+
+
+

templates/index.html

@@ -0,0 +1,35 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8">
+    <title>MakerlabX3DPrinting QA</title>
+    <meta http-equiv="Content-Security-Policy" content="upgrade-insecure-requests">
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css">
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
+    <link rel="stylesheet" href="static/style.css">
+</head>
+<body>
+<div class = "sample-query">
+<div class="query-heading"><h4>Sample Queries</h4></div>
+<div id="button-container">
+</div></div>
+<div class="chat-container">
+    <div class="chat-header">
+        <h4>Makerlab Q&A Bot</h4>
+        <i class="fa fa-close chat-close"></i>
+    </div>
+    <div class="chat-bot-icon">
+        <i class="fa fa-android"></i> <!-- Replace with your bot icon -->
+    </div>
+    <div class="chat-body chat-messages round"></div>
+    <div class="chat-input">
+        <input type="text" placeholder="Type your message">
+        <button class="chat_submit">Send</button>
+    </div>
+</div>
+<!--<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.5.1/socket.io.js" integrity="sha512-sY2t8W1xNQ2yB+1RFXJv+wwhdN7CHX9Z+fhM7JH/3B3q1x7VJBOwKe+zb7VW0EC8XG5M5rjBQd7+47F5fQlhKQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>-->
+<script src="https://cdn.socket.io/4.4.1/socket.io.min.js"></script>
+<script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
+<script src="{{ url_for('static', filename='chatbot.js') }}"></script>
+</body>
+</html>

utils.py

@@ -0,0 +1,335 @@
+import os
+import pickle
+import re
+import time
+from typing import List, Union
+from urllib.parse import urlparse, urljoin
+
+import faiss
+import requests
+from PyPDF2 import PdfReader
+from bs4 import BeautifulSoup
+from langchain import OpenAI, LLMChain
+from langchain.agents import ConversationalAgent
+from langchain.agents import Tool, AgentExecutor, LLMSingleActionAgent, AgentOutputParser
+from langchain.prompts import BaseChatPromptTemplate
+from langchain.chains import ConversationalRetrievalChain
+from langchain.docstore.document import Document
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.memory import ConversationBufferWindowMemory
+from langchain.schema import AgentAction, AgentFinish, HumanMessage
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.vectorstores.faiss import FAISS
+
+book_url = 'https://g.co/kgs/2VFC7u'
+book_file = "Book.pdf"
+url = 'https://makerlab.illinois.edu/'
+
+pickle_file = "open_ai.pkl"
+index_file = "open_ai.index"
+
+gpt_3_5 = OpenAI(model_name='gpt-3.5-turbo',temperature=0)
+
+embeddings = OpenAIEmbeddings()
+
+chat_history = []
+
+memory = ConversationBufferWindowMemory(memory_key="chat_history")
+
+gpt_3_5_index = None
+
+class CustomOutputParser(AgentOutputParser):
+
+    def parse(self, llm_output: str) -> Union[AgentAction, AgentFinish]:
+        # Check if agent replied without using tools
+        if "AI:" in llm_output:
+            return AgentFinish(return_values={"output": llm_output.split("AI:")[-1].strip()},
+                               log=llm_output)
+        # Check if agent should finish
+        if "Final Answer:" in llm_output:
+            return AgentFinish(
+                # Return values is generally always a dictionary with a single `output` key
+                # It is not recommended to try anything else at the moment :)
+                return_values={"output": llm_output.split("Final Answer:")[-1].strip()},
+                log=llm_output,
+            )
+        # Parse out the action and action input
+        regex = r"Action: (.*?)[\n]*Action Input:[\s]*(.*)"
+        match = re.search(regex, llm_output, re.DOTALL)
+        if not match:
+            raise ValueError(f"Could not parse LLM output: `{llm_output}`")
+        action = match.group(1).strip()
+        action_input = match.group(2)
+        # Return the action and action input
+        return AgentAction(tool=action, tool_input=action_input.strip(" ").strip('"'), log=llm_output)
+
+# Set up a prompt template
+class CustomPromptTemplate(BaseChatPromptTemplate):
+    # The template to use
+    template: str
+    # The list of tools available
+    tools: List[Tool]
+
+    def format_messages(self, **kwargs) -> str:
+        # Get the intermediate steps (AgentAction, Observation tuples)
+        # Format them in a particular way
+        intermediate_steps = kwargs.pop("intermediate_steps")
+        thoughts = ""
+        for action, observation in intermediate_steps:
+            thoughts += action.log
+            thoughts += f"\nObservation: {observation}\nThought: "
+        # Set the agent_scratchpad variable to that value
+        kwargs["agent_scratchpad"] = thoughts
+        # Create a tools variable from the list of tools provided
+        kwargs["tools"] = "\n".join([f"{tool.name}: {tool.description}" for tool in self.tools])
+        # Create a list of tool names for the tools provided
+        kwargs["tool_names"] = ", ".join([tool.name for tool in self.tools])
+        formatted = self.template.format(**kwargs)
+        return [HumanMessage(content=formatted)]
+
+def get_search_index():
+    global gpt_3_5_index
+    if os.path.isfile(pickle_file) and os.path.isfile(index_file) and os.path.getsize(pickle_file) > 0:
+        # Load index from pickle file
+        with open(pickle_file, "rb") as f:
+            search_index = pickle.load(f)
+    else:
+        search_index = create_index()
+
+    gpt_3_5_index = search_index
+
+
+def create_index():
+    source_chunks = create_chunk_documents()
+    search_index = search_index_from_docs(source_chunks)
+    faiss.write_index(search_index.index, index_file)
+    # Save index to pickle file
+    with open(pickle_file, "wb") as f:
+        pickle.dump(search_index, f)
+    return search_index
+
+
+def create_chunk_documents():
+    sources = fetch_data_for_embeddings(url, book_file, book_url)
+    # print("sources" + str(len(sources)))
+
+    splitter = CharacterTextSplitter(separator=" ", chunk_size=800, chunk_overlap=0)
+
+    source_chunks = splitter.split_documents(sources)
+
+    for chunk in source_chunks:
+        print("Size of chunk: " + str(len(chunk.page_content) + len(chunk.metadata)))
+        if chunk.page_content is None or chunk.page_content == '':
+            print("removing chunk: "+ chunk.page_content)
+            source_chunks.remove(chunk)
+        elif len(chunk.page_content) >=1000:
+            print("splitting document")
+            source_chunks.extend(splitter.split_documents([chunk]))
+    # print("Chunks: " + str(len(source_chunks)) + "and type " + str(type(source_chunks)))
+    return source_chunks
+
+
+def fetch_data_for_embeddings(url, book_file, book_url):
+    sources = get_website_data(url)
+    sources.extend(get_document_data(book_file, book_url))
+    return sources
+
+def get_website_data(index_url):
+    # Get all page paths from index
+    paths = get_paths(index_url)
+
+    # Filter out invalid links and join them with the base URL
+    links = get_links(index_url, paths)
+
+    return get_content_from_links(links, index_url)
+
+
+def get_content_from_links(links, index_url):
+    content_list = []
+    for link in set(links):
+        if link.startswith(index_url):
+            page_data = requests.get(link).content
+            soup = BeautifulSoup(page_data, "html.parser")
+
+            # Get page content
+            content = soup.get_text(separator="\n")
+            # print(link)
+
+            # Get page metadata
+            metadata = {"source": link}
+
+            content_list.append(Document(page_content=content, metadata=metadata))
+    time.sleep(1)
+    # print("content list" + str(len(content_list)))
+    return content_list
+
+
+def get_paths(index_url):
+    index_data = requests.get(index_url).content
+    soup = BeautifulSoup(index_data, "html.parser")
+    paths = set([a.get('href') for a in soup.find_all('a', href=True)])
+    return paths
+
+
+def get_links(index_url, paths):
+    links = []
+    for path in paths:
+        url = urljoin(index_url, path)
+        parsed_url = urlparse(url)
+        if parsed_url.scheme in ["http", "https"] and "squarespace" not in parsed_url.netloc:
+            links.append(url)
+    return links
+
+
+def get_document_data(book_file, book_url):
+    document_list = []
+    with open(book_file, 'rb') as f:
+        pdf_reader = PdfReader(f)
+        for i in range(len(pdf_reader.pages)):
+            page_text = pdf_reader.pages[i].extract_text()
+            metadata = {"source": book_url}
+            document_list.append(Document(page_content=page_text, metadata=metadata))
+
+    # print("document list" + str(len(document_list)))
+    return document_list
+
+def search_index_from_docs(source_chunks):
+    # Create index from chunk documents
+    # print("Size of chunk" + str(len(source_chunks)))
+    search_index = FAISS.from_texts([doc.page_content for doc in source_chunks], embeddings, metadatas=[doc.metadata for doc in source_chunks])
+    return search_index
+
+
+def get_qa_chain(gpt_3_5_index):
+    global gpt_3_5
+    print("index: " + str(gpt_3_5_index))
+    return ConversationalRetrievalChain.from_llm(gpt_3_5, chain_type="stuff", get_chat_history=get_chat_history,
+                                                 retriever=gpt_3_5_index.as_retriever(), return_source_documents=True, verbose=True)
+
+def get_chat_history(inputs) -> str:
+    res = []
+    for human, ai in inputs:
+        res.append(f"Human:{human}\nAI:{ai}")
+    return "\n".join(res)
+
+
+def generate_answer(question) -> str:
+    global chat_history, gpt_3_5_index
+    gpt_3_5_chain = get_qa_chain(gpt_3_5_index)
+    result = gpt_3_5_chain(
+        {"question": question, "chat_history": chat_history,"vectordbkwargs": {"search_distance": 0.8}})
+    print("REsult: " + str(result))
+    chat_history = [(question, result["answer"])]
+    sources = []
+
+    for document in result['source_documents']:
+        source = document.metadata['source']
+        sources.append(source)
+
+    source = ',\n'.join(set(sources))
+    return result['answer'] + '\nSOURCES: ' + source
+
+
+def get_agent_chain(prompt, tools):
+    global gpt_3_5
+    # output_parser = CustomOutputParser()
+    llm_chain = LLMChain(llm=gpt_3_5, prompt=prompt)
+    agent = ConversationalAgent(llm_chain=llm_chain, tools=tools, verbose=True)
+    agent_chain = AgentExecutor.from_agent_and_tools(agent=agent, tools=tools, verbose=True, memory=memory,
+                                                     intermediate_steps=True)
+    return agent_chain
+
+
+def get_prompt_and_tools():
+    tools = get_tools()
+
+    prefix = """Have a conversation with a human, answering the following questions as best you can. 
+    Always try to use Vectorstore first. 
+    Your name is Makerlab Bot because you are a personal assistant of Makerlab. You have access to the following tools:"""
+    suffix = """Begin! If you use any tool, ALWAYS return a "SOURCES" part in your answer"
+    
+    {chat_history}
+    Question: {input}
+    {agent_scratchpad}
+    SOURCES:"""
+    prompt = ConversationalAgent.create_prompt(
+        tools,
+        prefix=prefix,
+        suffix=suffix,
+        input_variables=["input", "chat_history", "agent_scratchpad"]
+    )
+    # print("Template: " + prompt.template)
+    return prompt, tools
+
+
+def get_tools():
+    tools = [
+        Tool(
+            name="Vectorstore",
+            func=generate_answer,
+            description="useful for when you need to answer questions about the Makerlab or 3D Printing.",
+            return_direct=True
+        )]
+    return tools
+
+def get_custom_agent(prompt, tools):
+
+    llm_chain = LLMChain(llm=gpt_3_5, prompt=prompt)
+
+    output_parser = CustomOutputParser()
+    tool_names = [tool.name for tool in tools]
+    agent = LLMSingleActionAgent(
+        llm_chain=llm_chain,
+        output_parser=output_parser,
+        stop=["\nObservation:"],
+        allowed_tools=tool_names
+    )
+    agent_executor = AgentExecutor.from_agent_and_tools(agent=agent, tools=tools, verbose=True, memory=memory,
+                                                        intermediate_steps=True)
+    return agent_executor
+
+def get_prompt_and_tools_for_custom_agent():
+    template = """
+    Have a conversation with a human, answering the following questions as best you can. 
+    Always try to use Vectorstore first. 
+    Your name is Makerlab Bot because you are a personal assistant of Makerlab. You have access to the following tools:
+    
+    {tools}
+
+    To answer for the new input, use the following format:
+    
+    New Input: the input question you must answer
+    Thought: Do I need to use a tool? Yes
+    Action: the action to take, should be one of [{tool_names}]
+    Action Input: the input to the action
+    Observation: the result of the action
+    ... (this Thought/Action/Action Input/Observation can repeat N times)
+    Thought: I now know the final answer
+    Final Answer: the final answer to the original input question. SOURCES: the sources referred to find the final answer
+
+    
+    When you have a response to say to the Human and DO NOT need to use a tool:
+    1. DO NOT return "SOURCES" if you did not use any tool.
+    2. You MUST use this format:
+    ```
+    Thought: Do I need to use a tool? No
+    AI: [your response here]
+    ```
+
+    Begin! Remember to speak as a personal assistant when giving your final answer.
+    ALWAYS return a "SOURCES" part in your answer, if you used any tool. 
+    
+    Previous conversation history:
+    {chat_history}
+    New input: {input}
+    {agent_scratchpad}
+    SOURCES:"""
+    tools = get_tools()
+    prompt = CustomPromptTemplate(
+        template=template,
+        tools=tools,
+        # This omits the `agent_scratchpad`, `tools`, and `tool_names` variables because those are generated dynamically
+        # This includes the `intermediate_steps` variable because that is needed
+        input_variables=["input", "intermediate_steps", "chat_history"]
+    )
+    return prompt, tools