pr/1 (#1)

- removed streamlit (63c9094e90c4a6e1b2a4dfa7a25d90ed0028856e)
- chatbot widget with flask, socket, html, js, css (a2b71cf094e99b79543d5cfd30c6a45cdb608609)
- git ignore (07de00b304e66f34dfe7a47ad7e6ce8b711b588f)
- added index file (964b90ef783e63834ccd6bfc17087dd74d3177b2)

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.index filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+env/
+.idea

README.md

@@ -1,12 +1 @@
----
-title: Makerlab Bot
-emoji: 🌍
-colorFrom: purple
-colorTo: green
-sdk: streamlit
-sdk_version: 1.17.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# makerlab-bot

app.py

@@ -0,0 +1,24 @@
+from flask import Flask, render_template
+from flask_socketio import SocketIO, emit
+from main import run
+
+app = Flask(__name__)
+app.config['SECRET_KEY'] = 'secret!'
+socketio = SocketIO(app)
+
+
+@app.route('/')
+def index():
+    return render_template('index.html')
+
+
+@socketio.on('message')
+def handle_message(data):
+    question = data['question']
+    print("question: " + question)
+    response = run(question)
+    emit('response', {'response': response})
+
+
+if __name__ == '__main__':
+    socketio.run(app)

main.py

@@ -0,0 +1,22 @@
+from langchain.chains.qa_with_sources.loading import load_qa_with_sources_chain
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.llms import OpenAI
+
+from utils import generate_answer
+from utils import get_search_index
+
+open_ai_pkl = "open_ai.pkl"
+open_ai_index = "open_ai.index"
+
+gpt_3_5 = OpenAI(model_name='gpt-3.5-turbo',temperature=0)
+
+open_ai_embeddings = OpenAIEmbeddings()
+
+def run(question):
+
+    gpt_3_5_index = get_search_index(open_ai_pkl, open_ai_index, open_ai_embeddings)
+
+    gpt_3_5_chain = load_qa_with_sources_chain(gpt_3_5, chain_type="stuff", verbose=True)
+
+    answer = generate_answer(gpt_3_5_chain, gpt_3_5_index, question)
+    return answer

open_ai.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d25013b3bf7b7195d01ec1cc9ac1527638d8db68d94556b3dcc69b7dd8ff704
+size 3016749

open_ai.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c903018b7b3ad6f89b802f0e36fc92c88fb793c4f6e2499687b8823050a4df0
+size 3373815

requirements.txt

@@ -0,0 +1,7 @@
+faiss-cpu
+langchain
+beautifulsoup4
+PyPDF2
+openai
+flask
+flask-socketio

static/chatbot.js

@@ -0,0 +1,72 @@
+$(document).ready(function() {
+    // Initialize variables
+    var $messages = $('.chat-messages');
+    var $input = $('.chat-input input');
+    var $submit = $('.chat_submit');
+    var session_id = '';
+    $messages.children().each(function() {
+        $(this).addClass('chat-message');
+    });
+
+    // Initialize SocketIO connection
+    var socket = io.connect('http://' + document.domain + ':' + location.port);
+
+    // Function to send message to Flask-SocketIO app
+    function sendMessage(message) {
+        console.log("message: " + message )
+        socket.emit('message', {'question': message});
+    }
+
+    // Function to display message
+    function displayMessage(message, isUser) {
+        var $message = $('<div>').addClass('chat-message round');
+        var $messageText = $('<p>').html(message.replace(/(https?:\/\/[^\s]+)/g, '<a href="$1">$1</a>'));
+
+        $message.append($messageText);
+        if (isUser) {
+            $message.addClass('user');
+        } else {
+            $message.addClass('bot')
+        }
+        if ($messages) {
+            $messages.append($message);
+            if ($messages[0]) {
+                $messages.animate({scrollTop: $messages[0].scrollHeight}, 300);
+            }
+        } else {
+            $('.chat-container').append($message);
+            $('.chat-container').animate({scrollTop: 0}, 300);
+        }
+    }
+
+
+    socket.on('response', function(data) {
+        console.log("Received response: " + data.response)
+        var response = data.response;
+        displayMessage(response, false);
+    });
+
+
+    // Send message on submit
+    $submit.click(function(event) {
+        event.preventDefault();
+        var message = $input.val().trim();
+        console.log("Submit clicked: " + message)
+        if (message !== '') {
+            displayMessage(message, true);
+            sendMessage(message);
+            $input.val('');
+        }
+    });
+
+    // Send message on enter key press
+    $input.keydown(function(event) {
+        if (event.keyCode === 13) {
+            event.preventDefault();
+            $submit.click();
+        }
+    });
+
+    // Initial message
+    displayMessage('Ask me anything');
+});

static/style.css

@@ -0,0 +1,152 @@
+.chat-container {
+    position: fixed;
+    bottom: 30px;
+    right: 30px;
+    z-index: 999;
+    background-color: #fff;
+    border-radius: 10px;
+    box-shadow: 0px 0px 20px rgba(0, 0, 0, 0.2);
+    max-width: 400px;
+    min-width: 300px;
+}
+
+.round {
+    border-radius: 10px;
+    -webkit-border-radius: 10px;
+    -moz-border-radius: 30px;
+
+}
+
+.chat-header {
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    padding: 10px;
+    background-color: rgb(113, 239, 234);
+    color: #fff;
+    border-top-left-radius: 10px;
+    border-top-right-radius: 10px;
+}
+
+.chat-header h4 {
+    margin: 0;
+}
+
+.chat-close {
+    cursor: pointer;
+}
+
+.chat-body {
+    height: 300px;
+    overflow-y: scroll;
+    padding: 10px;
+    word-wrap: break-word;
+    display:flex;
+    flex-direction: column;
+}
+
+.chat-message {
+    margin: 10px;
+}
+
+.chat-message p {
+    margin: 0;
+    padding: 10px;
+    font-size: 16px;
+    line-height: 1.4;
+    position: relative;
+    word-wrap: break-word;
+    border-radius: 10px;
+}
+
+.chat-message.user {
+    display: flex;
+    align-self: flex-end;
+    justify-content: flex-end;
+    text-align: right;
+    align-items: center;
+    background-color: rgba(113, 239, 234, 0.75);
+    border-top-right-radius: 0px;
+    border-bottom-right-radius: 0px;
+    border-bottom-left-radius: 10px;
+    word-wrap: break-word;
+    color: #000;
+}
+
+
+.chat-message.bot {
+    display: flex;
+    align-self: flex-start;
+    justify-content: flex-start;
+    text-align: left;
+    align-items: center;
+    background-color: rgba(113, 239, 234, 0.75);
+    border-top-left-radius: 0px;
+    border-bottom-right-radius: 10px;
+    border-bottom-left-radius: 0px;
+    word-wrap: break-word;
+}
+
+.chat-message.bot p {
+    margin: 0;
+    padding: 10px;
+    font-size: 16px;
+    line-height: 1.4;
+    position: relative;
+    word-wrap: break-word;
+    border-radius: 10px;
+    overflow-wrap: anywhere;
+}
+
+.chat-message.user:after {
+    content: "";
+    position: relative;
+    top: 0;
+    right: -15px;
+    width: 0;
+    height: 0;
+    border-top: 15px solid transparent;
+    border-bottom: 15px solid transparent;
+    border-left: 15px solid #71EFEABF;
+    border-top-right-radius: 10px;
+}
+
+.chat-message.bot:before {
+    content: "";
+    position: relative;
+    top: 0;
+    left: -15px;
+    width: 0;
+    height: 0;
+    border-top: 15px solid transparent;
+    border-bottom: 15px solid transparent;
+    border-right: 15px solid #71EFEABF;
+    border-top-left-radius: 10px;
+}
+
+
+.chat-input {
+    display: flex;
+    margin-top: 10px;
+}
+
+.chat-input input {
+    flex-grow: 1;
+    border: none;
+    border-radius: 5px;
+    padding: 8px 10px;
+    font-size: 16px;
+    margin-right: 10px;
+    box-shadow: 0px 0px 5px rgba(0, 0, 0, 0.1);
+}
+
+.chat-input button {
+    background-color: #FFA500;
+    color: #fff;
+    border: none;
+    border-radius: 5px;
+    padding: 8px 10px;
+    font-size: 16px;
+    cursor: pointer;
+    box-shadow: 0px 0px 5px rgba(0, 0, 0, 0.1);
+}

templates/index.html

@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8">
+    <title>MakerlabX3DPrinting QA</title>
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css">
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
+    <link rel="stylesheet" href="{{ url_for('static', filename='style.css') }}">
+</head>
+<body>
+<div class="chat-container">
+    <div class="chat-header">
+        <h4>Makerlab Q&A Bot</h4>
+        <i class="fa fa-close chat-close"></i>
+    </div>
+    <div class="chat-body chat-messages round"></div>
+    <div class="chat-input">
+        <input type="text" placeholder="Type your message">
+        <button class="chat_submit">Send</button>
+    </div>
+</div>
+<!--<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.5.1/socket.io.js" integrity="sha512-sY2t8W1xNQ2yB+1RFXJv+wwhdN7CHX9Z+fhM7JH/3B3q1x7VJBOwKe+zb7VW0EC8XG5M5rjBQd7+47F5fQlhKQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>-->
+<script src="https://cdn.socket.io/4.4.1/socket.io.min.js"></script>
+<script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
+<script src="{{ url_for('static', filename='chatbot.js') }}"></script>
+</body>
+</html>

utils.py

@@ -0,0 +1,136 @@
+import os
+import pickle
+import time
+from urllib.parse import urlparse, urljoin
+
+import faiss
+import requests
+from PyPDF2 import PdfReader
+from bs4 import BeautifulSoup
+from langchain.docstore.document import Document
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.vectorstores.faiss import FAISS
+
+book_url = 'https://g.co/kgs/2VFC7u'
+book_file = "Book.pdf"
+url = 'https://makerlab.illinois.edu/'
+def get_search_index(pickle_file, index_file, embeddings):
+
+    if os.path.isfile(pickle_file) and os.path.isfile(index_file) and os.path.getsize(pickle_file) > 0:
+        # Load index from pickle file
+        with open(pickle_file, "rb") as f:
+            search_index = pickle.load(f)
+    else:
+        source_chunks = create_chunk_documents()
+
+        search_index = search_index_from_docs(source_chunks, embeddings=embeddings)
+
+        faiss.write_index(search_index.index, index_file)
+
+        # Save index to pickle file
+        with open(pickle_file, "wb") as f:
+            pickle.dump(search_index, f)
+
+    return search_index
+
+
+def create_chunk_documents():
+    sources = fetch_data_for_embeddings(url, book_file, book_url)
+    # print("sources" + str(len(sources)))
+
+    splitter = CharacterTextSplitter(separator=" ", chunk_size=800, chunk_overlap=0)
+
+    source_chunks = splitter.split_documents(sources)
+
+    for chunk in source_chunks:
+        print("Size of chunk: " + str(len(chunk.page_content) + len(chunk.metadata)))
+        if chunk.page_content is None or chunk.page_content == '':
+            print("removing chunk: "+ chunk.page_content)
+            source_chunks.remove(chunk)
+        elif len(chunk.page_content) >=1000:
+            print("splitting document")
+            source_chunks.extend(splitter.split_documents([chunk]))
+    # print("Chunks: " + str(len(source_chunks)) + "and type " + str(type(source_chunks)))
+    return source_chunks
+
+
+def fetch_data_for_embeddings(url, book_file, book_url):
+    sources = get_website_data(url)
+    sources.extend(get_document_data(book_file, book_url))
+    return sources
+
+def get_website_data(index_url):
+    # Get all page paths from index
+    paths = get_paths(index_url)
+
+    # Filter out invalid links and join them with the base URL
+    links = get_links(index_url, paths)
+
+    return get_content_from_links(links, index_url)
+
+
+def get_content_from_links(links, index_url):
+    content_list = []
+    for link in set(links):
+        if link.startswith(index_url):
+            page_data = requests.get(link).content
+            soup = BeautifulSoup(page_data, "html.parser")
+
+            # Get page content
+            content = soup.get_text(separator="\n")
+            # print(link)
+
+            # Get page metadata
+            metadata = {"source": link}
+
+            content_list.append(Document(page_content=content, metadata=metadata))
+    time.sleep(1)
+    # print("content list" + str(len(content_list)))
+    return content_list
+
+
+def get_paths(index_url):
+    index_data = requests.get(index_url).content
+    soup = BeautifulSoup(index_data, "html.parser")
+    paths = set([a.get('href') for a in soup.find_all('a', href=True)])
+    return paths
+
+
+def get_links(index_url, paths):
+    links = []
+    for path in paths:
+        url = urljoin(index_url, path)
+        parsed_url = urlparse(url)
+        if parsed_url.scheme in ["http", "https"] and "squarespace" not in parsed_url.netloc:
+            links.append(url)
+    return links
+
+
+def get_document_data(book_file, book_url):
+    document_list = []
+    with open(book_file, 'rb') as f:
+        pdf_reader = PdfReader(f)
+        for i in range(len(pdf_reader.pages)):
+            page_text = pdf_reader.pages[i].extract_text()
+            metadata = {"source": book_url}
+            document_list.append(Document(page_content=page_text, metadata=metadata))
+
+    # print("document list" + str(len(document_list)))
+    return document_list
+
+def search_index_from_docs(source_chunks, embeddings):
+    # Create index from chunk documents
+    # print("Size of chunk" + str(len(source_chunks)))
+    search_index = FAISS.from_texts([doc.page_content for doc in source_chunks], embeddings, metadatas=[doc.metadata for doc in source_chunks])
+    return search_index
+def generate_answer(chain, index, question):
+    #Get answer
+    answer = chain(
+        {
+            "input_documents": index.similarity_search(question, k=4),
+            "question": question,
+        },
+        return_only_outputs=True,
+    )["output_text"]
+
+    return answer