deleted old files

Core_Advanced_RAG_components.ipynb

@@ -1,392 +0,0 @@
-{
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "provenance": [],
-      "gpuType": "T4",
-      "authorship_tag": "ABX9TyNTRxOWLfv3tkZHe66pK63p",
-      "include_colab_link": true
-    },
-    "kernelspec": {
-      "name": "python3",
-      "display_name": "Python 3"
-    },
-    "language_info": {
-      "name": "python"
-    },
-    "accelerator": "GPU"
-  },
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "view-in-github",
-        "colab_type": "text"
-      },
-      "source": [
-        "<a href=\"https://colab.research.google.com/github/almutareb/advanced-rag-system-anatomy/blob/main/Core_Advanced_RAG_components.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "Install requirements"
-      ],
-      "metadata": {
-        "id": "Hz8JZq6Ob8rt"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "import sys\n",
-        "import os\n",
-        "!pip install -qU langchain langchain-community --no-warn-script-location > /dev/null\n",
-        "!pip install -qU beautifulsoup4 --no-warn-script-location > /dev/null\n",
-        "!pip install -qU faiss-cpu --no-warn-script-location > /dev/null\n",
-        "# use the gpu optimized version of FAISS for better performance\n",
-        "#!pip install -qU faiss-gpu --no-warn-script-location > /dev/null\n",
-        "!pip install -qU chromadb --no-warn-script-location > /dev/null\n",
-        "!pip install -qU validators --no-warn-script-location > /dev/null\n",
-        "!pip install -qU sentence_transformers typing-extensions==4.8.0 unstructured --no-warn-script-location > /dev/null\n",
-        "!pip install -qU gradio==3.48.0 --no-warn-script-location > /dev/null"
-      ],
-      "metadata": {
-        "id": "SXTdFuTvboyV"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "Download Documents"
-      ],
-      "metadata": {
-        "id": "pETUBgFAk4Fx"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "from langchain.document_loaders.recursive_url_loader import RecursiveUrlLoader\n",
-        "from bs4 import BeautifulSoup as Soup\n",
-        "\n",
-        "# List of URLs to scrape\n",
-        "urls = [\"https://langchain-doc.readthedocs.io/en/latest\"\n",
-        "        \"https://python.langchain.com/docs/get_started\"]\n",
-        "\n",
-        "# Initialize an empty list to store the documents\n",
-        "docs = []\n",
-        "# Looping through each URL in the list - this could take some time!\n",
-        "for url in urls:\n",
-        "  # max_depth set to 2 for demo purpose, should be increased for real scenario results, e.g. at least 5\n",
-        "  loader = RecursiveUrlLoader(url=url, max_depth=4, extractor=lambda x: Soup(x, \"html.parser\").text)\n",
-        "  docs.extend(loader.load())\n",
-        "print(f'Downloaded a total of {len(docs)} documents')"
-      ],
-      "metadata": {
-        "id": "eVav9lGgk3X3"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "Chunking documents"
-      ],
-      "metadata": {
-        "id": "0iurKj94w1jm"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
-        "import time\n",
-        "\n",
-        "text_splitter = RecursiveCharacterTextSplitter(\n",
-        "    chunk_size = 500,       # The size of each text chunk\n",
-        "    chunk_overlap  = 50,    # Overlap between chunks to ensure continuity\n",
-        ")\n",
-        "\n",
-        "# Stage one: read all the docs, split them into chunks.\n",
-        "st = time.time() # Start time for performance measurement\n",
-        "print('Loading documents ...')\n",
-        "\n",
-        "# Split each document into chunks using the configured text splitter\n",
-        "chunks = text_splitter.create_documents([doc.page_content for doc in docs], metadatas=[doc.metadata for doc in docs])\n",
-        "\n",
-        "et = time.time() - st # Calculate time taken for splitting\n",
-        "print(f'created {len(chunks)} chunks in {et} seconds.')"
-      ],
-      "metadata": {
-        "id": "zSZJQeA_w2B3"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "Build VectorStore: Vectorization"
-      ],
-      "metadata": {
-        "id": "oQGtHuTxkmFq"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "from langchain.vectorstores import FAISS\n",
-        "from langchain.vectorstores.utils import filter_complex_metadata\n",
-        "from langchain.embeddings import HuggingFaceEmbeddings\n",
-        "\n",
-        "# Path for saving the FAISS index\n",
-        "FAISS_INDEX_PATH = \"./vectorstore/lc-faiss-multi-mpnet-500\"\n",
-        "\n",
-        "\n",
-        "#Stage two: embed the docs.\n",
-        "# use multi-qa-mpnet-base-dot-v1 sentence transformer to convert pieces of text in vectors to store them in the vector store\n",
-        "model_name = \"sentence-transformers/multi-qa-mpnet-base-dot-v1\"\n",
-        "\n",
-        "# use the GPU for faster processing\n",
-        "#model_kwargs = {\"device\": \"cuda\"}\n",
-        "\n",
-        "# Initialize HuggingFace embeddings with the specified model\n",
-        "embeddings = HuggingFaceEmbeddings(\n",
-        "    model_name=model_name,\n",
-        "#    model_kwargs=model_kwargs  # uncomment when using a GPU, like T4 - requires extended RAM!\n",
-        "    )\n",
-        "\n",
-        "print(f'Loading chunks into vector store ...')\n",
-        "st = time.time() # Start time for performance measurement\n",
-        "\n",
-        "# Create a FAISS vector store from the document chunks and save it locally\n",
-        "db = FAISS.from_documents(filter_complex_metadata(chunks), embeddings)\n",
-        "# persist vectorstore\n",
-        "db.save_local(FAISS_INDEX_PATH)\n",
-        "\n",
-        "et = time.time() - st\n",
-        "print(f'Time taken: {et} seconds.')"
-      ],
-      "metadata": {
-        "id": "qu6sDsq6c9fg"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "Load LLM"
-      ],
-      "metadata": {
-        "id": "updDdzwj0RdJ"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "from dotenv import load_dotenv\n",
-        "# HF libraries\n",
-        "from langchain.llms import HuggingFaceHub\n",
-        "\n",
-        "# Load environment variables from a .env file\n",
-        "CONFIG = load_dotenv(\".env\")\n",
-        "\n",
-        "# Retrieve the Hugging Face API token from environment variables\n",
-        "HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN')\n",
-        "\n",
-        "# load HF Token\n",
-        "HUGGINGFACEHUB_API_TOKEN=os.getenv('HUGGINGFACEHUB_API_TOKEN')\n",
-        "\n",
-        "# Load the model from the Hugging Face Hub\n",
-        "model_id = HuggingFaceHub(repo_id=\"mistralai/Mistral-7B-Instruct-v0.1\", model_kwargs={\n",
-        "    \"temperature\":0.1,\n",
-        "    \"max_new_tokens\":1024,\n",
-        "    \"repetition_penalty\":1.2,\n",
-        "    \"return_full_text\":False\n",
-        "    })\n"
-      ],
-      "metadata": {
-        "id": "GlnNrNdbg2E6"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "Retriever"
-      ],
-      "metadata": {
-        "id": "2m3BIm090jtr"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "from langchain.embeddings import HuggingFaceHubEmbeddings\n",
-        "# vectorestore\n",
-        "from langchain.vectorstores import FAISS\n",
-        "\n",
-        "# Load and Initialize the vector store as a retriever for the RAG pipeline\n",
-        "db = FAISS.load_local(FAISS_INDEX_PATH, embeddings)\n",
-        "\n",
-        "retriever = db.as_retriever()"
-      ],
-      "metadata": {
-        "id": "jzqPsuds0kSs"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "Template and Chat logic"
-      ],
-      "metadata": {
-        "id": "Bld8lOEv0Uq-"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "# retrieval chain\n",
-        "from langchain.chains import RetrievalQA\n",
-        "# prompt template\n",
-        "from langchain.prompts import PromptTemplate\n",
-        "from langchain.memory import ConversationBufferMemory\n",
-        "\n",
-        "\n",
-        "global qa\n",
-        "template = \"\"\"\n",
-        "You are the friendly documentation buddy Arti, who helps novice programmers in using LangChain with simple explanations and examples.\\\n",
-        "    Use the following context (delimited by <ctx></ctx>) and the chat history (delimited by <hs></hs>) to answer the question :\n",
-        "------\n",
-        "<ctx>\n",
-        "{context}\n",
-        "</ctx>\n",
-        "------\n",
-        "<hs>\n",
-        "{history}\n",
-        "</hs>\n",
-        "------\n",
-        "{question}\n",
-        "Answer:\n",
-        "\"\"\"\n",
-        "# Create a PromptTemplate object with specified input variables and the defined template\n",
-        "prompt = PromptTemplate.from_template(\n",
-        "    template=template,\n",
-        ")\n",
-        "prompt.format(context=\"context\", history=\"history\", question=\"question\")\n",
-        "\n",
-        "# Create a memory buffer to manage conversation history\n",
-        "memory = ConversationBufferMemory(memory_key=\"history\", input_key=\"question\")\n",
-        "\n",
-        "# Initialize the RetrievalQA object with the specified model,\n",
-        "# retriever, and additional configurations\n",
-        "qa = RetrievalQA.from_chain_type(llm=model_id, chain_type=\"stuff\", retriever=retriever, verbose=True, return_source_documents=True, chain_type_kwargs={\n",
-        "    \"verbose\": True,\n",
-        "    \"memory\": memory,\n",
-        "    \"prompt\": prompt\n",
-        "}\n",
-        "    )"
-      ],
-      "metadata": {
-        "id": "K255Ldxq0Xg6"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "UI - Gradio"
-      ],
-      "metadata": {
-        "id": "pA5d0LL2kObx"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "history=[]\n",
-        "query=\"draft a function to calculate a mxn matrix\"\n",
-        "question=query\n",
-        "response=qa({\"query\": query, \"history\": history, \"question\": question})\n",
-        "print(*response)"
-      ],
-      "metadata": {
-        "id": "bKeoyhXPrQ2C"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "print(response['result'])"
-      ],
-      "metadata": {
-        "id": "78wRMjjn0cl3"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "import gradio as gr\n",
-        "\n",
-        "# Function to add a new input to the chat history\n",
-        "def add_text(history, text):\n",
-        "  # Append the new text to the history with a placeholder for the response\n",
-        "    history = history + [(text, None)]\n",
-        "    return history, \"\"\n",
-        "\n",
-        "# Function representing the bot's response mechanism\n",
-        "def bot(history):\n",
-        "    response = infer(history[-1][0], history)\n",
-        "    history[-1][1] = response['result']\n",
-        "    return history\n",
-        "\n",
-        "# Function to infer the response using the RAG model\n",
-        "def infer(question, history):\n",
-        "    query =  question\n",
-        "    result = qa({\"query\": query, \"history\": history, \"question\": question})\n",
-        "    return result\n",
-        "\n",
-        "# Building the Gradio interface\n",
-        "with gr.Blocks() as demo:\n",
-        "    with gr.Column(elem_id=\"col-container\"):\n",
-        "        chatbot = gr.Chatbot([], elem_id=\"chatbot\")\n",
-        "        clear = gr.Button(\"Clear\")\n",
-        "\n",
-        "        # Create a row for the question input\n",
-        "        with gr.Row():\n",
-        "            question = gr.Textbox(label=\"Question\", placeholder=\"Type your question and hit Enter \")\n",
-        "\n",
-        "    # Define the action when the question is submitted\n",
-        "    question.submit(add_text, [chatbot, question], [chatbot, question], queue=False).then(\n",
-        "        bot, chatbot, chatbot\n",
-        "    )\n",
-        "\n",
-        "    # Define the action for the clear button\n",
-        "    clear.click(lambda: None, None, chatbot, queue=False)\n",
-        "\n",
-        "# Launch the Gradio demo interface\n",
-        "demo.launch(share=False)"
-      ],
-      "metadata": {
-        "id": "OHVkFa6MkCir"
-      },
-      "execution_count": null,
-      "outputs": []
-    }
-  ]
-}

docs/advanced_rag_architecture.drawio

@@ -1,115 +0,0 @@
-<mxfile host="app.diagrams.net" modified="2024-02-02T10:57:09.662Z" agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0" etag="jBjrxQrE8FMZUdqYmkDs" version="22.1.21" type="github">
-  <diagram id="C5RBs43oDa-KdzZeNtuy" name="Page-1">
-    <mxGraphModel dx="1434" dy="774" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
-      <root>
-        <mxCell id="WIyWlLk6GJQsqaUBKTNV-0" />
-        <mxCell id="WIyWlLk6GJQsqaUBKTNV-1" parent="WIyWlLk6GJQsqaUBKTNV-0" />
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-59" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="7HGE-dyt3ShhVV6eNgTS-4" target="7HGE-dyt3ShhVV6eNgTS-22">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-4" value="Retrieval System" style="swimlane;whiteSpace=wrap;html=1;fillColor=#e1d5e7;strokeColor=#9673a6;startSize=23;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
-          <mxGeometry x="280" y="360" width="290" height="280" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-7" value="&lt;div&gt;Indicies&lt;/div&gt;" style="strokeWidth=2;html=1;shape=mxgraph.flowchart.database;whiteSpace=wrap;fillColor=#ffe6cc;strokeColor=#d79b00;" vertex="1" parent="7HGE-dyt3ShhVV6eNgTS-4">
-          <mxGeometry x="44" y="40" width="60" height="60" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-32" value="Re-ranking" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="7HGE-dyt3ShhVV6eNgTS-4">
-          <mxGeometry x="14" y="130" width="120" height="40" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-34" value="Hypothetical Questions and HyDE" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="7HGE-dyt3ShhVV6eNgTS-4">
-          <mxGeometry x="160" y="130" width="120" height="40" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-37" value="Fine-tuning Embeddings" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="7HGE-dyt3ShhVV6eNgTS-4">
-          <mxGeometry x="14" y="195" width="120" height="40" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-33" value="Hyperparamter Tuning" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="7HGE-dyt3ShhVV6eNgTS-4">
-          <mxGeometry x="160" y="50" width="120" height="40" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-56" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=1;exitDx=0;exitDy=0;entryX=0;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="7HGE-dyt3ShhVV6eNgTS-14" target="7HGE-dyt3ShhVV6eNgTS-19">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-14" value="&lt;div&gt;Data Preparation and Management&lt;/div&gt;" style="swimlane;whiteSpace=wrap;html=1;fillColor=#ffe6cc;strokeColor=#d79b00;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
-          <mxGeometry x="40" y="40" width="360" height="240" as="geometry">
-            <mxRectangle x="410" y="40" width="240" height="30" as="alternateBounds" />
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-23" value="Chunking &amp;amp; Vectorization" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="7HGE-dyt3ShhVV6eNgTS-14">
-          <mxGeometry x="14" y="50" width="120" height="40" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-24" value="Metadata and Summaries" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="7HGE-dyt3ShhVV6eNgTS-14">
-          <mxGeometry x="161" y="50" width="120" height="40" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-25" value="User Profile Management" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;" vertex="1" parent="7HGE-dyt3ShhVV6eNgTS-14">
-          <mxGeometry x="14" y="190" width="120" height="40" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-26" value="Data Cleaning" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="7HGE-dyt3ShhVV6eNgTS-14">
-          <mxGeometry x="14" y="120" width="120" height="40" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-31" value="Complex Formats Handling" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="7HGE-dyt3ShhVV6eNgTS-14">
-          <mxGeometry x="161" y="120" width="120" height="40" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-57" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="7HGE-dyt3ShhVV6eNgTS-19" target="7HGE-dyt3ShhVV6eNgTS-4">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-19" value="&lt;div&gt;User Input processing&lt;/div&gt;" style="swimlane;horizontal=0;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
-          <mxGeometry x="40" y="360" width="200" height="280" as="geometry" />
-        </mxCell>
-        <mxCell id="WIyWlLk6GJQsqaUBKTNV-3" value="User Authentication" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;" parent="7HGE-dyt3ShhVV6eNgTS-19" vertex="1">
-          <mxGeometry x="40" y="20" width="120" height="40" as="geometry" />
-        </mxCell>
-        <mxCell id="WIyWlLk6GJQsqaUBKTNV-7" value="Query Rewriter" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#e1d5e7;strokeColor=#9673a6;" parent="7HGE-dyt3ShhVV6eNgTS-19" vertex="1">
-          <mxGeometry x="40" y="80" width="120" height="40" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-8" value="Input Guardrail" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#f8cecc;strokeColor=#b85450;" vertex="1" parent="7HGE-dyt3ShhVV6eNgTS-19">
-          <mxGeometry x="40" y="140" width="120" height="40" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-5" value="chat history&amp;nbsp; " style="strokeWidth=2;html=1;shape=mxgraph.flowchart.multi-document;whiteSpace=wrap;fillColor=#ffe6cc;strokeColor=#d79b00;" vertex="1" parent="7HGE-dyt3ShhVV6eNgTS-19">
-          <mxGeometry x="56" y="210" width="88" height="60" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-60" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0;exitDx=0;exitDy=0;entryX=1;entryY=1;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="7HGE-dyt3ShhVV6eNgTS-22" target="7HGE-dyt3ShhVV6eNgTS-44">
-          <mxGeometry relative="1" as="geometry">
-            <mxPoint x="790" y="280" as="targetPoint" />
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-22" value="Information Processing and Generation" style="swimlane;horizontal=0;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
-          <mxGeometry x="600" y="360" width="200" height="280" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-9" value="Response Generation" style="rounded=1;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=2;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="7HGE-dyt3ShhVV6eNgTS-22">
-          <mxGeometry x="40" y="20" width="120" height="40" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-28" value="Output Guardrails and Moderation" style="rounded=1;whiteSpace=wrap;html=1;absoluteArcSize=1;arcSize=14;strokeWidth=2;fillColor=#f8cecc;strokeColor=#b85450;" vertex="1" parent="7HGE-dyt3ShhVV6eNgTS-22">
-          <mxGeometry x="40" y="80" width="120" height="40" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-29" value="Caching" style="strokeWidth=2;html=1;shape=mxgraph.flowchart.multi-document;whiteSpace=wrap;fillColor=#ffe6cc;strokeColor=#d79b00;" vertex="1" parent="7HGE-dyt3ShhVV6eNgTS-22">
-          <mxGeometry x="56" y="140" width="88" height="60" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-30" value="Personalization and Customization" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;" vertex="1" parent="7HGE-dyt3ShhVV6eNgTS-22">
-          <mxGeometry x="40" y="215" width="120" height="40" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-61" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=0;exitDx=0;exitDy=0;entryX=1;entryY=0;entryDx=0;entryDy=0;dashed=1;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="7HGE-dyt3ShhVV6eNgTS-44" target="7HGE-dyt3ShhVV6eNgTS-14">
-          <mxGeometry relative="1" as="geometry">
-            <Array as="points">
-              <mxPoint x="440" y="40" />
-              <mxPoint x="440" y="40" />
-            </Array>
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-44" value="Feedback and Continuous Improvement" style="swimlane;whiteSpace=wrap;html=1;startSize=23;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
-          <mxGeometry x="480" y="40" width="320" height="240" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-46" value="Data Refinement" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="7HGE-dyt3ShhVV6eNgTS-44">
-          <mxGeometry x="170" y="50" width="120" height="40" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-47" value="System Monitoring" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="7HGE-dyt3ShhVV6eNgTS-44">
-          <mxGeometry x="170" y="120" width="120" height="40" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-48" value="Generation Evaluation" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="7HGE-dyt3ShhVV6eNgTS-44">
-          <mxGeometry x="20" y="120" width="120" height="40" as="geometry" />
-        </mxCell>
-        <mxCell id="7HGE-dyt3ShhVV6eNgTS-49" value="User Feedback" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;" vertex="1" parent="7HGE-dyt3ShhVV6eNgTS-44">
-          <mxGeometry x="20" y="50" width="120" height="40" as="geometry" />
-        </mxCell>
-      </root>
-    </mxGraphModel>
-  </diagram>
-</mxfile>

mail_automation_draft.drawio

@@ -1,140 +0,0 @@
-<mxfile host="app.diagrams.net" modified="2024-05-21T13:40:30.482Z" agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0" etag="dPw-sfjJEinoEt61k41I" version="24.4.4" type="github">
-  <diagram name="Page-1" id="JIRIjCkc2eHKuGLTyx-l">
-    <mxGraphModel dx="1434" dy="-1564" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
-      <root>
-        <mxCell id="0" />
-        <mxCell id="1" parent="0" />
-        <mxCell id="0ipz06AKfEBxF18oT8nj-1" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" parent="1" source="0ipz06AKfEBxF18oT8nj-19" target="0ipz06AKfEBxF18oT8nj-6" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-2" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="0ipz06AKfEBxF18oT8nj-3" target="0ipz06AKfEBxF18oT8nj-31" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-3" value="Ticket Creation" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="217.5" y="2640" width="120" height="80" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-4" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="0ipz06AKfEBxF18oT8nj-6" target="0ipz06AKfEBxF18oT8nj-8" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-5" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="0ipz06AKfEBxF18oT8nj-6" target="0ipz06AKfEBxF18oT8nj-21" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-6" value="Fetch Emails" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="210.5" y="2480" width="120" height="80" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-7" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="0ipz06AKfEBxF18oT8nj-8" target="0ipz06AKfEBxF18oT8nj-10" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-8" value="Entity Recognition" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="410.5" y="2480" width="120" height="80" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-9" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="0ipz06AKfEBxF18oT8nj-10" target="0ipz06AKfEBxF18oT8nj-14" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-10" value="Classification &amp;amp; Summarization" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="630" y="2480" width="120" height="80" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-11" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" parent="1" source="0ipz06AKfEBxF18oT8nj-17" target="0ipz06AKfEBxF18oT8nj-15" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-12" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="0ipz06AKfEBxF18oT8nj-14" target="0ipz06AKfEBxF18oT8nj-17" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-13" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="0ipz06AKfEBxF18oT8nj-14" target="0ipz06AKfEBxF18oT8nj-24" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-14" value="Data Enrichment" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="630" y="2640" width="120" height="80" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-15" value="Draft Response" style="ellipse;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" parent="1" vertex="1">
-          <mxGeometry x="410.5" y="2800" width="120" height="80" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-16" value="" style="curved=1;endArrow=classic;html=1;rounded=0;exitX=0.75;exitY=1;exitDx=0;exitDy=0;entryX=0.25;entryY=1;entryDx=0;entryDy=0;" parent="1" source="0ipz06AKfEBxF18oT8nj-19" target="0ipz06AKfEBxF18oT8nj-19" edge="1">
-          <mxGeometry width="50" height="50" relative="1" as="geometry">
-            <mxPoint x="200" y="2730" as="sourcePoint" />
-            <mxPoint x="80" y="2580" as="targetPoint" />
-            <Array as="points">
-              <mxPoint x="130" y="2600" />
-              <mxPoint x="90" y="2610" />
-              <mxPoint x="60" y="2620" />
-              <mxPoint x="53" y="2570" />
-            </Array>
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-17" value="Data Complete" style="rhombus;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="430.5" y="2640" width="80" height="80" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-18" value="No" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;fillColor=none;" parent="1" vertex="1">
-          <mxGeometry x="460.5" y="2748" width="40" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-19" value="Periodically Check Emails" style="shape=process;whiteSpace=wrap;html=1;backgroundOutline=1;" parent="1" vertex="1">
-          <mxGeometry x="30" y="2490" width="120" height="60" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-20" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="0ipz06AKfEBxF18oT8nj-21" target="0ipz06AKfEBxF18oT8nj-22" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-21" value="Download Attachements" style="whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="210.5" y="2360" width="120" height="60" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-22" value="Extract Text and Images from PDF" style="whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="399.5" y="2360" width="120" height="60" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-23" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.596;exitY=1.033;exitDx=0;exitDy=0;exitPerimeter=0;" parent="1" source="0ipz06AKfEBxF18oT8nj-22" target="0ipz06AKfEBxF18oT8nj-8" edge="1">
-          <mxGeometry relative="1" as="geometry">
-            <mxPoint x="281" y="2490" as="sourcePoint" />
-            <mxPoint x="281" y="2430" as="targetPoint" />
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-24" value="CRM, SQL, Product KB" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="650" y="2800" width="80" height="80" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-25" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;entryX=1;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="0ipz06AKfEBxF18oT8nj-17" target="0ipz06AKfEBxF18oT8nj-3" edge="1">
-          <mxGeometry relative="1" as="geometry">
-            <mxPoint x="620.5" y="2690" as="sourcePoint" />
-            <mxPoint x="520.5" y="2690" as="targetPoint" />
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-26" value="Yes" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" parent="0ipz06AKfEBxF18oT8nj-25" vertex="1" connectable="0">
-          <mxGeometry x="-0.1183" y="-5" relative="1" as="geometry">
-            <mxPoint as="offset" />
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-27" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=1;entryY=0;entryDx=0;entryDy=0;" parent="1" source="0ipz06AKfEBxF18oT8nj-15" target="0ipz06AKfEBxF18oT8nj-8" edge="1">
-          <mxGeometry width="50" height="50" relative="1" as="geometry">
-            <mxPoint x="520" y="2980" as="sourcePoint" />
-            <mxPoint x="570" y="2930" as="targetPoint" />
-            <Array as="points">
-              <mxPoint x="470" y="2930" />
-              <mxPoint x="770" y="2930" />
-              <mxPoint x="770" y="2690" />
-              <mxPoint x="770" y="2450" />
-              <mxPoint x="513" y="2450" />
-            </Array>
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-28" value="Mail / WhatsApp /..." style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" parent="0ipz06AKfEBxF18oT8nj-27" vertex="1" connectable="0">
-          <mxGeometry x="-0.6717" y="3" relative="1" as="geometry">
-            <mxPoint as="offset" />
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-29" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="0ipz06AKfEBxF18oT8nj-31" target="0ipz06AKfEBxF18oT8nj-32" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-30" value="slack / discord / teams" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" parent="0ipz06AKfEBxF18oT8nj-29" vertex="1" connectable="0">
-          <mxGeometry x="-0.3" y="2" relative="1" as="geometry">
-            <mxPoint as="offset" />
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-31" value="Fetch Relevant historical data &amp;amp; BP" style="whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="217.5" y="2810" width="120" height="60" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-32" value="Recommendations to Agent" style="whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" parent="1" vertex="1">
-          <mxGeometry x="217.5" y="2950" width="120" height="60" as="geometry" />
-        </mxCell>
-        <mxCell id="1TZoFbZY8AwqjJc08LRQ-1" value="email, customer name, contract id, date, category, summary, issue category,T&amp;amp;C, Process descriptions, customer history, issue summary" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;fillColor=none;" vertex="1" parent="1">
-          <mxGeometry x="265" y="3028" width="770" height="30" as="geometry" />
-        </mxCell>
-      </root>
-    </mxGraphModel>
-  </diagram>
-</mxfile>

requirements.txt

@@ -1,6 +1,6 @@
  langchain 
  langchain-community 
- langchain_huggingface
+ langchain-huggingface
  beautifulsoup4 
  faiss-cpu 
  chromadb