notebooks/12-Improve_Query.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/towardsai/ai-tutor-rag-system/blob/main/notebooks/12-Improve_Query.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-zE1h0uQV7uT"
      },
      "source": [
        "# Install Packages and Setup Variables\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QPJzr-I9XQ7l",
        "outputId": "5d48c88b-a0a9-49ff-d788-e076d1cb4ead"
      },
      "outputs": [],
      "source": [
        "!pip install -q llama-index==0.10.57 openai==1.37.0 llama-index-finetuning llama-index-embeddings-huggingface llama-index-embeddings-cohere llama-index-readers-web cohere==5.6.2 tiktoken==0.7.0 chromadb==0.5.5 html2text sentence_transformers pydantic llama-index-vector-stores-chroma==0.1.10 kaleido==0.2.1 llama-index-llms-gemini==0.1.11"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "riuXwpSPcvWC"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "\n",
        "# Set the following API Keys in the Python environment. Will be used later.\n",
        "os.environ[\"OPENAI_API_KEY\"] = \"<YOUR_OPENAI_KEY>\"\n",
        "os.environ[\"GOOGLE_API_KEY\"] = \"<YOUR_API_KEY>\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "jIEeZzqLbz0J"
      },
      "outputs": [],
      "source": [
        "# Allows running asyncio in environments with an existing event loop, like Jupyter notebooks.\n",
        "\n",
        "import nest_asyncio\n",
        "\n",
        "nest_asyncio.apply()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bkgi2OrYzF7q"
      },
      "source": [
        "# Load a Model\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "9oGT6crooSSj"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/Users/louis/Documents/GitHub/ai-tutor-rag-system/.conda/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
            "  from .autonotebook import tqdm as notebook_tqdm\n"
          ]
        }
      ],
      "source": [
        "from llama_index.llms.gemini import Gemini\n",
        "\n",
        "\n",
        "llm = Gemini(model=\"models/gemini-1.5-flash\", temperature=1, max_tokens=512)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0BwVuJXlzHVL"
      },
      "source": [
        "# Create a VectoreStore\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "SQP87lHczHKc"
      },
      "outputs": [],
      "source": [
        "import chromadb\n",
        "\n",
        "# create client and a new collection\n",
        "# chromadb.EphemeralClient saves data in-memory.\n",
        "chroma_client = chromadb.PersistentClient(path=\"./mini-llama-articles\")\n",
        "chroma_collection = chroma_client.create_collection(\"mini-llama-articles\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "zAaGcYMJzHAN"
      },
      "outputs": [],
      "source": [
        "from llama_index.vector_stores.chroma import ChromaVectorStore\n",
        "\n",
        "# Define a storage context object using the created vector database.\n",
        "vector_store = ChromaVectorStore(chroma_collection=chroma_collection)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I9JbAzFcjkpn"
      },
      "source": [
        "# Load the Dataset (CSV)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ceveDuYdWCYk"
      },
      "source": [
        "## Download\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eZwf6pv7WFmD"
      },
      "source": [
        "The dataset includes several articles from the TowardsAI blog, which provide an in-depth explanation of the LLaMA2 model. Read the dataset as a long string.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wl_pbPvMlv1h",
        "outputId": "a453b612-20a8-4396-d22b-b19d2bc47816"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100  169k  100  169k    0     0   915k      0 --:--:-- --:--:-- --:--:--  911k\n"
          ]
        }
      ],
      "source": [
        "!curl -o ./mini-llama-articles.csv https://raw.githubusercontent.com/AlaFalaki/tutorial_notebooks/main/data/mini-llama-articles.csv"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VWBLtDbUWJfA"
      },
      "source": [
        "## Read File\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0Q9sxuW0g3Gd",
        "outputId": "49b27d8a-1f96-4e8d-fa0f-27afbf2c395c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "14"
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import csv\n",
        "\n",
        "rows = []\n",
        "\n",
        "# Load the file as a JSON\n",
        "with open(\"./mini-llama-articles.csv\", mode=\"r\", encoding=\"utf-8\") as file:\n",
        "    csv_reader = csv.reader(file)\n",
        "\n",
        "    for idx, row in enumerate(csv_reader):\n",
        "        if idx == 0:\n",
        "            continue\n",
        "            # Skip header row\n",
        "        rows.append(row)\n",
        "\n",
        "# The number of characters in the dataset.\n",
        "len(rows)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S17g2RYOjmf2"
      },
      "source": [
        "# Convert to Document obj\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "YizvmXPejkJE"
      },
      "outputs": [],
      "source": [
        "from llama_index.core import Document\n",
        "\n",
        "# Convert the chunks to Document objects so the LlamaIndex framework can process them.\n",
        "documents = [\n",
        "    Document(\n",
        "        text=row[1], metadata={\"title\": row[0], \"url\": row[2], \"source_name\": row[3]}\n",
        "    )\n",
        "    for row in rows\n",
        "]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qjuLbmFuWsyl"
      },
      "source": [
        "# Transforming\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "9z3t70DGWsjO"
      },
      "outputs": [],
      "source": [
        "from llama_index.core.text_splitter import TokenTextSplitter\n",
        "\n",
        "text_splitter = TokenTextSplitter(separator=\" \", chunk_size=512, chunk_overlap=128)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 331,
          "referenced_widgets": [
            "3fbabd8a8660461ba5e7bc08ef39139a",
            "df2365556ae242a2ab1a119f9a31a561",
            "5f4b9d32df8f446e858e4c289dc282f9",
            "5b588f83a15d42d9aca888e06bbd95ff",
            "ad073bca655540809e39f26538d2ec0d",
            "13b9c5395bca4c3ba21265240cb936cf",
            "47a4586384274577a726c57605e7f8d9",
            "96a3bdece738481db57e811ccb74a974",
            "5c7973afd79349ed997a69120d0629b2",
            "af9b6ae927dd4764b9692507791bc67e",
            "134210510d49476e959dd7d032bbdbdc",
            "5f9bb065c2b74d2e8ded32e1306a7807",
            "73a06bc546a64f7f99a9e4a135319dcd",
            "ce48deaf4d8c49cdae92bfdbb3a78df0",
            "4a172e8c6aa44e41a42fc1d9cf714fd0",
            "0245f2604e4d49c8bd0210302746c47b",
            "e956dfab55084a9cbe33c8e331b511e7",
            "cb394578badd43a89850873ad2526542",
            "193aef33d9184055bb9223f56d456de6",
            "abfc9aa911ce4a5ea81c7c451f08295f",
            "e7937a1bc68441a080374911a6563376",
            "e532ed7bfef34f67b5fcacd9534eb789"
          ]
        },
        "id": "P9LDJ7o-Wsc-",
        "outputId": "01070c1f-dffa-4ab7-ad71-b07b76b12e03"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Parsing nodes:   0%|          | 0/14 [00:00<?, ?it/s]"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Parsing nodes: 100%|██████████| 14/14 [00:00<00:00, 28.28it/s]\n",
            "100%|██████████| 108/108 [01:36<00:00,  1.12it/s]\n",
            "100%|██████████| 108/108 [01:22<00:00,  1.30it/s]\n",
            "100%|██████████| 108/108 [00:29<00:00,  3.72it/s]\n",
            "Generating embeddings: 100%|██████████| 108/108 [00:02<00:00, 38.77it/s]\n"
          ]
        }
      ],
      "source": [
        "from llama_index.core.extractors import (\n",
        "    SummaryExtractor,\n",
        "    QuestionsAnsweredExtractor,\n",
        "    KeywordExtractor,\n",
        ")\n",
        "from llama_index.embeddings.openai import OpenAIEmbedding\n",
        "from llama_index.core.ingestion import IngestionPipeline\n",
        "\n",
        "pipeline = IngestionPipeline(\n",
        "    transformations=[\n",
        "        text_splitter,\n",
        "        QuestionsAnsweredExtractor(questions=3, llm=llm),\n",
        "        SummaryExtractor(summaries=[\"prev\", \"self\"], llm=llm),\n",
        "        KeywordExtractor(keywords=10, llm=llm),\n",
        "        OpenAIEmbedding(),\n",
        "    ],\n",
        "    vector_store=vector_store,\n",
        ")\n",
        "\n",
        "nodes = pipeline.run(documents=documents, show_progress=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mPGa85hM2P3P",
        "outputId": "c106c463-2459-4b11-bbae-5bd5e2246011"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "108"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(nodes)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "23x20bL3_jRb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "updating: mini-llama-articles/ (stored 0%)\n",
            "updating: mini-llama-articles/chroma.sqlite3 (deflated 65%)\n",
            "  adding: mini-llama-articles/aaac4d54-4f82-40da-b769-a6aecfa59eb0/ (stored 0%)\n",
            "  adding: mini-llama-articles/aaac4d54-4f82-40da-b769-a6aecfa59eb0/data_level0.bin (deflated 96%)\n",
            "  adding: mini-llama-articles/aaac4d54-4f82-40da-b769-a6aecfa59eb0/length.bin (deflated 35%)\n",
            "  adding: mini-llama-articles/aaac4d54-4f82-40da-b769-a6aecfa59eb0/link_lists.bin (stored 0%)\n",
            "  adding: mini-llama-articles/aaac4d54-4f82-40da-b769-a6aecfa59eb0/header.bin (deflated 61%)\n"
          ]
        }
      ],
      "source": [
        "!zip -r vectorstore.zip mini-llama-articles"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OWaT6rL7ksp8"
      },
      "source": [
        "# Load Indexes\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SodY2Xpf_kxg",
        "outputId": "9f8b7153-ea58-4824-8363-c47e922612a8"
      },
      "outputs": [],
      "source": [
        "# !unzip vectorstore.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "mXi56KTXk2sp"
      },
      "outputs": [],
      "source": [
        "import chromadb\n",
        "from llama_index.vector_stores.chroma import ChromaVectorStore\n",
        "\n",
        "# Create your index\n",
        "db = chromadb.PersistentClient(path=\"./mini-llama-articles\")\n",
        "chroma_collection = db.get_or_create_collection(\"mini-llama-articles\")\n",
        "vector_store = ChromaVectorStore(chroma_collection=chroma_collection)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "jKXURvLtkuTS"
      },
      "outputs": [],
      "source": [
        "# Create your index\n",
        "from llama_index.core import VectorStoreIndex\n",
        "\n",
        "vector_index = VectorStoreIndex.from_vector_store(vector_store)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SLrn8A3jckmW"
      },
      "source": [
        "# Multi-Step Query Engine\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UmpfpVCje8h3"
      },
      "source": [
        "## GPT-4\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "CaxFzDz4cRMd"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/var/folders/l7/9qcp7g5x5rl9x8ltw0t85qym0000gn/T/ipykernel_1424/226941912.py:4: DeprecationWarning: Call to deprecated class method from_defaults. (ServiceContext is deprecated, please use `llama_index.settings.Settings` instead.) -- Deprecated since version 0.10.0.\n",
            "  service_context_gpt4 = ServiceContext.from_defaults(llm=gpt4)\n"
          ]
        }
      ],
      "source": [
        "from llama_index.core import ServiceContext\n",
        "from llama_index.llms.openai import OpenAI\n",
        "\n",
        "gpt4 = OpenAI(temperature=1, model=\"gpt-4o\")\n",
        "service_context_gpt4 = ServiceContext.from_defaults(llm=gpt4)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "8y-Ya3GyfcAk"
      },
      "outputs": [],
      "source": [
        "from llama_index.core.indices.query.query_transform.base import (\n",
        "    StepDecomposeQueryTransform,\n",
        ")\n",
        "\n",
        "step_decompose_transform_gpt4 = StepDecomposeQueryTransform(llm=gpt4, verbose=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "zntXdSbGf_qF"
      },
      "outputs": [],
      "source": [
        "from llama_index.core.query_engine.multistep_query_engine import MultiStepQueryEngine\n",
        "\n",
        "query_engine_gpt4 = vector_index.as_query_engine(service_context=service_context_gpt4)\n",
        "query_engine_gpt4 = MultiStepQueryEngine(\n",
        "    query_engine=query_engine_gpt4,\n",
        "    query_transform=step_decompose_transform_gpt4,\n",
        "    index_summary=\"Used to answer questions about the LLaMA2 Model\",\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8JPD8yAinVSq"
      },
      "source": [
        "# Query Dataset\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D2IByQ5-ox9U"
      },
      "source": [
        "## Default\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "b0gue7cyctt1"
      },
      "outputs": [],
      "source": [
        "# Define a query engine that is responsible for retrieving related pieces of text,\n",
        "# and using a LLM to formulate the final answer.\n",
        "query_engine = vector_index.as_query_engine()\n",
        "\n",
        "res = query_engine.query(\"How many parameters LLaMA2 model has?\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "id": "VKK3jMprctre",
        "outputId": "b6ed346c-714b-44a6-b8fa-bfaca1b38deb"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'The Llama 2 model is available in four different sizes: 7 billion, 13 billion, 34 billion, and 70 billion parameters.'"
            ]
          },
          "execution_count": 23,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "res.response"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "465dH4yQc7Ct",
        "outputId": "6f7eb440-cc24-4d20-ac35-fa747265d18d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Node ID\t 63380d3f-7aff-47cd-b2c1-e4baaed70a7e\n",
            "Title\t Meta's Llama 2: Revolutionizing Open Source Language Models for Commercial Use\n",
            "Text\t I. Llama 2: Revolutionizing Commercial Use Unlike its predecessor Llama 1, which was limited to research use, Llama 2 represents a major advancement as an open-source commercial model. Businesses can now integrate Llama 2 into products to create AI-powered applications. Availability on Azure and AWS facilitates fine-tuning and adoption. However, restrictions apply to prevent exploitation. Companies with over 700 million active daily users cannot use Llama 2. Additionally, its output cannot be used to improve other language models.  II. Llama 2 Model Flavors Llama 2 is available in four different model sizes: 7 billion, 13 billion, 34 billion, and 70 billion parameters. While 7B, 13B, and 70B have already been released, the 34B model is still awaited. The pretrained variant, trained on a whopping 2 trillion tokens, boasts a context window of 4096 tokens, twice the size of its predecessor Llama 1. Meta also released a Llama 2 fine-tuned model for chat applications that was trained on over 1 million human annotations. Such extensive training comes at a cost, with the 70B model taking a staggering 1720320 GPU hours to train. The context window's length determines the amount of content the model can process at once, making Llama 2 a powerful language model in terms of scale and efficiency.  III. Safety Considerations: A Top Priority for Meta Meta's commitment to safety and alignment shines through in Llama 2's design. The model demonstrates exceptionally low AI safety violation percentages, surpassing even ChatGPT in safety benchmarks. Finding the right balance between helpfulness and safety when optimizing a model poses significant challenges. While a highly helpful model may be capable of answering any question, including sensitive ones like \"How do I build a bomb?\", it also raises concerns about potential misuse. Thus, striking the perfect equilibrium between providing useful information and ensuring safety is paramount. However, prioritizing safety to an extreme extent can lead to a model that struggles to effectively address a diverse range of questions. This limitation could hinder the model's practical applicability and user experience. Thus, achieving\n",
            "Score\t 0.7167442801500137\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
            "Node ID\t 77d0679b-6de4-4467-bc39-7932e18ae282\n",
            "Title\t Meta's Llama 2: Revolutionizing Open Source Language Models for Commercial Use\n",
            "Text\t The model demonstrates exceptionally low AI safety violation percentages, surpassing even ChatGPT in safety benchmarks. Finding the right balance between helpfulness and safety when optimizing a model poses significant challenges. While a highly helpful model may be capable of answering any question, including sensitive ones like \"How do I build a bomb?\", it also raises concerns about potential misuse. Thus, striking the perfect equilibrium between providing useful information and ensuring safety is paramount. However, prioritizing safety to an extreme extent can lead to a model that struggles to effectively address a diverse range of questions. This limitation could hinder the model's practical applicability and user experience. Thus, achieving an optimum balance that allows the model to be both helpful and safe is of utmost importance. To strike the right balance between helpfulness and safety, Meta employed two reward models - one for helpfulness and another for safety - to optimize the model's responses. The 34B parameter model has reported higher safety violations than other variants, possibly contributing to the delay in its release.  IV. Helpfulness Comparison: Llama 2 Outperforms Competitors Llama 2 emerges as a strong contender in the open-source language model arena, outperforming its competitors in most categories. The 70B parameter model outperforms all other open-source models, while the 7B and 34B models outshine Falcon in all categories and MPT in all categories except coding. Despite being smaller, Llam a2's performance rivals that of Chat GPT 3.5, a significantly larger closed-source model. While GPT 4 and PalM-2-L, with their larger size, outperform Llama 2, this is expected due to their capacity for handling complex language tasks. Llama 2's impressive ability to compete with larger models highlights its efficiency and potential in the market. However, Llama 2 does face challenges in coding and math problems, where models like Chat GPT 4 excel, given their significantly larger size. Chat GPT 4 performed significantly better than Llama 2 for coding (HumanEval benchmark)and math problem tasks (GSM8k benchmark). Open-source AI technologies, like Llama 2, continue to advance, offering\n",
            "Score\t 0.6967843740521363\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n"
          ]
        }
      ],
      "source": [
        "for src in res.source_nodes:\n",
        "    print(\"Node ID\\t\", src.node_id)\n",
        "    print(\"Title\\t\", src.metadata[\"title\"])\n",
        "    print(\"Text\\t\", src.text)\n",
        "    print(\"Score\\t\", src.score)\n",
        "    print(\"-_\" * 20)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2y2AiInmpz7g"
      },
      "source": [
        "## GPT-4 Multi-Step\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "69kADAFilW1n",
        "outputId": "8a847a58-539f-4ba7-ca07-ef80ceb8b3e2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[1;3;33m> Current query: How many parameters LLaMA2 model has?\n",
            "\u001b[0m\u001b[1;3;38;5;200m> New query: What is the LLaMA2 Model?\n",
            "\u001b[0m\u001b[1;3;33m> Current query: How many parameters LLaMA2 model has?\n",
            "\u001b[0m\u001b[1;3;38;5;200m> New query: None\n",
            "\u001b[0m"
          ]
        }
      ],
      "source": [
        "response_gpt4 = query_engine_gpt4.query(\"How many parameters LLaMA2 model has?\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "_ul5p3AMldzk",
        "outputId": "8c5cadda-8e06-4398-81bc-8571d4710b2a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'The LLaMA2 model has parameters ranging from 7 billion to 70 billion.'"
            ]
          },
          "execution_count": 26,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "response_gpt4.response"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k5pJPBPRqjbG",
        "outputId": "0bdd8382-8392-483d-bb6a-51e7a146eeb3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Node ID\t 3f7709ed-985e-417f-b88d-e3eac6ae8a06\n",
            "Text\t \n",
            "Question: What is the LLaMA2 Model?\n",
            "Answer: The Llama 2 model is an open-source commercial language model developed by Meta, available in different sizes ranging from 7 billion to 70 billion parameters. It is designed to be integrated into AI-powered applications for businesses, with a focus on safety considerations in its design. The model's Ghost Attention feature enhances conversational continuity, and it possesses a groundbreaking temporal capability for organizing information based on time relevance to deliver contextually accurate responses.\n",
            "Score\t None\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
            "Node ID\t 63380d3f-7aff-47cd-b2c1-e4baaed70a7e\n",
            "Text\t I. Llama 2: Revolutionizing Commercial Use Unlike its predecessor Llama 1, which was limited to research use, Llama 2 represents a major advancement as an open-source commercial model. Businesses can now integrate Llama 2 into products to create AI-powered applications. Availability on Azure and AWS facilitates fine-tuning and adoption. However, restrictions apply to prevent exploitation. Companies with over 700 million active daily users cannot use Llama 2. Additionally, its output cannot be used to improve other language models.  II. Llama 2 Model Flavors Llama 2 is available in four different model sizes: 7 billion, 13 billion, 34 billion, and 70 billion parameters. While 7B, 13B, and 70B have already been released, the 34B model is still awaited. The pretrained variant, trained on a whopping 2 trillion tokens, boasts a context window of 4096 tokens, twice the size of its predecessor Llama 1. Meta also released a Llama 2 fine-tuned model for chat applications that was trained on over 1 million human annotations. Such extensive training comes at a cost, with the 70B model taking a staggering 1720320 GPU hours to train. The context window's length determines the amount of content the model can process at once, making Llama 2 a powerful language model in terms of scale and efficiency.  III. Safety Considerations: A Top Priority for Meta Meta's commitment to safety and alignment shines through in Llama 2's design. The model demonstrates exceptionally low AI safety violation percentages, surpassing even ChatGPT in safety benchmarks. Finding the right balance between helpfulness and safety when optimizing a model poses significant challenges. While a highly helpful model may be capable of answering any question, including sensitive ones like \"How do I build a bomb?\", it also raises concerns about potential misuse. Thus, striking the perfect equilibrium between providing useful information and ensuring safety is paramount. However, prioritizing safety to an extreme extent can lead to a model that struggles to effectively address a diverse range of questions. This limitation could hinder the model's practical applicability and user experience. Thus, achieving\n",
            "Score\t 0.7149311149257048\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
            "Node ID\t 8a4bda7f-9e2a-44ff-a59b-84f36b8b3431\n",
            "Text\t with their larger size, outperform Llama 2, this is expected due to their capacity for handling complex language tasks. Llama 2's impressive ability to compete with larger models highlights its efficiency and potential in the market. However, Llama 2 does face challenges in coding and math problems, where models like Chat GPT 4 excel, given their significantly larger size. Chat GPT 4 performed significantly better than Llama 2 for coding (HumanEval benchmark)and math problem tasks (GSM8k benchmark). Open-source AI technologies, like Llama 2, continue to advance, offering strong competition to closed-source models.  V. Ghost Attention: Enhancing Conversational Continuity One unique feature in Llama 2 is Ghost Attention, which ensures continuity in conversations. This means that even after multiple interactions, the model remembers its initial instructions, ensuring more coherent and consistent responses throughout the conversation. This feature significantly enhances the user experience and makes Llama 2 a more reliable language model for interactive applications. In the example below, on the left, it forgets to use an emoji after a few conversations. On the right, with Ghost Attention, even after having many conversations, it will remember the context and continue to use emojis in its response.  VI. Temporal Capability: A Leap in Information Organization Meta reported a groundbreaking temporal capability, where the model organizes information based on time relevance. Each question posed to the model is associated with a date, and it responds accordingly by considering the event date before which the question becomes irrelevant. For example, if you ask the question, \"How long ago did Barack Obama become president?\", its only relevant after 2008. This temporal awareness allows Llama 2 to deliver more contextually accurate responses, enriching the user experience further.  VII. Open Questions and Future Outlook Meta's open-sourcing of Llama 2 represents a seismic shift, now offering developers and researchers commercial access to a leading language model. With Llama 2 outperforming MosaicML's current MPT models, all eyes are on how Databricks will respond. Can MosaicML's next MPT iteration beat Llama 2? Is it worthwhile to compete\n",
            "Score\t 0.714000324321046\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n"
          ]
        }
      ],
      "source": [
        "for src in response_gpt4.source_nodes:\n",
        "    print(\"Node ID\\t\", src.node_id)\n",
        "    print(\"Text\\t\", src.text)\n",
        "    print(\"Score\\t\", src.score)\n",
        "    print(\"-_\" * 20)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jwcSCiMhp4Uh"
      },
      "source": [
        "# Test gemini-1.5-flash Multi-Step\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "id": "uH9gNfZuslHK"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/var/folders/l7/9qcp7g5x5rl9x8ltw0t85qym0000gn/T/ipykernel_1424/1136257440.py:6: DeprecationWarning: Call to deprecated class method from_defaults. (ServiceContext is deprecated, please use `llama_index.settings.Settings` instead.) -- Deprecated since version 0.10.0.\n",
            "  service_context_gpt3 = ServiceContext.from_defaults(llm=gpt3)\n"
          ]
        }
      ],
      "source": [
        "from llama_index.core import ServiceContext\n",
        "from llama_index.core.indices.query.query_transform.base import (\n",
        "    StepDecomposeQueryTransform,\n",
        ")\n",
        "from llama_index.core.query_engine.multistep_query_engine import MultiStepQueryEngine\n",
        "\n",
        "service_context_gemini = ServiceContext.from_defaults(llm=llm)\n",
        "\n",
        "step_decompose_transform = StepDecomposeQueryTransform(llm=llm, verbose=True)\n",
        "\n",
        "query_engine_gemini = vector_index.as_query_engine(\n",
        "    service_context=service_context_gemini\n",
        ")\n",
        "query_engine_gemini = MultiStepQueryEngine(\n",
        "    query_engine=query_engine_gemini,\n",
        "    query_transform=step_decompose_transform,\n",
        "    index_summary=\"Used to answer questions about the LLaMA2 Model\",\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9s6SkHI0p6VZ",
        "outputId": "1c87dbda-e026-4e28-f7eb-b01145c62b77"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[1;3;33m> Current query: How many parameters LLaMA2 model has?\n",
            "\u001b[0m\u001b[1;3;38;5;200m> New query: What are the main components or features of the LLaMA2 model?\n",
            "\u001b[0m\u001b[1;3;33m> Current query: How many parameters LLaMA2 model has?\n",
            "\u001b[0m\u001b[1;3;38;5;200m> New query: What is the range of model sizes available for the LLaMA2 model?\n",
            "\u001b[0m\u001b[1;3;33m> Current query: How many parameters LLaMA2 model has?\n",
            "\u001b[0m\u001b[1;3;38;5;200m> New query: What are the safety considerations in the LLaMA2 model?\n",
            "\u001b[0m"
          ]
        }
      ],
      "source": [
        "response_gemini = query_engine_gemini.query(\"How many parameters LLaMA2 model has?\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "FlgMkAhQsTIY",
        "outputId": "0996e879-3914-44b1-cdec-e4f0b0ba7a4e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'The LLaMA2 model has model sizes ranging from 7 billion to 70 billion parameters.'"
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "response_gemini.response"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DxOF2qth1gUC"
      },
      "source": [
        "# Test Retriever on Multistep\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "id": "In9BZbU10KAz"
      },
      "outputs": [],
      "source": [
        "import llama_index"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "id": "_-fBK2g2zkKb"
      },
      "outputs": [],
      "source": [
        "from llama_index.core.indices.query.schema import QueryBundle"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "id": "wqT7mlhx1KGB"
      },
      "outputs": [],
      "source": [
        "t = QueryBundle(\"How many parameters LLaMA2 model has?\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 304
        },
        "id": "OHpa3MqXyyvd",
        "outputId": "d9b39a47-751d-48a1-ce68-ebf0a50b938d"
      },
      "outputs": [
        {
          "ename": "NotImplementedError",
          "evalue": "This query engine does not support retrieve, use query directly",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNotImplementedError\u001b[0m                       Traceback (most recent call last)",
            "Cell \u001b[0;32mIn[34], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mquery_engine_gpt3\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mretrieve\u001b[49m\u001b[43m(\u001b[49m\u001b[43mt\u001b[49m\u001b[43m)\u001b[49m\n",
            "File \u001b[0;32m~/Documents/GitHub/ai-tutor-rag-system/.conda/lib/python3.11/site-packages/llama_index/core/base/base_query_engine.py:49\u001b[0m, in \u001b[0;36mBaseQueryEngine.retrieve\u001b[0;34m(self, query_bundle)\u001b[0m\n\u001b[1;32m     48\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mretrieve\u001b[39m(\u001b[38;5;28mself\u001b[39m, query_bundle: QueryBundle) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m List[NodeWithScore]:\n\u001b[0;32m---> 49\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mNotImplementedError\u001b[39;00m(\n\u001b[1;32m     50\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThis query engine does not support retrieve, use query directly\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     51\u001b[0m     )\n",
            "\u001b[0;31mNotImplementedError\u001b[0m: This query engine does not support retrieve, use query directly"
          ]
        }
      ],
      "source": [
        "query_engine_gemini.retrieve(t)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FCdPwVAQ6ixg"
      },
      "source": [
        "# HyDE Transform\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "id": "1x6He0T961Kg"
      },
      "outputs": [],
      "source": [
        "query_engine = vector_index.as_query_engine(llm=llm)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "id": "0GgtfeBC6m0H"
      },
      "outputs": [],
      "source": [
        "from llama_index.core.indices.query.query_transform import HyDEQueryTransform\n",
        "from llama_index.core.query_engine.transform_query_engine import TransformQueryEngine\n",
        "\n",
        "hyde = HyDEQueryTransform(include_original=True)\n",
        "hyde_query_engine = TransformQueryEngine(query_engine, hyde)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {
        "id": "mm3nYnIE6mwl"
      },
      "outputs": [],
      "source": [
        "response = hyde_query_engine.query(\"How many parameters LLaMA2 model has?\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "id": "PjTJ2poc6mt5",
        "outputId": "32fc89c2-474d-4791-e4b0-2a1de262b571"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'The LLaMA 2 model has four different model sizes: 7 billion, 13 billion, 34 billion, and 70 billion parameters.'"
            ]
          },
          "execution_count": 38,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "response.response"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "StgikqWZ6mrl",
        "outputId": "f0552af4-524e-444b-b8cb-67a665fad474"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Node ID\t 63380d3f-7aff-47cd-b2c1-e4baaed70a7e\n",
            "Text\t I. Llama 2: Revolutionizing Commercial Use Unlike its predecessor Llama 1, which was limited to research use, Llama 2 represents a major advancement as an open-source commercial model. Businesses can now integrate Llama 2 into products to create AI-powered applications. Availability on Azure and AWS facilitates fine-tuning and adoption. However, restrictions apply to prevent exploitation. Companies with over 700 million active daily users cannot use Llama 2. Additionally, its output cannot be used to improve other language models.  II. Llama 2 Model Flavors Llama 2 is available in four different model sizes: 7 billion, 13 billion, 34 billion, and 70 billion parameters. While 7B, 13B, and 70B have already been released, the 34B model is still awaited. The pretrained variant, trained on a whopping 2 trillion tokens, boasts a context window of 4096 tokens, twice the size of its predecessor Llama 1. Meta also released a Llama 2 fine-tuned model for chat applications that was trained on over 1 million human annotations. Such extensive training comes at a cost, with the 70B model taking a staggering 1720320 GPU hours to train. The context window's length determines the amount of content the model can process at once, making Llama 2 a powerful language model in terms of scale and efficiency.  III. Safety Considerations: A Top Priority for Meta Meta's commitment to safety and alignment shines through in Llama 2's design. The model demonstrates exceptionally low AI safety violation percentages, surpassing even ChatGPT in safety benchmarks. Finding the right balance between helpfulness and safety when optimizing a model poses significant challenges. While a highly helpful model may be capable of answering any question, including sensitive ones like \"How do I build a bomb?\", it also raises concerns about potential misuse. Thus, striking the perfect equilibrium between providing useful information and ensuring safety is paramount. However, prioritizing safety to an extreme extent can lead to a model that struggles to effectively address a diverse range of questions. This limitation could hinder the model's practical applicability and user experience. Thus, achieving\n",
            "Score\t 0.7504822493620628\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
            "Node ID\t 5f575ba5-e68d-4c50-90bf-1125c4bd51f8\n",
            "Text\t Models Meta AI, formerly known as Facebook Artificial Intelligence Research (FAIR), is an artificial intelligence laboratory that aims to share open-source frameworks, tools, libraries, and models for research exploration and large-scale production deployment. In 2018, they released the open-source PyText, a modeling framework focused on NLP systems. Then, in August 2022, they announced the release of BlenderBot 3, a chatbot designed to improve conversational skills and safety. In November 2022, Meta developed a large language model called Galactica, which assists scientists with tasks such as summarizing academic papers and annotating molecules and proteins. Released in February 2023, LLaMA (Large Language Model Meta AI) is a transformer-based foundational large language model by Meta that ventures into both the AI and academic spaces. The model aims to help researchers, scientists, and engineers advance their work in exploring AI applications. It will be released under a non-commercial license to prevent misuse, and access will be granted to academic researchers, individuals, and organizations affiliated with the government, civil society, academia, and industry research facilities on a selective case-by-case basis. The sharing of codes and weights allows other researchers to test new approaches in LLMs. The LLaMA models have a range of 7 billion to 65 billion parameters. LLaMA-65B can be compared to DeepMind's Chinchilla and Google's PaLM. Publicly available unlabeled data was used to train these models, and training smaller foundational models require less computing power and resources. LLaMA 65B and 33B have been trained on 1.4 trillion tokens in 20 different languages, and according to the Facebook Artificial Intelligence Research (FAIR) team, the model's performance varies across languages. The data sources used for training included CCNet (67%), GitHub, Wikipedia, ArXiv, Stack Exchange, and books. LLaMA, like other large scale language models, has issues related to biased & toxic generation and hallucination.  6. Eleuther's GPT-Neo Models Founded in July 2020 by Connor Leahy, Sid Black, and Leo Gao, EleutherAI is a non-profit AI research lab\n",
            "Score\t 0.7375396701691563\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n"
          ]
        }
      ],
      "source": [
        "for src in response.source_nodes:\n",
        "    print(\"Node ID\\t\", src.node_id)\n",
        "    print(\"Text\\t\", src.text)\n",
        "    print(\"Score\\t\", src.score)\n",
        "    print(\"-_\" * 20)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "id": "17Jbo1FH6mjH"
      },
      "outputs": [],
      "source": [
        "query_bundle = hyde(\"How many parameters LLaMA2 model has?\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {
        "id": "UZEK63K77W7X"
      },
      "outputs": [],
      "source": [
        "hyde_doc = query_bundle.embedding_strs[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 214
        },
        "id": "wyzwkpSn7Yi1",
        "outputId": "9b03f8dc-a26e-45e4-eec1-22366bd68dd2"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'The LLaMA2 model has a total of 12 parameters. These parameters include the weights and biases of the neural network layers, as well as the hyperparameters such as learning rate, batch size, and number of epochs. Additionally, the model may also include regularization parameters such as L1 or L2 regularization coefficients. Overall, these parameters are crucial in determining the performance and behavior of the LLaMA2 model in various machine learning tasks.\"'"
            ]
          },
          "execution_count": 42,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "hyde_doc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "authorship_tag": "ABX9TyMcBonOXFUEEHJsKREchiOp",
      "include_colab_link": true,
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.12.4"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0245f2604e4d49c8bd0210302746c47b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "134210510d49476e959dd7d032bbdbdc": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "13b9c5395bca4c3ba21265240cb936cf": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "193aef33d9184055bb9223f56d456de6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3fbabd8a8660461ba5e7bc08ef39139a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_df2365556ae242a2ab1a119f9a31a561",
              "IPY_MODEL_5f4b9d32df8f446e858e4c289dc282f9",
              "IPY_MODEL_5b588f83a15d42d9aca888e06bbd95ff"
            ],
            "layout": "IPY_MODEL_ad073bca655540809e39f26538d2ec0d"
          }
        },
        "47a4586384274577a726c57605e7f8d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4a172e8c6aa44e41a42fc1d9cf714fd0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e7937a1bc68441a080374911a6563376",
            "placeholder": "​",
            "style": "IPY_MODEL_e532ed7bfef34f67b5fcacd9534eb789",
            "value": " 108/108 [00:03&lt;00:00, 33.70it/s]"
          }
        },
        "5b588f83a15d42d9aca888e06bbd95ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_af9b6ae927dd4764b9692507791bc67e",
            "placeholder": "​",
            "style": "IPY_MODEL_134210510d49476e959dd7d032bbdbdc",
            "value": " 14/14 [00:00&lt;00:00, 21.41it/s]"
          }
        },
        "5c7973afd79349ed997a69120d0629b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5f4b9d32df8f446e858e4c289dc282f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_96a3bdece738481db57e811ccb74a974",
            "max": 14,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5c7973afd79349ed997a69120d0629b2",
            "value": 14
          }
        },
        "5f9bb065c2b74d2e8ded32e1306a7807": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_73a06bc546a64f7f99a9e4a135319dcd",
              "IPY_MODEL_ce48deaf4d8c49cdae92bfdbb3a78df0",
              "IPY_MODEL_4a172e8c6aa44e41a42fc1d9cf714fd0"
            ],
            "layout": "IPY_MODEL_0245f2604e4d49c8bd0210302746c47b"
          }
        },
        "73a06bc546a64f7f99a9e4a135319dcd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e956dfab55084a9cbe33c8e331b511e7",
            "placeholder": "​",
            "style": "IPY_MODEL_cb394578badd43a89850873ad2526542",
            "value": "Generating embeddings: 100%"
          }
        },
        "96a3bdece738481db57e811ccb74a974": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "abfc9aa911ce4a5ea81c7c451f08295f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ad073bca655540809e39f26538d2ec0d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "af9b6ae927dd4764b9692507791bc67e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cb394578badd43a89850873ad2526542": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ce48deaf4d8c49cdae92bfdbb3a78df0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_193aef33d9184055bb9223f56d456de6",
            "max": 108,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_abfc9aa911ce4a5ea81c7c451f08295f",
            "value": 108
          }
        },
        "df2365556ae242a2ab1a119f9a31a561": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_13b9c5395bca4c3ba21265240cb936cf",
            "placeholder": "​",
            "style": "IPY_MODEL_47a4586384274577a726c57605e7f8d9",
            "value": "Parsing nodes: 100%"
          }
        },
        "e532ed7bfef34f67b5fcacd9534eb789": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e7937a1bc68441a080374911a6563376": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e956dfab55084a9cbe33c8e331b511e7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}