notebooks/Larger_Context_Larger_N.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/towardsai/ai-tutor-rag-system/blob/main/notebooks/Larger_Context_Larger_N.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qtOtOvibOBfW"
      },
      "source": [
        "# Install Packages and Setup Variables\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I-hKKV6GEkro",
        "outputId": "ae3ff694-3b58-427f-f0c9-29e855c4efca"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m320.6/320.6 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m526.8/526.8 kB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.4/15.4 MB\u001b[0m \u001b[31m20.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m26.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m20.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.0/92.0 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.4/62.4 kB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.3/41.3 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m40.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.9/59.9 kB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m107.0/107.0 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.3/67.3 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m283.7/283.7 kB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m56.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.6/67.6 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m142.5/142.5 kB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.5/4.5 MB\u001b[0m \u001b[31m54.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.9/71.9 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.6/53.6 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.9/141.9 kB\u001b[0m \u001b[31m14.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m290.4/290.4 kB\u001b[0m \u001b[31m27.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.5/52.5 kB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.5/130.5 kB\u001b[0m \u001b[31m14.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m26.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m67.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m59.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.2/130.2 kB\u001b[0m \u001b[31m14.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m22.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m47.2/47.2 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.2/49.2 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for pypika (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "imageio 2.31.6 requires pillow<10.1.0,>=8.3.2, but you have pillow 10.3.0 which is incompatible.\n",
            "spacy 3.7.4 requires typer<0.10.0,>=0.3.0, but you have typer 0.12.3 which is incompatible.\n",
            "weasel 0.3.4 requires typer<0.10.0,>=0.3.0, but you have typer 0.12.3 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0m"
          ]
        }
      ],
      "source": [
        "!pip install -q llama-index==0.10.37 openai==1.30.1 tiktoken==0.7.0 chromadb==0.5.0 llama-index-llms-gemini==0.1.10 llama-index-vector-stores-chroma==0.1.7"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "5UZDtKWJWZ3c"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "\n",
        "# Set the following API Keys in the Python environment. Will be used later.\n",
        "os.environ[\"OPENAI_API_KEY\"] = \"[OPENAI_API_KEY]\"\n",
        "os.environ[\"GOOGLE_API_KEY\"] = \"[GOOGLE_API_KEY]\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P8un03bdrwIn"
      },
      "source": [
        "# Load Gemini Model\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "dFvjEffurv6T"
      },
      "outputs": [],
      "source": [
        "from llama_index.llms.gemini import Gemini\n",
        "\n",
        "llm = Gemini(model=\"models/gemini-pro\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fcX9C-AThh15"
      },
      "source": [
        "# Download the Vector Store\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_oi1avNUhhYd",
        "outputId": "4e4bd6d7-884d-43a3-d322-9e979114860e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "--2024-06-07 16:54:00--  https://github.com/AlaFalaki/tutorial_notebooks/raw/main/data/vectorstore.zip\n",
            "Resolving github.com (github.com)... 140.82.114.4\n",
            "Connecting to github.com (github.com)|140.82.114.4|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://raw.githubusercontent.com/AlaFalaki/tutorial_notebooks/main/data/vectorstore.zip [following]\n",
            "--2024-06-07 16:54:01--  https://raw.githubusercontent.com/AlaFalaki/tutorial_notebooks/main/data/vectorstore.zip\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.110.133, 185.199.111.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1479982 (1.4M) [application/zip]\n",
            "Saving to: ‘vectorstore.zip’\n",
            "\n",
            "vectorstore.zip     100%[===================>]   1.41M  --.-KB/s    in 0.07s   \n",
            "\n",
            "2024-06-07 16:54:01 (21.4 MB/s) - ‘vectorstore.zip’ saved [1479982/1479982]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!wget https://github.com/AlaFalaki/tutorial_notebooks/raw/main/data/vectorstore.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8BM4sU-bWZ0l",
        "outputId": "2dcb0bdc-d9ca-451f-cdb6-fa04c64ddb8d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Archive:  vectorstore.zip\n",
            "   creating: mini-llama-articles/\n",
            "   creating: mini-llama-articles/a361e92f-9895-41b6-ba72-4ad38e9875bd/\n",
            "  inflating: mini-llama-articles/a361e92f-9895-41b6-ba72-4ad38e9875bd/data_level0.bin  \n",
            "  inflating: mini-llama-articles/a361e92f-9895-41b6-ba72-4ad38e9875bd/header.bin  \n",
            " extracting: mini-llama-articles/a361e92f-9895-41b6-ba72-4ad38e9875bd/link_lists.bin  \n",
            "  inflating: mini-llama-articles/a361e92f-9895-41b6-ba72-4ad38e9875bd/length.bin  \n",
            "  inflating: mini-llama-articles/chroma.sqlite3  \n"
          ]
        }
      ],
      "source": [
        "!unzip vectorstore.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "VikY0MnrWZyC"
      },
      "outputs": [],
      "source": [
        "import chromadb\n",
        "from llama_index.vector_stores.chroma import ChromaVectorStore\n",
        "\n",
        "# Load the vector store from the local storage.\n",
        "db = chromadb.PersistentClient(path=\"./mini-llama-articles\")\n",
        "chroma_collection = db.get_or_create_collection(\"mini-llama-articles\")\n",
        "vector_store = ChromaVectorStore(chroma_collection=chroma_collection)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "o87JiKrUWZvG"
      },
      "outputs": [],
      "source": [
        "from llama_index.core import VectorStoreIndex\n",
        "\n",
        "# Create the index based on the vector store.\n",
        "index = VectorStoreIndex.from_vector_store(vector_store, llm=llm)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-H8c-pUpqu7W",
        "outputId": "0b7f036b-f70e-40cd-92f5-4027fbd51fa3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "top_2 results:\n",
            "\t The Llama 2 model is available in four different sizes: 7 billion, 13 billion, 34 billion, and 70 billion parameters.\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
            "top_4 results:\n",
            "\t The Llama 2 model is available in four different sizes: 7 billion, 13 billion, 34 billion, and 70 billion parameters.\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
            "top_6 results:\n",
            "\t The Llama 2 model comes in four different sizes with varying parameter counts: 7 billion, 13 billion, 34 billion, and 70 billion parameters.\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
            "top_8 results:\n",
            "\t The LLaMA2 model has parameter sizes of 7 billion, 13 billion, 34 billion, and 70 billion.\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
            "top_10 results:\n",
            "\t The LLaMA2 model has four different sizes of parameters: 7 billion, 13 billion, 34 billion, and 70 billion.\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
            "top_15 results:\n",
            "\t The LLaMA2 model has 7 billion parameters.\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
            "top_20 results:\n",
            "\t The LLaMA2 model has 7 billion parameters.\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
            "top_25 results:\n",
            "\t The LLaMA2 model has 7 billion parameters.\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
            "top_30 results:\n",
            "\t The LLaMA2 model has 7 billion parameters.\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n"
          ]
        }
      ],
      "source": [
        "for i in [2, 4, 6, 8, 10, 15, 20, 25, 30]:\n",
        "\n",
        "    query_engine = index.as_query_engine(similarity_top_k=i)\n",
        "\n",
        "    res = query_engine.query(\"How many parameters LLaMA2 model has?\")\n",
        "\n",
        "    print(f\"top_{i} results:\")\n",
        "    print(\"\\t\", res.response)\n",
        "    print(\"-_\" * 20)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eB83yG_o0cjO"
      },
      "source": [
        "# Evaluate\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TblvUrZ97TV6",
        "outputId": "8d4bf9ce-7309-41c8-9705-9e02f7de5203"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "--2024-06-05 19:43:23--  https://raw.githubusercontent.com/AlaFalaki/tutorial_notebooks/main/data/rag_eval_dataset.json\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 476714 (466K) [text/plain]\n",
            "Saving to: ‘rag_eval_dataset.json’\n",
            "\n",
            "\rrag_eval_dataset.js   0%[                    ]       0  --.-KB/s               \rrag_eval_dataset.js 100%[===================>] 465.54K  --.-KB/s    in 0.02s   \n",
            "\n",
            "2024-06-05 19:43:24 (25.0 MB/s) - ‘rag_eval_dataset.json’ saved [476714/476714]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!wget https://raw.githubusercontent.com/AlaFalaki/tutorial_notebooks/main/data/rag_eval_dataset.json"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fqRm2AMPrNE7"
      },
      "outputs": [],
      "source": [
        "# We can also load the dataset from a previously saved json file.\n",
        "from llama_index.core.evaluation import EmbeddingQAFinetuneDataset\n",
        "\n",
        "rag_eval_dataset = EmbeddingQAFinetuneDataset.from_json(\"./rag_eval_dataset.json\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1GagTcRz7XkU",
        "outputId": "2c03eebc-2362-4934-fb19-8bdcb6ceb44d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "top_2 faithfulness_score: 1.0\n",
            "top_2 relevancy_score: 1.0\n",
            "top_4 faithfulness_score: 1.0\n",
            "top_4 relevancy_score: 0.95\n",
            "top_6 faithfulness_score: 1.0\n",
            "top_6 relevancy_score: 0.95\n",
            "top_8 faithfulness_score: 1.0\n",
            "top_8 relevancy_score: 1.0\n",
            "top_10 faithfulness_score: 1.0\n",
            "top_10 relevancy_score: 1.0\n",
            "top_15 faithfulness_score: 0.95\n",
            "top_15 relevancy_score: 0.95\n",
            "top_20 faithfulness_score: 1.0\n",
            "top_20 relevancy_score: 0.95\n",
            "top_25 faithfulness_score: 0.95\n",
            "top_25 relevancy_score: 1.0\n",
            "top_30 faithfulness_score: 0.95\n",
            "top_30 relevancy_score: 0.95\n"
          ]
        }
      ],
      "source": [
        "from llama_index.core.evaluation import (\n",
        "    RelevancyEvaluator,\n",
        "    FaithfulnessEvaluator,\n",
        "    BatchEvalRunner,\n",
        ")\n",
        "from llama_index.llms.openai import OpenAI\n",
        "\n",
        "llm_gpt4 = OpenAI(temperature=0, model=\"gpt-4o\")\n",
        "\n",
        "faithfulness_evaluator = FaithfulnessEvaluator(llm=llm_gpt4)\n",
        "relevancy_evaluator = RelevancyEvaluator(llm=llm_gpt4)\n",
        "\n",
        "# Run evaluation\n",
        "queries = list(rag_eval_dataset.queries.values())\n",
        "batch_eval_queries = queries[:20]\n",
        "\n",
        "runner = BatchEvalRunner(\n",
        "    {\"faithfulness\": faithfulness_evaluator, \"relevancy\": relevancy_evaluator},\n",
        "    workers=32,\n",
        ")\n",
        "\n",
        "for i in [2, 4, 6, 8, 10, 15, 20, 25, 30]:\n",
        "    # Set Faithfulness and Relevancy evaluators\n",
        "    query_engine = index.as_query_engine(similarity_top_k=i, llm=llm)\n",
        "\n",
        "    eval_results = await runner.aevaluate_queries(\n",
        "        query_engine, queries=batch_eval_queries\n",
        "    )\n",
        "    faithfulness_score = sum(\n",
        "        result.passing for result in eval_results[\"faithfulness\"]\n",
        "    ) / len(eval_results[\"faithfulness\"])\n",
        "    print(f\"top_{i} faithfulness_score: {faithfulness_score}\")\n",
        "\n",
        "    relevancy_score = sum(result.passing for result in eval_results[\"relevancy\"]) / len(\n",
        "        eval_results[\"relevancy\"]\n",
        "    )\n",
        "    print(f\"top_{i} relevancy_score: {relevancy_score}\")"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "authorship_tag": "ABX9TyO54/MUoEirbXFWGbR7On3U",
      "include_colab_link": true,
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.12.4"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}