{ "cells": [ { "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "view-in-github" }, "source": [ "\"Open\n" ] }, { "cell_type": "markdown", "metadata": { "id": "qtOtOvibOBfW" }, "source": [ "# Install Packages and Setup Variables\n" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "I-hKKV6GEkro", "outputId": "ae3ff694-3b58-427f-f0c9-29e855c4efca" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m320.6/320.6 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m526.8/526.8 kB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.4/15.4 MB\u001b[0m \u001b[31m20.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m26.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m20.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.0/92.0 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.4/62.4 kB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.3/41.3 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m40.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.9/59.9 kB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m107.0/107.0 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.3/67.3 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m283.7/283.7 kB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m56.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.6/67.6 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m142.5/142.5 kB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.5/4.5 MB\u001b[0m \u001b[31m54.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.9/71.9 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.6/53.6 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.9/141.9 kB\u001b[0m \u001b[31m14.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m290.4/290.4 kB\u001b[0m \u001b[31m27.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.5/52.5 kB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.5/130.5 kB\u001b[0m \u001b[31m14.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m26.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m67.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m59.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.2/130.2 kB\u001b[0m \u001b[31m14.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m22.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m47.2/47.2 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.2/49.2 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Building wheel for pypika (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", "imageio 2.31.6 requires pillow<10.1.0,>=8.3.2, but you have pillow 10.3.0 which is incompatible.\n", "spacy 3.7.4 requires typer<0.10.0,>=0.3.0, but you have typer 0.12.3 which is incompatible.\n", "weasel 0.3.4 requires typer<0.10.0,>=0.3.0, but you have typer 0.12.3 which is incompatible.\u001b[0m\u001b[31m\n", "\u001b[0m" ] } ], "source": [ "!pip install -q llama-index==0.10.37 openai==1.30.1 tiktoken==0.7.0 chromadb==0.5.0 llama-index-llms-gemini==0.1.10 llama-index-vector-stores-chroma==0.1.7" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "id": "5UZDtKWJWZ3c" }, "outputs": [], "source": [ "import os\n", "\n", "# Set the following API Keys in the Python environment. Will be used later.\n", "os.environ[\"OPENAI_API_KEY\"] = \"[OPENAI_API_KEY]\"\n", "os.environ[\"GOOGLE_API_KEY\"] = \"[GOOGLE_API_KEY]\"" ] }, { "cell_type": "markdown", "metadata": { "id": "P8un03bdrwIn" }, "source": [ "# Load Gemini Model\n" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "id": "dFvjEffurv6T" }, "outputs": [], "source": [ "from llama_index.llms.gemini import Gemini\n", "\n", "llm = Gemini(model=\"models/gemini-pro\")" ] }, { "cell_type": "markdown", "metadata": { "id": "fcX9C-AThh15" }, "source": [ "# Download the Vector Store\n" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "_oi1avNUhhYd", "outputId": "4e4bd6d7-884d-43a3-d322-9e979114860e" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--2024-06-07 16:54:00-- https://github.com/AlaFalaki/tutorial_notebooks/raw/main/data/vectorstore.zip\n", "Resolving github.com (github.com)... 140.82.114.4\n", "Connecting to github.com (github.com)|140.82.114.4|:443... connected.\n", "HTTP request sent, awaiting response... 302 Found\n", "Location: https://raw.githubusercontent.com/AlaFalaki/tutorial_notebooks/main/data/vectorstore.zip [following]\n", "--2024-06-07 16:54:01-- https://raw.githubusercontent.com/AlaFalaki/tutorial_notebooks/main/data/vectorstore.zip\n", "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.110.133, 185.199.111.133, ...\n", "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 1479982 (1.4M) [application/zip]\n", "Saving to: ‘vectorstore.zip’\n", "\n", "vectorstore.zip 100%[===================>] 1.41M --.-KB/s in 0.07s \n", "\n", "2024-06-07 16:54:01 (21.4 MB/s) - ‘vectorstore.zip’ saved [1479982/1479982]\n", "\n" ] } ], "source": [ "!wget https://github.com/AlaFalaki/tutorial_notebooks/raw/main/data/vectorstore.zip" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "8BM4sU-bWZ0l", "outputId": "2dcb0bdc-d9ca-451f-cdb6-fa04c64ddb8d" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Archive: vectorstore.zip\n", " creating: mini-llama-articles/\n", " creating: mini-llama-articles/a361e92f-9895-41b6-ba72-4ad38e9875bd/\n", " inflating: mini-llama-articles/a361e92f-9895-41b6-ba72-4ad38e9875bd/data_level0.bin \n", " inflating: mini-llama-articles/a361e92f-9895-41b6-ba72-4ad38e9875bd/header.bin \n", " extracting: mini-llama-articles/a361e92f-9895-41b6-ba72-4ad38e9875bd/link_lists.bin \n", " inflating: mini-llama-articles/a361e92f-9895-41b6-ba72-4ad38e9875bd/length.bin \n", " inflating: mini-llama-articles/chroma.sqlite3 \n" ] } ], "source": [ "!unzip vectorstore.zip" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "id": "VikY0MnrWZyC" }, "outputs": [], "source": [ "import chromadb\n", "from llama_index.vector_stores.chroma import ChromaVectorStore\n", "\n", "# Load the vector store from the local storage.\n", "db = chromadb.PersistentClient(path=\"./mini-llama-articles\")\n", "chroma_collection = db.get_or_create_collection(\"mini-llama-articles\")\n", "vector_store = ChromaVectorStore(chroma_collection=chroma_collection)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "id": "o87JiKrUWZvG" }, "outputs": [], "source": [ "from llama_index.core import VectorStoreIndex\n", "\n", "# Create the index based on the vector store.\n", "index = VectorStoreIndex.from_vector_store(vector_store, llm=llm)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "-H8c-pUpqu7W", "outputId": "0b7f036b-f70e-40cd-92f5-4027fbd51fa3" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "top_2 results:\n", "\t The Llama 2 model is available in four different sizes: 7 billion, 13 billion, 34 billion, and 70 billion parameters.\n", "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n", "top_4 results:\n", "\t The Llama 2 model is available in four different sizes: 7 billion, 13 billion, 34 billion, and 70 billion parameters.\n", "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n", "top_6 results:\n", "\t The Llama 2 model comes in four different sizes with varying parameter counts: 7 billion, 13 billion, 34 billion, and 70 billion parameters.\n", "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n", "top_8 results:\n", "\t The LLaMA2 model has parameter sizes of 7 billion, 13 billion, 34 billion, and 70 billion.\n", "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n", "top_10 results:\n", "\t The LLaMA2 model has four different sizes of parameters: 7 billion, 13 billion, 34 billion, and 70 billion.\n", "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n", "top_15 results:\n", "\t The LLaMA2 model has 7 billion parameters.\n", "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n", "top_20 results:\n", "\t The LLaMA2 model has 7 billion parameters.\n", "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n", "top_25 results:\n", "\t The LLaMA2 model has 7 billion parameters.\n", "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n", "top_30 results:\n", "\t The LLaMA2 model has 7 billion parameters.\n", "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n" ] } ], "source": [ "for i in [2, 4, 6, 8, 10, 15, 20, 25, 30]:\n", "\n", " query_engine = index.as_query_engine(similarity_top_k=i)\n", "\n", " res = query_engine.query(\"How many parameters LLaMA2 model has?\")\n", "\n", " print(f\"top_{i} results:\")\n", " print(\"\\t\", res.response)\n", " print(\"-_\" * 20)" ] }, { "cell_type": "markdown", "metadata": { "id": "eB83yG_o0cjO" }, "source": [ "# Evaluate\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "TblvUrZ97TV6", "outputId": "8d4bf9ce-7309-41c8-9705-9e02f7de5203" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--2024-06-05 19:43:23-- https://raw.githubusercontent.com/AlaFalaki/tutorial_notebooks/main/data/rag_eval_dataset.json\n", "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n", "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 476714 (466K) [text/plain]\n", "Saving to: ‘rag_eval_dataset.json’\n", "\n", "\rrag_eval_dataset.js 0%[ ] 0 --.-KB/s \rrag_eval_dataset.js 100%[===================>] 465.54K --.-KB/s in 0.02s \n", "\n", "2024-06-05 19:43:24 (25.0 MB/s) - ‘rag_eval_dataset.json’ saved [476714/476714]\n", "\n" ] } ], "source": [ "!wget https://raw.githubusercontent.com/AlaFalaki/tutorial_notebooks/main/data/rag_eval_dataset.json" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "fqRm2AMPrNE7" }, "outputs": [], "source": [ "# We can also load the dataset from a previously saved json file.\n", "from llama_index.core.evaluation import EmbeddingQAFinetuneDataset\n", "\n", "rag_eval_dataset = EmbeddingQAFinetuneDataset.from_json(\"./rag_eval_dataset.json\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "1GagTcRz7XkU", "outputId": "2c03eebc-2362-4934-fb19-8bdcb6ceb44d" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "top_2 faithfulness_score: 1.0\n", "top_2 relevancy_score: 1.0\n", "top_4 faithfulness_score: 1.0\n", "top_4 relevancy_score: 0.95\n", "top_6 faithfulness_score: 1.0\n", "top_6 relevancy_score: 0.95\n", "top_8 faithfulness_score: 1.0\n", "top_8 relevancy_score: 1.0\n", "top_10 faithfulness_score: 1.0\n", "top_10 relevancy_score: 1.0\n", "top_15 faithfulness_score: 0.95\n", "top_15 relevancy_score: 0.95\n", "top_20 faithfulness_score: 1.0\n", "top_20 relevancy_score: 0.95\n", "top_25 faithfulness_score: 0.95\n", "top_25 relevancy_score: 1.0\n", "top_30 faithfulness_score: 0.95\n", "top_30 relevancy_score: 0.95\n" ] } ], "source": [ "from llama_index.core.evaluation import (\n", " RelevancyEvaluator,\n", " FaithfulnessEvaluator,\n", " BatchEvalRunner,\n", ")\n", "from llama_index.llms.openai import OpenAI\n", "\n", "llm_gpt4 = OpenAI(temperature=0, model=\"gpt-4o\")\n", "\n", "faithfulness_evaluator = FaithfulnessEvaluator(llm=llm_gpt4)\n", "relevancy_evaluator = RelevancyEvaluator(llm=llm_gpt4)\n", "\n", "# Run evaluation\n", "queries = list(rag_eval_dataset.queries.values())\n", "batch_eval_queries = queries[:20]\n", "\n", "runner = BatchEvalRunner(\n", " {\"faithfulness\": faithfulness_evaluator, \"relevancy\": relevancy_evaluator},\n", " workers=32,\n", ")\n", "\n", "for i in [2, 4, 6, 8, 10, 15, 20, 25, 30]:\n", " # Set Faithfulness and Relevancy evaluators\n", " query_engine = index.as_query_engine(similarity_top_k=i, llm=llm)\n", "\n", " eval_results = await runner.aevaluate_queries(\n", " query_engine, queries=batch_eval_queries\n", " )\n", " faithfulness_score = sum(\n", " result.passing for result in eval_results[\"faithfulness\"]\n", " ) / len(eval_results[\"faithfulness\"])\n", " print(f\"top_{i} faithfulness_score: {faithfulness_score}\")\n", "\n", " relevancy_score = sum(result.passing for result in eval_results[\"relevancy\"]) / len(\n", " eval_results[\"relevancy\"]\n", " )\n", " print(f\"top_{i} relevancy_score: {relevancy_score}\")" ] } ], "metadata": { "colab": { "authorship_tag": "ABX9TyO54/MUoEirbXFWGbR7On3U", "include_colab_link": true, "provenance": [] }, "kernelspec": { "display_name": "Python 3", "name": "python3" }, "language_info": { "name": "python", "version": "3.12.4" } }, "nbformat": 4, "nbformat_minor": 0 }