notebooks/06-Evaluate_RAG.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/towardsai/ai-tutor-rag-system/blob/main/notebooks/06-Evaluate_RAG.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5BGJ3fxhOk2V"
      },
      "source": [
        "# Install Packages and Setup Variables\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QPJzr-I9XQ7l"
      },
      "outputs": [],
      "source": [
        "!pip install -q llama-index==0.10.57 openai==1.37.0 tiktoken==0.7.0 chromadb==0.5.5 llama-index-vector-stores-chroma==0.1.10 llama-index-llms-gemini==0.1.11"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "riuXwpSPcvWC"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "\n",
        "# Set the following API Keys in the Python environment. Will be used later.\n",
        "os.environ[\"OPENAI_API_KEY\"] = \"<YOUR_API_KEY>\"\n",
        "os.environ[\"GOOGLE_API_KEY\"] = \"<YOUR_API_KEY>\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "km-KQOrgr3VB"
      },
      "outputs": [],
      "source": [
        "# Allows running asyncio in environments with an existing event loop, like Jupyter notebooks.\n",
        "\n",
        "import nest_asyncio\n",
        "\n",
        "nest_asyncio.apply()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0BwVuJXlzHVL"
      },
      "source": [
        "# Create a VectoreStore\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "SQP87lHczHKc"
      },
      "outputs": [],
      "source": [
        "import chromadb\n",
        "\n",
        "# create client and a new collection\n",
        "# chromadb.EphemeralClient saves data in-memory.\n",
        "chroma_client = chromadb.PersistentClient(path=\"./mini-llama-articles\")\n",
        "chroma_collection = chroma_client.create_collection(\"mini-llama-articles\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "zAaGcYMJzHAN"
      },
      "outputs": [],
      "source": [
        "from llama_index.vector_stores.chroma import ChromaVectorStore\n",
        "\n",
        "# Define a storage context object using the created vector database.\n",
        "vector_store = ChromaVectorStore(chroma_collection=chroma_collection)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I9JbAzFcjkpn"
      },
      "source": [
        "# Load the Dataset (CSV)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_Tif8-JoRH68"
      },
      "source": [
        "## Download\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4fQaa1LN1mXL"
      },
      "source": [
        "The dataset includes several articles from the TowardsAI blog, which provide an in-depth explanation of the LLaMA2 model.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "fQtpDvUzKNzI"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100  169k  100  169k    0     0   500k      0 --:--:-- --:--:-- --:--:--  501k\n"
          ]
        }
      ],
      "source": [
        "!curl -o ./mini-dataset.csv https://raw.githubusercontent.com/AlaFalaki/tutorial_notebooks/main/data/mini-llama-articles.csv"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zk-4alIxROo8"
      },
      "source": [
        "## Load the Articles\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "_WER5lt0N7c5"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "14"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import csv\n",
        "\n",
        "rows = []\n",
        "\n",
        "# Load the file as a JSON\n",
        "with open(\"./mini-llama-articles.csv\", mode=\"r\", encoding=\"utf-8\") as file:\n",
        "    csv_reader = csv.reader(file)\n",
        "\n",
        "    for idx, row in enumerate(csv_reader):\n",
        "        if idx == 0:\n",
        "            continue\n",
        "            # Skip header row\n",
        "        rows.append(row)\n",
        "\n",
        "# The number of characters in the dataset.\n",
        "len(rows)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wxEStggPdxYs"
      },
      "source": [
        "# Convert to Document obj\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "lFvW_886dxKX"
      },
      "outputs": [],
      "source": [
        "from llama_index.core import Document\n",
        "from llama_index.core.schema import TextNode\n",
        "\n",
        "# Convert the chunks to Document objects so the LlamaIndex framework can process them.\n",
        "documents = [\n",
        "    Document(\n",
        "        text=row[1],\n",
        "        metadata={\"title\": row[0], \"url\": row[2], \"source_name\": row[3]},\n",
        "    )\n",
        "    for row in rows\n",
        "]\n",
        "# By default, the node/chunks ids are set to random uuids. To ensure same id's per run, we manually set them.\n",
        "for idx, doc in enumerate(documents):\n",
        "    doc.id_ = f\"doc_{idx}\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "Njoc3XEVkKkf",
        "outputId": "b40d03b6-4f19-465a-890c-9363481e3eca"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Document(id_='doc_0', embedding=None, metadata={'title': \"Beyond GPT-4: What's New?\", 'url': 'https://pub.towardsai.net/beyond-gpt-4-whats-new-cbd61a448eb9#dda8', 'source_name': 'towards_ai'}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={}, text='LLM Variants and Meta\\'s Open Source Before shedding light on four major trends, I\\'d share the latest Meta\\'s Llama 2 and Code Llama. Meta\\'s Llama 2 represents a sophisticated evolution in LLMs. This suite spans models pretrained and fine-tuned across a parameter spectrum of 7 billion to 70 billion. A specialized derivative, Llama 2-Chat, has been engineered explicitly for dialogue-centric applications. Benchmarking revealed Llama 2\\'s superior performance over most extant open-source chat models. Human-centric evaluations, focusing on safety and utility metrics, positioned Llama 2-Chat as a potential contender against proprietary, closed-source counterparts. The development trajectory of Llama 2 emphasized rigorous fine-tuning methodologies. Meta\\'s transparent delineation of these processes aims to catalyze community-driven advancements in LLMs, underscoring a commitment to collaborative and responsible AI development. Code Llama is built on top of Llama 2 and is available in three models: Code Llama, the foundational code model;Codel Llama - Python specialized for Python;and Code Llama - Instruct, which is fine-tuned for understanding natural language instructions. Based on its benchmark testing, Code Llama outperformed state-of-the-art publicly available LLMs (except GPT-4) on code tasks. Llama 2, Llama 2-Chat, and Code Llama are key steps in LLM development but still have a way to go compared to GPT-4. Meta\\'s open access and commitment to improving these models promise transparent and faster LLM progress in the future. Please refer to the LLM and Llama variants below:  From LLMs to Multimodal LLMs, like OpenAI\\'s ChatGPT (GPT-3.5), primarily focus on understanding and generating human language. They\\'ve been instrumental in tasks like text generation, translation, and even creative writing. However, their scope is limited to text. Enter multimodal models like GPT-4. These are a new breed of AI models that can understand and generate not just text, but also images, sounds, and potentially other types of data. The term \"multimodal\" refers to their ability to process multiple modes or types of data simultaneously. This is a game-changer. Imagine an AI that can not only read a description of a dress but also visualize it or even design it! Multimodal AI models are moving us towards more holistic AI systems. These systems can potentially understand our world in a more comprehensive manner, bridging the gap between different forms of data and providing richer, more integrated solutions. As we stand on the cusp of this new era, it\\'s exciting to envision the myriad of applications and innovations that Multimodal models will bring to the table. The future of AI looks more integrated and versatile than ever before.  From Connections to Vector DB The AI landscape is witnessing a fascinating transition: from Language Model (LLM) connections or integrations, e.g., LangChain and LlamaIndex, to the rise of Vector Databases (Vector DB) such as Weaviate, Milvus, Pinecone, Chroma, and Vespa.ai. But what\\'s driving this shift, and why does it matter? LLM connections, like the LlamaIndex, primarily focus on linking and understanding vast amounts of external data. They\\'ve been pivotal in creating semantic connections, enabling more intuitive search experiences, and enhancing data accessibility. However, as the volume and variety of data grow, the need for more advanced storage and retrieval mechanisms becomes evident. This is where Vector DBs come into play. Unlike traditional databases that store data in rows and columns, Vector DBs store data in high-dimensional space, allowing for more efficient and accurate similarity searches. Tools like Weaviate and Milvus are designed to handle massive datasets, making them ideal for tasks like image recognition, recommendation systems, and more. The rise of Vector DBs represents a broader trend in AI: the quest for more efficient, scalable, and versatile data handling solutions. As we navigate this evolution, it\\'s clear that the combination of LLMs and Vector DBs will redefine how we store, access, and understand data in the AI-driven future.  From Agents to OS The AI realm is abuzz with innovations, and one of the most intriguing shifts we\\'re witnessing is the transition from LLM agents to using LLMs as Operating Systems (OS). Let\\'s delve into this evolution and its implications. LLM agents, like AutoGPT, AgentGPT, BabyAGI, and HuggingGPT, have been groundbreaking in automating tasks based on user requests. These agents leverage the power of Language Models (LLMs) to understand and execute commands, making them invaluable in tasks ranging from content generation to data analysis. Their adaptability and intelligence have made them a staple in many AI toolkits. However, the vision for AI doesn\\'t stop there. The concept of LLM as an OS is emerging as the next big thing. Imagine an operating system where the core is a language model, orchestrating everything around it. Such a system would not just execute tasks but would understand context, anticipate needs, and offer solutions in real time. It\\'s like turning the LLM into the brain of the digital ecosystem, making devices and applications more intuitive and responsive than ever. The move towards LLM as OS signifies a paradigm shift in how we perceive and utilize AI. It\\'s not just about automation anymore; it\\'s about creating a seamless, intelligent interface between humans and technology. As we stand on the brink of this transformation, the potential for LLM-driven OS to revolutionize our digital interactions is immense.  From Fine-tuning to Plugins The world of LLMs is undergoing a transformative shift, moving from intricate fine-tuning processes to the more dynamic realm of plugins. Let\\'s unpack this evolution. Historically, fine-tuning has been the cornerstone of LLM optimization. There are two primary ways to fine-tune LLMs: feeding data into the LLM in real-time and directly fine-tuning on the LLM. From a technical standpoint, this involves three methods: Transfer Learning: Adapting a pre-trained model to new tasks.Sequential Fine-tuning: Refining models in stages for specific tasks.Task-specific Fine-tuning: Tailoring models for a particular function. Moreover, LLM techniques like In-context learning, Few-shot learning, and Zero-shot learning have further enhanced the model\\'s adaptability, allowing them to understand and generate content with minimal data. However, the future of LLMs is leaning towards plugins. With the introduction of tools like GPT-4 Plugins, the focus is on extending LLMs seamlessly. Instead of running LLMs as a service, they\\'re envisioned as platforms. This means integrating LLMs with various tools, enhancing their capabilities, and offering a more modular and scalable approach to AI applications. The journey from fine-tuning to plugins represents a move from static optimization to dynamic adaptability, ensuring that LLMs remain at the forefront of AI innovation.  In a Nutshell The AI domain is witnessing rapid shifts, with LLMs playing a central role. Initially, the move was from LLMs to Multimodal models, expanding from text to include images and sounds. Simultaneously, the trend shifted from LLM connections, which linked external data, to Vector Databases for efficient high-dimensional storage. Another evolution saw LLM agents, which automated tasks, transitioning towards LLMs as Operating Systems. This change aims for more intuitive, context-aware devices and applications. Furthermore, the traditional fine-tuning processes of LLMs are now being replaced by dynamic plugins, turning LLMs into platforms integrated with various tools. Leading this LLM revolution are OpenAI\\'s GPT-4 and Meta\\'s LLaMA2. Their pioneering efforts are setting the stage for an AI future that\\'s more integrated, responsive, and attuned to human interactions.  More Readings Harnessing the Power of LLMs in Practice: A Survey on ChatGPT and Beyond: https://arxiv.org/abs/2304.13712Sparks of Artificial General Intelligence: Early experiments with GPT-4: https://arxiv.org/abs/2303.12712GPT4All-J: https://huggingface.co/nomic-ai/gpt4all-jIntroducing Code Llama, a state-of-the-art large language model for coding: https://ai.meta.com/blog/code-llama-large-language-model-coding/Llama 2: Open Foundation and Fine-Tuned Chat Models: https://ai.meta.com/research/publications/llama-2-open-foundation-and-fine-tuned-chat-models/', mimetype='text/plain', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n')"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "documents[0]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S17g2RYOjmf2"
      },
      "source": [
        "# Transforming\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "STACTMUR1z9N"
      },
      "outputs": [],
      "source": [
        "from llama_index.core.node_parser import TokenTextSplitter\n",
        "from llama_index.core.schema import BaseNode\n",
        "import hashlib\n",
        "\n",
        "\n",
        "def deterministic_id_func(i: int, doc: BaseNode) -> str:\n",
        "    \"\"\"Deterministic ID function for the text splitter.\n",
        "    This will be used to generate a unique repeatable identifier for each node.\"\"\"\n",
        "    unique_identifier = doc.id_ + str(i)\n",
        "    hasher = hashlib.sha256()\n",
        "    hasher.update(unique_identifier.encode(\"utf-8\"))\n",
        "    return hasher.hexdigest()\n",
        "\n",
        "\n",
        "text_splitter = TokenTextSplitter(\n",
        "    separator=\" \", chunk_size=512, chunk_overlap=128, id_func=deterministic_id_func\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "CtdsIUQ81_hT"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/Users/omar/Documents/ai_repos/ai-tutor-rag-system/env/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
            "  from .autonotebook import tqdm as notebook_tqdm\n",
            "Parsing nodes: 100%|██████████| 14/14 [00:00<00:00, 75.67it/s] \n",
            "Generating embeddings: 100%|██████████| 108/108 [00:02<00:00, 53.30it/s]\n"
          ]
        }
      ],
      "source": [
        "from llama_index.embeddings.openai import OpenAIEmbedding\n",
        "from llama_index.core.ingestion import IngestionPipeline\n",
        "\n",
        "pipeline = IngestionPipeline(\n",
        "    transformations=[\n",
        "        text_splitter,\n",
        "        OpenAIEmbedding(),\n",
        "    ],\n",
        "    vector_store=vector_store,\n",
        ")\n",
        "\n",
        "nodes = pipeline.run(documents=documents, show_progress=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "n5WRy0g71Hwu",
        "outputId": "4caee0cf-3b6a-43a9-b12f-668f241641c1"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "TextNode(id_='4ab5bd897f01474fc9b0049f95e31edae3ccd9e74d0f0acd3932b50a74d608b6', embedding=[-0.022489557042717934, 0.010829868726432323, -0.017510632053017616, -0.013220878317952156, 0.00476795481517911, 0.01368501503020525, -0.028073269873857498, 0.025499416515231133, -0.03817176818847656, -0.028706183657050133, 0.028424888849258423, 0.028059205040335655, -0.02846708334982395, -0.01441638357937336, 0.008023947477340698, 0.019254662096500397, 0.014894585125148296, 0.003285880433395505, 0.004690598696470261, -0.004845311399549246, -0.002776032779365778, 0.00021833348728250712, -0.0064733074977993965, -0.019775059074163437, 0.004556983709335327, 0.02648395113646984, 0.026272978633642197, -0.028537405654788017, -0.017580954357981682, 0.0022995888721197844, 0.012756740674376488, 0.014036634936928749, -0.02931096777319908, -0.0015875602839514613, -0.0138326957821846, -0.017580954357981682, 0.016948040574789047, -0.005618873052299023, 0.035780761390924454, -0.010970516130328178, 0.01465548388659954, 0.007644199300557375, 0.006318595260381699, -0.030604926869273186, -0.027806038036942482, 8.427870488958433e-05, 0.023009954020380974, -0.026357367634773254, -0.025372834876179695, 0.0009801381966099143, -0.004335463512688875, 0.04509163275361061, -0.03293967619538307, 0.020140742883086205, 0.002515834756195545, -0.004085814114660025, 0.006199044641107321, -0.001410871627740562, 0.02624484896659851, 0.01378346886485815, -0.002285524271428585, -0.003994393162429333, -0.017651278525590897, 0.021378440782427788, -0.010893159545958042, -0.005780618172138929, -0.030267372727394104, 0.032855287194252014, 0.008924093097448349, -0.008656862191855907, -0.0007274119998328388, 0.015386851504445076, 0.008474020287394524, -0.022967759519815445, 0.028917154297232628, 0.007320709526538849, -0.018101351335644722, -0.020604878664016724, -0.02482430823147297, 0.0063959513790905476, 0.016877716407179832, 0.0035284976474940777, -0.0007964172400534153, 0.0344868004322052, 0.01715901307761669, -0.005158252082765102, 0.021026821807026863, -0.019198402762413025, -0.011969114653766155, -0.026934023946523666, -0.0012078116415068507, -0.0008223491604439914, 0.05054876208305359, 0.02251768670976162, -0.03150507062673569, 0.006642085034400225, -0.014613290317356586, 0.013931148685514927, -0.02115340530872345, -0.021941032260656357, -0.02552754618227482, -0.019423440098762512, -0.018129481002688408, -0.019915705546736717, -0.015696275979280472, 0.010182889178395271, 0.01728559471666813, 0.021406570449471474, 0.004676533862948418, 0.03184262663125992, -0.016976170241832733, 0.04542918875813484, 0.00532351341098547, -0.04028148576617241, -0.017398113384842873, 0.007071060128509998, 0.0276653915643692, -0.010801739059388638, -0.008895963430404663, 0.02243329957127571, 0.027201253920793533, 0.022250456735491753, 0.008066141977906227, -0.0038220996502786875, 0.02105495147407055, 0.0001253741793334484, -0.016554227098822594, -0.003452899632975459, -0.020056353881955147, 0.001645577372983098, 0.023263119161128998, 0.023867905139923096, 0.03364291414618492, 0.0041244919411838055, -0.037862345576286316, 0.024374235421419144, -0.02002822421491146, -0.012974744662642479, -0.025330640375614166, -0.01776379719376564, -0.004106910899281502, 0.025893229991197586, 0.0028709699399769306, 0.01352327037602663, -0.02012667804956436, 0.017890380695462227, 0.021265923976898193, 0.014669548720121384, 0.02652614563703537, 0.0006192891160026193, 0.005383288487792015, -0.03322097286581993, -0.02143470011651516, 0.011751110665500164, 0.001861823140643537, 0.014683613553643227, 0.019296856597065926, 0.006838991306722164, -0.015696275979280472, -0.0026213203091174364, -0.01507742702960968, 0.014402318745851517, -0.013122424483299255, 0.0060091703198850155, 0.027159059420228004, 0.03161758929491043, 0.030042335391044617, -0.018199805170297623, 0.001641182112507522, -0.018031027168035507, -0.03113938681781292, 0.013417785055935383, -0.04419148713350296, 0.020703332498669624, -0.0010856239823624492, 0.011877693235874176, 0.0033790594898164272, -0.005720842629671097, -0.014950844459235668, -0.020436102524399757, 0.0013387897051870823, 0.00120429543312639, 0.003709581447765231, 0.0075457459315657616, -0.022067613899707794, -0.01146981492638588, 0.0022644270211458206, 0.010485281236469746, 0.001576132606714964, -0.01064702682197094, 0.029085932299494743, 0.016976170241832733, -0.0023470574524253607, -0.023670997470617294, -0.6188496351242065, -0.032292697578668594, -0.0018881945870816708, -0.03206766024231911, -0.0015699792420491576, -0.015907248482108116, -0.018579553812742233, -0.005580195225775242, -0.02303808368742466, 0.038284286856651306, -0.02125185914337635, -0.003692000638693571, 0.01055560540407896, -0.01630106195807457, 0.002658240497112274, -0.0228552408516407, 0.0021519088186323643, -0.02351628616452217, 0.019760994240641594, 0.007320709526538849, -0.011758143082261086, -0.0022943145595490932, 0.002684611827135086, -0.007384001277387142, -0.017130883410573006, -0.002331234747543931, -0.0124824782833457, 0.009451521560549736, 0.009233517572283745, 0.012960679829120636, -0.045907389372587204, 0.01960628107190132, 0.004237010143697262, -0.026174526661634445, 0.04047838971018791, -0.008614667691290379, -0.011631559580564499, 0.018298257142305374, -0.005538000725209713, 0.014085860922932625, -0.023769451305270195, -0.015794729813933372, 0.013178683817386627, 0.013741274364292622, -0.015400916337966919, 0.02902967296540737, 0.02407887578010559, -0.0062799169681966305, -0.02171599492430687, -0.013980375602841377, 0.0038994557689875364, 9.400316776009277e-05, 0.020562684163451195, -0.008305243216454983, 0.001870613661594689, 0.012637190520763397, 0.04036587104201317, -0.01109709870070219, 0.0041104271076619625, 0.006061913445591927, -0.0005656672292388976, 0.010956451296806335, -0.03246147558093071, -0.027960751205682755, -0.026554275304079056, 0.017552824690937996, 0.006575277075171471, 0.012876291759312153, 0.007566843181848526, 0.0006012686644680798, 0.0006219262722879648, 0.0273700300604105, 0.012967712245881557, -0.015949442982673645, -0.003273573936894536, 0.014753937721252441, 0.009887529537081718, -0.008755316026508808, 0.014177282340824604, 0.03184262663125992, 0.01597757264971733, -0.015147751197218895, 0.004535886459052563, -0.009205387905240059, 0.03670903295278549, 0.011476847343146801, -0.0021114726550877094, -0.011279940605163574, 0.027074670419096947, 0.011181487701833248, 0.019198402762413025, 0.012292603962123394, -0.03797486424446106, -0.032911546528339386, 0.014950844459235668, 0.02133624628186226, -0.017215270549058914, 0.012264474295079708, 0.018874913454055786, -0.03232082724571228, -0.015007102862000465, -0.01691991090774536, 0.03226456791162491, 0.008741251192986965, 0.033333491533994675, 0.027271578088402748, -0.03811550885438919, -0.008431825786828995, 0.016216672956943512, -0.034177377820014954, -0.009887529537081718, 0.004964861553162336, -0.016230737790465355, -0.016793327406048775, 0.0190436914563179, -0.025091538205742836, -0.0014706469373777509, -0.01700429990887642, -0.0035232233349233866, -0.008431825786828995, 0.03203953430056572, -0.013881921768188477, 0.009282744489610195, -0.017398113384842873, 0.01880458928644657, 0.026399562135338783, -0.009029578417539597, -0.02469772659242153, -0.01411399058997631, 0.018438905477523804, -0.01486645545810461, -0.010921289213001728, 0.012524672783911228, -0.015414981171488762, 0.003029198618605733, 0.013157586567103863, 0.04433213546872139, 0.004757406655699015, 0.007320709526538849, -0.013600626960396767, -0.02661053277552128, -0.009071772918105125, 0.018452970311045647, -0.004258107393980026, -0.03904378414154053, -0.031111259013414383, -0.019395310431718826, 0.007974721491336823, 0.00753871351480484, -0.007180062122642994, -0.01479613222181797, -0.009226485155522823, 0.008867833763360977, 0.018692070618271828, 0.0015866812318563461, -0.011504977010190487, -0.005668099969625473, -0.029986076056957245, -0.004170202650129795, -0.016512032598257065, 0.008248983882367611, -0.0002192125393776223, -0.007000736426562071, 0.0017906202701851726, 0.004426884464919567, -0.010703285224735737, -0.022967759519815445, 0.027679456397891045, -0.025640064850449562, -0.02097056247293949, -0.004455014131963253, -0.03530255705118179, 0.007039414253085852, 0.012243377044796944, -0.014282767660915852, 0.02431797794997692, -0.003632225329056382, -0.004908602684736252, -0.007876267656683922, -0.0011735287262126803, 0.0032014918979257345, 0.030970610678195953, 0.00027382338885217905, -0.02171599492430687, 0.01592131331562996, 0.01710275374352932, 0.037862345576286316, 0.014331994578242302, -0.021125275641679764, 0.010991613380610943, 0.016512032598257065, 0.016877716407179832, -0.022630205377936363, 0.033389750868082047, -0.017355918884277344, 0.011118195950984955, 0.007278515491634607, -0.01790444552898407, 0.013108359649777412, 0.03755291923880577, 0.009078805334866047, 0.015400916337966919, -0.016793327406048775, -0.010133662261068821, 0.010829868726432323, -0.03029550239443779, 0.012095697224140167, -0.019198402762413025, -0.010548572987318039, -0.0057243588380515575, 0.003667387180030346, -0.006547147873789072, -0.008874866180121899, -0.012032405473291874, 0.0010363972978666425, 0.031589459627866745, -0.005727875046432018, 0.007447292562574148, -0.026357367634773254, 0.0109775485470891, 0.003607611870393157, 0.017046494409441948, -0.013649853877723217, 0.012855194509029388, -0.010407925583422184, 0.004834762774407864, -0.020154807716608047, 0.02073146216571331, -0.007482454646378756, -0.03133629262447357, 0.004335463512688875, 0.009627331048250198, 0.032770901918411255, 0.004455014131963253, 0.023009954020380974, -0.01798883266746998, 0.02247549220919609, 0.001355491578578949, 0.044219616800546646, -0.005696229636669159, -0.007285547908395529, 0.0034511415287852287, 2.4956714696600102e-05, -0.031083129346370697, 0.012749708257615566, -0.010112565010786057, 0.03831241652369499, -0.014395286329090595, -0.017173076048493385, -0.005541516933590174, -0.03673716261982918, 0.011504977010190487, -0.0021765222772955894, 0.005056282505393028, 0.02925470843911171, -0.009444489143788815, 0.004764438606798649, 0.0032120405230671167, 0.015358722768723965, 0.0036181604955345392, -0.013417785055935383, 0.0012675868347287178, -0.002943051978945732, 0.015302463434636593, 0.02209574356675148, -0.005077379755675793, -0.003825615858659148, -0.011554203927516937, -0.007000736426562071, -0.013860825449228287, -0.02111121080815792, -0.01001411210745573, 0.024866502732038498, -0.04067529737949371, 0.042700622230768204, 0.020886175334453583, 0.004103394690901041, 0.014894585125148296, -0.003488061483949423, 0.03203953430056572, -0.024852437898516655, -0.03777795657515526, 0.014613290317356586, -0.008635764941573143, -0.03198327496647835, -0.019662540405988693, -0.018579553812742233, 0.007405098062008619, -0.0017202964518219233, 0.00044391912524588406, -0.008895963430404663, -0.008544344455003738, -0.012299636378884315, 0.01227150671184063, 0.00034810291253961623, 0.008094271644949913, 0.014683613553643227, -0.006234206724911928, 0.008052077144384384, -0.020759591832756996, 0.02016887255012989, 0.012257441878318787, -0.00979610811918974, -0.005608324892818928, 0.011645624414086342, -0.01616041362285614, -0.03367104381322861, -0.00027338386280462146, 0.00030239243642427027, -0.033896081149578094, 0.0037728729657828808, -0.007362904027104378, 0.008558409288525581, -0.041491053998470306, 0.015597823075950146, 0.012728611938655376, 0.02039390802383423, 0.0034300442785024643, 0.019437503069639206, 0.013530302792787552, -0.007890332490205765, 0.002086859429255128, -0.010949418880045414, 0.019001496955752373, 0.048551566898822784, 0.018931172788143158, 0.003938133828341961, 0.0009801381966099143, -0.0057524885050952435, 0.003681452013552189, -0.024613337591290474, 0.0019743412267416716, 0.00083157914923504, 0.010365731082856655, -0.016371386125683784, -0.019578151404857635, 0.01804509200155735, 0.000988049665465951, 0.04005644842982292, 0.003213798627257347, 0.010351666249334812, -0.013811598531901836, -0.00954294204711914, -0.013094295747578144, 0.014444512315094471, -0.0026828537229448557, 0.010239148512482643, 0.024416429921984673, 0.011554203927516937, -0.009922690689563751, 0.007974721491336823, 0.028326435014605522, -0.006793281063437462, -0.021448764950037003, 0.012166020460426807, 0.004585112910717726, 0.008741251192986965, 0.008649829775094986, -0.016933975741267204, 0.011322135105729103, -0.0005085291340947151, -0.003305219579488039, 0.014036634936928749, 0.015344657935202122, 0.004138556774705648, 0.02450081892311573, 0.008023947477340698, 0.013502173125743866, 0.009824237786233425, -0.014472641982138157, -0.003920552786439657, 0.010140694677829742, -0.007004252634942532, -0.0392969511449337, 0.0036533225793391466, 0.004820697940886021, -0.040872205048799515, -0.0015937135322019458, 0.028256110846996307, 0.006543631665408611, 0.0013546126428991556, -0.011371362023055553, -0.00489805405959487, -0.03575263172388077, -0.00665614940226078, -0.030351761728525162, 0.006933928467333317, -0.015302463434636593, -0.004057684447616339, -0.003370269201695919, -0.00360409589484334, 0.0011708915699273348, -0.01611821912229061, 0.00015108633670024574, -0.016286997124552727, -0.011153358034789562, -0.05805934593081474, -0.013396687805652618, -0.0014935021754354239, -0.0040471358224749565, 0.010639994405210018, -0.002587916562333703, 0.0031751205679029226, 0.021941032260656357, -0.008895963430404663, -0.02040797285735607, 0.0025527547113597393, -0.01992977038025856, 0.005615356843918562, -0.023305313661694527, -0.0038642939180135727, -0.02077365666627884, -0.012384024448692799, 0.022264521569013596, -0.008769379928708076, -0.013797533698379993, -0.020182935521006584, 0.0024630918633192778, 0.005253189243376255, 0.02464146725833416, 0.016174478456377983, 0.006772183813154697, 0.022925565019249916, -0.02337563782930374, -0.0009230001596733928, -0.01850922964513302, -0.019845381379127502, -0.03575263172388077, 0.003143474692478776, 0.008396663703024387, 0.03558385372161865, 0.008488085120916367, 0.010316504165530205, -0.020858045667409897, 0.0006166520179249346, -0.014233541674911976, 0.012264474295079708, -0.0017071107868105173, 0.01286222692579031, -0.014015537686645985, 0.012151956558227539, -0.0037974861916154623, -0.005457128398120403, -0.029901688918471336, 0.01696210540831089, -0.030182983726263046, 0.02012667804956436, 0.01038682833313942, 0.00979610811918974, 0.01720120571553707, 0.022405169904232025, 0.002146634506061673, -0.045260410755872726, 0.00016075585153885186, 0.0010592525359243155, 0.014310897327959538, -0.0038783587515354156, -0.011125228367745876, -0.00249122129753232, -0.014753937721252441, -0.015246204100549221, 0.00893112551420927, -0.02469772659242153, 0.006431113462895155, -0.015218074433505535, -0.004261623602360487, -0.009317906573414803, -0.01479613222181797, 0.004824214149266481, -0.029339097440242767, -0.009233517572283745, 0.029142191633582115, 0.016807392239570618, 0.031026870012283325, -0.013586562126874924, 0.01060483232140541, -0.03825615718960762, -0.016371386125683784, 0.0005313843721523881, -0.023614738136529922, -0.010478248819708824, -0.006104107480496168, 0.011540139093995094, 0.016427643597126007, 0.021842578426003456, 0.005741939879953861, 0.017580954357981682, 0.015696275979280472, -0.0022556365001946688, -0.006336176302284002, -0.010316504165530205, 0.00018888538761530071, -0.02040797285735607, -0.013234943151473999, 0.012672352604568005, -0.003428286174312234, 0.009690622799098492, 0.004658953286707401, -0.0021817965898662806, 0.0020112611819058657, -0.01486645545810461, -0.004092846531420946, -0.018340451642870903, -0.041097242385149, -0.02139250561594963, 0.008445890620350838, -0.006329143885523081, -0.0020851013250648975, -0.04621681571006775, -0.017679408192634583, 0.018340451642870903, 0.008248983882367611, 0.014627354219555855, 0.012151956558227539, 0.03699032962322235, -0.0291703213006258, -0.002804162446409464, 0.029929818585515022, 0.016849586740136147, -0.008066141977906227, -0.007503551431000233, -0.04402271285653114, 0.013755339197814465, 0.015274333767592907, 0.007376968860626221, 0.040731556713581085, 0.026216719299554825, -0.015386851504445076, 0.016610486432909966, 0.014247605577111244, 0.003319284413009882, 0.010077403858304024, -0.0047187283635139465, -0.010288374498486519, 0.0036427739541977644, -0.012334798462688923, -0.025175927206873894, -0.023670997470617294, -0.019156208261847496, 0.011209617368876934, 0.013206813484430313, 0.030829962342977524, -0.02473991923034191, -0.02039390802383423, 0.02393822930753231, 0.008628732524812222, 0.05066128075122833, 0.013502173125743866, 0.008874866180121899, 0.02379758097231388, 0.009810172952711582, -0.010625929571688175, 0.011385426856577396, 0.01260202843695879, -0.014268702827394009, 0.020759591832756996, 0.01578066498041153, 0.002853388898074627, -0.009739848785102367, 0.012285571545362473, 0.010119597427546978, -0.03924069181084633, -0.04843904823064804, 0.00979610811918974, 0.02147689461708069, 0.005604808684438467, -0.03597766533493996, -0.03029550239443779, -0.025316575542092323, -0.0006333538913168013, -0.02552754618227482, 0.023066213354468346, 0.010232116095721722, -0.00808020681142807, 0.011174455285072327, 0.008938157930970192, 0.010253213346004486, -0.0006386282038874924, -0.004715212155133486, 0.0220535509288311, 0.004750374238938093, 0.0012702239910140634, -0.003263025311753154, 0.010211018845438957, -0.006227174308151007, 0.03763730823993683, -0.009339002892374992, 0.011589366011321545, 0.01029540691524744, -0.0010935354512184858, 0.006272884551435709, -0.008115368895232677, 0.004416335839778185, 0.03701845929026604, -0.035499464720487595, -0.010935354046523571, -0.008136466145515442, -0.016497967764735222, 0.01635732129216194, -0.01556969340890646, 0.014142120257019997, -0.004500724375247955, -0.013931148685514927, 0.003860777709633112, 0.02383977547287941, 0.02383977547287941, -0.03203953430056572, 0.0069655743427574635, 0.00284987292252481, -0.032348956912755966, -0.02708873525261879, 0.0057384236715734005, 0.032489605247974396, -0.037384141236543655, 0.007679361384361982, 0.00850214995443821, -0.021589413285255432, 0.021279988810420036, 0.021181534975767136, -0.0037447435315698385, -0.029282838106155396, 0.03237708657979965, -0.03381169214844704, 0.0217581894248724, 0.0013704354641959071, -0.010119597427546978, -0.026891829445958138, -0.0018547907238826156, -0.014978974126279354, -0.029085932299494743, 0.004535886459052563, -0.025316575542092323, -0.022123873233795166, -0.011715948581695557, 0.016807392239570618, 0.006733505986630917, -0.01059779990464449, 0.013333396054804325, 0.0033526881597936153, 0.003055569948628545, -0.0007427953532896936, -0.015907248482108116, -0.001497897319495678, 0.02303808368742466, 0.0010144211119040847, 0.0036533225793391466, 0.03043614886701107, -0.0035232233349233866, 0.012116794474422932, -0.013910051435232162, -0.005288351327180862, -0.009866432286798954, -0.05817186459898949, -0.02082991600036621, 0.009810172952711582, 0.0017132640350610018, -2.734662666625809e-05, 0.0012798935640603304, -0.016807392239570618, -0.010351666249334812, -0.012566866353154182, -0.002035874640569091, 0.010140694677829742, 0.0028692118357867002, 0.00724335340783, 0.023263119161128998, -0.0001748206268530339, 0.008523247204720974, -0.017918508499860764, -0.020562684163451195, -0.018973367288708687, -0.003681452013552189, -0.03175823763012886, -0.0187483299523592, -0.033614784479141235, 0.05037998408079147, -0.006336176302284002, -0.03887500613927841, -0.021097145974636078, -0.01507742702960968, -0.038424935191869736, -0.020464232191443443, 0.0025597871281206608, 0.01822793483734131, 0.012883324176073074, 0.01691991090774536, 0.002827017568051815, 0.03586514666676521, 0.006561212241649628, 0.004992991220206022, -0.026962153613567352, 0.01293255016207695, -0.0007502672378905118, -0.013635789044201374, 0.01276377309113741, 0.01776379719376564, -0.021069016307592392, 0.01860768347978592, 0.012095697224140167, -0.027806038036942482, -0.024247653782367706, 0.026934023946523666, -0.01507742702960968, -0.010042241774499416, 0.021884772926568985, -0.009078805334866047, 0.007021833676844835, -0.0124824782833457, 0.020281389355659485, -0.03133629262447357, -0.009324938990175724, 0.026596467941999435, -0.006592858117073774, 0.01635732129216194, -0.015035232529044151, 0.017018364742398262, 0.009036610834300518, 0.008734218776226044, 0.01610415428876877, -0.012144924141466618, 0.006304530426859856, 0.004483143333345652, 0.044078972190618515, 0.01936718076467514, 0.010464184917509556, 0.029367227107286453, -0.016737069934606552, -0.0018125964561477304, -0.0076371668837964535, 0.015007102862000465, 0.0009669525315985084, -0.008741251192986965, 0.04368515685200691, -0.01686365157365799, 0.009852367453277111, -0.010119597427546978, 0.0030696347821503878, -0.022461427375674248, -0.01714494824409485, -0.016216672956943512, -0.0027637260500341654, -0.022700529545545578, 0.014402318745851517, 0.019943835213780403, -0.021786319091916084, -0.0032419280614703894, -0.012876291759312153, 0.02261614054441452, 0.003713097656145692, -0.014142120257019997, -0.0058650067076087, -0.001107600168325007, -0.0034863033797591925, 0.019437503069639206, 0.01006333902478218, -0.006417048629373312, 0.01448670681566, 0.003507400630041957, -0.0218566432595253, 0.015485305339097977, 0.19926957786083221, -0.03164571896195412, 0.01357249729335308, 0.02873431332409382, 0.017046494409441948, -0.016694875434041023, 0.04447278380393982, 0.008213821798563004, 0.0015277849743142724, -0.004535886459052563, 0.006993704009801149, 0.005847425665706396, -0.016526097431778908, 7.318072312045842e-05, 0.019240597262978554, -0.026272978633642197, -0.021322181448340416, -0.03575263172388077, -0.017609084025025368, -0.004008457530289888, 0.006051364820450544, 0.0010768334614112973, -0.007074576336890459, -0.02265833504498005, -0.0030450213234871626, 0.005330545362085104, -4.8842091928236187e-05, -0.003489819588139653, 0.018481099978089333, 0.0006667576963081956, -0.016751134768128395, 0.005154735874384642, -0.008354470133781433, 0.0019479697803035378, 0.00027360362582840025, -0.009704687632620335, 0.010534508153796196, -0.0005085291340947151, 0.006188496015965939, 0.025119667872786522, 0.02199729159474373, 0.021350311115384102, 0.003723646281287074, -0.025049345567822456, 0.015429046005010605, 0.011061936616897583, -0.003920552786439657, 0.008790477178990841, 0.007834073156118393, 0.007398066110908985, -0.022461427375674248, 0.02271459437906742, 0.02341783232986927, 0.020618943497538567, -0.006733505986630917, 0.0007511462899856269, -0.019732864573597908, 0.015021167695522308, 0.00022184968111105263, -0.005073863547295332, -0.027313772588968277, -0.008509182371199131, -0.00019910431001335382, 0.024416429921984673, -0.010689220391213894, 0.012475445866584778, -0.0075879404321312904, -0.02393822930753231, 0.007953624241054058, -0.029001543298363686, -0.012166020460426807, -0.014782067388296127, 0.0001852593122748658, -0.011540139093995094, -0.019845381379127502, -0.03901565447449684, 0.02209574356675148, 0.02774977870285511, 0.00071290775667876, 0.04866408184170723, -0.003966263495385647, -0.03881875053048134, 0.004648404661566019, -0.012032405473291874, 0.005295383743941784, -0.02389603480696678, 0.009767978452146053, -0.015218074433505535, -0.0217581894248724, -0.011083033867180347, -0.016512032598257065, -0.009500748477876186, 0.004208880476653576, 0.024866502732038498, -1.9572547898860648e-05, 0.004578080493956804, 0.007524648681282997, 0.0060091703198850155, -0.010450120083987713, -0.03113938681781292, -0.007165997289121151, 0.059240784496068954, 0.02133624628186226, 0.0028305337764322758, -0.0009414601372554898, 0.019353115931153297, -0.0032559928949922323, 0.00015734955377411097, -0.0021712479647248983, 0.00013306585606187582, 0.009669525548815727, -0.02794668637216091, 0.011441685259342194, 0.0036111280787736177, -0.0025545128155499697, 0.011962082237005234, 0.011209617368876934, -0.009831270202994347, 0.016413578763604164, -0.010407925583422184, -0.006990187801420689, -0.02223639190196991, -0.0056891972199082375, 0.011448717676103115, -3.656289118225686e-05, -0.0274684838950634, 0.006751086562871933, 0.0009871706133708358, -0.027693521231412888, 0.01706055924296379, 0.0004386448417790234, -0.04005644842982292, 0.009353067725896835, 0.003994393162429333, 0.0006364305736497045, -0.008614667691290379, 0.014824260957539082, -0.006733505986630917, -0.02864992432296276, 0.013600626960396767, -0.00876234844326973, 0.024289848282933235, 0.004683566279709339, -0.02147689461708069, 0.01436715666204691, -0.014542966149747372, 0.002616046229377389, 0.015583758242428303, -0.013052101247012615, -0.019915705546736717, -0.020014159381389618, 0.008023947477340698, 0.00514770345762372, 0.009641395881772041, -0.0008403696701861918, 0.024374235421419144, -0.01189879048615694, -0.0006496162968687713, 0.022827111184597015, 0.008509182371199131, -0.028284240514039993, 0.0031803948804736137, 0.01556969340890646, 0.006895250640809536, -0.015907248482108116, -0.0070183174684643745, -0.18081660568714142, -0.015063362196087837, 0.011385426856577396, -0.040309615433216095, 0.018438905477523804, 0.006192012224346399, 0.012721579521894455, 0.007735620252788067, -0.010921289213001728, 0.0010759544093161821, 0.021701930090785027, -0.01852329447865486, -0.004891021642833948, -0.0062166256830096245, -0.0007252143695950508, 0.007932526990771294, -0.008790477178990841, 0.009620298631489277, 0.012327766045928001, 0.0008922334527596831, 0.03150507062673569, -0.027018411085009575, 0.009901593439280987, -0.00951481331139803, 0.017820056527853012, 0.012398089282214642, 0.00930384173989296, 0.04239119961857796, -0.017130883410573006, -0.03133629262447357, -0.01114632561802864, -0.01469767838716507, 0.0291703213006258, 0.006420564837753773, 0.03125190734863281, 0.030829962342977524, 0.011744078248739243, -0.019353115931153297, -0.018551424145698547, -0.015893183648586273, 0.02227858640253544, 0.013445914722979069, 0.02421952411532402, -0.008614667691290379, -0.03189888596534729, 0.007651231717318296, 0.007433227729052305, -0.02153315395116806, -0.0012078116415068507, -0.01303100399672985, 0.016933975741267204, 0.00989456195384264, 0.014317929744720459, -0.004198331851512194, 0.024753984063863754, 0.030492408201098442, -0.005344610195606947, 0.0034810290671885014, -0.006708892527967691, -0.02012667804956436, -0.0005155614926479757, -0.00027514193789102137, 0.0008816848858259618, -0.0005181986489333212, 0.008530279621481895, -0.035218168050050735, -0.008115368895232677, 0.004978926386684179, -0.04008457809686661, 0.0020552135538309813, -0.03386795148253441, 0.003231379436329007, -0.027806038036942482, -0.014360124245285988, 0.0275669377297163, 0.0389031358063221, -0.0305767972022295, 0.016765199601650238, 0.04413522779941559, 0.008263048715889454, -0.01992977038025856, 0.002658240497112274, -0.0190436914563179, 0.021139340475201607, -0.004001425579190254, 0.0054395473562181, 0.01587911881506443, 0.025738518685102463, -0.01528839860111475, -0.00930384173989296, 0.02002822421491146, -0.010000047273933887, 0.017609084025025368, -0.011561236344277859, -0.0008579505956731737, 0.014085860922932625, 0.010731414891779423, -0.034683708101511, -0.006125204730778933, -0.016455773264169693, 0.007566843181848526, 0.006336176302284002, -0.00893112551420927, -0.010970516130328178, 0.02011261321604252, 0.022222327068448067, -0.02275678887963295, 0.02115340530872345, 0.02841082401573658, -0.0028199851512908936, -0.02007041871547699, -0.008692024275660515, 0.0023910098243504763, 0.0035408043768256903, 0.00166315829847008, -0.0007827920489944518, -0.018579553812742233, -0.02845301851630211, 0.03237708657979965, -0.0019884060602635145, 0.05164581537246704, -0.008368534967303276, 0.0025756098330020905, 0.021983226761221886, -0.009978950023651123, -0.034599319100379944, -0.10464184731245041, -0.03547133505344391, 0.01926872693002224, 0.03142068162560463, -0.0163432564586401, 0.0330803245306015, -0.00514770345762372, 0.04126601666212082, -0.008045044727623463, 0.03966263309121132, -0.005538000725209713, -0.00419481610879302, 0.024135135114192963, 0.01601976715028286, 0.01448670681566, -0.011863628402352333, 0.000535779632627964, -0.015893183648586273, -0.02794668637216091, 0.029929818585515022, 0.017116818577051163, -0.01043605525046587, 0.01050637848675251, -0.03206766024231911, 0.013474044390022755, -0.02030951902270317, -0.03819989785552025, 0.019114013761281967, 0.005682164803147316, -0.0036849682219326496, 0.012039437890052795, -0.015302463434636593, 0.008635764941573143, -0.01648390293121338, 0.010914256796240807, -0.004581596702337265, 0.015400916337966919, -0.01983131654560566, 0.015738470479846, -0.0001563606201671064, -0.01033760141581297, 0.023024018853902817, 0.01227150671184063, -0.00850214995443821, -0.025457223877310753, 0.004444465506821871, -0.009592168964445591, 0.01917027309536934, 0.003646290162578225, -0.028157657012343407, -0.04008457809686661, -0.002616046229377389, -0.04829839989542961, 0.009177258238196373, 0.01340372022241354, -0.010464184917509556, -0.0014407592825591564, 0.025330640375614166, -0.010745479725301266, -0.008973319083452225, -0.0007515858160331845, -0.01818574033677578, -0.01399444043636322, 0.0025580290239304304, -0.014500771649181843, 0.006339692510664463, -0.00867092702537775, -0.0244586244225502, 0.025654129683971405, -0.004630823619663715, 0.0003617281618062407, 0.023333443328738213, -0.012018340639770031, 0.0038502290844917297, -0.016948040574789047, -0.006097075063735247, -0.00090541917597875, -0.0163432564586401, 0.011272908188402653, -0.0011269391980022192, -0.017791926860809326, -0.011526074260473251, 0.01686365157365799, -0.013818630948662758, -0.005214511416852474, 0.011371362023055553, -0.01436715666204691, -0.008656862191855907, 0.0101266298443079, -0.05403682217001915, -0.007250385824590921, 0.03265838325023651, 0.006838991306722164, -0.018073221668601036, 0.023291248828172684, 0.0007696063257753849, -0.011315102688968182, 0.0051125418394804, 0.005956427659839392, 0.0005973129300400615, -0.020984627306461334, -0.021870708093047142, -0.07206784933805466, 0.019845381379127502, 0.0018653393490239978, -0.010421990416944027, 0.014240573160350323, -0.012686417438089848, 0.007531681098043919, -0.010239148512482643, 0.009838302619755268, 0.02658240497112274, -0.02827017568051815, 0.011990210972726345, -0.023249054327607155, -0.03400859981775284, -0.00909287016838789, -0.00926164723932743, 0.018734265118837357, -0.00954294204711914, 0.0049261837266385555, -0.001507566892541945, 0.032995935529470444, -0.02578071318566799, 0.02524625137448311, 0.019001496955752373, -0.00014284526696428657, -0.002306621288880706, -0.043797675520181656, 0.014219476841390133, -0.009831270202994347, -0.01582285948097706, 0.006895250640809536, -0.014057731255888939, 0.013474044390022755, 0.01486645545810461, 0.0017739183967933059, -0.016286997124552727, 0.0008311396231874824, 0.026357367634773254, 0.007946591824293137, 0.03443054109811783, -0.01361469179391861, -0.04272875189781189, 0.015485305339097977, -0.006090042646974325, -0.002596707083284855, -0.008220854215323925, -0.02257394604384899, -0.008755316026508808, 0.04601990804076195, 0.010372763499617577, 0.041716091334819794, 0.00188467837870121, -0.019859446212649345, -0.05068941041827202, -0.011997243389487267, 0.001956760184839368, 0.024796178564429283, -0.009578104130923748, 0.014099925756454468, 0.01291145384311676, 0.03744040057063103, 0.008199757896363735, 0.02040797285735607, -0.015541564673185349, -0.003920552786439657, -0.013537335209548473, -0.004630823619663715, -0.008277113549411297, -0.0011849564034491777, -0.03220830857753754, 0.009985982440412045, -0.004106910899281502, 0.015091491863131523, -0.000993323978036642, 0.0028094365261495113, -0.00954294204711914, 0.001296595437452197, 0.014514836482703686, -0.016765199601650238, 0.01714494824409485, 0.018579553812742233, -0.02516186237335205, -0.002225748961791396, -0.0005595139227807522, 0.02568225935101509, 2.2415717467083596e-05, -0.019353115931153297, 0.00808020681142807, -0.015400916337966919, -0.001359886839054525, 0.0024103489704430103, 0.0016622792463749647, -0.007046446669846773, 0.019156208261847496, 0.0021079564467072487, -0.017876315861940384, -0.001709747826680541, -0.008888931013643742, 0.027876362204551697, -0.008938157930970192, 0.0003894181572832167, -0.010091467760503292, 0.004514789208769798, -0.007496519014239311, -0.007897364906966686, -0.009613266214728355, -0.045204151421785355, -0.021884772926568985, 0.039465729147195816, 0.03831241652369499, 0.015963507816195488, -0.008959254249930382, -0.012130859307944775, 0.004205364268273115, -0.012447316199541092, 0.016427643597126007, -0.004585112910717726, -0.014978974126279354, -0.024613337591290474, 0.03310845419764519, 0.02846708334982395, 0.015555629506707191, 0.05316480994224548, 0.00037579290801659226, 0.0204501673579216, 0.006691311486065388, 0.03043614886701107, -0.0024208975955843925, 0.025119667872786522, 0.02021106518805027, -0.005291867535561323, -0.010520443320274353, -0.014782067388296127, -0.010049274191260338, -0.005232092458754778, -0.013713144697248936, -0.021350311115384102, 0.01728559471666813, 0.012166020460426807, 0.09485276788473129, 0.018171675503253937, -0.01244028378278017, 0.009458553977310658, -0.009268679656088352, 0.0109775485470891, 0.008895963430404663, 0.02303808368742466, 0.0071097384206950665, -0.011315102688968182, -0.002434962196275592, -0.011751110665500164, -0.01244028378278017, -0.014838325791060925, 0.02105495147407055, -0.0037869377993047237, -0.010358698666095734, 0.015625953674316406, 0.005625905469059944, -0.0023329928517341614, 0.0421099029481411, -0.012243377044796944, 0.006339692510664463, 0.005203962791711092, -0.01168078649789095, 0.022180132567882538, 0.029142191633582115, 0.004163170233368874, 0.0028938252944499254, -0.018199805170297623, 0.032770901918411255, 0.01822793483734131, -0.050239335745573044, -0.019381245598196983, -0.009233517572283745, -0.008663894608616829, 0.0049402485601603985, -0.013438882306218147, -0.005485258065164089, 0.005464160814881325, 0.00570326205343008, 0.00209740805439651, -0.014268702827394009, -0.026652727276086807, 0.02516186237335205, -0.012841129675507545, -0.01490864995867014, -0.020225130021572113, -0.028002945706248283], metadata={'title': \"Beyond GPT-4: What's New?\", 'url': 'https://pub.towardsai.net/beyond-gpt-4-whats-new-cbd61a448eb9#dda8', 'source_name': 'towards_ai'}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='doc_0', node_type=<ObjectType.DOCUMENT: '4'>, metadata={'title': \"Beyond GPT-4: What's New?\", 'url': 'https://pub.towardsai.net/beyond-gpt-4-whats-new-cbd61a448eb9#dda8', 'source_name': 'towards_ai'}, hash='3b095b0e25cdf965d950cdbd7feb8024030e7645998c1a33dc4427affca624ab'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='e470fa0d001e50b3ec3088022462a94ea7c87dd80106411b7d120f90b379e977', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='71418de3d50e604c2581574f1abf2248e5cc3ab7c74a3182c37cb1152d0cfd21')}, text='LLM Variants and Meta\\'s Open Source Before shedding light on four major trends, I\\'d share the latest Meta\\'s Llama 2 and Code Llama. Meta\\'s Llama 2 represents a sophisticated evolution in LLMs. This suite spans models pretrained and fine-tuned across a parameter spectrum of 7 billion to 70 billion. A specialized derivative, Llama 2-Chat, has been engineered explicitly for dialogue-centric applications. Benchmarking revealed Llama 2\\'s superior performance over most extant open-source chat models. Human-centric evaluations, focusing on safety and utility metrics, positioned Llama 2-Chat as a potential contender against proprietary, closed-source counterparts. The development trajectory of Llama 2 emphasized rigorous fine-tuning methodologies. Meta\\'s transparent delineation of these processes aims to catalyze community-driven advancements in LLMs, underscoring a commitment to collaborative and responsible AI development. Code Llama is built on top of Llama 2 and is available in three models: Code Llama, the foundational code model;Codel Llama - Python specialized for Python;and Code Llama - Instruct, which is fine-tuned for understanding natural language instructions. Based on its benchmark testing, Code Llama outperformed state-of-the-art publicly available LLMs (except GPT-4) on code tasks. Llama 2, Llama 2-Chat, and Code Llama are key steps in LLM development but still have a way to go compared to GPT-4. Meta\\'s open access and commitment to improving these models promise transparent and faster LLM progress in the future. Please refer to the LLM and Llama variants below:  From LLMs to Multimodal LLMs, like OpenAI\\'s ChatGPT (GPT-3.5), primarily focus on understanding and generating human language. They\\'ve been instrumental in tasks like text generation, translation, and even creative writing. However, their scope is limited to text. Enter multimodal models like GPT-4. These are a new breed of AI models that can understand and generate not just text, but also images, sounds, and potentially other types of data. The term \"multimodal\" refers to their ability to process multiple modes or', start_char_idx=0, end_char_idx=2117, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n')"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "nodes[0]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EV0ll57p46Dc"
      },
      "source": [
        "# Load Indexes\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "HbT3-kRO4Qpt"
      },
      "outputs": [],
      "source": [
        "# Create your index\n",
        "from llama_index.core import VectorStoreIndex\n",
        "\n",
        "index = VectorStoreIndex.from_vector_store(vector_store)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "sb61DWU84bHP"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
            "I0000 00:00:1721833770.545195 6292275 config.cc:230] gRPC experiments enabled: call_status_override_on_cancellation, event_engine_dns, event_engine_listener, http2_stats_fix, monitoring_experiment, pick_first_new, trace_record_callops, work_serializer_clears_time_cache\n",
            "I0000 00:00:1721833770.551333 6292275 check_gcp_environment_no_op.cc:29] ALTS: Platforms other than Linux and Windows are not supported\n"
          ]
        }
      ],
      "source": [
        "from llama_index.llms.gemini import Gemini\n",
        "\n",
        "# Define a query engine that is responsible for retrieving related pieces of text,\n",
        "# and using a LLM to formulate the final answer.\n",
        "\n",
        "llm = Gemini(model=\"models/gemini-1.5-flash\", temperature=1, max_tokens=512)\n",
        "query_engine = index.as_query_engine(llm=llm, similarity_top_k=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "G32W2LMMCmnv"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "I0000 00:00:1721833773.198083 6292275 check_gcp_environment_no_op.cc:29] ALTS: Platforms other than Linux and Windows are not supported\n"
          ]
        }
      ],
      "source": [
        "res = query_engine.query(\"How many parameters LLaMA2 model has?\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "obc20cU5Cxf2",
        "outputId": "df3839ea-527d-4ae3-8a66-2520c643bdf2"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'LLaMA2 model has four different sizes: 7 billion, 13 billion, 34 billion, and 70 billion parameters. \\n'"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "res.response"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oIAO-saJCzYe",
        "outputId": "7aa982e4-bc27-478c-bee1-c72d30a4b9bd"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Node ID\t f707756065d1f788b41fb97fcef81979e1fd241dbfa4034a24bec8e57b648482\n",
            "Title\t Meta's Llama 2: Revolutionizing Open Source Language Models for Commercial Use\n",
            "Text\t I. Llama 2: Revolutionizing Commercial Use Unlike its predecessor Llama 1, which was limited to research use, Llama 2 represents a major advancement as an open-source commercial model. Businesses can now integrate Llama 2 into products to create AI-powered applications. Availability on Azure and AWS facilitates fine-tuning and adoption. However, restrictions apply to prevent exploitation. Companies with over 700 million active daily users cannot use Llama 2. Additionally, its output cannot be used to improve other language models.  II. Llama 2 Model Flavors Llama 2 is available in four different model sizes: 7 billion, 13 billion, 34 billion, and 70 billion parameters. While 7B, 13B, and 70B have already been released, the 34B model is still awaited. The pretrained variant, trained on a whopping 2 trillion tokens, boasts a context window of 4096 tokens, twice the size of its predecessor Llama 1. Meta also released a Llama 2 fine-tuned model for chat applications that was trained on over 1 million human annotations. Such extensive training comes at a cost, with the 70B model taking a staggering 1720320 GPU hours to train. The context window's length determines the amount of content the model can process at once, making Llama 2 a powerful language model in terms of scale and efficiency.  III. Safety Considerations: A Top Priority for Meta Meta's commitment to safety and alignment shines through in Llama 2's design. The model demonstrates exceptionally low AI safety violation percentages, surpassing even ChatGPT in safety benchmarks. Finding the right balance between helpfulness and safety when optimizing a model poses significant challenges. While a highly helpful model may be capable of answering any question, including sensitive ones like \"How do I build a bomb?\", it also raises concerns about potential misuse. Thus, striking the perfect equilibrium between providing useful information and ensuring safety is paramount. However, prioritizing safety to an extreme extent can lead to a model that struggles to effectively address a diverse range of questions. This limitation could hinder the model's practical applicability and user experience. Thus, achieving\n",
            "Score\t 0.7122364245314191\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
            "Node ID\t 636f98cf8754c3a4759da02aa11a3f2aa7cdeb848a4980ec99300ece4a2e92fd\n",
            "Title\t Meta's Llama 2: Revolutionizing Open Source Language Models for Commercial Use\n",
            "Text\t The model demonstrates exceptionally low AI safety violation percentages, surpassing even ChatGPT in safety benchmarks. Finding the right balance between helpfulness and safety when optimizing a model poses significant challenges. While a highly helpful model may be capable of answering any question, including sensitive ones like \"How do I build a bomb?\", it also raises concerns about potential misuse. Thus, striking the perfect equilibrium between providing useful information and ensuring safety is paramount. However, prioritizing safety to an extreme extent can lead to a model that struggles to effectively address a diverse range of questions. This limitation could hinder the model's practical applicability and user experience. Thus, achieving an optimum balance that allows the model to be both helpful and safe is of utmost importance. To strike the right balance between helpfulness and safety, Meta employed two reward models - one for helpfulness and another for safety - to optimize the model's responses. The 34B parameter model has reported higher safety violations than other variants, possibly contributing to the delay in its release.  IV. Helpfulness Comparison: Llama 2 Outperforms Competitors Llama 2 emerges as a strong contender in the open-source language model arena, outperforming its competitors in most categories. The 70B parameter model outperforms all other open-source models, while the 7B and 34B models outshine Falcon in all categories and MPT in all categories except coding. Despite being smaller, Llam a2's performance rivals that of Chat GPT 3.5, a significantly larger closed-source model. While GPT 4 and PalM-2-L, with their larger size, outperform Llama 2, this is expected due to their capacity for handling complex language tasks. Llama 2's impressive ability to compete with larger models highlights its efficiency and potential in the market. However, Llama 2 does face challenges in coding and math problems, where models like Chat GPT 4 excel, given their significantly larger size. Chat GPT 4 performed significantly better than Llama 2 for coding (HumanEval benchmark)and math problem tasks (GSM8k benchmark). Open-source AI technologies, like Llama 2, continue to advance, offering\n",
            "Score\t 0.7047492944862754\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
            "Node ID\t 2f429ec2a936a3dcd37504333de59d17ccd6f07f944ae6f5057aa8d29668662b\n",
            "Title\t Meta's Llama 2: Revolutionizing Open Source Language Models for Commercial Use\n",
            "Text\t with their larger size, outperform Llama 2, this is expected due to their capacity for handling complex language tasks. Llama 2's impressive ability to compete with larger models highlights its efficiency and potential in the market. However, Llama 2 does face challenges in coding and math problems, where models like Chat GPT 4 excel, given their significantly larger size. Chat GPT 4 performed significantly better than Llama 2 for coding (HumanEval benchmark)and math problem tasks (GSM8k benchmark). Open-source AI technologies, like Llama 2, continue to advance, offering strong competition to closed-source models.  V. Ghost Attention: Enhancing Conversational Continuity One unique feature in Llama 2 is Ghost Attention, which ensures continuity in conversations. This means that even after multiple interactions, the model remembers its initial instructions, ensuring more coherent and consistent responses throughout the conversation. This feature significantly enhances the user experience and makes Llama 2 a more reliable language model for interactive applications. In the example below, on the left, it forgets to use an emoji after a few conversations. On the right, with Ghost Attention, even after having many conversations, it will remember the context and continue to use emojis in its response.  VI. Temporal Capability: A Leap in Information Organization Meta reported a groundbreaking temporal capability, where the model organizes information based on time relevance. Each question posed to the model is associated with a date, and it responds accordingly by considering the event date before which the question becomes irrelevant. For example, if you ask the question, \"How long ago did Barack Obama become president?\", its only relevant after 2008. This temporal awareness allows Llama 2 to deliver more contextually accurate responses, enriching the user experience further.  VII. Open Questions and Future Outlook Meta's open-sourcing of Llama 2 represents a seismic shift, now offering developers and researchers commercial access to a leading language model. With Llama 2 outperforming MosaicML's current MPT models, all eyes are on how Databricks will respond. Can MosaicML's next MPT iteration beat Llama 2? Is it worthwhile to compete\n",
            "Score\t 0.7009494958788721\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
            "Node ID\t 9c39187e90964fb2503e9a6f9b6da69b965c9c8b53c57f3c0e4de3593e582bd9\n",
            "Title\t Meta's Llama 2: Revolutionizing Open Source Language Models for Commercial Use\n",
            "Text\t the question, \"How long ago did Barack Obama become president?\", its only relevant after 2008. This temporal awareness allows Llama 2 to deliver more contextually accurate responses, enriching the user experience further.  VII. Open Questions and Future Outlook Meta's open-sourcing of Llama 2 represents a seismic shift, now offering developers and researchers commercial access to a leading language model. With Llama 2 outperforming MosaicML's current MPT models, all eyes are on how Databricks will respond. Can MosaicML's next MPT iteration beat Llama 2? Is it worthwhile to compete with Llama 2 or join hands with the open-source community to make the open-source models better? Meanwhile, Microsoft's move to host Llama 2 on Azure despite having significant investment in ChatGPT raises interesting questions. Will users prefer the capabilities and transparency of an open-source model like Llama 2 over closed, proprietary options? The stakes are high, as Meta's bold democratization play stands to reshape preferences and partnerships in the AI space. One thing is certain - the era of open language model competition has begun.  VIII. Conclusion With the launch of Llama 2, Meta has achieved a landmark breakthrough in open-source language models, unleashing new potential through its commercial accessibility. Llama 2's formidable capabilities in natural language processing, along with robust safety protocols and temporal reasoning, set new benchmarks for the field. While select limitations around math and coding exist presently, Llama 2's strengths far outweigh its weaknesses. As Meta continues honing Llama technology, this latest innovation promises to be truly transformative. By open-sourcing such an advanced model, Meta is propelling democratization and proliferation of AI across industries. From healthcare to education and beyond, Llama 2 stands to shape the landscape by putting groundbreaking language modeling into the hands of all developers and researchers. The possibilities unlocked by this open-source approach signal a shift towards a more collaborative, creative AI future.\n",
            "Score\t 0.6923412027694422\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
            "Node ID\t b9507b49d2385ea4d7c6d656582f18b1a7ae64d6075ce9e2654788b4e8bcae8a\n",
            "Title\t Fine-Tuning a Llama-2 7B Model for Python Code Generation\n",
            "Text\t weights As we mention, we have trained \"modification weights\" on the base model, our final model requires merging the pretrained model and the adapters in a single model. You can find and download the model in my Hugging Face account edumunozsala/llama-27b-int4-python-code-20k. Give it a try!  Inferencing or generating Python code And finally, we will show you how you can download the model from the Hugging Face Hub and call the model to generate an accurate result: Thanks to Maxime Labonne for an excellent article [9] and Philipp Schmid who provides an inspiring code [8]. Their articles are a must-read for everyone interested in Llama 2 and model fine-tuning. And it is all I have to mention, I hope you find useful this article and claps are welcome!! You can Follow me and Subscribe to my articles, or even connect to me via Linkedin. The code is available in my Github Repository.  References [1] Llama-2 paper [2] Link to the original dataset in the Huggingface hub [3] Link to the used dataset in the Huggingface hub [4] Fine-tuning a GPT - LoRA by Chris Kuo/Dr. Dataman [5] Edward J. Hu, Yelong Shen, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, & Weizhu Chen. (2021). LoRA: Low-Rank Adaptation of Large Language Models. arXiv:2106.09685 [6]. QLoRa: Efficient Finetuning of QuantizedLLMs [7] Few-Shot Parameter-Efficient Fine-Tuning is Better and Cheaper than In-Context Learning [8] Extended Guide: Instruction-tune Llama 2 by Philipp Schmid. [9] Fine-Tune Your Own Llama 2 Model in a Colab Notebook by Maxime Labonne [10]. My Github Repository\n",
            "Score\t 0.6846097918258168\n",
            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n"
          ]
        }
      ],
      "source": [
        "for src in res.source_nodes:\n",
        "    print(\"Node ID\\t\", src.node_id)\n",
        "    print(\"Title\\t\", src.metadata[\"title\"])\n",
        "    print(\"Text\\t\", src.text)\n",
        "    print(\"Score\\t\", src.score)\n",
        "    print(\"-_\" * 20)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d4xxZHbdN0lK"
      },
      "source": [
        "# Evaluate the retrieval process and quality of answers\n",
        "\n",
        "We can evaluate our RAG system with a dataset of questions and associated chunks. Given a question, we can see if the RAG system retrieves the correct chunks of text that can answer the question.\n",
        "\n",
        "You can generate a synthetic dataset with an LLM such as `gemini-1.5-flash` or create an authentic and manually curated dataset.\n",
        "\n",
        "Note that a **well curated dataset will always be a better option**, especially for a specific domain or use case.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SuYIj1tD1Hwv"
      },
      "source": [
        "In our example, we will generate a synthetic dataset using `gemini-1.5-flash` to make it simple.\n",
        "\n",
        "This is the default prompt that the `generate_question_context_pairs` function will uses:\n",
        "\n",
        "```python\n",
        "DEFAULT_QA_GENERATE_PROMPT_TMPL = \"\"\"\\\n",
        "Context information is below.\n",
        "\n",
        "---------------------\n",
        "{context_str}\n",
        "---------------------\n",
        "\n",
        "Given the context information and no prior knowledge,\n",
        "generate only questions based on the below query.\n",
        "\n",
        "You are a Teacher/Professor. Your task is to setup \\\n",
        "{num_questions_per_chunk} questions for an upcoming \\\n",
        "quiz/examination. The questions should be diverse in nature \\\n",
        "across the document. Restrict the questions to the \\\n",
        "context information provided.\"\n",
        "\"\"\"\n",
        "```\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jhHLA3he1Hww",
        "outputId": "78e8c284-971e-45d2-a83b-03efa4759856"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "I0000 00:00:1721833843.536908 6292275 check_gcp_environment_no_op.cc:29] ALTS: Platforms other than Linux and Windows are not supported\n",
            "  0%|          | 0/108 [00:00<?, ?it/s]I0000 00:00:1721833843.855004 6292275 check_gcp_environment_no_op.cc:29] ALTS: Platforms other than Linux and Windows are not supported\n",
            "100%|██████████| 108/108 [01:30<00:00,  1.20it/s]\n"
          ]
        }
      ],
      "source": [
        "from llama_index.core.evaluation import generate_question_context_pairs\n",
        "from llama_index.llms.gemini import Gemini\n",
        "\n",
        "llm = Gemini(model=\"models/gemini-1.5-flash\", temperature=1, max_tokens=512)\n",
        "\n",
        "rag_eval_dataset = generate_question_context_pairs(\n",
        "    nodes, llm=llm, num_questions_per_chunk=1\n",
        ")\n",
        "# We can save the dataset as a json file for later use.\n",
        "rag_eval_dataset.save_json(\"./rag_eval_dataset.json\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "mNDd5i921Hww"
      },
      "outputs": [],
      "source": [
        "# We can also load the dataset from a previously saved json file.\n",
        "from llama_index.core.evaluation import EmbeddingQAFinetuneDataset\n",
        "\n",
        "rag_eval_dataset = EmbeddingQAFinetuneDataset.from_json(\"./rag_eval_dataset.json\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qOx3vDWA1Hww"
      },
      "source": [
        "### Evaluation for Hit Rate and Mean Reciprocal Rank (MRR)\n",
        "\n",
        "We will make use of `RetrieverEvaluator` available in Llama-index. We will measure the Hit Rate and Mean Reciprocal Rank (MRR).\n",
        "\n",
        "**Hit Rate:**\n",
        "\n",
        "Think of the Hit Rate like playing a game of guessing. You're given a question and you need to guess the correct answer from a list of options. The Hit Rate measures how often you guess the correct answer by only looking at your top few guesses. If you often find the right answer in your first few guesses, you have a high Hit Rate. So, in the context of a retrieval system, it's about how frequently the system finds the correct document within its top 'k' picks (where 'k' is a number you decide, like top 5 or top 10).\n",
        "\n",
        "**Mean Reciprocal Rank (MRR):**\n",
        "\n",
        "MRR is a bit like measuring how quickly you can find a treasure in a list of boxes. Imagine you have a row of boxes and only one of them has a treasure. The MRR calculates how close to the start of the row the treasure box is, on average. If the treasure is always in the first box you open, you're doing great and have an MRR of 1. If it's in the second box, the score is 1/2, since you took two tries to find it. If it's in the third box, your score is 1/3, and so on. MRR averages these scores across all your searches. So, for a retrieval system, MRR looks at where the correct document ranks in the system's guesses. If it's usually near the top, the MRR will be high, indicating good performance.\n",
        "In summary, Hit Rate tells you how often the system gets it right in its top guesses, and MRR tells you how close to the top the right answer usually is. Both metrics are useful for evaluating the effectiveness of a retrieval system, like how well a search engine or a recommendation system works.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "eARSzx8I1Hww"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "\n",
        "\n",
        "def display_results_retriever(name, eval_results):\n",
        "    \"\"\"Display results from evaluate.\"\"\"\n",
        "\n",
        "    metric_dicts = []\n",
        "    for eval_result in eval_results:\n",
        "        metric_dict = eval_result.metric_vals_dict\n",
        "        metric_dicts.append(metric_dict)\n",
        "\n",
        "    full_df = pd.DataFrame(metric_dicts)\n",
        "\n",
        "    hit_rate = full_df[\"hit_rate\"].mean()\n",
        "    mrr = full_df[\"mrr\"].mean()\n",
        "\n",
        "    metric_df = pd.DataFrame(\n",
        "        {\"Retriever Name\": [name], \"Hit Rate\": [hit_rate], \"MRR\": [mrr]}\n",
        "    )\n",
        "\n",
        "    return metric_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hD5YflG51Hww",
        "outputId": "981e6fc7-911f-4c15-835d-70096c8ba2e3"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "I0000 00:00:1721834042.976124 6292275 work_stealing_thread_pool.cc:320] WorkStealingThreadPoolImpl::PrepareFork\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    Retriever Name  Hit Rate       MRR\n",
            "0  Retriever top_2  0.083333  0.069444\n",
            "    Retriever Name  Hit Rate       MRR\n",
            "0  Retriever top_4  0.092593  0.071759\n",
            "    Retriever Name  Hit Rate       MRR\n",
            "0  Retriever top_6   0.12037  0.076698\n",
            "    Retriever Name  Hit Rate      MRR\n",
            "0  Retriever top_8   0.12963  0.07802\n",
            "     Retriever Name  Hit Rate       MRR\n",
            "0  Retriever top_10  0.148148  0.079872\n"
          ]
        }
      ],
      "source": [
        "from llama_index.core.evaluation import RetrieverEvaluator\n",
        "\n",
        "# We can evaluate the retievers with different top_k values.\n",
        "for i in [2, 4, 6, 8, 10]:\n",
        "    retriever = index.as_retriever(similarity_top_k=i)\n",
        "    retriever_evaluator = RetrieverEvaluator.from_metric_names(\n",
        "        [\"mrr\", \"hit_rate\"], retriever=retriever\n",
        "    )\n",
        "    eval_results = await retriever_evaluator.aevaluate_dataset(\n",
        "        rag_eval_dataset, workers=32\n",
        "    )\n",
        "    print(display_results_retriever(f\"Retriever top_{i}\", eval_results))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9y6uofcJ1Hwx"
      },
      "source": [
        "### Evaluation using Relevance and Faithfulness metrics.\n",
        "\n",
        "Here, we evaluate the answer generated by the LLM. Is the answer using the correct context? Is the answer faithful to the context? Is the answer relevant to the question?\n",
        "\n",
        "An LLM will answer these questions, more specifically `gpt-4o`.\n",
        "\n",
        "**`FaithfulnessEvaluator`**\n",
        "Evaluates if the answer is faithful to the retrieved contexts (in other words, whether there's an hallucination).\n",
        "\n",
        "**`RelevancyEvaluator`**\n",
        "Evaluates whether the retrieved context and answer are relevant to the user question.\n",
        "\n",
        "Now, let's see how the top_k value affects these two metrics.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ckjE4fcD1Hwx",
        "outputId": "6d251465-0e6d-4d67-b608-5c766f1f804c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "top_2 faithfulness_score: 0.9\n",
            "top_2 relevancy_score: 0.55\n",
            "top_4 faithfulness_score: 0.8\n",
            "top_4 relevancy_score: 0.55\n",
            "top_6 faithfulness_score: 0.45\n",
            "top_6 relevancy_score: 0.7\n",
            "top_8 faithfulness_score: 0.75\n",
            "top_8 relevancy_score: 0.6\n",
            "top_10 faithfulness_score: 0.75\n",
            "top_10 relevancy_score: 0.5\n"
          ]
        }
      ],
      "source": [
        "from llama_index.core.evaluation import (\n",
        "    RelevancyEvaluator,\n",
        "    FaithfulnessEvaluator,\n",
        "    BatchEvalRunner,\n",
        ")\n",
        "from llama_index.llms.openai import OpenAI\n",
        "\n",
        "# Define an LLM as a judge\n",
        "llm_gpt4 = OpenAI(temperature=0, model=\"gpt-4o\")\n",
        "\n",
        "# Initiate the faithfulnes and relevancy evaluator objects\n",
        "faithfulness_evaluator = FaithfulnessEvaluator(llm=llm_gpt4)\n",
        "relevancy_evaluator = RelevancyEvaluator(llm=llm_gpt4)\n",
        "\n",
        "# Extract the questions from the dataset\n",
        "queries = list(rag_eval_dataset.queries.values())\n",
        "# Limit to first 20 question to save cost (!!remove this line in production!!)\n",
        "batch_eval_queries = queries[:20]\n",
        "\n",
        "# The batch evaluator runs the evaluation in batches\n",
        "runner = BatchEvalRunner(\n",
        "    {\"faithfulness\": faithfulness_evaluator, \"relevancy\": relevancy_evaluator},\n",
        "    workers=32,\n",
        ")\n",
        "\n",
        "# Define a for-loop to try different `similarity_top_k` values\n",
        "for i in [2, 4, 6, 8, 10]:\n",
        "    # Set query engine with different number of returned chunks\n",
        "    query_engine = index.as_query_engine(similarity_top_k=i, llm=llm)\n",
        "\n",
        "    # Run the evaluation\n",
        "    eval_results = await runner.aevaluate_queries(\n",
        "        query_engine, queries=batch_eval_queries\n",
        "    )\n",
        "\n",
        "    # Printing the results\n",
        "    faithfulness_score = sum(\n",
        "        result.passing for result in eval_results[\"faithfulness\"]\n",
        "    ) / len(eval_results[\"faithfulness\"])\n",
        "    print(f\"top_{i} faithfulness_score: {faithfulness_score}\")\n",
        "\n",
        "    relevancy_score = sum(result.passing for result in eval_results[\"relevancy\"]) / len(\n",
        "        eval_results[\"relevancy\"]\n",
        "    )\n",
        "    print(f\"top_{i} relevancy_score: {relevancy_score}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YmlmP2Px4THB"
      },
      "source": [
        "### Correctness\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "aUulxzuh1Hwx"
      },
      "outputs": [],
      "source": [
        "from llama_index.core.evaluation import CorrectnessEvaluator\n",
        "\n",
        "query = (\n",
        "    \"Can you explain the theory of relativity proposed by Albert Einstein in\" \" detail?\"\n",
        ")\n",
        "\n",
        "reference = \"\"\"\n",
        "Certainly! Albert Einstein's theory of relativity consists of two main components: special relativity and general relativity. Special relativity, published in 1905, introduced the concept that the laws of physics are the same for all non-accelerating observers and that the speed of light in a vacuum is a constant, regardless of the motion of the source or observer. It also gave rise to the famous equation E=mc², which relates energy (E) and mass (m).\n",
        "\n",
        "General relativity, published in 1915, extended these ideas to include the effects of gravity. According to general relativity, gravity is not a force between masses, as described by Newton's theory of gravity, but rather the result of the warping of space and time by mass and energy. Massive objects, such as planets and stars, cause a curvature in spacetime, and smaller objects follow curved paths in response to this curvature. This concept is often illustrated using the analogy of a heavy ball placed on a rubber sheet, causing it to create a depression that other objects (representing smaller masses) naturally move towards.\n",
        "\n",
        "In essence, general relativity provided a new understanding of gravity, explaining phenomena like the bending of light by gravity (gravitational lensing) and the precession of the orbit of Mercury. It has been confirmed through numerous experiments and observations and has become a fundamental theory in modern physics.\n",
        "\"\"\"\n",
        "\n",
        "response = \"\"\"\n",
        "Certainly! Albert Einstein's theory of relativity consists of two main components: special relativity and general relativity. Special relativity, published in 1905, introduced the concept that the laws of physics are the same for all non-accelerating observers and that the speed of light in a vacuum is a constant, regardless of the motion of the source or observer. It also gave rise to the famous equation E=mc², which relates energy (E) and mass (m).\n",
        "\n",
        "However, general relativity, published in 1915, extended these ideas to include the effects of magnetism. According to general relativity, gravity is not a force between masses but rather the result of the warping of space and time by magnetic fields generated by massive objects. Massive objects, such as planets and stars, create magnetic fields that cause a curvature in spacetime, and smaller objects follow curved paths in response to this magnetic curvature. This concept is often illustrated using the analogy of a heavy ball placed on a rubber sheet with magnets underneath, causing it to create a depression that other objects (representing smaller masses) naturally move towards due to magnetic attraction.\n",
        "\"\"\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "CYIjkAP74bly"
      },
      "outputs": [],
      "source": [
        "evaluator = CorrectnessEvaluator(llm=llm_gpt4)\n",
        "\n",
        "result = evaluator.evaluate(\n",
        "    query=query,\n",
        "    response=response,\n",
        "    reference=reference,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-3b-bgvA4dAz",
        "outputId": "7ced2102-6372-4794-82ad-1c7e60438088"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "2.0"
            ]
          },
          "execution_count": 23,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "result.score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "id": "KNEhRQAo4dT0",
        "outputId": "4a5d7db9-b399-49ea-c90e-b1e076640a92"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'The generated answer is mostly relevant but contains significant inaccuracies. It incorrectly states that general relativity involves the effects of magnetism and magnetic fields, which is not true. General relativity deals with the warping of space and time by mass and energy, not magnetic fields. This fundamental error reduces the score significantly.'"
            ]
          },
          "execution_count": 24,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "result.feedback"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZOlwVWZb49H4"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "include_colab_link": true,
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.12.4"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}