worked on lesson How to deploy an open source LLM

notebooks/15-Use_OpenSource_Models.ipynb

@@ -21,18 +21,18 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 6,
+      "execution_count": null,
       "metadata": {
         "id": "QPJzr-I9XQ7l"
       },
       "outputs": [],
       "source": [
-        "!pip install -q llama-index==0.10.11 openai==1.12.0 llama-index-finetuning llama-index-llms-replicate llama-index-embeddings-huggingface llama-index-readers-web tiktoken==0.6.0 chromadb==0.4.22 pandas==2.2.0 html2text sentence_transformers pydantic replicate==0.23.1 kaleido==0.2.1"
+        "!pip install -q llama-index==0.10.11 openai==1.12.0 llama-index-finetuning llama-index-llms-together llama-index-llms-gemini llama-index-embeddings-huggingface llama-index-readers-web llama-index-vector-stores-chroma tiktoken==0.6.0 chromadb==0.4.22 pandas==2.2.0 html2text sentence_transformers pydantic kaleido==0.2.1"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 25,
+      "execution_count": null,
       "metadata": {
         "id": "riuXwpSPcvWC"
       },
@@ -40,121 +40,44 @@
       "source": [
         "import os\n",
         "\n",
-        "# Set the \"OPENAI_API_KEY\" and \"REPLICATE_API_TOKEN\" in the Python environment. Will be used by OpenAI client later.\n",
-        "# You can sign up on https://replicate.com/docs/get-started/python and get a token to use for free for this notebook.\n",
-        "os.environ[\"OPENAI_API_KEY\"] = \"<YOUR_OPENAI_KEY>\"\n",
-        "os.environ[\"REPLICATE_API_TOKEN\"] = \"<YOUR_REPLICATE_KEY>\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 2,
-      "metadata": {
-        "id": "jIEeZzqLbz0J"
-      },
-      "outputs": [],
-      "source": [
-        "# Allows running asyncio in environments with an existing event loop, like Jupyter notebooks.\n",
+        "# Set environment variables for the API keys\n",
+        "os.environ[\"OPENAI_API_KEY\"] = \"<YOUR_API_KEY>\"\n",
+        "os.environ[\"TOGETHER_AI_API_TOKEN\"] = \"<YOUR_API_KEY>\"\n",
+        "os.environ[\"GOOGLE_API_KEY\"] = \"<YOUR_API_KEY>\"\n",
         "\n",
+        "# Allows running asyncio in environments with an existing event loop, like Jupyter notebooks.\n",
         "import nest_asyncio\n",
         "\n",
         "nest_asyncio.apply()"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "Bkgi2OrYzF7q"
-      },
-      "source": [
-        "# Load a Model"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 3,
-      "metadata": {
-        "id": "A1yVgic9DeJ6"
-      },
-      "outputs": [
-        {
-          "name": "stderr",
-          "output_type": "stream",
-          "text": [
-            "/Users/louis/Documents/GitHub/ai-tutor-rag-system/.conda/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
-            "  from .autonotebook import tqdm as notebook_tqdm\n"
-          ]
-        }
-      ],
-      "source": [
-        "from llama_index.core.prompts import PromptTemplate\n",
-        "from llama_index.llms.replicate import Replicate\n",
-        "\n",
-        "# Use the repicate service to access the LLaMA2-70B chat model\n",
-        "llm = Replicate(\n",
-        "    model=\"meta/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf\",\n",
-        "    is_chat_model=True,\n",
-        "    additional_kwargs={\"max_new_tokens\": 512}\n",
-        ")\n"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "0BwVuJXlzHVL"
       },
       "source": [
-        "# Create a VectoreStore"
+        "# Create a vector store and ingest articles"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 4,
+      "execution_count": null,
       "metadata": {
         "id": "SQP87lHczHKc"
       },
       "outputs": [],
       "source": [
         "import chromadb\n",
-        "\n",
-        "# create client and a new collection\n",
-        "# chromadb.EphemeralClient saves data in-memory.\n",
-        "chroma_client = chromadb.PersistentClient(path=\"./mini-llama-articles\")\n",
-        "chroma_collection = chroma_client.create_collection(\"mini-llama-articles\")"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 11,
-      "metadata": {
-        "id": "zAaGcYMJzHAN"
-      },
-      "outputs": [],
-      "source": [
         "from llama_index.vector_stores.chroma import ChromaVectorStore\n",
         "\n",
-        "# Define a storage context object using the created vector database.\n",
+        "# create vector store\n",
+        "vector_store_name = \"mini-llama-articles\"\n",
+        "chroma_client = chromadb.PersistentClient(path=vector_store_name)\n",
+        "chroma_collection = chroma_client.get_or_create_collection(vector_store_name)\n",
         "vector_store = ChromaVectorStore(chroma_collection=chroma_collection)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "I9JbAzFcjkpn"
-      },
-      "source": [
-        "# Load the Dataset (CSV)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ceveDuYdWCYk"
-      },
-      "source": [
-        "## Download"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -166,7 +89,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 5,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -174,17 +97,7 @@
         "id": "wl_pbPvMlv1h",
         "outputId": "244bb539-f023-46f9-bc25-a05d0e526c14"
       },
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
-            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
-            "100  169k  100  169k    0     0   921k      0 --:--:-- --:--:-- --:--:--  926k\n"
-          ]
-        }
-      ],
+      "outputs": [],
       "source": [
         "!curl -o ./mini-llama-articles.csv https://raw.githubusercontent.com/AlaFalaki/tutorial_notebooks/main/data/mini-llama-articles.csv"
       ]
@@ -195,12 +108,12 @@
         "id": "VWBLtDbUWJfA"
       },
       "source": [
-        "## Read File"
+        "## Read articles from file"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 6,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -208,18 +121,7 @@
         "id": "0Q9sxuW0g3Gd",
         "outputId": "8b475ed3-23b1-43e1-c8fc-027fc26455c3"
       },
-      "outputs": [
-        {
-          "data": {
-            "text/plain": [
-              "14"
-            ]
-          },
-          "execution_count": 6,
-          "metadata": {},
-          "output_type": "execute_result"
-        }
-      ],
+      "outputs": [],
       "source": [
         "import csv\n",
         "\n",
@@ -231,10 +133,7 @@
         "\n",
         "  for idx, row in enumerate( csv_reader ):\n",
         "    if idx == 0: continue; # Skip header row\n",
-        "    rows.append( row )\n",
-        "\n",
-        "# The number of characters in the dataset.\n",
-        "len( rows )"
+        "    rows.append(row)"
       ]
     },
     {
@@ -243,168 +142,30 @@
         "id": "S17g2RYOjmf2"
       },
       "source": [
-        "# Convert to Document obj"
+        "## Ingest documents into vector store"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 7,
+      "execution_count": null,
       "metadata": {
         "id": "YizvmXPejkJE"
       },
       "outputs": [],
       "source": [
         "from llama_index.core import Document\n",
+        "from llama_index.core.text_splitter import TokenTextSplitter\n",
+        "from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n",
+        "from llama_index.core.ingestion import IngestionPipeline\n",
         "\n",
         "# Convert the chunks to Document objects so the LlamaIndex framework can process them.\n",
-        "documents = [Document(text=row[1], metadata={\"title\": row[0], \"url\": row[2], \"source_name\": row[3]}) for row in rows]"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "qjuLbmFuWsyl"
-      },
-      "source": [
-        "# Transforming"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 8,
-      "metadata": {
-        "id": "9z3t70DGWsjO"
-      },
-      "outputs": [],
-      "source": [
-        "from llama_index.core.text_splitter import TokenTextSplitter\n",
+        "documents = [Document(text=row[1], metadata={\"title\": row[0], \"url\": row[2], \"source_name\": row[3]}) for row in rows]\n",
         "\n",
         "# Define the splitter object that split the text into segments with 512 tokens,\n",
         "# with a 128 overlap between the segments.\n",
         "text_splitter = TokenTextSplitter(\n",
         "    separator=\" \", chunk_size=512, chunk_overlap=128\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 12,
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/",
-          "height": 650,
-          "referenced_widgets": [
-            "2711e1220eac4e4e8ff6e5fae93c5a1a",
-            "073ca56c113f4dad93b73d9fcd350a66",
-            "9bc95d130d7347548e112c59a113e169",
-            "9fdbd1ce1076410d8265699ff13df861",
-            "c1e0d83bdbb0484983901d6db36dc112",
-            "d237f91523d242f6a11ac294e0832fa2",
-            "c53e80972993487e94fd56cf34302f0b",
-            "2c47558aaa6c44adb2afdb5ec766d8dd",
-            "38efcc43969b40429cf99b03c9f7ccbc",
-            "e973a85fd8ce42c9949074c7747cf467",
-            "b68e7dbe575e4934bbbc67461d2ee167",
-            "a6d1480cebd044ee8ec996c26498a07d",
-            "23ac188881d1484aaf630309809bbc2b",
-            "f75e22947e4b4efcb10f7d157c9fe5d2",
-            "938bdfdc914e44fcb9ae942bb6b74496",
-            "fdc1f2cb889f43a88e0301b29b726657",
-            "d1271b18bd5f4f84aa450f8d58b17774",
-            "34e5dab56e354682adb687ffb19c695d",
-            "a2e452e75f964f96b84f22521f7533a5",
-            "644aeca8e37f4df294cec4a0425587f7",
-            "fe571f5475834c01a22255e68dc782e3",
-            "987018d8d0e34a58a993a836cb3300d4",
-            "9e79a1468ef6452899596ed496801394",
-            "387baf8595754e8e930f36426e9f6758",
-            "55650dcd92f14d5f8e05eda8295e4834",
-            "d4901be0f61c4cfb93fa8f05c0f5bd2f",
-            "3f77b9fcc78c41969ea0f7cffbab2ca9",
-            "2807299dd4f7402d897aae5bc7adefb6",
-            "a882c7b38aed4592944458efb288f025",
-            "846b99723d934015ade4d75987e92340",
-            "4111e05472284375bf6e591b83cdaab9",
-            "06de7570f46644ce89ebac09915d1df5",
-            "8688a4936b9f47c6a86288f6c56fe08c",
-            "ffd5823568564c05b5cd89b04132020a",
-            "5afa1845fd734f0d81c1833615ebcef2",
-            "f3f65dbeccec455cb169fb7b3b2f3748",
-            "99933154a95f4547811b56004ba96c99",
-            "c0a5b64331af4b5e89acc24905fecfc8",
-            "a1c85b0d1291481d984a7cc6009294d1",
-            "cc5115d155534a8b9187efeb3f18b917",
-            "ad4354f9a8134e7ca58571bf10bd0668",
-            "b5325917ecce4469be5f64936d88a9b9",
-            "3f9c588d74ee46ae96689c7112c43291",
-            "59052cd74dfc4da8aeb461ba43d6c1ce",
-            "36a5fc86bdee4f5a98922ae9abd687fd",
-            "830c2d0a44c245b9987ca5b9b3688300",
-            "f919db0110564db9a3f286dc622c48a6",
-            "2edec7086ca64edab9599fb64a73384a",
-            "b68df68efe8b43389d0880af68ebb6ef",
-            "9b7a963b1af749dc945d528ceff0487a",
-            "774b8be0d1cc4f1c9c3dcacdf0724fb7",
-            "b1855af26dcd4d1c8d84ec39144a10ea",
-            "65eb95ad32fd4e52a4367d23aebc7a9f",
-            "96f87f9f50374fb49fa2321f74c40522",
-            "edd7d9ce8a1942628d08ae84bd424ea2",
-            "777a1a034fe54c40a80176aa32de5bef",
-            "bc67387104744a3c8885b5d0a681977d",
-            "6f1fc70b6cf54f5fbe6dcb94828d412e",
-            "6b4ae5c8971f47f0b02a5417cc84e548",
-            "35523981fb7545b5bc70e32eaca6df61",
-            "5562e178fcbb4c3690c4555ca1a25649",
-            "77adc5c66da6481eba92996d3130e7e2",
-            "26bb673b58cd43a09004c15cd2ae0cc7",
-            "7656e8f0910244808ba32c9feb0d1cce",
-            "1c170007cc714f179e6761a26f648928",
-            "3abd1d02caf5489ba66907425f447651",
-            "6bd2c40c93e14c0bb93447af64799be9",
-            "b8111557128040b5a517ba77ebb0a244",
-            "e9ce05dd20304114b148076a5489fdc3",
-            "f35c6cab11eb4eebba14d2650e314daf",
-            "9f1c5dc44df3452d8637639861e5c978",
-            "3962d5f5f940453387e4e38c56c43555",
-            "6c227510710d4364b9983808b5b101f1",
-            "10043e32478b4400abb336d7d0fb6f18",
-            "19264bfe1e6048449dd4467afe582218",
-            "2086d0451a8b4fa2a395ff21e5d51d39",
-            "e53a03ea9bbb46928937065deec1ac08",
-            "73f6fa5f742d4341862f53da4263e5c5",
-            "6bc1d0ab84234e37b2fd34bd520215a9",
-            "679a9cc52b124d1b807d194dfc779e3c",
-            "98f05b81a1ba49baa3f7694f8d26a3ed",
-            "183d8bf73ede4872bab715efa0d011b4",
-            "80818af9902541bc80c1fa55436b6b91",
-            "8a1618e5df38497d92a2be9e3e982688",
-            "c067f30d6a14445dabd68393f736853b",
-            "e395cd15151743889282fff05ef628d4",
-            "0ae8682d08f74b8cbd9091c7d60096f8",
-            "a1f056b411d64699b93fd12da6e10162"
-          ]
-        },
-        "id": "P9LDJ7o-Wsc-",
-        "outputId": "08a795a9-53e3-4a2b-89d2-c0a8912d66b9"
-      },
-      "outputs": [
-        {
-          "name": "stderr",
-          "output_type": "stream",
-          "text": [
-            "Parsing nodes: 100%|██████████| 14/14 [00:00<00:00, 27.58it/s]\n",
-            "Generating embeddings: 100%|██████████| 108/108 [00:05<00:00, 21.15it/s]\n"
-          ]
-        }
-      ],
-      "source": [
-        "from llama_index.core.extractors import (\n",
-        "    SummaryExtractor,\n",
-        "    QuestionsAnsweredExtractor,\n",
-        "    KeywordExtractor,\n",
         ")\n",
-        "from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n",
-        "from llama_index.core.ingestion import IngestionPipeline\n",
         "\n",
         "# Create the pipeline to apply the transformation on each chunk,\n",
         "# and store the transformed text in the chroma vector store.\n",
@@ -416,80 +177,7 @@
         "    vector_store=vector_store\n",
         ")\n",
         "\n",
-        "nodes = pipeline.run(documents=documents, show_progress=True);"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 13,
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        },
-        "id": "mPGa85hM2P3P",
-        "outputId": "56c3980a-38a4-40e7-abdd-84ec1f26cb95"
-      },
-      "outputs": [
-        {
-          "data": {
-            "text/plain": [
-              "108"
-            ]
-          },
-          "execution_count": 13,
-          "metadata": {},
-          "output_type": "execute_result"
-        }
-      ],
-      "source": [
-        "len( nodes )"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 14,
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        },
-        "id": "OeeG3jxT0taW",
-        "outputId": "d1938534-9a12-4f5e-b7e1-5fd58d687d60"
-      },
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "  adding: mini-llama-articles/ (stored 0%)\n",
-            "  adding: mini-llama-articles/01877efc-b4a2-4da3-80b3-93cc40b27067/ (stored 0%)\n",
-            "  adding: mini-llama-articles/01877efc-b4a2-4da3-80b3-93cc40b27067/data_level0.bin"
-          ]
-        },
-        {
-          "name": "stderr",
-          "output_type": "stream",
-          "text": [
-            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
-            "To disable this warning, you can either:\n",
-            "\t- Avoid using `tokenizers` before the fork if possible\n",
-            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
-          ]
-        },
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            " (deflated 57%)\n",
-            "  adding: mini-llama-articles/01877efc-b4a2-4da3-80b3-93cc40b27067/length.bin (deflated 48%)\n",
-            "  adding: mini-llama-articles/01877efc-b4a2-4da3-80b3-93cc40b27067/link_lists.bin (stored 0%)\n",
-            "  adding: mini-llama-articles/01877efc-b4a2-4da3-80b3-93cc40b27067/header.bin (deflated 61%)\n",
-            "  adding: mini-llama-articles/chroma.sqlite3 (deflated 66%)\n"
-          ]
-        }
-      ],
-      "source": [
-        "# Compress the vector store directory to a zip file to be able to download and use later.\n",
-        "!zip -r vectorstore-bge-embedding.zip mini-llama-articles"
+        "nodes = pipeline.run(documents=documents, show_progress=True)"
       ]
     },
     {
@@ -498,50 +186,34 @@
         "id": "OWaT6rL7ksp8"
       },
       "source": [
-        "# Load Indexes"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "RF4U62oMr-iW"
-      },
-      "source": [
-        "If you have already uploaded the zip file for the vector store checkpoint, please uncomment the code in the following cell block to extract its contents. After doing so, you will be able to load the dataset from local storage."
+        "# Load vector store and create query engine"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 15,
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        },
-        "id": "XxPMJ4tq06qx",
-        "outputId": "8445e40a-b3c6-44ff-dfde-37cd4c73ffa2"
-      },
-      "outputs": [],
-      "source": [
-        "# !unzip vectorstore.zip"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 16,
+      "execution_count": null,
       "metadata": {
         "id": "mXi56KTXk2sp"
       },
       "outputs": [],
       "source": [
-        "# Load the vector store from the local storage.\n",
-        "db = chromadb.PersistentClient(path=\"./mini-llama-articles\")\n",
-        "chroma_collection = db.get_or_create_collection(\"mini-llama-articles\")\n",
-        "vector_store = ChromaVectorStore(chroma_collection=chroma_collection)"
+        "from llama_index.core import ServiceContext\n",
+        "from llama_index.core import VectorStoreIndex\n",
+        "\n",
+        "def from_vector_store_to_index(vector_store, llm, embed_model):\n",
+        "    # Define a ServiceContext that uses the BGE model for embedding which will be loaded from Huggingface.\n",
+        "    # The model will be downloaded to your local machine.\n",
+        "    service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model)\n",
+        "\n",
+        "    # Create the index based on the vector store.\n",
+        "    index = VectorStoreIndex.from_vector_store(vector_store, service_context=service_context)\n",
+        "\n",
+        "    return index"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 26,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/",
@@ -618,36 +290,23 @@
         "id": "RZ5iQ_KkJufJ",
         "outputId": "dd6029ee-10ed-4bf8-95d1-88ac5c636c47"
       },
-      "outputs": [
-        {
-          "name": "stderr",
-          "output_type": "stream",
-          "text": [
-            "/var/folders/l7/9qcp7g5x5rl9x8ltw0t85qym0000gn/T/ipykernel_8187/3245113941.py:5: DeprecationWarning: Call to deprecated class method from_defaults. (ServiceContext is deprecated, please use `llama_index.settings.Settings` instead.) -- Deprecated since version 0.10.0.\n",
-            "  service_context = ServiceContext.from_defaults(llm=llm, embed_model=\"local:BAAI/bge-small-en-v1.5\")\n"
-          ]
-        }
-      ],
-      "source": [
-        "from llama_index.core import ServiceContext\n",
-        "\n",
-        "# Define a ServiceContext that uses the BGE model for embedding which will be loads from Huggingface.\n",
-        "# The model will be downloaded in your local machine.\n",
-        "service_context = ServiceContext.from_defaults(llm=llm, embed_model=\"local:BAAI/bge-small-en-v1.5\")"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 27,
-      "metadata": {
-        "id": "jKXURvLtkuTS"
-      },
       "outputs": [],
       "source": [
-        "from llama_index.core import VectorStoreIndex\n",
+        "from llama_index.core.prompts import PromptTemplate\n",
+        "from llama_index.llms.together import TogetherLLM\n",
+        "\n",
+        "# Use the Together AI service to access the LLaMA2-70B chat model\n",
+        "llm = TogetherLLM(\n",
+        "    model=\"meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo\",\n",
+        "    api_key=os.environ[\"TOGETHER_AI_API_TOKEN\"]\n",
+        ")\n",
         "\n",
-        "# Create the index based on the vector store.\n",
-        "index = VectorStoreIndex.from_vector_store(vector_store, service_context=service_context)"
+        "# create index from vector store\n",
+        "index = from_vector_store_to_index(vector_store, llm, \"local:BAAI/bge-small-en-v1.5\")\n",
+        "\n",
+        "# Define a query engine that is responsible for retrieving related pieces of text,\n",
+        "# and using a LLM to formulate the final answer.\n",
+        "query_engine = index.as_query_engine()"
       ]
     },
     {
@@ -656,63 +315,25 @@
         "id": "8JPD8yAinVSq"
       },
       "source": [
-        "# Query Dataset"
+        "# Test query engine"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 28,
-      "metadata": {
-        "id": "8lBu8V7tJ2_8"
-      },
-      "outputs": [],
-      "source": [
-        "# Define a query engine that is responsible for retrieving related pieces of text,\n",
-        "# and using a LLM to formulate the final answer.\n",
-        "query_engine = index.as_query_engine()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 29,
+      "execution_count": null,
       "metadata": {
         "id": "rWAI0jUhJ7qH"
       },
       "outputs": [],
       "source": [
-        "res = query_engine.query(\"How many parameters LLaMA2 has?\")"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 30,
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/",
-          "height": 53
-        },
-        "id": "VKK3jMprctre",
-        "outputId": "06d62444-5e04-4e99-abf0-0ff1c76c0a82"
-      },
-      "outputs": [
-        {
-          "data": {
-            "text/plain": [
-              "'LLaMA2 has four different model sizes: 7 billion, 13 billion, 34 billion, and 70 billion parameters.'"
-            ]
-          },
-          "execution_count": 30,
-          "metadata": {},
-          "output_type": "execute_result"
-        }
-      ],
-      "source": [
-        "res.response"
+        "res = query_engine.query(\"How many parameters LLaMA2 has?\")\n",
+        "print(res.response)\n",
+        "# 'Llama 2 is available in four different model sizes: 7 billion, 13 billion, 34 billion, and 70 billion parameters.'"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 31,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -720,25 +341,9 @@
         "id": "nvSmOtqBoCY2",
         "outputId": "21a60031-4f39-4d1a-fbfa-ddba7d267936"
       },
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "Node ID\t 737ac31e-077f-448e-a777-a096ae0550c0\n",
-            "Title\t Meta's Llama 2: Revolutionizing Open Source Language Models for Commercial Use\n",
-            "Text\t I. Llama 2: Revolutionizing Commercial Use Unlike its predecessor Llama 1, which was limited to research use, Llama 2 represents a major advancement as an open-source commercial model. Businesses can now integrate Llama 2 into products to create AI-powered applications. Availability on Azure and AWS facilitates fine-tuning and adoption. However, restrictions apply to prevent exploitation. Companies with over 700 million active daily users cannot use Llama 2. Additionally, its output cannot be used to improve other language models.  II. Llama 2 Model Flavors Llama 2 is available in four different model sizes: 7 billion, 13 billion, 34 billion, and 70 billion parameters. While 7B, 13B, and 70B have already been released, the 34B model is still awaited. The pretrained variant, trained on a whopping 2 trillion tokens, boasts a context window of 4096 tokens, twice the size of its predecessor Llama 1. Meta also released a Llama 2 fine-tuned model for chat applications that was trained on over 1 million human annotations. Such extensive training comes at a cost, with the 70B model taking a staggering 1720320 GPU hours to train. The context window's length determines the amount of content the model can process at once, making Llama 2 a powerful language model in terms of scale and efficiency.  III. Safety Considerations: A Top Priority for Meta Meta's commitment to safety and alignment shines through in Llama 2's design. The model demonstrates exceptionally low AI safety violation percentages, surpassing even ChatGPT in safety benchmarks. Finding the right balance between helpfulness and safety when optimizing a model poses significant challenges. While a highly helpful model may be capable of answering any question, including sensitive ones like \"How do I build a bomb?\", it also raises concerns about potential misuse. Thus, striking the perfect equilibrium between providing useful information and ensuring safety is paramount. However, prioritizing safety to an extreme extent can lead to a model that struggles to effectively address a diverse range of questions. This limitation could hinder the model's practical applicability and user experience. Thus, achieving\n",
-            "Score\t 0.6191229199592215\n",
-            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
-            "Node ID\t c02a107f-b946-48c6-9180-98b058334b92\n",
-            "Title\t Beyond GPT-4: What's New?\n",
-            "Text\t LLM Variants and Meta's Open Source Before shedding light on four major trends, I'd share the latest Meta's Llama 2 and Code Llama. Meta's Llama 2 represents a sophisticated evolution in LLMs. This suite spans models pretrained and fine-tuned across a parameter spectrum of 7 billion to 70 billion. A specialized derivative, Llama 2-Chat, has been engineered explicitly for dialogue-centric applications. Benchmarking revealed Llama 2's superior performance over most extant open-source chat models. Human-centric evaluations, focusing on safety and utility metrics, positioned Llama 2-Chat as a potential contender against proprietary, closed-source counterparts. The development trajectory of Llama 2 emphasized rigorous fine-tuning methodologies. Meta's transparent delineation of these processes aims to catalyze community-driven advancements in LLMs, underscoring a commitment to collaborative and responsible AI development. Code Llama is built on top of Llama 2 and is available in three models: Code Llama, the foundational code model;Codel Llama - Python specialized for Python;and Code Llama - Instruct, which is fine-tuned for understanding natural language instructions. Based on its benchmark testing, Code Llama outperformed state-of-the-art publicly available LLMs (except GPT-4) on code tasks. Llama 2, Llama 2-Chat, and Code Llama are key steps in LLM development but still have a way to go compared to GPT-4. Meta's open access and commitment to improving these models promise transparent and faster LLM progress in the future. Please refer to the LLM and Llama variants below:  From LLMs to Multimodal LLMs, like OpenAI's ChatGPT (GPT-3.5), primarily focus on understanding and generating human language. They've been instrumental in tasks like text generation, translation, and even creative writing. However, their scope is limited to text. Enter multimodal models like GPT-4. These are a new breed of AI models that can understand and generate not just text, but also images, sounds, and potentially other types of data. The term \"multimodal\" refers to their ability to process multiple modes or\n",
-            "Score\t 0.5952622757578383\n",
-            "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n"
-          ]
-        }
-      ],
+      "outputs": [],
       "source": [
+        "# print the source nodes used to write the answer\n",
         "for src in res.source_nodes:\n",
         "  print(\"Node ID\\t\", src.node_id)\n",
         "  print(\"Title\\t\", src.metadata['title'])\n",
@@ -753,12 +358,12 @@
         "id": "iMkpzH7vvb09"
       },
       "source": [
-        "# Evaluate"
+        "# Evaluate the retriever"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 32,
+      "execution_count": null,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -766,15 +371,7 @@
         "id": "H8a3eKgKvckU",
         "outputId": "a0e0b170-a49e-4c83-fb5a-579046056af5"
       },
-      "outputs": [
-        {
-          "name": "stderr",
-          "output_type": "stream",
-          "text": [
-            "100%|██████████| 108/108 [04:46<00:00,  2.65s/it]\n"
-          ]
-        }
-      ],
+      "outputs": [],
       "source": [
         "from llama_index.core.evaluation import generate_question_context_pairs\n",
         "from llama_index.llms.openai import OpenAI\n",
@@ -790,7 +387,7 @@
         ")\n",
         "\n",
         "# We can save the evaluation dataset as a json file for later use.\n",
-        "rag_eval_dataset.save_json(\"./rag_eval_dataset_bge_embedding.json\")"
+        "rag_eval_dataset.save_json(\"./rag_eval_dataset.json\")"
       ]
     },
     {
@@ -804,100 +401,223 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 22,
+      "execution_count": null,
       "metadata": {
         "id": "3sA1K84U254o"
       },
       "outputs": [],
       "source": [
-        "# from llama_index.finetuning.embeddings.common import (\n",
-        "#     EmbeddingQAFinetuneDataset,\n",
-        "# )\n",
-        "# rag_eval_dataset = EmbeddingQAFinetuneDataset.from_json(\n",
-        "#     \"./rag_eval_dataset_bge_embedding.json\"\n",
-        "# )"
+        "from llama_index.finetuning.embeddings.common import (\n",
+        "    EmbeddingQAFinetuneDataset,\n",
+        ")\n",
+        "rag_eval_dataset = EmbeddingQAFinetuneDataset.from_json(\n",
+        "    \"./rag_eval_dataset.json\"\n",
+        ")"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 33,
+      "execution_count": null,
       "metadata": {
-        "id": "H7ubvcbk27vr"
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "uNLxDxoc2-Ac",
+        "outputId": "f5762021-37d3-423e-b4d6-23b1066a0e97"
       },
       "outputs": [],
       "source": [
-        "import pandas as pd\n",
+        "from llama_index.core.evaluation import RetrieverEvaluator, RelevancyEvaluator, FaithfulnessEvaluator, BatchEvalRunner\n",
+        "from llama_index.llms.openai import OpenAI\n",
+        "\n",
+        "async def run_evaluation(index, rag_eval_dataset, top_k_values, llm_judge, n_queries_to_evaluate=20):\n",
+        "    evaluation_results = {}\n",
+        "\n",
+        "    # ------------------- MRR and Hit Rate -------------------\n",
         "\n",
-        "#  A simple function to show the evaluation result.\n",
-        "def display_results_retriever(name, eval_results):\n",
-        "    \"\"\"Display results from evaluate.\"\"\"\n",
+        "    for top_k in top_k_values:\n",
+        "        # Get MRR and Hit Rate\n",
+        "        retriever = index.as_retriever(similarity_top_k=top_k)\n",
+        "        retriever_evaluator = RetrieverEvaluator.from_metric_names(\n",
+        "            [\"mrr\", \"hit_rate\"], retriever=retriever\n",
+        "        )\n",
+        "        eval_results = await retriever_evaluator.aevaluate_dataset(rag_eval_dataset)\n",
+        "        avg_mrr = sum(res.metric_vals_dict[\"mrr\"] for res in eval_results) / len(eval_results)\n",
+        "        avg_hit_rate = sum(res.metric_vals_dict[\"hit_rate\"] for res in eval_results) / len(eval_results)\n",
         "\n",
-        "    metric_dicts = []\n",
-        "    for eval_result in eval_results:\n",
-        "        metric_dict = eval_result.metric_vals_dict\n",
-        "        metric_dicts.append(metric_dict)\n",
+        "        # Collect the evaluation results\n",
+        "        evaluation_results[f\"mrr_@_{top_k}\"] = avg_mrr\n",
+        "        evaluation_results[f\"hit_rate_@_{top_k}\"] = avg_hit_rate\n",
         "\n",
-        "    full_df = pd.DataFrame(metric_dicts)\n",
+        "    # ------------------- Faithfulness and Relevancy -------------------\n",
         "\n",
-        "    hit_rate = full_df[\"hit_rate\"].mean()\n",
-        "    mrr = full_df[\"mrr\"].mean()\n",
+        "    # Extract the questions from the dataset\n",
+        "    queries = list(rag_eval_dataset.queries.values())\n",
+        "    batch_eval_queries = queries[:n_queries_to_evaluate]\n",
         "\n",
-        "    metric_df = pd.DataFrame(\n",
-        "        {\"Retriever Name\": [name], \"Hit Rate\": [hit_rate], \"MRR\": [mrr]}\n",
+        "    # Initiate the faithfulnes and relevancy evaluator objects\n",
+        "    faithfulness_evaluator = FaithfulnessEvaluator(llm=llm_judge)\n",
+        "    relevancy_evaluator = RelevancyEvaluator(llm=llm_judge)\n",
+        "\n",
+        "    # The batch evaluator runs the evaluation in batches\n",
+        "    runner = BatchEvalRunner(\n",
+        "        {\n",
+        "            \"faithfulness\": faithfulness_evaluator,\n",
+        "            \"relevancy\": relevancy_evaluator\n",
+        "        },\n",
+        "        workers=16,\n",
+        "    )\n",
+        "\n",
+        "    # Get faithfulness and relevancy scores\n",
+        "    query_engine = index.as_query_engine()\n",
+        "    eval_results = await runner.aevaluate_queries(\n",
+        "        query_engine, queries=batch_eval_queries\n",
         "    )\n",
+        "    faithfulness_score = sum(result.passing for result in eval_results['faithfulness']) / len(eval_results['faithfulness'])\n",
+        "    relevancy_score = sum(result.passing for result in eval_results['relevancy']) / len(eval_results['relevancy'])\n",
+        "    evaluation_results[\"faithfulness\"] = faithfulness_score\n",
+        "    evaluation_results[\"relevancy\"] = relevancy_score\n",
         "\n",
-        "    return metric_df"
+        "    return evaluation_results"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 34,
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        },
-        "id": "uNLxDxoc2-Ac",
-        "outputId": "f5762021-37d3-423e-b4d6-23b1066a0e97"
-      },
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "    Retriever Name  Hit Rate       MRR\n",
-            "0  Retriever top_2    0.6625  0.551042\n",
-            "    Retriever Name  Hit Rate       MRR\n",
-            "0  Retriever top_4  0.822917  0.600694\n",
-            "    Retriever Name  Hit Rate       MRR\n",
-            "0  Retriever top_6  0.879167  0.611111\n",
-            "    Retriever Name  Hit Rate       MRR\n",
-            "0  Retriever top_8  0.902083  0.614199\n",
-            "     Retriever Name  Hit Rate       MRR\n",
-            "0  Retriever top_10  0.922917  0.616467\n"
-          ]
-        }
-      ],
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
       "source": [
-        "from llama_index.core.evaluation import RetrieverEvaluator\n",
+        "# We evaluate the retrievers with different top_k values.\n",
+        "top_k_values = [2, 4, 6, 8, 10]\n",
+        "llm_judge = OpenAI(temperature=0, model=\"gpt-4o\")\n",
+        "evaluation_results = await run_evaluation(index, rag_eval_dataset, top_k_values, llm_judge, n_queries_to_evaluate=20)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "print(evaluation_results)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Use GPT-3.5-turbo as the LLM model\n",
+        "llm = OpenAI(model=\"gpt-3.5-turbo\")\n",
         "\n",
-        "# We can evaluate the retievers with different top_k values.\n",
-        "for i in [2, 4, 6, 8, 10]:\n",
-        "    retriever = index.as_retriever(similarity_top_k=i)\n",
-        "    retriever_evaluator = RetrieverEvaluator.from_metric_names(\n",
-        "        [\"mrr\", \"hit_rate\"], retriever=retriever\n",
-        "    )\n",
-        "    eval_results = await retriever_evaluator.aevaluate_dataset(rag_eval_dataset)\n",
-        "    print(display_results_retriever(f\"Retriever top_{i}\", eval_results))"
+        "# create index from vector store\n",
+        "index = from_vector_store_to_index(vector_store, llm, \"local:BAAI/bge-small-en-v1.5\")\n",
+        "\n",
+        "# run evaluation with GPT-3.5-turbo\n",
+        "top_k_values = [2, 4, 6, 8, 10]\n",
+        "llm_judge = OpenAI(temperature=0, model=\"gpt-4o\")\n",
+        "evaluation_results = await run_evaluation(index, rag_eval_dataset, top_k_values, llm_judge, n_queries_to_evaluate=20)"
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
-      "metadata": {
-        "id": "-DMSFJI8F6jl"
-      },
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "print(evaluation_results)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from llama_index.llms.gemini import Gemini\n",
+        "\n",
+        "# Use Gemini as the LLM model\n",
+        "llm = Gemini(model=\"models/gemini-1.5-flash\")\n",
+        "\n",
+        "# create index from vector store\n",
+        "index = from_vector_store_to_index(vector_store, llm, \"local:BAAI/bge-small-en-v1.5\")\n",
+        "\n",
+        "# run evaluation with Gemini\n",
+        "top_k_values = [2, 4, 6, 8, 10]\n",
+        "llm_judge = OpenAI(temperature=0, model=\"gpt-4o\")\n",
+        "evaluation_results = await run_evaluation(index, rag_eval_dataset, top_k_values, llm_judge, n_queries_to_evaluate=20)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "print(evaluation_results)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# Inference speed comparison"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "import time"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "llm = TogetherLLM(\n",
+        "    model=\"meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo\",\n",
+        "    api_key=os.environ[\"TOGETHER_AI_API_TOKEN\"]\n",
+        ")\n",
+        "\n",
+        "time_start = time.time()\n",
+        "llm.complete(\"List the 50 states in the United States of America. Write their names in a comma-separated list and nothing else.\")\n",
+        "time_end = time.time()\n",
+        "print(\"Time taken for Llama 3.1 70B on Together AI: {0:.2f} seconds\".format(time_end - time_start))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
       "outputs": [],
-      "source": []
+      "source": [
+        "llm = OpenAI(model=\"gpt-3.5-turbo\")\n",
+        "\n",
+        "time_start = time.time()\n",
+        "llm.complete(\"List the 50 states in the United States of America. Write their names in a comma-separated list and nothing else.\")\n",
+        "time_end = time.time()\n",
+        "print(\"Time taken for GPT 3.5 Turbo: {0:.2f} seconds\".format(time_end - time_start))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "llm = Gemini(model=\"models/gemini-1.5-flash\")\n",
+        "\n",
+        "time_start = time.time()\n",
+        "llm.complete(\"List the 50 states in the United States of America. Write their names in a comma-separated list and nothing else.\")\n",
+        "time_end = time.time()\n",
+        "print(\"Time taken for Gemini 1.5 Flash: {0:.2f} seconds\".format(time_end - time_start))"
+      ]
     }
   ],
   "metadata": {
@@ -907,7 +627,8 @@
       "provenance": []
     },
     "kernelspec": {
-      "display_name": "Python 3",
+      "display_name": "venv_ai_tutor",
+      "language": "python",
       "name": "python3"
     },
     "language_info": {
@@ -920,7 +641,7 @@
       "name": "python",
       "nbconvert_exporter": "python",
       "pygments_lexer": "ipython3",
-      "version": "3.11.8"
+      "version": "3.11.4"
     },
     "widgets": {
       "application/vnd.jupyter.widget-state+json": {