Created using Colab

notebooks/06-Evaluate_RAG.ipynb

@@ -24,10 +24,10 @@
       "execution_count": 1,
       "metadata": {
         "id": "QPJzr-I9XQ7l",
-        "outputId": "71591538-a161-4a0a-e2c4-057bd2de6941",
         "colab": {
           "base_uri": "https://localhost:8080/"
-        }
+        },
+        "outputId": "71591538-a161-4a0a-e2c4-057bd2de6941"
       },
       "outputs": [
         {
@@ -91,7 +91,7 @@
         "import os\n",
         "\n",
         "# Set the \"OPENAI_API_KEY\" in the Python environment. Will be used by OpenAI client later.\n",
-        "os.environ[\"OPENAI_API_KEY\"] = \"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\""
+        "os.environ[\"OPENAI_API_KEY\"] = \"[YOUR_OPENAI_KEY]\""
       ]
     },
     {
@@ -809,21 +809,20 @@
         }
       ],
       "source": [
-        "from llama_index.core.evaluation import RelevancyEvaluator, FaithfulnessEvaluator, CorrectnessEvaluator, BatchEvalRunner\n",
+        "from llama_index.core.evaluation import RelevancyEvaluator, FaithfulnessEvaluator, BatchEvalRunner\n",
         "from llama_index.llms.openai import OpenAI\n",
         "\n",
         "llm_gpt4 = OpenAI(temperature=0, model=\"gpt-4o\")\n",
         "\n",
         "faithfulness_evaluator = FaithfulnessEvaluator(llm=llm_gpt4)\n",
         "relevancy_evaluator = RelevancyEvaluator(llm=llm_gpt4)\n",
-        "correctness_evaluator = CorrectnessEvaluator(llm=llm_gpt4)\n",
         "\n",
         "# Run evaluation\n",
         "queries = list(rag_eval_dataset.queries.values())\n",
         "batch_eval_queries = queries[:20]\n",
         "\n",
         "runner = BatchEvalRunner(\n",
-        "{\"faithfulness\": faithfulness_evaluator, \"relevancy\": relevancy_evaluator, \"correctness\": correctness_evaluator},\n",
+        "{\"faithfulness\": faithfulness_evaluator, \"relevancy\": relevancy_evaluator},\n",
         "workers=32,\n",
         ")\n",
         "\n",
@@ -838,10 +837,7 @@
         "    print(f\"top_{i} faithfulness_score: {faithfulness_score}\")\n",
         "\n",
         "    relevancy_score = sum(result.passing for result in eval_results['relevancy']) / len(eval_results['relevancy'])\n",
-        "    print(f\"top_{i} relevancy_score: {relevancy_score}\")\n",
-        "\n",
-        "    correctness = sum(result.passing for result in eval_results['correctness']) / len(eval_results['correctness'])\n",
-        "    print(f\"top_{i} correctness: {correctness}\")\n"
+        "    print(f\"top_{i} relevancy_score: {relevancy_score}\")\n"
       ]
     },
     {