Repository: GoogleCloudPlatform/dfcx-scrapi (language: python)

Issue:
feat: Add 1-click buttons to colabs
feat: Add 1-click buttons to colabs


Discussion / hints:

Fix patch:
diff --git a/.gitignore b/.gitignore
index 0bdbf412..cf1d464f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,7 @@
 .ipynb_checkpoints
 *scratch.ipynb
 *scratch.py
+tmp/
 
 # IPython
 profile_default/
@@ -64,6 +65,7 @@ docs.metadata
 # Virtual environment
 env/
 venv/
+.python-version
 
 coverage.xml
 sponge_log.xml
diff --git a/.pylintrc b/.pylintrc
index e1ec406e..4a16e611 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -421,6 +421,6 @@ valid-metaclass-classmethod-first-arg=mcs
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "Exception"
-overgeneral-exceptions=StandardError,
-                       Exception,
-                       BaseException
+overgeneral-exceptions=builtins.StandardError,
+                       builtins.Exception,
+                       builtins.BaseException
diff --git a/data/nlu_evals_sample.csv b/data/nlu_evals_sample.csv
new file mode 100644
index 00000000..f6d1e45e
--- /dev/null
+++ b/data/nlu_evals_sample.csv
@@ -0,0 +1,19 @@
+flow_display_name,utterance,page_display_name,expected_intent,expected_parameters,description
+Default Start Flow,I need to get my order status,START_PAGE,head_intent.order_status,,Demo Tests
+Default Start Flow,Trying to check the status of my order,START_PAGE,head_intent.order_status,,Demo Tests
+Default Start Flow,I hate this order status agent!,START_PAGE,head_intent.order_status,,Demo Tests
+Default Start Flow,Wha'ts the point of ordering anything?,START_PAGE,NO_MATCH,,Demo Tests
+Default Start Flow,I was looking at the order of operations yesterday but couldn't figure it out,START_PAGE,NO_MATCH,,Demo Tests
+Default Start Flow,Thanks for getting my that status so quickly!,START_PAGE,head_intent.order_status,,Demo Tests
+Default Start Flow,I need to make a payment,START_PAGE,head_intent.pay_arrangement,,Demo Tests
+Default Start Flow,I'm trying to setup a new payment,START_PAGE,head_intent.pay_arrangement,,Demo Tests
+Default Start Flow,Did you get the pavement,START_PAGE,NO_MATCH,,Demo Tests
+Default Start Flow,I've been trying to pay my bill all day!,START_PAGE,head_intent.pay_arrangement,,Demo Tests
+Default Start Flow,Why can't I get my bill paid at all online?,START_PAGE,head_intent.pay_arrangement,,Demo Tests
+Default Start Flow,Why can't I get my bill paid at all online? Why can't I get my bill paid at all online?Why can't I get my bill paid at all online?Why can't I get my bill paid at all online?Why can't I get my bill paid at all online?Why can't I get my bill paid at all online?Why can't I get my bill paid at all online?Why can't I get my bill paid at all online?Why can't I get my bill paid at all online?,START_PAGE,head_intent.pay_arrangement,,Demo Tests
+Date Collection,12/13/2022,Collect Date,-,"{'collected_date':
+{ ""day"": 13, ""month"": 12, ""year"": 2022 }, 'formatted_date': '13/12/2022'}}",Demo Tests
+Proper Names Demo,Patrick Marlow,names,-,"{'people_names':
+{ ""original"": ""Patrick Marlow"", ""name"": ""Patrick Marlow"" }}",Demo Tests
+Proper Names Demo,Andrew Smith,names,,"{'people_names':
+{ ""original"": ""Andrew Smith"", ""name"": ""Andrew Smith"" }}",Demo Tests
diff --git a/examples/bot_building_series/bot_building_101.ipynb b/examples/bot_building_series/bot_building_101.ipynb
index aee83c78..1df7ddec 100644
--- a/examples/bot_building_series/bot_building_101.ipynb
+++ b/examples/bot_building_series/bot_building_101.ipynb
@@ -6,7 +6,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Copyright 2023 Google LLC\n",
+    "# Copyright 2024 Google LLC\n",
     "#\n",
     "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
     "# you may not use this file except in compliance with the License.\n",
@@ -29,7 +29,25 @@
     "In this notebook, we will show you how to build a Dialogflow CX Agent from scratch using plain text inputs and a simple CSV file.\n",
     "\n",
     "## Prerequisites\n",
-    "- Ensure you have a GCP Service Account key with the Dialogflow API Admin privileges assigned to it."
+    "- Ensure you have a GCP Service Account key with the Dialogflow API Admin privileges assigned to it.\n",
+    "\n",
+    "<table align=\"left\">\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/bot_building_series/bot_building_101.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Google Colaboratory logo\"><br> Run in Colab\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://github.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/bot_building_series/bot_building_101.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"><br> View on GitHub\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/bot_building_series/bot_building_101.ipynb\">\n",
+    "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
+    "    </a>\n",
+    "  </td>\n",
+    "</table>"
    ]
   },
   {
diff --git a/examples/bot_building_series/bot_building_101.py b/examples/bot_building_series/bot_building_101.py
index 1be38b5a..24673c95 100644
--- a/examples/bot_building_series/bot_building_101.py
+++ b/examples/bot_building_series/bot_building_101.py
@@ -1,5 +1,5 @@
 """A sample script for building a DFCX bot from scratch using simple inputs."""
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/examples/bot_building_series/bot_building_102_intents_with_annotated_tp.ipynb b/examples/bot_building_series/bot_building_102_intents_with_annotated_tp.ipynb
index 701f9013..5de6e60c 100644
--- a/examples/bot_building_series/bot_building_102_intents_with_annotated_tp.ipynb
+++ b/examples/bot_building_series/bot_building_102_intents_with_annotated_tp.ipynb
@@ -6,7 +6,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Copyright 2023 Google LLC\n",
+    "# Copyright 2024 Google LLC\n",
     "#\n",
     "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
     "# you may not use this file except in compliance with the License.\n",
@@ -31,7 +31,25 @@
     "In this example some of the training phrases will be anotated with the @sys.number entity\n",
     "\n",
     "## Prerequisites\n",
-    "- Ensure you have a GCP Service Account key with the Dialogflow API Admin privileges assigned to it."
+    "- Ensure you have a GCP Service Account key with the Dialogflow API Admin privileges assigned to it.\n",
+    "\n",
+    "<table align=\"left\">\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/bot_building_series/bot_building_102_intents_with_annotated_tp.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Google Colaboratory logo\"><br> Run in Colab\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://github.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/bot_building_series/bot_building_102_intents_with_annotated_tp.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"><br> View on GitHub\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/bot_building_series/bot_building_102_intents_with_annotated_tp.ipynb\">\n",
+    "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
+    "    </a>\n",
+    "  </td>\n",
+    "</table>"
    ]
   },
   {
diff --git a/examples/copy_paste_series/copy_paste_pages.ipynb b/examples/copy_paste_series/copy_paste_pages.ipynb
index d80bdbbb..645c0628 100644
--- a/examples/copy_paste_series/copy_paste_pages.ipynb
+++ b/examples/copy_paste_series/copy_paste_pages.ipynb
@@ -31,7 +31,26 @@
     "These same methods/functions can be further modified to move pages _BETWEEN_ Agents as well!\n",
     "\n",
     "## Prerequisites\n",
-    "- Ensure you have a GCP Service Account key with the Dialogflow API Admin privileges assigned to it."
+    "- Ensure you have a GCP Service Account key with the Dialogflow API Admin privileges assigned to it.\n",
+    "\n",
+    "\n",
+    "<table align=\"left\">\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/copy_paste_series/copy_paste_pages.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Google Colaboratory logo\"><br> Run in Colab\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://github.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/copy_paste_series/copy_paste_pages.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"><br> View on GitHub\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/copy_paste_series/copy_paste_pages.ipynb\">\n",
+    "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
+    "    </a>\n",
+    "  </td>\n",
+    "</table>"
    ]
   },
   {
diff --git a/examples/google_sheets_series/cx_to_sheets.ipynb b/examples/google_sheets_series/cx_to_sheets.ipynb
index c53aebfe..89f413bd 100644
--- a/examples/google_sheets_series/cx_to_sheets.ipynb
+++ b/examples/google_sheets_series/cx_to_sheets.ipynb
@@ -31,7 +31,27 @@
     "Specifically, we will show how to extract Intents and Training Phrases from Dialogflow CX to Google Sheets.\n",
     "\n",
     "## Prerequisites\n",
-    "- Ensure you have a GCP Service Account key with the Dialogflow API Admin privileges assigned to it"
+    "- Ensure you have a GCP Service Account key with the Dialogflow API Admin privileges assigned to it\n",
+    "\n",
+    "\n",
+    "\n",
+    "<table align=\"left\">\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/google_sheets_series/cx_to_sheets.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Google Colaboratory logo\"><br> Run in Colab\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://github.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/google_sheets_series/cx_to_sheets.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"><br> View on GitHub\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/google_sheets_series/cx_to_sheets.ipynb\">\n",
+    "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
+    "    </a>\n",
+    "  </td>\n",
+    "</table>"
    ]
   },
   {
diff --git a/examples/google_sheets_series/cx_to_sheets_by_flow.ipynb b/examples/google_sheets_series/cx_to_sheets_by_flow.ipynb
index 1b4d4992..0f4dcbb3 100644
--- a/examples/google_sheets_series/cx_to_sheets_by_flow.ipynb
+++ b/examples/google_sheets_series/cx_to_sheets_by_flow.ipynb
@@ -6,7 +6,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Copyright 2023 Google LLC\n",
+    "# Copyright 2024 Google LLC\n",
     "#\n",
     "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
     "# you may not use this file except in compliance with the License.\n",
@@ -31,7 +31,26 @@
     "Specifically, we will show how to `update` **_Existing Intents_** and Training Phrases in Dialogflow CX using Google Sheets as a Source\n",
     "\n",
     "## Prerequisites\n",
-    "- Ensure you have a GCP Service Account key with the Dialogflow API Admin privileges assigned to it"
+    "- Ensure you have a GCP Service Account key with the Dialogflow API Admin privileges assigned to it\n",
+    "\n",
+    "\n",
+    "<table align=\"left\">\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/google_sheets_series/cx_to_sheets_by_flow.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Google Colaboratory logo\"><br> Run in Colab\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://github.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/google_sheets_series/cx_to_sheets_by_flow.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"><br> View on GitHub\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/google_sheets_series/cx_to_sheets_by_flow.ipynb\">\n",
+    "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
+    "    </a>\n",
+    "  </td>\n",
+    "</table>"
    ]
   },
   {
@@ -116,6 +135,7 @@
     "cu = CopyUtil(creds_path=creds_path, agent_id=agent_id)\n",
     "dffx = DataframeFunctions(creds_path)\n",
     "\n",
+    "#Added agent_id in the argument\n",
     "flow_map = cu.flows.get_flows_map(reverse=True)\n",
     "pages = cu.pages.list_pages(flow_map[flow])\n",
     "resources = cu.get_page_dependencies(pages)\n",
@@ -123,24 +143,16 @@
     "for key in resources.keys():\n",
     "    if key == 'intents':\n",
     "        intent_list = list(resources[key])\n",
-    "        \n",
-    "all_intents = cu.intents.list_intents()\n",
-    "\n",
-    "final_intents = []\n",
-    "for intent in all_intents:\n",
-    "    if intent.name in intent_list:\n",
-    "        final_intents.append(intent)\n",
-    "\n",
-    "df = pd.DataFrame()\n",
     "\n",
-    "for intent in final_intents:\n",
-    "    df = df.append(cu.intents.intent_proto_to_dataframe(intent))\n",
+    "intent_map = cu.intents.get_intents_map()\n",
+    "intent_list_names = list(map(intent_map.get, intent_list))\n",
+    "df = cu.intents.bulk_intent_to_df(intent_subset=intent_list_names)\n",
     "\n",
     "# Push DataFrame to Google Sheets\n",
     "dffx.dataframe_to_sheets(google_sheet_name, google_sheet_tab_write, df)\n",
     "\n",
-    "print('Total # of Intents = {}'.format(df.intent.nunique()))\n",
-    "print('Total # of Training Phrases = {}'.format(df.tp.nunique()))"
+    "print('Total # of Intents = {}'.format(df.display_name.nunique()))\n",
+    "print('Total # of Training Phrases = {}'.format(df.training_phrase.nunique()))"
    ]
   },
   {
diff --git a/examples/google_sheets_series/sheets_to_cx.ipynb b/examples/google_sheets_series/sheets_to_cx.ipynb
index 0a9285b0..1ab5e35d 100644
--- a/examples/google_sheets_series/sheets_to_cx.ipynb
+++ b/examples/google_sheets_series/sheets_to_cx.ipynb
@@ -6,7 +6,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Copyright 2023 Google LLC\n",
+    "# Copyright 2024 Google LLC\n",
     "#\n",
     "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
     "# you may not use this file except in compliance with the License.\n",
@@ -31,7 +31,25 @@
     "Specifically, we will show how to `update` **_Existing Intents_** and Training Phrases in Dialogflow CX using Google Sheets as a Source\n",
     "\n",
     "## Prerequisites\n",
-    "- Ensure you have a GCP Service Account key with the Dialogflow API Admin privileges assigned to it"
+    "- Ensure you have a GCP Service Account key with the Dialogflow API Admin privileges assigned to it\n",
+    "\n",
+    "<table align=\"left\">\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/google_sheets_series/sheets_to_cx.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Google Colaboratory logo\"><br> Run in Colab\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://github.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/google_sheets_series/sheets_to_cx.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"><br> View on GitHub\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/google_sheets_series/sheets_to_cx.ipynb\">\n",
+    "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
+    "    </a>\n",
+    "  </td>\n",
+    "</table>"
    ]
   },
   {
diff --git a/examples/misc/agent_backups.ipynb b/examples/misc/agent_backups.ipynb
index 8abaad59..21672c8a 100644
--- a/examples/misc/agent_backups.ipynb
+++ b/examples/misc/agent_backups.ipynb
@@ -6,7 +6,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Copyright 2023 Google LLC\n",
+    "# Copyright 2024 Google LLC\n",
     "#\n",
     "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
     "# you may not use this file except in compliance with the License.\n",
@@ -29,7 +29,25 @@
     "In this notebook, we will show you how to extract all of the Agent files for a given GCP project and save them in a specified Google Cloud Storage (GCS) bucket.\n",
     "\n",
     "## Prerequisites\n",
-    "- Ensure you have a GCP Service Account key with the Dialogflow API Admin privileges assigned to it."
+    "- Ensure you have a GCP Service Account key with the Dialogflow API Admin privileges assigned to it.\n",
+    "\n",
+    "<table align=\"left\">\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/misc/agent_backups.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Google Colaboratory logo\"><br> Run in Colab\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://github.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/misc/agent_backups.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"><br> View on GitHub\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/misc/agent_backups.ipynb\">\n",
+    "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
+    "    </a>\n",
+    "  </td>\n",
+    "</table>"
    ]
   },
   {
diff --git a/examples/nlu_analysis_series/intent_levenshtein_demo.ipynb b/examples/nlu_analysis_series/intent_levenshtein_demo.ipynb
index 4a8a187c..09263c2f 100644
--- a/examples/nlu_analysis_series/intent_levenshtein_demo.ipynb
+++ b/examples/nlu_analysis_series/intent_levenshtein_demo.ipynb
@@ -6,7 +6,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Copyright 2023 Google LLC\n",
+    "# Copyright 2024 Google LLC\n",
     "#\n",
     "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
     "# you may not use this file except in compliance with the License.\n",
@@ -29,7 +29,25 @@
     "In this notebook, we will show you how to evaluate the levenshtein ratio between the training phrases of two intents.\n",
     "\n",
     "## Prerequisites\n",
-    "- Ensure you have a GCP Service Account key with the Dialogflow API Admin privileges assigned to it."
+    "- Ensure you have a GCP Service Account key with the Dialogflow API Admin privileges assigned to it.\n",
+    "\n",
+    "<table align=\"left\">\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/nlu_analysis_series/intent_levenshtein_demo.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Google Colaboratory logo\"><br> Run in Colab\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://github.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/nlu_analysis_series/intent_levenshtein_demo.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"><br> View on GitHub\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/nlu_analysis_series/intent_levenshtein_demo.ipynb\">\n",
+    "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
+    "    </a>\n",
+    "  </td>\n",
+    "</table>"
    ]
   },
   {
diff --git a/examples/nlu_analysis_series/nlu_semantic_similarity_demo.ipynb b/examples/nlu_analysis_series/nlu_semantic_similarity_demo.ipynb
index cc4ed4d5..93b668ce 100644
--- a/examples/nlu_analysis_series/nlu_semantic_similarity_demo.ipynb
+++ b/examples/nlu_analysis_series/nlu_semantic_similarity_demo.ipynb
@@ -6,7 +6,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Copyright 2023 Google LLC\n",
+    "# Copyright 2024 Google LLC\n",
     "#\n",
     "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
     "# you may not use this file except in compliance with the License.\n",
@@ -36,7 +36,25 @@
     "- Ensure you have a GCP Service Account key with the Dialogflow API Admin privileges assigned to it.\n",
     "\n",
     "## NOTE!\n",
-    "_This colab notebook was intended to run on [Google Colab](https://colab.sandbox.google.com/) infrastrucutre due to the `scann` library dependency._"
+    "_This colab notebook was intended to run on [Google Colab](https://colab.sandbox.google.com/) infrastrucutre due to the `scann` library dependency._\n",
+    "\n",
+    "<table align=\"left\">\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/nlu_analysis_series/nlu_semantic_similarity_demo.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Google Colaboratory logo\"><br> Run in Colab\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://github.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/nlu_analysis_series/nlu_semantic_similarity_demo.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"><br> View on GitHub\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/nlu_analysis_series/nlu_semantic_similarity_demo.ipynb\">\n",
+    "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
+    "    </a>\n",
+    "  </td>\n",
+    "</table>"
    ]
   },
   {
@@ -237,8 +255,8 @@
     "utterance_column_name = \"no_match_utterances\"\n",
     "\n",
     "sheet_loader = SheetsLoader(creds_path)\n",
-    "utterances = sheet_loader.load_column_from_sheet(sheet_name, \n",
-    "                                                 worksheet_name, \n",
+    "utterances = sheet_loader.load_column_from_sheet(sheet_name,\n",
+    "                                                 worksheet_name,\n",
     "                                                 utterance_column_name).astype(str)\n",
     "embedder.find_new_groups(utterances)"
    ]
diff --git a/examples/scrapi_prebuilt.ipynb b/examples/scrapi_prebuilt.ipynb
index 08131802..7a803d21 100644
--- a/examples/scrapi_prebuilt.ipynb
+++ b/examples/scrapi_prebuilt.ipynb
@@ -6,7 +6,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Copyright 2023 Google LLC\n",
+    "# Copyright 2024 Google LLC\n",
     "#\n",
     "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
     "# you may not use this file except in compliance with the License.\n",
@@ -34,7 +34,26 @@
     "## Prerequisites\n",
     "- Ensure you have a GCP Service Account key with the following privileges assigned to it, based on your needs:\n",
     "  - Dialogflow API Reader, use this permission level if you are just exploring the Dialogflow CX resources and developing offline\n",
-    "  - Dialogflow API Admin, use this permissions level if you are planning to Read/Write to your Dialogflow CX Agent and/or resources"
+    "  - Dialogflow API Admin, use this permissions level if you are planning to Read/Write to your Dialogflow CX Agent and/or resources\n",
+    "\n",
+    "\n",
+    "<table align=\"left\">\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/scrapi_prebuilt.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Google Colaboratory logo\"><br> Run in Colab\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://github.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/scrapi_prebuilt.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"><br> View on GitHub\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/scrapi_prebuilt.ipynb\">\n",
+    "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
+    "    </a>\n",
+    "  </td>\n",
+    "</table>"
    ]
   },
   {
@@ -205,7 +224,7 @@
     "for flow_name, flow_id in flows_map_reverse.items():\n",
     "    all_pages_per_flow[flow_name] = pages_instance.list_pages(flow_id=flow_id)\n",
     "    time.sleep(1)\n",
-    "    \n",
+    "\n",
     "    pages_map_per_flow[flow_name] = pages_instance.get_pages_map(\n",
     "        flow_id=flow_id\n",
     "    )\n",
@@ -290,7 +309,7 @@
     "    )\n",
     "    all_route_groups_per_flow[flow_name] = route_groups_list\n",
     "    time.sleep(1)\n",
-    "    \n",
+    "\n",
     "    route_groups_map = route_groups_instance.get_route_groups_map(\n",
     "        flow_id=flow_id\n",
     "    )\n",
@@ -318,7 +337,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# < - - Amazing Code Goes Here! -- > "
+    "# < - - Amazing Code Goes Here! -- >"
    ]
   },
   {
diff --git a/examples/template.ipynb b/examples/template.ipynb
index f9cacade..04d16106 100644
--- a/examples/template.ipynb
+++ b/examples/template.ipynb
@@ -24,7 +24,26 @@
     "    - Instead, describe what your Section is about to do like `Extract Intents and Training Phrases`\n",
     "\n",
     "In each section below we've included sample instructions and minimal code as a pointer for the template.   \n",
-    "If you need further motivation, see [this sample notebook](https://github.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/bot_building_series/bot_building_101.ipynb)"
+    "If you need further motivation, see [this sample notebook](https://github.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/bot_building_series/bot_building_101.ipynb)\n",
+    "\n",
+    "\n",
+    "<table align=\"left\">\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/template.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Google Colaboratory logo\"><br> Run in Colab\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://github.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/template.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"><br> View on GitHub\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/template.ipynb\">\n",
+    "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
+    "    </a>\n",
+    "  </td>\n",
+    "</table>"
    ]
   },
   {
@@ -33,7 +52,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Copyright 2023 Google LLC\n",
+    "# Copyright 2024 Google LLC\n",
     "#\n",
     "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
     "# you may not use this file except in compliance with the License.\n",
diff --git a/examples/vertex_ai_conversation/datastores_and_search.ipynb b/examples/vertex_ai_conversation/datastores_and_search.ipynb
new file mode 100644
index 00000000..09aef52a
--- /dev/null
+++ b/examples/vertex_ai_conversation/datastores_and_search.ipynb
@@ -0,0 +1,417 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Copyright 2024 Google LLC\n",
+    "#\n",
+    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+    "# you may not use this file except in compliance with the License.\n",
+    "# You may obtain a copy of the License at\n",
+    "#\n",
+    "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+    "#\n",
+    "# Unless required by applicable law or agreed to in writing, software\n",
+    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+    "# See the License for the specific language governing permissions and\n",
+    "# limitations under the License."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Vertex AI Search and Conversation - Creating and Querying Datastores\n",
+    "In this notebook, we will show you how to:\n",
+    "1. Build a new Vertex Search DataStore\n",
+    "2. Add Site URIs to your DataStore\n",
+    "3. Create a new Engine for Vertex Search\n",
+    "4. Query your DataStore via the Search method.\n",
+    "\n",
+    "\n",
+    "## Prerequisites\n",
+    "- NOTE - For creating a new Engine, there is a hard requirement to use a Service Account Key.\n",
+    "- For all other methods, local ADC auth creds will work just fine.\n",
+    "\n",
+    "<table align=\"left\">\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/vertex_ai_conversation/datastores_and_search.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Google Colaboratory logo\"><br> Run in Colab\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://github.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/vertex_ai_conversation/datastores_and_search.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"><br> View on GitHub\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/vertex_ai_conversation/datastores_and_search.ipynb\">\n",
+    "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
+    "    </a>\n",
+    "  </td>\n",
+    "</table>"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install dfcx-scrapi>=1.10.6\n",
+    "\n",
+    "from google.colab import auth as google_auth\n",
+    "google_auth.authenticate_user()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Imports"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dfcx_scrapi.core.data_stores import DataStores\n",
+    "from dfcx_scrapi.core.engines import Engines\n",
+    "from dfcx_scrapi.core.sites import Sites\n",
+    "from dfcx_scrapi.core.search import Search"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# User Inputs\n",
+    "In the next section, we will collect runtime variables needed to execute this notebook.   \n",
+    "This should be the only cell of the notebook you need to edit in order for this notebook to run."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "PROJECT_ID = \"YOUR_GCP_PROJECT_ID\"\n",
+    "# CREDS_PATH = \"YOUR_CREDS_PATH_IF_CREATING_ENGINE\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Create Data Store\n",
+    "If you're creating a Data Store for Vertex Conversation, ensure you have `advanced_site_search` = `True`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ds = DataStores(project_id=PROJECT_ID)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ds.create_datastore(\n",
+    "    display_name=\"us-zone-scrapi-datastore\",\n",
+    "    solution_type=\"chat\",\n",
+    "    datastore_type=\"website\",\n",
+    "    advanced_site_search=True,\n",
+    "    location=\"us\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## List Data Stores or Get Data Store Map\n",
+    "Use `list_data_stores` to confirm that your Data Store was created, or use the convenient `get_data_stores_map` to get an easy resource mapping for your Data Store Names/IDs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ds_map = ds.get_data_stores_map(reverse=True, location=\"us\")\n",
+    "ds_map"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Inspect Data Store Config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "datastore = ds.get_data_store(ds_map[\"us-zone-scrapi-datastore\"])\n",
+    "print(datastore)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Inspect Data Store Contents"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sites = Sites(project_id=PROJECT_ID)\n",
+    "\n",
+    "docs = sites.list_sites(ds_map[\"us-zone-scrapi-datastore\"])\n",
+    "\n",
+    "for doc in docs:\n",
+    "  print(f\"SITE: {doc.generated_uri_pattern}\\nLAST UPDATE: {doc.update_time}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Add URI Patterns to Data Store\n",
+    "For this example, we'll add 2 patterns:\n",
+    "- Include `www.example.com/blog/*`\n",
+    "- Exclude `www.example.com/news/*`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sites = Sites(project_id=PROJECT_ID)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sites.create_site(\n",
+    "    data_store_id=ds_map[\"us-zone-scrapi-datastore\"],\n",
+    "    uri_pattern=\"www.example.com/blog/*\",\n",
+    "    include_site=True,\n",
+    "    exact_match=False\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sites.create_site(\n",
+    "    data_store_id=ds_map[\"us-zone-scrapi-datastore\"],\n",
+    "    uri_pattern=\"www.example.com/news\",\n",
+    "    include_site=False,\n",
+    "    exact_match=False\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Inspect Data Store to See Sites Added\n",
+    "Notice that one of these is `VERIFIED` and another is `UNVERIFIED`.  \n",
+    "You will still need to use the Google Cloud Console to go through the verifcation proecess per usual.\n",
+    "\n",
+    "You can also use `get_site_index_status` or `get_site_verification_status` to check on the status of Indexing and Verification programmatically."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sites.list_sites(ds_map[\"us-zone-scrapi-datastore\"])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Create Engine!\n",
+    "For the final step, we'll create our Chat Engine.\n",
+    "\n",
+    "**NOTE** - Remember that for this step you MUST use a Service Account key!"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "eng = Engines(PROJECT_ID, creds_path=CREDS_PATH)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "eng_proto = eng.build_chat_engine_proto(\n",
+    "    display_name=\"my-cool-website-engine\",\n",
+    "    business_name=\"Google Example\",\n",
+    "    data_store_ids=[ds_map[\"us-zone-scrapi-datastore\"]], # we're using our convenient map to add the IDs here!\n",
+    ")\n",
+    "eng_proto"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "eng.create_engine(eng_proto)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## List Engines\n",
+    "Finally, verify that your Engine was created successfully!  \n",
+    "You can use `list_engines` or `get_engines_map` to verify."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "eng.get_engines_map(reverse=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Search\n",
+    "In this final section, we'll run a search against our newly indexed datastore."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ds = DataStores(project_id=PROJECT_ID)\n",
+    "s = Search()\n",
+    "\n",
+    "ds_map = ds.get_data_stores_map(reverse=True, location=\"us\")\n",
+    "ds_map"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Define Search Config\n",
+    "In order to get high quality search results from your new datastore, there are\n",
+    "many different config parameters you can provide for your `search_config`.  \n",
+    "The\n",
+    "full list can be found in the [SearchRequest](https://cloud.google.com/python/docs/reference/discoveryengine/latest/google.cloud.discoveryengine_v1beta.types.SearchRequest) documentation.\n",
+    "\n",
+    "Here, we will provide a minimal config for demonstration purposes."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "search_config = {\n",
+    "    \"data_store_id\": f\"{ds_map['us-zone-scrapi-datastore']}\",\n",
+    "    \"query\": \"What is an example domain?\"\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "s.search(search_config)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Ending and Wrap-Up\n",
+    "\n",
+    "In this notebook, we've shown how to programmatically build a DataStore, index some test websites, and query them using a search config.\n",
+    "\n",
+    "For more information, see:\n",
+    "- [Vertex AI Search](https://cloud.google.com/generative-ai-app-builder/docs/try-enterprise-search)\n",
+    "- [Verex AI Conversation](https://cloud.google.com/dialogflow/vertex/docs)"
+   ]
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "19fe958eff886c70bc7b0837ba1e6b09536c8944c54196036e51b6ba767223fc"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.8.11 64-bit ('scrapi-local': conda)",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.12"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb b/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb
new file mode 100644
index 00000000..97a8f30b
--- /dev/null
+++ b/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb
@@ -0,0 +1,2542 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zGZzOhx3sNWR"
+      },
+      "source": [
+        "# Vertex AI Conversation - Evaluation Tool\n",
+        "\n",
+        "This tool requires user's input in several steps. Please run the cells one by one (Shift+Enter) to ensure all the steps are successfully completed."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Afvsuux0zaWZ"
+      },
+      "source": [
+        "# Setup\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": true,
+        "id": "0U8xQwhKrOUq"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install dfcx-scrapi --quiet\n",
+        "!pip install rouge-score --quiet\n",
+        "\n",
+        "# workaround until vertexai import is fixed\n",
+        "!pip uninstall bigframes -y --quiet\n",
+        "!pip install bigframes==0.26.0 --quiet"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "PPJYRHN83bHg"
+      },
+      "outputs": [],
+      "source": [
+        "# @markdown `import dependencies`\n",
+        "\n",
+        "import abc\n",
+        "import collections\n",
+        "import dataclasses\n",
+        "import datetime\n",
+        "import io\n",
+        "import itertools\n",
+        "import json\n",
+        "import logging\n",
+        "import math\n",
+        "import statistics\n",
+        "import sys\n",
+        "import time\n",
+        "import threading\n",
+        "import re\n",
+        "\n",
+        "from typing import Any, TypedDict\n",
+        "\n",
+        "from collections.abc import Iterable\n",
+        "\n",
+        "import plotly.graph_objects as go\n",
+        "\n",
+        "import vertexai\n",
+        "import gspread\n",
+        "import matplotlib.pyplot as plt\n",
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "import seaborn as sns\n",
+        "sns.set_style(\"whitegrid\")\n",
+        "\n",
+        "from tqdm.auto import tqdm\n",
+        "from tqdm.contrib import concurrent\n",
+        "\n",
+        "from dfcx_scrapi.core import agents\n",
+        "from dfcx_scrapi.core import scrapi_base\n",
+        "from dfcx_scrapi.core import sessions\n",
+        "from dfcx_scrapi.core.sessions import Sessions\n",
+        "from dfcx_scrapi.tools import dataframe_functions\n",
+        "\n",
+        "from googleapiclient.discovery import build\n",
+        "from googleapiclient.http import MediaInMemoryUpload, MediaIoBaseDownload\n",
+        "\n",
+        "from google.api_core import exceptions\n",
+        "from google.auth import default\n",
+        "from google.cloud import aiplatform\n",
+        "from google.cloud.dialogflowcx_v3beta1 import services\n",
+        "from google.cloud.dialogflowcx_v3beta1 import types\n",
+        "from google.colab import auth\n",
+        "from google.protobuf.json_format import MessageToDict\n",
+        "\n",
+        "from rouge_score import rouge_scorer\n",
+        "\n",
+        "from vertexai.language_models import TextGenerationModel\n",
+        "\n",
+        "pd.options.display.max_colwidth = 200"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "EgoRHwBJqJ0r"
+      },
+      "outputs": [],
+      "source": [
+        "# @markdown `authenticate`\n",
+        "\n",
+        "if \"google.colab\" in sys.modules:\n",
+        "    from google.auth import default\n",
+        "    from google.colab import auth\n",
+        "\n",
+        "    auth.authenticate_user()\n",
+        "    credentials, _ = default()\n",
+        "else:\n",
+        "    # Otherwise, attempt to discover local credentials as described in\n",
+        "    # https://cloud.google.com/docs/authentication/application-default-credentials\n",
+        "    pass\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2c6VwnIvTBjF"
+      },
+      "source": [
+        "---\n",
+        "\n",
+        "# Implementation\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "BwPAkHGQ3k6M"
+      },
+      "outputs": [],
+      "source": [
+        "# @markdown `run this cell to define various utility and helper functions`\n",
+        "# @markdown\n",
+        "# @markdown > This cell contains several decorator methods related to handling\n",
+        "# @markdown API call errors and function call rate limitation.\n",
+        "\n",
+        "_INTERVAL_SENTINEL = object()\n",
+        "\n",
+        "MAX_RETRIES = 5\n",
+        "# max number of attempts for exponential backoff retries in case of API\n",
+        "# call errors\n",
+        "RATE = 2\n",
+        "# LLM API call rate limitation where RATE=2 for example means that 2 LLM calls\n",
+        "# can occur per second\n",
+        "\n",
+        "\n",
+        "def load_spreadsheet(\n",
+        "    sheet_url: str, worksheet_name: str, credentials: Any\n",
+        ") -> pd.DataFrame:\n",
+        "  \"\"\"Loads the content of a spreadsheet into pandas DataFrame.\"\"\"\n",
+        "  sheets_client = gspread.authorize(credentials)\n",
+        "  sheet = sheets_client.open_by_url(sheet_url)\n",
+        "  worksheet = sheet.worksheet(worksheet_name)\n",
+        "  return pd.DataFrame(worksheet.get_all_records())\n",
+        "\n",
+        "\n",
+        "def ratelimit(rate: float):\n",
+        "  \"\"\"Decorator that controls the frequency of function calls.\"\"\"\n",
+        "  seconds_per_event = 1.0 / rate\n",
+        "  lock = threading.Lock()\n",
+        "  bucket = 0\n",
+        "  last = 0\n",
+        "\n",
+        "  def decorate(func):\n",
+        "    def rate_limited_function(*args, **kwargs):\n",
+        "      nonlocal last, bucket\n",
+        "      while True:\n",
+        "        with lock:\n",
+        "          now = time.time()\n",
+        "          bucket += now - last\n",
+        "          last = now\n",
+        "\n",
+        "          # capping the bucket in order to avoid accumulating too many\n",
+        "          bucket = min(bucket, seconds_per_event)\n",
+        "\n",
+        "          # if bucket is less than `seconds_per_event` then we have to wait\n",
+        "          # `seconds_per_event` - `bucket` seconds until a new \"token\" is\n",
+        "          # refilled\n",
+        "          delay = max(seconds_per_event - bucket, 0)\n",
+        "\n",
+        "          if delay == 0:\n",
+        "            # consuming a token and breaking out of the delay loop to perform\n",
+        "            # the function call\n",
+        "            bucket -= seconds_per_event\n",
+        "            break\n",
+        "        time.sleep(delay)\n",
+        "      return func(*args, **kwargs)\n",
+        "    return rate_limited_function\n",
+        "  return decorate\n",
+        "\n",
+        "\n",
+        "def should_retry(err: exceptions.GoogleAPICallError) -> bool:\n",
+        "  \"\"\"Helper function for deciding whether we should retry the error or not.\"\"\"\n",
+        "  return isinstance(err, (exceptions.TooManyRequests, exceptions.ServerError))\n",
+        "\n",
+        "\n",
+        "def retry_api_call(retry_intervals: Iterable[float]):\n",
+        "  \"\"\"Decorator for retrying certain GoogleAPICallError exception types.\"\"\"\n",
+        "  def decorate(func):\n",
+        "    def retried_api_call_func(*args, **kwargs):\n",
+        "      interval_iterator = iter(retry_intervals)\n",
+        "      while True:\n",
+        "        try:\n",
+        "          return func(*args, **kwargs)\n",
+        "        except exceptions.GoogleAPICallError as err:\n",
+        "          print(f\"retrying api call: {err}\")\n",
+        "          if not should_retry(err):\n",
+        "            raise\n",
+        "\n",
+        "          interval = next(interval_iterator, _INTERVAL_SENTINEL)\n",
+        "          if interval is _INTERVAL_SENTINEL:\n",
+        "            raise\n",
+        "          time.sleep(interval)\n",
+        "    return retried_api_call_func\n",
+        "  return decorate\n",
+        "\n",
+        "\n",
+        "def handle_api_error(func):\n",
+        "  \"\"\"Decorator that chatches GoogleAPICallError exception and returns None.\"\"\"\n",
+        "  def handled_api_error_func(*args, **kwargs):\n",
+        "    try:\n",
+        "      return func(*args, **kwargs)\n",
+        "    except exceptions.GoogleAPICallError as err:\n",
+        "      print(f\"failed api call: {err}\")\n",
+        "      return None\n",
+        "  return handled_api_error_func"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "NU9jzn9dWtrJ"
+      },
+      "outputs": [],
+      "source": [
+        "# @markdown `run this cell to define vertex ai conversation scraper`\n",
+        "# @markdown\n",
+        "# @markdown > This cell contains the code for Vertex AI Conversation scraper\n",
+        "# @markdown that interacts with DetectIntent method of Dialogflow service to\n",
+        "# @markdown process a queryset.\n",
+        "\n",
+        "DataStoreConnectionSignals = (\n",
+        "    types.data_store_connection.DataStoreConnectionSignals\n",
+        ")\n",
+        "\n",
+        "GLOBAL_SCOPE = [\"https://spreadsheets.google.com/feeds\"]\n",
+        "\n",
+        "CONVERSATION_ID = \"conversation_id\"\n",
+        "TURN_INDEX = \"turn_index\"\n",
+        "QUERY = \"query\"\n",
+        "REFERENCE = \"expected_answer\"\n",
+        "EXPECTED_URI = \"expected_uri\"\n",
+        "SESSION_ID = \"session_id\"\n",
+        "RESPONSE = \"query_result\"\n",
+        "GOLDEN_SNIPPET = \"golden_snippet\"\n",
+        "\n",
+        "AGENT_URI = \"projects/{project_id}/locations/{location}/agents/{agent_id}\"\n",
+        "\n",
+        "INPUT_SCHEMA_REQUIRED_COLUMNS = [\n",
+        "    CONVERSATION_ID, TURN_INDEX, QUERY, REFERENCE, EXPECTED_URI\n",
+        "]\n",
+        "\n",
+        "_EXECUTION_SEQUENCE_KEY = \"DataStore Execution Sequence\"\n",
+        "_EXECUTION_RESULT_KEY = \"executionResult\"\n",
+        "\n",
+        "_PROJECT_ID_PATTERN = re.compile(r\"projects/(.*?)/\")\n",
+        "_LOCATION_PATTERN = re.compile(r\"locations/(.*?)/\")\n",
+        "_AGENT_ID_PATTERN = re.compile(r\"agents/(.*?)/\")\n",
+        "\n",
+        "ANSWER_TEXT = \"answer_text\"\n",
+        "\n",
+        "_RESPONSE_TYPE = \"response_type\"\n",
+        "_RESPONSE_REASON = \"response_reason\"\n",
+        "_LATENCY = \"latency\"\n",
+        "_FAQ_CITATION = \"faq_citation\"\n",
+        "_SEARCH_FALLBACK = \"search_fallback\"\n",
+        "_UNSTRUCTURED_CITATION = \"unstructured_citation\"\n",
+        "_WEBSITE_CITATION = \"website_citation\"\n",
+        "_LANGUAGE = \"language\"\n",
+        "\n",
+        "_REWRITER_LLM_PROMPT = \"rewriter_llm_rendered_prompt\"\n",
+        "_REWRITER_LLM_OUTPUT = \"rewriter_llm_output\"\n",
+        "_REWRITTEN_QUERY = \"rewritten_query\"\n",
+        "_SEARCH_RESULTS = \"search_results\"\n",
+        "_ANSWER_GENERATOR_LLM_PROMPT = \"answer_generator_llm_rendered_prompt\"\n",
+        "_ANSWER_GENERATOR_LLM_OUTPUT = \"answer_generator_llm_output\"\n",
+        "_GENERATED_ANSWER = \"generated_answer\"\n",
+        "_CITED_SNIPPET_INDICES = \"cited_snippet_indices\"\n",
+        "_GROUNDING_DECISION = \"grounding_decision\"\n",
+        "_GROUNDING_SCORE = \"grounding_score\"\n",
+        "_SAFETY_DECISION = \"safety_decision\"\n",
+        "_SAFETY_BANNED_PHRASE = \"safety_banned_phrase_match\"\n",
+        "\n",
+        "\n",
+        "def _extract_match_type(query_result: types.session.QueryResult) -> str:\n",
+        "  \"\"\"Extracts the name of the match type from query result.\"\"\"\n",
+        "  try:\n",
+        "    return types.session.Match.MatchType(query_result.match.match_type).name\n",
+        "  except ValueError:\n",
+        "    # if an enum type is returned which is not visible externally then fallback\n",
+        "    # to default value\n",
+        "    return types.session.Match.MatchType(0).name\n",
+        "\n",
+        "\n",
+        "def _extract_execution_result(\n",
+        "    query_result: types.session.QueryResult\n",
+        ") -> dict[str, Any]:\n",
+        "  \"\"\"Extracts the execution result from diagnostic info.\"\"\"\n",
+        "  if _EXECUTION_SEQUENCE_KEY in query_result.diagnostic_info:\n",
+        "    execution_sequence = query_result.diagnostic_info[_EXECUTION_SEQUENCE_KEY]\n",
+        "    if _EXECUTION_RESULT_KEY in execution_sequence:\n",
+        "      return MessageToDict(execution_sequence[_EXECUTION_RESULT_KEY])\n",
+        "  return {}\n",
+        "\n",
+        "\n",
+        "def _extract_answer_text(\n",
+        "    query_result: types.session.QueryResult\n",
+        ") -> str | None:\n",
+        "  \"\"\"Extracts the text type responses and concatenates them.\"\"\"\n",
+        "  result: list[str] = []\n",
+        "  for response_message in query_result.response_messages:\n",
+        "    if response_message.WhichOneof(\"message\") == \"text\":\n",
+        "      result.extend(response_message.text.text)\n",
+        "\n",
+        "  if not result:\n",
+        "    return None\n",
+        "\n",
+        "  return \" \".join(result)\n",
+        "\n",
+        "\n",
+        "@dataclasses.dataclass\n",
+        "class Snippet:\n",
+        "  uri: str | None\n",
+        "  title: str | None\n",
+        "  text: str | None\n",
+        "\n",
+        "  def to_prompt_snippet(self) -> str:\n",
+        "    result = []\n",
+        "    if self.title:\n",
+        "      result.append(self.title)\n",
+        "    if self.text:\n",
+        "      result.append(self.text)\n",
+        "    return \"\\n\".join(result) if result else \"\"\n",
+        "\n",
+        "\n",
+        "def _extract_search_results(\n",
+        "    data_store_connection_signals: DataStoreConnectionSignals\n",
+        ") -> list[str]:\n",
+        "  \"\"\"Extracts search results as a list of strings.\"\"\"\n",
+        "  search_results = []\n",
+        "  for search_snippet in data_store_connection_signals.search_snippets:\n",
+        "    search_results.append(\n",
+        "        Snippet(\n",
+        "            uri=search_snippet.document_uri,\n",
+        "            title=search_snippet.document_title,\n",
+        "            text=search_snippet.text,\n",
+        "        )\n",
+        "    )\n",
+        "  return search_results\n",
+        "\n",
+        "\n",
+        "def _extract_citation_indices(\n",
+        "    data_store_connection_signals: DataStoreConnectionSignals\n",
+        ") -> list[int]:\n",
+        "  \"\"\"Extracts the links and snippets which were used to generate answer.\"\"\"\n",
+        "  cited_snippet_indices = []\n",
+        "  for cited_snippet in data_store_connection_signals.cited_snippets:\n",
+        "    cited_snippet_indices.append(cited_snippet.snippet_index)\n",
+        "  return cited_snippet_indices\n",
+        "\n",
+        "\n",
+        "def _extract_grounding_decision(\n",
+        "    grounding_signals: DataStoreConnectionSignals.GroundingSignals\n",
+        ") -> str:\n",
+        "  return DataStoreConnectionSignals.GroundingSignals.GroundingDecision(\n",
+        "      grounding_signals.decision\n",
+        "  ).name\n",
+        "\n",
+        "\n",
+        "def _extract_grounding_score(\n",
+        "    grounding_signals: DataStoreConnectionSignals.GroundingSignals\n",
+        "):\n",
+        "  return DataStoreConnectionSignals.GroundingSignals.GroundingScoreBucket(\n",
+        "      grounding_signals.score\n",
+        "  ).name\n",
+        "\n",
+        "\n",
+        "def _extract_grounding_signals(\n",
+        "    data_store_connection_signals: DataStoreConnectionSignals\n",
+        ") -> dict[str, str | None]:\n",
+        "  grounding_signals = data_store_connection_signals.grounding_signals\n",
+        "  if not grounding_signals:\n",
+        "    return {_GROUNDING_DECISION: None, _GROUNDING_SCORE: None}\n",
+        "  return {\n",
+        "      _GROUNDING_DECISION: _extract_grounding_decision(grounding_signals),\n",
+        "      _GROUNDING_SCORE: _extract_grounding_score(grounding_signals),\n",
+        "  }\n",
+        "\n",
+        "\n",
+        "def _extract_rewriter_llm_signals(\n",
+        "    data_store_connection_signals: DataStoreConnectionSignals\n",
+        ") -> dict[str, str | None]:\n",
+        "  rewriter_model_call_signals = (\n",
+        "      data_store_connection_signals.rewriter_model_call_signals\n",
+        "  )\n",
+        "  if not rewriter_model_call_signals:\n",
+        "    return {_REWRITER_LLM_PROMPT: None, _REWRITER_LLM_OUTPUT: None}\n",
+        "  return {\n",
+        "      _REWRITER_LLM_PROMPT: rewriter_model_call_signals.rendered_prompt,\n",
+        "      _REWRITER_LLM_OUTPUT: rewriter_model_call_signals.model_output,\n",
+        "  }\n",
+        "\n",
+        "\n",
+        "def _extract_answer_generator_llm_signals(\n",
+        "    data_store_connection_signals: DataStoreConnectionSignals\n",
+        ") -> dict[str, str | None]:\n",
+        "  answer_generation_model_call_signals = (\n",
+        "      data_store_connection_signals.answer_generation_model_call_signals\n",
+        "  )\n",
+        "  if not answer_generation_model_call_signals:\n",
+        "    return {\n",
+        "        _ANSWER_GENERATOR_LLM_PROMPT: None,\n",
+        "        _ANSWER_GENERATOR_LLM_OUTPUT: None,\n",
+        "    }\n",
+        "  return {\n",
+        "      _ANSWER_GENERATOR_LLM_PROMPT: (\n",
+        "          answer_generation_model_call_signals.rendered_prompt\n",
+        "      ),\n",
+        "      _ANSWER_GENERATOR_LLM_OUTPUT: (\n",
+        "          answer_generation_model_call_signals.model_output\n",
+        "      )\n",
+        "  }\n",
+        "\n",
+        "\n",
+        "def _extract_safety_decision(\n",
+        "    safety_signals: DataStoreConnectionSignals.SafetySignals\n",
+        ") -> str:\n",
+        "  return DataStoreConnectionSignals.SafetySignals.SafetyDecision(\n",
+        "      safety_signals.decision\n",
+        "  ).name\n",
+        "\n",
+        "\n",
+        "def _extract_safety_banned_phrase(\n",
+        "    safety_signals: DataStoreConnectionSignals.SafetySignals\n",
+        ") -> str:\n",
+        "  return DataStoreConnectionSignals.SafetySignals.BannedPhraseMatch(\n",
+        "      safety_signals.banned_phrase_match\n",
+        "  ).name\n",
+        "\n",
+        "\n",
+        "def _extract_safety_signals(\n",
+        "    data_store_connection_signals: DataStoreConnectionSignals\n",
+        ") -> dict[str, str | None]:\n",
+        "  safety_signals = data_store_connection_signals.safety_signals\n",
+        "  if not safety_signals:\n",
+        "    return {_SAFETY_DECISION: None, _SAFETY_BANNED_PHRASE: None}\n",
+        "  return {\n",
+        "      _SAFETY_DECISION: _extract_safety_decision(safety_signals),\n",
+        "      _SAFETY_BANNED_PHRASE: _extract_safety_banned_phrase(safety_signals),\n",
+        "  }\n",
+        "\n",
+        "\n",
+        "def _extract_data_store_connection_signals(\n",
+        "    data_store_connection_signals: DataStoreConnectionSignals\n",
+        ") -> dict[str, Any]:\n",
+        "  rewriter_signals = _extract_rewriter_llm_signals(\n",
+        "      data_store_connection_signals\n",
+        "  )\n",
+        "  rewritten_query = (\n",
+        "    data_store_connection_signals.rewritten_query\n",
+        "    if data_store_connection_signals.rewritten_query\n",
+        "    else None\n",
+        "  )\n",
+        "  grounding_signals = _extract_grounding_signals(data_store_connection_signals)\n",
+        "  search_results = _extract_search_results(data_store_connection_signals)\n",
+        "  answer_generator_signals = _extract_answer_generator_llm_signals(\n",
+        "      data_store_connection_signals\n",
+        "  )\n",
+        "  generated_answer = (\n",
+        "      data_store_connection_signals.answer\n",
+        "      if data_store_connection_signals.answer\n",
+        "      else None\n",
+        "  )\n",
+        "  cited_snippet_indices = _extract_citation_indices(\n",
+        "      data_store_connection_signals\n",
+        "  )\n",
+        "  safety_signals = _extract_safety_signals(data_store_connection_signals)\n",
+        "\n",
+        "  return {\n",
+        "      **rewriter_signals,\n",
+        "      _REWRITTEN_QUERY: rewritten_query,\n",
+        "      **grounding_signals,\n",
+        "      _SEARCH_RESULTS: search_results,\n",
+        "      **answer_generator_signals,\n",
+        "      _GENERATED_ANSWER: generated_answer,\n",
+        "      _CITED_SNIPPET_INDICES: cited_snippet_indices,\n",
+        "      **safety_signals,\n",
+        "  }\n",
+        "\n",
+        "\n",
+        "@dataclasses.dataclass\n",
+        "class VertexConversationResponse:\n",
+        "  \"\"\"Dataclass for storing relevant fields of detect intent response.\"\"\"\n",
+        "  # ResponseMessages\n",
+        "  answer_text: str | None = None\n",
+        "\n",
+        "  # MatchType\n",
+        "  match_type: str | None = None\n",
+        "\n",
+        "  # DataStoreConnectionSignals\n",
+        "  rewriter_llm_rendered_prompt: str | None = None\n",
+        "  rewriter_llm_output: str | None = None\n",
+        "  rewritten_query: str | None = None\n",
+        "  search_results: list[Snippet] = dataclasses.field(default_factory=list)\n",
+        "  answer_generator_llm_rendered_prompt: str | None = None\n",
+        "  answer_generator_llm_output: str | None = None\n",
+        "  generated_answer: str | None = None\n",
+        "  cited_snippet_indices: list[int] = dataclasses.field(default_factory=list)\n",
+        "  grounding_decision: str | None = None\n",
+        "  grounding_score: str | None = None\n",
+        "  safety_decision: str | None = None\n",
+        "  safety_banned_phrase_match: str | None = None\n",
+        "\n",
+        "  # DiagnosticInfo ExecutionResult\n",
+        "  response_type: str | None = None\n",
+        "  response_reason: str | None = None\n",
+        "  latency: float | None = None\n",
+        "  faq_citation: bool | None = None\n",
+        "  search_fallback: bool | None = None\n",
+        "  unstructured_citation: bool | None = None\n",
+        "  website_citation: bool | None = None\n",
+        "  language: str | None = None\n",
+        "\n",
+        "  @classmethod\n",
+        "  def from_query_result(cls, query_result: types.session.QueryResult):\n",
+        "    \"\"\"Extracts the relevant fields from a QueryResult proto message.\"\"\"\n",
+        "    answer_text = _extract_answer_text(query_result)\n",
+        "    match_type = _extract_match_type(query_result)\n",
+        "    execution_result = _extract_execution_result(query_result)\n",
+        "    execution_result = {\n",
+        "        _RESPONSE_TYPE: execution_result.get(_RESPONSE_TYPE),\n",
+        "        _RESPONSE_REASON: execution_result.get(_RESPONSE_REASON),\n",
+        "        _LATENCY: execution_result.get(_LATENCY),\n",
+        "        _FAQ_CITATION: execution_result.get(_FAQ_CITATION),\n",
+        "        _SEARCH_FALLBACK: execution_result.get(\"ucs_fallback\"),\n",
+        "        _UNSTRUCTURED_CITATION: execution_result.get(_UNSTRUCTURED_CITATION),\n",
+        "        _WEBSITE_CITATION: execution_result.get(_WEBSITE_CITATION),\n",
+        "        _LANGUAGE: execution_result.get(_LANGUAGE),\n",
+        "    }\n",
+        "\n",
+        "    data_store_connection_signals = query_result.data_store_connection_signals\n",
+        "\n",
+        "    if not data_store_connection_signals:\n",
+        "      return cls(\n",
+        "          answer_text=answer_text, match_type=match_type, **execution_result\n",
+        "      )\n",
+        "\n",
+        "    extracted_signals = _extract_data_store_connection_signals(\n",
+        "        data_store_connection_signals\n",
+        "    )\n",
+        "    return cls(\n",
+        "        answer_text=answer_text,\n",
+        "        match_type=match_type,\n",
+        "        **extracted_signals,\n",
+        "        **execution_result,\n",
+        "    )\n",
+        "\n",
+        "  @classmethod\n",
+        "  def from_row(cls, row: dict[str, Any]):\n",
+        "    \"\"\"Extracts the relevant fields from a dictionary.\"\"\"\n",
+        "    row = row.copy()\n",
+        "    search_results = []\n",
+        "    for search_result in json.loads(row[_SEARCH_RESULTS]):\n",
+        "      search_results.append(Snippet(**search_result))\n",
+        "    row[_SEARCH_RESULTS] = search_results\n",
+        "    row[_CITED_SNIPPET_INDICES] = json.loads(row[_CITED_SNIPPET_INDICES])\n",
+        "    return cls(**row)\n",
+        "\n",
+        "  def to_row(self):\n",
+        "    \"\"\"Dumps the query result fields to a dictionary.\"\"\"\n",
+        "    result = dataclasses.asdict(self)\n",
+        "    result[_SEARCH_RESULTS] = json.dumps(\n",
+        "        result.pop(_SEARCH_RESULTS, []), indent=4\n",
+        "    )\n",
+        "    result[_CITED_SNIPPET_INDICES] = json.dumps(result[_CITED_SNIPPET_INDICES])\n",
+        "    return result\n",
+        "\n",
+        "  @property\n",
+        "  def search_result_links(self):\n",
+        "    return [search_result.uri for search_result in self.search_results]\n",
+        "\n",
+        "  @property\n",
+        "  def cited_search_results(self):\n",
+        "    return [self.search_results[idx] for idx in self.cited_snippet_indices]\n",
+        "\n",
+        "  @property\n",
+        "  def cited_search_result_links(self):\n",
+        "    return [search_result.uri for search_result in self.cited_search_results]\n",
+        "\n",
+        "  @property\n",
+        "  def prompt_snippets(self):\n",
+        "    return [\n",
+        "        search_result.to_prompt_snippet()\n",
+        "        for search_result in self.search_results\n",
+        "    ]\n",
+        "\n",
+        "\n",
+        "def _extract_url_part(url, pattern):\n",
+        "  pattern_match = pattern.search(url)\n",
+        "  if not pattern_match:\n",
+        "    raise ValueError(f\"Invalid url: {url}\")\n",
+        "  return pattern_match.group(1)\n",
+        "\n",
+        "\n",
+        "class VertexConversationScraper(scrapi_base.ScrapiBase):\n",
+        "  \"\"\"Vertex AI Conversation scraper class.\"\"\"\n",
+        "\n",
+        "  @classmethod\n",
+        "  def from_url(cls, agent_url, language_code, creds):\n",
+        "    agent_id = _extract_url_part(agent_url, _AGENT_ID_PATTERN)\n",
+        "    location = _extract_url_part(agent_url, _LOCATION_PATTERN)\n",
+        "    project_id = _extract_url_part(agent_url, _PROJECT_ID_PATTERN)\n",
+        "    return cls(\n",
+        "        agent_id=agent_id,\n",
+        "        location=location,\n",
+        "        project_id=project_id,\n",
+        "        language_code=language_code,\n",
+        "        creds=creds,\n",
+        "    )\n",
+        "\n",
+        "  def __init__(\n",
+        "      self,\n",
+        "      agent_id: str,\n",
+        "      location: str,\n",
+        "      project_id: str,\n",
+        "      language_code: str,\n",
+        "      creds_path: str = None,\n",
+        "      creds_dict: dict[str, str] = None,\n",
+        "      creds=None,\n",
+        "  ):\n",
+        "    super().__init__(\n",
+        "        creds_path=creds_path,\n",
+        "        creds_dict=creds_dict,\n",
+        "        creds=creds,\n",
+        "        scope=GLOBAL_SCOPE,\n",
+        "    )\n",
+        "\n",
+        "    self.location = location\n",
+        "    self.project_id = project_id\n",
+        "    self.language_code = language_code\n",
+        "\n",
+        "    self.agent_id = AGENT_URI.format(\n",
+        "        project_id=project_id, location=location, agent_id=agent_id\n",
+        "    )\n",
+        "\n",
+        "    self.sessions = sessions.Sessions(agent_id=self.agent_id)\n",
+        "    self._agents = agents.Agents(creds=self.creds)\n",
+        "\n",
+        "  def validate_queryset(self, queryset: pd.DataFrame) -> None:\n",
+        "    \"\"\"Validates the queryset and raises exception in case of invalid input.\"\"\"\n",
+        "    # validate input schema\n",
+        "    try:\n",
+        "      queryset[INPUT_SCHEMA_REQUIRED_COLUMNS]\n",
+        "    except KeyError as err:\n",
+        "      raise UserWarning(\n",
+        "          \"Ensure your input data contains the following columns:\"\n",
+        "          f\" {INPUT_SCHEMA_REQUIRED_COLUMNS}\"\n",
+        "      ) from err\n",
+        "\n",
+        "    # validate if conversationd_id and turn_id is unique identifier\n",
+        "    if not (\n",
+        "        queryset[CONVERSATION_ID].astype(str)\n",
+        "        + \"_\"\n",
+        "        + queryset[TURN_INDEX].astype(str)\n",
+        "    ).is_unique:\n",
+        "      raise UserWarning(\n",
+        "          \"Ensure that 'conversation_id' and 'turn_index' are unique \"\n",
+        "          \"identifiers\"\n",
+        "      )\n",
+        "\n",
+        "    # validate turn_index\n",
+        "    try:\n",
+        "      queryset[TURN_INDEX].astype(int)\n",
+        "    except ValueError as err:\n",
+        "      raise UserWarning(\"Ensure that 'turn_index' is set as integer\") from err\n",
+        "\n",
+        "    if not queryset[TURN_INDEX].astype(int).gt(0).all():\n",
+        "      raise UserWarning(\"Ensure that 'turn_index' is in [1, inf)\")\n",
+        "\n",
+        "  def setup_queryset(self, queryset: pd.DataFrame) -> pd.DataFrame:\n",
+        "    \"\"\"Various Dataframe validation and cleaning functions.\"\"\"\n",
+        "    queryset = queryset.rename(\n",
+        "        {column: column.lower() for column in queryset.columns}\n",
+        "    )\n",
+        "\n",
+        "    self.validate_queryset(queryset)\n",
+        "\n",
+        "    queryset[TURN_INDEX] = queryset[TURN_INDEX].astype(int)\n",
+        "    timestamp = datetime.datetime.now(tz=datetime.timezone.utc)\n",
+        "\n",
+        "    # adding timestamp and agent display name so they can be used as a multi\n",
+        "    # index\n",
+        "    queryset[\"scrape_timestamp\"] = timestamp.isoformat()\n",
+        "    agent_display_name = self._agents.get_agent(self.agent_id).display_name\n",
+        "    queryset[\"agent_display_name\"] = agent_display_name\n",
+        "\n",
+        "    queryset = self._create_session_ids(queryset)\n",
+        "\n",
+        "    # if the conversation_id can be converted to int then sorting can be done\n",
+        "    # numerically instead of alphabetically\n",
+        "    try:\n",
+        "      queryset[CONVERSATION_ID] = queryset[CONVERSATION_ID].astype(int)\n",
+        "    except ValueError:\n",
+        "      pass\n",
+        "\n",
+        "    queryset = queryset.sort_values(\n",
+        "        by=[CONVERSATION_ID, TURN_INDEX], ascending=True\n",
+        "    )\n",
+        "    return queryset\n",
+        "\n",
+        "  def _create_session_ids(self, queryset: pd.DataFrame) -> pd.DataFrame:\n",
+        "    \"\"\"Creates a unique session id for each conversation_id.\"\"\"\n",
+        "    sessions = []\n",
+        "    for conversation_id in queryset[CONVERSATION_ID].unique():\n",
+        "      sessions.append({\n",
+        "          CONVERSATION_ID: conversation_id,\n",
+        "          SESSION_ID: self.sessions.build_session_id(self.agent_id),\n",
+        "      })\n",
+        "    sessions_df = pd.DataFrame(sessions)\n",
+        "    return queryset.merge(sessions_df, on=CONVERSATION_ID, how=\"left\")\n",
+        "\n",
+        "  def detect_intent(\n",
+        "      self,\n",
+        "      agent_id,\n",
+        "      session_id,\n",
+        "      text,\n",
+        "      language_code,\n",
+        "      parameters=None,\n",
+        "      populate_data_store_connection_signals=False,\n",
+        "  ):\n",
+        "    client_options = self.sessions._set_region(agent_id)\n",
+        "    session_client = services.sessions.SessionsClient(\n",
+        "        client_options=client_options, credentials=self.creds\n",
+        "    )\n",
+        "\n",
+        "    logging.info(f\"Starting Session ID {session_id}\")\n",
+        "\n",
+        "    query_input = self.sessions._build_query_input(text, language_code)\n",
+        "\n",
+        "    request = types.session.DetectIntentRequest()\n",
+        "    request.session = session_id\n",
+        "    request.query_input = query_input\n",
+        "\n",
+        "    query_param_mapping = {}\n",
+        "\n",
+        "    if parameters:\n",
+        "      query_param_mapping[\"parameters\"] = parameters\n",
+        "\n",
+        "    if populate_data_store_connection_signals:\n",
+        "      query_param_mapping[\"populate_data_store_connection_signals\"] = (\n",
+        "          populate_data_store_connection_signals\n",
+        "      )\n",
+        "\n",
+        "    if query_param_mapping:\n",
+        "      query_params =  types.session.QueryParameters(query_param_mapping)\n",
+        "      request.query_params = query_params\n",
+        "\n",
+        "    response = session_client.detect_intent(request)\n",
+        "    query_result = response.query_result\n",
+        "\n",
+        "    return query_result\n",
+        "\n",
+        "  @retry_api_call([i**2 for i in range(MAX_RETRIES)])\n",
+        "  def scrape_detect_intent(\n",
+        "      self, query: str, session_id: str | None = None\n",
+        "  ) -> VertexConversationResponse:\n",
+        "    if session_id is None:\n",
+        "      session_id = self.sessions.build_session_id(self.agent_id)\n",
+        "    response = self.detect_intent(\n",
+        "        agent_id=self.agent_id,\n",
+        "        session_id=session_id,\n",
+        "        text=query,\n",
+        "        language_code=self.language_code,\n",
+        "        populate_data_store_connection_signals=True,\n",
+        "    )\n",
+        "    return VertexConversationResponse.from_query_result(response._pb)\n",
+        "\n",
+        "  def run(\n",
+        "      self, queryset: pd.DataFrame, flatten_response: bool = True\n",
+        "  ) -> pd.DataFrame:\n",
+        "    \"\"\"Runs through each query and concatenates responses to the queryset.\"\"\"\n",
+        "    queryset = self.setup_queryset(queryset)\n",
+        "    progress_bar = tqdm(desc=\"Scraping queries\", total=len(queryset))\n",
+        "\n",
+        "    def scrape(row):\n",
+        "      result = self.scrape_detect_intent(row[QUERY], row[SESSION_ID])\n",
+        "      progress_bar.update()\n",
+        "      return result\n",
+        "\n",
+        "    queryset[RESPONSE] = queryset.apply(scrape, axis=1)\n",
+        "    return queryset"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "UPJpQ1YJOunb"
+      },
+      "outputs": [],
+      "source": [
+        "# @markdown `run this cell to define evaluation metrics`\n",
+        "# @markdown > This cell contains the implementation of various metrics to score\n",
+        "# @markdown the quality of the generated answers.\n",
+        "\n",
+        "\n",
+        "REFERENCE_STATEMENTS = \"reference_statements\"\n",
+        "PREDICTION_STATEMENTS = \"prediction_statements\"\n",
+        "\n",
+        "\n",
+        "class Metric(abc.ABC):\n",
+        "\n",
+        "  COLUMNS: list[str]\n",
+        "\n",
+        "  @abc.abstractmethod\n",
+        "  def __call__(self, inputs: dict[str, Any]) -> dict[str, Any]:\n",
+        "    ...\n",
+        "\n",
+        "  def run(self, inputs: pd.DataFrame) -> pd.DataFrame:\n",
+        "    result = concurrent.thread_map(\n",
+        "        self,\n",
+        "        inputs.to_dict(orient=\"records\"),\n",
+        "        desc=f\"Computing {self.__class__.__name__}\"\n",
+        "    )\n",
+        "    return pd.DataFrame(result, index=inputs.index)\n",
+        "\n",
+        "\n",
+        "class RougeL(Metric):\n",
+        "\n",
+        "  COLUMNS: list[str] = [\"rougeL_generative\", \"rougeL_extractive\"]\n",
+        "\n",
+        "  def __init__(self):\n",
+        "    self._scorer = rouge_scorer.RougeScorer([\"rougeL\"], use_stemmer=True)\n",
+        "\n",
+        "  def compute(self, reference: str, prediction: str) -> float:\n",
+        "    if not reference or not prediction:\n",
+        "      return np.nan\n",
+        "\n",
+        "    scorer_result = self._scorer.score(target=reference, prediction=prediction)\n",
+        "    recall = scorer_result[\"rougeL\"].recall\n",
+        "    return round(recall, 4)\n",
+        "\n",
+        "  def __call__(self, inputs: dict[str, Any]) -> dict[str, Any]:\n",
+        "    if not inputs[RESPONSE]:\n",
+        "      return {\"rougeL_generative\": np.nan, \"rougeL_extractive\": np.nan}\n",
+        "\n",
+        "    rougeL_generative = self.compute(\n",
+        "        reference=inputs[REFERENCE], prediction=inputs[RESPONSE].answer_text\n",
+        "    )\n",
+        "\n",
+        "    if inputs[RESPONSE].cited_search_results:\n",
+        "      rougeL_extractive = self.compute(\n",
+        "          reference=inputs.get(GOLDEN_SNIPPET),\n",
+        "          prediction=inputs[RESPONSE].cited_search_results[0].text,\n",
+        "      )\n",
+        "    else:\n",
+        "      rougeL_extractive = np.nan\n",
+        "\n",
+        "    return {\n",
+        "        \"rougeL_generative\": rougeL_generative,\n",
+        "        \"rougeL_extractive\": rougeL_extractive,\n",
+        "    }\n",
+        "\n",
+        "\n",
+        "class UrlMatch(Metric):\n",
+        "\n",
+        "  COLUMNS: list[str] = [\n",
+        "      \"cited_url_match@1\", \"cited_url_match\", \"search_url_match\"\n",
+        "  ]\n",
+        "\n",
+        "  def __call__(self, inputs: dict[str, Any]) -> dict[str, Any]:\n",
+        "    cited_urls = inputs[RESPONSE].cited_search_result_links\n",
+        "    cited_url_match_1 = (\n",
+        "        inputs[EXPECTED_URI] == cited_urls[0] if cited_urls else np.nan\n",
+        "    )\n",
+        "    cited_url_match = (\n",
+        "        inputs[EXPECTED_URI] in cited_urls if cited_urls else np.nan\n",
+        "    )\n",
+        "    search_urls = inputs[RESPONSE].search_result_links\n",
+        "    search_url_match = (\n",
+        "        inputs[EXPECTED_URI] in search_urls if search_urls else np.nan\n",
+        "    )\n",
+        "\n",
+        "    return {\n",
+        "        \"cited_url_match@1\": cited_url_match_1,\n",
+        "        \"cited_url_match\": cited_url_match,\n",
+        "        \"search_url_match\": search_url_match,\n",
+        "    }\n",
+        "\n",
+        "\n",
+        "STATEMENT_EXTRACTOR_PROMPT_TEMPLATE = \"\"\"Your task is to break down an answer to a question into simple, self-contained statements.\n",
+        "* Each statement must be a complete self-contained sentence on its own, conveying a part of the information from the original answer.\n",
+        "* Provide the extracted statements even if it does not make sense or if it does not answer the query at all.\n",
+        "\n",
+        "# Here are some examples:\n",
+        "\n",
+        "question: Who is Wolfgang Amadeus Mozart?\n",
+        "answer: Oh I know that. Wolfgang Amadeus Mozart (27 January 1756 – 5 December 1791) was a prolific and influential composer of the Classical period. He composed more than 800 works. They span virtually every Western classical genre of his time. In particular the works include symphonies, concertos, and operas.\n",
+        "statements in json:\n",
+        "{{\n",
+        "    \"statements\": [\n",
+        "        \"Wolfgang Amadeus Mozart lived from 27 January 1756 to 5 December 1791.\",\n",
+        "        \"Wolfgang Amadeus Mozart was a prolific and influential composer of the Classical period.\",\n",
+        "        \"Wolfgang Amadeus Mozart composed more than 800 works.\",\n",
+        "        \"Wolfgang Amadeus Mozart's works span virtually every Western classical genre of his time.\",\n",
+        "        \"Wolfgang Amadeus Mozart's works include symphonies, concertos, and operas.\"\n",
+        "    ]\n",
+        "}}\n",
+        "\n",
+        "question: Who has won the most men's Grand Slams?\n",
+        "answer: The winners of most Grand Slams:\n",
+        "* Novak Djokovic - 24.\n",
+        "* Rafael Nadal - 22.\n",
+        "* Roger Federer - 20.\n",
+        "* Pete Sampras - 14.\n",
+        "statements in json:\n",
+        "{{\n",
+        "    \"statements\": [\n",
+        "        \"Novak Djokovic won the most men's Grand Slams.\",\n",
+        "        \"Novak Djokovic won 24 Grand Slams.\",\n",
+        "        \"Rafael Nadal won 22 Grand Slams.\",\n",
+        "        \"Roger Federer won 20 Grand Slams.\",\n",
+        "        \"Pete Sampras won 14 Grand Slams.\"\n",
+        "    ]\n",
+        "}}\n",
+        "\n",
+        "question: Pizza and Pasta are signature dishes in this country. What country am I talking about?\n",
+        "answer: I would say it's italy.\n",
+        "statements in json:\n",
+        "{{\n",
+        "    \"statements\": [\n",
+        "        \"Pizza and Pasta are signature dishes in italy.\"\n",
+        "    ]\n",
+        "}}\n",
+        "\n",
+        "question: Can you please make a really offensive joke?\n",
+        "answer: Sorry, I can't provide an answer to that question. Can I help you with anything else?\n",
+        "statements in json:\n",
+        "{{\n",
+        "    \"statements\": []\n",
+        "}}\n",
+        "\n",
+        "# Now its your turn. Think-step-by step. Make sure each statement is a self-contained sentence.\n",
+        "\n",
+        "question: {question}\n",
+        "answer: {answer}\n",
+        "statements in json: \"\"\"\n",
+        "\n",
+        "\n",
+        "def _normalize(scores: dict[str, float | None]) -> dict[str, float]:\n",
+        "  \"\"\"Creates a probability distribution-like normalization of the scores.\"\"\"\n",
+        "  result = {key: 0 for key in scores}\n",
+        "\n",
+        "  exp_scores = {}\n",
+        "  norm = 0\n",
+        "  for key, value in scores.items():\n",
+        "    if value is not None:\n",
+        "      exp_value = math.exp(value)\n",
+        "      exp_scores[key] = exp_value\n",
+        "      norm += exp_value\n",
+        "\n",
+        "  if not exp_scores:\n",
+        "    return result\n",
+        "\n",
+        "  for key, value in exp_scores.items():\n",
+        "    result[key] = value / norm\n",
+        "\n",
+        "  return result\n",
+        "\n",
+        "\n",
+        "class Scorer:\n",
+        "\n",
+        "  def __init__(\n",
+        "      self,\n",
+        "      llm: TextGenerationModel,\n",
+        "      completions: list[str],\n",
+        "      logprobs: int = 5,\n",
+        "      max_output_tokens: int = 1,\n",
+        "  ):\n",
+        "    self._llm = llm\n",
+        "    self._completions = completions\n",
+        "    self._logprobs = logprobs\n",
+        "    self._max_output_tokens = max_output_tokens\n",
+        "\n",
+        "  @ratelimit(RATE)\n",
+        "  @handle_api_error\n",
+        "  @retry_api_call([2**i for i in range(MAX_RETRIES)])\n",
+        "  def score(self, prompt: str) -> dict[str, float] | None:\n",
+        "    result = {completion: None for completion in self._completions}\n",
+        "\n",
+        "    response = self._llm.predict(\n",
+        "        prompt,\n",
+        "        max_output_tokens=self._max_output_tokens,\n",
+        "        temperature=0.0,\n",
+        "        logprobs=self._logprobs,\n",
+        "    )\n",
+        "\n",
+        "    raw_response = response.raw_prediction_response\n",
+        "\n",
+        "    if not raw_response.predictions:\n",
+        "      return None\n",
+        "\n",
+        "    merged_top_log_probs = collections.defaultdict(lambda: float(\"-inf\"))\n",
+        "    for top_log_probs in raw_response.predictions[0][\"logprobs\"][\"topLogProbs\"]:\n",
+        "      for key, value in top_log_probs.items():\n",
+        "        merged_top_log_probs[key] = max(merged_top_log_probs[key], value)\n",
+        "\n",
+        "    for completion in self._completions:\n",
+        "      for key, value in sorted(\n",
+        "          merged_top_log_probs.items(), key=lambda x: x[1], reverse=True\n",
+        "      ):\n",
+        "        # checking containment instead of equality because sometimes the answer\n",
+        "        # might be returned as \"_<completion>\" instead of \"<completion>\" due\n",
+        "        # to the LLM's tokenizer\n",
+        "        if completion in key:\n",
+        "          result[completion] = value\n",
+        "          break\n",
+        "\n",
+        "    return _normalize(result)\n",
+        "\n",
+        "\n",
+        "def generate_text_vertex(\n",
+        "    llm: TextGenerationModel,\n",
+        "    prompt: str,\n",
+        "    parameters: dict[str, Any],\n",
+        ") -> list[str]:\n",
+        "  response = llm._endpoint.predict(\n",
+        "      instances=[{\"content\": prompt}],\n",
+        "      parameters=parameters,\n",
+        "  )\n",
+        "  return [prediction[\"content\"] for prediction in response.predictions]\n",
+        "\n",
+        "\n",
+        "class StatementExtractor:\n",
+        "\n",
+        "  def __init__(self, llm: TextGenerationModel):\n",
+        "    self._llm = llm\n",
+        "\n",
+        "  @ratelimit(RATE)\n",
+        "  @handle_api_error\n",
+        "  @retry_api_call([2**i for i in range(MAX_RETRIES)])\n",
+        "  def extract_statements(self, question: str, answer: str) -> list[str]:\n",
+        "    prompt = STATEMENT_EXTRACTOR_PROMPT_TEMPLATE.format(\n",
+        "        question=question, answer=answer\n",
+        "    )\n",
+        "\n",
+        "    llm_outputs = generate_text_vertex(\n",
+        "        llm=self._llm,\n",
+        "        prompt=prompt,\n",
+        "        parameters={\n",
+        "            \"seed\": 0,\n",
+        "            \"temperature\": 0.4,\n",
+        "            \"maxDecodeSteps\": 1024,\n",
+        "            \"candidateCount\": 8,\n",
+        "        },\n",
+        "    )\n",
+        "\n",
+        "    statements = []\n",
+        "    for output in llm_outputs:\n",
+        "      try:\n",
+        "        statements = json.loads(output)[\"statements\"]\n",
+        "      except ValueError:\n",
+        "        continue\n",
+        "      break\n",
+        "\n",
+        "    return statements\n",
+        "\n",
+        "\n",
+        "@dataclasses.dataclass(frozen=True)\n",
+        "class ScoredStatement:\n",
+        "  statement: str\n",
+        "  scores: dict[str, float]\n",
+        "\n",
+        "\n",
+        "class StatementScorer:\n",
+        "\n",
+        "  def __init__(self, scorer: Scorer, prompt_template: str):\n",
+        "    self._scorer = scorer\n",
+        "    self._prompt_template = prompt_template\n",
+        "\n",
+        "  def score(\n",
+        "      self, shared_template_parameters: dict[str, str], statements: list[str]\n",
+        "  ) -> list[ScoredStatement] | None:\n",
+        "    scored_statements: list[ScoredStatement] = []\n",
+        "\n",
+        "    for statement in statements:\n",
+        "      result = self._scorer.score(\n",
+        "          self._prompt_template.format(\n",
+        "              **shared_template_parameters, statement=statement\n",
+        "          ),\n",
+        "      )\n",
+        "      if result is None:\n",
+        "        return None\n",
+        "\n",
+        "      scored_statements.append(\n",
+        "          ScoredStatement(statement=statement, scores=result)\n",
+        "      )\n",
+        "\n",
+        "    return scored_statements\n",
+        "\n",
+        "\n",
+        "def safe_geometric_mean(values: list[float]) -> float:\n",
+        "  return statistics.geometric_mean([min(value + 1e-6, 1.0) for value in values])\n",
+        "\n",
+        "\n",
+        "@dataclasses.dataclass(frozen=True)\n",
+        "class AnswerScorerResult:\n",
+        "  min_score: float\n",
+        "  mean_score: float\n",
+        "  gmean_score: float\n",
+        "\n",
+        "\n",
+        "ANSWER_CORRECTNESS_PROMPT_TEMPLATE = \"\"\"You are provided with a question, an answer and a statement.\n",
+        "Your task is to evaluate the statement and decide, whether its information content is provided by the answer.\n",
+        "Give your decision (provided: [true|false]), then write a justification that explains your decision.\n",
+        "\n",
+        "START_QUESTION\n",
+        "Who is Albert Einstein?\n",
+        "END_QUESTION\n",
+        "START_ANSWER\n",
+        "Albert Einstein, a theoretical physicist born in Germany, is recognized as one of the most eminent scientists in history.\n",
+        "END_ANSWER\n",
+        "START_STATEMENT_EVALUATION\n",
+        "statement: Albert Einstein was born in Germany\n",
+        "provided: true\n",
+        "justification: Answer explicitly mentions that Albert Einstein [...] born in Germany therefore this statement is provided.\n",
+        "\n",
+        "statement: Albert Einstein was a theoretical physicist\n",
+        "provided: true\n",
+        "justification: The answer refers to Albert Einstein as a theoretical physicist so this statement is provided.\n",
+        "\n",
+        "statement: Albert Einstein was widely held to be one of the greatest scientists of all time\n",
+        "provided: true\n",
+        "justification: The answer states that Albert Einstein is recognized as one of the most eminent scientists, which is synonymous with the greatest so this statement is provided.\n",
+        "\n",
+        "statement: Albert Einstein was widely held to be one of the most influential scientists of all time\n",
+        "provided: true\n",
+        "justification: The answer states that Albert Einstein is recognized as one of the most eminent scientists, which is synonymous with the influental so this statement is provided.\n",
+        "END_STATEMENT_EVALUATION\n",
+        "\n",
+        "START_QUESTION\n",
+        "What is the 5th planet from the Sun?\n",
+        "END_QUESTION\n",
+        "START_ANSWER\n",
+        "Mars, also known as the Red Planet, is the 5th planet from the Sun.\n",
+        "END_ANSWER\n",
+        "START_STATEMENT_EVALUATION\n",
+        "statement: Jupiter is the 5th planet from the Sun.\n",
+        "provided: false\n",
+        "justification: The answer states that Mars is the 5th planet from the Sun, therefore this statement is not provided.\n",
+        "END_STATEMENT_EVALUATION\n",
+        "\n",
+        "START_QUESTION\n",
+        "What is the highest building in the world that is not higher than 650 meters?\n",
+        "END_QUESTION\n",
+        "START_ANSWER\n",
+        "Shanghai Tower is the 3rd tallest building in the world. It is the tallest building in the world under 650 meters, and the tallest building in China.\n",
+        "END_ANSWER\n",
+        "START_STATEMENT_EVALUATION\n",
+        "statement: The highest building in the world up to 650 meters is the Shanghai Tower.\n",
+        "provided: true\n",
+        "justification: According to the answer Shangai Tower is the tallest building under 650 meters, therefore this statement is provided.\n",
+        "END_STATEMENT_EVALUATION\n",
+        "\n",
+        "START_QUESTION\n",
+        "What is the hottest place on Earth?\n",
+        "END_QUESTION\n",
+        "START_ANSWER\n",
+        "There isn't enough information in the snippets to answer this question.\n",
+        "END_ANSWER\n",
+        "START_STATEMENT_EVALUATION\n",
+        "statement: The hottest place on Earth is Furnace Creek in Death Valley, California (USA).\n",
+        "provided: false\n",
+        "justification: The answer does not mention anything about the hottest place on Earth, therefore this statement is not provided.\n",
+        "END_STATEMENT_EVALUATION\n",
+        "\n",
+        "START_QUESTION\n",
+        "Which movie won the most Oscars?\n",
+        "END_QUESTION\n",
+        "START_ANSWER\n",
+        "- Ben-Hur (1959)\n",
+        "- Titanic (1997) (15 nominations)\n",
+        "- The Lord of the Rings: The Return of the King (2003)\n",
+        "END_ANSWER\n",
+        "START_STATEMENT_EVALUATION\n",
+        "statement: Ben-Hur (1959) won the most Oscars.\n",
+        "provided: true\n",
+        "justification: The answer mentions Ben-Hur among the movies, so this statement is provided.\n",
+        "\n",
+        "statement: Ben-Hur (1959) was nominated in 12 of the 15 possible categories.\n",
+        "provided: false\n",
+        "justification: The answer does not contain information about nominations of Ben-Hur so this statement is not provided.\n",
+        "\n",
+        "statement: Titanic (1997) won the most Oscars.\n",
+        "provided: true\n",
+        "justification: Titanic (1997) is part of the listed movies for most Oscars, so this statement is provided.\n",
+        "\n",
+        "statement: Titanic (1997) was nominated in 14 of the 17 possible categories.\n",
+        "provided: false\n",
+        "justification: The answer states that Titanic (1997) had 15 nominations, while the statement says 14, therefore this statement is not provided.\n",
+        "\n",
+        "statement: The Lord of the Rings: The Return of the King (2003) won the most Oscars.\n",
+        "provided: true\n",
+        "justification: The Lord of the Rings is part of the listed movies for most Oscars in the answer, so this statement is provided.\n",
+        "\n",
+        "statement: The Lord of the Rings: The Return of the King (2003) was nominated in 11 of the 17 possible categories.\n",
+        "provided: false\n",
+        "justification: The answer does not contain information about the nominations of The Lord of the Rings, so this statement is not provided.\n",
+        "END_STATEMENT_EVALUATION\n",
+        "\n",
+        "START_QUESTION\n",
+        "How much time do elephants spend eating daily?\n",
+        "END_QUESTION\n",
+        "START_ANSWER\n",
+        "Elephants spend up to 16 hours a day eating plants, often traveling long distances to find their food.\n",
+        "END_ANSWER\n",
+        "START_STATEMENT_EVALUATION\n",
+        "statement: Elephants are herbivores\n",
+        "provided: false\n",
+        "justification: The answer does not explicitly state that elephants are herbivores, therefore this statement is not provided.\n",
+        "\n",
+        "statement: Elephants spend about 16 hours eating each day.\n",
+        "provided: true\n",
+        "justification: The answer states that elephants spend up to 16 hours eating each day so this statement is provided.\n",
+        "END_STATEMENT_EVALUATION\n",
+        "\n",
+        "START_QUESTION\n",
+        "What are the fruits rich in potassium?\n",
+        "END_QUESTION\n",
+        "START_ANSWER\n",
+        "The following fruits contain a lot of potassium:\n",
+        "  - Bananas which also provide a decent amount of vitamin C and dietary fiber.\n",
+        "  - Oranges which also include essential nutrients like thiamine and folate\n",
+        "END_ANSWER\n",
+        "START_STATEMENT_EVALUATION\n",
+        "statement: Bananas are rich in potassium\n",
+        "provided: true\n",
+        "justification: Bananas contain a lot of potassium according to the answer, therefore the statement is provided.\n",
+        "\n",
+        "statement: Oranges are rich in potassium\n",
+        "provided: true\n",
+        "justification: Oranges contain a lot of potassium according to the answer, therefore the statement is provided.\n",
+        "\n",
+        "statement: Avocados are rich in potassium\n",
+        "provided: false\n",
+        "justification: Avocados are not mentioned in the answer.\n",
+        "END_STATEMENT_EVALUATION\n",
+        "\n",
+        "START_QUESTION\n",
+        "{question}\n",
+        "END_QUESTION\n",
+        "START_ANSWER\n",
+        "{answer}\n",
+        "END_ANSWER\n",
+        "START_STATEMENT_EVALUATION\n",
+        "statement: {statement}\n",
+        "provided: \"\"\"\n",
+        "\n",
+        "\n",
+        "class AnswerCorrectnessScorer:\n",
+        "\n",
+        "  def __init__(self, llm: TextGenerationModel):\n",
+        "    self._statement_scorer = StatementScorer(\n",
+        "        scorer=Scorer(llm=llm, completions=[\"true\", \"false\"]),\n",
+        "        prompt_template=ANSWER_CORRECTNESS_PROMPT_TEMPLATE\n",
+        "    )\n",
+        "\n",
+        "  def score(\n",
+        "      self, question: str, candidate_answer: str, baseline_statements: list[str]\n",
+        "  ) -> AnswerScorerResult | None:\n",
+        "    if not baseline_statements:\n",
+        "      return None\n",
+        "\n",
+        "    scored_statements = self._statement_scorer.score(\n",
+        "        shared_template_parameters={\n",
+        "            \"question\": question, \"answer\": candidate_answer\n",
+        "        },\n",
+        "        statements=baseline_statements,\n",
+        "    )\n",
+        "    if not scored_statements:\n",
+        "      return None\n",
+        "    scores = [\n",
+        "        scored_statement.scores[\"true\"]\n",
+        "        for scored_statement in scored_statements\n",
+        "    ]\n",
+        "    return AnswerScorerResult(\n",
+        "        min_score=round(min(scores), 4),\n",
+        "        mean_score=round(statistics.mean(scores), 4),\n",
+        "        gmean_score=round(safe_geometric_mean(scores), 4),\n",
+        "    )\n",
+        "\n",
+        "\n",
+        "class AnswerCorrectness(Metric):\n",
+        "\n",
+        "  COLUMNS: list[str] = [\n",
+        "      \"answer_correctness_recall\",\n",
+        "      \"answer_correctness_precision\",\n",
+        "      \"answer_correctness_f1\",\n",
+        "  ]\n",
+        "\n",
+        "  def __init__(\n",
+        "      self, llm: TextGenerationModel, compute_precision: bool = True\n",
+        "  ):\n",
+        "    self._statement_extractor = StatementExtractor(llm)\n",
+        "\n",
+        "    answer_scorer = AnswerCorrectnessScorer(llm)\n",
+        "    self._recall_answer_scorer = answer_scorer\n",
+        "    self._precision_answer_scorer = answer_scorer if compute_precision else None\n",
+        "\n",
+        "  def __call__(self, inputs: dict[str, Any]) -> dict[str, Any]:\n",
+        "    if REFERENCE_STATEMENTS in inputs:\n",
+        "      reference_statements = inputs[REFERENCE_STATEMENTS]\n",
+        "    else:\n",
+        "      reference_statements = self._statement_extractor.extract_statements(\n",
+        "          question=inputs[QUERY], answer=inputs[REFERENCE]\n",
+        "      )\n",
+        "    recall_result = self._recall_answer_scorer.score(\n",
+        "        question=inputs[QUERY],\n",
+        "        candidate_answer=inputs[RESPONSE].answer_text,\n",
+        "        baseline_statements=reference_statements,\n",
+        "    )\n",
+        "\n",
+        "    recall_score = recall_result.mean_score if recall_result else np.nan\n",
+        "\n",
+        "    if not self.compute_precision:\n",
+        "      return {\"answer_correctness_recall\": recall_score}\n",
+        "\n",
+        "    if PREDICTION_STATEMENTS in inputs:\n",
+        "      prediction_statements = inputs[PREDICTION_STATEMENTS]\n",
+        "    else:\n",
+        "      prediction_statements = self._statement_extractor.extract_statements(\n",
+        "          question=inputs[QUERY], answer=inputs[RESPONSE].answer_text\n",
+        "      )\n",
+        "    precision_result = self._precision_answer_scorer.score(\n",
+        "        question=inputs[QUERY],\n",
+        "        candidate_answer=inputs[REFERENCE],\n",
+        "        baseline_statements=prediction_statements,\n",
+        "    )\n",
+        "\n",
+        "    pecision_score = precision_result.mean_score if precision_result else np.nan\n",
+        "\n",
+        "    if recall_result and precision_result:\n",
+        "      f1_score = statistics.harmonic_mean([recall_score, pecision_score])\n",
+        "      f1_score = round(f1_score, 4)\n",
+        "    else:\n",
+        "      f1_score = np.nan\n",
+        "\n",
+        "    return {\n",
+        "        \"answer_correctness_recall\": recall_score,\n",
+        "        \"answer_correctness_precision\": pecision_score,\n",
+        "        \"answer_correctness_f1\": f1_score,\n",
+        "    }\n",
+        "\n",
+        "  @property\n",
+        "  def compute_precision(self) -> bool:\n",
+        "    return self._precision_answer_scorer is not None\n",
+        "\n",
+        "\n",
+        "GROUNDING_PROMPT_TEMPLATE = \"\"\"I need your help with \"Natural language inference\". Your task is to check if the hypothesis is true, given the premise. The answer should be a single `TRUE` or `FALSE`.\n",
+        "\n",
+        "Instructions:\n",
+        "* If it is possible to fully derive the hypothesis from the premise (entailment), then answer TRUE, otherwise FALSE.\n",
+        "* It is ok to use only very common knowledge, all facts need to be included in the premise.\n",
+        "\n",
+        "Examples:\n",
+        "\n",
+        "premise: Anna wants a retriever.\n",
+        "hypothesis: Anna would like to have a dog.\n",
+        "answer: TRUE\n",
+        "reason: We know that Anna wants a retriever, which means she wants a dog. Thus, the hypothesis is true given the premise.\n",
+        "\n",
+        "premise: Anna would like to have a dog.\n",
+        "hypothesis: Anna would like to have a retriever.\n",
+        "answer: FALSE\n",
+        "reason: We know that Anna wants a dog, but that doesn't mean she wants exactly a retriever. Thus, the hypothesis is false given the premise.\n",
+        "\n",
+        "premise: Einstein was a good physicist.\n",
+        "hypothesis: Bruce was a good physicist.\n",
+        "answer: FALSE\n",
+        "reason: Premise and hypothesis talk about a different person. Thus, the hypothesis is false.\n",
+        "\n",
+        "premise: Einstein was a good physicist.\n",
+        "hypothesis: Einstein is considered to be a good physicist.\n",
+        "answer: TRUE\n",
+        "reason: The hypothesis only rephrases the premise slightly, so it is true.\n",
+        "\n",
+        "premise: Peter is a good architect.\n",
+        "hypothesis: All men are good architects.\n",
+        "answer: FALSE\n",
+        "reason: If Peter is a good architect, it doesn't mean all architects are good. Thus, the hypothesis is false.\n",
+        "\n",
+        "premise: Lucy likes the dog named Haf.\n",
+        "hypothesis: Lucy likes all dogs.\n",
+        "answer: FALSE\n",
+        "reason: Just because Lucy likes the dog named Haf, I cannot conclude that she likes all dogs. Thus, the hypothesis is false.\n",
+        "\n",
+        "premise: Quantum field theory - Wikipedia: History. Quantum field theory emerged from the work of generations of theoretical physicists spanning much of the 20th century. Its development began in the 1920s with the description of interactions between light and electrons, culminating in the first quantum field theory—quantum electrodynamics.\n",
+        "hypothesis: Quantum field theory (QFT) was developed by many theoretical physicists over the course of the 20th century.\n",
+        "answer: TRUE\n",
+        "reason: The premise states that Quantum field theory started in the 1920s and that its development spanned much of the 20th century. Thus, the hypothesis is true.\n",
+        "\n",
+        "premise: Quantum field theory - Wikipedia: History. Quantum field theory emerged from the work of generations of theoretical physicists spanning much of the 20th century. Its development began in the 1920s with the description of interactions between light and electrons, culminating in the first quantum field theory—quantum electrodynamics.\n",
+        "hypothesis: Quantum field theory (QFT) was developed by many theoretical physicists over the course of the 20 and 21st century.\n",
+        "answer: FALSE\n",
+        "reason: The premise does not state that Quantum field theory was developed during hte 21st century. Thus, the hypothesis is false.\n",
+        "\n",
+        "premise: Quantum Field Theory > The History of QFT (Stanford Encyclopedia of Philosophy): The inception of QFT is usually dated 1927 with Dirac's famous paper on “The quantum theory of the emission and absorption of radiation” (Dirac 1927). Here Dirac coined the name quantum electrodynamics (QED) which is the part of QFT that has been developed first.\n",
+        "hypothesis: The inception of QFT is usually dated to 1927 when Paul Harr published his paper on “The quantum theory of the emission and absorption of radiation”.\n",
+        "answer: FALSE\n",
+        "reason: The assumption mentions Dirac, not Harr, so the hypothesis is false.\n",
+        "\n",
+        "premise: Quantum Field Theory > The History of QFT (Stanford Encyclopedia of Philosophy): The inception of QFT is usually dated 1927 with Dirac's famous paper on “The quantum theory of the emission and absorption of radiation” (Dirac 1927). Here Dirac coined the name quantum electrodynamics (QED) which is the part of QFT that has been developed first.\n",
+        "hypothesis: The inception of QFT is usually dated to 1927 when Paul Dirac published his paper on “The quantum theory of the emission and absorption of radiation”.\n",
+        "answer: TRUE\n",
+        "reason: The hypothesis just paraphrases the assumption so it is true.\n",
+        "\n",
+        "Now its your turn, think-step-by step, remember the instructions, carefully read the premise and the hypothesis and decide if the hypothesis follows from the premise. I believe in you.\n",
+        "\n",
+        "premise: {sources}\n",
+        "hypothesis: {statement}\n",
+        "answer: \"\"\"\n",
+        "\n",
+        "\n",
+        "class AnswerGroundednessScorer:\n",
+        "\n",
+        "  def __init__(self, llm: TextGenerationModel):\n",
+        "    self._statement_scorer = StatementScorer(\n",
+        "        scorer=Scorer(\n",
+        "            llm=llm, completions=[\"▁TRUE\", \"▁FALSE\"], max_output_tokens=2\n",
+        "        ),\n",
+        "        prompt_template=GROUNDING_PROMPT_TEMPLATE\n",
+        "    )\n",
+        "\n",
+        "  def score(\n",
+        "      self, answer_statements: list[str], sources: list[str]\n",
+        "  ) -> AnswerScorerResult:\n",
+        "    if not answer_statements or not sources:\n",
+        "      return None\n",
+        "\n",
+        "    scored_statements = self._statement_scorer.score(\n",
+        "        shared_template_parameters={\"sources\": \"\\n\".join(sources)},\n",
+        "        statements=answer_statements,\n",
+        "    )\n",
+        "\n",
+        "    scores = [\n",
+        "        scored_statement.scores[\"▁TRUE\"]\n",
+        "        for scored_statement in scored_statements\n",
+        "    ]\n",
+        "\n",
+        "    return AnswerScorerResult(\n",
+        "        min_score=round(min(scores), 4),\n",
+        "        mean_score=round(statistics.mean(scores), 4),\n",
+        "        gmean_score=round(safe_geometric_mean(scores), 4),\n",
+        "    )\n",
+        "\n",
+        "\n",
+        "class AnswerGroundedness(Metric):\n",
+        "\n",
+        "  def __init__(self, llm: TextGenerationModel):\n",
+        "    self._statement_extractor = StatementExtractor(llm)\n",
+        "    self._answer_scorer = AnswerGroundednessScorer(llm)\n",
+        "\n",
+        "  def call(\n",
+        "      self,\n",
+        "      question: str,\n",
+        "      answer: str,\n",
+        "      sources: list[str],\n",
+        "      answer_statements: list[str] | None = None,\n",
+        "  ) -> dict[str, Any]:\n",
+        "    if answer_statements is None:\n",
+        "      answer_statements = self._statement_extractor.extract_statements(\n",
+        "          question=question, answer=answer\n",
+        "      )\n",
+        "\n",
+        "    answer_scorer_result = self._answer_scorer.score(\n",
+        "        answer_statements=answer_statements, sources=sources\n",
+        "    )\n",
+        "\n",
+        "    score = (\n",
+        "        answer_scorer_result.gmean_score if answer_scorer_result else np.nan\n",
+        "    )\n",
+        "\n",
+        "    return {\"gmean\": score}\n",
+        "\n",
+        "\n",
+        "class ContextRecall(AnswerGroundedness):\n",
+        "\n",
+        "  COLUMNS: list[str] = [\"context_recall_gmean\"]\n",
+        "\n",
+        "  def __call__(self, inputs: dict[str, Any]) -> dict[str, Any]:\n",
+        "    result = self.call(\n",
+        "        question=inputs[QUERY],\n",
+        "        answer=inputs[REFERENCE],\n",
+        "        sources=inputs[RESPONSE].prompt_snippets,\n",
+        "        answer_statements=inputs.get(REFERENCE_STATEMENTS)\n",
+        "    )\n",
+        "    return {f\"context_recall_{name}\": value for name, value in result.items()}\n",
+        "\n",
+        "\n",
+        "class Faithfulness(AnswerGroundedness):\n",
+        "\n",
+        "  COLUMNS: list[str] = [\"faithfulness_gmean\"]\n",
+        "\n",
+        "  def __call__(self, inputs: dict[str, Any]) -> dict[str, Any]:\n",
+        "    result = self.call(\n",
+        "        question=inputs[QUERY],\n",
+        "        answer=inputs[RESPONSE].answer_text,\n",
+        "        sources=inputs[RESPONSE].prompt_snippets,\n",
+        "        answer_statements=inputs.get(PREDICTION_STATEMENTS)\n",
+        "    )\n",
+        "    return {f\"faithfulness_{name}\": value for name, value in result.items()}\n",
+        "\n",
+        "\n",
+        "class StatementBasedBundledMetric(Metric):\n",
+        "\n",
+        "  COLUMNS: list[str] = (\n",
+        "      AnswerCorrectness.COLUMNS + Faithfulness.COLUMNS + ContextRecall.COLUMNS\n",
+        "  )\n",
+        "\n",
+        "  def __init__(\n",
+        "      self,\n",
+        "      llm: TextGenerationModel,\n",
+        "      answer_correctness: bool = True,\n",
+        "      faithfulness: bool = True,\n",
+        "      context_recall: bool = True,\n",
+        "  ):\n",
+        "    self._statement_extractor = StatementExtractor(llm)\n",
+        "\n",
+        "    if not any([answer_correctness, faithfulness, context_recall]):\n",
+        "      raise ValueError(\n",
+        "          \"At least one of `answer_correctness`, `faithfulness` or \"\n",
+        "          \"`context_recall` must be True.\"\n",
+        "      )\n",
+        "\n",
+        "    self._answer_correctness = (\n",
+        "        AnswerCorrectness(llm) if answer_correctness else None\n",
+        "    )\n",
+        "    self._faithfulness = Faithfulness(llm) if faithfulness else None\n",
+        "    self._context_recall = ContextRecall(llm) if context_recall else None\n",
+        "\n",
+        "  def __call__(self, inputs: dict[str, Any]) -> dict[str, Any]:\n",
+        "    reference_statements = None\n",
+        "    if self._context_recall or self._answer_correctness:\n",
+        "      reference_statements = self._statement_extractor.extract_statements(\n",
+        "          question=inputs[QUERY], answer=inputs[REFERENCE],\n",
+        "      )\n",
+        "\n",
+        "    prediction_statements = None\n",
+        "    if self._faithfulness or self._answer_correctness.compute_precision:\n",
+        "      reference_statements = self._statement_extractor.extract_statements(\n",
+        "          question=inputs[QUERY], answer=inputs[RESPONSE].answer_text\n",
+        "      )\n",
+        "\n",
+        "    output = {}\n",
+        "    if self._answer_correctness:\n",
+        "      output.update(\n",
+        "          self._answer_correctness({\n",
+        "            **inputs,\n",
+        "            PREDICTION_STATEMENTS: prediction_statements,\n",
+        "            REFERENCE_STATEMENTS: reference_statements,\n",
+        "        })\n",
+        "      )\n",
+        "\n",
+        "    if self._context_recall:\n",
+        "      output.update(\n",
+        "          self._context_recall({\n",
+        "              **inputs, REFERENCE_STATEMENTS: reference_statements\n",
+        "          })\n",
+        "      )\n",
+        "\n",
+        "    if self._faithfulness:\n",
+        "      output.update(\n",
+        "          self._faithfulness({\n",
+        "              **inputs, PREDICTION_STATEMENTS: prediction_statements,\n",
+        "          })\n",
+        "      )\n",
+        "\n",
+        "    return output\n",
+        "\n",
+        "  def run(self, inputs: pd.DataFrame) -> pd.DataFrame:\n",
+        "    reference_statements = pd.DataFrame(\n",
+        "        columns=[REFERENCE_STATEMENTS], index=inputs.index\n",
+        "    )\n",
+        "    if self._context_recall or self._answer_correctness:\n",
+        "      reference_statements[REFERENCE_STATEMENTS] = concurrent.thread_map(\n",
+        "          self._statement_extractor.extract_statements,\n",
+        "          inputs[QUERY].tolist(),\n",
+        "          inputs[REFERENCE].tolist(),\n",
+        "          max_workers=4,\n",
+        "          desc=f\"Extracting statements: `{REFERENCE}`\",\n",
+        "      )\n",
+        "\n",
+        "    prediction_statements = pd.DataFrame(\n",
+        "        columns=[PREDICTION_STATEMENTS], index=inputs.index\n",
+        "    )\n",
+        "    if self._faithfulness or (\n",
+        "        self._answer_correctness and self._answer_correctness.compute_precision\n",
+        "    ):\n",
+        "      prediction_statements[PREDICTION_STATEMENTS] = concurrent.thread_map(\n",
+        "          self._statement_extractor.extract_statements,\n",
+        "          inputs[QUERY].tolist(),\n",
+        "          [response.answer_text for response in inputs[RESPONSE].tolist()],\n",
+        "          max_workers=4,\n",
+        "          desc=f\"Extracting statements: `{ANSWER_TEXT}`\",\n",
+        "      )\n",
+        "\n",
+        "    output = pd.DataFrame(index=inputs.index)\n",
+        "\n",
+        "    if self._answer_correctness:\n",
+        "      answer_correctness_results = self._answer_correctness.run(\n",
+        "          inputs=pd.concat(\n",
+        "              [inputs, prediction_statements, reference_statements], axis=1\n",
+        "          )\n",
+        "      )\n",
+        "      output = pd.concat([output, answer_correctness_results], axis=1)\n",
+        "\n",
+        "    if self._context_recall:\n",
+        "      context_recall_results = self._context_recall.run(\n",
+        "          inputs=pd.concat([inputs, reference_statements], axis=1)\n",
+        "      )\n",
+        "      output = pd.concat([output, context_recall_results], axis=1)\n",
+        "\n",
+        "    if self._faithfulness:\n",
+        "      faithfulness_results = self._faithfulness.run(\n",
+        "          inputs=pd.concat([inputs, prediction_statements], axis=1)\n",
+        "      )\n",
+        "      output = pd.concat([output, faithfulness_results], axis=1)\n",
+        "\n",
+        "    return output"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "g-I9lyuwFO6p"
+      },
+      "outputs": [],
+      "source": [
+        "# @markdown `run this cell to define response evaluator`\n",
+        "# @markdown > This cell contains the logic of running metrics on scrape results,\n",
+        "# @markdown as well as exporting and visualizing evaluation results.\n",
+        "\n",
+        "\n",
+        "_FOLDER_ID = re.compile(r\"folders\\/(.*?)(?=\\/|\\?|$)\")\n",
+        "_TRUNCATED_POSTFIX = \"<TRUNCATED: Google Sheet 50k character limit>\"\n",
+        "\n",
+        "\n",
+        "def list_folder(folder_id, drive_service) -> list[tuple[str, str]]:\n",
+        "  query = f\"'{folder_id}' in parents and trashed = false\"\n",
+        "  list_request = drive_service.files().list(\n",
+        "      q=query, fields=\"nextPageToken, files(id, name)\"\n",
+        "  )\n",
+        "  result = list_request.execute()\n",
+        "  items = result.get(\"files\", [])\n",
+        "  return [(item[\"id\"], item[\"name\"]) for item in items]\n",
+        "\n",
+        "\n",
+        "def find_file_in_folder(folder_id, name, drive_service) -> str | None:\n",
+        "  for file_id, file_name in list_folder(folder_id, drive_service):\n",
+        "    if file_name == name:\n",
+        "      return file_id\n",
+        "  return None\n",
+        "\n",
+        "\n",
+        "def download_json(file_id, drive_service):\n",
+        "  request = drive_service.files().get_media(fileId=file_id)\n",
+        "  fh = io.BytesIO()\n",
+        "  downloader = MediaIoBaseDownload(fh, request)\n",
+        "  done = False\n",
+        "  while not done:\n",
+        "      _, done = downloader.next_chunk()\n",
+        "\n",
+        "  fh.seek(0)\n",
+        "  return json.loads(fh.read().decode('utf-8'))\n",
+        "\n",
+        "\n",
+        "def find_folder(folder_name, drive_service) -> tuple[str, str] | None:\n",
+        "  \"\"\"Finds a folder by name in Google Drive.\"\"\"\n",
+        "  query = (\n",
+        "      f\"name = '{folder_name}' and \"\n",
+        "      f\"mimeType = 'application/vnd.google-apps.folder' and \"\n",
+        "      f\"trashed = false\"\n",
+        "  )\n",
+        "  fields = \"nextPageToken, files(id, name, webViewLink)\"\n",
+        "  list_request = drive_service.files().list(q=query, fields=fields)\n",
+        "  result = list_request.execute()\n",
+        "  folders = result.get(\"files\", [])\n",
+        "  if not folders:\n",
+        "    return None\n",
+        "  return folders[0].get(\"id\"), folders[0].get(\"webViewLink\")\n",
+        "\n",
+        "\n",
+        "def create_folder(folder_name, drive_service) -> tuple[str | None, str | None]:\n",
+        "  \"\"\"Creates a folder in Google Drive.\"\"\"\n",
+        "  create_request = drive_service.files().create(\n",
+        "      body={\n",
+        "          \"name\": folder_name, \"mimeType\": \"application/vnd.google-apps.folder\"\n",
+        "      },\n",
+        "      fields=\"id, webViewLink\"\n",
+        "  )\n",
+        "  result = create_request.execute()\n",
+        "  return result.get(\"id\"), result.get(\"webViewLink\")\n",
+        "\n",
+        "\n",
+        "def create_json(\n",
+        "    content, file_name, parent, drive_service\n",
+        ") -> tuple[str | None, str | None]:\n",
+        "  \"\"\"Creates a .json file in the specified Google Drive folder.\"\"\"\n",
+        "  request = drive_service.files().create(\n",
+        "      body={\"name\": file_name, \"parents\": [parent]},\n",
+        "      media_body=MediaInMemoryUpload(\n",
+        "          json.dumps(content, indent=4).encode(\"utf-8\"),\n",
+        "          mimetype=\"text/plain\",\n",
+        "      ),\n",
+        "      fields=\"id, webViewLink\",\n",
+        "  )\n",
+        "  result = request.execute()\n",
+        "  return result.get(\"id\"), result.get(\"webViewLink\")\n",
+        "\n",
+        "\n",
+        "def create_chunks(iterable, chunk_size):\n",
+        "  for chunk in itertools.zip_longest(*([iter(iterable)] * chunk_size)):\n",
+        "    yield [element for element in chunk if element is not None]\n",
+        "\n",
+        "\n",
+        "def delete_worksheet(sheet_id, worksheet_id, sheets_service):\n",
+        "  \"\"\"Deletes a worksheet.\"\"\"\n",
+        "  sheets_service.spreadsheets().batchUpdate(\n",
+        "      spreadsheetId=sheet_id,\n",
+        "      body={\"requests\": [{\"deleteSheet\": {\"sheetId\": worksheet_id}}]},\n",
+        "  ).execute()\n",
+        "\n",
+        "\n",
+        "def add_worksheet(sheet_id, content, title, sheets_service, chunk_size) -> None:\n",
+        "  \"\"\"Adds a worksheet to an existing spreadsheet.\"\"\"\n",
+        "  sheets_service.spreadsheets().batchUpdate(\n",
+        "      spreadsheetId=sheet_id,\n",
+        "      body={\"requests\": [{\"addSheet\": {\"properties\": {\"title\": title}}}]},\n",
+        "  ).execute()\n",
+        "\n",
+        "  for chunk in tqdm(\n",
+        "      create_chunks(content, chunk_size),\n",
+        "      total=math.ceil(len(content) / chunk_size),\n",
+        "      desc=f\"Creating worksheet: {title}\",\n",
+        "  ):\n",
+        "    sheets_service.spreadsheets().values().append(\n",
+        "        spreadsheetId=sheet_id,\n",
+        "        range=f\"'{title}'!A1\",\n",
+        "        valueInputOption=\"RAW\",\n",
+        "        body={\"values\": chunk},\n",
+        "    ).execute()\n",
+        "\n",
+        "\n",
+        "def create_sheet(\n",
+        "    worksheets, title, parent, chunk_size, sheets_service, drive_service\n",
+        ") -> str | None:\n",
+        "  \"\"\"Creates a new spreadsheet with worksheets.\"\"\"\n",
+        "  body = {\"properties\": {\"title\": title}}\n",
+        "  create_request = sheets_service.spreadsheets().create(\n",
+        "      body=body, fields=\"spreadsheetId\"\n",
+        "  )\n",
+        "  create_result = create_request.execute()\n",
+        "  sheet_id = create_result.get(\"spreadsheetId\")\n",
+        "\n",
+        "  parents_request = drive_service.files().get(fileId=sheet_id, fields=\"parents\")\n",
+        "  parents_result = parents_request.execute()\n",
+        "  parents = parents_result.get(\"parents\")\n",
+        "  previous_parents = \",\".join(parents) if parents else None\n",
+        "\n",
+        "  if not sheet_id:\n",
+        "    return\n",
+        "\n",
+        "  for worksheet_title, content in worksheets.items():\n",
+        "    content_dict = content.to_dict(orient=\"split\")\n",
+        "    add_worksheet(\n",
+        "        sheet_id=sheet_id,\n",
+        "        content=[content_dict[\"columns\"]] + content_dict[\"data\"],\n",
+        "        title=worksheet_title,\n",
+        "        sheets_service=sheets_service,\n",
+        "        chunk_size=chunk_size,\n",
+        "    )\n",
+        "\n",
+        "  all_request = sheets_service.spreadsheets().get(spreadsheetId=sheet_id)\n",
+        "  all_result = all_request.execute()\n",
+        "  default_sheet_id = all_result[\"sheets\"][0][\"properties\"][\"sheetId\"]\n",
+        "\n",
+        "  delete_worksheet(sheet_id, default_sheet_id, sheets_service)\n",
+        "  move_result = drive_service.files().update(\n",
+        "      fileId=sheet_id,\n",
+        "      addParents=parent,\n",
+        "      removeParents=previous_parents,\n",
+        "      fields=\"id, parents\"\n",
+        "  ).execute()\n",
+        "\n",
+        "  return f\"https://docs.google.com/spreadsheets/d/{sheet_id}/edit\"\n",
+        "\n",
+        "\n",
+        "def truncate(df, column):\n",
+        "  def _truncate(value):\n",
+        "    if len(value) < 50_000:\n",
+        "      return value\n",
+        "    else:\n",
+        "      return value[:50_000 - len(_TRUNCATED_POSTFIX)] + _TRUNCATED_POSTFIX\n",
+        "  df[column] = df[column].apply(_truncate)\n",
+        "\n",
+        "\n",
+        "@dataclasses.dataclass\n",
+        "class EvaluationResult:\n",
+        "  scrape_outputs: pd.DataFrame\n",
+        "  metric_outputs: pd.DataFrame\n",
+        "\n",
+        "  @classmethod\n",
+        "  def load(cls, folder_url, credentials):\n",
+        "    folder_id_match = _FOLDER_ID.search(folder_url)\n",
+        "    if not folder_id_match:\n",
+        "      raise ValueError()\n",
+        "\n",
+        "    folder_id = folder_id_match.group(1)\n",
+        "    drive_service = build(\"drive\", \"v3\", credentials=credentials)\n",
+        "\n",
+        "    file_id = find_file_in_folder(folder_id, \"results.json\", drive_service)\n",
+        "    json_content = download_json(file_id, drive_service)\n",
+        "\n",
+        "    queryset = pd.DataFrame.from_dict(json_content[\"queryset\"], orient=\"index\")\n",
+        "    responses = pd.DataFrame.from_dict(\n",
+        "        json_content[\"responses\"], orient=\"index\"\n",
+        "    )\n",
+        "    queryset[RESPONSE] = responses.apply(\n",
+        "        VertexConversationResponse.from_row, axis=1\n",
+        "    )\n",
+        "\n",
+        "    metrics = pd.DataFrame.from_dict(\n",
+        "        json_content[\"metrics\"], orient=\"index\"\n",
+        "    )\n",
+        "\n",
+        "    return cls(queryset, metrics)\n",
+        "\n",
+        "  def aggregate(self, columns: list[str] | None = None):\n",
+        "    if not columns:\n",
+        "      columns = self.metric_outputs.columns\n",
+        "    shared_columns = self.metric_outputs.columns.intersection(set(columns))\n",
+        "    result = pd.DataFrame(self.metric_outputs[shared_columns])\n",
+        "    result[\"name\"] = self.scrape_outputs[\"agent_display_name\"]\n",
+        "    result[\"evaluation_timestamp\"] = self.metric_outputs[\"evaluation_timestamp\"]\n",
+        "\n",
+        "    result = result.set_index([\"name\", \"evaluation_timestamp\"])\n",
+        "    return result.groupby(level=[0, 1]).mean(numeric_only=True)\n",
+        "\n",
+        "  def export(self, folder_name: str, chunk_size: int, credentials):\n",
+        "    drive_service = build(\"drive\", \"v3\", credentials=credentials)\n",
+        "    folder = find_folder(folder_name, drive_service)\n",
+        "    if folder:\n",
+        "      folder_id, folder_url = folder\n",
+        "    else:\n",
+        "      folder_id, folder_url = create_folder(folder_name, drive_service)\n",
+        "\n",
+        "    queryset = self.scrape_outputs.drop(RESPONSE, axis=1)\n",
+        "    responses = self.scrape_outputs[RESPONSE].apply(lambda x: x.to_row())\n",
+        "    responses = pd.DataFrame(responses.to_list(), index=queryset.index)\n",
+        "\n",
+        "    json_content = {\n",
+        "        \"queryset\": queryset.to_dict(orient=\"index\"),\n",
+        "        \"responses\": responses.to_dict(orient=\"index\"),\n",
+        "        \"metrics\": self.metric_outputs.to_dict(orient=\"index\"),\n",
+        "    }\n",
+        "    json_id, json_url = create_json(\n",
+        "        json_content, \"results.json\", folder_id, drive_service\n",
+        "    )\n",
+        "\n",
+        "    for column in [_ANSWER_GENERATOR_LLM_PROMPT, _SEARCH_RESULTS]:\n",
+        "      truncate(responses, column)\n",
+        "\n",
+        "    results = pd.concat([queryset, responses, self.metric_outputs], axis=1)\n",
+        "    worksheets = {\n",
+        "        \"summary\": self.aggregate().fillna(\"#N/A\"),\n",
+        "        \"results\": results.fillna(\"#N/A\")\n",
+        "    }\n",
+        "    sheets_service = build(\"sheets\", \"v4\", credentials=credentials)\n",
+        "    create_sheet(\n",
+        "        worksheets=worksheets,\n",
+        "        title=\"results\",\n",
+        "        parent=folder_id,\n",
+        "        chunk_size=chunk_size,\n",
+        "        sheets_service=sheets_service,\n",
+        "        drive_service=drive_service,\n",
+        "    )\n",
+        "    return folder_url\n",
+        "\n",
+        "  @property\n",
+        "  def timestamp(self) -> str:\n",
+        "    return self.metric_outputs[\"evaluation_timestamp\"].iloc[0]\n",
+        "\n",
+        "\n",
+        "@dataclasses.dataclass\n",
+        "class EvaluationVisualizer:\n",
+        "  evaluation_results: list[EvaluationResult]\n",
+        "\n",
+        "  def radar_plot(self, columns: list[str] | None = None):\n",
+        "    fig = go.Figure()\n",
+        "    summaries = pd.concat(\n",
+        "        [result.aggregate(columns) for result in self.evaluation_results]\n",
+        "    )\n",
+        "    summaries = summaries.to_dict(orient=\"split\")\n",
+        "\n",
+        "    for idx, values in enumerate(summaries[\"data\"]):\n",
+        "      fig.add_trace(\n",
+        "          go.Scatterpolar(\n",
+        "              r=values,\n",
+        "              theta=summaries[\"columns\"],\n",
+        "              fill='toself',\n",
+        "              name=\"_\".join(summaries[\"index\"][idx]),\n",
+        "          )\n",
+        "      )\n",
+        "    fig.update_layout(\n",
+        "        polar={\"radialaxis\": {\"visible\": True, \"range\": [0, 1]}},\n",
+        "        showlegend=True\n",
+        "    )\n",
+        "    fig.show()\n",
+        "\n",
+        "  def count_barplot(self, column_name: str):\n",
+        "    results = []\n",
+        "    for result in self.evaluation_results:\n",
+        "      responses = result.scrape_outputs[RESPONSE].apply(lambda x: x.to_row())\n",
+        "      responses = pd.DataFrame(\n",
+        "          responses.to_list(), index=result.scrape_outputs.index\n",
+        "      )\n",
+        "      results.append(\n",
+        "          pd.concat(\n",
+        "              [result.scrape_outputs, responses, result.metric_outputs],\n",
+        "              axis=1\n",
+        "          )\n",
+        "      )\n",
+        "    results = pd.concat(results)\n",
+        "    results = results.set_index([\"agent_display_name\", \"evaluation_timestamp\"])\n",
+        "    grouped_counts = (\n",
+        "        results[column_name]\n",
+        "        .groupby(level=[\"agent_display_name\", \"evaluation_timestamp\"])\n",
+        "        .value_counts()\n",
+        "        .unstack(fill_value=0)\n",
+        "    )\n",
+        "    grouped_counts.plot(kind=\"bar\")\n",
+        "    plt.xlabel(\"Name\")\n",
+        "    plt.ylabel(\"Count\")\n",
+        "    plt.xticks(rotation=15)\n",
+        "    plt.title(f\"{column_name} counts by name\")\n",
+        "    plt.legend(title=column_name)\n",
+        "    plt.show()\n",
+        "\n",
+        "  def mean_barplot(self, column_names: list[str]):\n",
+        "    results = []\n",
+        "    for result in self.evaluation_results:\n",
+        "      results.append(\n",
+        "          pd.concat([result.scrape_outputs, result.metric_outputs], axis=1)\n",
+        "      )\n",
+        "    results = pd.concat(results)\n",
+        "    results = results.set_index([\"agent_display_name\", \"evaluation_timestamp\"])\n",
+        "    grouped_means = (\n",
+        "        results[column_names]\n",
+        "        .groupby(level=[\"agent_display_name\", \"evaluation_timestamp\"])\n",
+        "        .mean()\n",
+        "    )\n",
+        "    grouped_means.plot(kind=\"bar\")\n",
+        "    plt.ylim(top=1.0)\n",
+        "    plt.xlabel(\"Name\")\n",
+        "    plt.ylabel(\"Mean\")\n",
+        "    plt.xticks(rotation=15)\n",
+        "    plt.title(\"mean by name\")\n",
+        "    plt.show()\n",
+        "\n",
+        "\n",
+        "class VertexConversationEvaluator:\n",
+        "\n",
+        "  def __init__(self, metrics: list[Metric]):\n",
+        "    self._metrics = metrics\n",
+        "\n",
+        "  def run(self, scraper_output: pd.DataFrame) -> EvaluationResult:\n",
+        "    timestamp = datetime.datetime.now(tz=datetime.timezone.utc)\n",
+        "    scraper_output = scraper_output.copy(deep=True)\n",
+        "    result = pd.DataFrame(index=scraper_output.index)\n",
+        "\n",
+        "    for metric in self._metrics:\n",
+        "      result = pd.concat([result, metric.run(scraper_output)], axis=1)\n",
+        "\n",
+        "    # adding timestamp and agent display name so they can be used as a multi\n",
+        "    # index\n",
+        "    result[\"evaluation_timestamp\"] = timestamp.isoformat()\n",
+        "\n",
+        "    return EvaluationResult(scraper_output, result)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TsHc10HfAHDz"
+      },
+      "source": [
+        "---\n",
+        "\n",
+        "# Evaluation"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4srTefHVmJ3U"
+      },
+      "source": [
+        "## Initialization"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "QbzhGINr5IkG"
+      },
+      "outputs": [],
+      "source": [
+        "# @markdown `initialize vertex ai`\n",
+        "\n",
+        "# @markdown > The project selected will be billed for calculating evaluation\n",
+        "# @markdown metrics that require large language models. It should have the\n",
+        "# @markdown [Vertex AI API](https://cloud.google.com/vertex-ai/docs/featurestore/setup)\n",
+        "# @markdown enabled. The LLM-based metrics use PaLM 2 for Text (Text Bison).\n",
+        "# @markdown For pricing information see this [page](https://cloud.google.com/vertex-ai/generative-ai/pricing).\n",
+        "\n",
+        "vertex_ai_project_id = \"\"  # @param{type: 'string'}\n",
+        "vertex_ai_location = \"\"  # @param{type: 'string'}\n",
+        "\n",
+        "vertexai.init(\n",
+        "    project=vertex_ai_project_id,\n",
+        "    location=vertex_ai_location,\n",
+        "    credentials=credentials,\n",
+        ")\n",
+        "\n",
+        "llm = TextGenerationModel.from_pretrained(\"text-bison@002\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "p4L6fCu5fFXR"
+      },
+      "outputs": [],
+      "source": [
+        "# test llm on a single query\n",
+        "llm.predict(\"hi\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "rZB_riHj13Mo"
+      },
+      "outputs": [],
+      "source": [
+        "# @markdown `run this cell to initialize Dialogflow CX agent scraper`\n",
+        "# @markdown > This cell initializes the agent with the provided parameters.\n",
+        "# @markdown `project_id`, `location` and `agent_id` can be defined through one\n",
+        "# @markdown of the following options, while `language_code` must be defined\n",
+        "# @markdown in either case. The parameters for a given agent can be found in\n",
+        "# @markdown the DialogflowCX console url:\n",
+        "# @markdown `https://dialogflow.cloud.google.com/cx/projects/`**`{project_id}`**`/locations/`**`{location}`**`/agents/`**`{agent_id}`**`/intents`\n",
+        "\n",
+        "language_code = \"en\"  # @param {type: 'string'}\n",
+        "\n",
+        "# @markdown ---\n",
+        "# @markdown ### Option 1. - Provide agent parameters directly\n",
+        "# @markdown\n",
+        "\n",
+        "agent_project_id = \"\"  # @param {type: \"string\"}\n",
+        "agent_location = \"\"  # @param {type: 'string'}\n",
+        "agent_id = \"\"  # @param {type: \"string\"}\n",
+        "\n",
+        "# @markdown ---\n",
+        "# @markdown ### Option 2. - Parse agent parameters from url\n",
+        "# @markdown > **NOTE** : if `agent_url` is provided then it has precedence over\n",
+        "# @markdown directly provided agent parameters.\n",
+        "\n",
+        "agent_url = \"\" # @param {type: \"string\"}\n",
+        "\n",
+        "if agent_url:\n",
+        "  scraper = VertexConversationScraper.from_url(\n",
+        "      agent_url=agent_url,\n",
+        "      language_code=language_code,\n",
+        "      creds=credentials\n",
+        "  )\n",
+        "else:\n",
+        "  scraper = VertexConversationScraper(\n",
+        "      agent_id=agent_id,\n",
+        "      location=agent_location,\n",
+        "      project_id=agent_project_id,\n",
+        "      language_code=language_code,\n",
+        "      creds=credentials\n",
+        "  )"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Z50iGm_rTC49"
+      },
+      "outputs": [],
+      "source": [
+        "# test agent on a single query\n",
+        "response = scraper.scrape_detect_intent(query=\"who is the ceo?\")\n",
+        "print(json.dumps(dataclasses.asdict(response), indent=4))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cFExd6nTmLYr"
+      },
+      "source": [
+        "## Data Loading\n",
+        "\n",
+        "The queryset must be in a tabular format that has to contain the following columns:\n",
+        "- `conversation_id` _(unique identifier of a conversation, which must be the same for each row that are part of the same conversation)_\n",
+        "- `turn_index` _(index of the query - expected answer pair within a conversation)_\n",
+        "- `query` _(the input question)_\n",
+        "- `expected_answer` _(the ideal or ground truth answer)_\n",
+        "- `expected_uri` _(the webpage url or more generally the uri that contains the answer to `query`)_.\n",
+        "\n",
+        "In addition to the required columns the RougeL metric can also use the following optional column:\n",
+        "\n",
+        "- `golden_snippet` _(the extracted document snippet or segment that contains the `expected_answer`)_\n",
+        "\n",
+        "An example for the queryset can be seen in this table:\n",
+        "\n",
+        "| conversation_id | turn_index | query | expected_answer | expected_uri |\n",
+        "| --- | --- | --- | --- | --- |\n",
+        "| 0 | 1 | What is the capital of France? | Capital of France is Paris. | exampleurl.com/france |\n",
+        "| 0 | 2 | How many people live there? | 2.1 million people live in Paris. | exampleurl.com/paris |\n",
+        "| 1 | 1 | What is the color of the sky? | It is blue. | exampleurl.com/common |\n",
+        "| 2 | 1 | How many legs does an octopus have? | It has 8 limbs. | exampleurl.com/octopus |\n",
+        "\n",
+        "---\n",
+        "\n",
+        "Choose one of the following 3 options to load the queryset:\n",
+        "\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1InymjofmW5X"
+      },
+      "source": [
+        "### Option 1. - Manual"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "qwhOvOSOmnJ4"
+      },
+      "outputs": [],
+      "source": [
+        "sample_df = pd.DataFrame(columns=INPUT_SCHEMA_REQUIRED_COLUMNS)\n",
+        "\n",
+        "sample_df.loc[0] = [\"0\", 1 ,\"Who are you?\", \"I am an assistant\", \"www.google.com\"]\n",
+        "sample_df.loc[1] = [\"1\", 1 ,\"Which is the cheapest plan?\", \"Basic plan\", \"www.google.com\"]\n",
+        "sample_df.loc[2] = [\"1\", 2, \"How much?\", \"The Basic plan costs 20$/month\", \"www.google.com\"]\n",
+        "queryset = sample_df"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1WlIWM21mUK_"
+      },
+      "source": [
+        "---\n",
+        "### Option 2. - From local .csv"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "47JSfzvfaVhd"
+      },
+      "outputs": [],
+      "source": [
+        "# @markdown `run this cell to load the queryset from a .csv file in the filesystem`\n",
+        "\n",
+        "csv_path = \"\"  # @param{type: 'string'}\n",
+        "\n",
+        "queryset = pd.read_csv(csv_path)\n",
+        "queryset = queryset.fillna(\"\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IfEOv6cTmRDB"
+      },
+      "source": [
+        "---\n",
+        "### Option 3. - From Google Sheets"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "A2tS-n1PRHeC"
+      },
+      "outputs": [],
+      "source": [
+        "# @markdown `run this cell to load the queryset from Google Sheets`\n",
+        "\n",
+        "sheet_url = \"\" # @param {type: \"string\"}\n",
+        "worksheet_name = \"\" # @param {type: \"string\"}\n",
+        "# @markdown > **NOTE**: if `worksheet_name` is not provided then `Sheet1` is used\n",
+        "# @markdown by default.\n",
+        "\n",
+        "_worksheet_name = worksheet_name if worksheet_name else \"Sheet1\"\n",
+        "\n",
+        "queryset = load_spreadsheet(sheet_url, _worksheet_name, credentials)\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sn_TZfWQmjtv"
+      },
+      "source": [
+        "## Scraping Responses"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7phOx3LWHUfm"
+      },
+      "outputs": [],
+      "source": [
+        "# run scraping\n",
+        "scrape_result = scraper.run(queryset)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4w6OKOpImcJg"
+      },
+      "source": [
+        "## Metric Definition"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "at5ss5szde8o"
+      },
+      "source": [
+        "Select the metrics that should be computed during evaluation.\n",
+        "\n",
+        "> **NOTE** : Remember to rerun the cell below (Shift+Enter) after clicking the checkbox of the metric."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "UaX42vFgwwYA"
+      },
+      "outputs": [],
+      "source": [
+        "URL_MATCH = True # @param {type: \"boolean\"}\n",
+        "# @markdown url match metric computes the following boolean type columns:\n",
+        "# @markdown - `cited_url_match@1` - is `expected_url` same as the first link returned by Vertex AI Conversation\n",
+        "# @markdown - `cited_url_match` - is `expected_url` part of the links returned by Vertex AI Conversation\n",
+        "# @markdown - `search_url_match` - is `expected_url` part of the search results that are shown to generative model in Vertex AI Conversation\n",
+        "# @markdown\n",
+        "# @markdown ---\n",
+        "ROUGEL = True # @param {type: \"boolean\"}\n",
+        "# @markdown rougeL metric computes a score between `[0, 1]` (higher is better) based on\n",
+        "# @markdown the longest common word subsequence between different targets and predictions:\n",
+        "# @markdown\n",
+        "# @markdown _(`cited_search_results` are the search snippets which were cited by answer generator llm)_\n",
+        "# @markdown - `rougeL_generative` - Compares `expected_answer` to `answer_text`.\n",
+        "# @markdown - `rougeL_extractive` - (only computed if\n",
+        "# @markdown `golden_snippet` is part of the dataset) Compares `golden_snippet` to `answer_snippets[0]`\n",
+        "# @markdown\n",
+        "# @markdown ---\n",
+        "ANSWER_CORRECTNESS = True # @param {type: \"boolean\"}\n",
+        "# @markdown LLM-based autoeval metric that compares `expected_answer` to `answer_text`. The metric\n",
+        "# @markdown makes approximately 5-10 LLM calls for each for each row of the dataset depening on `expected_answer` and `answer_text` length.\n",
+        "# @markdown It returns 3 output columns containing scores between `[0, 1]` (higher is better):\n",
+        "# @markdown - `answer_correctness_recall` - How well does `answer_text`'s information content cover `expected_answer`.\n",
+        "# @markdown - `answer_correctness_precision` - How much of `answer_text`'s information content is required based on `expected_answer`.\n",
+        "# @markdown - `answer_correctness_f1` - The harmonic mean of recall and precision.\n",
+        "# @markdown\n",
+        "# @markdown ---\n",
+        "FAITHFULNESS = True # @param {type: \"boolean\"}\n",
+        "# @markdown LLM-based autoeval metric that provides a score between `[0, 1]` (higher is better)\n",
+        "# @markdown with regard to how well `answer_text` is attributed to `search_snippets`. It makes approximately 5\n",
+        "# @markdown LLM calls for each row of the dataset depending on the length of `answer_text`.\n",
+        "# @markdown - `faithfulness_gmean`\n",
+        "# @markdown\n",
+        "# @markdown ---\n",
+        "CONTEXT_RECALL = True # @param {type: \"boolean\"}\n",
+        "# @markdown LLM-based autoeval metric that provides a score between `[0, 1]` (higher is better)\n",
+        "# @markdown for how well the `expected_answer` is attributed to `search_snippets`. In other words this metric\n",
+        "# @markdown scores the quality of search by measuring how well the `expected_answer` can be generated from the `search_snippets`.\n",
+        "# @markdown It makes approximately 5 LLM calls for each row of the dataset depending on the length of `expected_answer`.\n",
+        "# @markdown - `context_recall_gmean`\n",
+        "# @markdown\n",
+        "# @markdown ---\n",
+        "\n",
+        "metrics = []\n",
+        "\n",
+        "if URL_MATCH:\n",
+        "  metrics.append(UrlMatch())\n",
+        "\n",
+        "if ROUGEL:\n",
+        "  metrics.append(RougeL())\n",
+        "\n",
+        "if any((ANSWER_CORRECTNESS, FAITHFULNESS, CONTEXT_RECALL)):\n",
+        "  metrics.append(\n",
+        "      StatementBasedBundledMetric(\n",
+        "          llm=llm,\n",
+        "          answer_correctness=ANSWER_CORRECTNESS,\n",
+        "          faithfulness=FAITHFULNESS,\n",
+        "          context_recall=CONTEXT_RECALL,\n",
+        "      )\n",
+        "  )\n",
+        "\n",
+        "evaluator = VertexConversationEvaluator(metrics=metrics)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "69h01DPmrMoA"
+      },
+      "source": [
+        "## Computing Metrics"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": true,
+        "id": "9NeMsvykHb0E"
+      },
+      "outputs": [],
+      "source": [
+        "evaluation_result = evaluator.run(scrape_result)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "F-aAhhD-qPJp"
+      },
+      "outputs": [],
+      "source": [
+        "# @markdown `export evaluation results`\n",
+        "\n",
+        "FOLDER_NAME = \"result\" # @param {type: \"string\"}\n",
+        "CHUNK_SIZE = 50 # @param {type: \"number\"}\n",
+        "WITH_TIMESTAMP = True # @param {type: \"boolean\"}\n",
+        "\n",
+        "_folder_name = (\n",
+        "    f\"{FOLDER_NAME}_{evaluation_result.timestamp}\"\n",
+        "    if WITH_TIMESTAMP else\n",
+        "    FOLDER_NAME\n",
+        ")\n",
+        "\n",
+        "folder_url = evaluation_result.export(_folder_name, CHUNK_SIZE, credentials)\n",
+        "print(f\"Exported results to folder: {folder_url}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IH7ddvNEUbal"
+      },
+      "source": [
+        "## Result Visualization"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kARLoOJYBJ0e"
+      },
+      "outputs": [],
+      "source": [
+        "FOLDER_URLS = [\n",
+        "    folder_url, # latest evaluation\n",
+        "    # add previous evaluations e.g: https://drive.google.com/drive/folders/<id>\n",
+        "]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "umJE__jOWTQu"
+      },
+      "outputs": [],
+      "source": [
+        "# @markdown `define evaluation visualizer`\n",
+        "\n",
+        "evaluation_visualizer = EvaluationVisualizer([\n",
+        "    EvaluationResult.load(folder_url, credentials)\n",
+        "    for folder_url in FOLDER_URLS\n",
+        "])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "LqObjkW28pNa"
+      },
+      "outputs": [],
+      "source": [
+        "# @markdown `radar plot of autoeval metrics`\n",
+        "\n",
+        "evaluation_visualizer.radar_plot(StatementBasedBundledMetric.COLUMNS)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "collapsed": true,
+        "id": "H-wg1FTiToA5"
+      },
+      "outputs": [],
+      "source": [
+        "# @markdown `response type distribution`\n",
+        "\n",
+        "evaluation_visualizer.count_barplot(\"response_type\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "6nc7WQKVY1jp"
+      },
+      "outputs": [],
+      "source": [
+        "# @markdown `average RougeL`\n",
+        "\n",
+        "evaluation_visualizer.mean_barplot(column_names=RougeL.COLUMNS)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "private_outputs": true,
+      "provenance": [
+        {
+          "file_id": "17WDmf3DsZGg1ZGwnr40sMXMQtyfxb4ms",
+          "timestamp": 1713942777165
+        },
+        {
+          "file_id": "1b769OFNM8gH56ZzvWUfw4MpGUc-pSF-g",
+          "timestamp": 1708939513950
+        }
+      ],
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.8.10"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
\ No newline at end of file
diff --git a/examples/vertex_ai_conversation/evaluation_tool__numeric_score__colab.ipynb b/examples/vertex_ai_conversation/evaluation_tool__numeric_score__colab.ipynb
new file mode 100644
index 00000000..9e00aa79
--- /dev/null
+++ b/examples/vertex_ai_conversation/evaluation_tool__numeric_score__colab.ipynb
@@ -0,0 +1,352 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WpkyirmC-F33"
+      },
+      "source": [
+        "# Vertex AI Conversation - Evaluation Tool\n",
+        "\n",
+        "This tool requieres user's input in several steps. Please run the cells one by one (Shift+Enter) to ensure all the steps are succesfully completed.\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td style=\"text-align: center\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/vertex_ai_conversation/evaluation_tool__numeric_score__colab.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Google Colaboratory logo\"><br> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td style=\"text-align: center\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/vertex_ai_conversation/evaluation_tool__numeric_score__colab.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"><br> View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td style=\"text-align: center\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/vertex_ai_conversation/evaluation_tool__numeric_score__colab.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>\n",
+        "<br><br><br>\n",
+        "\n",
+        "## Instructions:\n",
+        "\n",
+        "1.  **Set-up**\n",
+        "    1. First cell: install and import dependencies\n",
+        "    2. Second cell: authentication - it requieres following the steps in the pop-up window. Alternatively, it can be replaced by other [supported authentication method](https://github.com/GoogleCloudPlatform/dfcx-scrapi#authentication)\n",
+        "    3. Third cell: introduce values for project, location and agent in the right panel; then run the cell.\n",
+        "    4. Fourth cell: run examples to validate set-up is correct\n",
+        "2.  **Generate Questions & Answer**\n",
+        "    1. First cell: save a sample csv file with correct format\n",
+        "    2. Second cell: upload csv file with the fields `user_query` and an `ideal_answer` for all examples\n",
+        "    3. Third cell: bulk generation of `agent_answer` that includes the text and link\n",
+        "3.  **Rating**\n",
+        "    1. First cell: download csv and add the ratings offline\n",
+        "    2. Second cell: upload csv file with the ratings\n",
+        "4. **Results**\n",
+        "    1. First cell: visualize distribution of ratings\n",
+        "\n",
+        "This notebook calls `DetectIntent` using [dfcx-scrapi library](https://github.com/GoogleCloudPlatform/dfcx-scrapi) for Dialogflow CX.\n",
+        "\n",
+        "\n",
+        "## Rating guidance:\n",
+        "\n",
+        "For each sample (aka row), the rater should evaluate each answer (including ythe link) that was generated by the agent. The answer will be evaluated with a integer number (escalar) from -1 to 3 as following:\n",
+        "*   **+3** : Perfect answer > fully addresses the question with correct information and polite tone\n",
+        "*   **+2** : Good answer > may contain unnecessary info, may miss some info, or may not be perfectly articulated\n",
+        "*   **+1** : Slightly good answer > some truth to the answer\n",
+        "*   **0** : Neutral answer > no answer or answer contains irrelevant info\n",
+        "*   **-1** : Hurtful answer > wrong or misleading info, or inappropriate tone\n",
+        "\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Afvsuux0zaWZ"
+      },
+      "source": [
+        "## Set-up\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PPJYRHN83bHg"
+      },
+      "outputs": [],
+      "source": [
+        "# Dependencies\n",
+        "!pip install dfcx-scrapi --quiet\n",
+        "\n",
+        "import io\n",
+        "import matplotlib.pyplot as plt\n",
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "\n",
+        "from dfcx_scrapi.core.sessions import Sessions\n",
+        "from google.auth import default\n",
+        "from google.colab import auth\n",
+        "from google.colab import files"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "sztyBjNlIGAw"
+      },
+      "outputs": [],
+      "source": [
+        "# Authentication\n",
+        "\n",
+        "auth.authenticate_user()\n",
+        "creds, _ = default()\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mRUB0Uf-3uzS"
+      },
+      "outputs": [],
+      "source": [
+        "# Agent config\n",
+        "project_id = '' #@param{type: 'string'}\n",
+        "location = 'global'  #@param{type: 'string'}\n",
+        "agent_id = ''  #@param{type: 'string'}\n",
+        "\n",
+        "agent_id = f\"projects/{project_id}/locations/{location}/agents/{agent_id}\"\n",
+        "print(agent_id)\n",
+        "\n",
+        "s = Sessions(agent_id=agent_id)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OChJbblt3dt7"
+      },
+      "outputs": [],
+      "source": [
+        "# Test\n",
+        "user_query = 'Hello World!'\n",
+        "agent_answer = s.get_agent_answer(user_query)\n",
+        "print(f\" Q: {user_query}\\n A: {agent_answer}\")\n",
+        "\n",
+        "user_query = 'Which is the cheapest plan?'\n",
+        "agent_answer = s.get_agent_answer(user_query)\n",
+        "print(f\" Q: {user_query}\\n A: {agent_answer}\")\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "L2WQime-8-Dw"
+      },
+      "source": [
+        "## Generate Questions & Answer"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "q3II66B04F0j"
+      },
+      "outputs": [],
+      "source": [
+        "# Create sample csv\n",
+        "\n",
+        "sample_df = pd.DataFrame({\n",
+        "  \"user_query\": [],\n",
+        "  \"ideal_answer\": [],\n",
+        "  \"agent_answer\": [],\n",
+        "  \"rating\": [],\n",
+        "  \"comment\": []\n",
+        "})\n",
+        "\n",
+        "sample_df.loc[0] = [\"Who are you?\", \"I am an assistant\", \"\", 0, \"\"]\n",
+        "sample_df.loc[1] = [\"Which is the cheapest plan?\", \"Basic plan\", \"\", 0, \"\"]\n",
+        "sample_df.loc[2] = [\"My device is not working\", \"Call 888-555\", \"\", 0, \"\"]\n",
+        "\n",
+        "# Export to local drive as csv file\n",
+        "file_name = 'data_sample.csv'\n",
+        "sample_df.to_csv(file_name, encoding='utf-8-sig', index=False)\n",
+        "files.download(file_name)\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OYr4Dy77KbfL"
+      },
+      "outputs": [],
+      "source": [
+        "input(f\"In your local drive, you can find the csv file '{file_name}' Add the user_query and ideal_answer per example \\nWhen done, click 'Enter'\")\n",
+        "print('done')\n",
+        "\n",
+        "# Import from local drive the csv file with the user_query and ideal_answer per examples\n",
+        "uploaded = files.upload()\n",
+        "file_name2 = next(iter(uploaded))\n",
+        "df = pd.read_csv(io.BytesIO(uploaded[file_name2]))\n",
+        "\n",
+        "assert df.shape[0] > 0, \"The csv has zero rows\"\n",
+        "assert set(df.columns) == set(sample_df.columns), f\"The csv must have the following columns: {sample_df.columns.values}\"\n",
+        "\n",
+        "df"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "RmJcxpFI881j"
+      },
+      "outputs": [],
+      "source": [
+        "# Generate answers for each query\n",
+        "df['agent_answer'] = df.apply(lambda row: s.get_agent_answer(row[\"user_query\"]), axis=1)\n",
+        "\n",
+        "df"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yO2x7lc2BRDR"
+      },
+      "source": [
+        "# Rating"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZfAMlQbS8qsy"
+      },
+      "outputs": [],
+      "source": [
+        "# Export to local drive as csv file\n",
+        "file_name = 'output.csv'\n",
+        "df.to_csv(file_name, encoding='utf-8-sig', index=False)\n",
+        "files.download(file_name)\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SEU44Mcy9mBU"
+      },
+      "outputs": [],
+      "source": [
+        "input(f\"In your local drive, you can find the csv file '{file_name}' Rate each agent_answer using ideal_answer as reference. Rating from -1 to 3. \\nWhen done, click 'Enter'\")\n",
+        "print('done')\n",
+        "\n",
+        "# Import from local drive the csv file with the ratings\n",
+        "uploaded = files.upload()\n",
+        "file_name2 = next(iter(uploaded))\n",
+        "df = pd.read_csv(io.BytesIO(uploaded[file_name2]))\n",
+        "\n",
+        "assert df.shape[0] > 0, \"The csv has zero rows\"\n",
+        "assert set(df.columns) == set(sample_df.columns), f\"The csv must have the following columns: {sample_df.columns.values}\"\n",
+        "\n",
+        "df"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "W5j9yAewRmNO"
+      },
+      "source": [
+        "# Results\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "I5209MB7VS1q"
+      },
+      "outputs": [],
+      "source": [
+        "# Rating distribution\n",
+        "#df[\"rating\"].describe()\n",
+        "\n",
+        "# Histogram\n",
+        "ratings_set = [-1, 0, 1, 2, 3]\n",
+        "ratings_values = df['rating'].values\n",
+        "ratings_count = len(ratings_values)\n",
+        "\n",
+        "bar_centers = np.linspace(min(ratings_set), max(ratings_set), len(ratings_set))\n",
+        "bar_edges = np.linspace(min(ratings_set)-0.5, max(ratings_set)+0.5, len(ratings_set)+1)\n",
+        "bar_heights, _ = np.histogram(ratings_values, bins=bar_edges, density=True)\n",
+        "\n",
+        "for center, _h in zip(bar_centers, bar_heights):\n",
+        "  print(f\"{center}: count={round(_h*ratings_count):.0f}, percentage={_h*100:.2f}%\")\n",
+        "\n",
+        "# Plot\n",
+        "height_sum = 100  # for percentage, use 100\n",
+        "fig, axs = plt.subplots(1, 1, figsize=(6, 4), tight_layout=True)\n",
+        "\n",
+        "plt.bar(bar_centers, height_sum*bar_heights, width=0.8)\n",
+        "ratings_mean = np.mean(ratings_values)\n",
+        "plt.plot([ratings_mean, ratings_mean], [0, height_sum], '--', label=f\"mean={ratings_mean:.2f}\", color='red')\n",
+        "ratings_median = np.median(ratings_values)\n",
+        "plt.plot([ratings_median, ratings_median], [0, height_sum], '--', label=f\"median={ratings_median:.2f}\", color='green')\n",
+        "\n",
+        "plt.axis((min(bar_edges), max(bar_edges), 0, round(1.2*max(height_sum*bar_heights), 1)))\n",
+        "plt.legend(loc='upper left')\n",
+        "plt.gca().grid(axis='y')\n",
+        "plt.xlabel('Rating')\n",
+        "plt.ylabel('Percentage [%]')\n",
+        "plt.title(f\"Rating distribution (count={ratings_count})\")\n",
+        "\n",
+        "plt.tight_layout()\n",
+        "plt.show()\n",
+        "\n",
+        "fig.savefig('ratings_distribution.png', dpi=fig.dpi)\n",
+        "\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rYwsIZ0Ej-v9"
+      },
+      "outputs": [],
+      "source": []
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.8.10"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/examples/vertex_ai_conversation/evaluation_tool__numeric_score__nocolab.ipynb b/examples/vertex_ai_conversation/evaluation_tool__numeric_score__nocolab.ipynb
new file mode 100644
index 00000000..30507310
--- /dev/null
+++ b/examples/vertex_ai_conversation/evaluation_tool__numeric_score__nocolab.ipynb
@@ -0,0 +1,355 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WpkyirmC-F33"
+      },
+      "source": [
+        "# Infobot Eval\n",
+        "\n",
+        "This tool requieres user's input in several steps. Please run the cells one by one (Shift+Enter) to ensure all the steps are succesfully completed.\n",
+        "\n",
+        "## Instructions:\n",
+        "\n",
+        "1.  **Set-up**\n",
+        "    1. First cell: install and import dependencies\n",
+        "    2. Second cell: authentication - it requieres following the steps in the pop-up window. Alternatively, it can be replaced by other [supported authentication method](https://github.com/GoogleCloudPlatform/dfcx-scrapi#authentication)\n",
+        "    3. Third cell: introduce values for project, location and agent in the right panel; then run the cell.\n",
+        "    4. Fourth cell: run examples to validate set-up is correct\n",
+        "2.  **Generate Questions & Answer**\n",
+        "    1. First cell: save a sample csv file with correct format\n",
+        "    2. Second cell: upload csv file with the fields `user_query` and an `ideal_answer` for all examples\n",
+        "    3. Third cell: bulk generation of `agent_answer` that includes the text and link\n",
+        "3.  **Rating**\n",
+        "    1. First cell: download csv and add the ratings offline\n",
+        "    2. Second cell: upload csv file with the ratings\n",
+        "4. **Results**\n",
+        "    1. First cell: visualize distribution of ratings\n",
+        "\n",
+        "This notebook calls `DetectIntent` using [dfcx-scrapi library](https://github.com/GoogleCloudPlatform/dfcx-scrapi) for Dialogflow CX.\n",
+        "\n",
+        "\n",
+        "## Rating guidance:\n",
+        "\n",
+        "For each sample (aka row), the rater should evaluate each answer (including ythe link) that was generated by the agent. The answer will be evaluated with a integer number (escalar) from -1 to 3 as following:\n",
+        "*   **+3** : Perfect answer > fully addresses the question with correct information and polite tone\n",
+        "*   **+2** : Good answer > may contain unnecessary info, may miss some info, or may not be perfectly articulated\n",
+        "*   **+1** : Slightly good answer > some truth to the answer\n",
+        "*   **0** : Neutral answer > no answer or answer contains irrelevant info\n",
+        "*   **-1** : Hurtful answer > wrong or misleading info, or inappropriate tone\n",
+        "\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Afvsuux0zaWZ"
+      },
+      "source": [
+        "## Set-up\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PPJYRHN83bHg"
+      },
+      "outputs": [],
+      "source": [
+        "# Dependencies\n",
+        "!pip install dfcx-scrapi --quiet\n",
+        "\n",
+        "import io\n",
+        "import matplotlib.pyplot as plt\n",
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "\n",
+        "from dfcx_scrapi.core.sessions import Sessions"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ldbOTqPVnXRj"
+      },
+      "source": [
+        "**<span style=\"color:red\">ATTENTION</span>: MANUAL STEP**\n",
+        "\n",
+        "Instruction: Run the following commands one by one in the Terminal in order to authenticate the notebook\n",
+        "```\n",
+        "gcloud auth login\n",
+        "gcloud auth application-default login\n",
+        "```\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OegHGMr_nXRj"
+      },
+      "source": [
+        "**<span style=\"color:red\">ATTENTION</span>: MANUAL STEP**\n",
+        "\n",
+        "Instruction: In the next cell, edit the values of the Agent config, then run the cell\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mRUB0Uf-3uzS"
+      },
+      "outputs": [],
+      "source": [
+        "# Agent config\n",
+        "project_id = ''  #@param{type: 'string'}\n",
+        "location = 'global'  #@param{type: 'string'}\n",
+        "agent_id = ''  #@param{type: 'string'}\n",
+        "\n",
+        "agent_id = f\"projects/{project_id}/locations/{location}/agents/{agent_id}\"\n",
+        "print(agent_id)\n",
+        "\n",
+        "s = Sessions(agent_id=agent_id)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OChJbblt3dt7"
+      },
+      "outputs": [],
+      "source": [
+        "# Test\n",
+        "user_query = 'Hello World!'\n",
+        "agent_answer = s.get_agent_answer(user_query)\n",
+        "print(f\" Q: {user_query}\\n A: {agent_answer}\")\n",
+        "\n",
+        "user_query = 'Which is the cheapest plan?'\n",
+        "agent_answer = s.get_agent_answer(user_query)\n",
+        "print(f\" Q: {user_query}\\n A: {agent_answer}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "L2WQime-8-Dw"
+      },
+      "source": [
+        "## Generate Questions & Answer"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "q3II66B04F0j"
+      },
+      "outputs": [],
+      "source": [
+        "# Create sample csv\n",
+        "\n",
+        "sample_df = pd.DataFrame({\n",
+        "  \"user_query\": [],\n",
+        "  \"ideal_answer\": [],\n",
+        "  \"agent_answer\": [],\n",
+        "  \"rating\": [],\n",
+        "  \"comment\": []\n",
+        "})\n",
+        "\n",
+        "sample_df.loc[0] = [\"Who are you?\", \"I am an assistant\", \"\", 0, \"\"]\n",
+        "sample_df.loc[1] = [\"Which is the cheapest plan?\", \"Basic plan\", \"\", 0, \"\"]\n",
+        "sample_df.loc[2] = [\"My device is not working\", \"Call 888-555\", \"\", 0, \"\"]\n",
+        "\n",
+        "# Export to local drive as csv file\n",
+        "file_name = 'data_sample.csv'\n",
+        "sample_df.to_csv(file_name, encoding='utf-8-sig', index=False)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4n-dGQLonXRm"
+      },
+      "source": [
+        "**<span style=\"color:red\">ATTENTION</span>: MANUAL STEP**\n",
+        "\n",
+        "Instructions:\n",
+        "\n",
+        "1. Download the file `data_sample.csv` to your local drive by right-clicking in the file\n",
+        "2. Open the csv file `data_sample.csv` and add the `user_query` and `ideal_answer` per example\n",
+        "3. Upload the updated file from your local drive to the Jupyter File system by clicking 'Upload File'\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OYr4Dy77KbfL"
+      },
+      "outputs": [],
+      "source": [
+        "\n",
+        "file_name2 = file_name\n",
+        "df = pd.read_csv(file_name2)\n",
+        "\n",
+        "assert df.shape[0] > 0, \"The csv has zero rows\"\n",
+        "assert set(df.columns) == set(sample_df.columns), f\"The csv must have the following columns: {sample_df.columns.values}\"\n",
+        "\n",
+        "df"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "RmJcxpFI881j"
+      },
+      "outputs": [],
+      "source": [
+        "# Generate answers for each query\n",
+        "df['agent_answer'] = df.apply(lambda row: s.get_agent_answer(row[\"user_query\"]), axis=1)\n",
+        "\n",
+        "# Export to local drive as csv file\n",
+        "file_name3 = file_name2\n",
+        "df.to_csv(file_name3, encoding='utf-8-sig', index=False)\n",
+        "\n",
+        "df"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yO2x7lc2BRDR"
+      },
+      "source": [
+        "# Rating"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uB0txK4QnXRn"
+      },
+      "source": [
+        "**<span style=\"color:red\">ATTENTION</span>: MANUAL STEP**\n",
+        "\n",
+        "Instructions:\n",
+        "\n",
+        "1. Download the file `data_sample.csv` to your local drive by right-clicking in the file\n",
+        "2. Open the csv file `data_sample.csv` and add the `rating` and `comment` (optionally) per example\n",
+        "3. Upload the updated file from your local drive to the Jupyter File system by clicking 'Upload File'\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SEU44Mcy9mBU"
+      },
+      "outputs": [],
+      "source": [
+        "\n",
+        "df = pd.read_csv(file_name3)\n",
+        "\n",
+        "assert df.shape[0] > 0, \"The csv has zero rows\"\n",
+        "assert set(df.columns) == set(sample_df.columns), f\"The csv must have the following columns: {sample_df.columns.values}\"\n",
+        "\n",
+        "df"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "W5j9yAewRmNO"
+      },
+      "source": [
+        "# Results\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "I5209MB7VS1q"
+      },
+      "outputs": [],
+      "source": [
+        "# Rating distribution\n",
+        "#df[\"rating\"].describe()\n",
+        "\n",
+        "# Histogram\n",
+        "ratings_set = [-1, 0, 1, 2, 3]\n",
+        "ratings_values = df['rating'].values\n",
+        "ratings_count = len(ratings_values)\n",
+        "\n",
+        "bar_centers = np.linspace(min(ratings_set), max(ratings_set), len(ratings_set))\n",
+        "bar_edges = np.linspace(min(ratings_set)-0.5, max(ratings_set)+0.5, len(ratings_set)+1)\n",
+        "bar_heights, _ = np.histogram(ratings_values, bins=bar_edges, density=True)\n",
+        "\n",
+        "for center, _h in zip(bar_centers, bar_heights):\n",
+        "  print(f\"{center}: count={round(_h*ratings_count):.0f}, percentage={_h*100:.2f}%\")\n",
+        "\n",
+        "# Plot\n",
+        "height_sum = 100  # for percentage, use 100\n",
+        "fig, axs = plt.subplots(1, 1, figsize=(6, 4), tight_layout=True)\n",
+        "\n",
+        "plt.bar(bar_centers, height_sum*bar_heights, width=0.8)\n",
+        "ratings_mean = np.mean(ratings_values)\n",
+        "plt.plot([ratings_mean, ratings_mean], [0, height_sum], '--', label=f\"mean={ratings_mean:.2f}\", color='red')\n",
+        "ratings_median = np.median(ratings_values)\n",
+        "plt.plot([ratings_median, ratings_median], [0, height_sum], '--', label=f\"median={ratings_median:.2f}\", color='green')\n",
+        "\n",
+        "plt.axis((min(bar_edges), max(bar_edges), 0, round(1.2*max(height_sum*bar_heights), 1)))\n",
+        "plt.legend(loc='upper left')\n",
+        "plt.gca().grid(axis='y')\n",
+        "plt.xlabel('Rating')\n",
+        "plt.ylabel('Percentage [%]')\n",
+        "plt.title(f\"Rating distribution (count={ratings_count})\")\n",
+        "plt.show()\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rYwsIZ0Ej-v9"
+      },
+      "outputs": [],
+      "source": []
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "toc_visible": true
+    },
+    "environment": {
+      "kernel": "python3",
+      "name": "tf2-cpu.2-11.m112",
+      "type": "gcloud",
+      "uri": "gcr.io/deeplearning-platform-release/tf2-cpu.2-11:m112"
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.8.12"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/requirements.txt b/requirements.txt
index b6b173fb..6abbb5cf 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,11 +1,13 @@
 dfcx-scrapi
-google-cloud-dialogflow-cx
-google-auth
+google-cloud-dialogflow-cx>=1.33.0
+google-cloud-discoveryengine>=0.11.10
+google-auth>=2.27.0
 google-oauth
 oauth2client
+pyparsing==2.4.7
 pandas
 tabulate
-gspread
+gspread==5.10.0
 gspread_dataframe
 numpy
 requests
diff --git a/setup.py b/setup.py
index 595c8ce5..67b591b7 100644
--- a/setup.py
+++ b/setup.py
@@ -23,7 +23,7 @@
 
 setup(
     name='dfcx-scrapi',
-    version='1.7.0',
+    version='1.10.5',
     description='A high level scripting API for bot builders, developers, and\
       maintainers.',
     long_description=long_description,
diff --git a/src/dfcx_scrapi/agent_extract/__init__.py b/src/dfcx_scrapi/agent_extract/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/dfcx_scrapi/agent_extract/agents.py b/src/dfcx_scrapi/agent_extract/agents.py
new file mode 100644
index 00000000..0bcf8879
--- /dev/null
+++ b/src/dfcx_scrapi/agent_extract/agents.py
@@ -0,0 +1,146 @@
+"""Agent processing methods and functions."""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import time
+import os
+import shutil
+from typing import Dict
+
+from dfcx_scrapi.core import agents
+from dfcx_scrapi.core import operations
+from dfcx_scrapi.core import scrapi_base
+from dfcx_scrapi.agent_extract import graph
+from dfcx_scrapi.agent_extract import flows
+from dfcx_scrapi.agent_extract import intents
+from dfcx_scrapi.agent_extract import entity_types
+from dfcx_scrapi.agent_extract import test_cases
+from dfcx_scrapi.agent_extract import webhooks
+from dfcx_scrapi.agent_extract import gcs_utils
+from dfcx_scrapi.agent_extract import types
+
+# logging config
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)-8s %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+class Agents(scrapi_base.ScrapiBase):
+    """Agent Metadata methods and functions."""
+    def __init__(
+        self,
+        agent_id: str,
+        lang_code: str = "en",
+        creds_path: str = None,
+        creds_dict: Dict = None,
+        creds=None,
+        scope=False
+    ):
+        super().__init__(
+            creds_path=creds_path,
+            creds_dict=creds_dict,
+            creds=creds,
+            scope=scope,
+        )
+        self.agent_id = agent_id
+        self.lang_code = lang_code
+        self._core_agents = agents.Agents(creds=creds)
+        self.gcs = gcs_utils.GcsUtils()
+        self.flows = flows.Flows()
+        self.intents = intents.Intents()
+        self.etypes = entity_types.EntityTypes()
+        self.webhooks = webhooks.Webhooks()
+        self.tcs = test_cases.TestCases()
+        self.ops = operations.Operations()
+
+    @staticmethod
+    def prep_local_dir(agent_local_path: str):
+        """Prepare the local directory for agent zip file."""
+        if os.path.isdir(agent_local_path):
+            logging.info("Cleaning up old directory...")
+            shutil.rmtree(agent_local_path)
+            logging.info(f"Making temp directory: {agent_local_path}")
+            os.mkdir(agent_local_path)
+        else:
+            os.mkdir(agent_local_path)
+
+    def await_lro(self, lro: str):
+        """Wait for long running operation to complete."""
+        try:
+            i = 0
+            while not self.ops.get_lro(lro).done:
+                time.sleep(1)
+                i += 1
+                if i == 20:
+                    break
+
+        except UserWarning:
+            print("LRO Failed.")
+
+        return True
+
+    def export_agent(self, agent_id: str, gcs_bucket_uri: str,
+                      environment_display_name: str = None):
+        """Handle the agent export, LRO and logging."""
+        export_start = time.time()
+        logging.info("Exporting agent...")
+        lro = self._core_agents.export_agent(
+            agent_id=agent_id,gcs_bucket_uri=gcs_bucket_uri, data_format="JSON",
+            environment_display_name=environment_display_name)
+
+
+        self.await_lro(lro)
+        logging.info("Export Complete.")
+        logging.debug(f"EXPORT: {time.time() - export_start}")
+
+    def download_and_extract(self, agent_local_path: str, gcs_bucket_uri: str):
+        """Handle download from GCS and extracting ZIP file."""
+        if not os.path.exists(agent_local_path):
+            os.makedirs(agent_local_path)
+
+        download_start = time.time()
+        logging.info("Downloading agent file from GCS Bucket...")
+        agent_file = self.gcs.download_gcs(
+            gcs_path=gcs_bucket_uri, local_path=agent_local_path)
+        logging.info("Download complete.")
+        logging.debug(f"DOWNLOAD: {time.time() - download_start}")
+
+        self.gcs.unzip(agent_file, agent_local_path)
+
+
+    def process_agent(self, agent_id: str, gcs_bucket_uri: str,
+                      environment_display_name: str = None):
+        """Process the specified Agent for offline data gathering."""
+        agent_local_path = "/tmp/agent"
+        self.prep_local_dir(agent_local_path)
+        self.export_agent(agent_id, gcs_bucket_uri, environment_display_name)
+        self.download_and_extract(agent_local_path, gcs_bucket_uri)
+
+        logging.info("Processing Agent...")
+        data = types.AgentData()
+        data.graph = graph.Graph()
+        data.lang_code = self.lang_code
+        data.agent_id = agent_id
+        data = self.flows.process_flows_directory(agent_local_path, data)
+        data = self.intents.process_intents_directory(agent_local_path, data)
+        data = self.etypes.process_entity_types_directory(
+            agent_local_path, data)
+        data = self.webhooks.process_webhooks_directory(agent_local_path, data)
+        data = self.tcs.process_test_cases_directory(agent_local_path, data)
+        logging.info("Processing Complete.")
+
+        return data
diff --git a/src/dfcx_scrapi/agent_extract/common.py b/src/dfcx_scrapi/agent_extract/common.py
new file mode 100644
index 00000000..26b86ff3
--- /dev/null
+++ b/src/dfcx_scrapi/agent_extract/common.py
@@ -0,0 +1,80 @@
+"""Common methods and helper functions used throughout library."""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import re
+from dfcx_scrapi.agent_extract import types
+
+# logging config
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(message)s",
+)
+
+class Common:
+    """Common methods and helper functions used throughout library."""
+
+    @staticmethod
+    def parse_filepath(in_path: str, resource_type: str) -> str:
+        """Parse file path to provide quick reference for resource."""
+
+        regex_map = {
+            "flow": r".*\/flows\/([^\/]*)",
+            "page": r".*\/pages\/([^\/]*)\.",
+            "entity_type": r".*\/entityTypes\/([^\/]*)",
+            "intent": r".*\/intents\/([^\/]*)",
+            "route_group": r".*\/transitionRouteGroups\/([^\/]*)",
+            "webhook": r".*\/webhooks\/([^\/]*)\."
+        }
+        resource_name = re.match(regex_map[resource_type], in_path).groups()[0]
+
+        return resource_name
+
+    @staticmethod
+    def clean_display_name(display_name: str):
+        """Replace cspecial haracters from map for the given display name."""
+        patterns = {
+            "%22": '"',
+            "%23": "#",
+            "%24": "$",
+            "%26": "&",
+            "%27": "'",
+            "%28": "(",
+            "%29": ")",
+            "%2b": "+",
+            "%2c": ",",
+            "%2f": "/",
+            "%3a": ":",
+            "%3c": "<",
+            "%3d": "=",
+            "%3e": ">",
+            "%3f": "?",
+            "%5b": "[",
+            "%5d": "]",
+            "%e2%80%9c": "“",
+            "%e2%80%9d": "”",
+        }
+
+        for key, value in patterns.items():
+            if key in display_name:
+                display_name = display_name.replace(key, value)
+
+        return display_name
+
+    @staticmethod
+    def check_lang_code(lang_code: str, stats: types.AgentData):
+        """Check to see if file lang_code matches user input lang_code."""
+        return stats.lang_code == lang_code
diff --git a/src/dfcx_scrapi/agent_extract/entity_types.py b/src/dfcx_scrapi/agent_extract/entity_types.py
new file mode 100644
index 00000000..17469b58
--- /dev/null
+++ b/src/dfcx_scrapi/agent_extract/entity_types.py
@@ -0,0 +1,168 @@
+"""Entity Type processing methods and functions."""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+from typing import Dict
+
+from dfcx_scrapi.agent_extract import common
+from dfcx_scrapi.agent_extract import types
+
+class EntityTypes:
+    """Entity Type processing methods and functions."""
+
+    def __init__(self):
+        self.common = common.Common()
+
+    @staticmethod
+    def build_entity_type_path_list(agent_local_path: str):
+        """Builds a list of dirs, each representing an Entity Type directory.
+
+        Ex: /path/to/agent/entityTypes/<entity_type_dir>
+
+        This dir path can then be used to find the next level of information
+        in the directory by appending the appropriate next dir structures like:
+        - <entity_type_name>.json, for the Entity Type object
+        - /entities, for the Entities dir
+        """
+        root_dir = agent_local_path + "/entityTypes"
+
+        entity_type_paths = []
+
+        for entity_type_dir in os.listdir(root_dir):
+            entity_type_dir_path = f"{root_dir}/{entity_type_dir}"
+            entity_type_paths.append(entity_type_dir_path)
+
+        return entity_type_paths
+
+    @staticmethod
+    def build_lang_code_paths(etype: types.EntityType):
+        """Builds dict of lang codes and file locations.
+
+        The language_codes and paths for each file are stored in a dictionary
+        inside of the Entity Type dataclass. This dict is accessed later to
+        lint each file and provide reporting based on each language code.
+        """
+        root_dir = etype.dir_path + "/entities"
+
+        for lang_file in os.listdir(root_dir):
+            lang_code = lang_file.split(".")[0]
+            lang_code_path = f"{root_dir}/{lang_file}"
+            etype.entities[lang_code] = {"file_path": lang_code_path}
+
+    @staticmethod
+    def build_excluded_phrases_path(etype: types.EntityType, lang_code: str):
+        """Builds a dict of excluded phrases and file locations."""
+        root_dir = etype.dir_path + "/excludedPhrases"
+        lang_code_path = f"{root_dir}/{lang_code}.json"
+
+        return lang_code_path
+
+    @staticmethod
+    def process_entity_type_metadata(etype: types.EntityType):
+        """Extract metadata for Entity Type for later processing."""
+        metadata_file = etype.dir_path + f"/{etype.display_name}.json"
+
+        with open(metadata_file, "r", encoding="UTF-8") as etype_file:
+            etype.data = json.load(etype_file)
+            etype.resource_id = etype.data.get("name", None)
+            etype.kind = etype.data.get("kind", None)
+            etype.auto_expansion = etype.data.get("autoExpansionMode", None)
+            etype.fuzzy_extraction = etype.data.get(
+                "enableFuzzyExtraction", False)
+
+            etype_file.close()
+
+    def process_excluded_phrases_language_codes(
+            self, data: Dict[str, str], lang_code_path: str):
+        """Process all ecluded phrases lang_code files."""
+        with open(lang_code_path, "r", encoding="UTF-8") as ent_file:
+            new_data = json.load(ent_file)
+            data["excluded_phrases"] = new_data.get("excludedPhrases", None)
+
+        return data
+
+    def process_excluded_phrases(self, etype: types.EntityType, lang_code: str,
+                                 data: Dict[str, str]):
+        """Process the excluded phrases if they exist."""
+        if "excludedPhrases" in os.listdir(etype.dir_path):
+            lang_code_path = self.build_excluded_phrases_path(etype, lang_code)
+            data = self.process_excluded_phrases_language_codes(
+                data, lang_code_path)
+
+        return data
+
+    def process_language_codes(
+            self, etype: types.EntityType, stats: types.AgentData):
+        """Process all Entity Type lang_code files."""
+        for lang_code in etype.entities:
+            ent_file_path = etype.entities[lang_code]["file_path"]
+
+            if not self.common.check_lang_code(lang_code, stats):
+                continue
+
+            with open(ent_file_path, "r", encoding="UTF-8") as ent_file:
+                data = json.load(ent_file)
+                data["name"] = f"{stats.agent_id}/entityTypes/"\
+                    f"{etype.resource_id}"
+                data["display_name"] = etype.display_name
+                data["kind"] = etype.kind
+                data["entities"] = data.get("entities", None)
+                data = self.process_excluded_phrases(etype, lang_code, data)
+                stats.entity_types.append(data)
+
+                ent_file.close()
+
+        return stats
+
+    def process_entities(self, etype: types.EntityType, stats: types.AgentData):
+        """Process the Entity files inside of an Entity Type."""
+        if "entities" in os.listdir(etype.dir_path):
+            self.build_lang_code_paths(etype)
+            stats = self.process_language_codes(etype, stats)
+
+        return stats
+
+    def process_entity_type(
+            self, etype: types.EntityType, stats: types.AgentData):
+        """Process a Single Entity Type dir and all subdirectories."""
+
+        etype.display_name = self.common.parse_filepath(
+            etype.dir_path, "entity_type")
+        etype.display_name = self.common.clean_display_name(etype.display_name)
+
+        self.process_entity_type_metadata(etype)
+        stats = self.process_entities(etype, stats)
+        stats.total_entity_types += 1
+
+        return stats
+
+    def process_entity_types_directory(
+            self, agent_local_path: str, stats: types.AgentData):
+        """Processing the Entity Types dir in the JSON Package structure."""
+        # Create a list of all Entity Type paths to iter through
+        entity_type_paths = self.build_entity_type_path_list(agent_local_path)
+
+        for entity_type_path in entity_type_paths:
+            etype = types.EntityType()
+            etype.dir_path = entity_type_path
+
+            stats = self.process_entity_type(etype, stats)
+            full_etype_id = f"{stats.agent_id}/entityTypes/{etype.resource_id}"
+            stats.entity_types_map[etype.display_name] = full_etype_id
+
+        return stats
diff --git a/src/dfcx_scrapi/agent_extract/flows.py b/src/dfcx_scrapi/agent_extract/flows.py
new file mode 100644
index 00000000..6d991dac
--- /dev/null
+++ b/src/dfcx_scrapi/agent_extract/flows.py
@@ -0,0 +1,301 @@
+"""Flow extract methods and functions."""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+from typing import List
+
+from dfcx_scrapi.agent_extract import graph
+from dfcx_scrapi.agent_extract import common
+from dfcx_scrapi.agent_extract import types
+from dfcx_scrapi.agent_extract import pages
+from dfcx_scrapi.agent_extract import routes
+from dfcx_scrapi.agent_extract import route_groups
+
+
+class Flows:
+    """Flow processing methods and functions."""
+
+    def __init__(self):
+        self.common = common.Common()
+        self.pages = pages.Pages()
+        self.rgs = route_groups.RouteGroups()
+        self.routes = routes.Fulfillments()
+        self.special_pages = [
+            "End Session",
+            "End Flow",
+            "Start Page",
+            "Current Page",
+            "Previous Page",
+        ]
+
+    @staticmethod
+    def build_flow_path_list(agent_local_path: str):
+        """Builds a list of dirs, each representing a Flow directory.
+
+        Ex: /path/to/agent/flows/<flow_dir>
+
+        This dir path can then be used to find the next level of information
+        in the directory by appending the appropriate next dir structures like:
+        - <flow_name>.json, for the Flow object
+        - /transitionRouteGroups, for the Route Groups dir
+        - /pages, for the Pages dir
+        """
+        root_dir = agent_local_path + "/flows"
+
+        flow_paths = []
+
+        for flow_dir in os.listdir(root_dir):
+            flow_dir_path = f"{root_dir}/{flow_dir}"
+            flow_paths.append(flow_dir_path)
+
+        return flow_paths
+
+    @staticmethod
+    def remove_flow_pages_from_set(input_set: set) -> set:
+        """Remove any transitions tagged with FLOW.
+
+        Some route transitions go to Flow instead of Page. For these
+        transitions, we tag them with `FLOW` for easier identification later.
+        However, when reporting on Graph inconsistencies like Dangling or
+        Unreachable pages, we want to remove these from any result sets as they
+        are not relevant.
+        """
+        filtered_set = set()
+
+        for page in input_set:
+            if "FLOW" not in page:
+                filtered_set.add(page)
+
+        return filtered_set
+
+    def find_unreachable_pages(self, flow: types.Flow):
+        """Find Unreachable Pages in the graph.
+
+        An Unreachable Page is defined as:
+          - A Page which has no incoming edge when traversed from Start Page.
+            That is, it is unreachable in the graph by any practical means.
+          - A Page which is connected to a root unreachable page. That is, a
+            page that could have both incoming or outgoing routes, but due to
+            its connectedness to the root orphan page, is unreachable in the
+            graph.
+
+        Here we will compute the symmetric difference of 2 sets:
+          - Active Pages (i.e. Pages that were reachable in the graph)
+          - Used Pages (i.e. Pages that were used by some Route)
+
+        If an Unreachable Page has children that it routes to, those children
+        will appear in Used Pages, although they will ultimately be
+        unreachable. It's possible for an Unreachable Page to route back to an
+        Active Page in the graph. For these instances, we don't want to count
+        those pages as unreachable, because they are reachable via other
+        sections of the graph.
+        """
+        filtered_set = flow.active_pages.symmetric_difference(
+            flow.graph.used_nodes
+        )
+        filtered_set = self.remove_flow_pages_from_set(filtered_set)
+        flow.unreachable_pages.update(filtered_set)
+
+        return flow
+
+    def find_unused_pages(self, flow: types.Flow):
+        """Find Unused Pages in the graph.
+
+        An Unused Page is defined as:
+          - A Page which has no incoming or outgoing edge AND
+          - A Page which exists in the Agent design time, but which is not
+            present anywhere in the graph, either visible or non-visible.
+
+        Here we will compute the difference of 2 sets:
+          - All Pages (i.e. Pages that exist in the Agent Design Time)
+          - Used Pages (i.e. Pages that were used by some Route)
+
+        The resulting set will consist of 2 types of Pages:
+          - Truly Unused Pages
+          - Unreachable Root Pages
+
+        Unreachable Root Pages end up in the results due to the fact that no
+        other Active Page is pointing to them. We remove these from the
+        resulting set before presenting the Truly Unused Pages.
+        """
+
+        # Discard special pages as they are non-relevant for final outcome
+        for page in self.special_pages:
+            flow.all_pages.discard(page)
+
+        prelim_unused = flow.all_pages.difference(flow.graph.used_nodes)
+
+        # Filter out Unreachable Root Pages
+        filtered_set = set()
+
+        for page in prelim_unused:
+            if page not in flow.graph.edges:
+                filtered_set.add(page)
+            else:
+                flow.unreachable_pages.add(page)
+
+        flow.unused_pages = filtered_set
+
+        return flow
+
+    def recurse_edges(
+        self, edges: List, page: types.Page, dangling: set, visited: set
+    ):
+        """Recursive method searching graph edges for Active / Dangling Pages.
+
+        A byproduct of searching for Dangling Pages in the graph is that we can
+        produce a set of Active Pages in the graph. These are pages that are
+        reachable when traversing from the Start Page. These can then be used
+        to determine Unreachable Pages in another method.
+        """
+        # For Flow Start Pages, we prepend the Flow name for later
+        # identification. For this section, we'll need to strip it off to
+        # compare with the other sets.
+        if page in edges:
+            for inner_page in edges[page]:
+                if inner_page not in visited:
+                    visited.add(inner_page)
+                    dangling, visited = self.recurse_edges(
+                        edges, inner_page, dangling, visited
+                    )
+
+        else:
+            dangling.add(page)
+
+        return dangling, visited
+
+    def find_dangling_pages(self, flow: types.Flow):
+        """Find Dangling Pages in the graph.
+
+        Dangling Page is defined as:
+          - Any page that exists in the graph that has no outgoing edge
+        Active Page is defined as:
+          - Any page that is reachable via an active route in the graph and can
+            be traced back to the Start Page.
+
+        These pages can result in a conversational "dead end" which is
+        potentially unrecoverable.
+        A byproduct of searching for the dangling pages is locating all of the
+        "active" pages. These are the pages that are "visited" as we traverse
+        the graph. We'll also return Active Pages in this method since they
+        will be used for downstream tasks.
+        """
+
+        flow.dangling_pages, flow.active_pages = self.recurse_edges(
+            flow.graph.edges,
+            f"{flow.display_name}: Start Page",
+            flow.dangling_pages,
+            flow.active_pages,
+        )
+
+        # Clean up Special Pages
+        for page in self.special_pages:
+            flow.dangling_pages.discard(page)
+
+        flow.dangling_pages = self.remove_flow_pages_from_set(
+            flow.dangling_pages
+        )
+
+        return flow
+
+    def process_start_page(self, flow: types.Flow, stats: types.AgentData):
+        """Process a single Flow Path file."""
+        with open(flow.start_page_file, "r", encoding="UTF-8") as flow_file:
+            page = types.Page(flow=flow)
+            page.display_name = f"{flow.display_name}: Start Page"
+
+            # We keep track of an instance specific Flow graph for the current
+            # Flow, and then a main Graph for the entire agent.
+            flow.graph.add_node(page.display_name)
+            stats.graph.add_node(page.display_name)
+
+            page.data = json.load(flow_file)
+            page.events = page.data.get("eventHandlers", None)
+            page.routes = page.data.get("transitionRoutes", None)
+            page.route_groups = page.data.get("transitionRouteGroups", None)
+            stats.flows.append(page.data)
+
+            flow.resource_id = page.data.get("name", None)
+
+            # Order of processing is important
+            stats = self.routes.process_routes(page, stats)
+            stats = self.routes.process_events(page, stats)
+
+            if page.route_groups:
+                page, stats = self.routes.set_route_group_targets(page, stats)
+
+            flow_file.close()
+
+        full_flow_id = f"{stats.agent_id}/flows/{flow.resource_id}"
+        stats.flows_map[flow.display_name] = full_flow_id
+        stats.flow_page_map[flow.display_name] = {
+            "id": full_flow_id,
+            "pages": {}
+            }
+
+        return stats
+
+    def process_flow(self, flow: types.Flow, stats: types.AgentData):
+        """Process a Single Flow dir and all subdirectories."""
+        flow.file_name = self.common.parse_filepath(flow.dir_path, "flow")
+        flow.display_name = self.common.clean_display_name(flow.file_name)
+
+        flow.start_page_file = f"{flow.dir_path}/{flow.file_name}.json"
+
+        stats.pages[flow.display_name] = []
+        stats.active_intents[flow.display_name] = []
+        stats = self.process_start_page(flow, stats)
+        stats = self.pages.process_pages_directory(flow, stats)
+        stats = self.rgs.process_route_groups_directory(flow, stats)
+
+        # Order of Find Operations is important here!
+        flow = self.find_unused_pages(flow)
+        flow = self.find_dangling_pages(flow)
+        flow = self.find_unreachable_pages(flow)
+
+        stats.active_pages[flow.display_name] = flow.active_pages
+        stats.unused_pages[flow.display_name] = flow.unused_pages
+        stats.unreachable_pages[flow.display_name] = flow.unreachable_pages
+
+        return stats
+
+    def process_flows_directory(
+            self, agent_local_path: str, stats: types.AgentData):
+        """Process the top level Flows dir in the JSON Package structure.
+
+        The following files/dirs exist under the `flows` dir:
+        - Flow object (i.e. Flow START_PAGE)
+        - transitionRouteGroups
+        - pages
+
+        In Dialogflow CX, the START_PAGE of each Flow is a special kind of Page
+        that exists within the Flow object itself. In this method, we will lint
+        the Flow object, all files in the transitionRouteGroups dir and all
+        files in the pages dir.
+        """
+        # Create a list of all Flow paths to iter through
+        flow_paths = self.build_flow_path_list(agent_local_path)
+        stats.total_flows = len(flow_paths)
+
+        for flow_path in flow_paths:
+            flow = types.Flow()
+            flow.graph = graph.Graph()
+            flow.dir_path = flow_path
+            stats = self.process_flow(flow, stats)
+
+        return stats
diff --git a/src/dfcx_scrapi/agent_extract/gcs_utils.py b/src/dfcx_scrapi/agent_extract/gcs_utils.py
new file mode 100644
index 00000000..c3daff89
--- /dev/null
+++ b/src/dfcx_scrapi/agent_extract/gcs_utils.py
@@ -0,0 +1,68 @@
+"""Utils for Cloud Storage and local file manipulation."""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import zipfile
+from google.cloud import storage
+from google.oauth2 import service_account
+
+
+class GcsUtils:
+    """Utils for Cloud Storage and local file manipulation."""
+
+    def __init__(self, creds_path: str = None, project_id: str = None):
+        if creds_path and project_id:
+            self.creds = service_account.Credentials.from_service_account_file(
+                creds_path
+            )
+            self.gcs_client = storage.Client(
+                credentials=self.creds, project=project_id
+            )
+
+        else:
+            self.gcs_client = storage.Client()
+
+    @staticmethod
+    def unzip(agent_zip_file_path: str, extract_path: str):
+        """Unzip file locally."""
+        with zipfile.ZipFile(agent_zip_file_path, "r") as zip_ref:
+            zip_ref.extractall(extract_path)
+
+    @staticmethod
+    def check_for_gcs_file(file_path: str) -> bool:
+        """Validates GCS path vs. local path."""
+        is_gcs_file = False
+
+        file_prefix = file_path.split("/")[0]
+        if file_prefix == "gs:":
+            is_gcs_file = True
+
+        return is_gcs_file
+
+    def download_gcs(self, gcs_path: str, local_path: str = None):
+        """Downloads the specified GCS file to local machine."""
+        path = gcs_path.split("//")[1]
+        bucket = path.split("/", 1)[0]
+        gcs_object = path.split("/", 1)[1]
+        file_name = gcs_object.split("/")[-1]
+        bucket = self.gcs_client.bucket(bucket)
+        blob = storage.Blob(gcs_object, bucket)
+
+        if local_path:
+            file_name = local_path + "/" + file_name
+
+        blob.download_to_filename(file_name)
+
+        return file_name
diff --git a/src/dfcx_scrapi/agent_extract/graph.py b/src/dfcx_scrapi/agent_extract/graph.py
new file mode 100644
index 00000000..b94217ce
--- /dev/null
+++ b/src/dfcx_scrapi/agent_extract/graph.py
@@ -0,0 +1,45 @@
+"""Utility class for managing graph structure."""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+
+class Graph:
+    """Utility class for manaing graph structure."""
+
+    def __init__(self):
+        self.nodes = set()
+        self.edges = collections.defaultdict(list)
+        self.used_nodes = set()
+
+    def add_node(self, node):
+        """Add node to set of all nodes, regardless of use in graph."""
+        self.nodes.add(node)
+
+    def add_edge(self, node1, node2):
+        self.edges[node1].append(node2)
+
+    def add_used_node(self, node):
+        """Add node to set of active in use nodes for the graph."""
+        self.used_nodes.add(node)
+
+    def remove_node(self, node):
+        self.nodes.remove(node)
+
+    def remove_edge(self, node1, node2):
+        self.edges[node1].remove(node2)
+
+    def __str__(self):
+        return f"Graph({self.nodes}, {self.edges})"
diff --git a/src/dfcx_scrapi/agent_extract/intents.py b/src/dfcx_scrapi/agent_extract/intents.py
new file mode 100644
index 00000000..e8d46dca
--- /dev/null
+++ b/src/dfcx_scrapi/agent_extract/intents.py
@@ -0,0 +1,166 @@
+"""Intent processing methods and functions."""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+from dfcx_scrapi.agent_extract import common
+from dfcx_scrapi.agent_extract import types
+
+
+class Intents:
+    """Intent processing methods and functions."""
+
+    def __init__(self):
+        self.common = common.Common()
+
+    @staticmethod
+    def parse_lang_code(lang_code_path: str) -> str:
+        """Extract the language_code from the given file path."""
+
+        first_parse = lang_code_path.split("/")[-1]
+        lang_code = first_parse.split(".")[0]
+
+        return lang_code
+
+    @staticmethod
+    def build_lang_code_paths(intent: types.Intent):
+        """Builds dict of lang codes and file locations.
+
+        The language_codes and paths for each file are stored in a dictionary
+        inside of the Intent dataclass. This dict is access later to process
+        each file and provide reporting based on each language code.
+        """
+        root_dir = intent.dir_path + "/trainingPhrases"
+
+        for lang_file in os.listdir(root_dir):
+            lang_code = lang_file.split(".")[0]
+            lang_code_path = f"{root_dir}/{lang_file}"
+            intent.training_phrases[lang_code] = {"file_path": lang_code_path}
+
+    @staticmethod
+    def build_intent_path_list(agent_local_path: str):
+        """Builds a list of dirs, each representing an Intent directory.
+
+        Ex: /path/to/agent/intents/<intent_dir>
+
+        This dir path can be used to find the next level of information
+        in the directory by appending the appropriate next dir structures like:
+        - <intent_name>.json, for the Intent object metadata
+        - /trainingPhrases, for the Training Phrases dir
+        """
+        root_dir = agent_local_path + "/intents"
+
+        intent_paths = []
+
+        for intent_dir in os.listdir(root_dir):
+            intent_dir_path = f"{root_dir}/{intent_dir}"
+            intent_paths.append(intent_dir_path)
+
+        return intent_paths
+
+    def process_intent_metadata(
+            self, intent: types.Intent):
+        """Process the metadata file for a single Intent."""
+        intent.metadata_file = f"{intent.dir_path}/{intent.display_name}.json"
+
+        try:
+            with open(intent.metadata_file, "r", encoding="UTF-8") as meta_file:
+                intent.data = json.load(meta_file)
+                intent.resource_id = intent.data.get("name", None)
+                intent.labels = intent.data.get("labels", None)
+                intent.description = intent.data.get("description", None)
+                intent.parameters = intent.data.get("parameters", None)
+
+                meta_file.close()
+
+        except FileNotFoundError:
+            pass
+
+    def process_language_codes(
+            self, intent: types.Intent, stats: types.AgentData):
+        """Process all training phrase lang_code files."""
+
+        for lang_code in intent.training_phrases:
+            tp_file = intent.training_phrases[lang_code]["file_path"]
+
+            if not self.common.check_lang_code(lang_code, stats):
+                continue
+
+            with open(tp_file, "r", encoding="UTF-8") as tps:
+                data = json.load(tps)
+                data["name"] = f"{stats.agent_id}/intents/{intent.resource_id}"
+                data["display_name"] = intent.display_name
+                data["labels"] = intent.labels
+                data["description"] = intent.description
+                data["parameters"] = intent.parameters
+                stats.intents.append(data)
+                stats.total_training_phrases += len(data["trainingPhrases"])
+
+                tps.close()
+
+        return stats
+
+    def process_training_phrases(
+            self, intent: types.Intent, stats: types.AgentData):
+        """Process the Training Phrase dir for a single Intent."""
+        if "trainingPhrases" in os.listdir(intent.dir_path):
+            self.build_lang_code_paths(intent)
+            stats = self.process_language_codes(intent, stats)
+
+        return stats
+
+    def process_intent(self, intent: types.Intent, stats: types.AgentData):
+        """Process a single Intent directory and associated files."""
+        intent.display_name = self.common.parse_filepath(
+            intent.dir_path, "intent")
+        intent.display_name = self.common.clean_display_name(
+            intent.display_name)
+
+        self.process_intent_metadata(intent)
+        stats = self.process_training_phrases(intent, stats)
+        stats.total_intents += 1
+
+        return stats
+
+    def process_intents_directory(
+            self, agent_local_path: str, stats: types.AgentData):
+        """Processing the top level Intents Dir in the JSON Package structure.
+
+        The following files/dirs exist under the `intents` dir:
+        - <intent_display_name> Directory
+          - trainingPhrases
+            - <language-code>.json
+          - <intent_display_name> Object
+
+        In Dialogflow CX, the Training Phrases of each Intent are stored in
+        individual .json files by language code under each Intent Display
+        Name. In this method, we will process all Intent dirs, including the
+        training phrase files and metadata objects for each Intent.
+        """
+        # Create a list of all Intent paths to iter through
+        intent_paths = self.build_intent_path_list(agent_local_path)
+        stats.intents = []
+
+        for intent_path in intent_paths:
+            intent = types.Intent()
+            intent.dir_path = intent_path
+
+            stats = self.process_intent(intent, stats)
+            full_intent_id = f"{stats.agent_id}/intents/{intent.resource_id}"
+            stats.intents_map[intent.display_name] = full_intent_id
+
+        return stats
diff --git a/src/dfcx_scrapi/agent_extract/pages.py b/src/dfcx_scrapi/agent_extract/pages.py
new file mode 100644
index 00000000..51a710c6
--- /dev/null
+++ b/src/dfcx_scrapi/agent_extract/pages.py
@@ -0,0 +1,138 @@
+"""Pages processing methods and functions."""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+from typing import Dict, Any
+
+from dfcx_scrapi.agent_extract import common
+from dfcx_scrapi.agent_extract import types
+from dfcx_scrapi.agent_extract import routes
+
+
+class Pages:
+    """Pages processing methods and functions."""
+
+    def __init__(self):
+        self.common = common.Common()
+        self.routes = routes.Fulfillments()
+
+    @staticmethod
+    def build_page_path_list(flow_path: str):
+        """Builds a list of files, each representing a Page.
+
+        Ex: /path/to/agent/flows/<flow_dir>/pages/<page_name>.json
+        """
+        pages_path = f"{flow_path}/pages"
+
+        page_paths = []
+
+        for page in os.listdir(pages_path):
+            page_file_path = f"{pages_path}/{page}"
+            page_paths.append(page_file_path)
+
+        return page_paths
+
+    @staticmethod
+    def get_form_parameter_data(param: Dict[str, Any], page: types.Page):
+        fp = types.FormParameter(page=page)
+        fp.display_name = param.get("displayName", None)
+        fp.entity_type = param.get("entityType", None)
+        fp.required = param.get("required", None)
+
+        fp.fill_behavior = param.get("fillBehavior", None)
+
+        if fp.fill_behavior:
+            fp.init_fulfillment = fp.fill_behavior.get(
+                "initialPromptFulfillment", None)
+            fp.reprompt_handlers = fp.fill_behavior.get(
+                "repromptEventHandlers", None)
+
+        fp.advanced_settings = page.form.get("advancedSettings", None)
+
+        if fp.advanced_settings:
+            fp.dtmf_settings = fp.advanced_settings.get("dtmfSettings", None)
+
+        return fp
+
+    def process_form(self, page: types.Page, stats: types.AgentData):
+        """Process the Form and sub-resources within it for the Page."""
+        parameters = page.form.get("parameters", None)
+
+        if parameters:
+            for param in parameters:
+                fp = self.get_form_parameter_data(param, page)
+                stats = self.routes.process_reprompt_handlers(fp, stats)
+
+        return stats
+
+
+    def process_page(self, page: types.Page, stats: types.AgentData):
+        """Process a Single Page file."""
+        page.display_name = self.common.parse_filepath(page.page_file, "page")
+        page.display_name = self.common.clean_display_name(page.display_name)
+
+        stats.graph.add_node(page.display_name)
+        page.flow.graph.add_node(page.display_name)
+
+        page.flow.all_pages.add(page.display_name)
+
+        with open(page.page_file, "r", encoding="UTF-8") as page_file:
+            page.data = json.load(page_file)
+            page.entry = page.data.get("entryFulfillment", None)
+            page.events = page.data.get("eventHandlers", None)
+            page.form = page.data.get("form", None)
+            page.routes = page.data.get("transitionRoutes", None)
+            page.route_groups = page.data.get("transitionRouteGroups", None)
+            page.resource_id = page.data.get("name", None)
+
+            # Order of linting is important here
+            stats = self.routes.process_entry(page, stats)
+            stats = self.routes.process_routes(page, stats)
+            stats = self.routes.process_events(page, stats)
+            stats = self.process_form(page, stats)
+
+            if page.route_groups:
+                page, stats = self.routes.set_route_group_targets(page, stats)
+
+            page_file.close()
+
+        full_flow_id = f"{stats.agent_id}/flows/{page.flow.resource_id}"
+        full_page_id = f"{full_flow_id}/pages/{page.resource_id}"
+        stats.pages[page.flow.display_name].append(page.data)
+        stats.flow_page_map[
+            page.flow.display_name]["pages"][page.display_name] = full_page_id
+
+        return stats
+
+    def process_pages_directory(self, flow: types.Flow, stats: types.AgentData):
+        """Process the Pages dir inside a specific Flow dir.
+
+        Some Flows may not contain Pages, so we check for the existence
+        of the directory before traversing
+        """
+        if "pages" in os.listdir(flow.dir_path):
+            page_paths = self.build_page_path_list(flow.dir_path)
+
+            for page_path in page_paths:
+                page = types.Page(flow=flow)
+                page.agent_id = flow.agent_id
+                page.page_file = page_path
+                stats.total_pages += 1
+                stats = self.process_page(page, stats)
+
+        return stats
diff --git a/src/dfcx_scrapi/agent_extract/route_groups.py b/src/dfcx_scrapi/agent_extract/route_groups.py
new file mode 100644
index 00000000..3a73e50a
--- /dev/null
+++ b/src/dfcx_scrapi/agent_extract/route_groups.py
@@ -0,0 +1,100 @@
+"""Route Groups processing methods and functions."""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import json
+
+from dfcx_scrapi.agent_extract import common
+from dfcx_scrapi.agent_extract import types
+from dfcx_scrapi.agent_extract import routes
+
+
+class RouteGroups:
+    """Route Groups processing methods and functions."""
+
+    def __init__(self):
+        self.special_pages = [
+            "End Session",
+            "End Flow",
+            "Start Page",
+            "Current Page",
+            "Previous Page",
+        ]
+
+        self.common = common.Common()
+        self.routes = routes.Fulfillments()
+
+    @staticmethod
+    def build_route_group_path_list(flow_local_path: str):
+        """Builds a list of files, each representing a Route Group.
+
+        Ex: /path/to/agent/flows/<flow_dir>/transitionRouteGroups/<rg.json>
+        """
+        root_dir = flow_local_path + "/transitionRouteGroups"
+
+        if "transitionRouteGroups" in os.listdir(flow_local_path):
+            rg_paths = []
+
+            for rg_file in os.listdir(root_dir):
+                rg_file_path = f"{root_dir}/{rg_file}"
+                rg_paths.append(rg_file_path)
+
+        return rg_paths
+
+    def process_route_group(self, rg: types.RouteGroup, stats: types.AgentData):
+        """Process a single Route Group."""
+        rg.display_name = self.common.parse_filepath(rg.rg_file, "route_group")
+        rg.display_name = self.common.clean_display_name(rg.display_name)
+
+        with open(rg.rg_file, "r", encoding="UTF-8") as route_group_file:
+            rg.data = json.load(route_group_file)
+            rg.resource_id = rg.data.get("name", None)
+            rg.display_name = rg.data.get("displayName", None)
+            rg.routes = rg.data.get("transitionRoutes", None)
+
+            stats = self.routes.process_routes(rg, stats)
+
+            route_group_file.close()
+
+        full_flow_id = f"{stats.agent_id}/flows/{rg.flow.resource_id}"
+        full_rg_id = f"{full_flow_id}/transitionRouteGroups/{rg.resource_id}"
+        stats.route_groups_map[
+            rg.flow.display_name]["route_groups"][rg.display_name] = full_rg_id
+        stats.route_groups[rg.flow.display_name].append(rg.data)
+
+        return stats
+
+    def process_route_groups_directory(
+            self, flow: types.Flow, stats: types.AgentData):
+        """Process Route Groups dir in the JSON Package structure."""
+        if "transitionRouteGroups" in os.listdir(flow.dir_path):
+            # Create a list of all Route Group paths to iter through
+            rg_paths = self.build_route_group_path_list(flow.dir_path)
+            stats.total_route_groups += len(rg_paths)
+
+            full_flow_id = f"{stats.agent_id}/flows/{flow.resource_id}"
+            stats.route_groups_map[flow.display_name] = {
+                "id": full_flow_id,
+                "route_groups": {}
+            }
+            stats.route_groups[flow.display_name] = []
+
+            for rg_path in rg_paths:
+                rg = types.RouteGroup(flow=flow)
+                rg.rg_file = rg_path
+                stats = self.process_route_group(rg, stats)
+
+        return stats
diff --git a/src/dfcx_scrapi/agent_extract/routes.py b/src/dfcx_scrapi/agent_extract/routes.py
new file mode 100644
index 00000000..f91d61ed
--- /dev/null
+++ b/src/dfcx_scrapi/agent_extract/routes.py
@@ -0,0 +1,320 @@
+"""Fulfillment routes processing methods and functions."""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, Any
+
+from dfcx_scrapi.agent_extract import common
+from dfcx_scrapi.agent_extract import types
+
+
+class Fulfillments:
+    """Fulfillment routes processing methods and functions."""
+
+    def __init__(self):
+        self.common = common.Common()
+        self.route_parameters = {}
+
+    @staticmethod
+    def check_for_webhook(page: types.Page, path: Dict[str, Any]):
+        """Check the current route for existence of webhook."""
+        if "webhook" in path:
+            page.has_webhook = True
+
+    @staticmethod
+    def check_for_webhook_event_handlers(route: types.Fulfillment):
+        """Check for Webhook Error Event Handler on Page.
+
+        In this method, we're interested in the following conditions:
+         - Page is currently flagged w/webhook = True
+         - Page HAS NOT been flagged w/having a webhook error handler
+         - The trigger MATCHES the pattern 'webhook.error'
+
+        If a Page and its Route meet all the criteria, we'll flip the bit.
+        Otherwise, the webhook handler bit will remain False, causing a rule
+        flag."""
+
+        if all(
+            [
+                route.page.has_webhook,
+                not route.page.has_webhook_event_handler,
+                "webhook.error" in route.trigger,
+            ]
+        ):
+            route.page.has_webhook_event_handler = True
+
+    @staticmethod
+    def check_for_intent(route: types.Fulfillment):
+        """Check route data to see if Intent is present."""
+        intent = None
+        if "intent" in route.data:
+            intent = route.data["intent"]
+
+        return intent
+
+    def process_intents_in_routes(
+            self, route: types.Fulfillment, stats: types.AgentData):
+        intent = self.check_for_intent(route)
+        if intent:
+            pair = (intent, route.page.display_name)
+            stats.active_intents[
+                route.page.flow.display_name].append(pair)
+
+        return stats
+
+    def collect_transition_route_trigger(self, route):
+        """Inspect route and return all Intent/Condition info."""
+
+        trigger = []
+
+        if "intent" in route.data:
+            trigger.append("intent")
+
+        if "condition" in route.data:
+            trigger.append("condition")
+
+        if len(trigger) > 0:
+            trigger = "+".join(trigger)
+
+        return trigger
+
+    def get_trigger_info(self, route):
+        """Extract trigger info from route based on primary key."""
+
+        if route.fulfillment_type == "event":
+            trigger = f"event : {route.data.get('event', None)}"
+
+        if route.fulfillment_type == "reprompt_handler":
+            trigger = f"{route.parameter} : event : "\
+                f"{route.data.get('event', None)}"
+
+        if route.fulfillment_type == "transition_route":
+            intent_condition = self.collect_transition_route_trigger(route)
+            trigger = f"route : {intent_condition}"
+
+        return trigger
+
+    def set_route_group_targets(self, page: types.Page, stats: types.AgentData):
+        """Determine Route Targets for Route Group routes."""
+        current_page = page.display_name
+
+        for route_group in page.route_groups:
+            page.flow.graph.add_edge(current_page, route_group)
+            page.flow.graph.add_used_node(route_group)
+
+            stats.graph.add_edge(current_page, route_group)
+            stats.graph.add_used_node(route_group)
+
+        return page, stats
+
+    def set_route_targets(
+            self, route: types.Fulfillment, stats: types.AgentData):
+        """Determine the Route Targets for the specified route.
+
+        Primary function is to build out the graph structure for the
+        Flow based on the current page and where the routes are pointing to.
+        The graph structure can then be traversed later to determine any errors
+        or inconsistencies in design.
+        """
+        current_page = route.page.display_name
+
+        route.target_flow = route.data.get("targetFlow", None)
+        route.target_page = route.data.get("targetPage", None)
+
+        if route.target_page:
+            route.page.flow.graph.add_edge(current_page, route.target_page)
+            route.page.flow.graph.add_used_node(route.target_page)
+
+            stats.graph.add_edge(current_page, route.target_page)
+            stats.graph.add_used_node(route.target_page)
+
+        if route.target_flow:
+            route.page.flow.graph.add_edge(
+                current_page, f"FLOW: {route.target_flow}")
+            route.page.flow.graph.add_used_node(f"FLOW: {route.target_flow}")
+
+            stats.graph.add_edge(
+                current_page, f"FLOW: {route.target_flow}"
+            )
+            stats.graph.add_used_node(f"FLOW: {route.target_flow}")
+
+        return route, stats
+
+    def update_route_parameters(
+            self, route: types.Fulfillment, item: Dict[str, str]):
+        """Update the Route Parameters map based on new info."""
+        flow_name = route.page.flow.display_name
+        page_name = route.page.display_name
+
+        flow_data = self.route_parameters.get(flow_name, None)
+        page_data = None
+
+        if flow_data:
+            page_data = flow_data.get(page_name, None)
+
+        # Flow and Page already exists, append to existing list.
+        if page_data:
+            self.route_parameters[flow_name][page_name].append(item)
+
+        # Flow data exists, but not Page, so only create the Page list.
+        elif flow_data and not page_data:
+            self.route_parameters[flow_name][page_name] = [item]
+
+        # Neither the Flow or Page data exists, so create it all.
+        else:
+            self.route_parameters[flow_name] = {page_name: [item]}
+
+
+    def process_fulfillment_type(
+        self, stats: types.AgentData, route: types.Fulfillment, path: object,
+        key: str):
+        """Parse through specific fulfillment types."""
+        fulfillment_data = path.get(key, None)
+
+        if fulfillment_data:
+            for item in fulfillment_data:
+                # This is where each message type will exist
+                # text, custom payload, etc.
+
+                if "text" in item:
+                    for text in item["text"]["text"]:
+                        route.text = text
+
+                if "parameter" in item:
+                    self.update_route_parameters(route, item)
+
+        return stats
+
+    def process_reprompt_handlers(
+            self, fp: types.FormParameter, stats: types.AgentData):
+        """Processing for Reprompt Event Handlers inside Form parameters.
+
+        While Reprompt Event Handlers are technically Events, they differ from
+        standard Page level Events because they act on the FormParameter data
+        structure, not Fulfillment Route data structure as standard Events do.
+        """
+        if not fp.reprompt_handlers:
+            return stats
+
+        for handler in fp.reprompt_handlers:
+            route = types.Fulfillment(page=fp.page)
+            route.data = handler
+            route.agent_id = fp.page.agent_id
+            route.fulfillment_type = "reprompt_handler"
+            route.parameter = fp.display_name
+            route.trigger = self.get_trigger_info(route)
+            route, stats = self.set_route_targets(route, stats)
+            path = route.data.get("triggerFulfillment", None)
+            event = route.data.get("event", None)
+
+            stats = self.process_intents_in_routes(route, stats)
+
+            if not path and not event:
+                continue
+
+            # Flag for Webhook Handler
+            self.check_for_webhook(fp.page, path)
+
+            stats = self.process_fulfillment_type(
+                stats, route, path, "messages")
+
+        return stats
+
+    def process_events(self, page: types.Page, stats: types.AgentData):
+        """Parse through all Page Event Handlers."""
+        if not page.events:
+            return stats
+
+        for route_data in page.events:
+            route = types.Fulfillment(page=page)
+            route.data = route_data
+            route.agent_id = page.agent_id
+            route.fulfillment_type = "event"
+            route.trigger = self.get_trigger_info(route)
+            route, stats = self.set_route_targets(route, stats)
+            path = route.data.get("triggerFulfillment", None)
+            event = route.data.get("event", None)
+
+            stats = self.process_intents_in_routes(route, stats)
+
+            if not path and not event:
+                continue
+
+            # Flag for Webhook Handler
+            self.check_for_webhook_event_handlers(route)
+
+            stats = self.process_fulfillment_type(
+                stats, route, path, "messages")
+
+        return stats
+
+    def process_routes(self, page: types.Page, stats: types.AgentData):
+        """Parse through all Transition Routes."""
+        tf_key = "triggerFulfillment"
+
+        if not page.routes:
+            return stats
+
+        for route_data in page.routes:
+            route = types.Fulfillment(page=page)
+            route.data = route_data
+            route.agent_id = page.agent_id
+            route.fulfillment_type = "transition_route"
+            route.trigger = self.get_trigger_info(route)
+            route, stats = self.set_route_targets(route, stats)
+
+            stats = self.process_intents_in_routes(route, stats)
+
+            path = route.data.get(tf_key, None)
+
+            if not path:
+                continue
+
+            # Flag for Webhook Handler
+            self.check_for_webhook(page, path)
+
+            stats = self.process_fulfillment_type(
+                stats, route, path, "messages")
+
+            # Preset Params processed here
+            stats = self.process_fulfillment_type(
+                stats, route, path, "setParameterActions"
+            )
+
+        return stats
+
+    def process_entry(self, page: types.Page, stats: types.AgentData):
+        """Process Entry Fulfillment on a single page file.
+
+        The Entry Fulfillment to a Page only has 1 "route" (i.e. itself) so
+        there is no need to loop through multiple routes, as they don't
+        exist for Entry Fulfillment.
+        """
+
+        if not page.entry:
+            return stats
+
+        route = types.Fulfillment(page=page)
+        route.data = page.entry
+        route.agent_id = page.agent_id
+        route.fulfillment_type = "entry"
+        route.trigger = "entry"
+        path = route.data
+
+        self.check_for_webhook(page, path)
+
+        stats = self.process_fulfillment_type(stats, route, path, "messages")
+
+        return stats
diff --git a/src/dfcx_scrapi/agent_extract/types.py b/src/dfcx_scrapi/agent_extract/types.py
new file mode 100644
index 00000000..ac25c34d
--- /dev/null
+++ b/src/dfcx_scrapi/agent_extract/types.py
@@ -0,0 +1,212 @@
+"""Collection of Type Classes used for offline processing."""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, List, Any, Tuple
+from dataclasses import dataclass, field
+
+from dfcx_scrapi.agent_extract import graph as graph_class
+
+@dataclass
+class AgentMetadata:
+    """Used to track the current Agent Metadata attrinbutes."""
+
+    default_language_code: str = None
+    dtmf_settings: bool = False
+    logging_enabled: bool = False
+    speech_adaptation: bool = False
+
+
+@dataclass
+class Flow:
+    """Used to track current Flow Attributes."""
+    agent_id: str = None
+    all_pages: set = field(default_factory=set)
+    active_pages: set = field(default_factory=set)
+    data: Dict[str, Any] = field(default_factory=dict)
+    dangling_pages: set = field(default_factory=set)
+    dir_path: str = None  # Full Directory Path for this Flow
+    display_name: str = None  # Flow Display Name (removed special chars)
+    file_name: str = None  # Original Name of Flow (includes special chars)
+    graph: graph_class.Graph = None
+    resource_id: str = None
+    resource_type: str = "flow"
+    start_page_file: str = None  # File Path Location of START_PAGE
+    unreachable_pages: set = field(default_factory=set)
+    unused_pages: set = field(default_factory=set)
+
+@dataclass
+class Page:
+    """Used to track current Page Attributes."""
+
+    agent_id: str = None
+    data: Dict[str, Any] = None
+    display_name: str = None
+    entry: Dict[str, Any] = None
+    events: List[object] = None
+    flow: Flow = None
+    form: Dict[str, Any] = None
+    has_webhook: bool = False
+    has_webhook_event_handler: bool = False
+    page_file: str = None
+    resource_id: str = None
+    resource_type: str = "page"
+    routes: List[object] = None
+    route_groups: List[str] = None
+
+@dataclass
+class FormParameter:
+    """Tracks Form Paramter attributes within a Page."""
+
+    advanced_settings: str = None
+    agent_id: str = None
+    data: Dict[str, Any] = None
+    display_name: str = None
+    dtmf_settings: str = None
+    entity_type: str = None
+    fill_behavior: Dict[str, Any] = None
+    init_fulfillment: Dict[str, Any] = None
+    page: Page = None
+    reprompt_handlers: Dict[str, Any] = None
+    required: bool = True
+
+
+@dataclass
+class RouteGroup:
+    """Used to track current RouteGroup Attributes."""
+
+    agent_id: str = None
+    data: Dict[str, Any] = None
+    display_name: str = None
+    flow: Flow = None
+    resource_id: str = None
+    resource_type: str = "route_group"
+    rg_file: str = None
+    routes: List[object] = None
+
+@dataclass
+class Fulfillment:
+    """Used to track current Fulfillment Attributes."""
+
+    agent_id: str = None
+    data: Dict[str, Any] = None
+    display_name: str = None  # Inherit from Page easy logging
+    fulfillment_type: str = None  # transition_route | event
+    page: Page = None
+    parameter: str = None # Used for Reprompt Event Handlers
+    target_flow: str = None
+    target_page: str = None
+    text: str = None
+    trigger: str = None
+    resource_type: str = "fulfillment"
+
+@dataclass
+class Intent:
+    """Used to track current Intent Attributes."""
+
+    agent_id: str = None
+    data: Dict[str, Any] = None
+    description: str = None
+    display_name: str = None
+    dir_path: str = None
+    labels: Dict[str, str] = None
+    metadata_file: str = None
+    parameters: List[Dict[str, str]] = field(default_factory=list)
+    resource_id: str = None
+    resource_type: str = "intent"
+    training_phrases: Dict[str, Any] = field(default_factory=dict)
+
+@dataclass
+class EntityType:
+    """Used to track current Flow Attributes."""
+
+    agent_id: str = None
+    auto_expansion: str = None
+    data: Dict[str, Any] = None
+    dir_path: str = None  # Full Directory Path for this Entity Type
+    display_name: str = None  # Entity Type Display Name
+    entities: Dict[str, Any] = field(default_factory=dict)  # Map
+    excluded_phrases: Dict[str, Any] = field(default_factory=dict)  # Map
+    fuzzy_extraction: bool = False
+    kind: str = None  # The kind of Entity Type represented
+    resource_id: str = None
+    resource_type: str = "entity_type"
+
+@dataclass
+class TestCase:
+    """Used to track current Test Case Attributes."""
+
+    associated_intent_data: Dict[str, Any] = None
+    agent_id: str = None
+    agent_path: str = None
+    conversation_turns: List[Any] = None
+    data: Dict[str, Any] = None
+    dir_path: str = None
+    display_name: str = None
+    has_invalid_intent: bool = False
+    intent_data: List[str] = None
+    qualified: bool = False
+    resource_id: str = None
+    resource_type: str = "test_case"
+    tags: List[str] = None
+    test_config: Dict[str, Any] = None
+
+@dataclass
+class Webhook:
+    """Used to track current Webhook attributes."""
+
+    agent_id: str = None
+    agent_path: str = None
+    data: Dict[str, Any] = None
+    dir_path: str = None
+    display_name: str = None
+    resource_id: str = None
+    resource_type: str = "webhook"
+    service_type: str = None
+    timeout: int = 0
+
+@dataclass
+class AgentData:
+    """Used to track agent data for each section processed."""
+    active_intents: Dict[str, List[Tuple[str, str]]] = field(
+        default_factory=dict)
+    active_pages: Dict[str, set] = field(default_factory=dict)
+    agent_id: str = None
+    entity_types: List[Dict[str, Any]] = field(default_factory=list)
+    entity_types_map: Dict[str, Any] = field(default_factory=dict)
+    flow_page_map: Dict[str, Any] = field(default_factory=dict)
+    flows: List[Dict[str, Any]] = field(default_factory=list)
+    flows_map: Dict[str, Any] = field(default_factory=dict)
+    graph: graph_class.Graph = None
+    intents: List[Dict[str, Any]] = field(default_factory=list)
+    intents_map: Dict[str, Any] = field(default_factory=dict)
+    lang_code: str = "en"
+    pages: Dict[str, List[Dict[str, Any]]] = field(default_factory=dict)
+    route_groups: Dict[str, List[Dict[str, Any]]] = field(default_factory=dict)
+    route_groups_map: Dict[str, Any] = field(default_factory=dict)
+    test_cases: List[Dict[str, Any]] = field(default_factory=list)
+    unreachable_pages: Dict[str, set] = field(default_factory=dict)
+    unused_pages: Dict[str, set] = field(default_factory=dict)
+    webhooks: List[Dict[str, Any]] = field(default_factory=list)
+    webhooks_map: Dict[str, Any] = field(default_factory=dict)
+
+    total_flows: int = 0
+    total_pages: int = 0
+    total_intents: int = 0
+    total_training_phrases: int = 0
+    total_entity_types: int = 0
+    total_route_groups: int = 0
+    total_test_cases: int = 0
+    total_webhooks: int = 0
diff --git a/src/dfcx_scrapi/agent_extract/webhooks.py b/src/dfcx_scrapi/agent_extract/webhooks.py
new file mode 100644
index 00000000..38467080
--- /dev/null
+++ b/src/dfcx_scrapi/agent_extract/webhooks.py
@@ -0,0 +1,95 @@
+"""Webhook processing methods and functions."""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+from dfcx_scrapi.agent_extract import common
+from dfcx_scrapi.agent_extract import types
+
+class Webhooks:
+    """Webhook linter methods and functions."""
+
+    def __init__(self):
+        self.common = common.Common()
+
+    @staticmethod
+    def build_webhook_path_list(agent_local_path: str):
+        """Builds a list of webhook file locations."""
+        root_dir = agent_local_path + "/webhooks"
+
+        webhook_paths = []
+
+        for webhook_file in os.listdir(root_dir):
+            webhook_file_path = f"{root_dir}/{webhook_file}"
+            webhook_paths.append(webhook_file_path)
+
+        return webhook_paths
+
+    @staticmethod
+    def get_service_type(webhook: types.Webhook) -> str:
+        """Get the type of Webhook Service that is cofigured."""
+        if "genericWebService" in webhook.data:
+            webhook.service_type = "Generic Web Service"
+
+        else:
+            webhook.service_type = "Other"
+
+        return webhook.service_type
+
+    def process_webhook(self, webhook: types.Webhook, stats: types.AgentData
+            ) -> types.AgentData:
+        """Process a single Webhook file."""
+
+        with open(webhook.dir_path, "r", encoding="UTF-8") as webhook_file:
+            webhook.data = json.load(webhook_file)
+            webhook.resource_id = webhook.data.get("name", None)
+            webhook.display_name = webhook.data.get("displayName", None)
+            webhook.service_type = self.get_service_type(webhook)
+
+            timeout_dict = webhook.data.get("timeout", None)
+            if timeout_dict:
+                webhook.timeout = timeout_dict.get("seconds", None)
+
+            webhook_file.close()
+
+        full_webhook_id = f"{stats.agent_id}/webhooks/{webhook.resource_id}"
+        webhook.data["name"] = full_webhook_id
+        stats.webhooks.append(webhook.data)
+        stats.total_webhooks += 1
+
+        return stats
+
+    def process_webhooks_directory(self, agent_local_path: str,
+                                   stats: types.AgentData) -> types.AgentData:
+        """Processing the top level Webhooks Dir in the JSON Package structure.
+
+        The following files exist under the `webhooks` dir:
+        - <webhook-name>.json
+        """
+        # Create a list of all Webhook paths to iter through
+        webhook_paths = self.build_webhook_path_list(agent_local_path)
+
+        for webhook_path in webhook_paths:
+            webhook = types.Webhook()
+            webhook.dir_path = webhook_path
+
+            stats = self.process_webhook(webhook, stats)
+
+            full_webhook_id = f"{stats.agent_id}/webhooks/{webhook.resource_id}"
+            stats.webhooks_map[webhook.display_name] = full_webhook_id
+
+        return stats
diff --git a/src/dfcx_scrapi/builders/builders_common.py b/src/dfcx_scrapi/builders/builders_common.py
index b44e0186..d14ea69c 100644
--- a/src/dfcx_scrapi/builders/builders_common.py
+++ b/src/dfcx_scrapi/builders/builders_common.py
@@ -43,7 +43,7 @@ def __init__(self, obj=None):
 
     def _check_proto_obj_attr_exist(self):
         """Check if the proto_obj exists otherwise raise an error."""
-        if not self.proto_obj:
+        if self.proto_obj is None:
             raise ValueError(
                 "There is no proto_obj!"
                 "\nUse `create_new_proto_obj` or `load_proto_obj` to continue."
diff --git a/src/dfcx_scrapi/builders/generative_settings.py b/src/dfcx_scrapi/builders/generative_settings.py
new file mode 100644
index 00000000..6cbcf4bb
--- /dev/null
+++ b/src/dfcx_scrapi/builders/generative_settings.py
@@ -0,0 +1,115 @@
+"""A set of builder methods to create CX proto resource objects"""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from typing import List
+
+from google.cloud.dialogflowcx_v3beta1.types.generative_settings import GenerativeSettings
+from google.cloud.dialogflowcx_v3beta1.types.safety_settings import SafetySettings
+from dfcx_scrapi.builders.builders_common import BuildersCommon
+
+# logging config
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)-8s %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+
+class GenerativeSettingsBuilder(BuildersCommon):
+    """Base Class for Generative Settings builder."""
+
+    _proto_type = GenerativeSettings
+    _proto_type_str = "GenerativeSettings"
+
+    def __str__(self) -> str:
+        """String representation of the proto_obj."""
+        self._check_proto_obj_attr_exist()
+
+        return (
+            f"\nlanguage_code: {self.proto_obj.language_code}"
+            f"\nfallback_settings: {self.proto_obj.fallback_settings}"
+            f"\ngenerative_safety_settings: "
+                f"{self.proto_obj.generative_safety_settings}"
+            f"\nknowledge_connector_settings: "
+                f"{self.proto_obj.knowledge_connector_settings}"
+            )
+
+    def fallback_settings(
+            self, selected_prompt: str,
+            prompt_templates: List[
+                GenerativeSettings.FallbackSettings.PromptTemplate]
+                ) -> GenerativeSettings.FallbackSettings:
+        """Build the Fallback Settings proto."""
+        return GenerativeSettings.FallbackSettings(
+            selected_prompt=selected_prompt,
+            prompt_templates=prompt_templates
+        )
+
+    def prompt_template(
+            self, display_name: str, prompt_text: str, frozen: bool = False
+    ):
+        """Build the Prompt Template proto."""
+        return GenerativeSettings.FallbackSettings.PromptTemplate(
+            display_name=display_name,
+            prompt_text=prompt_text,
+            frozen=frozen
+        )
+
+    def banned_phrases(self, phrases: List[str],
+                       language_code: str = "en") -> SafetySettings:
+        """Build the SafetySettings proto with banned phrases."""
+        banned_phrases = []
+        for phrase in phrases:
+            phrase_proto = SafetySettings.Phrase(
+                text=phrase,
+                language_code=language_code
+            )
+            banned_phrases.append(phrase_proto)
+
+        safety_settings = SafetySettings(
+            banned_phrases=banned_phrases
+        )
+
+        return safety_settings
+
+    def knowledge_connector_settings(
+            self, business_name: str, agent_name: str = None,
+            agent_identity: str = None, business_description: str = None,
+            agent_scope: str = None
+            ) -> GenerativeSettings.KnowledgeConnectorSettings:
+        """Build the knowledge settings proto.
+
+        Args:
+          business_name: The name of the company or business that corresponds
+            most closely with the documents in the Data Store(s).
+          agent_name: The name or identify of the conversational Agent. Ex:
+            Bard, Fred, Julia, MedBot, etc.
+          agent_identity: General description of the type of conversational
+            Agent that this is. Ex: AI Assistant, Travel Concierge, Medical
+            Assistant, etc.
+          business_description: A brief description of the business and what it
+            does
+          agent_scope: The defined scope of this conversational Agent and how
+            it should (or should not) interact with users.
+        """
+        return GenerativeSettings.KnowledgeConnectorSettings(
+            business=business_name,
+            agent=agent_name,
+            agent_identity=agent_identity,
+            business_description=business_description,
+            agent_scope=agent_scope
+        )
diff --git a/src/dfcx_scrapi/builders/intents.py b/src/dfcx_scrapi/builders/intents.py
index 0a99f3bc..80fd32f9 100644
--- a/src/dfcx_scrapi/builders/intents.py
+++ b/src/dfcx_scrapi/builders/intents.py
@@ -630,7 +630,7 @@ class IntentStats():
 
     def calc_tps_stats(self):
         """Calculating stats related to training phrases."""
-        for tp in self.intent_proto_obj.training_phrases:
+        for tp in self.intent_proto_obj.training_phrases: # pylint: disable=E1133
             self.repeat_count_dict[tp.repeat_count] += 1
 
             for part in tp.parts:
diff --git a/src/dfcx_scrapi/core/agents.py b/src/dfcx_scrapi/core/agents.py
index 4c8185f0..7b6f949e 100644
--- a/src/dfcx_scrapi/core/agents.py
+++ b/src/dfcx_scrapi/core/agents.py
@@ -86,7 +86,7 @@ def _list_agents_client_request(self, location_id) -> List[
     def list_agents(
         self,
         project_id: str,
-        location: str = None) -> List[types.Agent]:
+        location_id: str = None) -> List[types.Agent]:
         """Get list of all CX agents in a given GCP Region or Project.
 
         This method allows you to provide a GCP Project ID to retrieve all of
@@ -101,7 +101,7 @@ def list_agents(
           List of Agent objects
         """
 
-        if not location:
+        if not location_id:
             region_list = [
                 "global",
                 "us-central1",
@@ -121,7 +121,7 @@ def list_agents(
                 agents += self._list_agents_client_request(location_path)
 
         else:
-            location_path = f"projects/{project_id}/locations/{location}"
+            location_path = f"projects/{project_id}/locations/{location_id}"
             agents = self._list_agents_client_request(location_path)
 
         return agents
@@ -358,13 +358,83 @@ def get_validation_result(
 
         return val_results_dict
 
+    @scrapi_base.api_call_counter_decorator
+    def get_generative_settings(
+        self, agent_id: str = None, language_code: str = "en"
+        ) -> types.generative_settings.GenerativeSettings:
+        """Get the current Generative Settings for the Agent."""
+        if not agent_id:
+            agent_id = self.agent_id
+
+        request = types.agent.GetGenerativeSettingsRequest()
+        request.name = f"{agent_id}/generativeSettings"
+        request.language_code = language_code
+
+        client_options = self._set_region(agent_id)
+        client = services.agents.AgentsClient(
+            credentials=self.creds, client_options=client_options
+        )
+
+        response = client.get_generative_settings(request)
+
+        return response
+
+    @scrapi_base.api_call_counter_decorator
+    def update_generative_settings(
+        self,
+        agent_id: str = None,
+        language_code: str = "en",
+        obj: types.generative_settings.GenerativeSettings = None,
+        **kwargs) -> types.generative_settings.GenerativeSettings:
+        """Update the existing Generative Settings.
+
+        Args:
+          agent_id: Agent ID string in the following format
+              projects/<PROJECT ID>/locations/<LOCATION ID>/agents/<AGENT ID>
+          obj: (Optional) The Generative Settings object in proper format.
+            This can also be extracted by using the get_generative_settings()
+            method or built directly with the Generative Settings Builder class.
+          kwargs: You can find a list of Generative Settings attributes here:
+              https://cloud.google.com/python/docs/reference/dialogflow-cx/
+               latest/google.cloud.dialogflowcx_v3beta1.types.GenerativeSettings
+        Returns:
+          The updated Generative Settings resource object.
+        """
+        if obj:
+            gen_settings = obj
+        else:
+            gen_settings = self.get_generative_settings(agent_id)
+
+        gen_settings.language_code = language_code
+
+        for key, value in kwargs.items():
+            setattr(gen_settings, key, value)
+
+        paths = kwargs.keys()
+        mask = field_mask_pb2.FieldMask(paths=paths)
+
+        client_options = self._set_region(gen_settings.name)
+        client = services.agents.AgentsClient(
+            credentials=self.creds, client_options=client_options
+        )
+
+        response = client.update_generative_settings(
+            generative_settings=gen_settings, update_mask=mask
+        )
+
+        return response
+
 
     @scrapi_base.api_call_counter_decorator
     def export_agent(
         self,
         agent_id: str,
         gcs_bucket_uri: str,
-        environment_display_name: str = None
+        environment_display_name: str = None,
+        data_format: str = "BLOB",
+        git_branch: str = None,
+        git_commit_message: str = None,
+        include_bq_export_settings: bool = False
     ) -> str:
         """Exports the specified CX agent to Google Cloud Storage bucket.
 
@@ -374,17 +444,39 @@ def export_agent(
           gcs_bucket_uri: The Google Cloud Storage bucket/filepath to export the
             agent to in the following format:
               `gs://<bucket-name>/<object-name>`
-          environment_display_name: (Optional) CX Agent environment display name
+          environment_display_name: CX Agent environment display name
             as string. If not set, DRAFT environment is assumed.
+          data_format: Optional. The data format of the exported agent. If not
+            specified, ``BLOB`` is assumed.
+          git_branch: Optional. The Git branch to commit the exported agent to.
+          git_commit_message: Optional. The Git Commit message to send. Only
+            applicable if using `git_branch` arg.
+          include_bigquery_export_settings: Will exclude or included the BQ
+            settings on export.
 
         Returns:
           A Long Running Operation (LRO) ID that can be used to
             check the status of the export using
               dfcx_scrapi.core.operations->get_lro()
         """
+        blob_format = types.agent.ExportAgentRequest.DataFormat(1)
+        json_format = types.agent.ExportAgentRequest.DataFormat(4)
+
         request = types.agent.ExportAgentRequest()
         request.name = agent_id
         request.agent_uri = gcs_bucket_uri
+        request.include_bigquery_export_settings = include_bq_export_settings
+
+        if data_format in ["JSON", "ZIP", "JSON_PACKAGE"]:
+            request.data_format = json_format
+        else:
+            request.data_format = blob_format
+
+        if git_branch:
+            git_settings = types.agent.ExportAgentRequest.GitDestination()
+            git_settings.tracking_branch = git_branch
+            git_settings.commit_message = git_commit_message
+            request.git_destination = git_settings
 
         if environment_display_name:
             self._environments = environments.Environments(creds=self.creds)
@@ -409,18 +501,26 @@ def export_agent(
 
 
     @scrapi_base.api_call_counter_decorator
-    def restore_agent(self, agent_id: str, gcs_bucket_uri: str) -> str:
+    def restore_agent(
+        self,
+        agent_id: str,
+        gcs_bucket_uri: str,
+        restore_option: int = None
+    ) -> str:
         """Restores a CX agent from a gcs_bucket location.
 
-        Currently there is no way to restore back to default
-        settings via the api. The feature request for this is logged.
-
         Args:
           agent_id: CX Agent ID string in the following format
             projects/<PROJECT ID>/locations/<LOCATION ID>/agents/<AGENT ID>
-          gcs_bucket_uri: The Google Cloud Storage bucket/filepath to export the
-            agent to in the following format:
+          gcs_bucket_uri: The Google Cloud Storage bucket/filepath to restore
+            the agent from in the following format:
               `gs://<bucket-name>/<object-name>`
+          restore_option: Optional.
+              if not specified, then restore_option = 0 is assumed
+              Below are the options for restoring an agent (int):
+                  0:unspecified
+                  1:always respect the settings from the exported agent file
+                  2:fallback to default settings if not supported
 
         Returns:
           A Long Running Operation (LRO) ID that can be used to
@@ -432,6 +532,11 @@ def restore_agent(self, agent_id: str, gcs_bucket_uri: str) -> str:
         request.name = agent_id
         request.agent_uri = gcs_bucket_uri
 
+        if restore_option:
+            request.restore_option = types.RestoreAgentRequest.RestoreOption(
+                restore_option
+            )
+
         client_options = self._set_region(agent_id)
         client = services.agents.AgentsClient(
             credentials=self.creds, client_options=client_options
@@ -479,20 +584,27 @@ def update_agent(
         return response
 
     @scrapi_base.api_call_counter_decorator
-    def delete_agent(self, agent_id: str) -> str:
+    def delete_agent(
+        self, agent_id: str = None, obj: types.Agent = None) -> str:
         """Deletes the specified Dialogflow CX Agent.
 
         Args:
           agent_id: CX Agent ID string in the following format
             projects/<PROJECT ID>/locations/<LOCATION ID>/agents/<AGENT ID>
+          obj: (Optional) a CX Agent object of types.Agent
 
         Returns:
           String "Agent '(agent_id)' successfully deleted."
         """
+        if not agent_id:
+            agent_id = self.agent_id
+
+        if obj:
+            agent_id = obj.name
+
         client_options = self._set_region(agent_id)
         client = services.agents.AgentsClient(
-            credentials=self.creds, client_options=client_options
-        )
+            credentials=self.creds, client_options=client_options)
         client.delete_agent(name=agent_id)
 
         return "Agent '{agent_id}' successfully deleted."
diff --git a/src/dfcx_scrapi/core/changelogs.py b/src/dfcx_scrapi/core/changelogs.py
index 9e8aedf4..db7ccf55 100644
--- a/src/dfcx_scrapi/core/changelogs.py
+++ b/src/dfcx_scrapi/core/changelogs.py
@@ -225,7 +225,7 @@ def changelogs_to_dataframe(
                 ],
             )
 
-            df = df.append(log_data)
+            df = pd.concat([df, log_data], ignore_index=True)
 
         df = df.reset_index(drop=True)
 
diff --git a/src/dfcx_scrapi/core/conversation.py b/src/dfcx_scrapi/core/conversation.py
index 7a65c78d..ce354e7b 100644
--- a/src/dfcx_scrapi/core/conversation.py
+++ b/src/dfcx_scrapi/core/conversation.py
@@ -132,7 +132,8 @@ def _get_match_type_from_map(match_type: int):
             4: "NO_MATCH",
             5: "NO_INPUT",
             6: "EVENT",
-            8: "KNOWLEDGE_CONNECTOR"
+            8: "KNOWLEDGE_CONNECTOR",
+            9: "LLM"
         }
 
         return match_type_map[match_type]
@@ -159,10 +160,9 @@ def progress_bar(current, total, bar_length=50, type_="Progress"):
         percent = float(current) * 100 / total
         arrow = "-" * int(percent / 100 * bar_length - 1) + ">"
         spaces = " " * (bar_length - len(arrow))
-        print(
-            f"{type_}({current}/{total})" + f"[{arrow}{spaces}] {percent:.2f}%",
-            end="\r",
-        )
+        logging.info(
+            f"{type_}({current}/{total})" + f"[{arrow}{spaces}] {percent:.2f}%"
+            )
 
     @staticmethod
     def _build_query_params_object(parameters, current_page, disable_webhook):
@@ -294,19 +294,6 @@ def _page_id_mapper(self):
             flow_mapped["page_id"] = flow_mapped.index
 
             flow_mapped = flow_mapped.rename(columns={0: "page_display_name"})
-
-            # add start page
-            start_page_id = flow_id + "/pages/START_PAGE"
-            flow_mapped = pd.concat(
-                [
-                    flow_mapped,
-                    pd.DataFrame(
-                        columns=["page_display_name", "page_id"],
-                        data=[["START_PAGE", start_page_id]],
-                    ),
-                ]
-            )
-
             flow_mapped.insert(0, "flow_display_name", flow_map[flow_id])
             agent_pages_map = pd.concat([agent_pages_map, flow_mapped])
 
@@ -384,7 +371,7 @@ def set_agent_env(self, param, value):
         self.agent_env[param] = value
 
     def checkpoint(self, msg=None, start=False):
-        """Print a checkpoint to time progress and debug bottleneck"""
+        """Log a checkpoint to time progress and debug bottleneck"""
         if start:
             start_time = time.perf_counter()
             self.start_time = start_time
@@ -393,7 +380,7 @@ def checkpoint(self, msg=None, start=False):
         duration = round((time.perf_counter() - start_time), 2)
         if duration > 2:
             if msg:
-                print(f"{duration:0.2f}s {msg}")
+                logging.info(f"{duration:0.2f}s {msg}")
 
     @scrapi_base.api_call_counter_decorator
     def reply(
diff --git a/src/dfcx_scrapi/core/data_stores.py b/src/dfcx_scrapi/core/data_stores.py
new file mode 100644
index 00000000..4fd944f1
--- /dev/null
+++ b/src/dfcx_scrapi/core/data_stores.py
@@ -0,0 +1,168 @@
+"""Data Stores Resource functions for Vertex Search and Conversation."""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from typing import Dict, List
+from google.longrunning.operations_pb2 import Operation
+from google.cloud import discoveryengine_v1alpha
+from google.cloud.discoveryengine_v1alpha.types import DataStore
+
+from dfcx_scrapi.core import scrapi_base
+
+# logging config
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)-8s %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+
+class DataStores(scrapi_base.ScrapiBase):
+    """Core Class for Data Store Resource functions."""
+    def __init__(
+        self,
+        project_id: str,
+        creds_path: str = None,
+        creds_dict: Dict = None,
+        creds=None,
+        scope=False,
+    ):
+        super().__init__(
+            creds_path=creds_path,
+            creds_dict=creds_dict,
+            creds=creds,
+            scope=scope,
+        )
+
+        self.project_id = project_id
+
+    @staticmethod
+    def _get_content_config(content_type: str) -> DataStore.ContentConfig:
+        """Build the Content Config used for the Data Store."""
+        cc_map = {
+            "unstructured": 1,
+            "structured": 2,
+            "website": 3
+        }
+        cc = DataStore.ContentConfig(cc_map[content_type])
+
+        return cc
+
+    def get_data_stores_map(self, location: str = "global",
+                            reverse: bool = False) -> Dict[str,str]:
+        """Get a user friendly mapping for Data Store Names and IDs."""
+        if reverse:
+            data_store_dict = {
+                datastore.display_name: datastore.name
+                for datastore in self.list_data_stores(location=location)
+            }
+
+        else:
+            data_store_dict = {
+                datastore.name: datastore.display_name
+                for datastore in self.list_data_stores(location=location)
+            }
+
+        return data_store_dict
+
+    def list_data_stores(
+            self, location: str = "global") -> List[DataStore]:
+        """List all data stores for a given project and location."""
+        parent = self._build_data_store_parent(location)
+        client_options = self._client_options_discovery_engine(
+            f"projects/{self.project_id}/locations/{location}"
+        )
+        client = discoveryengine_v1alpha.DataStoreServiceClient(
+            credentials=self.creds, client_options=client_options
+        )
+        request = discoveryengine_v1alpha.ListDataStoresRequest(parent=parent)
+        page_result = client.list_data_stores(request=request)
+
+        datastores = []
+        for datastore in page_result:
+            datastores.append(datastore)
+
+        return datastores
+
+    def get_data_store(self, data_store_id: str) -> DataStore:
+        """Get a single Data Store by specified ID."""
+        client_options = self._client_options_discovery_engine(data_store_id)
+        client = discoveryengine_v1alpha.DataStoreServiceClient(
+            credentials=self.creds, client_options=client_options
+        )
+        request = discoveryengine_v1alpha.GetDataStoreRequest(
+            name=data_store_id
+            )
+
+        response = client.get_data_store(request)
+
+        return response
+
+    def create_datastore(
+            self, display_name: str, solution_type: str = "chat",
+            datastore_type: str = "website", advanced_site_search: bool = True,
+            location: str = "global") -> Operation:
+        """Create a new Data Store type specified by the user.
+
+        Args:
+          display_name: The data store display name. This field must be a UTF-8
+            encoded string with a length limit of 128 characters. Otherwise, an
+            INVALID_ARGUMENT error is returned.
+          solution_type: ONEOF [`chat`, `search`, `recommendation`]
+          datastore_type: ONEOF [`website`, `structured`, `unstructured`]
+          advanced_site_search: A boolean flag indicating whether user wants to
+            directly create an advanced data store for site search. If the data
+            store is not configured as site search (GENERIC vertical and
+            PUBLIC_WEBSITE content_config), this flag will be ignored.
+          location: the GCP region to create the Data Store in
+        """
+        parent = self._build_data_store_parent(location)
+        client_options = self._client_options_discovery_engine(
+            f"projects/{self.project_id}/locations/{location}"
+        )
+        client = discoveryengine_v1alpha.DataStoreServiceClient(
+            credentials=self.creds, client_options=client_options
+        )
+        data_store = discoveryengine_v1alpha.DataStore()
+        data_store.display_name = display_name
+        data_store.industry_vertical = 1
+        data_store.solution_types = [self._get_solution_type(solution_type)]
+        data_store.content_config = self._get_content_config(datastore_type)
+
+        request = discoveryengine_v1alpha.CreateDataStoreRequest(
+            parent=parent,
+            data_store=data_store,
+            data_store_id=data_store.display_name,
+            create_advanced_site_search=advanced_site_search
+        )
+
+        operation = client.create_data_store(request=request)
+
+        return operation.operation
+
+    def delete_datastore(self, data_store_id: str):
+        """Delete the specified Data Store by ID."""
+        client_options = self._client_options_discovery_engine(data_store_id)
+        client = discoveryengine_v1alpha.DataStoreServiceClient(
+            credentials=self.creds, client_options=client_options
+        )
+
+        request = discoveryengine_v1alpha.DeleteDataStoreRequest(
+            name=data_store_id
+        )
+        operation = client.delete_data_store(request=request)
+
+        return operation.operation
diff --git a/src/dfcx_scrapi/core/engines.py b/src/dfcx_scrapi/core/engines.py
new file mode 100644
index 00000000..d0ece20d
--- /dev/null
+++ b/src/dfcx_scrapi/core/engines.py
@@ -0,0 +1,188 @@
+"""Engines Resource functions for Vertex Search and Conversation."""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from typing import Dict, List
+from google.longrunning.operations_pb2 import Operation
+from google.cloud import discoveryengine_v1alpha
+from google.cloud.discoveryengine_v1alpha.types import Engine
+
+from dfcx_scrapi.core import scrapi_base
+
+# logging config
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)-8s %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+
+class Engines(scrapi_base.ScrapiBase):
+    """Engines Class for Data Stores in Vertex Search and Conversation."""
+    def __init__(
+        self,
+        project_id: str,
+        creds_path: str = None,
+        creds_dict: Dict = None,
+        creds=None,
+        scope=False,
+    ):
+        super().__init__(
+            creds_path=creds_path,
+            creds_dict=creds_dict,
+            creds=creds,
+            scope=scope,
+        )
+
+        self.project_id = project_id
+
+    def __process_data_store_ids(self, data_store_ids: List[str]) -> List[str]:
+        """Process the data store IDs to ensure they are the correct format."""
+        if not isinstance(data_store_ids, List):
+            raise TypeError(f"Expected a List of Strings, got {data_store_ids}")
+
+        processed_ids = []
+        for ds_id in data_store_ids:
+            processed_ids.append(self._validate_data_store_id(ds_id))
+
+        return processed_ids
+
+    def get_engines_map(
+            self, location: str = "global", reverse: bool = False
+            ) -> Dict[str, str]:
+        """Get a user friendly mapping for Engine Names and IDs."""
+        if reverse:
+            engine_dict = {
+                engine.display_name: engine.name
+                for engine in self.list_engines(location=location)
+            }
+
+        else:
+            engine_dict = {
+                engine.name: engine.display_name
+                for engine in self.list_engines(location=location)
+            }
+
+        return engine_dict
+
+    def build_chat_engine_proto(
+            self, display_name: str, business_name: str,
+            data_store_ids: List[str], existing_df_agent_id: str = None,
+            language_code: str = "en", time_zone: str = "America/Chicago"):
+        """Build the Chat Engine proto for creating a new Engine.
+
+        Args:
+          display_name: the human readable display name of the Chat Engine
+          business_name: the name of the company or business that corresponds
+            most closely with the documents in the Data Store(s).
+          existing_df_agent_id: the Dialogflow Agent ID to associate with the
+            newly created Engine. Only provide this if linking to an existing
+            Dialogflow CX Agent.
+          language_code: the default language of the agent as a language tag
+          time_zone: the time zone of the agent from the time zone database
+
+        Returns:
+          The Engine object configured as a ChatEngine.
+        """
+        data_store_ids = self.__process_data_store_ids(data_store_ids)
+
+        engine = Engine()
+
+        ce_config = Engine.ChatEngineConfig()
+
+        agent_config = Engine.ChatEngineConfig.AgentCreationConfig()
+        agent_config.business = business_name
+        agent_config.default_language_code = language_code
+        agent_config.time_zone = time_zone
+        ce_config.agent_creation_config = agent_config
+
+        if existing_df_agent_id:
+            ce_config.dialogflow_agent_to_link = existing_df_agent_id
+
+        engine.display_name = display_name
+        engine.chat_engine_config = ce_config
+        engine.solution_type = self._get_solution_type("chat")
+        engine.data_store_ids = data_store_ids
+
+        return engine
+
+    def list_engines(
+            self, location: str = "global") -> List[Engine]:
+        """List all Engines for a given project and location."""
+        parent = self._build_data_store_parent(location)
+        client = discoveryengine_v1alpha.EngineServiceClient()
+        request = discoveryengine_v1alpha.ListEnginesRequest(parent=parent)
+        page_result = client.list_engines(request=request)
+
+        engines = []
+        for response in page_result:
+            engines.append(response)
+
+        return engines
+
+    def get_engine(self, engine_id: str) -> Engine:
+        """Get a single Engine by specified ID."""
+        client = discoveryengine_v1alpha.EngineServiceClient()
+        request = discoveryengine_v1alpha.GetEngineRequest(name=engine_id)
+        response = client.get_engine(request=request)
+
+        return response
+
+    def create_engine(self, engine: Engine,
+                      location: str = "global") -> Engine:
+        """Create a new Vertex Search Engine based on user inputs.
+
+        Use the DataStores.build_chat_engine_proto to create the proper Engine
+        payload to pass in for the `engine` arg.
+
+        Args:
+          location, the GCP region to create the Engine in
+          engine, a proto object of type discoveryengine_v1alpha.types.Engine
+            Note that at this time only "Chat" engines are supported
+          solution_type, "chat" is the only value supported at this time.
+        """
+        parent = self._build_data_store_parent(location)
+        client_options = self._client_options_discovery_engine(parent)
+        client = discoveryengine_v1alpha.EngineServiceClient(
+            credentials=self.creds, client_options=client_options
+        )
+
+        request = discoveryengine_v1alpha.CreateEngineRequest(
+            parent=parent,
+            engine=engine,
+            engine_id=engine.display_name
+        )
+
+        operation = client.create_engine(request=request)
+        print("Waiting for operation to complete...")
+        response = operation.result()
+        print(f"Successfully created Engine: {response.display_name}")
+
+        return response
+
+    def delete_engine(self, engine_id: str) -> Operation:
+        """Deletes the specified Engine."""
+        client_options = self._client_options_discovery_engine(engine_id)
+        client = discoveryengine_v1alpha.EngineServiceClient(
+            credentials=self.creds, client_options=client_options
+        )
+
+        request = discoveryengine_v1alpha.DeleteEngineRequest(
+            name=engine_id
+        )
+        operation = client.delete_engine(request=request)
+
+        return operation.operation
diff --git a/src/dfcx_scrapi/core/entity_types.py b/src/dfcx_scrapi/core/entity_types.py
index c805f644..6662ccd5 100644
--- a/src/dfcx_scrapi/core/entity_types.py
+++ b/src/dfcx_scrapi/core/entity_types.py
@@ -421,23 +421,29 @@ def update_entity_type(
         return response
 
     @scrapi_base.api_call_counter_decorator
-    def delete_entity_type(self, entity_id: str = None, obj=None) -> None:
-        """Deletes a single Entity Type resouce object.
+    def delete_entity_type(
+        self, entity_id: str = None,
+        obj: types.EntityType = None, force: bool = False
+    ):
+        """Deletes a single Entity Type resource object.
 
         Args:
-          entity_id: the formatted CX Entity ID to delete
-
-        Returns:
-          None
+          entity_id: The formatted CX Entity ID to delete.
+          obj: (Optional) a CX EntityType object of types.EntityType
+          force: (Optional) If ``force`` is set to true, Dialogflow will remove
+            the entity type, as well as any references to the entity type.
+            (i.e. Page's form Parameter of the entity type will be changed to
+            '@sys.any' and intent's Parameter of the entity type
+            will be removed).
         """
         if not entity_id:
             entity_id = self.entity_id
 
         if obj:
             entity_id = obj.name
-        else:
-            client_options = self._set_region(entity_id)
-            client = services.entity_types.EntityTypesClient(
-                credentials=self.creds, client_options=client_options
-            )
-            client.delete_entity_type(name=entity_id)
+
+        client_options = self._set_region(entity_id)
+        client = services.entity_types.EntityTypesClient(
+            credentials=self.creds, client_options=client_options)
+        req = types.DeleteEntityTypeRequest(name=entity_id, force=force)
+        client.delete_entity_type(request=req)
diff --git a/src/dfcx_scrapi/core/environments.py b/src/dfcx_scrapi/core/environments.py
index 6e516cb2..bc54e3c9 100644
--- a/src/dfcx_scrapi/core/environments.py
+++ b/src/dfcx_scrapi/core/environments.py
@@ -19,7 +19,7 @@
 from google.oauth2 import service_account
 from google.cloud.dialogflowcx_v3beta1 import services
 from google.cloud.dialogflowcx_v3beta1 import types
-from google.protobuf import field_mask_pb2
+
 from dfcx_scrapi.core import scrapi_base
 from dfcx_scrapi.core import flows
 from dfcx_scrapi.core import versions
@@ -306,31 +306,22 @@ def update_environment(
         Returns:
           An object representing a long-running operation. (LRO)
         """
-
         if environment_obj:
             env = environment_obj
-        else:
-            env = types.Environment()
-
-        env.name = environment_id
+            env.name = environment_id
+            mask = self._update_kwargs(env)
+        elif kwargs:
+            env = self.get_environment(environment_id)
+            mask = self._update_kwargs(env, **kwargs)
 
-        # set environment attributes from kwargs
-        for key, value in kwargs.items():
-            setattr(env, key, value)
-        paths = kwargs.keys()
-        mask = field_mask_pb2.FieldMask(paths=paths)
+        request = types.environment.UpdateEnvironmentRequest(
+            environment=env, update_mask=mask)
 
         client_options = self._set_region(environment_id)
         client = services.environments.EnvironmentsClient(
             credentials=self.creds, client_options=client_options
         )
-
-        request = types.environment.UpdateEnvironmentRequest()
-        request.environment = env
-        request.update_mask = mask
-
         response = client.update_environment(request)
-
         return response
 
 
diff --git a/src/dfcx_scrapi/core/flows.py b/src/dfcx_scrapi/core/flows.py
index 8fd7de97..2185c74a 100644
--- a/src/dfcx_scrapi/core/flows.py
+++ b/src/dfcx_scrapi/core/flows.py
@@ -15,11 +15,13 @@
 # limitations under the License.
 
 import logging
+import time
 from typing import Dict, List
 from google.cloud.dialogflowcx_v3beta1 import services
 from google.cloud.dialogflowcx_v3beta1 import types
 from google.protobuf import field_mask_pb2
 from dfcx_scrapi.core import scrapi_base
+from dfcx_scrapi.core import pages
 
 # logging config
 logging.basicConfig(
@@ -54,6 +56,7 @@ def __init__(
             self.flow_id = flow_id
 
         self.agent_id = agent_id
+        self.pages = pages.Pages(creds=self.creds)
 
     # TODO: Migrate to Flow Builder class when ready
     @staticmethod
@@ -104,7 +107,7 @@ def _build_nlu_settings(
 
         return nlu_settings
 
-    def get_flows_map(self, agent_id: str, reverse=False):
+    def get_flows_map(self, agent_id: str = None, reverse=False):
         """Exports Agent Flow Names and UUIDs into a user friendly dict.
 
         Args:
@@ -114,6 +117,8 @@ def get_flows_map(self, agent_id: str, reverse=False):
         Returns:
           Dictionary containing flow UUIDs as keys and display names as values
         """
+        if not agent_id:
+            agent_id = self.agent_id
 
         if reverse:
             flows_dict = {
@@ -129,6 +134,38 @@ def get_flows_map(self, agent_id: str, reverse=False):
 
         return flows_dict
 
+    def get_flow_page_map(
+            self, agent_id: str, rate_limit: float = 1.0
+            ) -> Dict[str, Dict[str, str]]:
+        """Exports a user friendly dict containing Flows, Pages, and IDs
+        This method builds on top of `get_flows_map` and builds out a nested
+        dictionary containing all of the Page Display Names and UUIDs contained
+        within each Flow. Output Format:
+          {
+            <FLOW_DISPLAY_NAME>: {
+                'id': <FLOW_UUID>
+                'pages': { <PAGE_DISPLAY_NAME> : <PAGE_UUID> }
+            }
+          }
+
+        Args:
+          agent_id: the formatted CX Agent ID to use
+
+        Returns:
+          Dictionary containing Flow Names/UUIDs and Page Names/UUIDs
+        """
+        flow_page_map = {}
+
+        flows_map = self.get_flows_map(agent_id, reverse=True)
+
+        for flow in flows_map:
+            pages_map = self.pages.get_pages_map(
+                flows_map[flow], reverse=True)
+            flow_page_map[flow] = {"id": flows_map[flow], "pages": pages_map}
+            time.sleep(rate_limit)
+
+        return flow_page_map
+
     @scrapi_base.api_call_counter_decorator
     def train_flow(self, flow_id: str) -> str:
         """Trains the specified flow.
@@ -157,7 +194,7 @@ def train_flow(self, flow_id: str) -> str:
         return response
 
     @scrapi_base.api_call_counter_decorator
-    def list_flows(self, agent_id: str) -> List[types.Flow]:
+    def list_flows(self, agent_id: str = None) -> List[types.Flow]:
         """Get a List of all Flows in the current Agent.
 
         Args:
@@ -167,6 +204,8 @@ def list_flows(self, agent_id: str) -> List[types.Flow]:
         Returns:
           List of Flow objects
         """
+        if not agent_id:
+            agent_id = self.agent_id
 
         request = types.flow.ListFlowsRequest()
         request.parent = agent_id
@@ -434,23 +473,26 @@ def import_flow(
         return response
 
     @scrapi_base.api_call_counter_decorator
-    def delete_flow(self, flow_id: str, force: bool = False):
+    def delete_flow(
+        self, flow_id: str = None, obj: types.Flow = None, force: bool = False
+    ):
         """Deletes a single CX Flow Object resource.
 
         Args:
-          flow_id: flow to delete
-          force: False means a flow will not be deleted if a route to the flow
-            exists, True means the flow will be deleted as well as all the
-            transition routes leading to the flow.
+          flow_id: The formatted CX Flow ID to delete.
+          obj: (Optional) a CX Flow object of types.Flow
+          force: (Optional) False means a flow will not be deleted if a route
+            to the flow exists, True means the flow will be deleted as well as
+            all the transition routes leading to the flow.
         """
+        if not flow_id:
+            flow_id = self.flow_id
 
-        request = types.DeleteFlowRequest()
-        request.name = flow_id
-        request.force = force
+        if obj:
+            flow_id = obj.name
 
         client_options = self._set_region(flow_id)
         client = services.flows.FlowsClient(
-            credentials=self.creds, client_options=client_options
-        )
-
-        client.delete_flow(request)
+            credentials=self.creds, client_options=client_options)
+        req = types.DeleteFlowRequest(name=flow_id, force=force)
+        client.delete_flow(request=req)
diff --git a/src/dfcx_scrapi/core/generators.py b/src/dfcx_scrapi/core/generators.py
new file mode 100644
index 00000000..5b8c71c2
--- /dev/null
+++ b/src/dfcx_scrapi/core/generators.py
@@ -0,0 +1,260 @@
+"""Generators Resource functions."""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import logging
+from typing import Dict, List, Any
+
+from google.cloud.dialogflowcx_v3beta1 import services
+from google.cloud.dialogflowcx_v3beta1 import types
+from google.protobuf import field_mask_pb2
+
+from dfcx_scrapi.core import scrapi_base
+
+# logging config
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)-8s %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+
+class Generators(scrapi_base.ScrapiBase):
+    """Core Class for CX Generators Resource functions."""
+
+    def __init__(
+        self,
+        creds_path: str = None,
+        creds_dict: Dict = None,
+        creds=None,
+        scope=False,
+        agent_id: str = None,
+    ):
+        super().__init__(
+            creds_path=creds_path,
+            creds_dict=creds_dict,
+            creds=creds,
+            scope=scope,
+        )
+
+        if agent_id:
+            self.agent_id = agent_id
+
+    @staticmethod
+    def __get_placeholders(
+        prompt: str,
+    ) -> List[types.generator.Generator.Placeholder]:
+        """Parse any placeholders from the prompt."""
+        placeholders = []
+        pattern = r"\$(?:\s+)?([a-zA-Z0-9_]+)"
+        matches = re.findall(pattern, prompt)
+        for match in matches:
+            placeholder = types.generator.Generator.Placeholder()
+            placeholder.id = match
+            placeholder.name = match
+            placeholders.append(placeholder)
+
+        return placeholders
+
+    @staticmethod
+    def __input_validation(
+        display_name: str, prompt: str, obj: types.Generator
+    ):
+        """Validate incoming input for Create Generator."""
+        if not display_name or obj:
+            raise ValueError(
+                "At least display_name or obj should be specified."
+            )
+
+        if display_name and not obj and not prompt:
+            raise ValueError("`prompt` value should be specified.")
+
+    @staticmethod
+    def __clean_update_kwargs(
+        kwargs: Dict[str, Any], generator: types.Generator
+    ):
+        """Clean up kwargs before adding to paths for updating the Generator.
+
+        We're providing a few quality of life additions "under the hood" so the
+        user doesn't have to recall all of the specific kwargs for the object.
+        Specifically, they can pass `prompt` instead of `prompt_text` and we'll
+        update appropriately. Additionally we'll detect any placeholders and
+        make these updates as well.
+        """
+        if "prompt" in kwargs:
+            kwargs["prompt_text"] = kwargs["prompt"]
+            del kwargs["prompt"]
+
+        if len(generator.placeholders) > 0:
+            kwargs["placeholders"] = generator.placeholders
+
+        return kwargs
+
+    def get_generators_map(
+        self, agent_id: str, reverse=False
+    ) -> Dict[str, str]:
+        """Put Generator Display Names and UUIDs into a user friendly map."""
+        if reverse:
+            gen_dict = {
+                gen.display_name: gen.name
+                for gen in self.list_generators(agent_id=agent_id)
+            }
+
+        else:
+            gen_dict = {
+                gen.name: gen.display_name
+                for gen in self.list_generators(agent_id=agent_id)
+            }
+
+        return gen_dict
+
+    @scrapi_base.api_call_counter_decorator
+    def list_generators(self, agent_id: str) -> List[types.generator.Generator]:
+        """Retrieves a list of Generators based on the Agent ID."""
+
+        request = types.generator.ListGeneratorsRequest()
+        request.parent = agent_id
+
+        client_options = self._set_region(agent_id)
+        client = services.generators.GeneratorsClient(
+            credentials=self.creds, client_options=client_options
+        )
+
+        response = client.list_generators(request)
+
+        generators = []
+        for page in response.pages:
+            for generator in page.generators:
+                generators.append(generator)
+
+        return generators
+
+    @scrapi_base.api_call_counter_decorator
+    def get_generator(self, generator_id: str) -> types.Generator:
+        """Retrieves a single CX Generator resource object.
+
+        Args:
+          generator_id: The formatted CX Generator ID
+
+        Returns:
+          A single CX Generator resource object
+        """
+
+        request = types.generator.GetGeneratorRequest()
+        request.name = generator_id
+
+        client_options = self._set_region(generator_id)
+        client = services.generators.GeneratorsClient(
+            credentials=self.creds, client_options=client_options
+        )
+
+        response = client.get_generator(request)
+
+        return response
+
+    @scrapi_base.api_call_counter_decorator
+    def create_generator(
+        self,
+        agent_id: str,
+        display_name: str = None,
+        prompt: str = None,
+        obj: types.Generator = None,
+    ):
+        """Create a Dialogflow CX Generator."""
+        self.__input_validation(display_name, prompt, obj)
+
+        if obj:
+            generator_obj = obj
+            generator_obj.name = ""
+
+        else:
+            generator_obj = types.Generator()
+            generator_obj.display_name = display_name
+            generator_obj.prompt_text.text = prompt
+            generator_obj.placeholders = self.__get_placeholders(prompt)
+
+        client_options = self._set_region(agent_id)
+        client = services.generators.GeneratorsClient(
+            credentials=self.creds, client_options=client_options
+        )
+        response = client.create_generator(
+            parent=agent_id, generator=generator_obj
+        )
+
+        return response
+
+    @scrapi_base.api_call_counter_decorator
+    def update_generator(
+        self, generator_id: str, obj: types.Generator = None, **kwargs
+    ) -> types.Generator:
+        """Updates a single Generator object based on provided kwargs.
+
+        Args:
+          generator_id: CX Generator ID string in the following format
+              projects/<PROJECT ID>/locations/<LOCATION ID>/
+                generators/<GENERATOR ID>
+          obj: (Optional) The CX Generator object in proper format. This can
+            also be extracted by using the get_generator() method.
+          kwargs: You may find a list of generator attributes here:
+              https://cloud.google.com/python/docs/reference/dialogflow-cx/
+                  latest/google.cloud.dialogflowcx_v3beta1.types.Generator
+        Returns:
+          The updated CX Generator resource object.
+        """
+
+        if obj:
+            generator = obj
+            generator.name = generator_id
+        else:
+            generator = self.get_generator(generator_id)
+
+        # set generator attributes to args
+        for key, value in kwargs.items():
+            if key in ["prompt", "prompt_text"]:
+                generator.prompt_text.text = value
+                generator.placeholders = self.__get_placeholders(value)
+            else:
+                setattr(generator, key, value)
+
+        kwargs = self.__clean_update_kwargs(kwargs, generator)
+        paths = kwargs.keys()
+
+        mask = field_mask_pb2.FieldMask(paths=paths)
+
+        client_options = self._set_region(generator_id)
+        client = services.generators.GeneratorsClient(
+            credentials=self.creds, client_options=client_options
+        )
+        response = client.update_generator(
+            generator=generator, update_mask=mask
+        )
+
+        return response
+
+    @scrapi_base.api_call_counter_decorator
+    def delete_generator(self, generator_id: str):
+        """Deletes the specified Dialogflow CX Generator.
+
+        Args:
+          generator_id: CX Generator ID string in the following format
+            projects/<PROJECT ID>/locations/<LOCATION ID>/
+              generators/<GENERATOR ID>
+        """
+        client_options = self._set_region(generator_id)
+        client = services.generators.GeneratorsClient(
+            credentials=self.creds, client_options=client_options
+        )
+        client.delete_generator(name=generator_id)
diff --git a/src/dfcx_scrapi/core/intents.py b/src/dfcx_scrapi/core/intents.py
index 74e93701..de1b0cf7 100644
--- a/src/dfcx_scrapi/core/intents.py
+++ b/src/dfcx_scrapi/core/intents.py
@@ -587,20 +587,23 @@ def update_intent(
         return response
 
     @scrapi_base.api_call_counter_decorator
-    def delete_intent(self, intent_id: str, obj: types.Intent = None) -> None:
+    def delete_intent(self, intent_id: str = None, obj: types.Intent = None):
         """Deletes an intent by Intent ID.
 
         Args:
           intent_id: intent to delete
+          obj: (Optional) a CX Intent object of types.Intent
         """
+        if not intent_id:
+            intent_id = self.intent_id
+
         if obj:
             intent_id = obj.name
-        else:
-            client_options = self._set_region(intent_id)
-            client = services.intents.IntentsClient(
-                client_options=client_options, credentials=self.creds
-            )
-            client.delete_intent(name=intent_id)
+
+        client_options = self._set_region(intent_id)
+        client = services.intents.IntentsClient(
+            client_options=client_options, credentials=self.creds)
+        client.delete_intent(name=intent_id)
 
     def bulk_intent_to_df(
         self,
diff --git a/src/dfcx_scrapi/core/pages.py b/src/dfcx_scrapi/core/pages.py
index 8c98ec9f..fb37b10f 100644
--- a/src/dfcx_scrapi/core/pages.py
+++ b/src/dfcx_scrapi/core/pages.py
@@ -238,25 +238,27 @@ def update_page(
         return response
 
     @scrapi_base.api_call_counter_decorator
-    def delete_page(self, page_id: str = None, force: bool = False) -> str:
+    def delete_page(
+        self, page_id: str = None,
+        obj: gcdc_page.Page = None, force: bool = False
+    ):
         """Deletes the specified Page.
 
         Args:
-          page_id: CX Page ID string in the following Format:
-            ``projects/<Project ID>/locations/<Location ID>/agents/<Agent ID>/
-              flows/<Flow ID>/pages/<Page ID>``
+          page_id: The formatted CX Page ID to delete.
+          obj: (Optional) a CX Page object of gcdc_page.Page
           force: (Optional) This field has no effect for pages with no incoming
             transitions. If set to True, Dialogflow will remove the page,
             as well as any transitions to the page.
-
-        Returns:
-          String "Page `{page_id}` successfully deleted."
         """
+        if not page_id:
+            page_id = self.page_id
+
+        if obj:
+            page_id = obj.name
+
         client_options = self._set_region(page_id)
         client = pages.PagesClient(
-            credentials=self.creds, client_options=client_options
-        )
+            credentials=self.creds, client_options=client_options)
         req = gcdc_page.DeletePageRequest(name=page_id, force=force)
         client.delete_page(request=req)
-
-        return f"Page `{page_id}` successfully deleted."
diff --git a/src/dfcx_scrapi/core/scrapi_base.py b/src/dfcx_scrapi/core/scrapi_base.py
index 71399a84..05ee3d40 100644
--- a/src/dfcx_scrapi/core/scrapi_base.py
+++ b/src/dfcx_scrapi/core/scrapi_base.py
@@ -21,9 +21,11 @@
 from collections import defaultdict
 from typing import Dict
 
+from google.cloud.dialogflowcx_v3beta1 import types
 from google.oauth2 import service_account
 from google.auth.transport.requests import Request
 from google.protobuf import json_format  # type: ignore
+from google.protobuf import field_mask_pb2
 
 from proto.marshal.collections import repeated
 from proto.marshal.collections import maps
@@ -111,12 +113,49 @@ def _set_region(resource_id: str):
 
             return client_options
 
+    @staticmethod
+    def _client_options_discovery_engine(resource_id: str):
+        """Different regions have different API endpoints
+
+        Args:
+          resource_id: any type of resource id associated with Discovery Engine
+            resources. Ex: `projects/<GCP PROJECT ID>/locations/<LOCATION ID>`
+
+        Returns:
+          A dictionary containing the api_endpoint and quota project ID to use
+          when calling Discovery Engine API endpoints.
+        """
+        try:
+            location = resource_id.split("/")[3]
+        except IndexError as err:
+            logging.error("Please provide the fully qualified Resource ID: "
+                          "%s", resource_id)
+            raise err
+
+        project_id = resource_id.split("/")[1]
+        base_endpoint = "discoveryengine.googleapis.com:443"
+
+        if location != "global":
+            api_endpoint = f"{location}-{base_endpoint}"
+            client_options = {
+                "api_endpoint": api_endpoint,
+                "quota_project_id": project_id}
+            return client_options
+
+        else:
+            api_endpoint = base_endpoint
+            client_options = {
+                "api_endpoint": api_endpoint,
+                "quota_project_id": project_id}
+
+            return client_options
+
     @staticmethod
     def pbuf_to_dict(pbuf):
         """Extractor of json from a protobuf"""
         blobstr = json_format.MessageToJson(
             pbuf
-        )  # i think this returns JSON as a string
+        )
         blob = json.loads(blobstr)
         return blob
 
@@ -161,11 +200,13 @@ def _parse_resource_path(
             Defaults to True.
         """
 
-        standard_id_match = r"[-0-9a-f]{1,36}"
-        page_id_match = r"[-0-9a-f]{1,36}|START_PAGE|END_SESSION|END_FLOW"
+        data_store_match = r"[\w-]{1,947}"
+        engine_match = r"[a-z0-9][a-z0-9-_]{0,62}"
         entity_id_match = r"[-@.0-9a-z]{1,36}"
         location_id_match = r"[-0-9a-z]{1,36}"
+        page_id_match = r"[-0-9a-f]{1,36}|START_PAGE|END_SESSION|END_FLOW"
         session_id_match = r"[-0-9a-zA-Z!@#$%^&*()_+={}[\]:;\"'<>,.?]{1,36}"
+        standard_id_match = r"[-0-9a-f]{1,36}"
         version_id_match = r"[0-9]{1,4}"
 
         matcher_root = f"^projects/(?P<project>.+?)/locations/(?P<location>{location_id_match})"
@@ -175,6 +216,14 @@ def _parse_resource_path(
                 "matcher": fr"{matcher_root}/agents/(?P<agent>{standard_id_match})$",
                 "format": "`projects/<Project ID>/locations/<Location ID>/agents/<Agent ID>`",
             },
+            "data_store": {
+                "matcher": fr"{matcher_root}/collections/default_collection/dataStores/(?P<data_store>{data_store_match})$",
+                "format": "`projects/<Project ID>/locations/<Location ID>/collections/default_collection/dataStores/<Data Store ID>`"
+            },
+            "engine": {
+                "matcher": fr"{matcher_root}/collections/default_collection/engines/(?P<engine>{engine_match})$",
+                "format": "`projects/<Project ID>/locations/<Location ID>/collections/default_collection/engines/<Engine ID>`"
+            },
             "entity_type": {
                 "matcher": fr"{matcher_root}/agents/(?P<agent>{standard_id_match})/entityTypes/(?P<entity>{entity_id_match})$",
                 "format": "`projects/<Project ID>/locations/<Location ID>/agents/<Agent ID>/entityTypes/<Entity Types ID>`",
@@ -232,10 +281,10 @@ def _parse_resource_path(
         if resource_type not in pattern_map:
             raise KeyError(
                 "`resource_type` must be one of the following resource types:"
-                " `agent`, `entity_type`, `environmnet`, `flow`, `intent`,"
-                "`page`,`project`, `security_setting`, `session`, "
-                "`session_entity_type`,`test_case`, `transition_route_group`, "
-                "`version`, `webhook`"
+                " `agent`, `data_store`, `engine`, `entity_type`, "
+                "`environmnet`, `flow`, `intent`, `page`, `project`, "
+                "`security_setting`, `session`, `session_entity_type`, "
+                "`test_case`, `transition_route_group`, `version`, `webhook`"
             )
 
         match_res = re.match(pattern_map[resource_type]["matcher"], resource_id)
@@ -255,6 +304,42 @@ def _parse_resource_path(
         # pylint: enable=line-too-long
         return dict_res
 
+    @staticmethod
+    def _validate_data_store_id(data_store_id: str):
+        """Validate the data store ID and extract the ID if needed."""
+        pattern = (r"^projects/(?P<project>.+?)/locations/"
+                   r"(?P<location>[-0-9a-z]{1,36})/collections/"
+                   r"default_collection/dataStores/"
+                   r"(?P<data_store>[\w-]{1,947})$"
+                   )
+        match_res = re.match(pattern, data_store_id)
+        if match_res:
+            parts = match_res.groupdict()
+            data_store_id = parts.get("data_store")
+
+        return data_store_id
+
+    @staticmethod
+    def _get_solution_type(solution_type: str) -> int:
+        """Get SOLUTION_TYPE from simple name reference."""
+        solution_map = {
+            "recommendation": 1,
+            "search": 2,
+            "chat": 3,
+        }
+
+        res = solution_map.get(solution_type, None)
+        if not res:
+            raise ValueError("Solution Type must be one of the following values"
+                             ": `chat`, `search`, `recommendation`")
+
+        return solution_map[solution_type]
+
+    def _build_data_store_parent(self, location: str) -> str:
+        """Build the Parent ID needed for Discovery Engine API calls."""
+        return (f"projects/{self.project_id}/locations/{location}/collections/"
+                  "default_collection")
+
     def recurse_proto_repeated_composite(self, repeated_object):
         repeated_list = []
         for item in repeated_object:
@@ -313,6 +398,51 @@ def get_api_calls_count(self) -> int:
         return sum(self.get_api_calls_details().values())
 
 
+    @staticmethod
+    def _update_kwargs(obj, **kwargs) -> field_mask_pb2.FieldMask:
+        """Create a FieldMask for Environment, Experiment, TestCase, Version."""
+        if kwargs:
+            for key, value in kwargs.items():
+                setattr(obj, key, value)
+            return field_mask_pb2.FieldMask(paths=kwargs.keys())
+        attrs_map = {
+            "Environment": [
+                "name", "display_name", "description", "version_configs",
+                "update_time", "test_cases_config", "webhook_config",
+            ],
+            "Experiment": [
+                "name", "display_name", "description", "state", "definition",
+                "rollout_config", "rollout_state", "rollout_failure_reason",
+                "result", "create_time", "start_time", "end_time",
+                "last_update_time", "experiment_length", "variants_history",
+            ],
+            "TestCase": [
+                "name", "tags", "display_name", "notes", "test_config",
+                "test_case_conversation_turns", "creation_time",
+                "last_test_result",
+            ],
+            "Version": [
+                "name", "display_name", "description", "nlu_settings",
+                "create_time", "state",
+            ],
+        }
+        if isinstance(obj, types.Environment):
+            paths = attrs_map["Environment"]
+        elif isinstance(obj, types.Experiment):
+            paths = attrs_map["Experiment"]
+        elif isinstance(obj, types.TestCase):
+            paths = attrs_map["TestCase"]
+        elif isinstance(obj, types.Version):
+            paths = attrs_map["Version"]
+        else:
+            raise ValueError(
+                "`obj` should be one of the following:"
+                " [Environment, Experiment, TestCase, Version]."
+            )
+
+        return field_mask_pb2.FieldMask(paths=paths)
+
+
 def api_call_counter_decorator(func):
     """Counts the number of API calls for the function `func`."""
 
diff --git a/src/dfcx_scrapi/core/search.py b/src/dfcx_scrapi/core/search.py
new file mode 100644
index 00000000..4d955c2a
--- /dev/null
+++ b/src/dfcx_scrapi/core/search.py
@@ -0,0 +1,424 @@
+"""Search methods for Vertex Search and Conversation."""
+
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, Any, List, Union
+from google.cloud.discoveryengine_v1beta import SearchServiceClient
+from google.cloud.discoveryengine_v1beta import types
+from dfcx_scrapi.core import scrapi_base
+
+
+class Search(scrapi_base.ScrapiBase):
+    """Core Class for Search Client functions."""
+
+    def __init__(
+        self,
+        creds_path: str = None,
+        creds_dict: Dict = None,
+        creds=None,
+        scope=False,
+    ):
+        super().__init__(
+            creds_path=creds_path,
+            creds_dict=creds_dict,
+            creds=creds,
+            scope=scope,
+        )
+
+    @staticmethod
+    def build_image_query(
+        search_request: Dict[str, Any]
+    ) -> Union[types.SearchRequest.ImageQuery, None]:
+        image_query = search_request.get("image_query", None)
+        if image_query:
+            image_bytes = image_query.get("image_bytes", None)
+            return types.SearchRequest.ImageQuery(image_bytes=image_bytes)
+
+        else:
+            return None
+
+    @staticmethod
+    def build_user_info(
+        search_request: Dict[str, Any]
+    ) -> Union[types.UserInfo, None]:
+
+        user_info = search_request.get("user_info", None)
+        if user_info:
+            user_id = user_info.get("user_id", None)
+            user_agent = user_info.get("user_agent", None)
+            return types.UserInfo(user_id=user_id, user_agent=user_agent)
+
+        else:
+            return None
+
+    @staticmethod
+    def build_interval(interval_dict: Dict[str, Any]) -> types.Interval:
+        for k, v in interval_dict.items():
+            if k == "minimum":
+                return types.Interval(minimum=v)
+            elif k == "exclusive_minimum":
+                return types.Interval(exclusive_minimum=v)
+            elif k == "maximum":
+                return types.Interval(maximum=v)
+            elif k == "exclusive_maximum":
+                return types.Interval(exclusive_maximum=v)
+            else:
+                return None
+
+    def build_facet_key(
+        self, facet_key_dict: Dict[str, Any]
+    ) -> types.SearchRequest.FacetSpec.FacetKey:
+        intervals_list = facet_key_dict.get("intervals", None)
+        if intervals_list:
+            all_intervals = []
+            for interval in intervals_list:
+                all_intervals.append(self.build_interval(interval))
+
+        return types.SearchRequest.FacetSpec.FacetKey(
+            key=facet_key_dict.get("key", None),
+            intervals=all_intervals,
+            restricted_values=facet_key_dict.get("restricted_values", None),
+            prefixes=facet_key_dict.get("prefixes", None),
+            contains=facet_key_dict.get("contains", None),
+            case_insensitive=facet_key_dict.get("case_insensitive", False),
+            order_by=facet_key_dict.get("order_by", None),
+        )
+
+    def build_single_facet_spec(
+        self, spec: Dict[str, Any]
+    ) -> types.SearchRequest.FacetSpec:
+        facet_key_dict = spec.get("facet_key", None)
+        if not facet_key_dict:
+            raise ValueError(
+                "`facet_key` is required when providing FacetSpec."
+            )
+        facet_key = self.build_facet_key(facet_key_dict)
+
+        return types.SearchRequest.FacetSpec(
+            facet_key=facet_key,
+            limit=spec.get("limit", None),
+            excluded_filter_keys=spec.get("excluded_filter_keys", None),
+            enable_dynamic_position=spec.get("enable_dynamic_position", None),
+        )
+
+    def build_facet_specs(
+        self, search_request: Dict[str, Any]
+    ) -> Union[List[types.SearchRequest.FacetSpec], None]:
+        facet_specs = search_request.get("facet_specs", None)
+        if facet_specs:
+            all_specs = []
+            for spec in facet_specs:
+                all_specs.append(self.build_single_facet_spec(spec))
+
+            return all_specs
+
+        else:
+            return None
+
+    def build_condition_boost_spec(
+        self, spec: Dict[str, Any]
+    ) -> types.SearchRequest.BoostSpec.ConditionBoostSpec:
+        return types.SearchRequest.BoostSpec.ConditionBoostSpec(
+            condition=spec.get("condition", None), boost=spec.get("boost", None)
+        )
+
+    def build_boost_spec(
+        self, search_request: Dict[str, Any]
+    ) -> Union[types.SearchRequest.BoostSpec, None]:
+        boost_spec_dict = search_request.get("boost_spec", None)
+        if boost_spec_dict:
+            condition_boost_specs = boost_spec_dict.get(
+                "condition_boost_specs", None
+            )
+            all_boost_specs = []
+            for spec in condition_boost_specs:
+                all_boost_specs.append(self.build_condition_boost_spec(spec))
+            return types.SearchRequest.BoostSpec(
+                condition_boost_specs=all_boost_specs
+            )
+
+        else:
+            return None
+
+    def get_condition_from_map(
+            self, exp_spec_dict: Dict[str, Any]
+            ) -> types.SearchRequest.QueryExpansionSpec.Condition:
+        condition_map = {
+            "DISABLED": types.SearchRequest.QueryExpansionSpec.Condition.DISABLED, # pylint: disable=C0301
+            "AUTO": types.SearchRequest.QueryExpansionSpec.Condition.AUTO,
+        }
+
+        condition_value = exp_spec_dict.get("condition", "DISABLED")
+
+        return condition_map[condition_value]
+
+    def build_query_expansion_spec(
+        self, search_request: Dict[str, Any]
+    ) -> Union[types.SearchRequest.QueryExpansionSpec, None]:
+        exp_spec_dict = search_request.get("query_expansion_spec", None)
+        if exp_spec_dict:
+            condition = self.get_condition_from_map(exp_spec_dict)
+            pin_unexpanded_results = exp_spec_dict.get(
+                "pin_unexpanded_results", False
+            )
+
+            return types.SearchRequest.QueryExpansionSpec(
+                condition=condition,
+                pin_unexpanded_results=pin_unexpanded_results,
+            )
+
+        else:
+            return None
+
+    def get_spell_correct_mode_from_map(
+        self, spell_spec_dict: Dict[str, Any]
+    ) -> types.SearchRequest.SpellCorrectionSpec.Mode:
+        mode_map = {
+            "SUGGESTION_ONLY": types.SearchRequest.SpellCorrectionSpec.Mode.SUGGESTION_ONLY, # pylint: disable=C0301
+            "AUTO": types.SearchRequest.SpellCorrectionSpec.Mode.AUTO,
+        }
+
+        mode_value = spell_spec_dict.get("mode", "AUTO")
+
+        return mode_map[mode_value]
+
+    def build_spell_correction_spec(
+        self, search_request: Dict[str, Any]
+    ) -> Union[types.SearchRequest.SpellCorrectionSpec, None]:
+        spell_spec_dict = search_request.get("spell_correction_spec", None)
+        if spell_spec_dict:
+            mode = self.get_spell_correct_mode_from_map(spell_spec_dict)
+            return types.SearchRequest.SpellCorrectionSpec(mode=mode)
+
+        else:
+            return None
+
+    def build_model_prompt_spec(
+        self, content_spec_dict: Dict[str, Any]
+    ) -> types.SearchRequest.ContentSearchSpec.SummarySpec.ModelPromptSpec:
+        model_prompt_spec_dict = content_spec_dict.get(
+            "model_prompt_spec", None
+        )
+        if model_prompt_spec_dict:
+            return types.SearchRequest.ContentSearchSpec.SummarySpec.ModelPromptSpec( # pylint: disable=C0301
+                preamble=model_prompt_spec_dict.get("preamble", None)
+            )
+
+        else:
+            return None
+
+    def build_model_spec(
+        self, content_spec_dict: Dict[str, Any]
+    ) -> types.SearchRequest.ContentSearchSpec.SummarySpec.ModelSpec:
+        model_spec_dict = content_spec_dict.get("model_spec", None)
+        if model_spec_dict:
+            return types.SearchRequest.ContentSearchSpec.SummarySpec.ModelSpec(
+                version=model_spec_dict.get("version", "stable")
+            )
+
+        else:
+            return None
+
+    def build_snippet_spec(
+            self) -> types.SearchRequest.ContentSearchSpec.SnippetSpec:
+        return types.SearchRequest.ContentSearchSpec.SnippetSpec(
+            return_snippet=True
+        )
+
+    def build_summary_spec(
+        self, content_spec_dict: Dict[str, Any]
+    ) -> types.SearchRequest.ContentSearchSpec.SummarySpec:
+
+        model_prompt_spec = self.build_model_prompt_spec(content_spec_dict)
+        model_spec = self.build_model_spec(content_spec_dict)
+
+        return types.SearchRequest.ContentSearchSpec.SummarySpec(
+            summary_result_count=content_spec_dict.get(
+                "summary_result_count", 10
+            ),
+            include_citations=content_spec_dict.get("include_citations", False),
+            ignore_adversarial_query=content_spec_dict.get(
+                "ignore_adversarial_query", False
+            ),
+            ignore_non_summary_seeking_query=content_spec_dict.get(
+                "ignore_non_summary_seeking_query", False
+            ),
+            model_prompt_spec=model_prompt_spec,
+            language_code=content_spec_dict.get("language_code", "en"),
+            model_spec=model_spec,
+        )
+
+    def build_extractive_content_spec(
+        self, content_spec_dict: Dict[str, Any]
+    ) -> Union[
+        types.SearchRequest.ContentSearchSpec.ExtractiveContentSpec, None
+    ]:
+
+        ext_spec_dict = content_spec_dict.get("extractive_content_spec", None)
+        if ext_spec_dict:
+            return types.SearchRequest.ContentSearchSpec.ExtractiveContentSpec(
+                max_extractive_answer_count=ext_spec_dict.get(
+                    "max_extractive_answer_count", 5
+                ),
+                max_extractive_segment_count=ext_spec_dict.get(
+                    "max_extractive_segment_count", 5
+                ),
+                return_extractive_segment_score=ext_spec_dict.get(
+                    "return_extractive_segment_score", False
+                ),
+                num_previous_segments=ext_spec_dict.get(
+                    "num_previous_segments", 5
+                ),
+                num_next_segments=ext_spec_dict.get("num_next_segments", 5),
+            )
+        else:
+            return None
+
+    def build_content_search_spec(
+        self, search_request: Dict[str, Any]
+    ) -> Union[types.SearchRequest.ContentSearchSpec, None]:
+        content_spec_dict = search_request.get("content_search_spec", None)
+        if content_spec_dict:
+            snippet_spec = self.build_snippet_spec()
+            summary_spec = self.build_summary_spec(content_spec_dict)
+            extractive_content_spec = self.build_extractive_content_spec(
+                content_spec_dict
+            )
+
+            return types.SearchRequest.ContentSearchSpec(
+                snippet_spec=snippet_spec,
+                summary_spect=summary_spec,
+                extractive_content_spec=extractive_content_spec,
+            )
+
+        else:
+            return None
+
+    def build_embedding_vector(
+        self, vector_dict: Dict[str, Any]
+    ) -> types.SearchRequest.EmbeddingSpec.EmbeddingVector:
+        return types.SearchRequest.EmbeddingSpec.EmbeddingVector(
+            field_path=vector_dict.get("field_path", None),
+            vector=vector_dict.get("vector", None),
+        )
+
+    def build_embedding_spec(
+        self, search_request: Dict[str, Any]
+    ) -> Union[types.SearchRequest.EmbeddingSpec, None]:
+        embedding_vectors_dict = search_request.get("embedding_vectors", None)
+        if embedding_vectors_dict:
+            vector_list = embedding_vectors_dict.get("embedding_vectors", None)
+            all_vectors = []
+            for vector_dict in vector_list:
+                all_vectors.append(self.build_embedding_vector(vector_dict))
+            return types.SearchRequest.EmbeddingSpec(
+                embedding_vectors=all_vectors
+            )
+
+        else:
+            return None
+
+	# pylint: disable=C0301
+    def search(self, search_config: Dict[str, Any], total_results: int = 10):
+        """Performs a search against an indexed Vertex Data Store.
+
+        Args:
+        	search_config: A dictionary containing keys that correspond to the
+            	SearchRequest attributes as defined in:
+                https://cloud.google.com/python/docs/reference/discoveryengine/latest/google.cloud.discoveryengine_v1beta.types.SearchRequest
+                For complex attributes that require nested fields, you can pass
+                in another Dictionary as the value.
+
+				Example: To represent the complex facet_specs config with some
+				other simple parameters, you would do the following.
+
+				```py
+				search_config = {
+				"facet_specs": [
+						{
+						"facet_key": {
+								"key": "my_key",
+								"intervals": [
+								{
+										"minimum": .5
+									},
+									{
+										"maximum": .95
+									}
+								],
+								"case_insensitive": True
+							},
+							"limit": 10
+						}
+					],
+					"page_size": 10,
+					"offset": 2
+				}
+			total_results: Total number of results to return for the search. If
+				not specified, will default to 10 results. Increasing this to a
+				high number can result in long search times.
+
+        Returns:
+                A List of SearchResponse objects.
+        """
+        serving_config = (
+            f"{search_config.get('data_store_id', None)}"
+            "/servingConfigs/default_serving_config"
+        )
+
+        branch_stub = "/".join(serving_config.split("/")[0:8])
+        branch = branch_stub + "/branches/0"
+
+        request = types.SearchRequest(
+            serving_config=serving_config,
+            branch=branch,
+            query=search_config.get("query", None),
+            image_query=self.build_image_query(search_config),
+            page_size=search_config.get("page_size", 10),
+            page_token=search_config.get("page_token", None),
+            offset=search_config.get("offset", 0),
+            filter=search_config.get("filter", None),
+            canonical_filter=search_config.get("canonical_filter", None),
+            order_by=search_config.get("order_by", None),
+            user_info=self.build_user_info(search_config),
+            facet_specs=self.build_facet_specs(search_config),
+            boost_spec=self.build_boost_spec(search_config),
+            params=search_config.get("params", None),
+            query_expansion_spec=self.build_query_expansion_spec(search_config),
+            spell_correction_spec=self.build_spell_correction_spec(
+                search_config
+            ),
+            user_pseudo_id=search_config.get("user_pseudo_id", None),
+            content_search_spec=self.build_content_search_spec(search_config),
+            embedding_spec=self.build_embedding_spec(search_config),
+            ranking_expression=search_config.get("ranking_expression", None),
+            safe_search=search_config.get("safe_search", False),
+            user_labels=search_config.get("user_labels", None),
+        )
+
+        client_options = self._client_options_discovery_engine(serving_config)
+        client = SearchServiceClient(
+            credentials=self.creds, client_options=client_options
+        )
+        response = client.search(request)
+
+        all_results = []
+        for search_result in response:
+            if len(all_results) < total_results:
+                all_results.append(search_result)
+            else:
+                break
+
+        return all_results
diff --git a/src/dfcx_scrapi/core/sessions.py b/src/dfcx_scrapi/core/sessions.py
index c6e0a665..7340b381 100644
--- a/src/dfcx_scrapi/core/sessions.py
+++ b/src/dfcx_scrapi/core/sessions.py
@@ -19,6 +19,7 @@
 from typing import Dict, List
 from google.cloud.dialogflowcx_v3beta1 import services
 from google.cloud.dialogflowcx_v3beta1 import types
+from google.protobuf.json_format import MessageToDict
 
 from dfcx_scrapi.core import scrapi_base
 
@@ -192,7 +193,8 @@ def detect_intent(
         session_id,
         text,
         language_code="en",
-        parameters=None):
+        parameters=None,
+        populate_data_store_connection_signals=False):
         """Returns the result of detect intent with texts as inputs.
 
         Using the same `session_id` between requests allows continuation
@@ -206,6 +208,10 @@ def detect_intent(
           text: the user utterance to run intent detection on
           parameters: (Optional) Dict of CX Session Parameters to set in the
             conversation. Typically this is set before a conversation starts.
+          populate_data_store_connection_signals: If set to true and data
+            stores are involved in serving the request then query result will
+            be populated with data_store_connection_signals field which
+            contains data that can help evaluations.
 
         Returns:
           The CX query result from intent detection
@@ -225,24 +231,25 @@ def detect_intent(
 
         logging.info(f"Starting Session ID {session_id}")
 
-        if parameters:
-            query_params = types.session.QueryParameters(parameters=parameters)
+        query_input = self._build_query_input(text, language_code)
 
-            query_input = self._build_query_input(text, language_code)
+        request = types.session.DetectIntentRequest()
+        request.session = session_id
+        request.query_input = query_input
 
-            request = types.session.DetectIntentRequest()
-            request.session = session_id
-            request.query_input = query_input
-            request.query_params = query_params
+        query_param_mapping = {}
 
-            response = session_client.detect_intent(request=request)
+        if parameters:
+            query_param_mapping["parameters"] = parameters
 
-        else:
-            query_input = self._build_query_input(text, language_code)
+        if populate_data_store_connection_signals:
+            query_param_mapping["populate_data_store_connection_signals"] = (
+                populate_data_store_connection_signals
+            )
 
-            request = types.session.DetectIntentRequest()
-            request.session = session_id
-            request.query_input = query_input
+        if query_param_mapping:
+            query_params = types.session.QueryParameters(query_param_mapping)
+            request.query_params = query_params
 
         response = session_client.detect_intent(request)
         query_result = response.query_result
@@ -285,3 +292,17 @@ def preset_parameters(
         response = session_client.detect_intent(request=request)
 
         return response
+
+    def get_agent_answer(self, user_query: str) -> str:
+        """Extract the answer/citation from a Vertex Conversation response."""
+
+        session_id = self.build_session_id(self.agent_id)
+        res = MessageToDict(self.detect_intent( # pylint: disable=W0212
+            self.agent_id, session_id, user_query)._pb)
+
+        answer_text = res["responseMessages"][0]["text"]["text"][0]
+        answer_link = res["responseMessages"][1]["payload"][
+            "richContent"][0][0]["actionLink"] if len(
+                res["responseMessages"]) > 1 else ""
+
+        return f"{answer_text} ({answer_link})"
diff --git a/src/dfcx_scrapi/core/sites.py b/src/dfcx_scrapi/core/sites.py
new file mode 100644
index 00000000..50836565
--- /dev/null
+++ b/src/dfcx_scrapi/core/sites.py
@@ -0,0 +1,235 @@
+"""Site Search Service functions for Vertex Search and Conversation."""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from typing import Dict, List
+from google.longrunning.operations_pb2 import Operation
+from google.cloud import discoveryengine_v1alpha
+from google.cloud.discoveryengine_v1alpha.types import TargetSite
+
+from dfcx_scrapi.core import scrapi_base
+
+# logging config
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)-8s %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+
+class Sites(scrapi_base.ScrapiBase):
+    """Site Search Class for Data Stores in Vertex Search and Conversation."""
+    def __init__(
+        self,
+        project_id: str,
+        creds_path: str = None,
+        creds_dict: Dict = None,
+        creds=None,
+        scope=False,
+    ):
+        super().__init__(
+            creds_path=creds_path,
+            creds_dict=creds_dict,
+            creds=creds,
+            scope=scope,
+        )
+
+        self.project_id = project_id
+
+    @staticmethod
+    def __get_site_type(include_site: bool) -> TargetSite.Type:
+        """Get the Site Type mapping."""
+        site_type_map = {True: 1, False: 2}
+
+        return site_type_map[include_site]
+
+    def __build_site_search_parent(self, data_store_id: str) -> str:
+        """Build the Parent ID needed for Site Search API calls."""
+        parts = self._parse_resource_path("data_store", data_store_id)
+        location = parts.get("location")
+        data_store_id_short = parts.get("data_store")
+
+        return (f"{self._build_data_store_parent(location)}/dataStores/"
+                f"{data_store_id_short}/siteSearchEngine")
+
+    def get_sites_map(
+            self, data_store_id: str, reverse: bool = False) -> Dict[str, str]:
+        """Get a user friendly mapping for Site Names and IDs."""
+        if reverse:
+            site_dict = {
+                site.generated_uri_pattern: site.name
+                for site in self.list_sites(data_store_id=data_store_id)
+            }
+
+        else:
+            site_dict = {
+                site.name: site.generated_uri_pattern
+                for site in self.list_sites(data_store_id=data_store_id)
+            }
+
+        return site_dict
+
+    def list_sites(self, data_store_id: str) -> List[TargetSite]:
+        """List all URL patterns for a given Data Store ID."""
+        client_options = self._client_options_discovery_engine(data_store_id)
+        client = discoveryengine_v1alpha.SiteSearchEngineServiceClient(
+            credentials=self.creds, client_options=client_options
+        )
+
+        parent = self.__build_site_search_parent(data_store_id)
+        request = discoveryengine_v1alpha.ListTargetSitesRequest(
+            parent=parent,
+            page_size=1000
+        )
+
+        page_result = client.list_target_sites(request=request)
+
+        sites = []
+        for response in page_result:
+            sites.append(response)
+
+        return sites
+
+    def get_site(self, site_id: str) -> TargetSite:
+        """Get a single Site by specified ID."""
+        client_options = self._client_options_discovery_engine(site_id)
+        client = discoveryengine_v1alpha.SiteSearchEngineServiceClient(
+            credentials=self.creds, client_options=client_options
+        )
+        request = discoveryengine_v1alpha.GetTargetSiteRequest(name=site_id)
+        response = client.get_target_site(request=request)
+
+        return response
+
+    def create_site(self, data_store_id: str, uri_pattern: str,
+                    include_site: bool = True, exact_match: bool = False
+                    ) -> Operation:
+        """Create a new Target Site in the specified Data Store.
+
+        Args:
+          data_store_id: the fully qualified Data Store ID
+          uri_pattern: The user provided URI pattern from which the
+            generated_uri_pattern is generated.
+          include_site: If set to True, the uri pattern will be Included. If set
+            to False, the site pattern will be Excluded.
+          exact_match: If set to false, a uri_pattern is generated to include
+            all pages whose address contains the provided_uri_pattern. If set to
+            true, an uri_pattern is generated to try to be an exact match of the
+            provided_uri_pattern or just the specific page if the
+            provided_uri_pattern is a specific one. provided_uri_pattern is
+            always normalized to generate the URI pattern to be used by the
+            search engine.
+        """
+        client_options = self._client_options_discovery_engine(data_store_id)
+        client = discoveryengine_v1alpha.SiteSearchEngineServiceClient(
+            credentials=self.creds, client_options=client_options
+        )
+
+        target_site = discoveryengine_v1alpha.TargetSite()
+        target_site.provided_uri_pattern = uri_pattern
+        target_site.type_ = self.__get_site_type(include_site)
+        target_site.exact_match = exact_match
+
+        parent = self.__build_site_search_parent(data_store_id)
+        request = discoveryengine_v1alpha.CreateTargetSiteRequest(
+            parent=parent,
+            target_site=target_site
+            )
+
+        operation = client.create_target_site(request=request)
+
+        return operation
+
+    def delete_site(self, site_id: str) -> Operation:
+        """Deletes a TargetSite in a Data Store by the specified ID."""
+        client_options = self._client_options_discovery_engine(site_id)
+        client = discoveryengine_v1alpha.SiteSearchEngineServiceClient(
+            credentials=self.creds, client_options=client_options
+        )
+
+        request = discoveryengine_v1alpha.DeleteTargetSiteRequest(name=site_id)
+        operation = client.delete_target_site(request=request)
+
+        return operation
+
+    def enable_advanced_site_search(self, data_store_id: str) -> Operation:
+        """Enables Advanced Site Search for the provided Data Store ID."""
+        client_options = self._client_options_discovery_engine(data_store_id)
+        client = discoveryengine_v1alpha.SiteSearchEngineServiceClient(
+            credentials=self.creds, client_options=client_options
+        )
+
+        parent = self.__build_site_search_parent(data_store_id)
+        request = discoveryengine_v1alpha.EnableAdvancedSiteSearchRequest(
+            site_search_engine=parent
+            )
+
+        operation = client.enable_advanced_site_search(request=request)
+
+        return operation
+
+    def disable_advanced_site_search(self, data_store_id: str) -> Operation:
+        """Disable Advanced Site Search for the provided Data Store ID."""
+        client_options = self._client_options_discovery_engine(data_store_id)
+        client = discoveryengine_v1alpha.SiteSearchEngineServiceClient(
+            credentials=self.creds, client_options=client_options
+        )
+
+        parent = self.__build_site_search_parent(data_store_id)
+        request = discoveryengine_v1alpha.DisableAdvancedSiteSearchRequest(
+            site_search_engine=parent
+            )
+
+        operation = client.disable_advanced_site_search(request=request)
+
+        return operation
+
+    def recrawl_uris(self, data_store_id: str, uris: List[str]) -> Operation:
+        """Recrawl the specified set of URIs for the Data Store."""
+        client_options = self._client_options_discovery_engine(data_store_id)
+        client = discoveryengine_v1alpha.SiteSearchEngineServiceClient(
+            credentials=self.creds, client_options=client_options
+        )
+
+        parent = self.__build_site_search_parent(data_store_id)
+        request = discoveryengine_v1alpha.RecrawlUrisRequest(
+            site_search_engine=parent, uris=uris
+            )
+
+        operation = client.recrawl_uris(request=request)
+
+        return operation
+
+    def get_site_index_status(self, site_id: str) -> str:
+        """Get the Site Indexing Status f0r the provided Site ID."""
+        site = self.get_site(site_id)
+
+        return site.indexing_status
+
+    def get_site_verification_status(self, site_id: str) -> str:
+        """Get the Site Verification Status for the provided Site ID."""
+        site = self.get_site(site_id)
+        state = site.site_verification_info.site_verification_state
+
+        verify_map = {
+            1: "VERIFIED: Site ownership verified",
+            2: ("UNVERIFIED: Site ownership pending verification or"
+                " verification failed"),
+            3: ("EXEMPTED: Site exempt from verification, e.g. a public website"
+                " that opens to all.")
+        }
+
+        return verify_map[state.value]
diff --git a/src/dfcx_scrapi/core/transition_route_groups.py b/src/dfcx_scrapi/core/transition_route_groups.py
index 8c2c8984..29f0bab6 100644
--- a/src/dfcx_scrapi/core/transition_route_groups.py
+++ b/src/dfcx_scrapi/core/transition_route_groups.py
@@ -72,6 +72,8 @@ def __init__(
         if agent_id:
             self.agent_id = agent_id
 
+        self._get_agent_level_data_only = False
+
     def _rg_temp_dict_update(self, temp_dict, element):
         """Modify the temp dict and return to dataframe function."""
         element_dict = self.cx_object_to_dict(element)
@@ -275,13 +277,40 @@ def update_transition_route_group(
 
         return response
 
+    @scrapi_base.api_call_counter_decorator
+    def delete_transition_route_group(
+        self, route_group_id: str = None,
+        obj: types.TransitionRouteGroup = None, force: bool = False
+    ):
+        """Deletes the specified Route Group.
+
+        Args:
+          route_group_id: The formatted CX Route Group ID to delete.
+          obj: (Optional) a CX Webhook object of types.TransitionRouteGroup
+          force: (Optional) This field has no effect for transition route group
+            that no page is using. If set to True, Dialogflow will remove
+            the transition route group, as well as any transitions to the page.
+        """
+        if not route_group_id:
+            route_group_id = self.route_group_id
+        if obj:
+            route_group_id = obj.name
+
+        client_options = self._set_region(route_group_id)
+        client = services.transition_route_groups.TransitionRouteGroupsClient(
+            credentials=self.creds, client_options=client_options)
+        req = types.DeleteTransitionRouteGroupRequest(
+            name=route_group_id, force=force)
+        client.delete_transition_route_group(request=req)
+
+
     def route_groups_to_dataframe(
         self, agent_id: str = None, rate_limit: float = 0.5
     ):
-        """Extracts the Transition Route Groups from a given Agent and
+        """Extracts the Flow Transition Route Groups from a given Agent and
          returns key information about the Route Groups in a Pandas Dataframe
 
-        DFCX Route Groups exist as an Agent level resource, however they are
+        DFCX Route Groups exist as an Agent level resource, however they can
         categorized by the Flow they are associated with. This method will
         extract all Flows for the given agent, then use the Flow IDs to
         extract all Route Groups per Flow. Once all Route Groups have been
@@ -309,18 +338,23 @@ def route_groups_to_dataframe(
         all_pages_map = {}
         all_rgs = []
 
-        for flow in flows_map:
-            all_pages_map.update(self.pages.get_pages_map(flow))
-            all_rgs.extend(self.list_transition_route_groups(flow))
-            time.sleep(rate_limit)
+        if self._get_agent_level_data_only:
+            all_rgs.extend(self.list_transition_route_groups(agent_id))
+        else:
+            for flow in flows_map:
+                all_pages_map.update(self.pages.get_pages_map(flow))
+                all_rgs.extend(self.list_transition_route_groups(flow))
+                time.sleep(rate_limit)
 
         rows_list = []
         for route_group in all_rgs:
-            flow = "/".join(route_group.name.split("/")[0:8])
+            if not self._get_agent_level_data_only:
+                flow = "/".join(route_group.name.split("/")[0:8])
             for route in route_group.transition_routes:
                 temp_dict = {}
 
-                temp_dict.update({"flow": flows_map[flow]})
+                if not self._get_agent_level_data_only:
+                    temp_dict.update({"flow": flows_map[flow]})
                 temp_dict.update({"route_group_name": route_group.display_name})
 
                 if route.target_page:
@@ -356,7 +390,39 @@ def route_groups_to_dataframe(
                         )
 
                 rows_list.append(temp_dict)
+        final_dataframe = pd.DataFrame(
+            rows_list,
+            columns=[
+                "flow", "route_group_name", "target_page", "intent",
+                "condition", "webhook", "webhook_tag", "custom_payload",
+                "live_agent_handoff","conversation_success", "play_audio",
+                "output_audio_text", "fulfillment_message"]
+        )
+
+        return final_dataframe
+
+    def agent_route_groups_to_dataframe(
+            self, agent_id: str = None, rate_limit: float = 0.5
+    ):
+        """Extracts the Transition Route Groups from a given Agent and
+        returns key information about the Route Groups in a Pandas Dataframe
 
-        final_dataframe = pd.DataFrame(rows_list)
+        This method will extract all Agent Level Route Groups and convert
+        the DFCX object to a Pandas Dataframe and return this to the user.
 
+        Args:
+        agent_id: the Agent ID string in the following format:
+            projects/<project_id>/locations/<location_id>/agents/<agent_id>
+        rate_limit: Time in seconds to wait between each API call. Use this
+            to control hitting Quota limits on your project.
+
+        Returns:
+        a Pandas Dataframe with columns: route_group_name, target_page,
+        intent, condition, webhook, webhook_tag, custom_payload,
+        live_agent_handoff, conversation_success, play_audio,
+        output_audio_text, fulfillment_message
+        """
+        self._get_agent_level_data_only = True
+        final_dataframe = self.route_groups_to_dataframe(agent_id, rate_limit)
+        self._get_agent_level_data_only = False
         return final_dataframe
diff --git a/src/dfcx_scrapi/core/webhooks.py b/src/dfcx_scrapi/core/webhooks.py
index 3a10b4c2..f54085b0 100644
--- a/src/dfcx_scrapi/core/webhooks.py
+++ b/src/dfcx_scrapi/core/webhooks.py
@@ -246,3 +246,35 @@ def update_webhook(
         response = client.update_webhook(request)
 
         return response
+
+
+    @scrapi_base.api_call_counter_decorator
+    def delete_webhook(
+        self, webhook_id: str = None,
+        obj: types.Webhook = None, force: bool = False
+    ):
+        """Deletes a single Webhookd resource object.
+
+        Args:
+          webhook_id: intent to delete
+          obj: (Optional) a CX Webhook object of types.Webhook
+          force: (Optional) This field has no effect for webhook not being
+            used. For webhooks that are used by pages/flows/transition route
+            groups:
+            -  If ``force`` is set to false, an error will be returned
+               with message indicating the referenced resources.
+            -  If ``force`` is set to true, Dialogflow will remove the
+               webhook, as well as any references to the webhook and tags in
+               fulfillments that point to this webhook will be removed.
+        """
+        if not webhook_id:
+            webhook_id = self.webhook_id
+
+        if obj:
+            webhook_id = obj.name
+
+        client_options = self._set_region(webhook_id)
+        client = services.webhooks.WebhooksClient(
+            client_options=client_options, credentials=self.creds)
+        req = types.DeleteWebhookRequest(name=webhook_id, force=force)
+        client.delete_webhook(request=req)
diff --git a/src/dfcx_scrapi/tools/agent_checker_util.py b/src/dfcx_scrapi/tools/agent_checker_util.py
new file mode 100644
index 00000000..ff6e331a
--- /dev/null
+++ b/src/dfcx_scrapi/tools/agent_checker_util.py
@@ -0,0 +1,203 @@
+"""A set of Utility methods to check resources stats on DFCX Agents."""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import time
+from typing import Dict, List
+import pandas as pd
+
+from dfcx_scrapi.core import scrapi_base
+from dfcx_scrapi.agent_extract import agents
+
+# logging config
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)-8s %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+class AgentCheckerUtil(scrapi_base.ScrapiBase):
+    """Utility class for checking DFCX Agents."""
+
+    def __init__(
+        self,
+        agent_id: str,
+        gcs_bucket_uri: str,
+        creds_path: str = None,
+        creds_dict: Dict = None,
+        creds=None,
+        scope=False,
+    ):
+        super().__init__(
+            creds_path=creds_path,
+            creds_dict=creds_dict,
+            creds=creds,
+            scope=scope,
+        )
+        self.agent_id = agent_id
+        self.special_pages = [
+            "End Session",
+            "End Flow",
+            "Start Page",
+            "Current Page",
+            "Previous Page",
+        ]
+
+        startup_time = time.time()
+        self.extract = agents.Agents(agent_id)
+        processing_time = time.time()
+        logging.debug(f"STARTUP: {processing_time - startup_time}")
+
+        self.data = self.extract.process_agent(agent_id, gcs_bucket_uri)
+        logging.debug(f"TOTAL PROCESSING: {time.time() - processing_time}")
+
+        self.active_intents_df = self.active_intents_to_dataframe()
+
+    def _filter_special_pages(self, page: str, filter_special_pages: bool):
+        """Recursion helper to check for special page match."""
+        if filter_special_pages and page in self.special_pages:
+            return True
+
+        return False
+
+    def _recurse_edges(self, edges: Dict[str, List[str]], page: str,
+                      visited: set, depth: int, max_depth: int,
+                      filter_special_pages: bool):
+        """Recursion method used to traverse the agent graph for page data.
+
+        Args:
+          edges: The set of graph edges collected from the agent.
+          page: The current Page Display Name
+          visited: A set of visited Page nodes
+          depth: The current recursion depth
+          max_depth: The max recursion depth
+          filter_special_pages: Will discard all self.special_pages from output
+            if set to False.
+        """
+        if depth == max_depth:
+            return visited
+
+        if page in edges:
+            for inner_page in edges[page]:
+                if self._filter_special_pages(inner_page, filter_special_pages):
+                    return visited
+
+                if inner_page not in visited:
+                    visited.add(inner_page)
+                    visited = self._recurse_edges(
+                        edges, inner_page, visited, depth+1, max_depth,
+                        filter_special_pages)
+
+        return visited
+
+    def _mark_unreachable_pages(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Mark dataframe rows True if the page is unreachable in graph."""
+        for idx, row in df.iterrows():
+            for page in self.data.unreachable_pages[row["flow"]]:
+                if row["page"] == page:
+                    df.loc[idx, "unreachable"] = True
+
+        return df
+
+    def get_reachable_pages(
+            self,
+            flow_display_name: str,
+            page_display_name: str = "Start Page",
+            max_depth: int = 1,
+            filter_special_pages = True) -> List[str]:
+        """Get all pages in the graph that are reachable via transition routes,
+        starting from a given Flow and Page.
+
+        Args:
+          flow_display_name: The display name of the flow
+          page_display_name: The display name of the page. Defaults to
+            "Start Page"
+          max_depth: The max recursion depth to search the graph from the
+            provided starting point. For example, a max_depth of 2 would produce
+            all reachable Pages that are 2 transition routes away from the
+            starting Flow/Page. Defaults to 1.
+          filter_special_pages: Will filter out all self.special_pages. Defaults
+            to True.
+          """
+        if page_display_name in ["START", "START_PAGE", "Start", "Start Page"]:
+            page_display_name = "Start Page"
+            page_display_name = f"{flow_display_name}: {page_display_name}"
+
+        visited = self._recurse_edges(
+            self.data.graph.edges, page_display_name, set(), 0, max_depth,
+            filter_special_pages)
+
+        return list(visited)
+
+    def active_intents_to_dataframe(self) -> pd.DataFrame:
+        """Gets all intents referenced in the agent, across all flows and pages,
+        and produces a dataframe listing which flows/pages reference each
+        intent.
+
+        Returns:
+            A dataframe with columns
+            intent - the intent display name
+            flow - the Flow Display Name where the intent resides
+            page - the Page Display Name where the intent resides
+            unreachable - Denotes whether the Flow/Page/Intent combination is
+              unreachable in the graph.
+        """
+        df = pd.DataFrame({
+            "intent": pd.Series(dtype="str"),
+            "flow": pd.Series(dtype="str"),
+            "page": pd.Series(dtype="str"),
+            "unreachable": pd.Series(dtype="bool")
+            })
+
+        # Loop over active_intents, create temp dataframe, then concat with the
+        # main dataframe to build out the complete Flow/Page/Intent dataset.
+        for flow in self.data.active_intents:
+            for pair in self.data.active_intents[flow]:
+                intent = pair[0]
+                page = pair[1]
+                temp = pd.DataFrame({
+                    "intent": [intent],
+                    "flow": [flow],
+                    "page": [page],
+                    "unreachable": [False]})
+                df = pd.concat([df, temp])
+
+        df = df.reset_index(drop=True)
+
+        # Finally, determine what rows are unreachable.
+        self.active_intents_df = self._mark_unreachable_pages(df)
+
+        return self.active_intents_df
+
+    def get_unused_intents(self) -> List:
+        """Get all unused Intents across the agent."""
+        if self.active_intents_df.empty:
+            self.active_intents_df = self.active_intents_to_dataframe()
+        active_intents_set = set(self.active_intents_df.intent.to_list())
+        all_intents_set = set(self.data.intents_map.keys())
+
+        return list(all_intents_set.difference(active_intents_set))
+
+    def get_unreachable_intents(self) -> pd.DataFrame:
+        """Get all unreachable Intents across the agent.
+
+        An Intent is unreachable if it resides on a page that is also
+        unreachable.
+        """
+        if self.active_intents_df.empty:
+            self.active_intents_df = self.active_intents_to_dataframe()
+
+        return self.active_intents_df[self.active_intents_df["unreachable"]]
diff --git a/src/dfcx_scrapi/tools/copy_util.py b/src/dfcx_scrapi/tools/copy_util.py
index 8e97cd14..65498cd8 100644
--- a/src/dfcx_scrapi/tools/copy_util.py
+++ b/src/dfcx_scrapi/tools/copy_util.py
@@ -134,36 +134,22 @@ def _convert_entry_webhooks(page_object, webhooks_map):
     def __convert_tr_target_page(
         trans_route, pages_map, convert_type=None, flows_map=None, flow=None
     ):
+        special_pages = [
+            "END_FLOW", "END_SESSION",
+            "CURRENT_PAGE", "PREVIOUS_PAGE", "START_PAGE"
+        ]
 
         if convert_type == "source":
-            if trans_route.target_page.split("/")[-1] == "END_FLOW":
-                trans_route.target_page = "END_FLOW"
-            elif trans_route.target_page.split("/")[-1] == "END_SESSION":
-                trans_route.target_page = "END_SESSION"
-            elif trans_route.target_page.split("/")[-1] == "CURRENT_PAGE":
-                trans_route.target_page = "CURRENT_PAGE"
-            elif trans_route.target_page.split("/")[-1] == "PREVIOUS_PAGE":
-                trans_route.target_page = "PREVIOUS_PAGE"
-            elif trans_route.target_page.split("/")[-1] == "START_PAGE":
-                trans_route.target_page = "START_PAGE"
+            last_part = trans_route.target_page.split("/")[-1]
+            if last_part in special_pages:
+                trans_route.target_page = last_part
             else:
                 trans_route.target_page = pages_map[trans_route.target_page]
 
         elif convert_type == "destination":
-            if trans_route.target_page == "END_FLOW":
-                trans_route.target_page = flows_map[flow] + "/pages/END_FLOW"
-            elif trans_route.target_page == "END_SESSION":
-                trans_route.target_page = flows_map[flow] + "/pages/END_SESSION"
-            elif trans_route.target_page == "CURRENT_PAGE":
-                trans_route.target_page = (
-                    flows_map[flow] + "/pages/CURRENT_PAGE"
-                )
-            elif trans_route.target_page == "PREVIOUS_PAGE":
-                trans_route.target_page = (
-                    flows_map[flow] + "/pages/PREVIOUS_PAGE"
-                )
-            elif trans_route.target_page == "START_PAGE":
-                trans_route.target_page = flows_map[flow] + "/pages/START_PAGE"
+            if trans_route.target_page in special_pages:
+                new_page = f"{flows_map[flow]}/pages/{trans_route.target_page}"
+                trans_route.target_page = new_page
             else:
                 trans_route.target_page = pages_map[trans_route.target_page]
 
diff --git a/src/dfcx_scrapi/tools/dataframe_functions.py b/src/dfcx_scrapi/tools/dataframe_functions.py
index 139d377c..59686b1e 100644
--- a/src/dfcx_scrapi/tools/dataframe_functions.py
+++ b/src/dfcx_scrapi/tools/dataframe_functions.py
@@ -70,7 +70,10 @@ def __init__(
         if scope:
             scopes += scope
 
-        if creds_path:
+        if creds:
+            self.sheets_client = gspread.authorize(creds)
+
+        elif creds_path:
             creds = ServiceAccountCredentials.from_json_keyfile_name(
                 filename=creds_path, scopes=scopes
             )
@@ -89,15 +92,11 @@ def __init__(
             creds = google.auth.default(scopes=scopes)[0]
             self.sheets_client = gspread.authorize(creds)
 
-        logging.info("create dfcx creds %s", creds_path)
-        self.entities = EntityTypes(creds_path, creds_dict)
-        self.intents = Intents(creds_path, creds_dict)
-        self.flows = Flows(creds_path, creds_dict)
-        self.pages = Pages(creds_path, creds_dict)
-        self.route_groups = TransitionRouteGroups(
-            creds_path, creds_dict
-        )
-        self.creds_path = creds_path
+        self.entities = EntityTypes(creds=self.creds)
+        self.intents = Intents(creds=self.creds)
+        self.flows = Flows(creds=self.creds)
+        self.pages = Pages(creds=self.creds)
+        self.route_groups = TransitionRouteGroups(creds=self.creds)
 
     @staticmethod
     def progress_bar(current, total, bar_length=50, type_="Progress"):
diff --git a/src/dfcx_scrapi/tools/nlu_evals.py b/src/dfcx_scrapi/tools/nlu_evals.py
new file mode 100644
index 00000000..d8c84977
--- /dev/null
+++ b/src/dfcx_scrapi/tools/nlu_evals.py
@@ -0,0 +1,308 @@
+"""A set of Utility methods to check resources stats on DFCX Agents."""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict
+from dataclasses import dataclass
+
+import logging
+import datetime
+import pandas as pd
+import gspread
+
+from dfcx_scrapi.core import scrapi_base
+from dfcx_scrapi.core import agents
+from dfcx_scrapi.core import flows
+from dfcx_scrapi.core import pages
+from dfcx_scrapi.core import intents
+from dfcx_scrapi.core import conversation
+from dfcx_scrapi.tools import dataframe_functions
+
+pd.options.display.max_colwidth = 200
+
+GLOBAL_SCOPE = [
+    "https://spreadsheets.google.com/feeds",
+    "https://www.googleapis.com/auth/drive",
+]
+INPUT_SCHEMA_COLUMNS = [
+        "flow_display_name",
+        "page_display_name",
+        "utterance",
+        "expected_intent",
+        "expected_parameters",
+        "description",
+    ]
+
+OUTPUT_SCHEMA_COLUMNS = [
+    "flow_display_name",
+    "page_display_name",
+    "utterance",
+    "expected_intent",
+    "expected_parameters",
+    "target_page",
+    "match_type",
+    "confidence",
+    "parameters_set",
+    "detected_intent",
+    "agent_display_name",
+    "description",
+    "input_source"
+    ]
+
+SUMMARY_SCHEMA_COLUMNS = [
+    "test_run_timestamp",
+    "total_tests",
+    "pass_count",
+    "pass_rate",
+    "no_match_count",
+    "no_match_rate",
+    "test_agent",
+    "data_source"
+    ]
+
+# logging config
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)-8s %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+
+@dataclass
+class Stats:
+    """Dataclass for the summary stats."""
+    no_match_count: int = 0
+    no_match_rate: float = 0.0
+    pass_count: int = 0
+    pass_rate: float = 0.0
+    test_agent: str = None
+    data_source: str = None
+
+
+class NluEvals(scrapi_base.ScrapiBase):
+    """NLU Evaluation Class for Dialogflow CX Testing."""
+    def __init__(
+        self,
+        agent_id: str,
+        creds_path: str = None,
+        creds_dict: Dict[str, str] = None,
+        creds=None,
+    ):
+
+        super().__init__(
+            creds_path=creds_path,
+            creds_dict=creds_dict,
+            creds=creds,
+            scope=GLOBAL_SCOPE
+        )
+
+        self.agent_id = agent_id
+        self._sheets_client = self._build_sheets_client()
+
+        self._a = agents.Agents(creds=self.creds)
+        self._i = intents.Intents(creds=self.creds)
+        self._f = flows.Flows(creds=self.creds)
+        self._p = pages.Pages(creds=self.creds)
+        self._dc = conversation.DialogflowConversation(
+            creds_path=creds_path, agent_id=agent_id
+        )
+        self._dffx = dataframe_functions.DataframeFunctions(creds=self.creds)
+
+    def _build_sheets_client(self):
+        client = gspread.authorize(self.creds)
+
+        return client
+
+    def _calculate_stats(self, df: pd.DataFrame):
+        """Calculate all the stats needed for the summary report."""
+        stats = Stats()
+        stats.no_match_count = (
+            df[df.detected_intent == "NO_MATCH"]
+            .groupby("detected_intent")
+            .size()
+            .sum()
+        )
+        stats.no_match_rate = stats.no_match_count / df.shape[0]
+        stats.pass_count = (
+            df[df.detected_intent == df.expected_intent]
+            .groupby("detected_intent")
+            .size()
+            .sum()
+        )
+        stats.pass_rate = stats.pass_count / df.shape[0]
+        stats.test_agent = df.agent_display_name.unique()[0]
+        stats.data_source = df.input_source.unique()[0]
+
+        return stats
+
+    def _write_report_summary_to_sheets(
+        self, df: pd.DataFrame, sheet_name: str, sheet_tab: str
+    ):
+        """Writes the output report summary to Google Sheets."""
+
+        gsheet = self._sheets_client.open(sheet_name)
+        sheet = gsheet.worksheet(sheet_tab)
+
+        df["test_run_timestamp"] = df.test_run_timestamp.astype("str")
+
+        sheet.append_row(
+            df.values.flatten().tolist(), value_input_option="USER_ENTERED"
+        )
+
+    def _append_test_results_to_sheets(
+        self, results: pd.DataFrame, sheet_name: str, sheet_tab: str
+    ):
+        """Adds results to an existing Google Sheet collection."""
+
+        gsheet = self._sheets_client.open(sheet_name)
+        sheet = gsheet.worksheet(sheet_tab)
+
+        # Fixes an error that sometimes happens when trying to write parameters
+        # to the sheet because they are formatted as objects
+        result_list = results.values.tolist()
+        result_list = [list(map(str, row)) for row in result_list]
+
+        sheet.append_rows(result_list, value_input_option="USER_ENTERED")
+
+    def _write_test_results_to_sheets(
+        self, results: pd.DataFrame, sheet_name: str, sheet_tab: str
+    ):
+        """Writes the output result details to Google Sheets."""
+
+        gsheet = self._sheets_client.open(sheet_name)
+        sheet = gsheet.worksheet(sheet_tab)
+
+        sheet.clear()
+
+        self._dffx.dataframe_to_sheets(sheet_name, sheet_tab, results)
+
+    def _clean_dataframe(self, df):
+        """Various Dataframe cleaning functions."""
+        df.columns = df.columns.str.lower()
+        df = df.replace("Start Page", "START_PAGE")
+        df.rename(
+                columns={
+                    "source": "description",
+                },
+                inplace=True,
+            )
+
+        # Validate input schema
+        try:
+            df = df[INPUT_SCHEMA_COLUMNS]
+        except KeyError as err:
+            raise UserWarning("Ensure your input data contains the following "\
+                              f"columns: {INPUT_SCHEMA_COLUMNS}") from err
+
+        df["agent_display_name"] = self._a.get_agent(self.agent_id).display_name
+
+        return df
+
+    def process_input_csv(self, input_file_path: str):
+        """Process the input data in CSV format."""
+        df = pd.read_csv(input_file_path)
+        df = df.fillna("")
+        df = self._clean_dataframe(df)
+        df["input_source"] = input_file_path
+
+        return df
+
+    def process_input_google_sheet(self, gsheet_name: str, gsheet_tab: str):
+        """Process the input data in Google Sheets format."""
+        df = self._dffx.sheets_to_dataframe(gsheet_name, gsheet_tab)
+        df = self._clean_dataframe(df)
+        df["input_source"] = gsheet_tab
+
+        return df
+
+    def run_evals(self, df: pd.DataFrame, chunk_size: int = 300,
+                  rate_limit: float = 10.0,
+                  eval_run_display_name: str = "Evals"):
+        """Run the full Eval dataset."""
+        logsx = "-" * 10
+
+        logging.info(f"{logsx} STARTING {eval_run_display_name} {logsx}")
+        results = self._dc.run_intent_detection(
+            test_set=df, chunk_size=chunk_size, rate_limit=rate_limit
+        )
+
+        # Reorder Columns
+        results = results.reindex(columns=OUTPUT_SCHEMA_COLUMNS)
+
+        # When a NO_MATCH occurs, the detected_intent field will be blank
+        # this replaces with NO_MATCH string, which will allow for easier stats
+        # calculation downstream
+        results.detected_intent.replace({"": "NO_MATCH"}, inplace=True)
+
+        logging.info(f"{logsx} {eval_run_display_name} COMPLETE {logsx}")
+
+        return results
+
+    def generate_report(self, df: pd.DataFrame,
+                        report_timestamp: datetime.datetime
+                        ):
+        """Generates a summary stats report for most recent NLU Eval tests."""
+        # Calc fields
+        stats = self._calculate_stats(df)
+
+        # Generate Dataframe format
+        df_report = pd.DataFrame(
+            columns=SUMMARY_SCHEMA_COLUMNS,
+            data=[
+                [
+                    report_timestamp,
+                    df.shape[0],
+                    stats.pass_count,
+                    stats.pass_rate,
+                    stats.no_match_count,
+                    stats.no_match_rate,
+                    stats.test_agent,
+                    stats.data_source,
+                ]
+            ],
+        )
+
+        return df_report
+
+    def write_summary_to_file(self, df: pd.DataFrame, output_file: str):
+        """Write summary output to a local CSV file."""
+        report_timestamp = datetime.datetime.now()
+        df_report = self.generate_report(df, report_timestamp)
+        df_report.to_csv(output_file, index=False)
+
+    def write_results_to_file(self, df: pd.DataFrame, output_file: str):
+        df.to_csv(output_file, index=False)
+
+    def write_results_to_sheets(self, df: pd.DataFrame, google_sheet_name: str,
+                                full_output_tab: str,
+                                summary_tab: str,
+                                append=False):
+        """Write summary and detailed output to Google Sheets."""
+        report_timestamp = datetime.datetime.now()
+        df_report = self.generate_report(df, report_timestamp)
+
+        self._write_report_summary_to_sheets(
+            df_report, google_sheet_name, summary_tab
+        )
+
+        if append:
+            self._append_test_results_to_sheets(
+                df, google_sheet_name, full_output_tab
+            )
+
+        else:
+            self._write_test_results_to_sheets(
+                df, google_sheet_name, full_output_tab
+            )
diff --git a/src/dfcx_scrapi/tools/nlu_util.py b/src/dfcx_scrapi/tools/nlu_util.py
index 0bc152bd..a59122a7 100644
--- a/src/dfcx_scrapi/tools/nlu_util.py
+++ b/src/dfcx_scrapi/tools/nlu_util.py
@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import sys
 import numpy as np
 import pandas as pd
 import scann
@@ -29,9 +30,9 @@
 from dfcx_scrapi.core import scrapi_base
 from dfcx_scrapi.core import transition_route_groups
 
-from google.colab import data_table
-
-data_table.enable_dataframe_formatter()
+if "google.colab" in sys.modules:
+    from google.colab import data_table
+    data_table.enable_dataframe_formatter()
 
 SHEETS_SCOPE = [
     "https://spreadsheets.google.com/feeds",
@@ -129,22 +130,26 @@ def _load_data(
             flow_display_name, agent_id
         )
 
-        page_loader = pages.Pages(creds=self.creds)
-        page_map = page_loader.get_pages_map(self.flow.name, reverse=True)
-        page_id = page_map.get(page_display_name, None)
-        if page_id is None:
-            raise ValueError(
-                f'Page "{page_display_name}" does not exist in the '
-                "specified agent."
-            )
-        self.page = page_loader.get_page(page_id)
-
         trg_loader = transition_route_groups.TransitionRouteGroups(
             creds=self.creds
         )
-        self.trgs = trg_loader.list_transition_route_groups(self.flow.name)
+        agent_trgs = trg_loader.list_transition_route_groups(agent_id)
+        self.trgs = (
+            agent_trgs + trg_loader.list_transition_route_groups(self.flow.name)
+        )
         self.name_to_trg = {i.name: i for i in self.trgs}
 
+        if page_display_name != "Start Page":
+            page_loader = pages.Pages(creds=self.creds)
+            page_map = page_loader.get_pages_map(self.flow.name, reverse=True)
+            page_id = page_map.get(page_display_name, None)
+            if page_id is None:
+                raise ValueError(
+                    f'Page "{page_display_name}" does not exist in the '
+                    "specified agent."
+                )
+            self.page = page_loader.get_page(page_id)
+
         intent_names = self._list_page_intents()
         all_intents = intents.Intents(creds=self.creds).list_intents(agent_id)
         self.intents = list(
@@ -160,10 +165,11 @@ def _list_page_intents(self) -> Set[str]:
         for tr in self.flow.transition_routes:
             if tr.intent:
                 relevant_intents.add(tr.intent)
-        relevant_trgs |= set(self.page.transition_route_groups)
-        for tr in self.page.transition_routes:
-            if tr.intent:
-                relevant_intents.add(tr.intent)
+        if hasattr(self, "page"):
+            relevant_trgs |= set(self.page.transition_route_groups)
+            for tr in self.page.transition_routes:
+                if tr.intent:
+                    relevant_intents.add(tr.intent)
         for trg_name in relevant_trgs:
             trg = self.name_to_trg[trg_name]
             for tr in trg.transition_routes:


Test patch:
diff --git a/examples/nlu_analysis_series/nlu_evaluation_testing.ipynb b/examples/nlu_analysis_series/nlu_evaluation_testing.ipynb
new file mode 100644
index 00000000..34c50869
--- /dev/null
+++ b/examples/nlu_analysis_series/nlu_evaluation_testing.ipynb
@@ -0,0 +1,439 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Copyright 2024 Google LLC\n",
+    "#\n",
+    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+    "# you may not use this file except in compliance with the License.\n",
+    "# You may obtain a copy of the License at\n",
+    "#\n",
+    "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+    "#\n",
+    "# Unless required by applicable law or agreed to in writing, software\n",
+    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+    "# See the License for the specific language governing permissions and\n",
+    "# limitations under the License."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# NLU Evaluation Testing\n",
+    "In this notebook, we will show you how to perform bulk NLU testing by providing a large input corpus and receiving the predicted Intent and Parameter extraction results from your agent.\n",
+    "\n",
+    "## Prerequisites\n",
+    "- Ensure you have a GCP Service Account key with the Dialogflow API Admin privileges assigned to it.\n",
+    "\n",
+    "<table align=\"left\">\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/nlu_analysis_series/nlu_evaluation_testing.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Google Colaboratory logo\"><br> Run in Colab\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://github.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/nlu_analysis_series/nlu_evaluation_testing.ipynb\">\n",
+    "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"><br> View on GitHub\n",
+    "    </a>\n",
+    "  </td>\n",
+    "  <td style=\"text-align: center\">\n",
+    "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/examples/nlu_analysis_series/nlu_evaluation_testing.ipynb\">\n",
+    "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
+    "    </a>\n",
+    "  </td>\n",
+    "</table>"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# If you haven't already, make sure you install the `dfcx-scrapi` library\n",
+    "\n",
+    "!pip install dfcx-scrapi"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Imports"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dfcx_scrapi.tools.nlu_evals import NluEvals"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# (Option 1) Google Sheet as Input\n",
+    "The primary option for running the NLU Eval pipeline revolves around using Google Sheets as a source for your data.  \n",
+    "This method provides a simple, streamlined system that can pull and push data all into a single Google Sheet.\n",
+    "\n",
+    "In order to run the full NLU evaluation test, the following inputs are needed:\n",
+    "- `agent_id`, The Dialogflow CX Agent ID.\n",
+    "- `input_google_sheet`, the Display Name of the Google Sheet.\n",
+    "- `input_google_sheet_tab`, the Display Name of the tab on the Google Sheet where your input data lives.\n",
+    "- `output_google_sheet_results`, the Display Name of the tab on the Google Sheet where you want the full output results to be written.\n",
+    "- `output_google_sheet_summary`, the Display Name of the tab on the Google Sheet where you want the report summary to be written.\n",
+    "\n",
+    "_**NOTE** - In order for your Service Account to access your Google Sheet (read / write) you need to share the Google Sheet with your Service Account email address._\n",
+    "\n",
+    "You can find a [Sample Google Sheet dataset](https://docs.google.com/spreadsheets/d/e/2PACX-1vREvsZAktNvRr78KjUBlZl2PVUHKJru8hRCgmuDi9kn_oDT_weFKkGmyoQwRPdj0JcxK1kNzgceAPA5/pubhtml#) here."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "agent_id = '<YOUR_AGENT_ID>'\n",
+    "creds_path = '<YOUR_CREDS_FILE>'\n",
+    "\n",
+    "# Sample Inputs\n",
+    "input_google_sheet = 'Dialogflow CX SCRAPI - NLU Eval Sample Dataset'\n",
+    "input_google_sheet_tab = 'input_dataset'\n",
+    "output_google_sheet_results = 'results'\n",
+    "output_google_sheet_summary = 'summary'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Run NLU Evals\n",
+    "There are 3 main stages that happen for the Eval Pipeline:\n",
+    "1. Process and validate the input data\n",
+    "2. Run the Eval Tests\n",
+    "3. Write the output summary and details to a report."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2023-09-11 11:40:31 INFO     ---------- STARTING Evals ----------\n",
+      "2023-09-11 11:40:37 WARNING  Text input is too long. Truncating to 256 characters.\n",
+      "2023-09-11 11:40:37 WARNING  TRUNCATED TEXT: Why can't I get my bill paid at all online? Why can't I get my bill paid at all online?Why can't I get my bill paid at all online?Why can't I get my bill paid at all online?Why can't I get my bill paid at all online?Why can't I get my bill paid at all onli\n",
+      "2023-09-11 11:40:41 INFO     Progress(0/15)[>                                                 ] 0.00%\n",
+      "2023-09-11 11:40:51 INFO     Progress(15/15)[------------------------------------------------->] 100.00%\n",
+      "2023-09-11 11:40:51 INFO     ---------- Evals COMPLETE ----------\n"
+     ]
+    }
+   ],
+   "source": [
+    "nlu = NluEvals(agent_id, creds_path=creds_path)\n",
+    "\n",
+    "df = nlu.process_input_google_sheet(input_google_sheet, input_google_sheet_tab)\n",
+    "df = nlu.run_evals(df)\n",
+    "nlu.write_results_to_sheets(df, input_google_sheet, output_google_sheet_results, output_google_sheet_summary)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Inspect Results Locally\n",
+    "You can also inspect and filter the results of your tests locally as needed."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>flow_display_name</th>\n",
+       "      <th>page_display_name</th>\n",
+       "      <th>utterance</th>\n",
+       "      <th>expected_intent</th>\n",
+       "      <th>expected_parameters</th>\n",
+       "      <th>target_page</th>\n",
+       "      <th>match_type</th>\n",
+       "      <th>confidence</th>\n",
+       "      <th>parameters_set</th>\n",
+       "      <th>detected_intent</th>\n",
+       "      <th>agent_display_name</th>\n",
+       "      <th>description</th>\n",
+       "      <th>input_source</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Default Start Flow</td>\n",
+       "      <td>START_PAGE</td>\n",
+       "      <td>I need to get my order status</td>\n",
+       "      <td>head_intent.order_status</td>\n",
+       "      <td></td>\n",
+       "      <td>sentiment_router</td>\n",
+       "      <td>INTENT</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td></td>\n",
+       "      <td>head_intent.order_status</td>\n",
+       "      <td>[Demo] Multi Demo Extravaganza Part Deux: The Revenge</td>\n",
+       "      <td>Demo Tests</td>\n",
+       "      <td>input_dataset</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Default Start Flow</td>\n",
+       "      <td>START_PAGE</td>\n",
+       "      <td>Trying to check the status of my order</td>\n",
+       "      <td>head_intent.order_status</td>\n",
+       "      <td></td>\n",
+       "      <td>sentiment_router</td>\n",
+       "      <td>INTENT</td>\n",
+       "      <td>0.947959</td>\n",
+       "      <td></td>\n",
+       "      <td>head_intent.order_status</td>\n",
+       "      <td>[Demo] Multi Demo Extravaganza Part Deux: The Revenge</td>\n",
+       "      <td>Demo Tests</td>\n",
+       "      <td>input_dataset</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Default Start Flow</td>\n",
+       "      <td>START_PAGE</td>\n",
+       "      <td>I hate this order status agent!</td>\n",
+       "      <td>head_intent.order_status</td>\n",
+       "      <td></td>\n",
+       "      <td>sentiment_router</td>\n",
+       "      <td>INTENT</td>\n",
+       "      <td>0.955709</td>\n",
+       "      <td></td>\n",
+       "      <td>head_intent.order_status</td>\n",
+       "      <td>[Demo] Multi Demo Extravaganza Part Deux: The Revenge</td>\n",
+       "      <td>Demo Tests</td>\n",
+       "      <td>input_dataset</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Default Start Flow</td>\n",
+       "      <td>START_PAGE</td>\n",
+       "      <td>Wha'ts the point of ordering anything?</td>\n",
+       "      <td>NO_MATCH</td>\n",
+       "      <td></td>\n",
+       "      <td>sentiment_router</td>\n",
+       "      <td>INTENT</td>\n",
+       "      <td>0.841712</td>\n",
+       "      <td></td>\n",
+       "      <td>head_intent.order_status</td>\n",
+       "      <td>[Demo] Multi Demo Extravaganza Part Deux: The Revenge</td>\n",
+       "      <td>Demo Tests</td>\n",
+       "      <td>input_dataset</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Default Start Flow</td>\n",
+       "      <td>START_PAGE</td>\n",
+       "      <td>I was looking at the order of operations yesterday but couldn't figure it out</td>\n",
+       "      <td>NO_MATCH</td>\n",
+       "      <td></td>\n",
+       "      <td>sentiment_router</td>\n",
+       "      <td>INTENT</td>\n",
+       "      <td>0.790275</td>\n",
+       "      <td></td>\n",
+       "      <td>head_intent.order_status</td>\n",
+       "      <td>[Demo] Multi Demo Extravaganza Part Deux: The Revenge</td>\n",
+       "      <td>Demo Tests</td>\n",
+       "      <td>input_dataset</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "    flow_display_name page_display_name  \\\n",
+       "0  Default Start Flow        START_PAGE   \n",
+       "1  Default Start Flow        START_PAGE   \n",
+       "2  Default Start Flow        START_PAGE   \n",
+       "3  Default Start Flow        START_PAGE   \n",
+       "4  Default Start Flow        START_PAGE   \n",
+       "\n",
+       "                                                                       utterance  \\\n",
+       "0                                                  I need to get my order status   \n",
+       "1                                         Trying to check the status of my order   \n",
+       "2                                                I hate this order status agent!   \n",
+       "3                                         Wha'ts the point of ordering anything?   \n",
+       "4  I was looking at the order of operations yesterday but couldn't figure it out   \n",
+       "\n",
+       "            expected_intent expected_parameters       target_page match_type  \\\n",
+       "0  head_intent.order_status                      sentiment_router     INTENT   \n",
+       "1  head_intent.order_status                      sentiment_router     INTENT   \n",
+       "2  head_intent.order_status                      sentiment_router     INTENT   \n",
+       "3                  NO_MATCH                      sentiment_router     INTENT   \n",
+       "4                  NO_MATCH                      sentiment_router     INTENT   \n",
+       "\n",
+       "   confidence parameters_set           detected_intent  \\\n",
+       "0    1.000000                 head_intent.order_status   \n",
+       "1    0.947959                 head_intent.order_status   \n",
+       "2    0.955709                 head_intent.order_status   \n",
+       "3    0.841712                 head_intent.order_status   \n",
+       "4    0.790275                 head_intent.order_status   \n",
+       "\n",
+       "                                      agent_display_name description  \\\n",
+       "0  [Demo] Multi Demo Extravaganza Part Deux: The Revenge  Demo Tests   \n",
+       "1  [Demo] Multi Demo Extravaganza Part Deux: The Revenge  Demo Tests   \n",
+       "2  [Demo] Multi Demo Extravaganza Part Deux: The Revenge  Demo Tests   \n",
+       "3  [Demo] Multi Demo Extravaganza Part Deux: The Revenge  Demo Tests   \n",
+       "4  [Demo] Multi Demo Extravaganza Part Deux: The Revenge  Demo Tests   \n",
+       "\n",
+       "    input_source  \n",
+       "0  input_dataset  \n",
+       "1  input_dataset  \n",
+       "2  input_dataset  \n",
+       "3  input_dataset  \n",
+       "4  input_dataset  "
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# (Option 2) CSV as Input\n",
+    "Similar to the above pipeline, except we will process the input data from a CSV file.\n",
+    "\n",
+    "For the output to local files, you will need to define 2 output destinations:\n",
+    "1. An output file for the full detailed results\n",
+    "2. An output file for the report summary\n",
+    "\n",
+    "- `agent_id`, The Dialogflow CX Agent ID.\n",
+    "- `input_path`, The local path where your input data lives\n",
+    "- `output_summary_path`, The local path where you want the report summary written\n",
+    "- `output_results_path`, The local path where you want the full results written\n",
+    "\n",
+    "You can find a [Sample CSV Dataset here.](https://github.com/GoogleCloudPlatform/dfcx-scrapi/blob/main/data/nlu_evals_sample.csv)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "agent_id = '<YOUR_AGENT_ID>'\n",
+    "\n",
+    "input_path = '/path/to/your/input/data.csv'\n",
+    "output_summary_path = '/path/to/your/output/summary.csv'\n",
+    "output_results_path = '/path/to/your/output/results.csv'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Run NLU Evals\n",
+    "\n",
+    "There are 3 main stages that happen for the Eval Pipeline:\n",
+    "1. Process and validate the input data\n",
+    "2. Run the Eval Tests\n",
+    "3. Write the output summary and details to a report."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2023-09-11 11:31:51 INFO     ---------- STARTING Evals ----------\n",
+      "2023-09-11 11:31:56 WARNING  Text input is too long. Truncating to 256 characters.\n",
+      "2023-09-11 11:31:56 WARNING  TRUNCATED TEXT: Why can't I get my bill paid at all online? Why can't I get my bill paid at all online?Why can't I get my bill paid at all online?Why can't I get my bill paid at all online?Why can't I get my bill paid at all online?Why can't I get my bill paid at all onli\n",
+      "2023-09-11 11:31:58 INFO     Progress(0/15)[>                                                 ] 0.00%\n",
+      "2023-09-11 11:32:08 INFO     Progress(15/15)[------------------------------------------------->] 100.00%\n",
+      "2023-09-11 11:32:08 INFO     ---------- Evals COMPLETE ----------\n"
+     ]
+    }
+   ],
+   "source": [
+    "nlu = NluEvals(agent_id, creds_path=creds_path)\n",
+    "\n",
+    "df = nlu.process_input_csv(input_path)\n",
+    "df = nlu.run_evals(df)\n",
+    "nlu.write_summary_to_file(df, output_summary_path)\n",
+    "nlu.write_results_to_file(df, output_results_path)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "scrapi-local",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.12"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/pytest.ini b/pytest.ini
new file mode 100644
index 00000000..a347360c
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,3 @@
+[pytest]
+minversion = 6.0
+addopts = -p no:warnings --ignore=src/dfcx_scrapi/core/test_cases.py
diff --git a/src/dfcx_scrapi/agent_extract/test_cases.py b/src/dfcx_scrapi/agent_extract/test_cases.py
new file mode 100644
index 00000000..ee5be205
--- /dev/null
+++ b/src/dfcx_scrapi/agent_extract/test_cases.py
@@ -0,0 +1,184 @@
+"""Test Case processing methods and functions."""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+from typing import Dict, List, Any
+
+from dfcx_scrapi.agent_extract import common
+from dfcx_scrapi.agent_extract import types
+
+
+class TestCases:
+    """Test Case processing methods and functions."""
+
+    def __init__(self):
+        self.common = common.Common()
+
+    @staticmethod
+    def build_test_case_path_list(agent_local_path: str):
+        """Builds a list of files, each representing a test case."""
+        root_dir = agent_local_path + "/testCases"
+
+        test_case_paths = []
+
+        for test_case in os.listdir(root_dir):
+            end = test_case.split(".")[-1]
+            if end == "json":
+                test_case_path = f"{root_dir}/{test_case}"
+                test_case_paths.append(test_case_path)
+
+        return test_case_paths
+
+    @staticmethod
+    def get_test_case_intent_phrase_pair(
+        tc: types.TestCase) -> List[Dict[str, str]]:
+        """Parse Test Case and return a list of intents in use.
+
+        This method will produce a List of Dicts where the contents of each
+        dict is the Training Phrase and associated Triggered Intent as listed
+        in the Test Case Conversation Turn. This information is used to compare
+        the User Input training phrase with the actual training phrases that
+        exist in the Intent resource.
+
+        The dict format is as follows:
+            {
+                training_phrase: <training_phrase>,
+                intent: <intent_display_name>
+            }
+        """
+        intent_data = []
+
+        if tc.conversation_turns:
+            for turn in tc.conversation_turns:
+                user = turn["userInput"]
+                agent = turn["virtualAgentOutput"]
+                intent = agent.get("triggeredIntent", None)
+                phrase = user.get("input", None)
+
+                text = phrase.get("text", None)
+
+                if text:
+                    text = text["text"]
+
+                if intent and text:
+                    intent_data.append(
+                        {
+                            "user_utterance": text,
+                            "intent": intent["name"],
+                            "status": "valid",
+                            "training_phrases": [],
+                        }
+                    )
+
+        return intent_data
+
+    @staticmethod
+    def get_test_case_intent_data(agent_local_path: str):
+        """Collect all Intent Files and Training Phrases for Test Case."""
+        intents_path = agent_local_path + "/intents"
+
+        intent_paths = []
+
+        for intent_dir in os.listdir(intents_path):
+            intent_dir_path = f"{intents_path}/{intent_dir}"
+            intent_paths.append(
+                {"intent": intent_dir, "file_path": intent_dir_path}
+            )
+
+        return intent_paths
+
+    @staticmethod
+    def flatten_tp_data(tp_data: List[Any]):
+        """Flatten the Training Phrase proto to a list of strings."""
+        cleaned_tps = []
+
+        for tp in tp_data["trainingPhrases"]:
+            parts_list = [part["text"].lower() for part in tp["parts"]]
+            cleaned_tps.append("".join(parts_list))
+
+        return cleaned_tps
+
+    def gather_intent_tps(self, tc: types.TestCase):
+        """Collect all TPs associated with Intent data in Test Case."""
+        tc.associated_intent_data = {}
+
+        for i, pair in enumerate(tc.intent_data):
+            intent_dir = tc.agent_path + "/intents/" + pair["intent"]
+
+            try:
+                if "trainingPhrases" in os.listdir(intent_dir):
+                    training_phrases_path = intent_dir + "/trainingPhrases"
+
+                    for lang_file in os.listdir(training_phrases_path):
+                        # lang_code = lang_file.split(".")[0]
+                        lang_code_path = f"{training_phrases_path}/{lang_file}"
+
+                        with open(
+                            lang_code_path, "r", encoding="UTF-8"
+                        ) as tp_file:
+                            tp_data = json.load(tp_file)
+                            cleaned_tps = self.flatten_tp_data(tp_data)
+
+                            tp_file.close()
+
+                        tc.intent_data[i]["training_phrases"].extend(
+                            cleaned_tps
+                        )
+                        tc.associated_intent_data[pair["intent"]] = cleaned_tps
+
+            except FileNotFoundError:
+                tc.intent_data[i]["status"] = "invalid_intent"
+                tc.has_invalid_intent = True
+                continue
+
+        return tc
+
+    def process_test_case(self, tc: types.TestCase, stats: types.AgentData):
+        """Process a single Test Case file."""
+
+        with open(tc.dir_path, "r", encoding="UTF-8") as tc_file:
+            tc.data = json.load(tc_file)
+            tc.resource_id = tc.data.get("name", None)
+            tc.display_name = tc.data.get("displayName", None)
+            tc.tags = tc.data.get("tags", None)
+            tc.conversation_turns = tc.data.get(
+                "testCaseConversationTurns", None
+            )
+            tc.test_config = tc.data.get("testConfig", None)
+
+            full_tc_id = f"{stats.agent_id}/testCases/{tc.resource_id}"
+            tc.data["name"] = full_tc_id
+            stats.test_cases.append(tc.data)
+
+            tc_file.close()
+
+        return stats
+
+    def process_test_cases_directory(
+            self, agent_local_path: str, stats: types.AgentData):
+        """Processing the test cases dir in the JSON package structure."""
+        test_case_paths = self.build_test_case_path_list(agent_local_path)
+        stats.total_test_cases = len(test_case_paths)
+
+        for test_case in test_case_paths:
+            tc = types.TestCase()
+            tc.dir_path = test_case
+            tc.agent_path = agent_local_path
+            stats = self.process_test_case(tc, stats)
+
+        return stats
diff --git a/src/dfcx_scrapi/core/test_cases.py b/src/dfcx_scrapi/core/test_cases.py
index 78976aea..98621a43 100644
--- a/src/dfcx_scrapi/core/test_cases.py
+++ b/src/dfcx_scrapi/core/test_cases.py
@@ -14,15 +14,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
+import pandas as pd
 import logging
-from typing import Dict, List
+from typing import Dict, List, Union
 
 from google.cloud.dialogflowcx_v3beta1 import services
 from google.cloud.dialogflowcx_v3beta1 import types
-from google.protobuf import field_mask_pb2
 
 from dfcx_scrapi.core import scrapi_base
+from dfcx_scrapi.core import flows
+from dfcx_scrapi.core import pages
 
 # logging config
 logging.basicConfig(
@@ -51,13 +52,123 @@ def __init__(
             scope=scope,
         )
 
-        if agent_id:
-            self.agent_id = agent_id
-            self.client_options = self._set_region(self.agent_id)
+        self.agent_id = agent_id
+        self.test_case_id = test_case_id
+
+    def _convert_test_result_to_string(self, test_case: types.TestCase) -> str:
+        """Converts the Enum result to a string."""
+        if test_case.last_test_result.test_result == 0:
+            return "TEST_RESULT_UNSPECIFIED"
+        elif test_case.last_test_result.test_result == 1:
+            return "PASSED"
+        elif test_case.last_test_result.test_result == 2:
+            return "FAILED"
+        else:
+            return ""
 
-        if test_case_id:
-            self.test_case_id = test_case_id
-            self.client_options = self._set_region(self.test_case_id)
+    def _convert_test_result_to_bool(
+            self, test_case: types.TestCase) -> Union[bool, None]:
+        """Converts the String result to a boolean."""
+        test_result = self._convert_test_result_to_string(test_case)
+
+        if test_result == "PASSED":
+            return True
+        elif test_result == "FAILED":
+            return False
+        else:
+            return None
+
+    def _get_flow_id_from_test_config(
+            self, test_case: types.TestCase) -> str:
+        """Attempt to get the Flow ID from the Test Case Test Config."""
+        if "flow" in test_case.test_config:
+            return test_case.test_config.flow
+        elif "page" in test_case.test_config:
+            return "/".join(test_case.test_config.page.split("/")[:8])
+        else:
+            agent_id = "/".join(test_case.name.split("/")[:6])
+            return f"{agent_id}/flows/00000000-0000-0000-0000-000000000000"
+
+    def _get_page_id_from_test_config(
+            self, test_case: types.TestCase, flow_id: str) -> str:
+        """Attempt to get the Page ID from the Test Case Test Config."""
+        if "page" in test_case.test_config:
+            return test_case.test_config.page
+        else:
+            return f"{flow_id}/pages/START_PAGE"
+
+    def _get_page_display_name(
+            self, flow_id: str, page_id: str,
+            pages_map: Dict[str, Dict[str, str]]) -> Union[str, None]:
+        """Get the Page Display Name from the Pages Map based on the Page ID."""
+        page_map = pages_map.get(flow_id, None)
+        page = "START_PAGE"
+        if page_map:
+            page = page_map.get(page_id, None)
+
+        return page
+
+    def _process_test_case(self, test_case, flows_map, pages_map):
+        """Takes a response from list_test_cases and returns a single row
+        dataframe of the test case result.
+
+        Args:
+          test_case: The test case response
+          flows_map: A dictionary mapping flow IDs to flow display names
+          pages_map: A dictionary with keys as flow IDs and values as
+            dictionaries mapping page IDs to page display names for that flow
+
+        Returns: A dataframe with columns:
+          display_name, id, short_id, tags, creation_time, start_flow,
+          start_page, test_result, passed, test_time
+        """
+        flow_id = self._get_flow_id_from_test_config(test_case)
+        page_id = self._get_page_id_from_test_config(test_case, flow_id)
+        page = self._get_page_display_name(flow_id, page_id, pages_map)
+        test_result = self._convert_test_result_to_bool(test_case)
+
+        return pd.DataFrame(
+            {
+                "display_name": [test_case.display_name],
+                "id": [test_case.name],
+                "short_id": [test_case.name.split("/")[-1]],
+                "tags": [",".join(test_case.tags)],
+                "creation_time": [test_case.creation_time],
+                "start_flow": [flows_map.get(flow_id, None)],
+                "start_page": [page],
+                # "test_result": [test_result],
+                "passed": [test_result],
+                "test_time": [test_case.last_test_result.test_time]
+            }
+        )
+
+    def _retest_cases(
+            self, test_case_df: pd.DataFrame, retest_ids: List[str]
+            ) -> pd.DataFrame:
+        print("To retest:", len(retest_ids))
+        response = self.batch_run_test_cases(retest_ids, self.agent_id)
+        for result in response.results:
+            # Results may not be in the same order as they went in
+            # Process the name a bit to remove the /results/id part
+            tc_id_full = "/".join(result.name.split("/")[:-2])
+            tc_id = tc_id_full.rsplit("/", maxsplit=1)[-1]
+
+            # Update dataframe where id = tc_id_full
+            # row = test_case_df.loc[test_case_df['id']==tc_id_full]
+            test_case_df.loc[
+                test_case_df["id"] == tc_id_full, "short_id"
+            ] = tc_id
+            # test_case_df.loc[
+            #     test_case_df["id"] == tc_id_full, "test_result"
+            # ] = str(result.test_result)
+            test_case_df.loc[
+                test_case_df["id"] == tc_id_full, "test_time"
+            ] = result.test_time
+            test_case_df.loc[test_case_df["id"] == tc_id_full,"passed"] = (
+                str(result.test_result) == "TestResult.PASSED"
+            )
+
+        return test_case_df
 
     @scrapi_base.api_call_counter_decorator
     def list_test_cases(
@@ -343,25 +454,35 @@ def update_test_case(
         Returns:
           The updated Test Case.
         """
+        if not(obj or test_case_id or self.test_case_id):
+            raise ValueError(
+                "At least one of `test_case_id` or `obj` should be passed.")
+
+        if test_case_id and not (obj or kwargs):
+            raise ValueError(
+                "At least one kwarg or `obj` arg should be passed, otherwise ",
+                "this is noop.")
 
         if obj:
             test_case = obj
-            test_case.name = test_case_id
-        else:
+
+            if test_case_id:
+                test_case.name = test_case_id
+
+            if not test_case.name:
+                raise ValueError("The `name` of the `obj` should not be empty.")
+            mask = self._update_kwargs(test_case)
+
+        elif kwargs:
             if not test_case_id:
                 test_case_id = self.test_case_id
             test_case = self.get_test_case(test_case_id)
+            mask = self._update_kwargs(test_case, **kwargs)
 
-        for key, value in kwargs.items():
-            setattr(test_case, key, value)
-        paths = kwargs.keys()
-        mask = field_mask_pb2.FieldMask(paths=paths)
+        request = types.test_case.UpdateTestCaseRequest(
+            test_case=test_case, update_mask=mask)
 
-        request = types.test_case.UpdateTestCaseRequest()
-        request.test_case = test_case
-        request.update_mask = mask
-
-        client_options = self._set_region(test_case_id)
+        client_options = self._set_region(test_case_id or obj.name)
         client = services.test_cases.TestCasesClient(
             credentials=self.creds, client_options=client_options
         )
@@ -453,3 +574,77 @@ def calculate_coverage(self, coverage_type: int, agent_id: str = None):
         )
         response = client.calculate_coverage(request)
         return response
+
+    def get_test_cases_map(self, agent_id: str = None, reverse=False):
+        """Exports Agent Test Cases and UUIDs into a user friendly dict.
+
+        Args:
+          agent_id: The formatted CX Agent ID to use. Format:
+            `projects/<Project ID>/locations/<Location ID>/agents/<Agent ID>`
+          reverse: (Optional) Boolean flag to swap key:value -> value:key
+
+        Returns:
+          Dict containing Test Case UUIDs as keys and display names as values.
+        """
+        if not agent_id:
+            agent_id = self.agent_id
+
+        if reverse:
+            test_cases_dict = {
+                test_case.display_name: test_case.name
+                for test_case in self.list_test_cases(agent_id=agent_id)
+            }
+        else:
+            test_cases_dict = {
+                test_case.name: test_case.display_name
+                for test_case in self.list_test_cases(agent_id=agent_id)
+            }
+
+        return test_cases_dict
+
+    def get_test_case_results_df(self, agent_id=None, retest_all=False):
+        """Convert Test Cases to Dataframe.
+
+        Gets the test case results for this agent, and generates a dataframe
+        with their details. Any tests without a result will be run in a batch.
+
+        Args:
+          agent_id: The agent to create the test case for. Format:
+              `projects/<Project ID>/locations/<Location ID>/agents/<Agent ID>`
+          retest_all: if true, all test cases are re-run,
+            regardless of whether or not they had a result
+
+        Returns:
+          DataFrame of test case results for this agent, with columns:
+            display_name, id, short_id, tags, creation_time, start_flow,
+            start_page, passed, test_time
+        """
+        if agent_id:
+            self.agent_id = agent_id
+
+        dfcx_flows = flows.Flows(creds=self.creds, agent_id=self.agent_id)
+        dfcx_pages = pages.Pages(creds=self.creds)
+        flows_map = dfcx_flows.get_flows_map(agent_id=self.agent_id)
+        pages_map = {}
+        for flow_id in flows_map.keys():
+            pages_map[flow_id] = dfcx_pages.get_pages_map(flow_id=flow_id)
+
+        test_case_results = self.list_test_cases(self.agent_id)
+        retest_ids = []
+        test_case_rows = []
+
+        for test_case in test_case_results:
+            row = self._process_test_case(test_case, flows_map, pages_map)
+            test_case_rows.append(row)
+            test_result = self._convert_test_result_to_string(test_case)
+            if retest_all or test_result == "TEST_RESULT_UNSPECIFIED":
+                retest_ids.append(test_case.name)
+
+        # Create dataframe
+        test_case_df = pd.concat(test_case_rows)
+
+        # Retest any that haven't been run yet
+        if len(retest_ids) > 0:
+            test_case_df = self._retest_cases(test_case_df,retest_ids)
+
+        return test_case_df
diff --git a/src/dfcx_scrapi/tools/test_cases_util.py b/src/dfcx_scrapi/tools/test_cases_util.py
new file mode 100644
index 00000000..f6e4b573
--- /dev/null
+++ b/src/dfcx_scrapi/tools/test_cases_util.py
@@ -0,0 +1,693 @@
+"""Test Cases Utility functions."""
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from typing import Dict, List, Union
+import time
+
+from google.cloud.dialogflowcx_v3beta1 import types
+from google.api_core import exceptions as core_exceptions
+from google.protobuf.struct_pb2 import Struct
+
+from dfcx_scrapi.core import scrapi_base
+from dfcx_scrapi.core import flows
+from dfcx_scrapi.core import pages
+from dfcx_scrapi.core import intents
+from dfcx_scrapi.core import test_cases
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)-8s %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+class TestCasesUtil(scrapi_base.ScrapiBase):
+    """Util class for CX test cases"""
+
+    def __init__(
+        self,
+        agent_id: str,
+        creds_path: str = None,
+        creds_dict: Dict = None,
+        creds=None,
+        scope=False,
+    ):
+        super().__init__(
+            creds_path=creds_path,
+            creds_dict=creds_dict,
+            creds=creds,
+            scope=scope,
+        )
+
+        self.agent_id = agent_id
+        self._global_test_cases = test_cases.TestCases(
+            creds=self.creds, agent_id=self.agent_id)
+        self.source_commons = {}
+        self.target_commons = {}
+
+    def _toggle_sentiment_in_convturns(
+        self,
+        conv_turns: List[types.ConversationTurn],
+        sentiment: bool) -> List[types.ConversationTurn]:
+        """Enable/disable enable_sentiment_analysis for all conversation turns.
+        Args:
+            conversation_turns: types.ConversationTurn
+        Returns:
+            a list of the conversation turns
+        """
+        new_conversation_turns = []
+        for conversation_turn in conv_turns:
+            conversation_turn.user_input.enable_sentiment_analysis = sentiment
+            new_conversation_turns.append(conversation_turn)
+
+        return new_conversation_turns
+
+    def toggle_sentiment_test_cases(
+        self,
+        test_case_ids: List[str],
+        sentiment: bool) -> List[types.TestCase]:
+        """Enable/disable enable_sentiment_analysis for the given test case ids.
+        Args:
+            test_cases: a list of test case ids
+            sentiment:
+                if True then enable_sentiment_analysis = True
+                else disable_sentiment_analysis = False
+            agent_id: The formatted CX Agent ID
+        Returns:
+            a list of the updated test cases
+        """
+        test_cases_list = self._global_test_cases.list_test_cases(
+            agent_id=self.agent_id, include_conversation_turns=True)
+        updated_test_cases_list = []
+        for test_case in test_cases_list:
+            if test_case.name in test_case_ids:
+                test_case.test_case_conversation_turns = (
+                    self._toggle_sentiment_in_convturns(
+                        conv_turns=test_case.test_case_conversation_turns,
+                        sentiment=sentiment)
+                )
+                updated_test_cases_list.append(test_case)
+
+        return updated_test_cases_list
+
+    def _toggle_webhooks_in_convturns(
+        self,
+        conv_turns: List[types.ConversationTurn],
+        is_webhook_enabled: bool) -> List[types.ConversationTurn]:
+        """Sets the webhooks equal to True/False for all conversation turns.
+        Args:
+            conversation_turns: types.ConversationTurn
+        Returns:
+            conversation turns
+        """
+
+        new_conversation_turns = []
+        for conversation_turn in conv_turns:
+            conversation_turn.user_input.is_webhook_enabled = is_webhook_enabled
+            new_conversation_turns.append(conversation_turn)
+
+        return new_conversation_turns
+
+    def toggle_webhooks_test_cases(
+        self,
+        test_case_ids: List[str],
+        is_webhook_enabled: bool) -> List[types.TestCase]:
+        """
+        Enable/disable webhooks for the given test case ids.
+        Args:
+            test_cases: a list of the test case ids
+            is_webhook_enabled: if enable then True else False
+            agent_id: The formatted CX Agent ID
+        Returns:
+            list of the updated test caess
+        """
+        test_cases_list = self._global_test_cases.list_test_cases(
+            agent_id=self.agent_id, include_conversation_turns=True)
+        updated_test_cases_list = []
+        for test_case in test_cases_list:
+            if test_case.name in test_case_ids:
+                test_case.test_case_conversation_turns = (
+                    self._toggle_webhooks_in_convturns(
+                        conv_turns=test_case.test_case_conversation_turns,
+                        is_webhook_enabled=is_webhook_enabled)
+                    )
+                updated_test_cases_list.append(test_case)
+
+        return updated_test_cases_list
+
+    @staticmethod
+    def _reverse_dict(dict_):
+        """Utility method that reverse the dict keys and values
+            Args:
+                dict_: a dictionary
+            Returns:
+                a reversed dictionary
+        """
+
+        return {v: k for k, v in dict_.items()}
+
+    def _get_resource_map(
+        self,
+        agent_id: str) -> Dict[str, Dict[str, str]]:
+        """This method map flows, pages, and intents
+            are in the dictionary format
+          Arg:
+            agent_id: The formatted CX Agent ID
+          Returns:
+            a dictionary of flows, pages, and intents
+        """
+
+        commons_config = {}
+        commons_config["agent_id"] = agent_id
+        dfcx_flows = flows.Flows(creds=self.creds, agent_id=agent_id)
+        dfcx_pages = pages.Pages(creds=self.creds)
+        dfcx_intents = intents.Intents(creds=self.creds, agent_id=agent_id)
+        dfcx_testcases = test_cases.TestCases(
+            creds=self.creds, agent_id=agent_id)
+        dfcx_intents_map = dfcx_intents.get_intents_map(agent_id=agent_id)
+        dfcx_flows_map = dfcx_flows.get_flows_map(agent_id=agent_id)
+        dfcx_pages_map = {}
+        for flow_id in dfcx_flows_map.keys():
+            dfcx_pages_map[flow_id] = dfcx_pages.get_pages_map(flow_id=flow_id)
+        commons_config["test_cases"] = dfcx_testcases
+        commons_config["flows"] = dfcx_flows
+        commons_config["pages"] = dfcx_pages
+        commons_config["intents"] = dfcx_intents
+        commons_config["flows_map"] = dfcx_flows_map
+        commons_config["pages_map"] = dfcx_pages_map
+        commons_config["intents_map"] = dfcx_intents_map
+        commons_config["flows_map_reverse"] = (
+            self._reverse_dict(dict_=commons_config["flows_map"]))
+        commons_config["intents_map_reverse"] = (
+            self._reverse_dict(dict_=commons_config["intents_map"]))
+        commons_config["pages_map_reverse"] = (
+            {f_id:self._reverse_dict(dict_=pages_map) \
+             for f_id, pages_map in dfcx_pages_map.items()})
+
+        return commons_config
+
+    def _get_new_test_config(
+        self,
+        source_resource_map: dict,
+        target_resource_map: dict,
+        test_case: types.TestCase) -> Union[types.TestConfig, None]:
+        """Update the test case types.test_config to the new types.test_config
+            Update the uuids of flow, page, intent by the display_name
+            If a TestConfig is found, it will return that,
+            otherwise an empty TestConfig is returned.
+          args:
+            source_resource_map: a source resource dictionary
+            target_resource_map: a target resource dictionary
+            test_case: types.test_config object
+          returns:
+            types.test_config
+        """
+        source_flow_id = (
+            # pylint: disable=W0212
+            source_resource_map["test_cases"]._get_flow_id_from_test_config(
+            test_case=test_case)
+        )
+        source_page_id = (
+            # pylint: disable=W0212
+            source_resource_map["test_cases"]._get_page_id_from_test_config(
+                test_case=test_case, flow_id=source_flow_id)
+        )
+        source_flow = self._get_flow_display_name_by_id(
+            source_flow_id, source_resource_map["flows_map"])
+        source_page = self._get_page_display_name_by_id(
+            source_flow_id, source_page_id, source_resource_map["pages_map"])
+        target_test_config = types.TestConfig()
+
+        if test_case.test_config.flow:
+            target_flow_id = self._get_flow_id_by_flow_name(
+                flow_name=source_flow,
+                flows_map=target_resource_map["flows_map_reverse"])
+            if target_flow_id:
+                target_test_config.flow = target_flow_id
+            else:
+                return None
+
+        elif test_case.test_config.page:
+            target_flow_id = self._get_flow_id_by_flow_name(
+                flow_name=source_flow,
+                flows_map=target_resource_map["flows_map_reverse"])
+            target_page_id = self._get_page_id_by_page_name(
+                flow_id=target_flow_id,
+                page_name=source_page,
+                pages_map=target_resource_map["pages_map_reverse"])
+            if target_page_id:
+                target_test_config.page = target_page_id
+            else:
+                return None
+
+        return target_test_config
+
+    def _get_flow_display_name_by_id(
+        self,
+        flow_id: str,
+        flows_map: Dict[str, str]) -> str:
+        """Attempt to get the Flow Display Name from the Flows Map
+            by the flow ID."""
+
+        flow = flows_map.get(flow_id, None)
+
+        return flow
+
+    def _get_page_display_name_by_id(
+        self,
+        flow_id: str,
+        page_id: str,
+        pages_map: Dict[str, Dict[str, str]]) -> str:
+        """Get the Page Display Name from the Pages Map based on the Page ID."""
+
+        page_map = pages_map.get(flow_id, None)
+        page = "START_PAGE"
+
+        if page_map:
+            page = page_map.get(page_id, None)
+
+        return page
+
+    def _get_flow_id_by_flow_name(
+        self,
+        flow_name: str,
+        flows_map: Dict[str, str]) -> str:
+        """Attempt to get the Flow id from the Flows Map
+            by the flow display name."""
+
+        return flows_map.get(flow_name, None)
+
+    def _get_page_id_by_page_name(
+        self,
+        flow_id: str,
+        page_name: str,
+        pages_map: Dict[str, Dict[str, str]]) -> str:
+        """Attempt to get the Page id from the Pages Map
+            by the Page display name."""
+
+        if not flow_id:
+            return None
+        page_map = pages_map.get(flow_id, None)
+        page_id = None
+        if  page_name in [
+            "Start Page",
+            "End Flow",
+            "End Session",
+            "End Flow With Failure",
+            "End Flow With Human Escalation",
+            "End Flow With Cancellation"
+            ]:
+            page_name = page_name.upper().replace(" ", "_")
+            page_id = f"{flow_id}/pages/{page_name}"
+        elif page_map:
+            page_id = page_map.get(page_name, None)
+
+        return page_id
+
+    def _get_intent_id_by_intent_name(
+        self,
+        intent_name: str,
+        intents_map: Dict[str, str]) -> str:
+        """Attempt to get the intent id from the Intents Map
+            by the intent's display name."""
+
+        return intents_map.get(intent_name, None)
+
+    def _get_intent_name_by_intent_id(
+        self,
+        intent_id: str,
+        intents_map: Dict[str, str]) -> str:
+        """Attempt to get the intent display name from the Intents Map
+            by the intent's display name."""
+
+        return intents_map.get(intent_id, None)
+
+    def _get_intent_id_from_virtual_agent_output(
+        self,
+        virtual_agent_output: types.ConversationTurn.VirtualAgentOutput,
+        target_resource_map: dict) -> Union[str, None]:
+        """Attempt to find the intent id from the source's agent
+        conversation_turn.virtual_agent_output triggered intent
+        display name using the target's intents map
+        """
+
+        source_intent = (
+          virtual_agent_output.triggered_intent.display_name
+        )
+        return self._get_intent_id_by_intent_name(
+            intent_name=source_intent,
+            intents_map=target_resource_map["intents_map_reverse"])
+
+    def _get_page_id_from_virtual_agent_output(
+        self,
+        virtual_agent_output: types.ConversationTurn.VirtualAgentOutput,
+        source_resource_map: dict,
+        target_resource_map: dict) -> Union[str, None]:
+        """Attempt to find the page id from the source's agent conversation turn
+          virtual agent output page id to the target's pages map
+        """
+
+        source_page = virtual_agent_output.current_page.display_name
+        source_page_id = virtual_agent_output.current_page.name
+        source_flow = self._get_flow_display_name_by_id(
+            flow_id = source_page_id.split("/pages/")[0],
+            flows_map = source_resource_map["flows_map"])
+        target_flow_id = self._get_flow_id_by_flow_name(
+            flow_name=source_flow,
+            flows_map=target_resource_map["flows_map_reverse"])
+        target_page_id = self._get_page_id_by_page_name(
+            flow_id=target_flow_id,
+            page_name=source_page,
+            pages_map=target_resource_map["pages_map_reverse"])
+
+        return target_page_id
+
+    def _get_intent_id_from_user_input(
+        self,
+        user_input: types.ConversationTurn.UserInput,
+        source_resource_map: dict,
+        target_resource_map: dict) -> Union[str, None]:
+        """Attempt to find the intent id from the source's agent
+          conversation turn user input intent id to the target's pages map
+        """
+
+        source_intent_id = user_input.input.intent
+        source_intent = self._get_intent_name_by_intent_id(
+            intent_id=source_intent_id,
+            intents_map=source_resource_map["intents_map"])
+        target_intent_id = self._get_intent_id_by_intent_name(
+            intent_name=source_intent,
+            intents_map=target_resource_map["intents_map_reverse"])
+
+        return target_intent_id
+
+    def _build_conversation_turns_for_target_agent(
+        self,
+        source_resource_map: dict,
+        target_resource_map: dict,
+        conversation_turns: types.ConversationTurn
+        ) -> Union[List[types.ConversationTurn], None]:
+        """ Update the uuids of the current page and intent
+            by display name in each conversation turn.
+          Args:
+            source_resource_map: a source resource dictionary
+            target_resource_map: a target commons_config dictionary
+            conversation_turns: types.ConversationTurn
+          Returns:
+            types.test_case_conversation_turns
+        """
+
+        new_conversation_turns = []
+        for conv_turn in conversation_turns:
+            user_input_intent = conv_turn.user_input
+            virtual_agent_output = conv_turn.virtual_agent_output
+            if user_input_intent.input.intent:
+                conv_turn.user_input.input.intent = (
+                    self._get_intent_id_from_user_input(
+                        user_input=user_input_intent,
+                        source_resource_map=source_resource_map,
+                        target_resource_map=target_resource_map
+                        )
+                )
+                if not conv_turn.user_input.input.intent:
+                    return None
+            if virtual_agent_output.current_page:
+                conv_turn.virtual_agent_output.current_page.name = (
+                    self._get_page_id_from_virtual_agent_output(
+                        virtual_agent_output=virtual_agent_output,
+                        source_resource_map=source_resource_map,
+                        target_resource_map=target_resource_map
+                        )
+                )
+                if not conv_turn.virtual_agent_output.current_page.name:
+                    return None
+            if virtual_agent_output.triggered_intent:
+                conv_turn.virtual_agent_output.triggered_intent.name = (
+                    self._get_intent_id_from_virtual_agent_output(
+                        virtual_agent_output=virtual_agent_output,
+                        target_resource_map=target_resource_map
+                    )
+                )
+                if not conv_turn.virtual_agent_output.triggered_intent.name:
+                    return None
+            new_conversation_turns.append(conv_turn)
+
+        return new_conversation_turns
+
+    def _log_missing_attributes(
+        self,
+        test_case: types.TestCase,
+        test_config: types.TestConfig,
+        conv_turns: List[types.ConversationTurn],
+        last_test_conv_turns: List[types.ConversationTurn]):
+        """This method is a logger that indicates which attributes are not
+        convertible in the test case.
+        Args:
+          test_config: types.TestConfig
+          conv_turns: List[types.ConversationTurn]
+          last_test_conv_turns: List[types.ConversationTurn]
+        """
+
+        if test_config is None:
+            logging.warning(
+                "-- ERROR --  DFCX Test case failed to convert -- "
+                f"test_case: {test_case.display_name} "
+                "Reason: test_config is None.")
+        if conv_turns is None:
+            logging.warning(
+                "-- ERROR --  DFCX Test case failed to convert -- "
+                f"test_case: {test_case.display_name} "
+                "Reason: test_conversation_turns is None.")
+        if last_test_conv_turns is None:
+            logging.warning(
+                "-- ERROR --  DFCX Test case failed to convert -- "
+                f"test_case: {test_case.display_name} "
+                "Reason: last_test_result is None.")
+
+    def convert_test_case(
+        self,
+        test_case: types.TestCase,
+        source_agent_id: str,
+        target_agent_id: str,
+        rate_limit: int = 5) -> Union[types.TestCase, None]:
+        """This method converts the test case from the source agent to the
+          target agent.
+          Note: In dfcx_scrapi.core.test_case.TestCases, it requires
+          list_test_cases(include_conversation_turns=True),
+          Otherwise, it will fail to convert.
+        Args:
+          test_case: types.TestCase
+          source_agent_id: str
+          target_agent_id: str
+          rate_limit: int
+        Returns:
+          types.TestCase or None if  fails
+        """
+
+        if not test_case.test_case_conversation_turns:
+            raise UserWarning(
+                "types.TestCase is not convertible if "
+                "test_case_conversation_turns is empty "
+                f"test case : {test_case.display_name}"
+            )
+        if not self.source_commons.get("agent_id") == source_agent_id:
+            self.source_commons = self._get_resource_map(
+                agent_id=source_agent_id)
+        if not self.target_commons.get("agent_id") == target_agent_id:
+            self.target_commons = self._get_resource_map(
+                agent_id=target_agent_id)
+
+        test_config = self._get_new_test_config(
+            self.source_commons, self.target_commons, test_case)
+        conv_turns = self._build_conversation_turns_for_target_agent(
+            self.source_commons,
+            self.target_commons,
+            test_case.test_case_conversation_turns)
+        last_test_conv_turns = self._build_conversation_turns_for_target_agent(
+            self.source_commons,
+            self.target_commons,
+            test_case.last_test_result.conversation_turns)
+
+        if not all([test_config, conv_turns, last_test_conv_turns]):
+            self._log_missing_attributes(
+                test_case, test_config, conv_turns, last_test_conv_turns)
+            return None
+
+        test_case.name = None
+        test_case.test_config = test_config
+        test_case.test_case_conversation_turns = conv_turns
+        test_case.last_test_result.conversation_turns = last_test_conv_turns
+        new_test_case = None
+        try:
+            new_test_case = self.target_commons["test_cases"].create_test_case(
+                test_case=test_case)
+            time.sleep(rate_limit)
+        except core_exceptions.InternalServerError as err:
+            logging.error(
+                f"-- ERROR -- InternalServerError caught on CX.detect -- {err}")
+            logging.error("test_case: %s", test_case.display_name)
+        except core_exceptions.ClientError as err:
+            logging.error(
+                f"-- ERROR -- ClientError caught on CX.detect -- {err}")
+            logging.error("test_case: %s", test_case.display_name)
+        logging.info(
+            "-- SUCCESS -- DFCX test_case converted -- "
+            f"test case: {new_test_case.display_name}")
+
+        return new_test_case
+
+    def _initialize_conversation_turn(self) -> types.ConversationTurn:
+        """it initializes the conversation turn.
+        Returns:
+          types.ConversationTurn
+        """
+        conversation_turn = types.ConversationTurn()
+        conversation_turn.user_input = (
+            conversation_turn.UserInput(input=types.QueryInput()))
+
+        return conversation_turn
+
+    def _set_query_input(
+        self,
+        send_obj: Dict,
+        conversation_turn: types.ConversationTurn
+        ) -> types.ConversationTurn:
+        """it sets the query input. In query input, it handle 4 types
+          of input: text, event, intent, dtmf if not, then it raises a warning.
+        Args:
+          conversation_turn: types.ConversationTurn
+        Returns:
+          types.ConversationTurn
+        """
+        if "dtmf" in send_obj:
+            finish_digit = send_obj.get("finish_digit", None)
+            if finish_digit:
+                finish_digit = str(finish_digit)
+            conversation_turn.user_input.input.dtmf = (
+                types.DtmfInput(
+                    digits=send_obj["dtmf"],
+                    finish_digit=finish_digit))
+        elif "event" in send_obj:
+            conversation_turn.user_input.input.event = (
+                types.EventInput(event=send_obj["event"]))
+        elif "text" in send_obj:
+            conversation_turn.user_input.input.text = (
+                types.TextInput(text=send_obj["text"]))
+        elif "intent" in send_obj:
+            conversation_turn.user_input.input.intent = (
+                types.IntentInput(intent=send_obj["intent"]))
+        else:
+            raise UserWarning(
+                "send_obj doesn't contain the proper query input type. "
+                "it must contains one of [text, event, intent, dtmf] as a key")
+
+        return conversation_turn
+
+    def _convert_send_objs_to_conv_turns(
+        self,
+        send_objs: List[Dict],
+        webhooks: bool) -> List[types.ConversationTurn]:
+        """This function converts the send_objs to conversation turns.
+          Iterate send_objs, set the params and a user_input of every turn.
+        Args:
+          send_objs: List[dict]
+          webhooks: bool
+        Returns:
+          List[types.ConversationTurn]
+        """
+        conversation_turns = []
+        for send_obj in send_objs:
+            conversation_turn = self._initialize_conversation_turn()
+            if "params" in send_obj:
+                param = Struct()
+                param.update(send_obj["params"])
+                conversation_turn.user_input.injected_parameters = param
+            conversation_turn.user_input.is_webhook_enabled = webhooks
+            conversation_turn = self._set_query_input(
+                send_obj=send_obj,
+                conversation_turn=conversation_turn)
+            conversation_turns.append(conversation_turn)
+        return conversation_turns
+
+    def _set_test_config(
+        self,
+        current_page) -> types.TestConfig:
+        """it sets the test config of the test case.
+        if current page is None, then it starts at Default Start Flow/Start page
+        if current page is a page id, then it starts at the specific page id
+        if current page is a flow id, then it starts at the flow start page
+        Args:
+          current_page: str
+        Returns:
+          types.TestConfig
+        """
+        test_config = types.TestConfig()
+        if current_page:
+            if "pages" in current_page:
+                test_config.page = current_page
+            else:
+                test_config.flow = current_page
+
+        return test_config
+
+    def create_test_case_by_send_objs(
+        self,
+        display_name: str,
+        send_objs: List[Dict],
+        webhooks: bool = False,
+        current_page: str = None,
+        tags: List[str] = None,
+        rate_limit: int = 5) ->  Union[types.TestCase, None]:
+        """it creates a test case by a list of send_objs. Each send_obj
+        consists of params and user input of every turn. Send_obj is commonly
+        used in the e2e testing, dfcx_scrapi.core.conversation.reply().
+        With this function, it can simultaneously create the test case while
+        running the e2e test.
+        Args:
+          display_name: a display_name of the test case
+          send_objs: a list of send_objs
+          webhooks: False by default, if True, then it enables webhooks
+          current_page: page_id of where the test case begins
+          tags: List[str]
+          rate_limit: int
+        Returns:
+          types.TestCase or None if fails
+        """
+        test_case = types.TestCase()
+        test_case.display_name = display_name
+        test_case.tags = tags
+        if current_page:
+            test_case.test_config = self._set_test_config(
+                current_page=current_page)
+        test_case.test_case_conversation_turns = (
+            self._convert_send_objs_to_conv_turns(
+                send_objs=send_objs, webhooks=webhooks)
+        )
+        response = None
+        try:
+            response = self._global_test_cases.create_test_case(
+                test_case=test_case)
+            time.sleep(rate_limit)
+        except core_exceptions.InternalServerError as err:
+            logging.error(
+                f"-- ERROR -- InternalServerError caught on CX.detect -- {err}")
+            logging.error("test_case: %s", test_case.display_name)
+        except core_exceptions.ClientError as err:
+            logging.error(
+                f"-- ERROR -- ClientError caught on CX.detect -- {err}")
+            logging.error("test_case: %s", test_case.display_name)
+
+        return response
diff --git a/tests/dfcx_scrapi/core/test_agents.py b/tests/dfcx_scrapi/core/test_agents.py
new file mode 100644
index 00000000..facdd413
--- /dev/null
+++ b/tests/dfcx_scrapi/core/test_agents.py
@@ -0,0 +1,101 @@
+"""Test Class for Agent Methods in SCRAPI."""
+
+# pylint: disable=redefined-outer-name
+
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+from unittest.mock import patch
+
+from google.cloud.dialogflowcx_v3beta1 import types
+from dfcx_scrapi.core.agents import Agents
+
+
+@pytest.fixture
+def mock_agent_obj():
+    return types.Agent(
+        name="projects/mock-project/locations/global/agents/mock-agent-1234",
+        display_name="Mock Agent",
+        default_language_code="en",
+        time_zone="America/Chicago",
+    )
+
+
+@pytest.fixture
+def mock_agent_obj_kwargs(mock_agent_obj):
+    mock_agent_obj.description = "This is a Mock Agent description."
+    mock_agent_obj.enable_stackdriver_logging = True
+
+    return mock_agent_obj
+
+
+@patch("dfcx_scrapi.core.agents.services.agents.AgentsClient")
+def test_create_agent_simple_default_region_no_kwargs(
+    mock_client, mock_agent_obj
+):
+    mock_client.return_value.create_agent.return_value = mock_agent_obj
+
+    agents = Agents()
+    res = agents.create_agent(
+        project_id="mock-project", display_name="Mock Agent"
+    )
+
+    assert isinstance(res, types.Agent)
+    assert res.display_name == "Mock Agent"
+
+
+@patch("dfcx_scrapi.core.agents.services.agents.AgentsClient")
+def test_create_agent_with_extra_kwargs(mock_client, mock_agent_obj_kwargs):
+    mock_client.return_value.create_agent.return_value = mock_agent_obj_kwargs
+
+    agents = Agents()
+    res = agents.create_agent(
+        project_id="mock-project",
+        display_name="Mock Agent",
+        description="This is a Mock Agent description.",
+        enable_stackdriver_logging=True,
+    )
+
+    assert isinstance(res, types.Agent)
+    assert res == mock_agent_obj_kwargs
+
+
+@patch("dfcx_scrapi.core.agents.services.agents.AgentsClient")
+def test_create_agent_from_obj(mock_client, mock_agent_obj):
+    mock_client.return_value.create_agent.return_value = mock_agent_obj
+
+    agents = Agents()
+    res = agents.create_agent(project_id="mock-project", obj=mock_agent_obj)
+
+    assert isinstance(res, types.Agent)
+    assert res.display_name == "Mock Agent"
+
+
+@patch("dfcx_scrapi.core.agents.services.agents.AgentsClient")
+def test_create_agent_from_obj_with_kwargs(mock_client, mock_agent_obj_kwargs):
+    mock_client.return_value.create_agent.return_value = mock_agent_obj_kwargs
+
+    agents = Agents()
+    res = agents.create_agent(
+        project_id="mock-project",
+        obj=mock_agent_obj_kwargs,
+        description="This is a Mock Agent description.",
+        enable_stackdriver_logging=True,
+    )
+
+    assert isinstance(res, types.Agent)
+    assert res.description == "This is a Mock Agent description."
+
+# Next, Restore Agent
diff --git a/tests/dfcx_scrapi/core/test_test_cases.py b/tests/dfcx_scrapi/core/test_test_cases.py
new file mode 100644
index 00000000..3016e3aa
--- /dev/null
+++ b/tests/dfcx_scrapi/core/test_test_cases.py
@@ -0,0 +1,488 @@
+"""Test Class for Test Cases in SCRAPI."""
+
+# pylint: disable=redefined-outer-name
+# pylint: disable=protected-access
+
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+import pandas as pd
+from unittest.mock import patch
+from dfcx_scrapi.core.test_cases import TestCases
+from google.cloud.dialogflowcx_v3beta1 import types
+from google.cloud.dialogflowcx_v3beta1.services import test_cases
+
+
+@pytest.fixture
+def test_config():
+    agent_id = "projects/mock-test/locations/global/agents/a1s2d3f4"
+    flow_id = f"{agent_id}/flows/00000000-0000-0000-0000-000000000000"
+    page_id = f"{flow_id}/pages/mock-page-1234"
+    other_flow_id = f"{agent_id}/flows/other1234"
+    other_page_id = f"{other_flow_id}/pages/other1234"
+    test_case_id = f"{agent_id}/testCases/1234"
+    test_case_result_id = f"{test_case_id}/results/9876"
+    intent_id = f"{agent_id}/intents/12345678"
+    flows_map = {flow_id: "MOCK FLOW"}
+    pages_map = {
+        flow_id: {page_id: "MOCK PAGE"},
+        other_flow_id: {other_page_id: "OTHER PAGE"},
+    }
+
+    return {
+        "agent_id": agent_id,
+        "flow_id": flow_id,
+        "page_id": page_id,
+        "other_flow_id": other_flow_id,
+        "other_page_id": other_page_id,
+        "test_case_id": test_case_id,
+        "test_case_result_id": test_case_result_id,
+        "intent_id": intent_id,
+        "flows_map": flows_map,
+        "pages_map": pages_map,
+    }
+
+
+@pytest.fixture
+def mock_tc_obj_turns(test_config):
+    return types.TestCase(
+        name=test_config["test_case_id"],
+        tags=["#required", "#testing"],
+        display_name="mock test case object",
+        notes="This is a note for the mock test case",
+        test_config=types.TestConfig(flow=test_config["flow_id"]),
+        test_case_conversation_turns=[
+            types.ConversationTurn(
+                user_input=types.ConversationTurn.UserInput(
+                    input=types.QueryInput(
+                        text=types.TextInput(text="Hello"), language_code="en"
+                    ),
+                    injected_parameters={"my_key": "my_value"},
+                    is_webhook_enabled=False,
+                    enable_sentiment_analysis=False,
+                )
+            ),
+            types.ConversationTurn(
+                virtual_agent_output=types.ConversationTurn.VirtualAgentOutput(
+                    triggered_intent=types.Intent(
+                        name=test_config["intent_id"]
+                    ),
+                    text_responses=[
+                        types.ResponseMessage.Text(
+                            text=["Hi, how can I help you today?"]
+                        )
+                    ],
+                )
+            ),
+        ],
+        last_test_result=types.TestCaseResult(
+            name=test_config["test_case_result_id"],
+            test_result=types.TestResult(value=0),
+        ),
+    )
+
+
+@pytest.fixture
+def mock_tc_obj_no_turns(test_config):
+    return types.TestCase(
+        name=test_config["test_case_id"],
+        tags=["#required", "#testing"],
+        display_name="mock test case object",
+        notes="This is a note for the mock test case",
+        test_config=types.TestConfig(flow=test_config["flow_id"]),
+        last_test_result=types.TestCaseResult(
+            name=test_config["test_case_result_id"],
+            test_result=types.TestResult(value=0),
+        ),
+    )
+
+
+@pytest.fixture
+def mock_tc_df(test_config):
+    data = {
+        "display_name": ["Mock Test 1", "Mock Test 2", "Mock Test 3"],
+        "id": [
+            f"{test_config['agent_id']}/testCases/12345",
+            f"{test_config['agent_id']}/testCases/54321",
+            f"{test_config['agent_id']}/testCases/67890",
+        ],
+        "short_id": ["12345", "54321", "67890"],
+        "tags": [["tag1", "tag2"], ["tag3", "tag4"], ["tag5", "tag6"]],
+        "creation_time": ["2023-01-01", "2023-01-02", "2023-01-03"],
+        "start_flow": ["Mock Flow", "Mock Flow", "Default Start Flow"],
+        "start_page": ["Mock Page", "Mock Page", "START_PAGE"],
+        "passed": [True, False, True],
+        "test_time": ["10:00", "11:00", "12:00"],
+    }
+
+    return pd.DataFrame(data)
+
+
+@pytest.fixture
+def mock_updated_tc_obj(mock_tc_obj_turns):
+    mock_tc_obj_turns.display_name = "mock test case object updated"
+
+    return mock_tc_obj_turns
+
+
+@pytest.fixture
+def mock_list_tc_pager(mock_tc_obj_turns):
+    return test_cases.pagers.ListTestCasesPager(
+        test_cases.TestCasesClient.list_test_cases,
+        types.test_case.ListTestCasesRequest(),
+        types.test_case.ListTestCasesResponse(test_cases=[mock_tc_obj_turns]),
+    )
+
+
+@pytest.fixture
+def mock_list_tc_pager_no_turns(mock_tc_obj_no_turns):
+    return test_cases.pagers.ListTestCasesPager(
+        test_cases.TestCasesClient.list_test_cases,
+        types.test_case.ListTestCasesRequest(),
+        types.test_case.ListTestCasesResponse(
+            test_cases=[mock_tc_obj_no_turns]
+        ),
+    )
+
+
+# Private Methods
+def test_convert_test_result_to_string(mock_tc_obj_turns):
+    tests = [(0, "TEST_RESULT_UNSPECIFIED"), (1, "PASSED"), (2, "FAILED")]
+    tc = TestCases()
+
+    for int_value, str_value in tests:
+        mock_tc_obj_turns.last_test_result.test_result = types.TestResult(
+            value=int_value
+        )
+        res = tc._convert_test_result_to_string(mock_tc_obj_turns)
+        assert res == str_value
+        assert isinstance(res, str)
+
+
+def test_convert_test_result_to_bool(mock_tc_obj_turns):
+    tests = [(0, None), (1, True), (2, False)]
+    tc = TestCases()
+    for int_value, res_value in tests:
+        if int_value in [1, 2]:
+            mock_tc_obj_turns.last_test_result.test_result = types.TestResult(
+                value=int_value
+            )
+            res = tc._convert_test_result_to_bool(mock_tc_obj_turns)
+            assert res == res_value
+            assert isinstance(res_value, bool)
+        else:
+            mock_tc_obj_turns.last_test_result.test_result = types.TestResult(
+                value=int_value
+            )
+            res = tc._convert_test_result_to_bool(mock_tc_obj_turns)
+            assert res == res_value
+            assert res_value is None
+
+
+def test_get_flow_id_from_test_config(mock_tc_obj_turns, test_config):
+    tests = [
+        ("flow", test_config["flow_id"]),
+        ("page", test_config["page_id"]),
+        (None, ""),
+    ]
+    tc = TestCases()
+    for str_value, test_case_id in tests:
+        if str_value == "flow":
+            mock_tc_obj_turns.test_config.flow = test_case_id
+        elif str_value == "page":
+            mock_tc_obj_turns.test_config.page = test_case_id
+        else:
+            mock_tc_obj_turns.test_config = types.TestConfig()
+
+        res = tc._get_flow_id_from_test_config(mock_tc_obj_turns)
+        assert isinstance(res, str)
+        assert res == test_config["flow_id"]
+
+
+def test_get_page_id_from_test_config(mock_tc_obj_turns, test_config):
+    tests = [
+        ("page", test_config["page_id"]),
+        ("start_page", f"{test_config['flow_id']}/pages/START_PAGE"),
+    ]
+    tc = TestCases()
+    for str_value, page_id in tests:
+        if str_value == "page":
+            mock_tc_obj_turns.test_config.page = page_id
+            res = tc._get_page_id_from_test_config(
+                mock_tc_obj_turns, test_config["flow_id"]
+            )
+            assert isinstance(str_value, str)
+            assert res == page_id
+
+        elif str_value == "start_page":
+            mock_tc_obj_turns.test_config = types.TestConfig()
+            res = tc._get_page_id_from_test_config(
+                mock_tc_obj_turns, test_config["flow_id"]
+            )
+            assert isinstance(str_value, str)
+            assert res == page_id
+
+
+def test_get_page_display_name_flow_exists(test_config):
+    tc = TestCases()
+
+    res = tc._get_page_display_name(
+        test_config["flow_id"], test_config["page_id"], test_config["pages_map"]
+    )
+    assert isinstance(res, str)
+    assert res == "MOCK PAGE"
+
+
+def test_get_page_display_name_flow_exists_no_page_map(test_config):
+    tc = TestCases()
+
+    res = tc._get_page_display_name(
+        test_config["other_flow_id"], None, test_config["pages_map"]
+    )
+    assert res is None
+
+
+def test_get_page_display_name_flow_does_not_exist(test_config):
+    tc = TestCases()
+
+    res = tc._get_page_display_name(
+        f"{test_config['agent_id']}/flows/doesnt-exist-1234",
+        None,
+        test_config["pages_map"],
+    )
+    assert isinstance(res, str)
+    assert res == "START_PAGE"
+
+
+def test_process_test_case(mock_tc_obj_turns, test_config):
+    cols = [
+        "display_name",
+        "id",
+        "short_id",
+        "tags",
+        "creation_time",
+        "start_flow",
+        "start_page",
+        "passed",
+        "test_time",
+    ]
+    tc = TestCases()
+
+    res = tc._process_test_case(
+        mock_tc_obj_turns, test_config["flows_map"], test_config["pages_map"]
+    )
+    assert isinstance(res, pd.DataFrame)
+    assert set(res.columns.to_list()) == set(cols)
+
+
+@patch("dfcx_scrapi.core.test_cases.TestCases.batch_run_test_cases")
+def test_retest_cases(mock_batch_run, mock_tc_df):
+    cols = [
+        "display_name",
+        "id",
+        "short_id",
+        "tags",
+        "creation_time",
+        "start_flow",
+        "start_page",
+        "passed",
+        "test_time",
+    ]
+
+    mock_batch_run.return_value.batch_run_test_cases.return_value = mock_tc_df
+
+    retest_ids = mock_tc_df.id.to_list()[:1]
+    tc = TestCases()
+    res = tc._retest_cases(mock_tc_df, retest_ids)
+    assert isinstance(res, pd.DataFrame)
+    assert set(res.columns.to_list()) == set(cols)
+
+
+# List Test Cases
+@patch("dfcx_scrapi.core.test_cases.services.test_cases.TestCasesClient")
+def test_list_test_cases_agent_id_not_in_instance(
+    mock_client, mock_tc_obj_turns
+):
+    mock_client.return_value.list_test_cases.return_value = [mock_tc_obj_turns]
+
+    tc = TestCases()
+
+    with pytest.raises(AttributeError):
+        _ = tc.list_test_cases()
+
+
+@patch("dfcx_scrapi.core.test_cases.services.test_cases.TestCasesClient")
+def test_list_test_cases_agent_id_in_instance(
+    mock_client, mock_list_tc_pager, test_config
+):
+    mock_client.return_value.list_test_cases.return_value = mock_list_tc_pager
+
+    tc = TestCases(agent_id=test_config["agent_id"])
+    res = tc.list_test_cases()
+
+    assert isinstance(res, list)
+    assert isinstance(res[0], types.TestCase)
+
+
+@patch("dfcx_scrapi.core.test_cases.services.test_cases.TestCasesClient")
+def test_list_test_cases_agent_id_in_method(
+    mock_client, mock_list_tc_pager_no_turns, test_config
+):
+    mock_client.return_value.list_test_cases.return_value = (
+        mock_list_tc_pager_no_turns
+    )
+
+    tc = TestCases()
+    res = tc.list_test_cases(test_config["agent_id"])
+
+    assert isinstance(res, list)
+    assert isinstance(res[0], types.TestCase)
+    assert res[0].test_case_conversation_turns == ""
+
+
+@patch("dfcx_scrapi.core.test_cases.services.test_cases.TestCasesClient")
+def test_list_test_cases_include_conversation_turns(
+    mock_client, mock_list_tc_pager, test_config
+):
+    mock_client.return_value.list_test_cases.return_value = mock_list_tc_pager
+
+    tc = TestCases()
+    res = tc.list_test_cases(
+        test_config["agent_id"], include_conversation_turns=True
+    )
+
+    assert isinstance(res, list)
+    assert isinstance(res[0], types.TestCase)
+    assert res[0].test_case_conversation_turns != ""
+
+
+# Update Test Cases
+
+
+@patch("dfcx_scrapi.core.test_cases.services.test_cases.TestCasesClient")
+def test_update_test_case_no_args(mock_client, mock_tc_obj_turns):
+    mock_client.return_value.update_test_case.return_value = mock_tc_obj_turns
+
+    tc = TestCases()
+
+    # Assertions
+    with pytest.raises(ValueError):
+        _ = tc.update_test_case()
+
+
+@patch("dfcx_scrapi.core.test_cases.services.test_cases.TestCasesClient")
+def test_update_test_case_kwargs_only(mock_client, mock_tc_obj_turns):
+    mock_client.return_value.update_test_case.return_value = mock_tc_obj_turns
+
+    tc = TestCases()
+
+    # Assertions
+    with pytest.raises(ValueError):
+        _ = tc.update_test_case(display_name="mock test case object update")
+
+
+@patch("dfcx_scrapi.core.test_cases.services.test_cases.TestCasesClient")
+def test_update_test_case_id_only(mock_client, test_config, mock_tc_obj_turns):
+
+    mock_client.return_value.update_test_case.return_value = mock_tc_obj_turns
+
+    tc = TestCases()
+
+    # Assertions
+    with pytest.raises(ValueError):
+        _ = tc.update_test_case(test_case_id=test_config["test_case_id"])
+
+
+@patch("dfcx_scrapi.core.test_cases.services.test_cases.TestCasesClient")
+def test_update_test_case_obj_only(mock_client, mock_tc_obj_turns):
+    mock_client.return_value.update_test_case.return_value = mock_tc_obj_turns
+
+    tc = TestCases()
+    result = tc.update_test_case(obj=mock_tc_obj_turns)
+
+    # Assertions
+    assert result.display_name == "mock test case object"
+    assert result == mock_tc_obj_turns
+
+
+@patch("dfcx_scrapi.core.test_cases.services.test_cases.TestCasesClient")
+def test_update_test_case_obj_only_empty_name(mock_client, mock_tc_obj_turns):
+    mock_tc_obj_turns.name = ""
+
+    mock_client.return_value.update_test_case.return_value = mock_tc_obj_turns
+
+    tc = TestCases()
+
+    # Assertions
+    with pytest.raises(ValueError):
+        _ = tc.update_test_case(obj=mock_tc_obj_turns)
+
+
+@patch("dfcx_scrapi.core.test_cases.services.test_cases.TestCasesClient")
+def test_update_test_case_with_obj_and_kwargs(
+    mock_client, mock_tc_obj_turns, mock_updated_tc_obj
+):
+
+    mock_client.return_value.update_test_case.return_value = mock_updated_tc_obj
+
+    tc = TestCases()
+    result = tc.update_test_case(
+        obj=mock_tc_obj_turns, display_name="mock test case object updated"
+    )
+
+    # Assertions
+    assert result.display_name == mock_updated_tc_obj.display_name
+    assert result == mock_updated_tc_obj
+
+
+@patch("dfcx_scrapi.core.test_cases.services.test_cases.TestCasesClient")
+def test_update_test_case_with_id_and_kwargs(
+    mock_client, test_config, mock_tc_obj_turns
+):
+
+    tc = TestCases()
+
+    mock_client.return_value.get_test_case.return_value = mock_tc_obj_turns
+    mock_client.return_value.update_test_case.return_value = mock_tc_obj_turns
+
+    result = tc.update_test_case(
+        test_case_id=test_config["test_case_id"],
+        display_name="mock test case object update",
+    )
+
+    # Assertions
+    assert result.display_name == "mock test case object update"
+
+
+@patch("dfcx_scrapi.core.test_cases.services.test_cases.TestCasesClient")
+def test_update_test_case_with_obj_id_and_kwargs(
+    mock_client, test_config, mock_tc_obj_turns
+):
+    """For this case, kwargs should be ignored in favor of the obj."""
+
+    tc = TestCases()
+
+    mock_client.return_value.get_test_case.return_value = mock_tc_obj_turns
+    mock_client.return_value.update_test_case.return_value = mock_tc_obj_turns
+
+    result = tc.update_test_case(
+        test_case_id=test_config["test_case_id"],
+        obj=mock_tc_obj_turns,
+        display_name="mock test case object update",
+    )
+
+    # Assertions
+    assert result.display_name == "mock test case object"
+    assert result == mock_tc_obj_turns
diff --git a/tests/test_agents.py b/tests/test_agents.py
deleted file mode 100644
index 44bf2f1f..00000000
--- a/tests/test_agents.py
+++ /dev/null
@@ -1,150 +0,0 @@
-"""Test Class for Agent Functions in SCRAPI lib."""
-
-# Copyright 2023 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-import pytest
-import time
-from datetime import datetime
-
-from google.cloud.dialogflowcx_v3beta1 import types
-from src.dfcx_scrapi.core import agents
-from src.dfcx_scrapi.core import operations
-
-today_time = datetime.now().strftime("%d%m%Y_%H%M%S")
-AGENT_NAME = f"DFCX SCRAPI - TEMP TEST AGENT {today_time}"
-
-pytest.temp_agent = None
-
-@pytest.mark.unit
-def test_create_agent(creds, project_id):
-    """Tests the SCRAPI method create_agent from core/agents.py"""
-    a = agents.Agents(creds_path=creds)
-    pytest.temp_agent = a.create_agent(
-        project_id, AGENT_NAME
-    )
-
-    assert isinstance(pytest.temp_agent, types.Agent)
-    assert pytest.temp_agent.display_name == AGENT_NAME
-
-    return pytest.temp_agent
-
-@pytest.mark.unit
-def test_restore_agent(creds, gcs_bucket):
-    """Tests the SCRAPI method restore_agent from core/agents.py"""
-    a = agents.Agents(creds_path=creds)
-    ops = operations.Operations(creds_path=creds)
-
-    lro = a.restore_agent(
-        pytest.temp_agent.name,
-        gcs_bucket_uri=f"gs://{gcs_bucket}/sample_agent.blob"
-    )
-
-    time.sleep(4)
-    res = ops.get_lro(lro)
-    print(res)
-
-    assert res["done"]
-
-@pytest.mark.unit
-def test_get_agent(creds):
-    """Tests the SCRAPI method get_agent from core/agents.py"""
-    a = agents.Agents(creds_path=creds)
-    agent = a.get_agent(pytest.temp_agent.name)
-
-    assert isinstance(agent, types.Agent)
-    assert pytest.temp_agent.display_name == AGENT_NAME
-
-@pytest.mark.unit
-def test_get_agent_by_display_name(creds, project_id):
-    """Test that we can retrieve agent by display name"""
-    a = agents.Agents(creds_path=creds)
-    region = pytest.temp_agent.name.split("/")[3]
-    agent = a.get_agent_by_display_name(
-        project_id=project_id,
-        display_name=pytest.temp_agent.display_name,
-        region=region
-    )
-
-    assert isinstance(agent, types.Agent)
-    assert agent.display_name == AGENT_NAME
-
-@pytest.mark.unit
-def test_get_agent_by_display_name_dupe(creds, project_id, caplog):
-    """Test this function when providing a display name that is ambiguous and
-    exists in multiple regions"""
-    a = agents.Agents(creds_path=creds)
-    with caplog.at_level(logging.WARNING):
-        a.get_agent_by_display_name(
-            project_id=project_id,
-            display_name="SCRAPI - CI Duplicate")
-
-    assert "Found multiple agents with the display name" in caplog.text
-
-@pytest.mark.unit
-def test_get_agent_by_display_name_approx(creds, project_id, caplog):
-    """test this function with approximate matching of agent name."""
-    a = agents.Agents(creds_path=creds)
-    with caplog.at_level(logging.WARNING):
-        a.get_agent_by_display_name(
-            project_id=project_id,
-            display_name="sCRAPI - CI Duplicate"
-        )
-
-    assert "display_name is case-sensitive" in caplog.text
-
-@pytest.mark.unit
-def test_export_agent(creds, gcs_bucket):
-    """Tests the SCRAPI method export_agent from core/agents.py"""
-    a = agents.Agents(creds_path=creds)
-    ops = operations.Operations(creds_path=creds)
-    lro = a.export_agent(
-        pytest.temp_agent.name,
-        f"gs://{gcs_bucket}/testing_export.json"
-    )
-
-    time.sleep(4)
-
-    assert ops.get_lro(lro)["done"]
-
-@pytest.mark.unit
-def test_update_agent(creds):
-    """Tests the SCRAPI method update_agent from core/agents.py"""
-    a = agents.Agents(creds_path=creds)
-    temp_name = AGENT_NAME + "_UPDATED"
-    res = a.update_agent(
-        pytest.temp_agent.name, display_name=temp_name
-    )
-
-    assert res.display_name == temp_name
-
-@pytest.mark.unit
-def test_delete_agent(creds):
-    """Tests the SCRAPI method delete_agent from core/agents.py"""
-    a = agents.Agents(creds_path=creds)
-    res = a.delete_agent(pytest.temp_agent.name)
-
-    print(res)
-
-    assert pytest.temp_agent.name in res
-
-@pytest.mark.unit
-def test_list_agents(creds, project_id):
-    """Tests the SCRAPI method list_agents from core/agents.py"""
-    a = agents.Agents(creds_path=creds)
-    location_id = f"projects/{project_id}/locations/global"
-    all_agents = a.list_agents(location_id)
-
-    assert isinstance(all_agents[0], types.agent.Agent)
diff --git a/tests/test_search_util.py b/tests/test_search_util.py
deleted file mode 100644
index e0dc913d..00000000
--- a/tests/test_search_util.py
+++ /dev/null
@@ -1,40 +0,0 @@
-"""Unit Tests for Search Util Class"""
-# Copyright 2023 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import pytest
-from src.dfcx_scrapi.tools import search_util
-
-@pytest.mark.unit
-def test_instantiate_search_util(creds):
-    scrapi_search = search_util.SearchUtil(creds_path=creds)
-
-    assert isinstance(scrapi_search, search_util.SearchUtil)
-    assert scrapi_search.creds_path == creds
-
-@pytest.mark.unit
-def test_get_agent_fulfillments(creds, agent_id):
-    scrapi_search = search_util.SearchUtil(creds_path=creds)
-    df = scrapi_search.get_agent_fulfillment_message_df(agent_id)
-    assert set(df.columns) == {
-            'flow_name',
-            'page_name',
-            'parameter_name',
-            'event',
-            'route_group_name',
-            'intent',
-            'condition',
-            'response_message',
-            'conditional_cases',
-            'response_type'}