cache chroma_db, fine-tuned-embeddings, etc.

.gitattributes

@@ -1,2 +1,3 @@
 raw_documents/** filter=lfs diff=lfs merge=lfs -text
 models/** filter=lfs diff=lfs merge=lfs -text
+database/** filter=lfs diff=lfs merge=lfs -text

database/mock_qna.db

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c380902975056aca9cbc32ff2948725fc9901a59ae01e2cf1634f475e1c889f
+size 8192

database/mock_qna_source.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b604288137e94da640f1e5a88900390084eba746508cd7257dbcdba8cbe67f32
+size 2701

models/chroma_db/9b83ffa5-f19f-42a5-b97f-969906ca1a4f/data_level0.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:47c3649b50c934105cb86707e622c6c59af2c8a247948ab986ebfbbb7041def5
+oid sha256:d37c44e68139700bd5cfddc1f64e610ae6d974b559548175754eac7df1ac8065
 size 1676000

models/chroma_db/9b83ffa5-f19f-42a5-b97f-969906ca1a4f/length.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a8cbc166a0aba7021ff88582e00e169a953dfccffe96f92a59b2c9a9153419e4
+oid sha256:fc19b1997119425765295aeab72d76faa6927d4f83985d328c26f20468d6cc76
 size 4000

models/chroma_db/chroma.sqlite3

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:098b75a211dfc48c60fbc7e0b8f90ea29c08760f6fde4e1d65a5f67c63738d59
-size 11952128
+oid sha256:ffe0f3842c7835daddb5c11b8f70bb5dc6352abcb91c11f30c53a49d8c6d540c
+size 23486464

models/fine-tuned-embeddings/1_Pooling/config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfd7e0a022036d0ffa0f998824a918247d5a7473d968cdc92e318fd04098e682
+size 270

models/fine-tuned-embeddings/README.md

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:399b632f51b91d4c9c104040c22f21cfb73e671c14975f78af346a238ccd43f1
+size 2544

models/fine-tuned-embeddings/config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13582bcf2effc85b7bf3d3f5532e686bc1c9ce86bb009d10f0ec33cbe92299dd
+size 706

models/fine-tuned-embeddings/config_sentence_transformers.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:940d5f50db195fa6e5e6a4f122c095f77880de259d74b14a65779ed48bdd7c56
+size 124

models/fine-tuned-embeddings/eval/Information-Retrieval_evaluation_results.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:858293a2164d38e8abf7e46e701d54a46acc966b5b0ee71355693d339ecc648f
+size 6519

models/fine-tuned-embeddings/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc291c956c8b74f5f8336412568855a17957e71ecb95d0dc1b7429aadee084f4
+size 133462128

models/fine-tuned-embeddings/modules.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84e40c8e006c9b1d6c122e02cba9b02458120b5fb0c87b746c41e0207cf642cf
+size 349

models/fine-tuned-embeddings/sentence_bert_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84e39fda68ccbff05bfa723ae9c0e70e23e2ec373b76e0f8c6e71af72a693cbf
+size 52

models/fine-tuned-embeddings/special_tokens_map.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d5b662e421ea9fac075174bb0688ee0d9431699900b90662acd44b2a350503a
+size 695

models/fine-tuned-embeddings/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91f1def9b9391fdabe028cd3f3fcc4efd34e5d1f08c3bf2de513ebb5911a1854
+size 711649

models/fine-tuned-embeddings/tokenizer_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b29c7bfc889e53b36d9dd3e686dd4300f6525110eaa98c76a5dafceb2029f53
+size 1242

models/fine-tuned-embeddings/vocab.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07eced375cec144d27c900241f3e339478dec958f92fddbc551f295c992038a3
+size 231508

notebooks/create_mock_qna.ipynb

@@ -0,0 +1,311 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "23b388fd-2a24-48cf-9cf8-fd5cd19257d8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import sqlite3\n",
+    "\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1edf4aeb-bcb3-42f6-b3f7-9f9543b5ab12",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "04969710-e7b7-4017-8eb7-fc50ee99df6f",
+   "metadata": {},
+   "source": [
+    "### Parameters"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7cf683dc-93fc-4497-9641-75f0a3c1ba12",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "db_path = \"../database/mock_qna.db\"\n",
+    "nature_of_run = \"new\" if not os.path.exists(db_path) else \"existing\"\n",
+    "\n",
+    "qna_path = \"../database/mock_qna_source.csv\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b6cca63e-021b-4950-ab9f-0e3170194c35",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(f\"nature of run: `{nature_of_run}`\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "add28f2e-d695-42a5-97e5-3647dd768dce",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "qna_data = pd.read_csv( qna_path )\n",
+    "qna_cols = list(qna_data.columns)\n",
+    "qna_data.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "26fa3a67-71d9-4410-b0ea-9c1e08ca2f51",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "qna_data[:3]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2a20c4ee-ae53-4582-a660-54e40f8f1dd5",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1167bb3a-97fd-48b1-a0a9-eab6e4d54245",
+   "metadata": {},
+   "source": [
+    "### Initialize database connection & resources"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "095b8a2e-c3cb-4c09-b49d-ccb5df8467b0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "con = sqlite3.connect(db_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f2668a87-be3c-464d-a4ad-4e40590cbd0c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cur = con.cursor()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4437d3cb-b92b-40ef-b030-b7fb4499d0e7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if nature_of_run == \"new\":\n",
+    "    qna_cols_str = \", \".join(qna_cols)\n",
+    "    cur.execute(f\"\"\"CREATE TABLE qna_tbl (\n",
+    "                        {qna_cols_str}\n",
+    "                    )\n",
+    "                \"\"\")\n",
+    "    print(\"created table `qna_tbl`\")\n",
+    "    print(f\"columns for `qna_tbl` are {qna_cols_str}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a6153892-4d8b-487e-bd1d-05577ef1fcb5",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cdc0a81b-fb0a-46fa-9646-1a78c2781f02",
+   "metadata": {},
+   "source": [
+    "#### Test fetching empty table"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dce53aec-680e-4f0f-b6eb-71efe902231a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = cur.execute(\"SELECT chapter, question FROM qna_tbl\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "506527e2-4d6d-4817-bdaf-9a31fec3b006",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res.fetchone()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "69f74ed2-a1da-410a-b759-d334fcf37851",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e82debcf-c3e4-4c93-8e59-2c73ead63adc",
+   "metadata": {},
+   "source": [
+    "#### Test ingesting one record of data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e239f941-d19b-4400-acac-8a45b7b50fcc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = qna_data.values.tolist()\n",
+    "q_mark_list = [\"?\"] * len(qna_cols)\n",
+    "q_mark_str = \"(\" + \", \".join(q_mark_list) + \")\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "93b7130b-b007-4359-a0a2-bfe5fb7ddba2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cur.executemany(f\"INSERT INTO qna_tbl VALUES {q_mark_str}\", data[:1])\n",
+    "con.commit()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5f01dac9-c9f5-4536-85d4-667abd8f178d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "bf8b1f1d-08fd-4a07-9489-58ef14b8439d",
+   "metadata": {},
+   "source": [
+    "#### Test fetching one record of data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "26206800-54c0-495e-bf8f-5958421eddca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = cur.execute(\"SELECT chapter, question FROM qna_tbl\")\n",
+    "res.fetchone()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "54722955-7e72-4723-88ca-a0dbee361934",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "54ec1451-fe61-4a92-9148-d4a3d05aeed8",
+   "metadata": {},
+   "source": [
+    "#### Clean up and ingest full Q&A data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "64131faf-b2e7-4e70-8547-762a09ed2ad2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cur.execute(\"DELETE FROM qna_tbl\")\n",
+    "con.commit()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "06d55885-50b1-4c23-a364-1fb8fa4f4b36",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cur.executemany(f\"INSERT INTO qna_tbl VALUES {q_mark_str}\", data)\n",
+    "con.commit()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9e2a3d06-a077-4b32-8fce-600b3577cad9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = cur.execute(\"SELECT COUNT(*) FROM qna_tbl\")\n",
+    "res.fetchone()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9256ad33-f70a-482c-801e-01b5a52e8261",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/fine-tune-and-persist-vector-store.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "10638b27-aa20-43a6-bee6-b7b97f64996e",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/fine-tuning-embedding-model.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "ca2c990f-5215-4ab9-8143-1d79db28edc6",
    "metadata": {},
    "outputs": [],
@@ -16,7 +16,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "id": "2c535ad7-7846-4bef-8ba8-33e182490c3d",
    "metadata": {},
    "outputs": [],
@@ -30,7 +30,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": null,
    "id": "25f0c7a3-c52f-4417-aec8-4b6cfbf7a1b5",
    "metadata": {},
    "outputs": [],
@@ -44,7 +44,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": null,
    "id": "62f4d7f0-748a-405e-b5f1-6520fd02bedc",
    "metadata": {},
    "outputs": [],
@@ -56,7 +56,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "id": "12527049-a5cb-423c-8de5-099aee970c85",
    "metadata": {},
    "outputs": [],
@@ -66,18 +66,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "id": "abde5e6c-3474-460c-9fac-4f3352c38b53",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "0.9.39\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "import llama_index\n",
     "print(llama_index.__version__)"
@@ -93,7 +85,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "id": "978cf71f-1ce7-4598-92fe-18fe22ca37c6",
    "metadata": {},
    "outputs": [],
@@ -115,7 +107,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "id": "26f614c8-eb45-4cc1-b067-2c7299587982",
    "metadata": {},
    "outputs": [],
@@ -148,7 +140,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "id": "84cc4308-8ac4-4eba-9478-b81d5b645c48",
    "metadata": {},
    "outputs": [],
@@ -184,7 +176,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
    "id": "8f17c832-e9ae-477b-8bf7-a9c8410f1ed8",
    "metadata": {},
    "outputs": [],
@@ -192,7 +184,7 @@
     "finetune_engine = SentenceTransformersFinetuneEngine(\n",
     "    train_dataset,\n",
     "    model_id=\"BAAI/bge-small-en-v1.5\",\n",
-    "    model_output_path=\"test_model\",\n",
+    "    model_output_path=\"../models/fine-tuned-embeddings\",\n",
     "    batch_size=5,\n",
     "    val_dataset=val_dataset\n",
     ")"
@@ -200,60 +192,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
    "id": "a6498d0b-da9a-4f7f-8c85-c9bf4d772c72",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "e80f94e7c7a84014b3cbf270dde3fcaf",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Epoch:   0%|          | 0/2 [00:00<?, ?it/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "d02eb3c3b1454494a566557e8b73174f",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Iteration:   0%|          | 0/183 [00:00<?, ?it/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "0d73a19c286e43afa7c12cfb5fb49d34",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Iteration:   0%|          | 0/183 [00:00<?, ?it/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
     "finetune_engine.finetune()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": null,
    "id": "e057b405-aa0e-4e78-91e0-9bf40f01c1a9",
    "metadata": {},
    "outputs": [],
@@ -263,21 +212,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": null,
    "id": "72d9f97a-0902-4e65-8459-b34613e419f6",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "HuggingFaceEmbedding(model_name='test_model', embed_batch_size=10, callback_manager=<llama_index.callbacks.base.CallbackManager object at 0x3c7fadca0>, tokenizer_name='test_model', max_length=512, pooling=<Pooling.CLS: 'cls'>, normalize=True, query_instruction=None, text_instruction=None, cache_folder=None)"
-      ]
-     },
-     "execution_count": 14,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "embed_model"
    ]
@@ -285,11 +223,21 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "0709eaf7-b934-4f1d-84ea-c356a1dc5f11",
+   "id": "c4f4058c-edbb-43c4-bebe-8c36d410e819",
    "metadata": {},
    "outputs": [],
    "source": []
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "97ebae28-80ef-4f35-92ce-a370776e3b22",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fine_tuned_embed_model = SentenceTransformer(\"../models/fine-tuned-embeddings\")"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -300,7 +248,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": null,
    "id": "ac4a1a5b-974d-452e-8507-0950c962f9b2",
    "metadata": {},
    "outputs": [],
@@ -341,7 +289,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": null,
    "id": "a53cf893-ce9f-4d9d-ad4a-e9e17fb058d3",
    "metadata": {},
    "outputs": [],
@@ -359,7 +307,7 @@
     "        queries, corpus, relevant_docs, name=name\n",
     "    )\n",
     "    model = SentenceTransformer(model_id)\n",
-    "    output_path = \"results/\"\n",
+    "    output_path = \"../results/\"\n",
     "    Path(output_path).mkdir(exist_ok=True, parents=True)\n",
     "    return evaluator(model, output_path=output_path)"
    ]
@@ -390,49 +338,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": null,
    "id": "91f057aa-4b59-48ea-b3d5-23012a4d487f",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "f4bf05fbe14c4c379c0b3e1912b84d36",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Generating embeddings:   0%|          | 0/100 [00:00<?, ?it/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
-      "To disable this warning, you can either:\n",
-      "\t- Avoid using `tokenizers` before the fork if possible\n",
-      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "4f365d1cab004fe897949e2a3928c457",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "  0%|          | 0/200 [00:00<?, ?it/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
     "ada = OpenAIEmbedding()\n",
     "ada_val_results = evaluate(val_dataset, ada)"
@@ -440,7 +349,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": null,
    "id": "5d2f59c6-75d3-4970-bac3-dfe0eef00efe",
    "metadata": {},
    "outputs": [],
@@ -450,119 +359,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": null,
    "id": "7a697cd8-6f39-4d5b-84f4-f08cf58adc4a",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>is_hit</th>\n",
-       "      <th>retrieved</th>\n",
-       "      <th>expected</th>\n",
-       "      <th>query</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>False</td>\n",
-       "      <td>[5b9cd986-33dc-46f1-abae-e4e1dc9e3629, c3c1804...</td>\n",
-       "      <td>6a756f03-638d-480d-8222-1a6bf3790e3c</td>\n",
-       "      <td>011d84b2-0c26-4c5c-89d1-2a85498f30e0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>True</td>\n",
-       "      <td>[6a756f03-638d-480d-8222-1a6bf3790e3c, c3c1804...</td>\n",
-       "      <td>6a756f03-638d-480d-8222-1a6bf3790e3c</td>\n",
-       "      <td>70c5ddd7-eb86-4a41-af70-a23d2392f48d</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>True</td>\n",
-       "      <td>[c83dbd8a-7e62-445e-8c12-a8ad604ff65e, 2177824...</td>\n",
-       "      <td>c83dbd8a-7e62-445e-8c12-a8ad604ff65e</td>\n",
-       "      <td>a8f4290a-1281-4272-aab9-bf089954a45e</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>True</td>\n",
-       "      <td>[c83dbd8a-7e62-445e-8c12-a8ad604ff65e, 2177824...</td>\n",
-       "      <td>c83dbd8a-7e62-445e-8c12-a8ad604ff65e</td>\n",
-       "      <td>c1ef991a-1cc6-4dbf-b179-2df688c84301</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>True</td>\n",
-       "      <td>[21778248-2ed9-4147-bdb0-a60337a1a599, c83dbd8...</td>\n",
-       "      <td>21778248-2ed9-4147-bdb0-a60337a1a599</td>\n",
-       "      <td>1ce25e78-c1e1-487e-9455-9418baa0b60c</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "   is_hit                                          retrieved  \\\n",
-       "0   False  [5b9cd986-33dc-46f1-abae-e4e1dc9e3629, c3c1804...   \n",
-       "1    True  [6a756f03-638d-480d-8222-1a6bf3790e3c, c3c1804...   \n",
-       "2    True  [c83dbd8a-7e62-445e-8c12-a8ad604ff65e, 2177824...   \n",
-       "3    True  [c83dbd8a-7e62-445e-8c12-a8ad604ff65e, 2177824...   \n",
-       "4    True  [21778248-2ed9-4147-bdb0-a60337a1a599, c83dbd8...   \n",
-       "\n",
-       "                               expected                                 query  \n",
-       "0  6a756f03-638d-480d-8222-1a6bf3790e3c  011d84b2-0c26-4c5c-89d1-2a85498f30e0  \n",
-       "1  6a756f03-638d-480d-8222-1a6bf3790e3c  70c5ddd7-eb86-4a41-af70-a23d2392f48d  \n",
-       "2  c83dbd8a-7e62-445e-8c12-a8ad604ff65e  a8f4290a-1281-4272-aab9-bf089954a45e  \n",
-       "3  c83dbd8a-7e62-445e-8c12-a8ad604ff65e  c1ef991a-1cc6-4dbf-b179-2df688c84301  \n",
-       "4  21778248-2ed9-4147-bdb0-a60337a1a599  1ce25e78-c1e1-487e-9455-9418baa0b60c  "
-      ]
-     },
-     "execution_count": 24,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "df_ada[:5]"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": null,
    "id": "3f7186fb-f392-4531-8959-25161e3905e4",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "(0.955, 200)"
-      ]
-     },
-     "execution_count": 27,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "hit_rate_ada = df_ada[\"is_hit\"].mean()\n",
     "hit_rate_ada, len(df_ada)"
@@ -586,123 +396,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": null,
    "id": "b2905831-0eb9-4ea7-a0b9-5db286b0965e",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "784a67a3d51a400cad53c52bb16121fc",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "config.json:   0%|          | 0.00/743 [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "1c0edb74b4154cb49931180def479320",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "model.safetensors:   0%|          | 0.00/133M [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "af9cb2f4d3934e9a991969f0083fa495",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "tokenizer_config.json:   0%|          | 0.00/366 [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "2370d77040d94ffb9a4d8ca2f45faa97",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "0b7c293a142d4eaf91673c17222d232a",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "tokenizer.json:   0%|          | 0.00/711k [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "7fcb86d759084084a8e41aec12738e19",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "special_tokens_map.json:   0%|          | 0.00/125 [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ab4d747b58f74fdb86481b7f936bf0c4",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Generating embeddings:   0%|          | 0/100 [00:00<?, ?it/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "baa0bb9ae0da4dfc86c20308477415fa",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "  0%|          | 0/200 [00:00<?, ?it/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
     "bge = \"local:BAAI/bge-small-en-v1.5\"\n",
     "bge_val_results = evaluate(val_dataset, bge)"
@@ -710,7 +407,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": null,
    "id": "4e66270d-d3f6-429e-9e48-e8062866aa02",
    "metadata": {},
    "outputs": [],
@@ -720,119 +417,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": null,
    "id": "698c1eb7-eba4-4383-98aa-931fc4ad56a4",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>is_hit</th>\n",
-       "      <th>retrieved</th>\n",
-       "      <th>expected</th>\n",
-       "      <th>query</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>False</td>\n",
-       "      <td>[69a5696d-0c0e-482a-b6a9-f7b87f19945f, fa650c7...</td>\n",
-       "      <td>6a756f03-638d-480d-8222-1a6bf3790e3c</td>\n",
-       "      <td>011d84b2-0c26-4c5c-89d1-2a85498f30e0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>True</td>\n",
-       "      <td>[6a756f03-638d-480d-8222-1a6bf3790e3c, d89a649...</td>\n",
-       "      <td>6a756f03-638d-480d-8222-1a6bf3790e3c</td>\n",
-       "      <td>70c5ddd7-eb86-4a41-af70-a23d2392f48d</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>True</td>\n",
-       "      <td>[c83dbd8a-7e62-445e-8c12-a8ad604ff65e, 2177824...</td>\n",
-       "      <td>c83dbd8a-7e62-445e-8c12-a8ad604ff65e</td>\n",
-       "      <td>a8f4290a-1281-4272-aab9-bf089954a45e</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>True</td>\n",
-       "      <td>[c83dbd8a-7e62-445e-8c12-a8ad604ff65e, ad2e3eb...</td>\n",
-       "      <td>c83dbd8a-7e62-445e-8c12-a8ad604ff65e</td>\n",
-       "      <td>c1ef991a-1cc6-4dbf-b179-2df688c84301</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>True</td>\n",
-       "      <td>[21778248-2ed9-4147-bdb0-a60337a1a599, c83dbd8...</td>\n",
-       "      <td>21778248-2ed9-4147-bdb0-a60337a1a599</td>\n",
-       "      <td>1ce25e78-c1e1-487e-9455-9418baa0b60c</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "   is_hit                                          retrieved  \\\n",
-       "0   False  [69a5696d-0c0e-482a-b6a9-f7b87f19945f, fa650c7...   \n",
-       "1    True  [6a756f03-638d-480d-8222-1a6bf3790e3c, d89a649...   \n",
-       "2    True  [c83dbd8a-7e62-445e-8c12-a8ad604ff65e, 2177824...   \n",
-       "3    True  [c83dbd8a-7e62-445e-8c12-a8ad604ff65e, ad2e3eb...   \n",
-       "4    True  [21778248-2ed9-4147-bdb0-a60337a1a599, c83dbd8...   \n",
-       "\n",
-       "                               expected                                 query  \n",
-       "0  6a756f03-638d-480d-8222-1a6bf3790e3c  011d84b2-0c26-4c5c-89d1-2a85498f30e0  \n",
-       "1  6a756f03-638d-480d-8222-1a6bf3790e3c  70c5ddd7-eb86-4a41-af70-a23d2392f48d  \n",
-       "2  c83dbd8a-7e62-445e-8c12-a8ad604ff65e  a8f4290a-1281-4272-aab9-bf089954a45e  \n",
-       "3  c83dbd8a-7e62-445e-8c12-a8ad604ff65e  c1ef991a-1cc6-4dbf-b179-2df688c84301  \n",
-       "4  21778248-2ed9-4147-bdb0-a60337a1a599  1ce25e78-c1e1-487e-9455-9418baa0b60c  "
-      ]
-     },
-     "execution_count": 29,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "df_bge[:5]"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": null,
    "id": "9b1cb546-4605-4c48-bf4e-df812db97f13",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "(0.915, 200)"
-      ]
-     },
-     "execution_count": 30,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "hit_rate_bge = df_bge[\"is_hit\"].mean()\n",
     "hit_rate_bge, len(df_bge)"
@@ -848,21 +446,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 31,
+   "execution_count": null,
    "id": "1b12ca3d-6ca2-41f6-9ddb-b12b9354ca83",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "0.7955697668171072"
-      ]
-     },
-     "execution_count": 31,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "evaluate_st(val_dataset, \"BAAI/bge-small-en-v1.5\", name=\"bge\")"
    ]
@@ -893,47 +480,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 32,
+   "execution_count": null,
    "id": "bd42b288-1f1f-41aa-9fd4-1ae4b1df462b",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "47dbb97a78c04f7f8fc1264c1013b5ea",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Generating embeddings:   0%|          | 0/100 [00:00<?, ?it/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "31c9e93debe34cc790bf32e579134a1a",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "  0%|          | 0/200 [00:00<?, ?it/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "finetuned = \"local:test_model\"\n",
+    "finetuned = \"local:../models/fine-tuned-embeddings\"\n",
     "val_results_finetuned = evaluate(val_dataset, finetuned)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": null,
    "id": "b1d7112d-b1b8-47db-8a4b-6c024ef99dd6",
    "metadata": {},
    "outputs": [],
@@ -943,21 +501,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 34,
+   "execution_count": null,
    "id": "62a4dd29-0631-4c5b-88e1-be43d48e1043",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "0.97"
-      ]
-     },
-     "execution_count": 34,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "hit_rate_finetuned = df_finetuned[\"is_hit\"].mean()\n",
     "hit_rate_finetuned"
@@ -965,23 +512,12 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 35,
+   "execution_count": null,
    "id": "4332594b-c861-40fb-a58b-ba36717d0519",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "0.8573385846534823"
-      ]
-     },
-     "execution_count": 35,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "evaluate_st(val_dataset, \"test_model\", name=\"finetuned\")"
+    "evaluate_st(val_dataset, \"../models/fine-tuned-embeddings\", name=\"finetuned\")"
    ]
   },
   {
@@ -1002,7 +538,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 36,
+   "execution_count": null,
    "id": "3ca46cff-b186-463a-847d-a86c310268ec",
    "metadata": {},
    "outputs": [],
@@ -1014,68 +550,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 37,
+   "execution_count": null,
    "id": "d1d3053e-2395-48a0-af59-fd27180e1e7b",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>is_hit</th>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>model</th>\n",
-       "      <th></th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>ada</th>\n",
-       "      <td>0.955</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>bge</th>\n",
-       "      <td>0.915</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>fine_tuned</th>\n",
-       "      <td>0.970</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "            is_hit\n",
-       "model             \n",
-       "ada          0.955\n",
-       "bge          0.915\n",
-       "fine_tuned   0.970"
-      ]
-     },
-     "execution_count": 37,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "df_all = pd.concat([df_ada, df_bge, df_finetuned])\n",
     "df_all.groupby(\"model\").mean(\"is_hit\")"
@@ -1091,16 +569,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 38,
+   "execution_count": null,
    "id": "032cac38-c856-4aeb-9bbb-6d70ed53c614",
    "metadata": {},
    "outputs": [],
    "source": [
     "df_st_bge = pd.read_csv(\n",
-    "    \"results/Information-Retrieval_evaluation_bge_results.csv\"\n",
+    "    \"../results/Information-Retrieval_evaluation_bge_results.csv\"\n",
     ")\n",
     "df_st_finetuned = pd.read_csv(\n",
-    "    \"results/Information-Retrieval_evaluation_finetuned_results.csv\"\n",
+    "    \"../results/Information-Retrieval_evaluation_finetuned_results.csv\"\n",
     ")"
    ]
   },
@@ -1114,176 +592,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 39,
+   "execution_count": null,
    "id": "d2975262-c486-4a9a-a61f-ea535203a0f3",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>epoch</th>\n",
-       "      <th>steps</th>\n",
-       "      <th>cos_sim-Accuracy@1</th>\n",
-       "      <th>cos_sim-Accuracy@3</th>\n",
-       "      <th>cos_sim-Accuracy@5</th>\n",
-       "      <th>cos_sim-Accuracy@10</th>\n",
-       "      <th>cos_sim-Precision@1</th>\n",
-       "      <th>cos_sim-Recall@1</th>\n",
-       "      <th>cos_sim-Precision@3</th>\n",
-       "      <th>cos_sim-Recall@3</th>\n",
-       "      <th>...</th>\n",
-       "      <th>dot_score-Recall@1</th>\n",
-       "      <th>dot_score-Precision@3</th>\n",
-       "      <th>dot_score-Recall@3</th>\n",
-       "      <th>dot_score-Precision@5</th>\n",
-       "      <th>dot_score-Recall@5</th>\n",
-       "      <th>dot_score-Precision@10</th>\n",
-       "      <th>dot_score-Recall@10</th>\n",
-       "      <th>dot_score-MRR@10</th>\n",
-       "      <th>dot_score-NDCG@10</th>\n",
-       "      <th>dot_score-MAP@100</th>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>model</th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>bge</th>\n",
-       "      <td>-1</td>\n",
-       "      <td>-1</td>\n",
-       "      <td>0.705</td>\n",
-       "      <td>0.865</td>\n",
-       "      <td>0.92</td>\n",
-       "      <td>0.96</td>\n",
-       "      <td>0.705</td>\n",
-       "      <td>0.705</td>\n",
-       "      <td>0.288333</td>\n",
-       "      <td>0.865</td>\n",
-       "      <td>...</td>\n",
-       "      <td>0.705</td>\n",
-       "      <td>0.288333</td>\n",
-       "      <td>0.865</td>\n",
-       "      <td>0.184</td>\n",
-       "      <td>0.92</td>\n",
-       "      <td>0.096</td>\n",
-       "      <td>0.96</td>\n",
-       "      <td>0.792935</td>\n",
-       "      <td>0.833595</td>\n",
-       "      <td>0.795570</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>fine_tuned</th>\n",
-       "      <td>-1</td>\n",
-       "      <td>-1</td>\n",
-       "      <td>0.790</td>\n",
-       "      <td>0.900</td>\n",
-       "      <td>0.97</td>\n",
-       "      <td>0.98</td>\n",
-       "      <td>0.790</td>\n",
-       "      <td>0.790</td>\n",
-       "      <td>0.300000</td>\n",
-       "      <td>0.900</td>\n",
-       "      <td>...</td>\n",
-       "      <td>0.790</td>\n",
-       "      <td>0.300000</td>\n",
-       "      <td>0.900</td>\n",
-       "      <td>0.194</td>\n",
-       "      <td>0.97</td>\n",
-       "      <td>0.098</td>\n",
-       "      <td>0.98</td>\n",
-       "      <td>0.856264</td>\n",
-       "      <td>0.886738</td>\n",
-       "      <td>0.857339</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>2 rows × 32 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "            epoch  steps  cos_sim-Accuracy@1  cos_sim-Accuracy@3  \\\n",
-       "model                                                              \n",
-       "bge            -1     -1               0.705               0.865   \n",
-       "fine_tuned     -1     -1               0.790               0.900   \n",
-       "\n",
-       "            cos_sim-Accuracy@5  cos_sim-Accuracy@10  cos_sim-Precision@1  \\\n",
-       "model                                                                      \n",
-       "bge                       0.92                 0.96                0.705   \n",
-       "fine_tuned                0.97                 0.98                0.790   \n",
-       "\n",
-       "            cos_sim-Recall@1  cos_sim-Precision@3  cos_sim-Recall@3  ...  \\\n",
-       "model                                                                ...   \n",
-       "bge                    0.705             0.288333             0.865  ...   \n",
-       "fine_tuned             0.790             0.300000             0.900  ...   \n",
-       "\n",
-       "            dot_score-Recall@1  dot_score-Precision@3  dot_score-Recall@3  \\\n",
-       "model                                                                       \n",
-       "bge                      0.705               0.288333               0.865   \n",
-       "fine_tuned               0.790               0.300000               0.900   \n",
-       "\n",
-       "            dot_score-Precision@5  dot_score-Recall@5  dot_score-Precision@10  \\\n",
-       "model                                                                           \n",
-       "bge                         0.184                0.92                   0.096   \n",
-       "fine_tuned                  0.194                0.97                   0.098   \n",
-       "\n",
-       "            dot_score-Recall@10  dot_score-MRR@10  dot_score-NDCG@10  \\\n",
-       "model                                                                  \n",
-       "bge                        0.96          0.792935           0.833595   \n",
-       "fine_tuned                 0.98          0.856264           0.886738   \n",
-       "\n",
-       "            dot_score-MAP@100  \n",
-       "model                          \n",
-       "bge                  0.795570  \n",
-       "fine_tuned           0.857339  \n",
-       "\n",
-       "[2 rows x 32 columns]"
-      ]
-     },
-     "execution_count": 39,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "df_st_bge[\"model\"] = \"bge\"\n",
     "df_st_finetuned[\"model\"] = \"fine_tuned\"\n",

notebooks/persisted-embedding-model.ipynb

@@ -483,7 +483,7 @@
    },
    "outputs": [],
    "source": [
-    "r_list[1].to_dict()"
+    "r_list[0].to_dict()"
    ]
   },
   {
@@ -551,6 +551,18 @@
     "embed_model = HuggingFaceEmbedding(model_name=\"BAAI/bge-small-en-v1.5\")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6c98a573-b401-4191-99c0-1216833bb566",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.llms import OpenAI\n",
+    "from llama_index.memory import ChatMemoryBuffer\n",
+    "llm = OpenAI(model=\"gpt-3.5-turbo-1106\", temperature=0.0)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -614,6 +626,41 @@
     ")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "73ba6d06-ba69-4b5e-962a-9cf7d2dc4d94",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ab778a5d-d438-4f39-88f5-c67a1f1d575e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "system_content = (\"You are a helpful study assistant. \"\n",
+    "                  \"You do not respond as 'User' or pretend to be 'User'. \"\n",
+    "                  \"You only respond once as 'Assistant'.\"\n",
+    ")\n",
+    "memory = ChatMemoryBuffer.from_defaults(token_limit=15000)\n",
+    "chat_engine = index.as_chat_engine(\n",
+    "    chat_mode=\"context\",\n",
+    "    memory=memory,\n",
+    "    system_prompt=system_content\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8d6de457-43b5-4ea7-b5e3-150abe918671",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -653,6 +700,182 @@
    "metadata": {},
    "outputs": [],
    "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "301e8270-783d-4942-a05f-9683ca96fbda",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "506672cc-f447-414d-9c57-cd62a964dea8",
+   "metadata": {},
+   "source": [
+    "### ChromaDB method - load vectorstore with LLM"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d9c4a50e-915c-492d-be69-e4ebfd16744a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import chromadb\n",
+    "from llama_index import VectorStoreIndex, SimpleDirectoryReader\n",
+    "from llama_index.vector_stores import ChromaVectorStore\n",
+    "from llama_index.storage.storage_context import StorageContext\n",
+    "from llama_index import ServiceContext\n",
+    "from llama_index import Document\n",
+    "\n",
+    "from llama_index.embeddings import HuggingFaceEmbedding\n",
+    "\n",
+    "import time"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "97680b61-d87a-426d-9177-3670688e8e0c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "embed_model = HuggingFaceEmbedding(model_name=\"BAAI/bge-small-en-v1.5\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "808fa41d-2b3f-40ab-8cd3-01565b6d6e35",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.llms import OpenAI\n",
+    "from llama_index.memory import ChatMemoryBuffer\n",
+    "llm = OpenAI(model=\"gpt-3.5-turbo-1106\", temperature=0.0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "497b02bd-3ec7-4a4e-8af9-6417437a4bce",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "51d64b76-628e-418c-b394-807ea9cafd6c",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c0b28d70-c43d-4542-9e1b-4ce29a60f9d3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "db = chromadb.PersistentClient(path=\"../models/chroma_db\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6f1d4e93-0d74-456a-9c1d-938405a8ec9a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "chroma_collection = db.get_or_create_collection(\"quickstart\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "da0dd3b7-d798-4c0f-b735-cf1e67094c46",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# assign chroma as the vector_store to the context\n",
+    "vector_store = ChromaVectorStore(chroma_collection=chroma_collection)\n",
+    "storage_context = StorageContext.from_defaults(vector_store=vector_store)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0d62e372-8a33-4609-9ac4-fee3cbc4e8a9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# create your index\n",
+    "index = VectorStoreIndex.from_vector_store(\n",
+    "    vector_store=vector_store, service_context=service_context, storage_context=storage_context\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "26dedd3b-44f3-4a67-865a-693cd6d0a9ea",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "system_content = (\"You are a helpful study assistant. \"\n",
+    "                  \"You do not respond as 'User' or pretend to be 'User'. \"\n",
+    "                  \"You only respond once as 'Assistant'.\"\n",
+    ")\n",
+    "memory = ChatMemoryBuffer.from_defaults(token_limit=15000)\n",
+    "chat_engine = index.as_chat_engine(\n",
+    "    chat_mode=\"context\",\n",
+    "    memory=memory,\n",
+    "    system_prompt=system_content\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9e3da625-283a-4d57-a449-d5aa17d0c188",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "response = chat_engine.stream_chat(\"are you there?\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "62ed7a14-261f-4c68-8578-5dfb74bcfc58",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for r in response.response_gen:\n",
+    "    print(r, end=\"\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1d4ba65c-3135-4b96-a342-c5546949cb72",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9ca2555f-6975-4bc1-b804-c0c9beb2a515",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

notebooks/qna_prompting_with_function_calling.ipynb

@@ -0,0 +1,399 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9e975979-3b3d-4a8d-9db6-b7433cf0d8b4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os, random, json\n",
+    "import sqlite3\n",
+    "\n",
+    "import pandas as pd\n",
+    "from openai import OpenAI"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "98601634-bd9b-4566-b242-2b3c9d04b260",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "63db76a8-31de-4957-b7b9-291c2539f976",
+   "metadata": {},
+   "source": [
+    "### Parameters"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ff4d40aa-a42e-4ad7-9ca9-d894653d205e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "db_path = \"../database/mock_qna.db\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "98a20c7e-b1dc-42d5-929b-62978959abda",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a11295d9-9bf0-4c9d-b5b2-0feec01bf640",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "con = sqlite3.connect(db_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a1c1e976-0d75-42e3-8c2e-5045ee0f2c4a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cur = con.cursor()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d78b0cc7-0238-41be-bc9f-688fcac71f73",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = cur.execute(f\"\"\"SELECT COUNT(*)\n",
+    "                      FROM qna_tbl\n",
+    "                   \"\"\")\n",
+    "table_size = res.fetchone()[0]\n",
+    "print(f\"table size: {table_size}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "faaacff0-bc67-464d-bd7c-1d51b0901dd4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = cur.execute(f\"\"\"SELECT chapter, COUNT(*)\n",
+    "                      FROM qna_tbl\n",
+    "                      GROUP BY chapter\n",
+    "                   \"\"\")\n",
+    "chapter_counts = res.fetchall()\n",
+    "print(chapter_counts)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f83954ba-f92a-42ce-8d1c-758f4054b4c5",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "117bbc79-5f58-4b31-9df1-dac75d7ef5a8",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8dae73ca-845a-4d1e-8e1f-b1efb36dec8e",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6c4fddf3-6e7a-40c7-a6c2-2e06f976ec56",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "id = random.randint(1, table_size)\n",
+    "res = cur.execute(f\"\"\"SELECT question, option_1, option_2, option_3, option_4, correct_answer\n",
+    "                      FROM qna_tbl\n",
+    "                      WHERE id={id}\n",
+    "                   \"\"\")\n",
+    "result = res.fetchone()\n",
+    "result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f55b4a21-45b1-42a6-8ad1-352174b78806",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c5ef430b-807c-4090-8ed2-969c43ba228e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_qna_question(chapter_n):\n",
+    "    sql_string = f\"\"\"SELECT id, question, option_1, option_2, option_3, option_4, correct_answer\n",
+    "                     FROM qna_tbl\n",
+    "                     WHERE chapter='{chapter_n}'\n",
+    "                  \"\"\"\n",
+    "    res = cur.execute(sql_string)\n",
+    "    result = res.fetchone()\n",
+    "\n",
+    "    id       = result[0]\n",
+    "    question = result[1]\n",
+    "    option_1 = result[2]\n",
+    "    option_2 = result[3]\n",
+    "    option_3 = result[4]\n",
+    "    option_4 = result[5]\n",
+    "    c_answer = result[6]\n",
+    "\n",
+    "    qna_str  = \"Question: \\n\" + \\\n",
+    "               \"========= \\n\" + \\\n",
+    "                question.replace(\"\\\\n\", \"\\n\") + \"\\n\" + \\\n",
+    "               \"A) \" + option_1 + \"\\n\" + \\\n",
+    "               \"B) \" + option_2 + \"\\n\" + \\\n",
+    "               \"C) \" + option_3 + \"\\n\" + \\\n",
+    "               \"D) \" + option_4\n",
+    "    \n",
+    "    return id, qna_str, c_answer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b61cc8eb-5118-438a-b38f-e01fc92c7387",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "13702036-6457-464d-bd32-0e20dd7050e5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "qna_custom_functions = [\n",
+    "    {\n",
+    "        \"name\": \"get_qna_question\",\n",
+    "        \"description\": \"\"\"\n",
+    "                        Extract the chapter information from the body of the input text, the format looks as follow:\n",
+    "                        The output should be in the format with `Chapter_` as prefix.\n",
+    "                        Example 1: `Chapter_1` for first chapter\n",
+    "                        Example 2: For chapter 12 of the textbook, you should return `Chapter_12`\n",
+    "                        Example 3: `Chapter_5` for fifth chapter\n",
+    "                        Thereafter, the chapter_n argument will be passed to the function for Q&A question retrieval.\n",
+    "                       \"\"\",\n",
+    "        \"parameters\": {\n",
+    "            \"type\": \"object\",\n",
+    "            \"properties\": {\n",
+    "                \"chapter_n\": {\n",
+    "                    \"type\": \"string\",\n",
+    "                    \"description\": \"\"\"\n",
+    "                        which chapter to extract, the format of this function argumet is with `Chapter_` as prefix, \n",
+    "                        concatenated with chapter number in integer. For example, `Chapter_2`, `Chapter_10`.\n",
+    "                    \"\"\"\n",
+    "                }\n",
+    "            }\n",
+    "        }\n",
+    "    }\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1bbb95af-dd82-443f-b23c-97c9a2777e11",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "957fe647-c1f7-4db5-8f31-fb5e1f546c0c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "client = OpenAI()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "018fc414-d6df-408f-a14c-0a3857f4c52d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt = \"I am interested in chapter 13, can you test my understanding of this chapter?\"\n",
+    "response = client.chat.completions.create(\n",
+    "    model = 'gpt-3.5-turbo',\n",
+    "    messages = [{'role': 'user', 'content': prompt}],\n",
+    "    functions = qna_custom_functions,\n",
+    "    function_call = 'auto'\n",
+    ")\n",
+    "json_response = json.loads(response.choices[0].message.function_call.arguments)\n",
+    "print(json_response)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2408c546-335c-478a-b1ea-9c0921a9b7a0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "37ec1b9a-2cdd-4838-ab02-8260d392483f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt = \"I am interested in chapter thirteen, can you test my understanding of this chapter?\"\n",
+    "response = client.chat.completions.create(\n",
+    "    model = 'gpt-3.5-turbo',\n",
+    "    messages = [{'role': 'user', 'content': prompt}],\n",
+    "    functions = qna_custom_functions,\n",
+    "    function_call = 'auto'\n",
+    ")\n",
+    "json_response = json.loads(response.choices[0].message.function_call.arguments)\n",
+    "print(json_response)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6b8e9f05-bb9a-429b-a1fb-abbaced23230",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "18edebdd-2c7f-4589-8909-f816be5c4d1c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt = \"I am interested in 4th chapter, can you test my understanding of this chapter?\"\n",
+    "response = client.chat.completions.create(\n",
+    "    model = 'gpt-3.5-turbo',\n",
+    "    messages = [{'role': 'user', 'content': prompt}],\n",
+    "    functions = qna_custom_functions,\n",
+    "    function_call = 'auto'\n",
+    ")\n",
+    "json_response = json.loads(response.choices[0].message.function_call.arguments)\n",
+    "print(json_response)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d4325b3c-47d6-4d3f-a50a-45914b47a9c0",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c558b722-4438-4485-98c0-b4117bc3d46e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt = \"\"\"There are 15 chapters in the Health Insurance text book, I want to study the last chapter, \n",
+    "            can you test my understanding of this chapter?\n",
+    "        \"\"\"\n",
+    "response = client.chat.completions.create(\n",
+    "    model = 'gpt-3.5-turbo',\n",
+    "    messages = [{'role': 'user', 'content': prompt}],\n",
+    "    functions = qna_custom_functions,\n",
+    "    function_call = 'auto'\n",
+    ")\n",
+    "json_response = json.loads(response.choices[0].message.function_call.arguments)\n",
+    "print(json_response)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "074229dc-82d9-4a2b-9a08-019228da78a1",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "289fba25-f547-402a-bd13-0dc4ce7ddf8e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "id, qna_str, answer = get_qna_question(chapter_n=json_response[\"chapter_n\"])\n",
+    "print(qna_str)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "adc9f539-3654-4174-815b-e0939f513a20",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5b6ad929-e6a5-4978-8678-519375ef62eb",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/qna_prompting_with_pydantic.ipynb

@@ -0,0 +1,114 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6f0f5f02-c8e9-43a9-853d-12bb3c19dbe8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pydantic import BaseModel"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "94244a1e-e55a-4954-885e-4558797c6fe3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.llms import OpenAI"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "641f36c7-0aa3-4146-9840-bfb0d4d78b4d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.core.tools import BaseTool, FunctionTool"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cb20cd13-20fd-4303-acde-b7abe0b48e39",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ab4d1a52-84be-492f-8275-3da20d854cb6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class Song(BaseModel):\n",
+    "    \"\"\"A song with name and artist\"\"\"\n",
+    "\n",
+    "    name: str\n",
+    "    artist: str"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a5822b1d-32ef-4b68-8629-a727ff51cd0a",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "63332a44-9441-4f49-85a2-934e2c55a362",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "song_fn = FunctionTool.from_defaults(fn=Song)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ef0d7d67-9855-47ea-8569-7bfb20b03a07",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "response = OpenAI().complete(\"Generate a song\", tools=[song_fn])\n",
+    "tool_calls = response.additional_kwargs[\"tool_calls\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bca4c0b2-5165-4943-af1f-d3168ee88fcd",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

raw_documents/qna.txt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8b44d78e6dec3a285124f0a449ff5bae699ab4ff98ae3826a33a8eb4f182334
-size 1804
+oid sha256:96f148c23c11fe6df506f5286d2c90143b274ce2705501deaeac47fa63863825
+size 2134

requirements.txt

@@ -16,9 +16,10 @@ attrs==23.2.0
 Babel==2.14.0
 backoff==2.2.1
 bcrypt==4.1.2
-beautifulsoup4==4.12.2
+beautifulsoup4==4.12.3
 bleach==6.1.0
 blinker==1.7.0
+bs4==0.0.2
 build==1.0.3
 cachetools==5.3.2
 certifi==2023.11.17
@@ -37,6 +38,7 @@ decorator==5.1.1
 defusedxml==0.7.1
 Deprecated==1.2.14
 dill==0.3.7
+dirtyjson==1.0.8
 distro==1.9.0
 entrypoints==0.4
 exceptiongroup==1.2.0

streamlit_app.py

@@ -7,12 +7,15 @@ import base64
 from io import BytesIO
 import nest_asyncio
 
-from llama_index.llms import OpenAI
-from llama_index import SimpleDirectoryReader
-from llama_index import Document
-from llama_index import VectorStoreIndex
-from llama_index import ServiceContext
+import chromadb
+from llama_index import (VectorStoreIndex, 
+                         SimpleDirectoryReader,
+                         ServiceContext,
+                         Document)
+from llama_index.vector_stores import ChromaVectorStore
+from llama_index.storage.storage_context import StorageContext
 from llama_index.embeddings import HuggingFaceEmbedding
+from llama_index.llms import OpenAI
 from llama_index.memory import ChatMemoryBuffer
 
 from vision_api import get_transcribed_text
@@ -27,6 +30,8 @@ openai_api = os.getenv("OPENAI_API_KEY")
 input_files = ["./raw_documents/HI Chapter Summary Version 1.3.pdf",
                "./raw_documents/qna.txt"]
 embedding_model = "BAAI/bge-small-en-v1.5"
+persisted_vector_db = "./models/chroma_db"
+fine_tuned_path = "local:models/fine-tuned-embeddings"
 system_content = ("You are a helpful study assistant. "
                   "You do not respond as 'User' or pretend to be 'User'. "
                   "You only respond once as 'Assistant'."
@@ -104,7 +109,9 @@ def clear_chat_history():
                                    llm_model=selected_model, 
                                    temperature=temperature,
                                    embedding_model=embedding_model,
-                                   system_content=system_content)
+                                   fine_tuned_path=fine_tuned_path,
+                                   system_content=system_content,
+                                   persisted_path=persisted_vector_db)
     chat_engine.reset()
 
 st.sidebar.button("Clear Chat History", on_click=clear_chat_history)
@@ -124,23 +131,52 @@ def get_llm_object(selected_model, temperature):
     return llm
 
 @st.cache_resource
-def get_embedding_model(model_name):
-    embed_model = HuggingFaceEmbedding(model_name=model_name)
+def get_embedding_model(model_name, fine_tuned_path=None):
+    if fine_tuned_path is None:
+        print(f"loading from `{model_name}` from huggingface")
+        embed_model = HuggingFaceEmbedding(model_name=model_name)
+    else:
+        print(f"loading from local `{fine_tuned_path}`")
+        embed_model = fine_tuned_path
     return embed_model
 
 @st.cache_resource
-def get_query_engine(input_files, llm_model, temperature, 
-                     embedding_model, system_content):
-
-    document = get_document_object(input_files)
+def get_query_engine(input_files, llm_model, temperature,
+                     embedding_model, fine_tuned_path,
+                     system_content, persisted_path):
+    
     llm = get_llm_object(llm_model, temperature)
-    embedded_model = get_embedding_model(embedding_model)
-
-    service_context = ServiceContext.from_defaults(llm=llm, embed_model=embedded_model)
-    index = VectorStoreIndex.from_documents([document], service_context=service_context)
+    embedded_model = get_embedding_model(
+                        model_name=embedding_model, 
+                        fine_tuned_path=fine_tuned_path
+    )
+    service_context = ServiceContext.from_defaults(
+                        llm=llm, 
+                        embed_model=embedded_model
+    )
+    
+    if os.path.exists(persisted_path):
+        print("loading from vector database - chroma")
+        db = chromadb.PersistentClient(path=persisted_path)
+        chroma_collection = db.get_or_create_collection("quickstart")
+        vector_store = ChromaVectorStore(chroma_collection=chroma_collection)
+        storage_context = StorageContext.from_defaults(
+                                vector_store=vector_store
+        )
+        index = VectorStoreIndex.from_vector_store(
+            vector_store=vector_store, 
+            service_context=service_context, 
+            storage_context=storage_context
+        )
+    else:
+        print("create in-memory vector store")
+        document = get_document_object(input_files)
+        index = VectorStoreIndex.from_documents(
+                    [document], 
+                    service_context=service_context
+        )
+    
     memory = ChatMemoryBuffer.from_defaults(token_limit=15000)
-
-    # chat_engine = index.as_query_engine(streaming=True)
     chat_engine = index.as_chat_engine(
         chat_mode="context",
         memory=memory,
@@ -154,7 +190,9 @@ def generate_llm_response(prompt_input):
                                    llm_model=selected_model, 
                                    temperature=temperature,
                                    embedding_model=embedding_model,
-                                   system_content=system_content)
+                                   fine_tuned_path=fine_tuned_path,
+                                   system_content=system_content,
+                                   persisted_path=persisted_vector_db)
     
     # st.session_state.messages
     response = chat_engine.stream_chat(prompt_input)