added fine-tuning example

.gitignore

@@ -4,4 +4,6 @@
 results/
 
 *.sqlite
-ux/
+ux/
+data/
+notebooks/test_model

notebooks/fine-tuning-embedding-model.ipynb

@@ -6,6 +6,199 @@
    "id": "ca2c990f-5215-4ab9-8143-1d79db28edc6",
    "metadata": {},
    "outputs": [],
+   "source": [
+    "import json, os\n",
+    "\n",
+    "from llama_index import SimpleDirectoryReader\n",
+    "from llama_index.node_parser import SentenceSplitter\n",
+    "from llama_index.schema import MetadataMode"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2c535ad7-7846-4bef-8ba8-33e182490c3d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.finetuning import (\n",
+    "    generate_qa_embedding_pairs,\n",
+    "    EmbeddingQAFinetuneDataset,\n",
+    ")\n",
+    "from llama_index.finetuning import SentenceTransformersFinetuneEngine"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "12527049-a5cb-423c-8de5-099aee970c85",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.llms import OpenAI"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "abde5e6c-3474-460c-9fac-4f3352c38b53",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import llama_index\n",
+    "print(llama_index.__version__)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7dc65d7b-3cdb-4513-b09f-f7406ad59b35",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "978cf71f-1ce7-4598-92fe-18fe22ca37c6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "TRAIN_FILES = [\"../raw_documents/HI_Knowledge_Base.pdf\"]\n",
+    "VAL_FILES = [\"../raw_documents/HI Chapter Summary Version 1.3.pdf\"]\n",
+    "\n",
+    "TRAIN_CORPUS_FPATH = \"../data/train_corpus.json\"\n",
+    "VAL_CORPUS_FPATH = \"../data/val_corpus.json\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "663cd20e-c16e-4dda-924e-5f60eb25a772",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "26f614c8-eb45-4cc1-b067-2c7299587982",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def load_corpus(files, verbose=False):\n",
+    "    if verbose:\n",
+    "        print(f\"Loading files {files}\")\n",
+    "\n",
+    "    reader = SimpleDirectoryReader(input_files=files)\n",
+    "    docs = reader.load_data()\n",
+    "    if verbose:\n",
+    "        print(f\"Loaded {len(docs)} docs\")\n",
+    "\n",
+    "    parser = SentenceSplitter()\n",
+    "    nodes = parser.get_nodes_from_documents(docs, show_progress=verbose)\n",
+    "\n",
+    "    if verbose:\n",
+    "        print(f\"Parsed {len(nodes)} nodes\")\n",
+    "\n",
+    "    return nodes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a6ba52e5-4d7f-4c30-8979-8d84a1bc3ca4",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "84cc4308-8ac4-4eba-9478-b81d5b645c48",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if not os.path.exists(TRAIN_CORPUS_FPATH) or \\\n",
+    "   not os.path.exists(VAL_CORPUS_FPATH):\n",
+    "\n",
+    "    train_nodes = load_corpus(TRAIN_FILES, verbose=True)\n",
+    "    val_nodes = load_corpus(VAL_FILES, verbose=True)\n",
+    "    \n",
+    "    train_dataset = generate_qa_embedding_pairs(\n",
+    "        llm=OpenAI(model=\"gpt-3.5-turbo-1106\"), nodes=train_nodes\n",
+    "    )\n",
+    "    val_dataset = generate_qa_embedding_pairs(\n",
+    "        llm=OpenAI(model=\"gpt-3.5-turbo-1106\"), nodes=val_nodes\n",
+    "    )\n",
+    "    \n",
+    "    train_dataset.save_json(TRAIN_CORPUS_FPATH)\n",
+    "    val_dataset.save_json(VAL_CORPUS_FPATH)\n",
+    "    \n",
+    "else:\n",
+    "    train_dataset = EmbeddingQAFinetuneDataset.from_json(TRAIN_CORPUS_FPATH)\n",
+    "    val_dataset = EmbeddingQAFinetuneDataset.from_json(VAL_CORPUS_FPATH)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c3399443-5936-4dfe-b0ec-821d222e734d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8f17c832-e9ae-477b-8bf7-a9c8410f1ed8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "finetune_engine = SentenceTransformersFinetuneEngine(\n",
+    "    train_dataset,\n",
+    "    model_id=\"BAAI/bge-small-en-v1.5\",\n",
+    "    model_output_path=\"test_model\",\n",
+    "    val_dataset=val_dataset,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a6498d0b-da9a-4f7f-8c85-c9bf4d772c72",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "finetune_engine.finetune()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e057b405-aa0e-4e78-91e0-9bf40f01c1a9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "embed_model = finetune_engine.get_finetuned_model()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "72d9f97a-0902-4e65-8459-b34613e419f6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "embed_model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0709eaf7-b934-4f1d-84ea-c356a1dc5f11",
+   "metadata": {},
+   "outputs": [],
    "source": []
   }
  ],

requirements.txt

@@ -8,19 +8,26 @@ appnope==0.1.3
 argon2-cffi==23.1.0
 argon2-cffi-bindings==21.2.0
 arrow==1.3.0
+asgiref==3.7.2
 asttokens==2.4.1
 async-lru==2.0.4
 async-timeout==4.0.3
 attrs==23.2.0
 Babel==2.14.0
+backoff==2.2.1
+bcrypt==4.1.2
 beautifulsoup4==4.12.2
 bleach==6.1.0
 blinker==1.7.0
+build==1.0.3
 cachetools==5.3.2
 certifi==2023.11.17
 cffi==1.16.0
 charset-normalizer==3.3.2
+chroma-hnswlib==0.7.3
+chromadb==0.4.22
 click==8.1.7
+coloredlogs==15.0.1
 comm==0.2.0
 contourpy==1.2.0
 cycler==0.12.1
@@ -35,9 +42,11 @@ entrypoints==0.4
 exceptiongroup==1.2.0
 executing==2.0.1
 Faker==22.0.0
+fastapi==0.109.0
 fastjsonschema==2.19.1
 favicon==0.7.0
 filelock==3.13.1
+flatbuffers==23.5.26
 fonttools==4.47.0
 fqdn==1.5.1
 frozendict==2.4.0
@@ -45,12 +54,17 @@ frozenlist==1.4.1
 fsspec==2023.12.2
 gitdb==4.0.11
 GitPython==3.1.40
+google-auth==2.27.0
+googleapis-common-protos==1.62.0
 greenlet==3.0.3
+grpcio==1.60.0
 h11==0.14.0
 htbuilder==0.6.2
 httpcore==1.0.2
+httptools==0.6.1
 httpx==0.26.0
 huggingface-hub==0.20.1
+humanfriendly==10.0
 humanize==4.9.0
 idna==3.6
 importlib-metadata==6.11.0
@@ -80,11 +94,12 @@ jupyterlab-widgets==3.0.9
 jupyterlab_pygments==0.3.0
 jupyterlab_server==2.25.2
 kiwisolver==1.4.5
+kubernetes==29.0.0
 langchain==0.0.354
 langchain-community==0.0.8
 langchain-core==0.1.5
 langsmith==0.0.77
-llama-index==0.9.24
+llama-index==0.9.39
 Mako==1.3.0
 Markdown==3.5.1
 markdown-it-py==3.0.0
@@ -97,6 +112,8 @@ mdurl==0.1.2
 merkle-json==1.0.0
 millify==0.1.1
 mistune==3.0.2
+mmh3==4.1.0
+monotonic==1.6
 more-itertools==10.1.0
 mpmath==1.3.0
 multidict==6.0.4
@@ -111,7 +128,19 @@ nltk==3.8.1
 notebook==7.0.6
 notebook_shim==0.2.3
 numpy==1.26.2
+oauthlib==3.2.2
+onnxruntime==1.16.3
 openai==1.6.1
+opentelemetry-api==1.22.0
+opentelemetry-exporter-otlp-proto-common==1.22.0
+opentelemetry-exporter-otlp-proto-grpc==1.22.0
+opentelemetry-instrumentation==0.43b0
+opentelemetry-instrumentation-asgi==0.43b0
+opentelemetry-instrumentation-fastapi==0.43b0
+opentelemetry-proto==1.22.0
+opentelemetry-sdk==1.22.0
+opentelemetry-semantic-conventions==0.43b0
+opentelemetry-util-http==0.43b0
 overrides==7.4.0
 packaging==23.2
 pandas==2.1.4
@@ -120,13 +149,17 @@ parso==0.8.3
 pexpect==4.9.0
 pillow==10.2.0
 platformdirs==4.1.0
+posthog==3.3.3
 prometheus-client==0.19.0
 prompt-toolkit==3.0.43
 protobuf==4.25.1
 psutil==5.9.7
 ptyprocess==0.7.0
+pulsar-client==3.4.0
 pure-eval==0.2.2
 pyarrow==14.0.2
+pyasn1==0.5.1
+pyasn1-modules==0.3.0
 pycparser==2.21
 pydantic==2.5.3
 pydantic_core==2.14.6
@@ -135,6 +168,8 @@ Pygments==2.17.2
 pymdown-extensions==10.7
 pyparsing==3.1.1
 pypdf==3.17.4
+PyPika==0.48.9
+pyproject_hooks==1.0.0
 python-dateutil==2.8.2
 python-decouple==3.8
 python-dotenv==1.0.0
@@ -147,12 +182,18 @@ QtPy==2.4.1
 referencing==0.32.0
 regex==2023.12.25
 requests==2.31.0
+requests-oauthlib==1.3.1
 rfc3339-validator==0.1.4
 rfc3986-validator==0.1.1
 rich==13.7.0
 rpds-py==0.16.2
+rsa==4.9
 safetensors==0.4.1
+scikit-learn==1.4.0
+scipy==1.12.0
 Send2Trash==1.8.2
+sentence-transformers==2.3.0
+sentencepiece==0.1.99
 six==1.16.0
 smmap==5.0.1
 sniffio==1.3.0
@@ -160,6 +201,7 @@ soupsieve==2.5
 SQLAlchemy==2.0.24
 st-annotated-text==4.0.1
 stack-data==0.6.3
+starlette==0.35.1
 streamlit==1.29.0
 streamlit-aggrid==0.3.4.post3
 streamlit-camera-input-live==0.2.0
@@ -174,6 +216,7 @@ streamlit-vertical-slider==2.5.5
 sympy==1.12
 tenacity==8.2.3
 terminado==0.18.0
+threadpoolctl==3.2.0
 tiktoken==0.5.2
 tinycss2==1.2.1
 tokenizers==0.15.0
@@ -187,6 +230,7 @@ traitlets==5.14.0
 transformers==4.36.2
 trulens==0.13.4
 trulens-eval==0.20.0
+typer==0.9.0
 types-python-dateutil==2.8.19.14
 typing-inspect==0.9.0
 typing_extensions==4.9.0
@@ -194,12 +238,16 @@ tzdata==2023.4
 tzlocal==5.2
 uri-template==1.3.0
 urllib3==2.1.0
+uvicorn==0.27.0
+uvloop==0.19.0
 validators==0.22.0
+watchfiles==0.21.0
 wcwidth==0.2.12
 webcolors==1.13
 webencodings==0.5.1
 websocket-client==1.7.0
+websockets==12.0
 widgetsnbextension==4.0.9
 wrapt==1.16.0
 yarl==1.9.4
-zipp==3.17.0
+zipp==3.17.0