Hope final

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.json filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+**/.env
+**/.venv

.python-version

@@ -0,0 +1 @@
+3.13

Dockerfile

@@ -0,0 +1,44 @@
+FROM python:3.13-slim
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends gcc g++ cmake && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY src/ src/
+COPY packages/data_prep/generated/ packages/data_prep/generated/
+
+RUN uv pip install --system \
+    --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu \
+    "gradio>=6.12.0" \
+    "huggingface-hub>=0.30.0" \
+    "llama-cpp-python>=0.3.19" \
+    "llama-index>=0.14.20" \
+    "llama-index-embeddings-huggingface>=0.7.0" \
+    "numpy>=2.0.0"
+
+RUN useradd -m -u 1000 user
+USER user
+
+ENV HOME=/home/user \
+    PYTHONPATH=/app/src \
+    GRADIO_SERVER_NAME=0.0.0.0
+
+RUN python -c "\
+from huggingface_hub import hf_hub_download, list_repo_files; \
+files = list_repo_files('Jackrong/Qwen3.5-4B-Neo-GGUF'); \
+gguf = [f for f in files if f.endswith('.gguf')]; \
+target = next((f for f in gguf if 'Q4_K_M' in f.upper()), gguf[0]); \
+print(f'Downloading {target}...'); \
+hf_hub_download('Jackrong/Qwen3.5-4B-Neo-GGUF', target)"
+
+RUN python -c "\
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding; \
+HuggingFaceEmbedding(model_name='BAAI/bge-small-en-v1.5')"
+
+EXPOSE 7860
+
+CMD ["python", "-m", "big_data_application"]

README.md

@@ -9,3 +9,5 @@ license: mit
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+- uv add llama-cpp-python --index https://abetlen.github.io/llama-cpp-python/whl/cpu

packages/data_prep/.ipynb_checkpoints/main-checkpoint.ipynb

@@ -0,0 +1,294 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "6137a317",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from dotenv import load_dotenv\n",
+    "load_dotenv()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "4a7dc84b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import nest_asyncio\n",
+    "nest_asyncio.apply()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "a47086ec",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "SyntaxError",
+     "evalue": "invalid syntax (188846707.py, line 6)",
+     "output_type": "error",
+     "traceback": [
+      "  \u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[3]\u001b[39m\u001b[32m, line 6\u001b[39m\n\u001b[31m    \u001b[39m\u001b[31mfrom llama_index.llms.openai\u001b[39m\n                                ^\n\u001b[31mSyntaxError\u001b[39m\u001b[31m:\u001b[39m invalid syntax\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import datasets\n",
+    "from llama_index.core import PropertyGraphIndex, Document, Settings, load_index_from_storage\n",
+    "from llama_index.core.graph_stores import SimplePropertyGraphStore\n",
+    "from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n",
+    "from llama_index.llms.openailike"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "28b17221",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n"
+     ]
+    }
+   ],
+   "source": [
+    "og_data = datasets.load_dataset(\"gamino/wiki_medical_terms\", split=\"train[:100]\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "ca0aa288",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = [\n",
+    "    Document(id_=str(idx), text=text) for idx, text in enumerate(og_data[\"page_text\"])\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "53a4d694",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "100"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "6d77d82f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "graph_store = SimplePropertyGraphStore()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "e08d931b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "llm = OpenAI(\n",
+    "    model=\"DeepSeek-V3.2\",\n",
+    "    api_key=os.getenv(\"AZURE_API_KEY\"),\n",
+    "    api_base=\"https://thong-api-hub.services.ai.azure.com/models\",\n",
+    ")\n",
+    "Settings.llm = llm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "8d7d9e4a",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "ValueError",
+     "evalue": "Unknown model 'DeepSeek-V3.2'. Please provide a valid OpenAI model name in: o1, o1-2024-12-17, o1-pro, o1-pro-2025-03-19, o1-preview, o1-preview-2024-09-12, o1-mini, o1-mini-2024-09-12, o3-mini, o3-mini-2025-01-31, o3, o3-2025-04-16, o3-pro, o3-pro-2025-06-10, o4-mini, o4-mini-2025-04-16, gpt-5, gpt-5-2025-08-07, gpt-5-chat, gpt-5-chat-latest, gpt-5-mini, gpt-5-mini-2025-08-07, gpt-5-nano, gpt-5-nano-2025-08-07, gpt-5-pro, gpt-5-pro-2025-10-06, gpt-5.1, gpt-5.1-2025-11-13, gpt-5.1-chat-latest, gpt-5.2, gpt-5.2-2025-12-11, gpt-5.2-chat-latest, gpt-5.3, gpt-5.3-chat-latest, gpt-5.4, gpt-5.4-2026-03-05, gpt-5.4-mini, gpt-5.4-nano, gpt-5.4-chat-latest, gpt-4, gpt-4-32k, gpt-4-1106-preview, gpt-4-0125-preview, gpt-4-turbo-preview, gpt-4-vision-preview, gpt-4-1106-vision-preview, gpt-4-turbo-2024-04-09, gpt-4-turbo, gpt-4o, gpt-4o-audio-preview, gpt-4o-audio-preview-2024-12-17, gpt-4o-audio-preview-2024-10-01, gpt-4o-mini-audio-preview, gpt-4o-mini-audio-preview-2024-12-17, gpt-4o-2024-05-13, gpt-4o-2024-08-06, gpt-4o-2024-11-20, gpt-4.5-preview, gpt-4.5-preview-2025-02-27, chatgpt-4o-latest, gpt-4o-mini, gpt-4o-mini-2024-07-18, gpt-4-0613, gpt-4-32k-0613, gpt-4-0314, gpt-4-32k-0314, gpt-4.1, gpt-4.1-mini, gpt-4.1-nano, gpt-4.1-2025-04-14, gpt-4.1-mini-2025-04-14, gpt-4.1-nano-2025-04-14, gpt-3.5-turbo, gpt-3.5-turbo-16k, gpt-3.5-turbo-0125, gpt-3.5-turbo-1106, gpt-3.5-turbo-0613, gpt-3.5-turbo-16k-0613, gpt-3.5-turbo-0301, text-davinci-003, text-davinci-002, gpt-3.5-turbo-instruct, text-ada-001, text-babbage-001, text-curie-001, ada, babbage, curie, davinci, gpt-35-turbo-16k, gpt-35-turbo, gpt-35-turbo-0125, gpt-35-turbo-1106, gpt-35-turbo-0613, gpt-35-turbo-16k-0613",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
+      "\u001b[31mValueError\u001b[39m                                Traceback (most recent call last)",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[12]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m Settings.llm.complete(\u001b[33m\"Hello\"\u001b[39m)\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/llama_index_instrumentation/dispatcher.py:413\u001b[39m, in \u001b[36mDispatcher.span.<locals>.wrapper\u001b[39m\u001b[34m(func, instance, args, kwargs)\u001b[39m\n\u001b[32m    410\u001b[39m             _logger.debug(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mFailed to reset active_span_id: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m)\n\u001b[32m    412\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m413\u001b[39m     result = \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    414\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(result, asyncio.Future):\n\u001b[32m    415\u001b[39m         \u001b[38;5;66;03m# If the result is a Future, wrap it\u001b[39;00m\n\u001b[32m    416\u001b[39m         new_future = asyncio.ensure_future(result)\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/llama_index/core/llms/callbacks.py:447\u001b[39m, in \u001b[36mllm_completion_callback.<locals>.wrap.<locals>.wrapped_llm_predict\u001b[39m\u001b[34m(_self, *args, **kwargs)\u001b[39m\n\u001b[32m    444\u001b[39m     _self.rate_limiter.acquire()\n\u001b[32m    446\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m447\u001b[39m     f_return_val = \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_self\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    448\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m    449\u001b[39m     callback_manager.on_event_end(\n\u001b[32m    450\u001b[39m         CBEventType.LLM,\n\u001b[32m    451\u001b[39m         payload={EventPayload.EXCEPTION: e},\n\u001b[32m    452\u001b[39m         event_id=event_id,\n\u001b[32m    453\u001b[39m     )\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/llama_index/llms/openai/base.py:425\u001b[39m, in \u001b[36mOpenAI.complete\u001b[39m\u001b[34m(self, prompt, formatted, **kwargs)\u001b[39m\n\u001b[32m    420\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.modalities \u001b[38;5;129;01mand\u001b[39;00m \u001b[33m\"\u001b[39m\u001b[33maudio\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m.modalities:\n\u001b[32m    421\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[32m    422\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mAudio is not supported for completion. Use chat/achat instead.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    423\u001b[39m     )\n\u001b[32m--> \u001b[39m\u001b[32m425\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_use_chat_completions\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[32m    426\u001b[39m     complete_fn = chat_to_completion_decorator(\u001b[38;5;28mself\u001b[39m._chat)\n\u001b[32m    427\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/llama_index/llms/openai/base.py:444\u001b[39m, in \u001b[36mOpenAI._use_chat_completions\u001b[39m\u001b[34m(self, kwargs)\u001b[39m\n\u001b[32m    442\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[33m\"\u001b[39m\u001b[33muse_chat_completions\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m kwargs:\n\u001b[32m    443\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m kwargs[\u001b[33m\"\u001b[39m\u001b[33muse_chat_completions\u001b[39m\u001b[33m\"\u001b[39m]\n\u001b[32m--> \u001b[39m\u001b[32m444\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmetadata\u001b[49m.is_chat_model\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/llama_index/llms/openai/base.py:385\u001b[39m, in \u001b[36mOpenAI.metadata\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m    382\u001b[39m \u001b[38;5;129m@property\u001b[39m\n\u001b[32m    383\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mmetadata\u001b[39m(\u001b[38;5;28mself\u001b[39m) -> LLMMetadata:\n\u001b[32m    384\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m LLMMetadata(\n\u001b[32m--> \u001b[39m\u001b[32m385\u001b[39m         context_window=\u001b[43mopenai_modelname_to_contextsize\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_get_model_name\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m,\n\u001b[32m    386\u001b[39m         num_output=\u001b[38;5;28mself\u001b[39m.max_tokens \u001b[38;5;129;01mor\u001b[39;00m -\u001b[32m1\u001b[39m,\n\u001b[32m    387\u001b[39m         is_chat_model=is_chat_model(model=\u001b[38;5;28mself\u001b[39m._get_model_name()),\n\u001b[32m    388\u001b[39m         is_function_calling_model=is_function_calling_model(\n\u001b[32m    389\u001b[39m             model=\u001b[38;5;28mself\u001b[39m._get_model_name()\n\u001b[32m    390\u001b[39m         ),\n\u001b[32m    391\u001b[39m         model_name=\u001b[38;5;28mself\u001b[39m.model,\n\u001b[32m    392\u001b[39m         \u001b[38;5;66;03m# TODO: Temp for O1 beta\u001b[39;00m\n\u001b[32m    393\u001b[39m         system_role=MessageRole.USER\n\u001b[32m    394\u001b[39m         \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.model \u001b[38;5;129;01min\u001b[39;00m O1_MODELS\n\u001b[32m    395\u001b[39m         \u001b[38;5;28;01melse\u001b[39;00m MessageRole.SYSTEM,\n\u001b[32m    396\u001b[39m     )\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/llama_index/llms/openai/utils.py:383\u001b[39m, in \u001b[36mopenai_modelname_to_contextsize\u001b[39m\u001b[34m(modelname)\u001b[39m\n\u001b[32m    378\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[32m    379\u001b[39m         \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mOpenAI model \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmodelname\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m has been discontinued. \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    380\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mPlease choose another model.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    381\u001b[39m     )\n\u001b[32m    382\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m modelname \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m ALL_AVAILABLE_MODELS:\n\u001b[32m--> \u001b[39m\u001b[32m383\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[32m    384\u001b[39m         \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mUnknown model \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmodelname\u001b[38;5;132;01m!r}\u001b[39;00m\u001b[33m. Please provide a valid OpenAI model name in:\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    385\u001b[39m         \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m, \u001b[39m\u001b[33m'\u001b[39m.join(ALL_AVAILABLE_MODELS.keys())\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m    386\u001b[39m     )\n\u001b[32m    387\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m ALL_AVAILABLE_MODELS[modelname]\n",
+      "\u001b[31mValueError\u001b[39m: Unknown model 'DeepSeek-V3.2'. Please provide a valid OpenAI model name in: o1, o1-2024-12-17, o1-pro, o1-pro-2025-03-19, o1-preview, o1-preview-2024-09-12, o1-mini, o1-mini-2024-09-12, o3-mini, o3-mini-2025-01-31, o3, o3-2025-04-16, o3-pro, o3-pro-2025-06-10, o4-mini, o4-mini-2025-04-16, gpt-5, gpt-5-2025-08-07, gpt-5-chat, gpt-5-chat-latest, gpt-5-mini, gpt-5-mini-2025-08-07, gpt-5-nano, gpt-5-nano-2025-08-07, gpt-5-pro, gpt-5-pro-2025-10-06, gpt-5.1, gpt-5.1-2025-11-13, gpt-5.1-chat-latest, gpt-5.2, gpt-5.2-2025-12-11, gpt-5.2-chat-latest, gpt-5.3, gpt-5.3-chat-latest, gpt-5.4, gpt-5.4-2026-03-05, gpt-5.4-mini, gpt-5.4-nano, gpt-5.4-chat-latest, gpt-4, gpt-4-32k, gpt-4-1106-preview, gpt-4-0125-preview, gpt-4-turbo-preview, gpt-4-vision-preview, gpt-4-1106-vision-preview, gpt-4-turbo-2024-04-09, gpt-4-turbo, gpt-4o, gpt-4o-audio-preview, gpt-4o-audio-preview-2024-12-17, gpt-4o-audio-preview-2024-10-01, gpt-4o-mini-audio-preview, gpt-4o-mini-audio-preview-2024-12-17, gpt-4o-2024-05-13, gpt-4o-2024-08-06, gpt-4o-2024-11-20, gpt-4.5-preview, gpt-4.5-preview-2025-02-27, chatgpt-4o-latest, gpt-4o-mini, gpt-4o-mini-2024-07-18, gpt-4-0613, gpt-4-32k-0613, gpt-4-0314, gpt-4-32k-0314, gpt-4.1, gpt-4.1-mini, gpt-4.1-nano, gpt-4.1-2025-04-14, gpt-4.1-mini-2025-04-14, gpt-4.1-nano-2025-04-14, gpt-3.5-turbo, gpt-3.5-turbo-16k, gpt-3.5-turbo-0125, gpt-3.5-turbo-1106, gpt-3.5-turbo-0613, gpt-3.5-turbo-16k-0613, gpt-3.5-turbo-0301, text-davinci-003, text-davinci-002, gpt-3.5-turbo-instruct, text-ada-001, text-babbage-001, text-curie-001, ada, babbage, curie, davinci, gpt-35-turbo-16k, gpt-35-turbo, gpt-35-turbo-0125, gpt-35-turbo-1106, gpt-35-turbo-0613, gpt-35-turbo-16k-0613"
+     ]
+    }
+   ],
+   "source": [
+    "Settings.llm.complete(\"Hello\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "711fd553",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Loading weights: 100%|██████████| 199/199 [00:00<00:00, 2394.67it/s]\n",
+      "\u001b[1mBertModel LOAD REPORT\u001b[0m from: BAAI/bge-small-en-v1.5\n",
+      "Key                     | Status     |  | \n",
+      "------------------------+------------+--+-\n",
+      "embeddings.position_ids | UNEXPECTED |  | \n",
+      "\n",
+      "Notes:\n",
+      "- UNEXPECTED:\tcan be ignored when loading from different task/architecture; not ok if you expect identical arch.\n"
+     ]
+    }
+   ],
+   "source": [
+    "embed_model = HuggingFaceEmbedding(model_name=\"BAAI/bge-small-en-v1.5\")\n",
+    "\n",
+    "Settings.embed_model = embed_model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "6403ff0b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Applying transformations:   0%|          | 0/1 [00:00<?, ?it/s]Task was destroyed but it is pending!\n",
+      "task: <Task pending name='Task-1319' coro=<_async_in_context.<locals>.run_in_context() running at /media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/ipykernel/utils.py:60> wait_for=<Task pending name='Task-1321' coro=<Kernel.shell_main() running at /media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/ipykernel/kernelbase.py:597> cb=[Task.__wakeup()]> cb=[ZMQStream._run_callback.<locals>._log_error() at /media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/zmq/eventloop/zmqstream.py:563]>\n",
+      "/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/pydantic/main.py:112: RuntimeWarning: coroutine 'Kernel.shell_main' was never awaited\n",
+      "  'validate_assignment': lambda model, name, val: model.__pydantic_validator__.validate_assignment(model, name, val),  # pyright: ignore[reportAssignmentType]\n",
+      "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n",
+      "Task was destroyed but it is pending!\n",
+      "task: <Task pending name='Task-1321' coro=<Kernel.shell_main() running at /media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/ipykernel/kernelbase.py:597> cb=[Task.__wakeup()]>\n",
+      "Applying transformations: 100%|██████████| 1/1 [00:01<00:00,  1.66s/it]\n",
+      "Applying transformations: 100%|██████████| 2/2 [09:57<00:00, 298.86s/it]\n",
+      "Generating embeddings:   2%|▏         | 10/499 [00:01<01:10,  6.99it/s]"
+     ]
+    },
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
+      "\u001b[31mKeyboardInterrupt\u001b[39m                         Traceback (most recent call last)",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[11]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m index = \u001b[43mPropertyGraphIndex\u001b[49m\u001b[43m.\u001b[49m\u001b[43mfrom_documents\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m      2\u001b[39m \u001b[43m    \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mshow_progress\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mproperty_graph_store\u001b[49m\u001b[43m=\u001b[49m\u001b[43mgraph_store\u001b[49m\u001b[43m,\u001b[49m\u001b[43muse_async\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\n\u001b[32m      3\u001b[39m \u001b[43m)\u001b[49m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/llama_index/core/indices/base.py:122\u001b[39m, in \u001b[36mBaseIndex.from_documents\u001b[39m\u001b[34m(cls, documents, storage_context, show_progress, callback_manager, transformations, **kwargs)\u001b[39m\n\u001b[32m    113\u001b[39m     docstore.set_document_hash(doc.id_, doc.hash)\n\u001b[32m    115\u001b[39m nodes = run_transformations(\n\u001b[32m    116\u001b[39m     documents,  \u001b[38;5;66;03m# type: ignore\u001b[39;00m\n\u001b[32m    117\u001b[39m     transformations,\n\u001b[32m    118\u001b[39m     show_progress=show_progress,\n\u001b[32m    119\u001b[39m     **kwargs,\n\u001b[32m    120\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m122\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[32m    123\u001b[39m \u001b[43m    \u001b[49m\u001b[43mnodes\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnodes\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    124\u001b[39m \u001b[43m    \u001b[49m\u001b[43mstorage_context\u001b[49m\u001b[43m=\u001b[49m\u001b[43mstorage_context\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    125\u001b[39m \u001b[43m    \u001b[49m\u001b[43mcallback_manager\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcallback_manager\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    126\u001b[39m \u001b[43m    \u001b[49m\u001b[43mshow_progress\u001b[49m\u001b[43m=\u001b[49m\u001b[43mshow_progress\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    127\u001b[39m \u001b[43m    \u001b[49m\u001b[43mtransformations\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtransformations\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    128\u001b[39m \u001b[43m    \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    129\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/llama_index/core/indices/property_graph/base.py:136\u001b[39m, in \u001b[36mPropertyGraphIndex.__init__\u001b[39m\u001b[34m(self, nodes, llm, kg_extractors, property_graph_store, vector_store, use_async, embed_model, embed_kg_nodes, callback_manager, transformations, storage_context, show_progress, **kwargs)\u001b[39m\n\u001b[32m    130\u001b[39m \u001b[38;5;28mself\u001b[39m._embed_kg_nodes = embed_kg_nodes\n\u001b[32m    131\u001b[39m \u001b[38;5;28mself\u001b[39m._override_vector_store = (\n\u001b[32m    132\u001b[39m     vector_store \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m    133\u001b[39m     \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m storage_context.property_graph_store.supports_vector_queries\n\u001b[32m    134\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m136\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[34;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[32m    137\u001b[39m \u001b[43m    \u001b[49m\u001b[43mnodes\u001b[49m\u001b[43m=\u001b[49m\u001b[43mnodes\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    138\u001b[39m \u001b[43m    \u001b[49m\u001b[43mcallback_manager\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcallback_manager\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    139\u001b[39m \u001b[43m    \u001b[49m\u001b[43mstorage_context\u001b[49m\u001b[43m=\u001b[49m\u001b[43mstorage_context\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    140\u001b[39m \u001b[43m    \u001b[49m\u001b[43mtransformations\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtransformations\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    141\u001b[39m \u001b[43m    \u001b[49m\u001b[43mshow_progress\u001b[49m\u001b[43m=\u001b[49m\u001b[43mshow_progress\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    142\u001b[39m \u001b[43m    \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    143\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/llama_index/core/indices/base.py:79\u001b[39m, in \u001b[36mBaseIndex.__init__\u001b[39m\u001b[34m(self, nodes, objects, index_struct, storage_context, callback_manager, transformations, show_progress, **kwargs)\u001b[39m\n\u001b[32m     77\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m index_struct \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m     78\u001b[39m     nodes = nodes \u001b[38;5;129;01mor\u001b[39;00m []\n\u001b[32m---> \u001b[39m\u001b[32m79\u001b[39m     index_struct = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mbuild_index_from_nodes\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m     80\u001b[39m \u001b[43m        \u001b[49m\u001b[43mnodes\u001b[49m\u001b[43m \u001b[49m\u001b[43m+\u001b[49m\u001b[43m \u001b[49m\u001b[43mobjects\u001b[49m\u001b[43m,\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# type: ignore\u001b[39;49;00m\n\u001b[32m     81\u001b[39m \u001b[43m        \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# type: ignore\u001b[39;49;00m\n\u001b[32m     82\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     83\u001b[39m \u001b[38;5;28mself\u001b[39m._index_struct = index_struct\n\u001b[32m     84\u001b[39m \u001b[38;5;28mself\u001b[39m._storage_context.index_store.add_index_struct(\u001b[38;5;28mself\u001b[39m._index_struct)\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/llama_index/core/indices/base.py:189\u001b[39m, in \u001b[36mBaseIndex.build_index_from_nodes\u001b[39m\u001b[34m(self, nodes, **build_kwargs)\u001b[39m\n\u001b[32m    187\u001b[39m \u001b[38;5;250m\u001b[39m\u001b[33;03m\"\"\"Build the index from nodes.\"\"\"\u001b[39;00m\n\u001b[32m    188\u001b[39m \u001b[38;5;28mself\u001b[39m._docstore.add_documents(nodes, allow_update=\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m--> \u001b[39m\u001b[32m189\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_build_index_from_nodes\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnodes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mbuild_kwargs\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/llama_index/core/indices/property_graph/base.py:336\u001b[39m, in \u001b[36mPropertyGraphIndex._build_index_from_nodes\u001b[39m\u001b[34m(self, nodes, **build_kwargs)\u001b[39m\n\u001b[32m    332\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m_build_index_from_nodes\u001b[39m(\n\u001b[32m    333\u001b[39m     \u001b[38;5;28mself\u001b[39m, nodes: Optional[Sequence[BaseNode]], **build_kwargs: Any\n\u001b[32m    334\u001b[39m ) -> IndexLPG:\n\u001b[32m    335\u001b[39m \u001b[38;5;250m    \u001b[39m\u001b[33;03m\"\"\"Build index from nodes.\"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m336\u001b[39m     nodes = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_insert_nodes\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnodes\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43m[\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    338\u001b[39m     \u001b[38;5;66;03m# this isn't really used or needed\u001b[39;00m\n\u001b[32m    339\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m IndexLPG()\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/llama_index/core/indices/property_graph/base.py:265\u001b[39m, in \u001b[36mPropertyGraphIndex._insert_nodes\u001b[39m\u001b[34m(self, nodes)\u001b[39m\n\u001b[32m    259\u001b[39m     embeddings = asyncio.run(\n\u001b[32m    260\u001b[39m         \u001b[38;5;28mself\u001b[39m._embed_model.aget_text_embedding_batch(\n\u001b[32m    261\u001b[39m             node_texts, show_progress=\u001b[38;5;28mself\u001b[39m._show_progress\n\u001b[32m    262\u001b[39m         )\n\u001b[32m    263\u001b[39m     )\n\u001b[32m    264\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m265\u001b[39m     embeddings = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_embed_model\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_text_embedding_batch\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    266\u001b[39m \u001b[43m        \u001b[49m\u001b[43mnode_texts\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mshow_progress\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_show_progress\u001b[49m\n\u001b[32m    267\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    269\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m node, embedding \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(nodes, embeddings):\n\u001b[32m    270\u001b[39m     node.embedding = embedding\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/llama_index_instrumentation/dispatcher.py:413\u001b[39m, in \u001b[36mDispatcher.span.<locals>.wrapper\u001b[39m\u001b[34m(func, instance, args, kwargs)\u001b[39m\n\u001b[32m    410\u001b[39m             _logger.debug(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mFailed to reset active_span_id: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m)\n\u001b[32m    412\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m413\u001b[39m     result = \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    414\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(result, asyncio.Future):\n\u001b[32m    415\u001b[39m         \u001b[38;5;66;03m# If the result is a Future, wrap it\u001b[39;00m\n\u001b[32m    416\u001b[39m         new_future = asyncio.ensure_future(result)\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/llama_index/core/base/embeddings/base.py:508\u001b[39m, in \u001b[36mBaseEmbedding.get_text_embedding_batch\u001b[39m\u001b[34m(self, texts, show_progress, **kwargs)\u001b[39m\n\u001b[32m    506\u001b[39m     \u001b[38;5;28mself\u001b[39m.rate_limiter.acquire()\n\u001b[32m    507\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m.embeddings_cache:\n\u001b[32m--> \u001b[39m\u001b[32m508\u001b[39m     embeddings = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_get_text_embeddings\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcur_batch\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    509\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.embeddings_cache \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m    510\u001b[39m     embeddings = \u001b[38;5;28mself\u001b[39m._get_text_embeddings_cached(cur_batch)\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/llama_index/embeddings/huggingface/base.py:333\u001b[39m, in \u001b[36mHuggingFaceEmbedding._get_text_embeddings\u001b[39m\u001b[34m(self, texts)\u001b[39m\n\u001b[32m    322\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m_get_text_embeddings\u001b[39m(\u001b[38;5;28mself\u001b[39m, texts: List[\u001b[38;5;28mstr\u001b[39m]) -> List[List[\u001b[38;5;28mfloat\u001b[39m]]:\n\u001b[32m    323\u001b[39m \u001b[38;5;250m    \u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m    324\u001b[39m \u001b[33;03m    Generates Embeddings for text.\u001b[39;00m\n\u001b[32m    325\u001b[39m \n\u001b[32m   (...)\u001b[39m\u001b[32m    331\u001b[39m \n\u001b[32m    332\u001b[39m \u001b[33;03m    \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m333\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_embed\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtexts\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprompt_name\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mtext\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/llama_index/embeddings/huggingface/base.py:268\u001b[39m, in \u001b[36mHuggingFaceEmbedding._embed\u001b[39m\u001b[34m(self, inputs, prompt_name)\u001b[39m\n\u001b[32m    248\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m_embed\u001b[39m(\n\u001b[32m    249\u001b[39m     \u001b[38;5;28mself\u001b[39m,\n\u001b[32m    250\u001b[39m     inputs: List[Union[\u001b[38;5;28mstr\u001b[39m, BytesIO]],\n\u001b[32m    251\u001b[39m     prompt_name: Optional[\u001b[38;5;28mstr\u001b[39m] = \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[32m    252\u001b[39m ) -> List[List[\u001b[38;5;28mfloat\u001b[39m]]:\n\u001b[32m    253\u001b[39m \u001b[38;5;250m    \u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m    254\u001b[39m \u001b[33;03m    Generates Embeddings with input validation and retry mechanism.\u001b[39;00m\n\u001b[32m    255\u001b[39m \n\u001b[32m   (...)\u001b[39m\u001b[32m    266\u001b[39m \n\u001b[32m    267\u001b[39m \u001b[33;03m    \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m268\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_embed_with_retry\u001b[49m\u001b[43m(\u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprompt_name\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/tenacity/__init__.py:331\u001b[39m, in \u001b[36mBaseRetrying.wraps.<locals>.wrapped_f\u001b[39m\u001b[34m(*args, **kw)\u001b[39m\n\u001b[32m    329\u001b[39m copy = \u001b[38;5;28mself\u001b[39m.copy()\n\u001b[32m    330\u001b[39m wrapped_f.statistics = copy.statistics  \u001b[38;5;66;03m# type: ignore[attr-defined]\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m331\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mcopy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkw\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/tenacity/__init__.py:470\u001b[39m, in \u001b[36mRetrying.__call__\u001b[39m\u001b[34m(self, fn, *args, **kwargs)\u001b[39m\n\u001b[32m    468\u001b[39m retry_state = RetryCallState(retry_object=\u001b[38;5;28mself\u001b[39m, fn=fn, args=args, kwargs=kwargs)\n\u001b[32m    469\u001b[39m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m470\u001b[39m     do = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43miter\u001b[49m\u001b[43m(\u001b[49m\u001b[43mretry_state\u001b[49m\u001b[43m=\u001b[49m\u001b[43mretry_state\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    471\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(do, DoAttempt):\n\u001b[32m    472\u001b[39m         \u001b[38;5;28;01mtry\u001b[39;00m:\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/tenacity/__init__.py:371\u001b[39m, in \u001b[36mBaseRetrying.iter\u001b[39m\u001b[34m(self, retry_state)\u001b[39m\n\u001b[32m    369\u001b[39m result = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m    370\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m action \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m.iter_state.actions:\n\u001b[32m--> \u001b[39m\u001b[32m371\u001b[39m     result = \u001b[43maction\u001b[49m\u001b[43m(\u001b[49m\u001b[43mretry_state\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    372\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/tenacity/__init__.py:393\u001b[39m, in \u001b[36mBaseRetrying._post_retry_check_actions.<locals>.<lambda>\u001b[39m\u001b[34m(rs)\u001b[39m\n\u001b[32m    391\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m_post_retry_check_actions\u001b[39m(\u001b[38;5;28mself\u001b[39m, retry_state: \u001b[33m\"\u001b[39m\u001b[33mRetryCallState\u001b[39m\u001b[33m\"\u001b[39m) -> \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m    392\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m.iter_state.is_explicit_retry \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m.iter_state.retry_run_result):\n\u001b[32m--> \u001b[39m\u001b[32m393\u001b[39m         \u001b[38;5;28mself\u001b[39m._add_action_func(\u001b[38;5;28;01mlambda\u001b[39;00m rs: \u001b[43mrs\u001b[49m\u001b[43m.\u001b[49m\u001b[43moutcome\u001b[49m\u001b[43m.\u001b[49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[32m    394\u001b[39m         \u001b[38;5;28;01mreturn\u001b[39;00m\n\u001b[32m    396\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.after \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~/.local/share/uv/python/cpython-3.13.5-linux-x86_64-gnu/lib/python3.13/concurrent/futures/_base.py:449\u001b[39m, in \u001b[36mFuture.result\u001b[39m\u001b[34m(self, timeout)\u001b[39m\n\u001b[32m    447\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m CancelledError()\n\u001b[32m    448\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m._state == FINISHED:\n\u001b[32m--> \u001b[39m\u001b[32m449\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m__get_result\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    451\u001b[39m \u001b[38;5;28mself\u001b[39m._condition.wait(timeout)\n\u001b[32m    453\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m._state \u001b[38;5;129;01min\u001b[39;00m [CANCELLED, CANCELLED_AND_NOTIFIED]:\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~/.local/share/uv/python/cpython-3.13.5-linux-x86_64-gnu/lib/python3.13/concurrent/futures/_base.py:401\u001b[39m, in \u001b[36mFuture.__get_result\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m    399\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m._exception \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m    400\u001b[39m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m401\u001b[39m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28mself\u001b[39m._exception\n\u001b[32m    402\u001b[39m     \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[32m    403\u001b[39m         \u001b[38;5;66;03m# Break a reference cycle with the exception in self._exception\u001b[39;00m\n\u001b[32m    404\u001b[39m         \u001b[38;5;28mself\u001b[39m = \u001b[38;5;28;01mNone\u001b[39;00m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/tenacity/__init__.py:473\u001b[39m, in \u001b[36mRetrying.__call__\u001b[39m\u001b[34m(self, fn, *args, **kwargs)\u001b[39m\n\u001b[32m    471\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(do, DoAttempt):\n\u001b[32m    472\u001b[39m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m473\u001b[39m         result = \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    474\u001b[39m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m:  \u001b[38;5;66;03m# noqa: B902\u001b[39;00m\n\u001b[32m    475\u001b[39m         retry_state.set_exception(sys.exc_info())  \u001b[38;5;66;03m# type: ignore[arg-type]\u001b[39;00m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/llama_index/embeddings/huggingface/base.py:236\u001b[39m, in \u001b[36mHuggingFaceEmbedding._embed_with_retry\u001b[39m\u001b[34m(self, inputs, prompt_name)\u001b[39m\n\u001b[32m    234\u001b[39m         \u001b[38;5;28mself\u001b[39m._model.stop_multi_process_pool(pool=pool)\n\u001b[32m    235\u001b[39m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m236\u001b[39m         emb = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_model\u001b[49m\u001b[43m.\u001b[49m\u001b[43mencode\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    237\u001b[39m \u001b[43m            \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    238\u001b[39m \u001b[43m            \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43membed_batch_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    239\u001b[39m \u001b[43m            \u001b[49m\u001b[43mprompt_name\u001b[49m\u001b[43m=\u001b[49m\u001b[43mprompt_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    240\u001b[39m \u001b[43m            \u001b[49m\u001b[43mnormalize_embeddings\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mnormalize\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    241\u001b[39m \u001b[43m            \u001b[49m\u001b[43mshow_progress_bar\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mshow_progress_bar\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    242\u001b[39m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    243\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m emb.tolist()\n\u001b[32m    244\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/torch/utils/_contextlib.py:124\u001b[39m, in \u001b[36mcontext_decorator.<locals>.decorate_context\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m    120\u001b[39m \u001b[38;5;129m@functools\u001b[39m.wraps(func)\n\u001b[32m    121\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mdecorate_context\u001b[39m(*args, **kwargs):\n\u001b[32m    122\u001b[39m     \u001b[38;5;66;03m# pyrefly: ignore [bad-context-manager]\u001b[39;00m\n\u001b[32m    123\u001b[39m     \u001b[38;5;28;01mwith\u001b[39;00m ctx_factory():\n\u001b[32m--> \u001b[39m\u001b[32m124\u001b[39m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/sentence_transformers/util/decorators.py:41\u001b[39m, in \u001b[36mdeprecated_kwargs.<locals>.decorator.<locals>.wrapper\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m     39\u001b[39m         \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m     40\u001b[39m             kwargs.pop(old_name)\n\u001b[32m---> \u001b[39m\u001b[32m41\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/sentence_transformers/sentence_transformer/model.py:682\u001b[39m, in \u001b[36mSentenceTransformer.encode\u001b[39m\u001b[34m(self, inputs, prompt_name, prompt, batch_size, show_progress_bar, output_value, precision, convert_to_numpy, convert_to_tensor, device, normalize_embeddings, truncate_dim, pool, chunk_size, **kwargs)\u001b[39m\n\u001b[32m    680\u001b[39m             embeddings = torch.nn.functional.normalize(embeddings, p=\u001b[32m2\u001b[39m, dim=\u001b[32m1\u001b[39m)\n\u001b[32m    681\u001b[39m         \u001b[38;5;28;01mif\u001b[39;00m convert_to_numpy:\n\u001b[32m--> \u001b[39m\u001b[32m682\u001b[39m             embeddings = \u001b[43membeddings\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcpu\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    684\u001b[39m     all_embeddings.extend(embeddings)\n\u001b[32m    686\u001b[39m all_embeddings = [all_embeddings[idx] \u001b[38;5;28;01mfor\u001b[39;00m idx \u001b[38;5;129;01min\u001b[39;00m np.argsort(length_sorted_idx)]\n",
+      "\u001b[31mKeyboardInterrupt\u001b[39m: "
+     ]
+    }
+   ],
+   "source": [
+    "index = PropertyGraphIndex.from_documents(\n",
+    "    data, show_progress=True, property_graph_store=graph_store,use_async=False\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "abedc381",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index.storage_context.persist(\"./storage2\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "b5a54ce9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index.property_graph_store.save_networkx_graph(name=\"./kg.html\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

packages/data_prep/.ipynb_checkpoints/main2-checkpoint.ipynb

@@ -0,0 +1,176 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c23fd4cd",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mRunning cells with '.venv (Python 3.13.5)' requires the ipykernel package.\n",
+      "\u001b[1;31mInstall 'ipykernel' into the Python environment. \n",
+      "\u001b[1;31mCommand: '/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/packages/data_prep/.venv/bin/python -m pip install ipykernel -U --force-reinstall'"
+     ]
+    }
+   ],
+   "source": [
+    "from dotenv import load_dotenv\n",
+    "load_dotenv()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "d45ba734",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import nest_asyncio\n",
+    "nest_asyncio.apply()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "0c69d6c4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/.venv/lib/python3.13/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import datasets\n",
+    "from llama_index.core import PropertyGraphIndex, Document, Settings, load_index_from_storage, StorageContext\n",
+    "from llama_index.core.graph_stores import SimplePropertyGraphStore\n",
+    "from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n",
+    "from llama_index.llms.openai import OpenAI"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "1eb7b186",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "llm = OpenAI(\n",
+    "    model=\"DeepSeek-V3.2\",\n",
+    "    api_key=os.getenv(\"AZURE_API_KEY\"),\n",
+    "    api_base=\"https://thong-api-hub.services.ai.azure.com/models\",\n",
+    ")\n",
+    "Settings.llm = llm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "351a1c7a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n",
+      "Loading weights: 100%|██████████| 199/199 [00:00<00:00, 11205.98it/s]\n",
+      "\u001b[1mBertModel LOAD REPORT\u001b[0m from: BAAI/bge-small-en-v1.5\n",
+      "Key                     | Status     |  | \n",
+      "------------------------+------------+--+-\n",
+      "embeddings.position_ids | UNEXPECTED |  | \n",
+      "\n",
+      "Notes:\n",
+      "- UNEXPECTED:\tcan be ignored when loading from different task/architecture; not ok if you expect identical arch.\n"
+     ]
+    }
+   ],
+   "source": [
+    "embed_model = HuggingFaceEmbedding(model_name=\"BAAI/bge-small-en-v1.5\")\n",
+    "\n",
+    "Settings.embed_model = embed_model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "36defa9e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "storage_context = StorageContext.from_defaults(persist_dir=\"./storage\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "8d5f7f31",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index = load_index_from_storage(storage_context)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "dfdccca1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index.property_graph_store.save_networkx_graph(name=\"./kg.html\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "46bc10ba",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from typing import cast\n",
+    "tmp = cast(SimplePropertyGraphStore, index.property_graph_store)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dfb604fd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tmp.save_networkx_graph"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

packages/data_prep/enhance.ipynb

@@ -0,0 +1,510 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "c23fd4cd",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from dotenv import load_dotenv\n",
+    "load_dotenv()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "d45ba734",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import nest_asyncio\n",
+    "nest_asyncio.apply()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "0c69d6c4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from llama_index.core import PropertyGraphIndex, Settings, load_index_from_storage, StorageContext\n",
+    "from llama_index.core.graph_stores import SimplePropertyGraphStore\n",
+    "from llama_index.core.graph_stores.types import KG_SOURCE_REL, Relation, LabelledPropertyGraph\n",
+    "from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n",
+    "from llama_index.llms.openai_like import OpenAILike"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7b6858c1",
+   "metadata": {},
+   "source": [
+    "### Load the generated Graph"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "1eb7b186",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "llm = OpenAILike(\n",
+    "    model=\"DeepSeek-V3.2\",\n",
+    "    api_key=os.getenv(\"AZURE_API_KEY\"),\n",
+    "    api_base=\"https://thong-api-hub.openai.azure.com/openai/v1\",\n",
+    "    is_chat_model=True,\n",
+    "    timeout=300,\n",
+    ")\n",
+    "Settings.llm = llm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "351a1c7a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n",
+      "Loading weights: 100%|██████████| 199/199 [00:00<00:00, 2288.13it/s]\n",
+      "\u001b[1mBertModel LOAD REPORT\u001b[0m from: BAAI/bge-small-en-v1.5\n",
+      "Key                     | Status     |  | \n",
+      "------------------------+------------+--+-\n",
+      "embeddings.position_ids | UNEXPECTED |  | \n",
+      "\n",
+      "Notes:\n",
+      "- UNEXPECTED:\tcan be ignored when loading from different task/architecture; not ok if you expect identical arch.\n"
+     ]
+    }
+   ],
+   "source": [
+    "embed_model = HuggingFaceEmbedding(model_name=\"BAAI/bge-small-en-v1.5\")\n",
+    "\n",
+    "Settings.embed_model = embed_model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "36defa9e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "storage_context = StorageContext.from_defaults(persist_dir=\"./generated\")\n",
+    "\n",
+    "index = load_index_from_storage(storage_context)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "id": "8615e444",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from typing import cast\n",
+    "\n",
+    "index = cast(PropertyGraphIndex, index)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "id": "0cb1ccf6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "property_graph_store = cast(SimplePropertyGraphStore, index.property_graph_store)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "01c9ab47",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "graph = cast(LabelledPropertyGraph, property_graph_store.graph)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "id": "c455850d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1039"
+      ]
+     },
+     "execution_count": 32,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(graph.triplets)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "93f60fe1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Add source relationship, I forgor to add ImplicitExtract in the previous script\n",
+    "for node in graph.get_all_nodes():\n",
+    "    if \"triplet_source_id\" in node.properties:\n",
+    "        graph.add_relation(\n",
+    "            Relation(\n",
+    "                label=KG_SOURCE_REL,\n",
+    "                source_id=node.id,\n",
+    "                target_id=node.properties[\"triplet_source_id\"],\n",
+    "            )\n",
+    "        )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "id": "17606922",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "2383"
+      ]
+     },
+     "execution_count": 34,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(graph.triplets)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 53,
+   "id": "b893ab85",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Save it back\n",
+    "index.storage_context.persist(\"./shitass\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b911da97",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "id_to_int = { node: i for i, node in enumerate(graph.nodes)}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 66,
+   "id": "310a330e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "int_to_id = { v: k for k, v in id_to_int.items()}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5e71094c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "json.dump(id_to_int, open(\"generated/id_to_int.json\", 'w'))\n",
+    "json.dump(id_to_int, open(\"generated/int_to_id.json\", 'w'))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8c4d7d6e",
+   "metadata": {},
+   "source": [
+    "### Construct model + training"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "id": "fab9274c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "import torch.nn.functional as F\n",
+    "from torch_geometric.data import Data\n",
+    "from torch_geometric.nn import LightGCN"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "id": "dc054e89",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "edges = [\n",
+    "    [id_to_int[rel.source_id], id_to_int[rel.target_id]]\n",
+    "    for rel in graph.relations.values()\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "id": "bd9ed0ed",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# [2, num_edges]\n",
+    "edge_index = torch.tensor(edges, dtype=torch.long)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "id": "b6162da5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "initial_semantic_features = torch.tensor(embed_model.get_text_embedding_batch([\n",
+    "    getattr(node, \"text\", node.label) for node in graph.nodes.values()\n",
+    "]), dtype=torch.float)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "id": "37b00c3e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = Data(x=initial_semantic_features, edge_index=edge_index)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 49,
+   "id": "78592e84",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "torch.Size([1449, 384])"
+      ]
+     },
+     "execution_count": 49,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "initial_semantic_features.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1d138911",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "torch.save(data, \"generated/graph.pt\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 62,
+   "id": "f7ba02e9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "light_gcn = LightGCN(num_nodes=initial_semantic_features.shape[0], embedding_dim=initial_semantic_features.shape[1], num_layers=3).to(\"cuda\")\n",
+    "\n",
+    "# copy embedding qua\n",
+    "light_gcn.embedding = light_gcn.embedding.from_pretrained(initial_semantic_features).to(\"cuda\").requires_grad_()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 54,
+   "id": "6d2b44af",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "edge_index_pyg = edge_index.t().contiguous().to(\"cuda\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 63,
+   "id": "c1015cd8",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Epoch  20/300, Loss: 0.6267\n",
+      "Epoch  40/300, Loss: 0.5856\n",
+      "Epoch  60/300, Loss: 0.5324\n",
+      "Epoch  80/300, Loss: 0.4638\n",
+      "Epoch 100/300, Loss: 0.3999\n",
+      "Epoch 120/300, Loss: 0.3355\n",
+      "Epoch 140/300, Loss: 0.2812\n",
+      "Epoch 160/300, Loss: 0.2454\n",
+      "Epoch 180/300, Loss: 0.2091\n",
+      "Epoch 200/300, Loss: 0.1841\n",
+      "Epoch 220/300, Loss: 0.1624\n",
+      "Epoch 240/300, Loss: 0.1443\n",
+      "Epoch 260/300, Loss: 0.1302\n",
+      "Epoch 280/300, Loss: 0.1144\n",
+      "Epoch 300/300, Loss: 0.1047\n",
+      "Training complete.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from torch_geometric.utils import negative_sampling\n",
+    "\n",
+    "optimizer = torch.optim.Adam(light_gcn.parameters(), lr=1e-3)\n",
+    "num_epochs = 300\n",
+    "\n",
+    "for epoch in range(num_epochs):\n",
+    "    light_gcn.train()\n",
+    "    optimizer.zero_grad()\n",
+    "\n",
+    "    emb = light_gcn.get_embedding(edge_index_pyg)\n",
+    "\n",
+    "    pos_score = (emb[edge_index_pyg[0]] * emb[edge_index_pyg[1]]).sum(dim=-1)\n",
+    "\n",
+    "    neg_edge = negative_sampling(\n",
+    "        edge_index=edge_index_pyg,\n",
+    "        num_nodes=data.num_nodes,\n",
+    "        num_neg_samples=edge_index_pyg.size(1),\n",
+    "    )\n",
+    "    neg_score = (emb[neg_edge[0]] * emb[neg_edge[1]]).sum(dim=-1)\n",
+    "\n",
+    "    loss = -F.logsigmoid(pos_score - neg_score).mean()\n",
+    "\n",
+    "    loss.backward()\n",
+    "    optimizer.step()\n",
+    "\n",
+    "    if (epoch + 1) % 20 == 0:\n",
+    "        print(f\"Epoch {epoch+1:>3}/{num_epochs}, Loss: {loss.item():.4f}\")\n",
+    "\n",
+    "print(\"Training complete.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2cafe26c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Model saved.\n"
+     ]
+    }
+   ],
+   "source": [
+    "torch.save(light_gcn.state_dict(), \"generated/lightgcn.pth\")\n",
+    "print(\"Model saved.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "342967bd",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Saved embeddings: torch.Size([1449, 384])\n"
+     ]
+    }
+   ],
+   "source": [
+    "light_gcn.eval()\n",
+    "with torch.no_grad():\n",
+    "    updated_embeddings = light_gcn.get_embedding(edge_index_pyg).cpu()\n",
+    "\n",
+    "torch.save(updated_embeddings, \"generated/lightgcn_embeddings.pt\")\n",
+    "print(f\"Saved embeddings: {updated_embeddings.shape}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5461e373",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "np.save(\"generated/lightgcn_embeddings\", updated_embeddings.numpy())"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

packages/data_prep/generate.ipynb

@@ -0,0 +1,266 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "6137a317",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from dotenv import load_dotenv\n",
+    "load_dotenv()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "4a7dc84b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import nest_asyncio\n",
+    "nest_asyncio.apply()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "a47086ec",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/packages/data_prep/.venv/lib/python3.13/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import datasets\n",
+    "from llama_index.core import PropertyGraphIndex, Document, Settings, load_index_from_storage, VectorStoreIndex\n",
+    "from llama_index.core.graph_stores import SimplePropertyGraphStore\n",
+    "from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n",
+    "from llama_index.llms.openai_like import OpenAILike"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "28b17221",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n"
+     ]
+    }
+   ],
+   "source": [
+    "og_data = datasets.load_dataset(\"gamino/wiki_medical_terms\", split=\"train[:20]\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "ca0aa288",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = [\n",
+    "    Document(id_=str(idx), text=text) for idx, text in enumerate(og_data[\"page_text\"])\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "53a4d694",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "20"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "e08d931b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "llm = OpenAILike(\n",
+    "    model=\"DeepSeek-V3.2\",\n",
+    "    api_key=os.getenv(\"AZURE_API_KEY\"),\n",
+    "    api_base=\"https://thong-api-hub.openai.azure.com/openai/v1\",\n",
+    "    is_chat_model=True,\n",
+    "    timeout=300,\n",
+    "    is_function_calling_model=True, \n",
+    "    should_use_structured_outputs=True,\n",
+    ")\n",
+    "Settings.llm = llm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "8d7d9e4a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "CompletionResponse(text='你好！😊 很高兴见到你！\\n\\n有什么我可以帮助你的吗？无论是回答问题、聊天、协助解决问题，还是其他任何事情，我都很乐意为你提供帮助。请随时告诉我你需要什么！', additional_kwargs={'prompt_tokens': 5, 'completion_tokens': 41, 'total_tokens': 46}, raw=ChatCompletion(id='48c590e60d6e472da59fddacdeda1cd2', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content='你好！😊 很高兴见到你！\\n\\n有什么我可以帮助你的吗？无论是回答问题、聊天、协助解决问题，还是其他任何事情，我都很乐意为你提供帮助。请随时告诉我你需要什么！', refusal=None, role='assistant', annotations=None, audio=None, function_call=None, tool_calls=None, reasoning_content=None), content_filter_results={'hate': {'filtered': False, 'severity': 'safe'}, 'protected_material_code': {'filtered': False, 'detected': False}, 'protected_material_text': {'filtered': False, 'detected': False}, 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': {'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'safe'}}, stop_reason=None)], created=1777086170, model='deepseek-v3.2', object='chat.completion', service_tier=None, system_fingerprint=None, usage=CompletionUsage(completion_tokens=41, prompt_tokens=5, total_tokens=46, completion_tokens_details=None, prompt_tokens_details=None, audio_prompt_tokens=0, reasoning_tokens=0), prompt_filter_results=[{'prompt_index': 0, 'content_filter_results': {'hate': {'filtered': False, 'severity': 'safe'}, 'jailbreak': {'filtered': False, 'detected': False}, 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': {'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'safe'}}}]), logprobs=None, delta=None)"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "Settings.llm.complete(\"Hello\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "711fd553",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/media/data/BaiTap/Code/Nam4/DLLUD/Lab03/big-data-application/packages/data_prep/.venv/lib/python3.13/site-packages/torch/cuda/__init__.py:180: UserWarning: CUDA initialization: CUDA unknown error - this may be due to an incorrectly set up environment, e.g. changing env variable CUDA_VISIBLE_DEVICES after program start. Setting the available devices to be zero. (Triggered internally at /pytorch/c10/cuda/CUDAFunctions.cpp:119.)\n",
+      "  return torch._C._cuda_getDeviceCount() > 0\n",
+      "Loading weights: 100%|██████████| 199/199 [00:00<00:00, 2822.69it/s]\n",
+      "\u001b[1mBertModel LOAD REPORT\u001b[0m from: BAAI/bge-small-en-v1.5\n",
+      "Key                     | Status     |  | \n",
+      "------------------------+------------+--+-\n",
+      "embeddings.position_ids | UNEXPECTED |  | \n",
+      "\n",
+      "Notes:\n",
+      "- UNEXPECTED:\tcan be ignored when loading from different task/architecture; not ok if you expect identical arch.\n"
+     ]
+    }
+   ],
+   "source": [
+    "embed_model = HuggingFaceEmbedding(model_name=\"BAAI/bge-small-en-v1.5\")\n",
+    "\n",
+    "Settings.embed_model = embed_model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a5b54280",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.core.indices.property_graph import DynamicLLMPathExtractor, ImplicitPathExtractor\n",
+    "\n",
+    "kg_extractor = DynamicLLMPathExtractor(\n",
+    "    llm=llm,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6403ff0b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Applying transformations: 100%|██████████| 1/1 [00:00<00:00,  5.57it/s]\n",
+      "Applying transformations: 100%|██████████| 1/1 [02:28<00:00, 148.94s/it]\n",
+      "Generating embeddings: 100%|██████████| 11/11 [00:13<00:00,  1.24s/it]\n",
+      "Generating embeddings: 100%|██████████| 210/210 [00:24<00:00,  8.57it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "graph_store = SimplePropertyGraphStore()\n",
+    "\n",
+    "index = PropertyGraphIndex.from_documents(\n",
+    "    data,\n",
+    "    show_progress=True,\n",
+    "    property_graph_store=graph_store,\n",
+    "    llm=llm,\n",
+    "    embed_model=embed_model,\n",
+    "    kg_extractors=[kg_extractor, ImplicitPathExtractor()],\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "b5a54ce9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index.property_graph_store.save_networkx_graph(name=\"./kg.html\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "abedc381",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index.storage_context.persist(\"./generated\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

packages/data_prep/generated/default__vector_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0c7f4210f14c206400b066e59279a3fdd29ea38c8a475f75a75bf0117fcf083
+size 11773589

packages/data_prep/generated/docstore.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a73fa5772fee1f6f964029a3d38634f864600cc0de7db553b8dae0bcdd919fd
+size 574331

packages/data_prep/generated/graph.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10d161965eb144eb9e7851a3373fc10e40cf2bee95f65e39489ef77bf8b5dfe7
+size 2265813

packages/data_prep/generated/graph_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e0a77744010862225c69da83c585f4f8a42fd551b044ce530dbb1eb6e16742c
+size 18

packages/data_prep/generated/id_to_int.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a356d44914a150f95f5d1db996f8874961d5323ab8f2fe9d3e91e4f75f30438
+size 44218

packages/data_prep/generated/image__vector_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d17ed74c1649a438e518a8dc56a7772913dfe1ea7a7605bce069c63872431455
+size 72

packages/data_prep/generated/index_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d3c3dd402c1fe9eed6658c9ff43a7e360df4046b08103c21ec53f74efd895c3
+size 181

packages/data_prep/generated/int_to_id.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a356d44914a150f95f5d1db996f8874961d5323ab8f2fe9d3e91e4f75f30438
+size 44218

packages/data_prep/generated/lightgcn.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aec6644cf92f4a49f4ae3416bef018498da11b8f73e67712d94e0df6b64eee39
+size 2227821

packages/data_prep/generated/lightgcn_embeddings.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2cff0781fee7d2ea40b19e867e1c07b32012ca1fcb53203a5bfac10da32b8f8
+size 2225792

packages/data_prep/generated/lightgcn_embeddings.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:315c00d6317b813cb02b5ca682be2fcd70b5349d44e88a43ec587e3221ddc3df
+size 2227325

packages/data_prep/generated/property_graph_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f15eb29ec3ff3b4b8049cd5ff58ec8071e5400de464b3ee54a283d147a52c0f
+size 2301082

packages/data_prep/pyproject.toml

@@ -0,0 +1,29 @@
+[project]
+name = "data-prep"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+authors = [
+    { name = "thng292", email = "nguyenquangthong292@gmail.com" }
+]
+requires-python = ">=3.13"
+dependencies = [
+    "datasets>=4.8.4",
+    "ipykernel>=7.2.0",
+    "llama-index>=0.14.20",
+    "llama-index-embeddings-huggingface>=0.7.0",
+    "llama-index-llms-openai-like>=0.7.1",
+    "nest-asyncio>=1.6.0",
+    "notebook>=7.5.5",
+    "python-dotenv>=1.2.2",
+    "pyvis>=0.3.2",
+    "torch>=2.11.0",
+    "torch-geometric>=2.7.0",
+]
+
+[project.scripts]
+data-prep = "data_prep:main"
+
+[build-system]
+requires = ["uv_build>=0.10.4,<0.11.0"]
+build-backend = "uv_build"

packages/data_prep/src/data_prep/__init__.py

@@ -0,0 +1,12 @@
+def main() -> None:
+    print("Hello from data-prep!")
+
+
+# Load datasets
+# Build + index to SimplePropertyGraphStore
+# Save it `original`
+# Load to torch geometric
+# Message passing somehow?
+# Save it `enhanced`
+# Find weighted linear formulation by subclassing BaseRetriever
+# Build front-end with gradio, allow chat, uses llama.cpp

pyproject.toml

@@ -0,0 +1,32 @@
+[project]
+name = "big-data-application"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+authors = [
+    { name = "thng292", email = "nguyenquangthong292@gmail.com" }
+]
+requires-python = ">=3.13"
+dependencies = [
+    "gradio>=6.12.0",
+    "huggingface-hub>=0.30.0",
+    "ipykernel>=7.2.0",
+    "llama-cpp-python>=0.3.19",
+    "llama-index>=0.14.20",
+    "llama-index-embeddings-huggingface>=0.7.0",
+    "numpy>=2.0.0",
+    "python-dotenv>=1.2.2",
+]
+
+[project.scripts]
+big-data-application = "big_data_application:main"
+
+[[tool.uv.index]]
+url = "https://abetlen.github.io/llama-cpp-python/whl/cpu"
+
+[build-system]
+requires = ["uv_build>=0.10.4,<0.11.0"]
+build-backend = "uv_build"
+
+[tool.uv.workspace]
+members = ["packages/*"]

src/big_data_application/__init__.py

@@ -0,0 +1,189 @@
+import os
+import re
+import json
+
+import numpy as np
+import gradio as gr
+from llama_index.core import StorageContext, load_index_from_storage, Settings, QueryBundle
+from llama_index.core.retrievers import BaseRetriever
+from llama_index.core.schema import NodeWithScore, TextNode
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+from llama_cpp import Llama
+from huggingface_hub import hf_hub_download, list_repo_files
+
+
+DATA_DIR = os.environ.get("DATA_DIR", "packages/data_prep/generated")
+MODEL_REPO = "Jackrong/Qwen3.5-4B-Neo-GGUF"
+
+
+def download_gguf_model(repo_id: str) -> str:
+    files = list_repo_files(repo_id)
+    gguf_files = [f for f in files if f.endswith(".gguf")]
+    target = next((f for f in gguf_files if "Q4_K_M" in f.upper()), gguf_files[0])
+    print(f"Downloading {target} from {repo_id}...")
+    return hf_hub_download(repo_id=repo_id, filename=target)
+
+
+class HybridGraphRetriever(BaseRetriever):
+    """Combines original embedding similarity and LightGCN-enhanced embedding
+    similarity using a weighted linear formulation for hybrid ranking."""
+
+    def __init__(
+        self,
+        data_dir: str,
+        embed_model: HuggingFaceEmbedding,
+        alpha: float = 0.5,
+        top_k: int = 5,
+    ):
+        super().__init__()
+        self._embed_model = embed_model
+        self._alpha = alpha
+        self._top_k = top_k
+
+        with open(os.path.join(data_dir, "property_graph_store.json")) as f:
+            pg_data = json.load(f)
+        with open(os.path.join(data_dir, "id_to_int.json")) as f:
+            id_to_int = json.load(f)
+        lightgcn_all = np.load(os.path.join(data_dir, "lightgcn_embeddings.npy"))
+
+        node_ids: list[str] = []
+        node_texts: list[str] = []
+        node_labels: list[str] = []
+        orig_list: list[list[float]] = []
+        lgcn_list: list[np.ndarray] = []
+
+        for node_id, node_data in pg_data["nodes"].items():
+            if node_id not in id_to_int:
+                continue
+            emb = node_data.get("embedding")
+            if not emb:
+                continue
+            idx = id_to_int[node_id]
+            node_ids.append(node_id)
+            node_texts.append(node_data.get("text", ""))
+            node_labels.append(node_data.get("label", ""))
+            orig_list.append(emb)
+            lgcn_list.append(lightgcn_all[idx])
+
+        self._node_ids = node_ids
+        self._node_texts = node_texts
+        self._node_labels = node_labels
+
+        orig = np.array(orig_list, dtype=np.float32)
+        lgcn = np.stack(lgcn_list).astype(np.float32)
+        self._orig_normed = orig / (np.linalg.norm(orig, axis=1, keepdims=True) + 1e-8)
+        self._lgcn_normed = lgcn / (np.linalg.norm(lgcn, axis=1, keepdims=True) + 1e-8)
+
+        print(
+            f"HybridGraphRetriever ready: {len(node_ids)} nodes, "
+            f"alpha={alpha}, top_k={top_k}"
+        )
+
+    def _retrieve(self, query_bundle: QueryBundle) -> list[NodeWithScore]:
+        query_emb = np.array(
+            self._embed_model.get_query_embedding(query_bundle.query_str),
+            dtype=np.float32,
+        )
+        query_normed = query_emb / (np.linalg.norm(query_emb) + 1e-8)
+
+        sim_orig = self._orig_normed @ query_normed
+        sim_lgcn = self._lgcn_normed @ query_normed
+
+        # Weighted linear combination: score = alpha * sim_original + (1 - alpha) * sim_lightgcn
+        scores = self._alpha * sim_orig + (1 - self._alpha) * sim_lgcn
+
+        top_indices = np.argsort(scores)[::-1][: self._top_k]
+
+        return [
+            NodeWithScore(
+                node=TextNode(
+                    text=self._node_texts[i],
+                    id_=self._node_ids[i],
+                    metadata={"label": self._node_labels[i]},
+                ),
+                score=float(scores[i]),
+            )
+            for i in top_indices
+        ]
+
+
+def _strip_think_tags(text: str) -> str:
+    text = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL)
+    if "<think>" in text:
+        text = text[: text.index("<think>")]
+    return text.strip()
+
+
+def main() -> None:
+    print("Loading embedding model...")
+    embed_model = HuggingFaceEmbedding(model_name="BAAI/bge-small-en-v1.5")
+    Settings.embed_model = embed_model
+    Settings.llm = None
+
+    print("Loading property graph index...")
+    storage_context = StorageContext.from_defaults(persist_dir=DATA_DIR)
+    index = load_index_from_storage(storage_context)
+    print(f"Index loaded: {index.index_id}")
+
+    print("Building hybrid retriever...")
+    retriever = HybridGraphRetriever(
+        data_dir=DATA_DIR,
+        embed_model=embed_model,
+        alpha=0.5,
+        top_k=5,
+    )
+
+    print("Loading LLM...")
+    model_path = download_gguf_model(MODEL_REPO)
+    llm = Llama(
+        model_path=model_path,
+        n_ctx=4096,
+        n_threads=os.cpu_count() or 4,
+        verbose=False,
+    )
+
+    def chat(message: str, history: list[dict]):
+        nodes = retriever.retrieve(message)
+        context = "\n\n".join(
+            f"[{n.metadata.get('label', '')}] {n.text}" for n in nodes
+        )
+
+        messages: list[dict] = [
+            {
+                "role": "system",
+                "content": (
+                    "You are a helpful knowledge assistant. "
+                    "Answer questions based on the provided context from a knowledge graph. "
+                    "If the context doesn't contain relevant information, say so.\n\n"
+                    f"Context:\n{context}"
+                ),
+            }
+        ]
+        for msg in history:
+            messages.append({"role": msg["role"], "content": msg["content"]})
+        messages.append({"role": "user", "content": message})
+
+        response = llm.create_chat_completion(
+            messages=messages,
+            stream=True,
+            max_tokens=1024,
+        )
+
+        partial = ""
+        for chunk in response:
+            delta = chunk["choices"][0]["delta"]
+            content = delta.get("content", "")
+            if content:
+                partial += content
+                display = _strip_think_tags(partial)
+                if display:
+                    yield display
+
+    print("Starting Gradio app...")
+    demo = gr.ChatInterface(
+        fn=chat,
+        title="Knowledge Graph Chat",
+        description="Chat with an LLM powered by a knowledge graph with hybrid retrieval (original + LightGCN embeddings).",
+        type="messages",
+    )
+    demo.launch(server_name="0.0.0.0", server_port=7860)

src/big_data_application/__main__.py

@@ -0,0 +1,3 @@
+from big_data_application import main
+
+main()