Upload setup_qdrant.ipynb

setup_qdrant.ipynb

@@ -0,0 +1,211 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'/Users/arda/Desktop/A.I./Projects/FinanceChatbot'"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%pwd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Import libraries\n",
+    "import os\n",
+    "import warnings\n",
+    "from dotenv import load_dotenv\n",
+    "from langchain.document_loaders import PyPDFLoader, DirectoryLoader\n",
+    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+    "from langchain_qdrant import QdrantVectorStore\n",
+    "from langchain.embeddings import HuggingFaceEmbeddings\n",
+    "\n",
+    "# Ignore all warnings\n",
+    "warnings.filterwarnings(\"ignore\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Environment variables loaded successfully.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Load environment variables from .env file\n",
+    "load_dotenv()\n",
+    "\n",
+    "# Check if .env file exists and API keys are loaded\n",
+    "if not os.path.exists('.env'):\n",
+    "    print(\"Warning: .env file not found!\")\n",
+    "elif not os.getenv(\"QDRANT_API_KEY\") or not os.getenv(\"QDRANT_URL\"):\n",
+    "    print(\"Warning: QDRANT_API_KEY or QDRANT_URL not found in .env file!\")\n",
+    "else:\n",
+    "    print(\"Environment variables loaded successfully.\")\n",
+    "\n",
+    "# Settings\n",
+    "QDRANT_API_KEY = os.getenv(\"QDRANT_API_KEY\")\n",
+    "QDRANT_URL = os.getenv(\"QDRANT_URL\")\n",
+    "COLLECTION_NAME = \"finance-chatbot\"\n",
+    "DATA_DIR = \"Data\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Number of unique PDF files loaded: 3\n",
+      "Loaded files:\n",
+      "File 1: Data/Basics.pdf\n",
+      "File 2: Data/Financialterms.pdf\n",
+      "File 3: Data/Statementanalysis.pdf\n",
+      "Success: All 3 PDFs (Basics.pdf, Statementanalysis.pdf, Financialterms.pdf) have been loaded.\n",
+      "Total number of pages loaded: 547\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Load and extract data from PDFs\n",
+    "def load_pdf_file(data_dir):\n",
+    "    loader = DirectoryLoader(\n",
+    "        data_dir,\n",
+    "        glob=\"*.pdf\",\n",
+    "        loader_cls=PyPDFLoader\n",
+    "    )\n",
+    "    documents = loader.load()\n",
+    "    return documents\n",
+    "\n",
+    "extracted_data = load_pdf_file(DATA_DIR)\n",
+    "\n",
+    "# Verify the number of loaded PDFs by checking unique file sources\n",
+    "unique_files = set(doc.metadata.get('source', 'Unknown') for doc in extracted_data)\n",
+    "print(f\"Number of unique PDF files loaded: {len(unique_files)}\")\n",
+    "print(\"Loaded files:\")\n",
+    "for i, file in enumerate(unique_files, 1):\n",
+    "    print(f\"File {i}: {file}\")\n",
+    "\n",
+    "# Check if the expected number of PDFs (3) were loaded\n",
+    "if len(unique_files) == 3:\n",
+    "    print(\"Success: All 3 PDFs (Basics.pdf, Statementanalysis.pdf, Financialterms.pdf) have been loaded.\")\n",
+    "else:\n",
+    "    print(f\"Warning: Expected 3 PDFs, but {len(unique_files)} unique files were loaded. Check the Data directory.\")\n",
+    "\n",
+    "# Additional info: Total number of pages (documents)\n",
+    "print(f\"Total number of pages loaded: {len(extracted_data)}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Length of Text Chunks: 2756\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Split the data into text chunks\n",
+    "def text_split(extracted_data):\n",
+    "    text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=20)\n",
+    "    text_chunks = text_splitter.split_documents(extracted_data)\n",
+    "    return text_chunks\n",
+    "\n",
+    "text_chunks = text_split(extracted_data)\n",
+    "print(\"Length of Text Chunks:\", len(text_chunks))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Embedding Dimension: 384\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Download embeddings from Hugging Face\n",
+    "embeddings = HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L6-v2')\n",
+    "\n",
+    "# Verify embedding dimension\n",
+    "query_result = embeddings.embed_query(\"Hello world\")\n",
+    "print(\"Embedding Dimension:\", len(query_result))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Initialize Qdrant client and create/upload to collection\n",
+    "try:\n",
+    "    qdrant = QdrantVectorStore.from_documents(\n",
+    "        documents=text_chunks,\n",
+    "        embedding=embeddings,\n",
+    "        url=QDRANT_URL,\n",
+    "        api_key=QDRANT_API_KEY,\n",
+    "        collection_name=COLLECTION_NAME\n",
+    "    )\n",
+    "    print(\"Qdrant collection created and populated successfully.\")\n",
+    "except Exception as e:\n",
+    "    print(f\"Error creating Qdrant collection: {e}\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "finance_chatbot",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}