vector stores

ingest.py

@@ -1,34 +1,28 @@
-import os
-from langchain_community.vectorstores.faiss import FAISS
-from langchain.document_loaders import PyPDFLoader, DirectoryLoader
-from langchain.embeddings import HuggingFaceBgeEmbeddings
 from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.document_loaders import PyPDFLoader, DirectoryLoader # could have done any unstructured text loader like ppt and xlsx
 
-# Define the directory paths
-DATA_PATH = "data"
-VECTORSTORES_DIR = "vectorstores"
-DB_FAISS_PATH = os.path.join(VECTORSTORES_DIR, "db_faiss")
 
-# Create the vectorstores directory if it doesn't exist
-try:
-    os.makedirs(VECTORSTORES_DIR, exist_ok=True)
-except Exception as e:
-    print(f"Error creating directory: {e}")
+from langchain.embeddings import HuggingFaceBgeEmbeddings # we can replace huggingface with facetransformers
 
-# Create vector database
+from langchain.vectorstores import FAISS
+
+DATA_PATH = "$HOME/data/"
+DB_FAISS_PATH = "$HOME/vectorstores/db_faiss"
+
+#create vector database
 def create_vector_db():
-    # Load documents from the data directory
-    loader = DirectoryLoader(DATA_PATH, glob="*.pdf", loader_cls=PyPDFLoader)
+                                            # WE can change .pdf with any other unstructured text format
+    loader = DirectoryLoader(DATA_PATH, glob="*.pdf", loader_cls = PyPDFLoader)
     documents = loader.load()
 
-    # Split text from documents
     text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
+
     texts = text_splitter.split_documents(documents)
 
-    # Initialize embeddings
-    embeddings = HuggingFaceBgeEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2", model_kwargs={"device": "cpu"})
+    embeddings = HuggingFaceBgeEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2", model_kwargs={"device": "cpu"}) # change to GPU if you want
+
+    # cuda is not supported in my MAC M1! SADLY.
 
-    # Create FAISS vector database
     db = FAISS.from_documents(texts, embeddings)
     db.save_local(DB_FAISS_PATH)