vector stores

ingest.py

@@ -1,19 +1,27 @@
 from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.document_loaders import PyPDFLoader, DirectoryLoader
-from langchain.embeddings import HuggingFaceBgeEmbeddings
+from langchain.document_loaders import PyPDFLoader, DirectoryLoader # could have done any unstructured text loader like ppt and xlsx
+
+
+from langchain.embeddings import HuggingFaceBgeEmbeddings # we can replace huggingface with facetransformers
+
 from langchain.vectorstores import FAISS
 
-DATA_PATH = "/home/user/data"
-DB_FAISS_PATH = "/home/user/vectorstores/db_faiss"
+DATA_PATH = "data/"
+DB_FAISS_PATH = "vectorstores/db_faiss"
 
+#create vector database
 def create_vector_db():
-    loader = DirectoryLoader(DATA_PATH, glob="*.pdf", loader_cls=PyPDFLoader)
+                                            # WE can change .pdf with any other unstructured text format
+    loader = DirectoryLoader(DATA_PATH, glob="*.pdf", loader_cls = PyPDFLoader)
     documents = loader.load()
 
     text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
+
     texts = text_splitter.split_documents(documents)
 
-    embeddings = HuggingFaceBgeEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2", model_kwargs={"device": "cpu"})
+    embeddings = HuggingFaceBgeEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2", model_kwargs={"device": "cpu"}) # change to GPU if you want
+
+    # cuda is not supported in my MAC M1! SADLY.
 
     db = FAISS.from_documents(texts, embeddings)
     db.save_local(DB_FAISS_PATH)