Create ingest_data.py

ingest_data.py

@@ -0,0 +1,29 @@
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.document_loaders import UnstructuredFileLoader
+from langchain.vectorstores.faiss import FAISS
+from langchain.embeddings import OpenAIEmbeddings
+import pickle
+
+# Load Data
+loader = UnstructuredFileLoader("state_of_the_union.txt")
+raw_documents = loader.load()
+
+# Split text
+text_splitter = RecursiveCharacterTextSplitter()
+documents = text_splitter.split_documents(raw_documents)
+
+
+# Load Data to vectorstore
+embeddings = OpenAIEmbeddings()
+vectorstore = FAISS.from_documents(documents, embeddings)
+
+query = "What is Sales Handoff?"
+docs = vectorstore.similarity_search(query)
+db = FAISS.from_documents(docs, embeddings)
+
+print(docs[0].page_content)
+
+# Save vectorstore
+# with open("vectorstore.pkl", "wb") as f:
+#    pickle.dump(vectorstore, f)
+