Rename ransient ingenstion_data.py to ingest_data.py

ingest_data.py

@@ -0,0 +1,23 @@
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.document_loaders import UnstructuredFileLoader
+from langchain.vectorstores.faiss import FAISS
+from langchain.embeddings import OpenAIEmbeddings
+import pickle
+
+# Load Data
+loader = UnstructuredFileLoader("state_of_the_union.txt")
+raw_documents = loader.load()
+
+# Split text
+text_splitter = RecursiveCharacterTextSplitter()
+documents = text_splitter.split_documents(raw_documents)
+
+
+# Load Data to vectorstore
+embeddings = OpenAIEmbeddings()
+vectorstore = FAISS.from_documents(documents, embeddings)
+
+
+# Save vectorstore
+with open("vectorstore.pkl", "wb") as f:
+    pickle.dump(vectorstore, f)

ransient ingenstion_data.py

@@ -1,8 +0,0 @@
-from langchain.document_loaders import TextLoader
-loader = TextLoader('SMR4 publication.txt')
-from langchain.indexes import VectorstoreIndexCreator
-index = VectorstoreIndexCreator().from_loaders([loader])
-
-# Load Data
-loader = TextLoader("SMR4 publication.txt")
-raw_documents = loader.load()