omdena-mental-health / ingest_data.py
patti-j's picture
Create ingest_data.py
7944524
raw
history blame contribute delete
828 Bytes
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.document_loaders import UnstructuredFileLoader
from langchain.vectorstores.faiss import FAISS
from langchain.embeddings import OpenAIEmbeddings
import pickle
# Load Data
loader = UnstructuredFileLoader("state_of_the_union.txt")
raw_documents = loader.load()
# Split text
text_splitter = RecursiveCharacterTextSplitter()
documents = text_splitter.split_documents(raw_documents)
# Load Data to vectorstore
embeddings = OpenAIEmbeddings()
vectorstore = FAISS.from_documents(documents, embeddings)
query = "What is Sales Handoff?"
docs = vectorstore.similarity_search(query)
db = FAISS.from_documents(docs, embeddings)
print(docs[0].page_content)
# Save vectorstore
# with open("vectorstore.pkl", "wb") as f:
# pickle.dump(vectorstore, f)