Spaces:
Runtime error
Runtime error
File size: 860 Bytes
e67fa80 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 |
#!/usr/bin/env python
from langchain.document_loaders.csv_loader import CSVLoader
from langchain.text_splitter import CharacterTextSplitter
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.vectorstores import FAISS
def create_vector_db():
"""
This function creates a vector database
"""
# Load CSV data
loader = CSVLoader(file_path="mylib/combined.csv")
data = loader.load()
text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=50)
texts = text_splitter.split_documents(data)
embeddings = HuggingFaceEmbeddings(
model_name="sentence-transformers/all-MiniLM-L6-v2"
)
db = FAISS.from_documents(texts, embeddings)
return db.save_local("mylib/vector_db")
if __name__ == "__main__":
vector_db = create_vector_db()
print("Vector database created and indexed.")
|