Spaces:
Runtime error
Runtime error
from langchain.text_splitter import RecursiveCharacterTextSplitter | |
from langchain.document_loaders import PyPDFLoader, DirectoryLoader | |
from langchain.embeddings import HuggingFaceEmbeddings | |
from langchain.vectorstores import FAISS | |
DATA_PATH = "data/" | |
DB_FAISS_PATH = "vectorstores/db_faiss" | |
#model path: | |
#https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGML/blob/main/llama-2-7b-chat.ggmlv3.q8_0.bin | |
#create vector database | |
def create_vector_db(): | |
loader = DirectoryLoader( | |
DATA_PATH, | |
glob='*.pdf', | |
loader_cls=PyPDFLoader | |
) | |
documents = loader.load() | |
text_splitter = RecursiveCharacterTextSplitter(chunk_size = 500, chunk_overlap = 50) | |
texts = text_splitter.split_documents(documents) | |
embeddings = HuggingFaceEmbeddings(model_name = 'sentence-transformers/all-MiniLM-L6-v2', model_kwargs = {'device': 'cpu'}) | |
db = FAISS.from_documents(texts, embeddings) | |
db.save_local(DB_FAISS_PATH) | |
if __name__ == '__main__': | |
create_vector_db() | |