Spaces:
Paused
Paused
File size: 1,633 Bytes
624109c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 |
from langchain.text_splitter import (
RecursiveCharacterTextSplitter,
CharacterTextSplitter,
)
from langchain_community.vectorstores import FAISS
from langchain_community.document_loaders import PyPDFLoader
from langchain_huggingface import HuggingFaceEmbeddings
import sys, os
vector_db_path = None
pdf_data_path = None
def create_vector_db_from_text() -> FAISS:
text = """"""
text_splitter = CharacterTextSplitter(
chunk_size=500, chunk_overlap=100, length_function=len, separator="\n"
)
texts = text_splitter.split_text(text)
embedding = HuggingFaceEmbeddings(
model_name="sentence-transformers/all-MiniLM-L6-v2"
)
db = FAISS.from_texts(texts, embedding)
db.save_local(vector_db_path)
return db
def create_vector_db_from_file() -> FAISS:
loader = PyPDFLoader(pdf_data_path)
documents = loader.load()
text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100)
texts = text_splitter.split_documents(documents)
embedding = HuggingFaceEmbeddings(
model_name="sentence-transformers/all-MiniLM-L6-v2"
)
db = FAISS.from_documents(texts, embedding)
db.save_local(vector_db_path)
return db
folder_path = f"./data/courses/{sys.argv[1]}"
pdf_files = [f for f in os.listdir(folder_path) if f.lower()]
for file in pdf_files:
pdf_data_path = f"./data/courses/{sys.argv[1]}/{file}"
vector_db_path = f"./data/vectorstores/{sys.argv[1]}/{file[:-4]}"
create_vector_db_from_file()
print(f"Vector DB created successfully for file {file}")
|