Spaces:
Running
Running
import os | |
from langchain_community.embeddings import SentenceTransformerEmbeddings | |
from langchain_community.document_loaders import UnstructuredFileLoader, DirectoryLoader | |
from langchain.text_splitter import RecursiveCharacterTextSplitter | |
from langchain_community.vectorstores import Qdrant | |
embeddings = SentenceTransformerEmbeddings(model_name="NeuML/pubmedbert-base-embeddings") | |
loader = DirectoryLoader('data/', glob="**/*.pdf", show_progress=True, loader_cls=UnstructuredFileLoader) | |
documents = loader.load() | |
text_splitter = RecursiveCharacterTextSplitter(chunk_size=700, chunk_overlap=70) | |
texts = text_splitter.split_documents(documents) | |
url = "http://0.0.0.0:6333" # This is the same URL that must match Step 4d | |
qdrant_api_key = os.environ['QDRANT_API_KEY'] | |
url = os.environ['QDRANT_URL']+':6333' | |
qdrant = Qdrant.from_documents( | |
texts, | |
embeddings, | |
url=url, | |
prefer_grpc=False, | |
collection_name="vector_db" | |
) | |
print("Vector DB Successfully Created!") | |