aelitta commited on
Commit
010df3b
1 Parent(s): 685a86d

Update ingest.py

Browse files
Files changed (1) hide show
  1. ingest.py +26 -1
ingest.py CHANGED
@@ -1 +1,26 @@
1
- import os
2
  texts,
3
  embeddings,
4
  url=url,
5
  prefer_grpc=False,
6
  collection_name="vector_db"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  texts,
2
  embeddings,
3
  url=url,
4
  prefer_grpc=False,
5
  collection_name="vector_db"
6
+ import os
7
+ from langchain_community.embeddings import SentenceTransformerEmbeddings
8
+ from langchain_community.document_loaders import UnstructuredFileLoader, DirectoryLoader
9
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
10
+ from langchain_community.vectorstores import Qdrant
11
+
12
+ embeddings = SentenceTransformerEmbeddings(model_name="NeuML/pubmedbert-base-embeddings")
13
+
14
+
15
+ loader = DirectoryLoader('data/', glob="**/*.pdf", show_progress=True, loader_cls=UnstructuredFileLoader)
16
+ documents = loader.load()
17
+
18
+ text_splitter = RecursiveCharacterTextSplitter(chunk_size=700, chunk_overlap=70)
19
+ texts = text_splitter.split_documents(documents)
20
+
21
+ url = "http://0.0.0.0:6333" # This is the same URL that must match Step 4d
22
+ qdrant = Qdrant.from_documents(
23
+ texts,
24
+ embeddings,
25
+ url=url,
26
+ prefer_grpc=False,
27
+ collection_name="vector_db"
28
+ )
29
+
30
+ print("Vector DB Successfully Created!")
31
+