d221 commited on
Commit
248efe2
·
verified ·
1 Parent(s): f3b060a

Update ingest.py

Browse files
Files changed (1) hide show
  1. ingest.py +28 -28
ingest.py CHANGED
@@ -1,29 +1,29 @@
1
- import os
2
- from langchain.text_splitter import RecursiveCharacterTextSplitter
3
- from langchain.embeddings import SentenceTransformerEmbeddings
4
- from langchain.document_loaders import DirectoryLoader
5
- from langchain.document_loaders import PyPDFLoader
6
- from langchain.vectorstores import Qdrant
7
- from qdrant_client import QdrantClient
8
-
9
- embeddings = SentenceTransformerEmbeddings(model_name="NeuML/pubmedbert-base-embeddings")
10
-
11
- client = QdrantClient(
12
- url=os.getenv("QDRANT_URL", "https://868005ec-814c-4a06-b5f5-f4051fdf2a5d.europe-west3-0.gcp.cloud.qdrant.io"),
13
- api_key=os.getenv("QDRANT_API_KEY"),
14
- prefer_grpc=False
15
- )
16
-
17
- loader = DirectoryLoader('data/', glob="**/*.pdf", show_progress=True, loader_cls=PyPDFLoader)
18
- documents = loader.load()
19
- text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
20
- texts = text_splitter.split_documents(documents)
21
-
22
- qdrant = Qdrant.from_documents(
23
- texts,
24
- embeddings,
25
- client=client,
26
- collection_name="vector_db"
27
- )
28
-
29
  #print("Vector DB Successfully Created!")
 
1
+ import os
2
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
3
+ from langchain.embeddings import SentenceTransformerEmbeddings
4
+ from langchain.document_loaders import DirectoryLoader
5
+ from langchain.document_loaders import PyPDFLoader
6
+ from langchain.vectorstores import Qdrant
7
+ from qdrant_client import QdrantClient
8
+
9
+ embeddings = SentenceTransformerEmbeddings(model_name="NeuML/pubmedbert-base-embeddings")
10
+
11
+ client = QdrantClient(
12
+ url=os.getenv("QDRANT_URL", "https://QDRANT_URL.europe-west3-0.gcp.cloud.qdrant.io"),
13
+ api_key=os.getenv("QDRANT_API_KEY"),
14
+ prefer_grpc=False
15
+ )
16
+
17
+ loader = DirectoryLoader('data/', glob="**/*.pdf", show_progress=True, loader_cls=PyPDFLoader)
18
+ documents = loader.load()
19
+ text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
20
+ texts = text_splitter.split_documents(documents)
21
+
22
+ qdrant = Qdrant.from_documents(
23
+ texts,
24
+ embeddings,
25
+ client=client,
26
+ collection_name="vector_db"
27
+ )
28
+
29
  #print("Vector DB Successfully Created!")