djangomango
commited on
Commit
•
ab2623c
1
Parent(s):
8d8e8cb
Create ingest.py
Browse files
ingest.py
ADDED
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os
|
2 |
+
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
3 |
+
from langchain.vectorstores import Chroma
|
4 |
+
from langchain.embeddings import HuggingFaceBgeEmbeddings
|
5 |
+
from langchain.document_loaders import PyPDFLoader
|
6 |
+
|
7 |
+
model_name = "BAAI/bge-large-en"
|
8 |
+
model_kwargs = {'device': 'cpu'}
|
9 |
+
encode_kwargs = {'normalize_embeddings': False}
|
10 |
+
embeddings = HuggingFaceBgeEmbeddings(
|
11 |
+
model_name=model_name,
|
12 |
+
model_kwargs=model_kwargs,
|
13 |
+
encode_kwargs=encode_kwargs
|
14 |
+
)
|
15 |
+
|
16 |
+
loader = PyPDFLoader("resume.pdf")
|
17 |
+
documents = loader.load()
|
18 |
+
text_splitter = RecursiveCharacterTextSplitter(chunk_size=200, chunk_overlap=20)
|
19 |
+
texts = text_splitter.split_documents(documents)
|
20 |
+
|
21 |
+
vector_store = Chroma.from_documents(texts, embeddings, collection_metadata={"hnsw:space": "cosine"}, persist_directory="stores/pet_cosine")
|
22 |
+
|
23 |
+
print("Vector Store Created.......")
|