Spaces:
Sleeping
Sleeping
| from langchain.embeddings import GPT4AllEmbeddings | |
| from langchain.document_loaders import TextLoader, DirectoryLoader | |
| from langchain.text_splitter import RecursiveCharacterTextSplitter | |
| from langchain.embeddings import GPT4AllEmbeddings | |
| from langchain.vectorstores import Chroma | |
| import chromadb | |
| class Vectorstore_client: | |
| def __init__(self): | |
| self.persist_directory = "data/vectorstore" | |
| self.client = chromadb.PersistentClient(path=self.persist_directory) | |
| elections = ["2013", "2017", "2021"] | |
| for election in elections: | |
| # load all files from cleaned data set | |
| glob = "*" + election + ".txt" | |
| loader = DirectoryLoader( | |
| 'data/clean/', glob=glob, use_multithreading=True, loader_cls=TextLoader) | |
| docs_list = loader.load() | |
| # split documents | |
| text_splitter = RecursiveCharacterTextSplitter( | |
| chunk_size=1000, chunk_overlap=200) | |
| all_splits = text_splitter.split_documents(docs_list) | |
| all_texts = [text.page_content for text in all_splits] | |
| # generate ids for all documents | |
| ids_list = ["id{}".format(i) | |
| for i in range(1, len(all_texts) + 1)] | |
| # Store splits in database | |
| collection = self.client.get_or_create_collection( | |
| name=election) | |
| if collection.count() == 0: | |
| collection.add( | |
| documents=all_texts, | |
| ids=ids_list | |
| ) | |
| return | |
| def get_client(self): | |
| return self.client | |
| # class Vectorstore: | |
| # def __init__(self) -> None: | |
| # self.persist_directory = "/home/phisinger/Programmieren/wahlprogramm_analyse/data/vectorstore" | |
| # if False: | |
| # # load data from data persist_directory | |
| # print("use persisted db.") | |
| # self.vectordb = Chroma(persist_directory=persist_directory, | |
| # embedding_function=GPT4AllEmbeddings()) | |
| # else: | |
| # print("Build new vector DB") | |
| # self.build_vectorstore() | |
| # return self.vectordb | |
| # def build_vectorstore(self): | |
| # elections = ["2013", "2017", "2021"] | |
| # for election in elections: | |
| # # load all files from cleaned data set | |
| # glob = "*" + election + ".txt" | |
| # loader = DirectoryLoader( | |
| # '../data/clean/', glob=glob, use_multithreading=True, loader_cls=TextLoader) | |
| # docs_list = loader.load() | |
| # # split documents | |
| # text_splitter = RecursiveCharacterTextSplitter( | |
| # chunk_size=1000, chunk_overlap=200) | |
| # all_splits = text_splitter.split_documents(docs_list) | |
| # # store documents in vector store | |
| # self.vectordb = Chroma.from_documents( | |
| # documents=all_splits, embedding=GPT4AllEmbeddings(), persist_directory=self.persist_directory) | |
| # self.vectordb.persist() | |
| # def get(self): | |
| # return self.vectordb | |