Upload 21 files

.gitattributes

@@ -1,3 +1,5 @@
 # Auto detect text files and perform LF normalization
 * text=auto
+stores/ConserGPT/Circular25julio2023ReligionyAtencionEducativa.pdf_store/8bfd0a18-a06b-40ba-91ae-3f3ed1b0d1a3/data_level0.bin filter=lfs diff=lfs merge=lfs -text
+stores/ConserGPT/Instruccion26septiembre2023PremiosExtraordinariosMusica.pdf_store/61c07000-ae3f-47ca-94a7-80c0ed5dfd53/data_level0.bin filter=lfs diff=lfs merge=lfs -text
 zephyr-7b-alpha.Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,15 +1,6 @@
----
-title: ConserGPT
-sdk: gradio
-emoji: 🗿
-colorFrom: red
-colorTo: yellow
-pinned: true
----
-
 # Zephyr-7B-beta-RAG-Demo
 Zephyr 7B beta RAG Demo inside a Gradio app powered by BGE Embeddings, ChromaDB, and Zephyr 7B Alpha.
 
 Download zephyr-7b-alpha.Q5_K_S.gguf in this link : https://huggingface.co/TheBloke/zephyr-7B-alpha-GGUF/tree/main
 
-https://view.genial.ly/65805d10850fa600146ed98b/presentation-consergpt
+https://view.genial.ly/65805d10850fa600146ed98b/presentation-consergpt

app.py

@@ -26,7 +26,7 @@ config = {
 
 llm = CTransformers(
     model=local_llm,
-    model_type="mistral",
+    model_type="zephyr",
     lib="avx2",  # for CPU use
     **config
 )
@@ -34,7 +34,6 @@ llm = CTransformers(
 print("LLM Initialized...")
 
 
-
 prompt_template = """Utiliza la siguiente información para responder a la pregunta del usuario.
 Si no sabes la respuesta, di simplemente que no la sabes, no intentes inventarte una respuesta.
 
@@ -55,23 +54,23 @@ embeddings = HuggingFaceBgeEmbeddings(
     encode_kwargs=encode_kwargs
 )
 
-loader = PyPDFLoader(
-    "./Instruccion26septiembre2023PremiosExtraordinariosMusica.pdf")
-documents = loader.load()
-text_splitter = RecursiveCharacterTextSplitter(
-    chunk_size=1000, chunk_overlap=100)
-texts = text_splitter.split_documents(documents)
+# loader = PyPDFLoader(
+#     "./Instruccion26septiembre2023PremiosExtraordinariosMusica.pdf")
+# documents = loader.load()
+# text_splitter = RecursiveCharacterTextSplitter(
+#     chunk_size=1000, chunk_overlap=100)
+# texts = text_splitter.split_documents(documents)
 
-vector_store = Chroma.from_documents(texts, embeddings, collection_metadata={
-                                     "hnsw:space": "cosine"}, persist_directory="stores/ConserGPT")
+# vector_store = Chroma.from_documents(texts, embeddings, collection_metadata={
+#                                      "hnsw:space": "cosine"}, persist_directory="stores/ConserGPT")
 
-print("Vector Store Created.......")
+# print("Vector Store Created.......")
 
 
 prompt = PromptTemplate(template=prompt_template,
                         input_variables=['context', 'question'])
 load_vector_store = Chroma(
-    persist_directory="stores/ConserGPT", embedding_function=embeddings)
+    persist_directory="stores/ConserGPT/", embedding_function=embeddings)
 retriever = load_vector_store.as_retriever(search_kwargs={"k": 1})
 
 print("######################################################################")

ingest2.py

@@ -0,0 +1,38 @@
+import os
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores import Chroma
+from langchain.embeddings import HuggingFaceBgeEmbeddings
+from langchain.document_loaders import PyPDFLoader
+
+model_name = "BAAI/bge-large-en"
+model_kwargs = {'device': 'cpu'}
+encode_kwargs = {'normalize_embeddings': False}
+embeddings = HuggingFaceBgeEmbeddings(
+    model_name=model_name,
+    model_kwargs=model_kwargs,
+    encode_kwargs=encode_kwargs
+)
+
+pdf_folder = "./pdf_folder"  # Ruta a la carpeta que contiene los archivos PDF
+output_folder = "stores/ConserGPT"  # Carpeta de salida para los vector stores
+
+# Crear el directorio de salida si no existe
+os.makedirs(output_folder, exist_ok=True)
+
+# Iterar a través de los archivos PDF en la carpeta
+for pdf_file in os.listdir(pdf_folder):
+    if pdf_file.endswith(".pdf"):
+        pdf_path = os.path.join(pdf_folder, pdf_file)
+
+        loader = PyPDFLoader(pdf_path)
+        documents = loader.load()
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=1000, chunk_overlap=100)
+        texts = text_splitter.split_documents(documents)
+
+        vector_store = Chroma.from_documents(texts, embeddings, collection_metadata={
+                                             "hnsw:space": "cosine"}, persist_directory=os.path.join(output_folder, f"{pdf_file}_store"))
+
+        print(f"Vector Store created for {pdf_file}")
+
+print("All Vector Stores Created.......")

requirements.txt

@@ -2,8 +2,7 @@ chainlit
 ctransformers
 torch
 sentence_transformers
-chromadb
-langchain-community
+chromadb 
 langchain
 pypdf 
 PyPDF2

stores/ConserGPT/Circular25julio2023ReligionyAtencionEducativa.pdf_store/8bfd0a18-a06b-40ba-91ae-3f3ed1b0d1a3/data_level0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95042e844cfb77b20e578cf65635282a99d7c4dd20e589ac062f38bc389f8e58
+size 4236000

stores/ConserGPT/Instruccion26septiembre2023PremiosExtraordinariosMusica.pdf_store/61c07000-ae3f-47ca-94a7-80c0ed5dfd53/data_level0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95042e844cfb77b20e578cf65635282a99d7c4dd20e589ac062f38bc389f8e58
+size 4236000