simplified app.py

Use only pre-populated chroma
remove document sources

Reports/GBV AoR Strategy 2021-2025.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:fcee3331913a4cf854391589e2503a9c798059b856646b830bca3928d85f3b9e
-size 7629117

Reports/GBV_PocketGuide021718.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:fa1a2d24742803f7a2543b8f3297ab3c15ff3be96f2ebae2dfb5549c423ee2b7
-size 466663

Reports/UNFPA_GBV_E-Learning_Companion_Guide_ENGLISH.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9d109398e90862628ee6eb38f564707b74c51cbf629cc7f3b7e829a5bab8705a
-size 20350214

Reports/gbv_me_toolkit_eng_2nd_edition_apr_21_en_final.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:733ff134d0bd39db05f91d18080a833dcdd2baf396520ef198a43c52fd591abb
-size 1978746

Reports/gbv_sc_sops_2018_english_final.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f753dde4b06dc354ba6769d18b063ca4a7c11034239b145285b2b7d02f5f97d0
-size 5599140

Reports/gbv_toolkit_book_01_20_2015_en.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38ab25a323bec1bbb4efc3c9ce9f144a8613c0dbd03bbeb32808dcbc572481ac
-size 5088528

Reports/genderbased_eng.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:62674549a5f55638a93a365e22a869b3637fb37a2fed6f31f947f1829ef88186
-size 491998

Reports/handbook-for-coordinating-gbv-in-emergencies_fin.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f235cf3c6f21a5f2bc2589466bfd284cb633e0a684f786a089398ba79d603db8
-size 6877542

Reports/iasc_gender_handbook_2017.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:82c0127e6a48da59ec27634ac867a08a625ce5d7c1cbd261ecf0ae8a0e64b2ad
-size 6582298

app.py

@@ -1,14 +1,11 @@
 from langchain.embeddings.openai import OpenAIEmbeddings
 from langchain.vectorstores import Chroma
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.agents import AgentExecutor, Tool, load_tools
-from langchain.chains import RetrievalQA, RetrievalQAWithSourcesChain, LLMChain, LLMMathChain
+from langchain.agents import AgentExecutor, Tool
+from langchain.chains import RetrievalQAWithSourcesChain
 from langchain.chat_models import ChatOpenAI
-from langchain.document_loaders import DirectoryLoader
 from langchain.memory import ConversationBufferMemory
 from langchain.utilities import WikipediaAPIWrapper
 from langchain.agents import initialize_agent, AgentType
-from langchain.document_loaders import WebBaseLoader
 
 import gradio as gr
 
@@ -18,48 +15,17 @@ from app_modules.presets import *
 import os
 os.environ["OPENAI_API_KEY"] = os.environ["OPENAI_TOKEN"]
 
-# Flag to load chroma store
-flag_chroma = True
-
 # Define the LLM chat model
 model = 'gpt-3.5-turbo'
 #model = 'gpt-4'
 temperature = 0
 llm = ChatOpenAI(temperature=temperature, model=model)
 
-# Check flag to load vectorstore
-if flag_chroma:
-    # Load an existing database
-    persist_dir = "./chroma"
-    embeddings = OpenAIEmbeddings()
-    vectorstore = Chroma(persist_directory=persist_dir, embedding_function=embeddings)
-    vectorstore.persist()
-else:
-    # Document and sources loader
-    pdf_loader = DirectoryLoader('./Reports/', glob="**/*.pdf")
-    txt_loader = DirectoryLoader('./Reports/', glob="**/*.txt")
-    word_loader = DirectoryLoader('./Reports/', glob="**/*.docx")
-    web_based_loader = WebBaseLoader(["https://www.unwomen.org/en/what-we-do/ending-violence-against-women/faqs/types-of-violence", "https://2021.gho.unocha.org/global-trends/gender-and-gender-based-violence-humanitarian-action/"])
-
-    loaders = [pdf_loader, txt_loader, word_loader, web_based_loader]
-    docs = []
-    for loader in loaders:
-        docs.extend(loader.load())
-
-    # Text splitter
-    ## If chunks are bigger than 1000, it recursively splits them until fitting them within size
-    text_splitter = RecursiveCharacterTextSplitter(
-        separators=["\n\n", "\n", ".", "!", "?", ",", " ", ""],
-        chunk_size = 1000,
-        chunk_overlap  = 50
-    )
-    documents = text_splitter.split_documents(docs)
-
-    # Embed documents in Vectorstore
-    persist_dir = "chroma"
-    embeddings = OpenAIEmbeddings()
-    vectorstore = Chroma.from_documents(documents, embeddings, persist_directory=persist_dir)
-    vectorstore.persist()
+# Load existing vectorstore
+persist_dir = "./chroma"
+embeddings = OpenAIEmbeddings()
+vectorstore = Chroma(persist_directory=persist_dir, embedding_function=embeddings)
+vectorstore.persist()
 
 # Create Retrieval Chain with sources
 ## It returns a dictionary with at least the 'answer' and the 'sources'
@@ -132,7 +98,6 @@ with gr.Blocks(css=customCSS, theme=small_and_beautiful_theme, title="GBV Q&A Bo
         history[-1][1] = ""
         for character in bot_message:
             history[-1][1] += character
-            #time.sleep(0.05)
             yield history
 
     response = msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(