use chromadb and openai to query docs

Makefile

@@ -1,2 +1,4 @@
 deps:
-	poetry export --without-hashes --format=requirements.txt > requirements.txt
+	pip install -U virtualenv
+	virtualenv .venv
+	.venv/bin/pip install -r requirements.txt

README.md

@@ -12,10 +12,8 @@ pinned: false
 # Internal DOC QA
 
 ```bash
-poetry shell
-poetry install
-export OPENAI_API_KEY=<...>
-export VERBA_URL=<...>
-export VERBA_API_KEY=<...>
-verba start --model "gpt-3.5-turbo"
+make deps
+
+source .venv/bin/activate
+python app.py
 ```

app.py

@@ -1,75 +1,66 @@
-import os
-
 import gradio as gr
-import weaviate
-from langchain import LLMChain
-from langchain.chains import SequentialChain
+from langchain.document_loaders import PyPDFLoader, DirectoryLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.vectorstores import Chroma
+from langchain.retrievers import SVMRetriever
+from langchain.chains import RetrievalQA
 from langchain.chat_models import ChatOpenAI
-from langchain.prompts import ChatPromptTemplate
 
-collection_name = "Chunk"
 
-MODEL = "gpt-3.5-turbo"
-LANGUAGE = "en"  # nl / en
-llm = ChatOpenAI(temperature=0.0, openai_api_key=os.environ["OPENAI_API_KEY"])
+def load_data():
+    # load the documents
+    loader = DirectoryLoader('./data', glob="**/*.pdf", show_progress=True, loader_cls=PyPDFLoader)
+    docs = loader.load()
+    # replace all new lines with spaces
+    [setattr(doc, "page_content", doc.page_content.replace("\n", " ")) for doc in docs]
+    print(docs)
 
+    # split the documents into chunks
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size = 500, chunk_overlap = 50)
+    all_splits = text_splitter.split_documents(docs)
 
-def get_answer_given_the_context(llm, prompt, context) -> SequentialChain:
-    template = f"""
-    Provide an answer to the prompt given the context.
-    
-    <PROMPT>
-    
-    {prompt}
-    
-    <CONTEXT>
-    
-    {context}
-    
-    """
+    # construct vector store
+    vectorstore = Chroma.from_documents(documents=all_splits, embedding=OpenAIEmbeddings())
+    # https://python.langchain.com/docs/use_cases/question_answering.html#go-deeper-3
+    svm_retriever = SVMRetriever.from_documents(all_splits, OpenAIEmbeddings())
+    return svm_retriever, vectorstore
 
-    prompt_get_skills_intersection = ChatPromptTemplate.from_template(template=template)
-    skills_match_chain = LLMChain(
-        llm=llm,
-        prompt=prompt_get_skills_intersection,
-        output_key="answer",
-    )
+svm_retriever, vectorstore = load_data()
 
-    chain = SequentialChain(
-        chains=[skills_match_chain],
-        input_variables=["prompt", "context"],
-        output_variables=[
-            skills_match_chain.output_key,
-        ],
-        verbose=False,
-    )
-    return chain({"prompt": prompt, "context": context})["answer"]
+def process_question(question, svm_retriever=svm_retriever, vectorstore=vectorstore):
 
+    docs_svm=svm_retriever.get_relevant_documents(question)
+    print(len(docs_svm))
+    llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
+    qa_chain = RetrievalQA.from_chain_type(llm, retriever=vectorstore.as_retriever(), return_source_documents=True)
+    result = qa_chain({"query": question})
+
+    output = f"""
+============RESULT==============
+\n
+{result["result"]}
+\n
+============SOURCES=============
+    """
 
-def predict(prompt):
-    client = weaviate.Client(
-        url=os.environ["WEAVIATE_URL"],
-        auth_client_secret=weaviate.AuthApiKey(api_key=os.environ["WEAVIATE_API_KEY"]),
-        additional_headers={"X-OpenAI-Api-Key": os.environ["OPENAI_API_KEY"]},
-    )
+    # Initialize an empty list to hold the lines
+    lines = []
 
-    search_result = (
-        client.query.get(class_name=collection_name, properties=["text"])
-        .with_near_text({"concepts": prompt})
-        # .with_generate(single_prompt="{text}")
-        .with_limit(5)
-        .do()
-    )
-    context_list = [
-        element["text"] for element in search_result["data"]["Get"]["Chunk"]
-    ]
-    context = "\n".join(context_list)
+    source_docs = [(x.metadata["source"], x.page_content) for x in result["source_documents"]]
+    for i, doc in enumerate(source_docs):
+        lines.append(f"* CHUNK: {i} *")
+        lines.append(f"original doc: {doc[0]}")
+        lines.append(f"{doc[1]}")
+        lines.append('')  # for a newline between chunks
 
-    return get_answer_given_the_context(llm=llm, prompt=prompt, context=context)
+    # Join the lines with a newline character to get the multi-line string
+    output += '\n'.join(lines)
+    return output
 
 
 iface = gr.Interface(
-    fn=predict,  # the function to wrap
+    fn=process_question,  # the function to wrap
     inputs="text",  # the input type
     outputs="text",  # the output type
     examples=[

import_data.py

@@ -1,81 +0,0 @@
-import os
-import weaviate
-from llama_index import download_loader
-from llama_index.vector_stores import WeaviateVectorStore
-from llama_index import VectorStoreIndex, StorageContext
-from pathlib import Path
-import argparse
-
-
-def get_pdf_files(base_path, loader):
-    """
-    Get paths to all PDF files in a directory and its subdirectories.
-
-    Parameters:
-    - base_path (str): The path to the starting directory.
-
-    Returns:
-    - list of str: A list of paths to all PDF files found.
-    """
-    pdf_paths = []
-
-    # Check if the base path exists and is a directory
-    if not os.path.exists(base_path):
-        raise FileNotFoundError(f"The specified base path does not exist: {base_path}")
-    if not os.path.isdir(base_path):
-        raise NotADirectoryError(
-            f"The specified base_path is not a directory: {base_path}"
-        )
-
-    # Loop through all directories and files starting from the base path
-    for root, dirs, files in os.walk(base_path):
-        for filename in files:
-            # If a file has a .pdf extension, add its path to the list
-            if filename.endswith(".pdf"):
-                pdf_file = loader.load_data(file=Path(root, filename))
-                pdf_paths.extend(pdf_file)
-
-    return pdf_paths
-
-
-def main(args):
-    PDFReader = download_loader("PDFReader")
-    loader = PDFReader()
-
-    documents = get_pdf_files(args.pdf_dir, loader)
-
-    client = weaviate.Client(
-        url=os.environ["WEAVIATE_URL"],
-        auth_client_secret=weaviate.AuthApiKey(api_key=os.environ["WEAVIATE_API_KEY"]),
-        additional_headers={"X-OpenAI-Api-Key": os.environ["OPENAI_API_KEY"]},
-    )
-
-    # construct vector store
-    vector_store = WeaviateVectorStore(
-        weaviate_client=client, index_name=args.customer, text_key="content"
-    )
-
-    # setting up the storage for the embeddings
-    storage_context = StorageContext.from_defaults(vector_store=vector_store)
-
-    # set up the index
-    index = VectorStoreIndex(documents, storage_context=storage_context)
-    query_engine = index.as_query_engine()
-    response = query_engine.query(args.query)
-    print(response)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Process and query PDF files.")
-
-    parser.add_argument("--customer", default="Ausy", help="Customer name")
-    parser.add_argument("--pdf_dir", default="./data", help="Directory containing PDFs")
-    parser.add_argument(
-        "--query",
-        default="What is CX0 customer exprience office?",
-        help="Query to execute",
-    )
-
-    args = parser.parse_args()
-
-    main(args)

pyproject.toml

@@ -1,20 +0,0 @@
-[tool.poetry]
-name = "ausy-rag-demo"
-version = "0.1.0"
-description = ""
-authors = ["Vincent Claes <vincent.v.claes@gmail.com>"]
-readme = "README.md"
-packages = [{include = "ausy_rag_demo"}]
-
-[tool.poetry.dependencies]
-python = "^3.9"
-llama-index = "^0.8.29.post1"
-weaviate-client = "^3.24.1"
-pypdf = "^3.16.1"
-goldenverba = "^0.2.3"
-gradio = "^3.44.4"
-
-
-[build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"

requirements.txt

@@ -1,117 +1,7 @@
-aiofiles==23.2.1 ; python_version >= "3.9" and python_version < "4.0"
-aiohttp==3.8.5 ; python_version >= "3.9" and python_version < "4.0"
-aiosignal==1.3.1 ; python_version >= "3.9" and python_version < "4.0"
-altair==5.1.1 ; python_version >= "3.9" and python_version < "4.0"
-annotated-types==0.5.0 ; python_version >= "3.9" and python_version < "4.0"
-anyio==3.7.1 ; python_version >= "3.9" and python_version < "4.0"
-async-timeout==4.0.3 ; python_version >= "3.9" and python_version < "4.0"
-attrs==23.1.0 ; python_version >= "3.9" and python_version < "4.0"
-authlib==1.2.1 ; python_version >= "3.9" and python_version < "4.0"
-beautifulsoup4==4.12.2 ; python_version >= "3.9" and python_version < "4.0"
-blis==0.7.10 ; python_version >= "3.9" and python_version < "4.0"
-catalogue==2.0.9 ; python_version >= "3.9" and python_version < "4.0"
-certifi==2023.7.22 ; python_version >= "3.9" and python_version < "4.0"
-cffi==1.15.1 ; python_version >= "3.9" and python_version < "4.0"
-charset-normalizer==3.2.0 ; python_version >= "3.9" and python_version < "4.0"
-click==8.1.7 ; python_version >= "3.9" and python_version < "4.0"
-colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0" and (platform_system == "Windows" or sys_platform == "win32")
-confection==0.1.3 ; python_version >= "3.9" and python_version < "4.0"
-contourpy==1.1.1 ; python_version >= "3.9" and python_version < "4.0"
-cryptography==41.0.4 ; python_version >= "3.9" and python_version < "4.0"
-cycler==0.11.0 ; python_version >= "3.9" and python_version < "4.0"
-cymem==2.0.8 ; python_version >= "3.9" and python_version < "4.0"
-dataclasses-json==0.6.0 ; python_version >= "3.9" and python_version < "4.0"
-exceptiongroup==1.1.3 ; python_version >= "3.9" and python_version < "3.11"
-fastapi==0.103.1 ; python_version >= "3.9" and python_version < "4.0"
-ffmpy==0.3.1 ; python_version >= "3.9" and python_version < "4.0"
-filelock==3.12.4 ; python_version >= "3.9" and python_version < "4.0"
-fonttools==4.42.1 ; python_version >= "3.9" and python_version < "4.0"
-frozenlist==1.4.0 ; python_version >= "3.9" and python_version < "4.0"
-fsspec==2023.9.1 ; python_version >= "3.9" and python_version < "4.0"
-goldenverba==0.2.3 ; python_version >= "3.9" and python_version < "4.0"
-gradio-client==0.5.1 ; python_version >= "3.9" and python_version < "4.0"
-gradio==3.44.4 ; python_version >= "3.9" and python_version < "4.0"
-greenlet==2.0.2 ; python_version >= "3.9" and python_version < "4.0" and (platform_machine == "win32" or platform_machine == "WIN32" or platform_machine == "AMD64" or platform_machine == "amd64" or platform_machine == "x86_64" or platform_machine == "ppc64le" or platform_machine == "aarch64")
-h11==0.14.0 ; python_version >= "3.9" and python_version < "4.0"
-httpcore==0.18.0 ; python_version >= "3.9" and python_version < "4.0"
-httptools==0.6.0 ; python_version >= "3.9" and python_version < "4.0"
-httpx==0.25.0 ; python_version >= "3.9" and python_version < "4.0"
-huggingface-hub==0.17.2 ; python_version >= "3.9" and python_version < "4.0"
-idna==3.4 ; python_version >= "3.9" and python_version < "4.0"
-importlib-metadata==6.8.0 ; python_version >= "3.9" and python_version < "3.10"
-importlib-resources==6.1.0 ; python_version >= "3.9" and python_version < "4.0"
-jinja2==3.1.2 ; python_version >= "3.9" and python_version < "4.0"
-joblib==1.3.2 ; python_version >= "3.9" and python_version < "4.0"
-jsonschema-specifications==2023.7.1 ; python_version >= "3.9" and python_version < "4.0"
-jsonschema==4.19.1 ; python_version >= "3.9" and python_version < "4.0"
-kiwisolver==1.4.5 ; python_version >= "3.9" and python_version < "4.0"
-langchain==0.0.296 ; python_version >= "3.9" and python_version < "4.0"
-langcodes==3.3.0 ; python_version >= "3.9" and python_version < "4.0"
-langsmith==0.0.38 ; python_version >= "3.9" and python_version < "4.0"
-llama-index==0.8.29.post1 ; python_version >= "3.9" and python_version < "4.0"
-markupsafe==2.1.3 ; python_version >= "3.9" and python_version < "4.0"
-marshmallow==3.20.1 ; python_version >= "3.9" and python_version < "4.0"
-matplotlib==3.8.0 ; python_version >= "3.9" and python_version < "4.0"
-multidict==6.0.4 ; python_version >= "3.9" and python_version < "4.0"
-murmurhash==1.0.10 ; python_version >= "3.9" and python_version < "4.0"
-mypy-extensions==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
-nest-asyncio==1.5.8 ; python_version >= "3.9" and python_version < "4.0"
-nltk==3.8.1 ; python_version >= "3.9" and python_version < "4.0"
-numexpr==2.8.6 ; python_version >= "3.9" and python_version < "4.0"
-numpy==1.25.2 ; python_version >= "3.9" and python_version < "4.0"
-openai==0.28.0 ; python_version >= "3.9" and python_version < "4.0"
-orjson==3.9.7 ; python_version >= "3.9" and python_version < "4.0"
-packaging==23.1 ; python_version >= "3.9" and python_version < "4.0"
-pandas==2.1.0 ; python_version >= "3.9" and python_version < "4.0"
-pathy==0.10.2 ; python_version >= "3.9" and python_version < "4.0"
-pillow==10.0.1 ; python_version >= "3.9" and python_version < "4.0"
-preshed==3.0.9 ; python_version >= "3.9" and python_version < "4.0"
-pycparser==2.21 ; python_version >= "3.9" and python_version < "4.0"
-pydantic-core==2.6.3 ; python_version >= "3.9" and python_version < "4.0"
-pydantic==2.3.0 ; python_version >= "3.9" and python_version < "4.0"
-pydub==0.25.1 ; python_version >= "3.9" and python_version < "4.0"
-pyparsing==3.1.1 ; python_version >= "3.9" and python_version < "4.0"
-pypdf==3.16.1 ; python_version >= "3.9" and python_version < "4.0"
-python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0"
-python-dotenv==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
-python-multipart==0.0.6 ; python_version >= "3.9" and python_version < "4.0"
-pytz==2023.3.post1 ; python_version >= "3.9" and python_version < "4.0"
-pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "4.0"
-referencing==0.30.2 ; python_version >= "3.9" and python_version < "4.0"
-regex==2023.8.8 ; python_version >= "3.9" and python_version < "4.0"
-requests==2.31.0 ; python_version >= "3.9" and python_version < "4.0"
-rpds-py==0.10.3 ; python_version >= "3.9" and python_version < "4.0"
-semantic-version==2.10.0 ; python_version >= "3.9" and python_version < "4.0"
-setuptools-scm==8.0.1 ; python_version >= "3.9" and python_version < "4.0"
-setuptools==68.2.2 ; python_version >= "3.9" and python_version < "4.0"
-six==1.16.0 ; python_version >= "3.9" and python_version < "4.0"
-smart-open==6.4.0 ; python_version >= "3.9" and python_version < "4.0"
-sniffio==1.3.0 ; python_version >= "3.9" and python_version < "4.0"
-soupsieve==2.5 ; python_version >= "3.9" and python_version < "4.0"
-spacy-legacy==3.0.12 ; python_version >= "3.9" and python_version < "4.0"
-spacy-loggers==1.0.5 ; python_version >= "3.9" and python_version < "4.0"
-spacy==3.6.1 ; python_version >= "3.9" and python_version < "4.0"
-sqlalchemy==2.0.21 ; python_version >= "3.9" and python_version < "4.0"
-srsly==2.4.7 ; python_version >= "3.9" and python_version < "4.0"
-starlette==0.27.0 ; python_version >= "3.9" and python_version < "4.0"
-tenacity==8.2.3 ; python_version >= "3.9" and python_version < "4.0"
-thinc==8.1.12 ; python_version >= "3.9" and python_version < "4.0"
-tiktoken==0.5.1 ; python_version >= "3.9" and python_version < "4.0"
-tomli==2.0.1 ; python_version >= "3.9" and python_version < "3.11"
-toolz==0.12.0 ; python_version >= "3.9" and python_version < "4.0"
-tqdm==4.66.1 ; python_version >= "3.9" and python_version < "4.0"
-typer==0.9.0 ; python_version >= "3.9" and python_version < "4.0"
-typing-extensions==4.8.0 ; python_version >= "3.9" and python_version < "4.0"
-typing-inspect==0.9.0 ; python_version >= "3.9" and python_version < "4.0"
-tzdata==2023.3 ; python_version >= "3.9" and python_version < "4.0"
-urllib3==1.26.16 ; python_version >= "3.9" and python_version < "4.0"
-uvicorn==0.23.2 ; python_version >= "3.9" and python_version < "4.0"
-uvicorn[standard]==0.23.2 ; python_version >= "3.9" and python_version < "4.0"
-uvloop==0.17.0 ; (sys_platform != "win32" and sys_platform != "cygwin") and platform_python_implementation != "PyPy" and python_version >= "3.9" and python_version < "4.0"
-validators==0.22.0 ; python_version >= "3.9" and python_version < "4.0"
-wasabi==1.1.2 ; python_version >= "3.9" and python_version < "4.0"
-watchfiles==0.20.0 ; python_version >= "3.9" and python_version < "4.0"
-weaviate-client==3.24.1 ; python_version >= "3.9" and python_version < "4.0"
-websockets==11.0.3 ; python_version >= "3.9" and python_version < "4.0"
-yarl==1.9.2 ; python_version >= "3.9" and python_version < "4.0"
-zipp==3.17.0 ; python_version >= "3.9" and python_version < "3.10"
+openai
+chromadb
+langchain
+pypdf
+tiktoken
+scikit-learn
+gradio