added trulens implementation for evaluation

.gitignore

@@ -1,3 +1,8 @@
 .DS_Store
 
-.streamlit/
+.streamlit/
+results/
+
+*.sqlite
+ux/
+pages/

main.py

@@ -9,32 +9,62 @@ from llama_index import ServiceContext
 from llama_index.llms import OpenAI
 
 from llama_index.embeddings import HuggingFaceEmbedding
+from trulens_eval import Tru
 
+from utils import get_prebuilt_trulens_recorder
 
 openai.api_key = utils.get_openai_api_key()
 
-if __name__ == "__main__":
+def main():
+    
+    if not os.path.exists("./default.sqlite"):
+
+        documents = SimpleDirectoryReader(
+            input_files=["./raw_documents/HI_knowledge_base.pdf"]
+        ).load_data()
+
+        document = Document(text="\n\n".join([doc.text for doc in documents]))
+
+        ### gpt-4-1106-preview
+        ### gpt-3.5-turbo-1106 / gpt-3.5-turbo
+        llm = OpenAI(model="gpt-3.5-turbo-1106", temperature=0.1)
+        embed_model = HuggingFaceEmbedding(model_name="BAAI/bge-small-en-v1.5")
+
+        service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model)
+        index = VectorStoreIndex.from_documents([document], service_context=service_context)
+
+        query_engine = index.as_query_engine()
 
-    documents = SimpleDirectoryReader(
-        input_files=["./raw_documents/HI_knowledge_base.pdf"]
-    ).load_data()
+        separator = "\n\n"
+        eval_questions = []
+        with open('raw_documents/eval_questions.txt', 'r') as file:
+            content = file.read()
 
-    document = Document(text="\n\n".join([doc.text for doc in documents]))
+        for question in content.split(separator):
+            print(question)
+            print(separator)
+            eval_questions.append(question.strip())
 
-    ### gpt-4-1106-preview
-    ### gpt-3.5-turbo-1106 / gpt-3.5-turbo
-    llm = OpenAI(model="gpt-3.5-turbo-1106", temperature=0.1)
-    embed_model = HuggingFaceEmbedding(model_name="BAAI/bge-small-en-v1.5")
+        response = query_engine.query(eval_questions[0])
+        print(str(response))
 
-    service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model)
-    index = VectorStoreIndex.from_documents([document], service_context=service_context)
+        tru = Tru()
+        # tru.reset_database()
 
-    query_engine = index.as_query_engine()
+        tru_recorder = get_prebuilt_trulens_recorder(query_engine,
+                                                    app_id="Direct Query Engine")
+        with tru_recorder as recording:
+            for question in eval_questions:
+                response = query_engine.query(question)
+
+        records, feedback = tru.get_records_and_feedback(app_ids=[])
+
+        os.makedirs("results", exist_ok=True)
+        records.to_csv("results/records.csv", index=False)
+
+        print(tru.db.engine.url.render_as_string(hide_password=False))
+        # tru.run_dashboard()
+
+if __name__ == "__main__":
 
-    response = query_engine.query(
-        ("Intermediate and Long Term Care (ILTC) services are for those who need further care and"
-         "treatment after discharge from the hospital, who may need assistance with their activities of"
-         "daily living. This can be through"
-        )
-    )
-    print(str(response))
+    main()

raw_documents/eval_answers.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d6f42a68ccf96496a6dcd89016e53ebb1add84c42ecef1fffe08e211037c4df
+size 332

raw_documents/eval_questions.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:931b5c74d5696e5efb242c7d968765734a621d881642a1b16dbd1d004fd2900e
+size 1473

requirements.txt

@@ -1,2 +1,4 @@
 openai==1.6.1
-streamlit==1.29.0
+streamlit==1.29.0
+trulens==0.13.4
+trulens-eval==0.20.0

streamlit_app.py

@@ -1,13 +1,44 @@
 import streamlit as st
 import os
+import pandas as pd
 
 import openai
 from openai import OpenAI
 
+import pkg_resources
+import shutil
+import main
+
+### To trigger trulens evaluation
+main.main()
+
+### Finally, start streamlit app
+leaderboard_path = pkg_resources.resource_filename(
+                    "trulens_eval", "Leaderboard.py"
+                )
+evaluation_path = pkg_resources.resource_filename(
+                    "trulens_eval", "pages/Evaluations.py"
+                )
+ux_path = pkg_resources.resource_filename(
+                    "trulens_eval", "ux"
+                )
+shutil.copyfile(leaderboard_path, os.path.join("pages", "1_Leaderboard.py"))
+shutil.copyfile(evaluation_path, os.path.join("pages", "2_Evaluations.py"))
+if os.path.exists("./ux"):
+    shutil.rmtree("./ux")
+shutil.copytree(ux_path, "./ux")
+
 # App title
 st.set_page_config(page_title="💬 Open AI Chatbot")
 openai_api = os.getenv("OPENAI_API_KEY")
 
+data_df = pd.DataFrame(
+    {
+        "Completion": [30, 40, 100, 10],
+    }
+)
+data_df.index = ["Chapter 1", "Chapter 2", "Chapter 3", "Chapter 4"]
+
 # Replicate Credentials
 with st.sidebar:
     st.title("💬 Open AI Chatbot")
@@ -33,6 +64,20 @@ with st.sidebar:
                                            key="selected_model")
     temperature = st.sidebar.slider("temperature", min_value=0.01, max_value=2.0, 
                                     value=0.1, step=0.01)
+    st.data_editor(
+        data_df,
+        column_config={
+            "Completion": st.column_config.ProgressColumn(
+                            "Completion %",
+                            help="Percentage of content covered",
+                            format="%.1f%%",
+                            min_value=0,
+                            max_value=100,
+            ),
+        },
+        hide_index=False,
+    )
+
     st.markdown("📖 Reach out to SakiMilo to learn how to create this app!")
 
 # Store LLM generated responses

utils.py

@@ -1,4 +1,160 @@
 import os
+import numpy as np
+from trulens_eval import (
+    Feedback,
+    TruLlama,
+    OpenAI
+)
+
+from trulens_eval.feedback import Groundedness
+import nest_asyncio
+
+from llama_index import ServiceContext, VectorStoreIndex, StorageContext
+from llama_index.node_parser import SentenceWindowNodeParser
+from llama_index.indices.postprocessor import MetadataReplacementPostProcessor
+from llama_index.indices.postprocessor import SentenceTransformerRerank
+from llama_index import load_index_from_storage
+
+from llama_index.node_parser import HierarchicalNodeParser
+from llama_index.node_parser import get_leaf_nodes
+from llama_index import StorageContext
+from llama_index.retrievers import AutoMergingRetriever
+from llama_index.indices.postprocessor import SentenceTransformerRerank
+from llama_index.query_engine import RetrieverQueryEngine
+
+
+nest_asyncio.apply()
+openai = OpenAI()
+
+qa_relevance = (
+    Feedback(openai.relevance_with_cot_reasons, name="Answer Relevance")
+    .on_input_output()
+)
+
+qs_relevance = (
+    Feedback(openai.relevance_with_cot_reasons, name = "Context Relevance")
+    .on_input()
+    .on(TruLlama.select_source_nodes().node.text)
+    .aggregate(np.mean)
+)
+
+#grounded = Groundedness(groundedness_provider=openai, summarize_provider=openai)
+grounded = Groundedness(groundedness_provider=openai)
+
+groundedness = (
+    Feedback(grounded.groundedness_measure_with_cot_reasons, name="Groundedness")
+        .on(TruLlama.select_source_nodes().node.text)
+        .on_output()
+        .aggregate(grounded.grounded_statements_aggregator)
+)
+
+feedbacks = [qa_relevance, qs_relevance, groundedness]
 
 def get_openai_api_key():
-    return os.getenv("OPENAI_API_KEY")
+    return os.getenv("OPENAI_API_KEY")
+
+def get_trulens_recorder(query_engine, feedbacks, app_id):
+    tru_recorder = TruLlama(
+        query_engine,
+        app_id=app_id,
+        feedbacks=feedbacks
+    )
+    return tru_recorder
+
+def get_prebuilt_trulens_recorder(query_engine, app_id):
+    tru_recorder = TruLlama(
+        query_engine,
+        app_id=app_id,
+        feedbacks=feedbacks
+        )
+    return tru_recorder
+
+def build_sentence_window_index(
+    document, llm, embed_model="local:BAAI/bge-small-en-v1.5", save_dir="sentence_index"
+):
+    # create the sentence window node parser w/ default settings
+    node_parser = SentenceWindowNodeParser.from_defaults(
+        window_size=3,
+        window_metadata_key="window",
+        original_text_metadata_key="original_text",
+    )
+    sentence_context = ServiceContext.from_defaults(
+        llm=llm,
+        embed_model=embed_model,
+        node_parser=node_parser,
+    )
+    if not os.path.exists(save_dir):
+        sentence_index = VectorStoreIndex.from_documents(
+            [document], service_context=sentence_context
+        )
+        sentence_index.storage_context.persist(persist_dir=save_dir)
+    else:
+        sentence_index = load_index_from_storage(
+            StorageContext.from_defaults(persist_dir=save_dir),
+            service_context=sentence_context,
+        )
+
+    return sentence_index
+
+def get_sentence_window_query_engine(
+    sentence_index,
+    similarity_top_k=6,
+    rerank_top_n=2,
+):
+    # define postprocessors
+    postproc = MetadataReplacementPostProcessor(target_metadata_key="window")
+    rerank = SentenceTransformerRerank(
+        top_n=rerank_top_n, model="BAAI/bge-reranker-base"
+    )
+
+    sentence_window_engine = sentence_index.as_query_engine(
+        similarity_top_k=similarity_top_k, node_postprocessors=[postproc, rerank]
+    )
+    return sentence_window_engine
+
+def build_automerging_index(
+    documents,
+    llm,
+    embed_model="local:BAAI/bge-small-en-v1.5",
+    save_dir="merging_index",
+    chunk_sizes=None,
+):
+    chunk_sizes = chunk_sizes or [2048, 512, 128]
+    node_parser = HierarchicalNodeParser.from_defaults(chunk_sizes=chunk_sizes)
+    nodes = node_parser.get_nodes_from_documents(documents)
+    leaf_nodes = get_leaf_nodes(nodes)
+    merging_context = ServiceContext.from_defaults(
+        llm=llm,
+        embed_model=embed_model,
+    )
+    storage_context = StorageContext.from_defaults()
+    storage_context.docstore.add_documents(nodes)
+
+    if not os.path.exists(save_dir):
+        automerging_index = VectorStoreIndex(
+            leaf_nodes, storage_context=storage_context, service_context=merging_context
+        )
+        automerging_index.storage_context.persist(persist_dir=save_dir)
+    else:
+        automerging_index = load_index_from_storage(
+            StorageContext.from_defaults(persist_dir=save_dir),
+            service_context=merging_context,
+        )
+    return automerging_index
+
+def get_automerging_query_engine(
+    automerging_index,
+    similarity_top_k=12,
+    rerank_top_n=2,
+):
+    base_retriever = automerging_index.as_retriever(similarity_top_k=similarity_top_k)
+    retriever = AutoMergingRetriever(
+        base_retriever, automerging_index.storage_context, verbose=True
+    )
+    rerank = SentenceTransformerRerank(
+        top_n=rerank_top_n, model="BAAI/bge-reranker-base"
+    )
+    auto_merging_engine = RetrieverQueryEngine.from_args(
+        retriever, node_postprocessors=[rerank]
+    )
+    return auto_merging_engine