added ux, vision_api, qna.txt

.gitignore

@@ -4,6 +4,9 @@
 results/
 
 *.sqlite
-ux/
 data/
-notebooks/test_model
+
+notebooks/test_model
+screenshot_questions/
+
+# ux/

archive/init_setup.py

@@ -0,0 +1,27 @@
+import main
+
+import pkg_resources
+import shutil
+import os
+
+### To trigger trulens evaluation
+main.main()
+
+### Finally, start streamlit app
+leaderboard_path = pkg_resources.resource_filename(
+                    "trulens_eval", "Leaderboard.py"
+                )
+evaluation_path = pkg_resources.resource_filename(
+                    "trulens_eval", "pages/Evaluations.py"
+                )
+ux_path = pkg_resources.resource_filename(
+                    "trulens_eval", "ux"
+                )
+
+os.makedirs("./pages", exist_ok=True)
+shutil.copyfile(leaderboard_path, os.path.join("./pages", "1_Leaderboard.py"))
+shutil.copyfile(evaluation_path, os.path.join("./pages", "2_Evaluations.py"))
+
+if os.path.exists("./ux"):
+    shutil.rmtree("./ux")
+shutil.copytree(ux_path, "./ux")

raw_documents/qna.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8b44d78e6dec3a285124f0a449ff5bae699ab4ff98ae3826a33a8eb4f182334
+size 1804

streamlit_app.py

@@ -3,13 +3,11 @@ from streamlit_feedback import streamlit_feedback
 
 import os
 import pandas as pd
-import time
+import base64
+from io import BytesIO
+import nest_asyncio
 
-import openai
-
-# from openai import OpenAI
 from llama_index.llms import OpenAI
-
 from llama_index import SimpleDirectoryReader
 from llama_index import Document
 from llama_index import VectorStoreIndex
@@ -17,38 +15,17 @@ from llama_index import ServiceContext
 from llama_index.embeddings import HuggingFaceEmbedding
 from llama_index.memory import ChatMemoryBuffer
 
-import pkg_resources
-import shutil
-import main
-
-### To trigger trulens evaluation
-main.main()
-
-### Finally, start streamlit app
-leaderboard_path = pkg_resources.resource_filename(
-                    "trulens_eval", "Leaderboard.py"
-                )
-evaluation_path = pkg_resources.resource_filename(
-                    "trulens_eval", "pages/Evaluations.py"
-                )
-ux_path = pkg_resources.resource_filename(
-                    "trulens_eval", "ux"
-                )
+from vision_api import get_transcribed_text
 
-os.makedirs("./pages", exist_ok=True)
-shutil.copyfile(leaderboard_path, os.path.join("./pages", "1_Leaderboard.py"))
-shutil.copyfile(evaluation_path, os.path.join("./pages", "2_Evaluations.py"))
-
-if os.path.exists("./ux"):
-    shutil.rmtree("./ux")
-shutil.copytree(ux_path, "./ux")
+nest_asyncio.apply()
 
 # App title
 st.set_page_config(page_title="💬 Open AI Chatbot")
 openai_api = os.getenv("OPENAI_API_KEY")
 
 # "./raw_documents/HI_Knowledge_Base.pdf"
-input_files = ["./raw_documents/HI Chapter Summary Version 1.3.pdf"]
+input_files = ["./raw_documents/HI Chapter Summary Version 1.3.pdf",
+               "./raw_documents/qna.txt"]
 embedding_model = "BAAI/bge-small-en-v1.5"
 system_content = ("You are a helpful study assistant. "
                   "You do not respond as 'User' or pretend to be 'User'. "
@@ -104,25 +81,25 @@ with st.sidebar:
     st.markdown("📖 Reach out to SakiMilo to learn how to create this app!")
 
 if "init" not in st.session_state.keys():
-    st.session_state.init = {"warm_start": "No"}
+    st.session_state.init = {"warm_started": "No"}
     st.session_state.feedback = False
 
 # Store LLM generated responses
 if "messages" not in st.session_state.keys():
     st.session_state.messages = [{"role": "assistant", 
-                                  "content": "How may I assist you today?"}]
+                                  "content": "How may I assist you today?",
+                                  "type": "text"}]
 
 if "feedback_key" not in st.session_state:
     st.session_state.feedback_key = 0
 
-# Display or clear chat messages
-for message in st.session_state.messages:
-    with st.chat_message(message["role"]):
-        st.write(message["content"])
+if "release_file" not in st.session_state:
+    st.session_state.release_file = "false"
 
 def clear_chat_history():
     st.session_state.messages = [{"role": "assistant", 
-                                  "content": "How may I assist you today?"}]
+                                  "content": "How may I assist you today?",
+                                  "type": "text"}]
     chat_engine = get_query_engine(input_files=input_files, 
                                    llm_model=selected_model, 
                                    temperature=temperature,
@@ -187,23 +164,66 @@ def handle_feedback(user_response):
     st.toast("✔️ Feedback received!")
     st.session_state.feedback = False
 
+def handle_image_upload():
+    st.session_state.release_file = "true"
+
 # Warm start
-if st.session_state.init["warm_start"] == "No":
+if st.session_state.init["warm_started"] == "No":
     clear_chat_history()
-    st.session_state.init["warm_start"] = "Yes"
+    st.session_state.init["warm_started"] = "Yes"
+
+# Image upload option
+with st.sidebar:
+    image_file = st.file_uploader("Upload your image here...", 
+                                  type=["png", "jpeg", "jpg"],
+                                  on_change=handle_image_upload)
+
+    if st.session_state.release_file == "true" and image_file:
+        with st.spinner("Uploading..."):
+            b64string = base64.b64encode(image_file.read()).decode('utf-8')
+            message = {
+                    "role": "user", 
+                    "content": b64string,
+                    "type": "image"}
+            st.session_state.messages.append(message)
+
+            transcribed_msg = get_transcribed_text(b64string)
+            message = {
+                    "role": "admin", 
+                    "content": transcribed_msg,
+                    "type": "text"}
+            st.session_state.messages.append(message)
+            st.session_state.release_file = "false"
+
+# Display or clear chat messages
+for message in st.session_state.messages:
+    if message["role"] == "admin":
+        continue
+    with st.chat_message(message["role"]):
+        if message["type"] == "text":
+            st.write(message["content"])
+        elif message["type"] == "image":
+            img_io = BytesIO(base64.b64decode(message["content"].encode("utf-8")))
+            st.image(img_io)
 
 # User-provided prompt
 if prompt := st.chat_input(disabled=not openai_api):
     client = OpenAI()
-    st.session_state.messages.append({"role": "user", "content": prompt})
+    st.session_state.messages.append({"role": "user", 
+                                      "content": prompt, 
+                                      "type": "text"})
     with st.chat_message("user"):
         st.write(prompt)
 
+# Retrieve text prompt from image submission
+if prompt is None and \
+   st.session_state.messages[-1]["role"] == "admin":
+    prompt = st.session_state.messages[-1]["content"]
+
 # Generate a new response if last message is not from assistant
 if st.session_state.messages[-1]["role"] != "assistant":
     with st.chat_message("assistant"):
         with st.spinner("Thinking..."):
-            # response = generate_llm_response(client, prompt)
             response = generate_llm_response(prompt)
             placeholder = st.empty()
             full_response = ""
@@ -212,9 +232,12 @@ if st.session_state.messages[-1]["role"] != "assistant":
                 placeholder.markdown(full_response)
             placeholder.markdown(full_response)
 
-    message = {"role": "assistant", "content": full_response}
+    message = {"role": "assistant", 
+               "content": full_response,
+               "type": "text"}
     st.session_state.messages.append(message)
 
+# Trigger feedback
 if st.session_state.feedback:
     result = streamlit_feedback(
                 feedback_type="thumbs",

ux/add_logo.py

@@ -0,0 +1,50 @@
+import base64
+
+import pkg_resources
+import streamlit as st
+
+from trulens_eval import __package__
+from trulens_eval import __version__
+
+
+def add_logo_and_style_overrides():
+    logo = open(
+        pkg_resources.resource_filename('trulens_eval', 'ux/trulens_logo.svg'),
+        "rb"
+    ).read()
+
+    logo_encoded = base64.b64encode(logo).decode()
+    st.markdown(
+        f"""
+        <style>
+            [data-testid="stSidebarNav"] {{
+                background-image: url('data:image/svg+xml;base64,{logo_encoded}');
+                background-repeat: no-repeat;
+                background-size: 300px auto;
+                padding-top: 50px;
+                background-position: 20px 20px;
+            }}
+            [data-testid="stSidebarNav"]::before {{
+                margin-left: 20px;
+                margin-top: 20px;
+                font-size: 30px;
+                position: relative;
+                top: 100px;
+            }}
+            [data-testid="stSidebarNav"]::after {{
+                margin-left: 20px;
+                color: #aaaaaa;
+                content: "{__package__} {__version__}";
+                font-size: 10pt;
+            }}
+
+            /* For list items in st.dataframe */
+            #portal .clip-region .boe-bubble {{
+                height: auto;
+                border-radius: 4px;
+                padding: 8px;
+            }}
+        </style>
+        """,
+        unsafe_allow_html=True,
+    )

ux/apps.py

@@ -0,0 +1,31 @@
+# Code in support of the Apps.py page.
+
+from typing import Any, ClassVar, Optional
+
+import pydantic
+
+from trulens_eval.app import App
+from trulens_eval.utils.serial import JSON
+
+
+class ChatRecord(pydantic.BaseModel):
+
+    model_config: ClassVar[dict] = dict(
+        arbitrary_types_allowed = True
+    )
+
+    # Human input
+    human: Optional[str] = None
+
+    # Computer response
+    computer: Optional[str] = None
+
+    # Jsonified record. Available only after the app is run on human input and
+    # produced a computer output.
+    record_json: Optional[JSON] = None
+
+    # The final app state for continuing the session.
+    app: App
+
+    # The state of the app as was when this record was produced.
+    app_json: JSON

ux/components.py

@@ -0,0 +1,279 @@
+import json
+import random
+from typing import Dict, List, Optional
+
+import pandas as pd
+import streamlit as st
+
+from trulens_eval.app import ComponentView
+from trulens_eval.keys import REDACTED_VALUE
+from trulens_eval.keys import should_redact_key
+from trulens_eval.schema import Metadata
+from trulens_eval.schema import Record
+from trulens_eval.schema import RecordAppCall
+from trulens_eval.schema import Select
+from trulens_eval.utils.containers import is_empty
+from trulens_eval.utils.json import jsonify
+from trulens_eval.utils.pyschema import CLASS_INFO
+from trulens_eval.utils.pyschema import is_noserio
+from trulens_eval.utils.serial import GetItemOrAttribute
+from trulens_eval.utils.serial import JSON_BASES
+from trulens_eval.utils.serial import Lens
+
+
+def write_or_json(st, obj):
+    """
+    Dispatch either st.json or st.write depending on content of `obj`. If it is
+    a string that can parses into strictly json (dict), use st.json, otherwise
+    use st.write.
+    """
+
+    if isinstance(obj, str):
+        try:
+            content = json.loads(obj)
+            if not isinstance(content, str):
+                st.json(content)
+            else:
+                st.write(content)
+
+        except BaseException:
+            st.write(obj)
+
+
+def copy_to_clipboard(path, *args, **kwargs):
+    st.session_state.clipboard = str(path)
+
+
+def draw_selector_button(path) -> None:
+    st.button(
+        key=str(random.random()),
+        label=f"{Select.render_for_dashboard(path)}",
+        on_click=copy_to_clipboard,
+        args=(path,)
+    )
+
+
+def render_selector_markdown(path) -> str:
+    return f"[`{Select.render_for_dashboard(path)}`]"
+
+
+def render_call_frame(frame: RecordAppCall, path=None) -> str:  # markdown
+    path = path or frame.path
+
+    return (
+        f"__{frame.method.name}__ (__{frame.method.obj.cls.module.module_name}.{frame.method.obj.cls.name}__)"
+    )
+
+
+def dict_to_md(dictionary: dict) -> str:
+    if len(dictionary) == 0:
+        return "No metadata."
+    mdheader = "|"
+    mdseparator = "|"
+    mdbody = "|"
+    for key, value in dictionary.items():
+        mdheader = mdheader + str(key) + "|"
+        mdseparator = mdseparator + "-------|"
+        mdbody = mdbody + str(value) + "|"
+    mdtext = mdheader + "\n" + mdseparator + "\n" + mdbody
+    return mdtext
+
+
+def draw_metadata(metadata: Metadata) -> str:
+    if isinstance(metadata, Dict):
+        return dict_to_md(metadata)
+    else:
+        return str(metadata)
+
+
+def draw_call(call: RecordAppCall) -> None:
+    top = call.stack[-1]
+
+    path = Select.for_record(
+        top.path._append(
+            step=GetItemOrAttribute(item_or_attribute=top.method.name)
+        )
+    )
+
+    with st.expander(label=f"Call " + render_call_frame(top, path=path) + " " +
+                     render_selector_markdown(path)):
+
+        args = call.args
+        rets = call.rets
+
+        for frame in call.stack[::-1][1:]:
+            st.write("Via " + render_call_frame(frame, path=path))
+
+        st.subheader(f"Inputs {render_selector_markdown(path.args)}")
+        if isinstance(args, Dict):
+            st.json(args)
+        else:
+            st.write(args)
+
+        st.subheader(f"Outputs {render_selector_markdown(path.rets)}")
+        if isinstance(rets, Dict):
+            st.json(rets)
+        else:
+            st.write(rets)
+
+
+def draw_calls(record: Record, index: int) -> None:
+    """
+    Draw the calls recorded in a `record`.
+    """
+
+    calls = record.calls
+
+    app_step = 0
+
+    for call in calls:
+        app_step += 1
+
+        if app_step != index:
+            continue
+
+        draw_call(call)
+
+
+def draw_prompt_info(query: Lens, component: ComponentView) -> None:
+    prompt_details_json = jsonify(component.json, skip_specials=True)
+
+    st.caption(f"Prompt details")
+
+    path = Select.for_app(query)
+
+    prompt_types = {
+        k: v for k, v in prompt_details_json.items() if (v is not None) and
+        not is_empty(v) and not is_noserio(v) and k != CLASS_INFO
+    }
+
+    for key, value in prompt_types.items():
+        with st.expander(key.capitalize() + " " +
+                         render_selector_markdown(getattr(path, key)),
+                         expanded=True):
+
+            if isinstance(value, (Dict, List)):
+                st.write(value)
+            else:
+                if isinstance(value, str) and len(value) > 32:
+                    st.text(value)
+                else:
+                    st.write(value)
+
+
+def draw_llm_info(query: Lens, component: ComponentView) -> None:
+    llm_details_json = component.json
+
+    st.subheader(f"*LLM Details*")
+    # path_str = str(query)
+    # st.text(path_str[:-4])
+
+    llm_kv = {
+        k: v for k, v in llm_details_json.items() if (v is not None) and
+        not is_empty(v) and not is_noserio(v) and k != CLASS_INFO
+    }
+    # CSS to inject contained in a string
+    hide_table_row_index = """
+                <style>
+                thead tr th:first-child {display:none}
+                tbody th {display:none}
+                </style>
+                """
+    df = pd.DataFrame.from_dict(llm_kv, orient='index').transpose()
+
+    # Redact any column whose name indicates it might be a secret.
+    for col in df.columns:
+        if should_redact_key(col):
+            df[col] = REDACTED_VALUE
+
+    # TODO: What about columns not indicating a secret but some values do
+    # indicate it as per `should_redact_value` ?
+
+    # Iterate over each column of the DataFrame
+    for column in df.columns:
+        path = getattr(Select.for_app(query), str(column))
+        # Check if any cell in the column is a dictionary
+
+        if any(isinstance(cell, dict) for cell in df[column]):
+            # Create new columns for each key in the dictionary
+            new_columns = df[column].apply(
+                lambda x: pd.Series(x) if isinstance(x, dict) else pd.Series()
+            )
+            new_columns.columns = [
+                f"{key} {render_selector_markdown(path)}"
+                for key in new_columns.columns
+            ]
+
+            # Remove extra zeros after the decimal point
+            new_columns = new_columns.applymap(
+                lambda x: '{0:g}'.format(x) if isinstance(x, float) else x
+            )
+
+            # Add the new columns to the original DataFrame
+            df = pd.concat([df.drop(column, axis=1), new_columns], axis=1)
+
+        else:
+            # TODO: add selectors to the output here
+
+            pass
+
+    # Inject CSS with Markdown
+
+    st.markdown(hide_table_row_index, unsafe_allow_html=True)
+    st.table(df)
+
+
+def draw_agent_info(query: Lens, component: ComponentView) -> None:
+    # copy of draw_prompt_info
+    # TODO: dedup
+    prompt_details_json = jsonify(component.json, skip_specials=True)
+
+    st.subheader(f"*Agent Details*")
+
+    path = Select.for_app(query)
+
+    prompt_types = {
+        k: v for k, v in prompt_details_json.items() if (v is not None) and
+        not is_empty(v) and not is_noserio(v) and k != CLASS_INFO
+    }
+
+    for key, value in prompt_types.items():
+        with st.expander(key.capitalize() + " " +
+                         render_selector_markdown(getattr(path, key)),
+                         expanded=True):
+
+            if isinstance(value, (Dict, List)):
+                st.write(value)
+            else:
+                if isinstance(value, str) and len(value) > 32:
+                    st.text(value)
+                else:
+                    st.write(value)
+
+
+def draw_tool_info(query: Lens, component: ComponentView) -> None:
+    # copy of draw_prompt_info
+    # TODO: dedup
+    prompt_details_json = jsonify(component.json, skip_specials=True)
+
+    st.subheader(f"*Tool Details*")
+
+    path = Select.for_app(query)
+
+    prompt_types = {
+        k: v for k, v in prompt_details_json.items() if (v is not None) and
+        not is_empty(v) and not is_noserio(v) and k != CLASS_INFO
+    }
+
+    for key, value in prompt_types.items():
+        with st.expander(key.capitalize() + " " +
+                         render_selector_markdown(getattr(path, key)),
+                         expanded=True):
+
+            if isinstance(value, (Dict, List)):
+                st.write(value)
+            else:
+                if isinstance(value, str) and len(value) > 32:
+                    st.text(value)
+                else:
+                    st.write(value)

ux/styles.py

@@ -0,0 +1,143 @@
+from collections import defaultdict
+from enum import Enum
+import operator
+from typing import Callable, List, NamedTuple, Optional
+
+import numpy as np
+
+from trulens_eval.utils.serial import SerialModel
+
+
+class ResultCategoryType(Enum):
+    PASS = 0
+    WARNING = 1
+    FAIL = 2
+
+
+class CATEGORY:
+    """
+    Feedback result categories for displaying purposes: pass, warning, fail, or
+    unknown.
+    """
+
+    class Category(SerialModel):
+        name: str
+        adjective: str
+        threshold: float
+        color: str
+        icon: str
+        direction: Optional[str] = None
+        compare: Optional[Callable[[float, float], bool]] = None
+
+    class FeedbackDirection(NamedTuple):
+        name: str
+        ascending: bool
+        thresholds: List[float]
+
+    # support both directions by default
+    # TODO: make this configurable (per feedback definition & per app?)
+    directions = [
+        FeedbackDirection("HIGHER_IS_BETTER", False, [0, 0.6, 0.8]),
+        FeedbackDirection("LOWER_IS_BETTER", True, [0.2, 0.4, 1]),
+    ]
+
+    styling = {
+        "PASS": dict(color="#aaffaa", icon="✅"),
+        "WARNING": dict(color="#ffffaa", icon="⚠️"),
+        "FAIL": dict(color="#ffaaaa", icon="🛑"),
+    }
+
+    for category_name in ResultCategoryType._member_names_:
+        locals()[category_name] = defaultdict(dict)
+
+    for direction in directions:
+        a = sorted(
+            zip(["low", "medium", "high"], sorted(direction.thresholds)),
+            key=operator.itemgetter(1),
+            reverse=not direction.ascending,
+        )
+
+        for enum, (adjective, threshold) in enumerate(a):
+            category_name = ResultCategoryType(enum).name
+            locals()[category_name][direction.name] = Category(
+                name=category_name.lower(),
+                adjective=adjective,
+                threshold=threshold,
+                direction=direction.name,
+                compare=operator.ge
+                if direction.name == "HIGHER_IS_BETTER" else operator.le,
+                **styling[category_name],
+            )
+
+    UNKNOWN = Category(
+        name="unknown",
+        adjective="unknown",
+        threshold=np.nan,
+        color="#aaaaaa",
+        icon="?"
+    )
+
+    # order matters here because `of_score` returns the first best category
+    ALL = [PASS, WARNING, FAIL]  # not including UNKNOWN intentionally
+
+    @staticmethod
+    def of_score(score: float, higher_is_better: bool = True) -> Category:
+        direction_key = "HIGHER_IS_BETTER" if higher_is_better else "LOWER_IS_BETTER"
+
+        for cat in map(operator.itemgetter(direction_key), CATEGORY.ALL):
+            if cat.compare(score, cat.threshold):
+                return cat
+
+        return CATEGORY.UNKNOWN
+
+
+default_direction = "HIGHER_IS_BETTER"
+
+# These would be useful to include in our pages but don't yet see a way to do
+# this in streamlit.
+root_js = f"""
+    var default_pass_threshold = {CATEGORY.PASS[default_direction].threshold};
+    var default_warning_threshold = {CATEGORY.WARNING[default_direction].threshold};
+    var default_fail_threshold = {CATEGORY.FAIL[default_direction].threshold};
+"""
+
+# Not presently used. Need to figure out how to include this in streamlit pages.
+root_html = f"""
+<script>
+    {root_js}
+</script>
+"""
+
+stmetricdelta_hidearrow = """
+    <style> [data-testid="stMetricDelta"] svg { display: none; } </style>
+    """
+
+valid_directions = ["HIGHER_IS_BETTER", "LOWER_IS_BETTER"]
+
+cellstyle_jscode = {
+    k: f"""function(params) {{
+        let v = parseFloat(params.value);
+        """ + "\n".join(
+        f"""
+        if (v {'>=' if k == "HIGHER_IS_BETTER" else '<='} {cat.threshold}) {{
+            return {{
+                'color': 'black',
+                'backgroundColor': '{cat.color}'
+            }};
+        }}
+    """ for cat in map(operator.itemgetter(k), CATEGORY.ALL)
+    ) + f"""
+        // i.e. not a number
+        return {{
+            'color': 'black',
+            'backgroundColor': '{CATEGORY.UNKNOWN.color}'
+        }};
+    }}""" for k in valid_directions
+}
+
+hide_table_row_index = """
+    <style>
+        thead tr th:first-child {display:none}
+        tbody th {display:none}
+    </style>
+    """

vision_api.py

@@ -0,0 +1,38 @@
+import streamlit as st
+import os, base64, requests
+
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+
+def get_transcribed_text(base64_image):
+
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {OPENAI_API_KEY}"
+    }
+
+    payload = {
+        "model": "gpt-4-vision-preview",
+        "messages": [
+            {
+                "role": "user",
+                "content": [
+                    {
+                    "type": "text",
+                    "text": "transcribe the image into text for me."
+                    },
+                    {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": f"data:image/jpeg;base64,{base64_image}"
+                    }
+                    }
+                ]
+            }
+        ],
+        "max_tokens": 300
+    }
+
+    response = requests.post("https://api.openai.com/v1/chat/completions", headers=headers, json=payload)
+    transcribed_msg = response.json()["choices"][0]["message"]["content"]
+
+    return transcribed_msg