Update

app.py

@@ -1,9 +1,12 @@
-import numpy as np
-import open_clip
 import streamlit as st
 
 from app_lib.main import main
 
+with open("style.css", "r") as f:
+    style = f.read()
+with open("header.md", "r") as f:
+    header = f.read()
+
 if "sidebar_state" not in st.session_state:
     st.session_state.sidebar_state = "collapsed"
 if "disabled" not in st.session_state:
@@ -14,33 +17,9 @@ if "results" not in st.session_state:
 st.set_page_config(layout="wide", initial_sidebar_state=st.session_state.sidebar_state)
 
 st.session_state.sidebar_state = "collapsed"
-st.markdown(
-    """
-        <style>
-            textarea {
-                font-family: monospace !important;
-            }
-            input {
-                font-family: monospace !important;
-            }
-
-            [data-testid="stHorizontalBlock"] {
-                align-items: center;
-            }
-        </style>
-        """,
-    unsafe_allow_html=True,
-)
-
-st.markdown(
-    """
-        # I Bet You Did Not Mean That
-
-        Official HF Space for the paper [*I Bet You Did Not Mean That: Testing Semantci Importance via Betting*](https://arxiv.org/pdf/2405.19146), by [Jacopo Teneggi](https://jacopoteneggi.github.io) and [Jeremias Sulam](https://sites.google.com/view/jsulam).
+st.markdown(f"<style>{style}</style>", unsafe_allow_html=True)
 
-        ---
-    """,
-)
+st.markdown(header)
 
 if __name__ == "__main__":
     main()

app_lib/main.py

@@ -10,6 +10,7 @@ from app_lib.user_input import (
     get_model_name,
 )
 from app_lib.test import test
+from app_lib.viz import viz_results
 
 
 def _disable():
@@ -20,50 +21,58 @@ def main(device=torch.device("cuda" if torch.cuda.is_available() else "cpu")):
     columns = st.columns([0.40, 0.60])
 
     with columns[0]:
-        model_name = get_model_name()
+        st.header("Choose Image and Concepts")
 
-        row1 = st.columns(2)
-        row2 = st.columns(2)
+        image_col, concepts_col = st.columns(2)
 
-        with row1[0]:
+        with image_col:
             image = get_image()
             st.image(image, use_column_width=True)
-        with row1[1]:
-            class_name, class_ready, class_error = get_class_name()
-            concepts, concepts_ready, concepts_error = get_concepts()
-            cardinality = get_cardinality(concepts, concepts_ready)
 
-        with row2[0]:
             change_image_button = st.button(
                 "Change Image",
-                use_container_width=True,
+                use_container_width=False,
                 disabled=st.session_state.disabled,
             )
             if change_image_button:
                 st.session_state.sidebar_state = "expanded"
                 st.experimental_rerun()
-        with row2[1]:
-            ready = class_ready and concepts_ready
+        with concepts_col:
+            model_name = get_model_name()
+            class_name, class_ready, class_error = get_class_name()
+            concepts, concepts_ready, concepts_error = get_concepts()
+            cardinality = int(len(concepts) / 2)
+            # get_cardinality(concepts, concepts_ready)
 
-            error_message = ""
-            if class_error is not None:
-                error_message += f"- {class_error}\n"
-            if concepts_error is not None:
-                error_message += f"- {concepts_error}\n"
-            if error_message:
-                st.error(error_message)
+        ready = class_ready and concepts_ready
 
+        error_message = ""
+        if class_error is not None:
+            error_message += f"- {class_error}\n"
+        if concepts_error is not None:
+            error_message += f"- {concepts_error}\n"
+        if error_message:
+            st.error(error_message)
+
+        with st.container():
             test_button = st.button(
-                "Test",
+                "Test Concepts",
                 use_container_width=True,
                 on_click=_disable,
                 disabled=st.session_state.disabled or not ready,
             )
 
+            with st.popover("Advanced settings", disabled=st.session_state.disabled):
+                st.markdown("Hello World 👋")
+
     with columns[1]:
-        _, centercol, _ = st.columns(3)
-        with centercol:
-            if test_button:
+        st.header("Results")
+
+        if test_button:
+            st.session_state.results = None
+
+            _, centercol, _ = st.columns(3)
+            with centercol:
                 test(
                     image,
                     class_name,
@@ -73,3 +82,5 @@ def main(device=torch.device("cuda" if torch.cuda.is_available() else "cpu")):
                     model_name,
                     device,
                 )
+
+        viz_results()

app_lib/test.py

@@ -4,7 +4,7 @@ import open_clip
 import h5py
 import streamlit as st
 import numpy as np
-import time
+import pandas as pd
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
 import ml_collections
@@ -170,12 +170,26 @@ def test(image, class_name, concepts, cardinality, dataset_name, model_name, dev
                 results.append(future.result())
                 progress_bar.progress((idx + 1) / len(concepts))
 
-        # print(results)
-        # wealth = np.empty((testing_config.tau_max, len(concepts)))
-        # wealth[:] = np.nan
-        # for _results in results:
-        #     concept_idx = concepts.index(_results["concept"])
-        #     _wealth =
+        rejected = np.empty((testing_config.r, len(concepts)))
+        tau = np.empty((testing_config.r, len(concepts)))
+        wealth = np.empty((testing_config.r, testing_config.tau_max, len(concepts)))
+
+        for _results in results:
+            concept_idx = concepts.index(_results["concept"])
+
+            rejected[:, concept_idx] = np.array(_results["rejected"])
+            tau[:, concept_idx] = np.array(_results["tau"])
+            wealth[:, :, concept_idx] = np.array(_results["wealth"])
+
+        tau /= testing_config.tau_max
+
+        st.session_state.results = {
+            "significance_level": testing_config.significance_level,
+            "concepts": concepts,
+            "rejected": rejected,
+            "tau": tau,
+            "wealth": wealth,
+        }
 
     st.session_state.disabled = False
     st.experimental_rerun()

app_lib/user_input.py

@@ -20,10 +20,9 @@ def _validate_concepts(concepts):
         return (False, "Maximum 10 concepts allowed")
     return (True, None)
 
-
 def get_model_name():
     return st.selectbox(
-        "Choose a model to test",
+        "Model to test",
         options=list(SUPPORTED_MODELS.keys()),
         help="Name of the vision-language model to test the predictions of.",
         disabled=st.session_state.disabled,

app_lib/viz.py

@@ -0,0 +1,40 @@
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+
+
+def _viz_wealth(results):
+    wealth = results["wealth"]
+    concepts = results["concepts"]
+    significance_level = results["significance_level"]
+
+    wealth_mu = wealth.mean(axis=0)
+
+    wealth_df = []
+    for concept_idx, concept in enumerate(concepts):
+        for t in range(wealth.shape[1]):
+            wealth_df.append(
+                {"time": t, "concept": concept, "wealth": wealth_mu[t, concept_idx]}
+            )
+    wealth_df = pd.DataFrame(wealth_df)
+
+    fig = px.line(wealth_df, x="time", y="wealth", color="concept")
+    fig.update_yaxes(range=[0, 3 * 1 / significance_level])
+    st.plotly_chart(fig, use_container_width=True)
+
+
+def viz_results():
+    results = st.session_state.results
+
+    if results is None:
+        st.info("Run tests to show results", icon="ℹ️")
+    else:
+        rank_tab, wealth_tab = st.tabs(["Rank of importance", "Wealth process"])
+
+        with rank_tab:
+            st.subheader("Rank of Semantic Importance")
+        with wealth_tab:
+            st.subheader("Wealth Process of Testing Procedures")
+
+            if results is not None:
+                _viz_wealth(results)

header.md

@@ -0,0 +1,5 @@
+# 🤔 I Bet You Did Not Mean That
+
+Official HF Space for the paper [*I Bet You Did Not Mean That: Testing Semantic Importance via Betting*](https://arxiv.org/pdf/2405.19146), by [Jacopo Teneggi](https://jacopoteneggi.github.io) and [Jeremias Sulam](https://sites.google.com/view/jsulam).
+
+---

style.css

@@ -0,0 +1,39 @@
+textarea {
+    font-family: monospace !important;
+}
+
+input {
+    font-family: monospace !important;
+}
+
+h1 {
+    padding-top: 0 !important;
+}
+
+[data-testid="stHorizontalBlock"] [data-testid="stHorizontalBlock"] {
+    align-items: center;
+}
+
+[data-testid="stButton"] {
+    display: flex;
+    justify-content: center;
+}
+
+[data-testid="stVerticalBlock"]:has(> [data-testid="stPopover"]) {
+    display: block;
+}
+
+[data-testid="stPopover"] {
+    button {
+        padding: 0;
+        border: 0;
+
+        div:first-of-type>p {
+            font-size: small;
+        }
+    }
+
+    button:hover>div:first-of-type>p {
+        text-decoration: underline;
+    }
+}