Update

.pre-commit-config.yaml

@@ -8,3 +8,4 @@ repos:
     rev: 22.6.0
     hooks:
       - id: black-jupyter
+        args: ["--preview"]

.streamlit/config.toml

@@ -0,0 +1,2 @@
+[theme]
+base = "light"

app.py

@@ -6,6 +6,10 @@ if "sidebar_state" not in st.session_state:
     st.session_state.sidebar_state = "collapsed"
 if "disabled" not in st.session_state:
     st.session_state.disabled = False
+if "image_name" not in st.session_state:
+    st.session_state.image_name = None
+if "tested" not in st.session_state:
+    st.session_state.tested = False
 if "results" not in st.session_state:
     st.session_state.results = None
 

app_lib/defaults.py

@@ -0,0 +1,14 @@
+DATASET_NAME = "imagenette"
+MODEL_NAME = "open_clip:ViT-B-32"
+
+SIGNIFICANCE_LEVEL_VALUE = 0.05
+SIGNIFICANCE_LEVEL_STEP = 0.01
+
+TAU_MAX_VALUE = 200
+TAU_MAX_STEP = 50
+
+R_VALUE = 10
+R_STEP = 5
+
+CARDINALITY_VALUE = lambda concepts: int(len(concepts) / 2)
+CARDINALITY_STEP = 1

app_lib/main.py

@@ -1,7 +1,7 @@
 import streamlit as st
 import torch
 
-from app_lib.test import get_testing_config, test
+from app_lib.test import get_testing_config, load_precomputed_results, test
 from app_lib.user_input import (
     get_advanced_settings,
     get_class_name,
@@ -19,10 +19,6 @@ def _disable():
 def main(device=torch.device("cuda" if torch.cuda.is_available() else "cpu")):
     columns = st.columns([0.40, 0.60])
 
-    with columns[1]:
-        st.header("Results")
-        viz_results()
-
     with columns[0]:
         st.header("Choose Image and Concepts")
 
@@ -30,6 +26,13 @@ def main(device=torch.device("cuda" if torch.cuda.is_available() else "cpu")):
 
         with image_col:
             image_name, image = get_image()
+            if image_name != st.session_state.image_name:
+                st.session_state.image_name = image_name
+                st.session_state.tested = False
+
+            if image_name is not None and not st.session_state.tested:
+                st.session_state.results = load_precomputed_results(image_name)
+
             st.image(image, use_column_width=True)
 
             change_image_button = st.button(
@@ -85,5 +88,11 @@ def main(device=torch.device("cuda" if torch.cuda.is_available() else "cpu")):
                 cardinality,
                 dataset_name,
                 model_name,
-                device,
+                device=device,
             )
+
+            st.session_state.tested = True
+
+    with columns[1]:
+        st.header("Results")
+        viz_results()

app_lib/test.py

@@ -1,3 +1,4 @@
+import os
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
 import clip
@@ -145,6 +146,14 @@ def get_testing_config(**kwargs):
     return testing_config
 
 
+def load_precomputed_results(image_name):
+    results = np.load(
+        os.path.join("assets", "results", f"{image_name.split('.')[0]}.npy"),
+        allow_pickle=True,
+    ).item()
+    return results
+
+
 def test(
     testing_config,
     image,
@@ -153,23 +162,44 @@ def test(
     cardinality,
     dataset_name,
     model_name,
-    device,
+    device=torch.device("cuda" if torch.cuda.is_available() else "cpu"),
+    with_streamlit=True,
 ):
-    with st.spinner("Loading model"):
+    if with_streamlit:
+        with st.spinner("Loading model"):
+            model, preprocess, tokenizer = _load_model(model_name, device)
+    else:
         model, preprocess, tokenizer = _load_model(model_name, device)
 
-    with st.spinner("Encoding concepts"):
+    if with_streamlit:
+        with st.spinner("Encoding concepts"):
+            cbm = _encode_concepts(tokenizer, model, concepts, device)
+    else:
         cbm = _encode_concepts(tokenizer, model, concepts, device)
 
-    with st.spinner("Encoding image"):
+    if with_streamlit:
+        with st.spinner("Encoding image"):
+            h = _encode_image(model, preprocess, image, device)
+    else:
         h = _encode_image(model, preprocess, image, device)
-        z = h @ cbm.T
-        z = z.squeeze()
-
-    progress_bar = st.progress(
-        0,
-        text=f"Testing concepts (can take up to a minute) [0 / {len(concepts)} completed]",
-    )
+    z = h @ cbm.T
+    z = z.squeeze()
+
+    if with_streamlit:
+        progress_bar = st.progress(
+            0,
+            text=(
+                "Testing concepts (can take up to a minute) [0 /"
+                f" {len(concepts)} completed]"
+            ),
+        )
+        progress_bar.progress(
+            1 / (len(concepts) + 1),
+            text=(
+                "Testing concepts (can take up to a minute) [0 /"
+                f" {len(concepts)} completed]"
+            ),
+        )
 
     embedding = _load_dataset(dataset_name, model_name)
     semantics = embedding @ cbm.T
@@ -177,11 +207,6 @@ def test(
 
     classifier = _encode_class_name(tokenizer, model, class_name, device)
 
-    progress_bar.progress(
-        1 / (len(concepts) + 1),
-        text=f"Testing concepts (can take up to a minute) [0 / {len(concepts)} completed]",
-    )
-
     with ThreadPoolExecutor() as executor:
         futures = [
             executor.submit(
@@ -200,10 +225,14 @@ def test(
         results = []
         for idx, future in enumerate(as_completed(futures)):
             results.append(future.result())
-            progress_bar.progress(
-                (idx + 2) / (len(concepts) + 1),
-                text=f"Testing concepts (can take up to a minute) [{idx + 1} / {len(concepts)} completed]",
-            )
+            if with_streamlit:
+                progress_bar.progress(
+                    (idx + 2) / (len(concepts) + 1),
+                    text=(
+                        f"Testing concepts (can take up to a minute) [{idx + 1} /"
+                        f" {len(concepts)} completed]"
+                    ),
+                )
 
     rejected = np.empty((testing_config.r, len(concepts)))
     tau = np.empty((testing_config.r, len(concepts)))
@@ -218,7 +247,7 @@ def test(
 
     tau /= testing_config.tau_max
 
-    st.session_state.results = {
+    results = {
         "significance_level": testing_config.significance_level,
         "concepts": concepts,
         "rejected": rejected,
@@ -226,5 +255,9 @@ def test(
         "wealth": wealth,
     }
 
-    st.session_state.disabled = False
-    st.experimental_rerun()
+    if with_streamlit:
+        st.session_state.results = results
+        st.session_state.disabled = False
+        st.experimental_rerun()
+    else:
+        return results

app_lib/user_input.py

@@ -5,6 +5,7 @@ import streamlit as st
 from PIL import Image
 from streamlit_image_select import image_select
 
+import app_lib.defaults as defaults
 from app_lib.utils import SUPPORTED_DATASETS, SUPPORTED_MODELS
 
 IMAGE_DIR = os.path.join("assets", "images")
@@ -30,102 +31,97 @@ def _validate_concepts(concepts):
 
 
 def _get_significance_level():
-    STEP, DEFAULT = 0.01, 0.05
+    default = defaults.SIGNIFICANCE_LEVEL_VALUE
+    step = defaults.SIGNIFICANCE_LEVEL_STEP
     return st.slider(
         "Significance level",
-        help=" ".join(
-            [
-                "The level of significance of the tests.",
-                f"Defaults to {DEFAULT:.2F}.",
-            ]
-        ),
-        min_value=STEP,
+        help=f"The level of significance of the tests. Defaults to {default:.2F}.",
+        min_value=step,
         max_value=1.0,
-        value=DEFAULT,
-        step=STEP,
+        value=default,
+        step=step,
         disabled=st.session_state.disabled,
     )
 
 
 def _get_tau_max():
-    STEP, DEFAULT = 50, 200
+    default = defaults.TAU_MAX_VALUE
+    step = defaults.TAU_MAX_STEP
     return int(
         st.slider(
             "Length of test",
-            help=" ".join(
-                [
-                    "The maximum number of steps for each test.",
-                    f"Defaults to {DEFAULT}.",
-                ]
-            ),
-            min_value=STEP,
+            help=f"The maximum number of steps for each test. Defaults to {default}.",
+            min_value=step,
             max_value=1000,
-            step=STEP,
-            value=DEFAULT,
+            step=step,
+            value=default,
             disabled=st.session_state.disabled,
         )
     )
 
 
 def _get_number_of_tests():
-    STEP, DEFAULT = 5, 10
+    default = defaults.R_VALUE
+    step = defaults.R_STEP
     return int(
         st.slider(
             "Number of tests per concept",
-            help=" ".join(
-                [
-                    "The number of tests to average for each concept.",
-                    f"Defaults to {DEFAULT}.",
-                ]
+            help=(
+                "The number of tests to average for each concept. "
+                f"Defaults to {default}."
             ),
-            min_value=STEP,
+            min_value=step,
             max_value=100,
-            step=STEP,
-            value=DEFAULT,
+            step=step,
+            value=default,
             disabled=st.session_state.disabled,
         )
     )
 
 
 def _get_cardinality(concepts, concepts_ready):
-    DEFAULT = lambda concepts: int(len(concepts) / 2)
+    default = defaults.CARDINALITY_VALUE
+    step = defaults.CARDINALITY_STEP
     return st.slider(
         "Size of conditioning set",
-        help=" ".join(
-            [
-                "The number of concepts to condition model predictions on.",
-                "Defaults to half of the number of concepts.",
-            ]
+        help=(
+            "The number of concepts to condition model predictions on. "
+            "Defaults to half of the number of concepts.",
         ),
         min_value=1,
         max_value=max(2, len(concepts) - 1),
-        value=DEFAULT(concepts),
-        step=1,
+        value=default(concepts),
+        step=step,
         disabled=st.session_state.disabled or not concepts_ready,
     )
 
 
 def _get_dataset_name():
-    DEFAULT = SUPPORTED_DATASETS.index("imagenette")
+    options = SUPPORTED_DATASETS
+    default_idx = options.index(defaults.DATASET_NAME)
     return st.selectbox(
         "Dataset",
-        options=SUPPORTED_DATASETS,
-        index=DEFAULT,
-        help=" ".join(
-            [
-                "Name of the dataset to use to train sampler.",
-                "Defaults to Imagenette.",
-            ]
+        options=options,
+        index=default_idx,
+        help=(
+            "Name of the dataset to use to train sampler."
+            f"Defaults to {SUPPORTED_DATASETS[default_idx]}."
         ),
         disabled=st.session_state.disabled,
     )
 
 
 def get_model_name():
+    options = list(SUPPORTED_MODELS.keys())
+    default_idx = options.index(defaults.MODEL_NAME)
     return st.selectbox(
         "Model to test",
-        options=list(SUPPORTED_MODELS.keys()),
-        help="Name of the vision-language model to test the predictions of.",
+        options=options,
+        index=default_idx,
+        help=(
+            "Name of the vision-language model to test the predictions of."
+            f"Defaults to {options[default_idx]}"
+        ),
         disabled=st.session_state.disabled,
     )
 
@@ -148,13 +144,13 @@ def get_image():
 
 
 def get_class_name(image_name=None):
-    DEFAULT = (
+    default = (
         IMAGE_PRESETS[image_name.split(".")[0]]["class_name"] if image_name else ""
     )
     class_name = st.text_input(
-        "Class to test",
+        "Class to predict",
         help="Name of the class to build the zero-shot CLIP classifier with.",
-        value=DEFAULT,
+        value=default,
         disabled=st.session_state.disabled,
         placeholder="Type class name here",
     )
@@ -164,16 +160,19 @@ def get_class_name(image_name=None):
 
 
 def get_concepts(image_name=None):
-    DEFAULT = (
+    default = (
         "\n".join(IMAGE_PRESETS[image_name.split(".")[0]]["concepts"])
         if image_name
         else ""
     )
     concepts = st.text_area(
         "Concepts to test",
-        help="List of concepts to test the predictions of the model with. Write one concept per line. Maximum 10 concepts allowed.",
+        help=(
+            "List of concepts to test the predictions of the model with. "
+            "Write one concept per line. Maximum 10 concepts allowed."
+        ),
         height=160,
-        value=DEFAULT,
+        value=default,
         disabled=st.session_state.disabled,
         placeholder="Type one concept\nper line",
     )

app_lib/viz.py

@@ -6,6 +6,32 @@ import streamlit as st
 
 
 def _viz_rank(results):
+    tau = results["tau"]
+    concepts = results["concepts"]
+
+    tau_mu = tau.mean(axis=0)
+
+    sorted_idx = np.argsort(tau_mu)
+    sorted_tau = tau_mu[sorted_idx]
+    sorted_concepts = [concepts[idx] for idx in sorted_idx]
+
+    min_size, max_size = 14, 50
+
+    _, centercol, _ = st.columns(3)
+    with centercol:
+        with st.container():
+            for concept, tau in zip(sorted_concepts, sorted_tau):
+                style = (
+                    "text-align:center;"
+                    f"font-size:{max_size - tau * (max_size - min_size)}px"
+                )
+                st.write(
+                    f"<p style='{style}'>{concept}</p>",
+                    unsafe_allow_html=True,
+                )
+
+
+def _viz_test(results):
     rejected = results["rejected"]
     tau = results["tau"]
     concepts = results["concepts"]
@@ -55,7 +81,11 @@ def _viz_rank(results):
     )
     fig.add_vline(significance_level, line_dash="dash", line_color="black")
 
-    fig.update_layout(yaxis_title="Rank of importance", xaxis_title="")
+    fig.update_layout(
+        yaxis_title="Rank of importance",
+        xaxis_title="",
+        margin=dict(l=20, r=20, t=20, b=20),
+    )
     if rank_df["tau"].min() <= 0.3:
         fig.update_layout(
             legend=dict(
@@ -104,24 +134,62 @@ def viz_results():
     if results is None:
         st.info("Test concepts to show results", icon="ℹ️")
     else:
-        rank_tab, wealth_tab = st.tabs(["Rank of importance", "Wealth process"])
+        rank_tab, test_tab, wealth_tab = st.tabs(
+            ["Rank of importance", "Testing results", "Wealth process"]
+        )
 
         with rank_tab:
-            st.subheader("Rank of Semantic Importance")
+            st.subheader("Rank of Importance")
+            st.write(
+                """
+                This tab visually shows the rank of importance of the specified concepts 
+                for the prediction of the model on the input image. Larger font sizes indicate 
+                higher importance. See the other two tabs for more details.
+                """
+            )
+
+            if results is not None:
+                _viz_rank(results)
+                st.divider()
+
+        with test_tab:
+            st.subheader("Testing Results")
+            st.write(
+                """
+                Importance is measured by performing sequential tests of statistical independence. 
+                This tab shows the results of these tests and how the rank of importance is computed.
+                Concepts are sorted by increasing rejection time, where a shorter rejection time indicates 
+                higher importance.
+                """
+            )
             with st.expander("Details"):
-                st.write(
-                    "This tab shows the rank of semantic importance of the concepts for the predictions of the model on the image. Concepts are sorted by increasing rejection time, where a shorter rejection time indicates higher importance."
+                st.markdown(
+                    """
+                    Results are averaged over multiple random draws of conditioning subsets of
+                    concepts. The number of tests can be controlled under `Advanced settings`.
+
+                    - **Rejection rate**: The average number of times the test is rejected for a concept.
+                    - **Rejection time**: The (normalized) average number of steps before the test is 
+                    rejected for a concept.
+                    - **Significance level**: The level at which the test is rejected for a concept.
+                    """
                 )
 
             if results is not None:
-                _viz_rank(results)
+                _viz_test(results)
+                st.divider()
 
         with wealth_tab:
             st.subheader("Wealth Process of Testing Procedures")
-            with st.expander("Details"):
-                st.write(
-                    "This tab shows the average wealth process of the testing procedures for random draws of conditioning subsets with the same cardinality. The black dashed line represents the rejection threshold."
-                )
+            st.markdown(
+                """
+                Sequential tests instantiate a wealth process for each concept. Once the 
+                wealth reaches a value of 1/α, the test is rejected with Type I error control at
+                level α. This tab shows the average wealth process of the testing procedures for
+                each concept.
+                """
+            )
 
             if results is not None:
                 _viz_wealth(results)
+                st.divider()

assets/results/ace.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c82df211515a7d8c565c6e350d4183ba7c877d23ab03e0d639319a3429d8850b
+size 81407

assets/results/english_springer_1.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81c05c7f4cc612f2afefb06a67a970001605ccfe83ddc72e2eb46cafa27773dc
+size 81414

assets/results/english_springer_2.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2ef386512d02796f4570464e88ffaf6df8829858ad6a23836eaa7f86c6a1302
+size 81416

assets/results/french_horn.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:467bf67d0cbf873bb8c72b0bf68906e331573920cf019b78142245d37a9fafb9
+size 81412

assets/results/parachute.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33b001c2390ecdd8975fc8a39b257813a0ff7a05fc65c6a52a981581693b6a42
+size 81415

header.md

@@ -1,4 +1,5 @@
 # 🤔 I Bet You Did Not Mean That
 
-Test the importance of semantic concepts for the predictions of a classifier. [[paper]](https://arxiv.org/pdf/2405.19146) [[code]](https://github.com/Sulam-Group/IBYDMT)
+Test the effect of different concepts on the predictions of a classifier. Concepts are ranked by their *importance*: how much they change the prediction. [[paper]](https://arxiv.org/pdf/2405.19146) [[code]](https://github.com/Sulam-Group/IBYDMT)
+
 

ibydmt/test.py

@@ -120,7 +120,6 @@ class xSKIT(SequentialTester):
         cond_p: Callable[[Float[Array, "D"], list[int], int], Float[Array, "N D2"]],
         model: Callable[[Float[Array, "N D2"]], Float[Array, "N"]],
     ) -> Tuple[Float[Array, "1"], Float[Array, "1"]]:
-
         if len(self._queue) == 0:
             Cuj = C + [j]
 

precompute_results.py

@@ -0,0 +1,46 @@
+import json
+import os
+
+import numpy as np
+from PIL import Image
+
+import app_lib.defaults as defaults
+from app_lib.test import get_testing_config, test
+
+assets_dir = "assets"
+image_dir = os.path.join(assets_dir, "images")
+results_dir = os.path.join(assets_dir, "results")
+os.makedirs(results_dir, exist_ok=True)
+
+testing_config = get_testing_config(
+    significance_level=defaults.SIGNIFICANCE_LEVEL_VALUE,
+    tau_max=defaults.TAU_MAX_VALUE,
+    r=defaults.R_VALUE,
+)
+
+image_presets = json.load(open(os.path.join(assets_dir, "image_presets.json")))
+for _image_name, _image_presets in image_presets.items():
+    _image_name = f"{_image_name}.jpg"
+    _image_path = os.path.join(image_dir, _image_name)
+
+    _image = Image.open(_image_path)
+    _class_name = _image_presets["class_name"]
+    _concepts = _image_presets["concepts"]
+    _cardinality = defaults.CARDINALITY_VALUE(_concepts)
+
+    _results = test(
+        testing_config,
+        _image,
+        _class_name,
+        _concepts,
+        _cardinality,
+        defaults.DATASET_NAME,
+        defaults.MODEL_NAME,
+        with_streamlit=False,
+    )
+
+    np.save(
+        os.path.join(results_dir, _image_name.split(".")[0]),
+        _results,
+        allow_pickle=True,
+    )